Merge remote-tracking branch 'efi/chainsaw' into x86/efi

Resolved Conflicts:
	drivers/firmware/efivars.c
	fs/efivarsfs/file.c

Signed-off-by: H. Peter Anvin <hpa@linux.intel.com>

254 files changed, 9392 insertions, 5339 deletions

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 225543bf45a5..70c0f3da0476 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -1,7 +1,7 @@
 # Select 32 or 64 bit
 config 64BIT
         bool "64-bit kernel" if ARCH = "x86"
-        default ARCH = "x86_64"
+        default ARCH != "i386"
         ---help---
           Say yes to build a 64-bit kernel - formerly known as x86_64
           Say no to build a 32-bit kernel - formerly known as i386
@@ -28,7 +28,6 @@ config X86
         select HAVE_OPROFILE
         select HAVE_PCSPKR_PLATFORM
         select HAVE_PERF_EVENTS
-        select HAVE_IRQ_WORK
         select HAVE_IOREMAP_PROT
         select HAVE_KPROBES
         select HAVE_MEMBLOCK
@@ -40,10 +39,12 @@ config X86
         select HAVE_DMA_CONTIGUOUS if !SWIOTLB
         select HAVE_KRETPROBES
         select HAVE_OPTPROBES
+        select HAVE_KPROBES_ON_FTRACE
         select HAVE_FTRACE_MCOUNT_RECORD
         select HAVE_FENTRY if X86_64
         select HAVE_C_RECORDMCOUNT
         select HAVE_DYNAMIC_FTRACE
+        select HAVE_DYNAMIC_FTRACE_WITH_REGS
         select HAVE_FUNCTION_TRACER
         select HAVE_FUNCTION_GRAPH_TRACER
         select HAVE_FUNCTION_GRAPH_FP_TEST
@@ -106,14 +107,19 @@ config X86
         select GENERIC_CLOCKEVENTS_BROADCAST if X86_64 || (X86_32 && X86_LOCAL_APIC)
         select GENERIC_TIME_VSYSCALL if X86_64
         select KTIME_SCALAR if X86_32
+        select ALWAYS_USE_PERSISTENT_CLOCK
         select GENERIC_STRNCPY_FROM_USER
         select GENERIC_STRNLEN_USER
         select HAVE_CONTEXT_TRACKING if X86_64
         select HAVE_IRQ_TIME_ACCOUNTING
+        select VIRT_TO_BUS
         select MODULES_USE_ELF_REL if X86_32
         select MODULES_USE_ELF_RELA if X86_64
         select CLONE_BACKWARDS if X86_32
-        select GENERIC_SIGALTSTACK
+        select ARCH_USE_BUILTIN_BSWAP
+        select OLD_SIGSUSPEND3 if X86_32 || IA32_EMULATION
+        select OLD_SIGACTION if X86_32
+        select COMPAT_OLD_SIGACTION if IA32_EMULATION
 
 config INSTRUCTION_DECODER
         def_bool y
@@ -222,7 +228,7 @@ config ARCH_SUPPORTS_DEBUG_PAGEALLOC
 
 config HAVE_INTEL_TXT
         def_bool y
-        depends on EXPERIMENTAL && INTEL_IOMMU && ACPI
+        depends on INTEL_IOMMU && ACPI
 
 config X86_32_SMP
         def_bool y
@@ -320,6 +326,10 @@ config X86_BIGSMP
         ---help---
           This option is needed for the systems that have more than 8 CPUs
 
+config GOLDFISH
+       def_bool y
+       depends on X86_GOLDFISH
+
 if X86_32
 config X86_EXTENDED_PLATFORM
         bool "Support for extended (non-PC) x86 platforms"
@@ -402,6 +412,14 @@ config X86_UV
 # Following is an alphabetically sorted list of 32 bit extended platforms
 # Please maintain the alphabetic order if and when there are additions
 
+config X86_GOLDFISH
+       bool "Goldfish (Virtual Platform)"
+       depends on X86_32
+       ---help---
+         Enable support for the Goldfish virtual platform used primarily
+         for Android development. Unless you are building for the Android
+         Goldfish emulator say N here.
+
 config X86_INTEL_CE
         bool "CE4100 TV platform"
         depends on PCI
@@ -454,6 +472,16 @@ config X86_MDFLD
 
 endif
 
+config X86_INTEL_LPSS
+        bool "Intel Low Power Subsystem Support"
+        depends on ACPI
+        select COMMON_CLK
+        ---help---
+          Select to build support for Intel Low Power Subsystem such as
+          found on Intel Lynxpoint PCH. Selecting this option enables
+          things like clock tree (common clock framework) which are needed
+          by the LPSS peripheral drivers.
+
 config X86_RDC321X
         bool "RDC R-321x SoC"
         depends on X86_32
@@ -617,7 +645,7 @@ config PARAVIRT
 
 config PARAVIRT_SPINLOCKS
         bool "Paravirtualization layer for spinlocks"
-        depends on PARAVIRT && SMP && EXPERIMENTAL
+        depends on PARAVIRT && SMP
         ---help---
           Paravirtualized spinlocks allow a pvops backend to replace the
           spinlock implementation with something virtualization-friendly
@@ -729,7 +757,7 @@ config GART_IOMMU
 config CALGARY_IOMMU
         bool "IBM Calgary IOMMU support"
         select SWIOTLB
-        depends on X86_64 && PCI && EXPERIMENTAL
+        depends on X86_64 && PCI
         ---help---
           Support for hardware IOMMUs in IBM's xSeries x366 and x460
           systems. Needed to run systems with more than 3GB of memory
@@ -771,7 +799,7 @@ config IOMMU_HELPER
 
 config MAXSMP
         bool "Enable Maximum number of SMP Processors and NUMA Nodes"
-        depends on X86_64 && SMP && DEBUG_KERNEL && EXPERIMENTAL
+        depends on X86_64 && SMP && DEBUG_KERNEL
         select CPUMASK_OFFSTACK
         ---help---
           Enable maximum number of CPUS and NUMA Nodes for this architecture.
@@ -1029,6 +1057,24 @@ config MICROCODE_OLD_INTERFACE
         def_bool y
         depends on MICROCODE
 
+config MICROCODE_INTEL_LIB
+        def_bool y
+        depends on MICROCODE_INTEL
+
+config MICROCODE_INTEL_EARLY
+        bool "Early load microcode"
+        depends on MICROCODE_INTEL && BLK_DEV_INITRD
+        default y
+        help
+          This option provides functionality to read additional microcode data
+          at the beginning of initrd image. The data tells kernel to load
+          microcode to CPU's as early as possible. No functional change if no
+          microcode data is glued to the initrd, therefore it's safe to say Y.
+
+config MICROCODE_EARLY
+        def_bool y
+        depends on MICROCODE_INTEL_EARLY
+
 config X86_MSR
         tristate "/dev/cpu/*/msr - Model-specific register support"
         ---help---
@@ -1107,7 +1153,6 @@ config HIGHMEM64G
 endchoice
 
 choice
-        depends on EXPERIMENTAL
         prompt "Memory split" if EXPERT
         default VMSPLIT_3G
         depends on X86_32
@@ -1184,7 +1229,7 @@ config DIRECT_GBPAGES
 config NUMA
         bool "Numa Memory Allocation and Scheduler Support"
         depends on SMP
-        depends on X86_64 || (X86_32 && HIGHMEM64G && (X86_NUMAQ || X86_BIGSMP || X86_SUMMIT && ACPI) && EXPERIMENTAL)
+        depends on X86_64 || (X86_32 && HIGHMEM64G && (X86_NUMAQ || X86_BIGSMP || X86_SUMMIT && ACPI))
         default y if (X86_NUMAQ || X86_SUMMIT || X86_BIGSMP)
         ---help---
           Enable NUMA (Non Uniform Memory Access) support.
@@ -1253,10 +1298,6 @@ config NODES_SHIFT
           Specify the maximum number of NUMA Nodes available on the target
           system.  Increases memory reserved to accommodate various tables.
 
-config HAVE_ARCH_ALLOC_REMAP
-        def_bool y
-        depends on X86_32 && NUMA
-
 config ARCH_HAVE_MEMORY_PRESENT
         def_bool y
         depends on X86_32 && DISCONTIGMEM
@@ -1279,7 +1320,7 @@ config ARCH_DISCONTIGMEM_DEFAULT
 
 config ARCH_SPARSEMEM_ENABLE
         def_bool y
-        depends on X86_64 || NUMA || (EXPERIMENTAL && X86_32) || X86_32_NON_STANDARD
+        depends on X86_64 || NUMA || X86_32 || X86_32_NON_STANDARD
         select SPARSEMEM_STATIC if X86_32
         select SPARSEMEM_VMEMMAP_ENABLE if X86_64
 
@@ -1593,8 +1634,7 @@ config CRASH_DUMP
           For more details see Documentation/kdump/kdump.txt
 
 config KEXEC_JUMP
-        bool "kexec jump (EXPERIMENTAL)"
-        depends on EXPERIMENTAL
+        bool "kexec jump"
         depends on KEXEC && HIBERNATION
         ---help---
           Jump between original kernel and kexeced kernel and invoke
@@ -1699,7 +1739,7 @@ config HOTPLUG_CPU
 config BOOTPARAM_HOTPLUG_CPU0
         bool "Set default setting of cpu0_hotpluggable"
         default n
-        depends on HOTPLUG_CPU && EXPERIMENTAL
+        depends on HOTPLUG_CPU
         ---help---
           Set whether default state of cpu0_hotpluggable is on or off.
 
@@ -1728,7 +1768,7 @@ config BOOTPARAM_HOTPLUG_CPU0
 config DEBUG_HOTPLUG_CPU0
         def_bool n
         prompt "Debug CPU0 hotplug"
-        depends on HOTPLUG_CPU && EXPERIMENTAL
+        depends on HOTPLUG_CPU
         ---help---
           Enabling this option offlines CPU0 (if CPU0 can be offlined) as
           soon as possible and boots up userspace with CPU0 offlined. User
@@ -1912,6 +1952,7 @@ config APM_DO_ENABLE
           this feature.
 
 config APM_CPU_IDLE
+        depends on CPU_IDLE
         bool "Make CPU Idle calls when idle"
         ---help---
           Enable calls to APM CPU Idle/CPU Busy inside the kernel's idle loop.
@@ -2037,7 +2078,7 @@ config PCI_MMCONFIG
 
 config PCI_CNB20LE_QUIRK
         bool "Read CNB20LE Host Bridge Windows" if EXPERT
-        depends on PCI && EXPERIMENTAL
+        depends on PCI
         help
           Read the PCI windows out of the CNB20LE host bridge. This allows
           PCI hotplug to work on systems with the CNB20LE chipset which do
@@ -2188,6 +2229,15 @@ config GEOS
         ---help---
           This option enables system support for the Traverse Technologies GEOS.
 
+config TS5500
+        bool "Technologic Systems TS-5500 platform support"
+        depends on MELAN
+        select CHECK_SIGNATURE
+        select NEW_LEDS
+        select LEDS_CLASS
+        ---help---
+          This option enables system support for the Technologic Systems TS-5500.
+
 endif # X86_32
 
 config AMD_NB
@@ -2232,8 +2282,8 @@ config IA32_AOUT
           Support old a.out binaries in the 32bit emulation.
 
 config X86_X32
-        bool "x32 ABI for 64-bit mode (EXPERIMENTAL)"
-        depends on X86_64 && IA32_EMULATION && EXPERIMENTAL
+        bool "x32 ABI for 64-bit mode"
+        depends on X86_64 && IA32_EMULATION
         ---help---
           Include code to run binaries for the x32 native 32-bit ABI
           for 64-bit processors.  An x32 process gets access to the
diff --git a/arch/x86/Makefile b/arch/x86/Makefile
index e71fc4279aab..5c477260294f 100644
--- a/arch/x86/Makefile
+++ b/arch/x86/Makefile
@@ -2,7 +2,11 @@
 
 # select defconfig based on actual architecture
 ifeq ($(ARCH),x86)
+  ifeq ($(shell uname -m),x86_64)
+        KBUILD_DEFCONFIG := x86_64_defconfig
+  else
         KBUILD_DEFCONFIG := i386_defconfig
+  endif
 else
         KBUILD_DEFCONFIG := $(ARCH)_defconfig
 endif
diff --git a/arch/x86/boot/boot.h b/arch/x86/boot/boot.h
index 18997e5a1053..5b7531966b84 100644
--- a/arch/x86/boot/boot.h
+++ b/arch/x86/boot/boot.h
@@ -285,16 +285,26 @@ struct biosregs {
 void intcall(u8 int_no, const struct biosregs *ireg, struct biosregs *oreg);
 
 /* cmdline.c */
-int __cmdline_find_option(u32 cmdline_ptr, const char *option, char *buffer, int bufsize);
-int __cmdline_find_option_bool(u32 cmdline_ptr, const char *option);
+int __cmdline_find_option(unsigned long cmdline_ptr, const char *option, char *buffer, int bufsize);
+int __cmdline_find_option_bool(unsigned long cmdline_ptr, const char *option);
 static inline int cmdline_find_option(const char *option, char *buffer, int bufsize)
 {
-        return __cmdline_find_option(boot_params.hdr.cmd_line_ptr, option, buffer, bufsize);
+        unsigned long cmd_line_ptr = boot_params.hdr.cmd_line_ptr;
+
+        if (cmd_line_ptr >= 0x100000)
+                return -1;      /* inaccessible */
+
+        return __cmdline_find_option(cmd_line_ptr, option, buffer, bufsize);
 }
 
 static inline int cmdline_find_option_bool(const char *option)
 {
-        return __cmdline_find_option_bool(boot_params.hdr.cmd_line_ptr, option);
+        unsigned long cmd_line_ptr = boot_params.hdr.cmd_line_ptr;
+
+        if (cmd_line_ptr >= 0x100000)
+                return -1;      /* inaccessible */
+
+        return __cmdline_find_option_bool(cmd_line_ptr, option);
 }
 
 
diff --git a/arch/x86/boot/cmdline.c b/arch/x86/boot/cmdline.c
index 6b3b6f708c04..625d21b0cd3f 100644
--- a/arch/x86/boot/cmdline.c
+++ b/arch/x86/boot/cmdline.c
@@ -27,7 +27,7 @@ static inline int myisspace(u8 c)
  * Returns the length of the argument (regardless of if it was
  * truncated to fit in the buffer), or -1 on not found.
  */
-int __cmdline_find_option(u32 cmdline_ptr, const char *option, char *buffer, int bufsize)
+int __cmdline_find_option(unsigned long cmdline_ptr, const char *option, char *buffer, int bufsize)
 {
         addr_t cptr;
         char c;
@@ -41,8 +41,8 @@ int __cmdline_find_option(u32 cmdline_ptr, const char *option, char *buffer, int
                 st_bufcpy       /* Copying this to buffer */
         } state = st_wordstart;
 
-        if (!cmdline_ptr || cmdline_ptr >= 0x100000)
-                return -1;      /* No command line, or inaccessible */
+        if (!cmdline_ptr)
+                return -1;      /* No command line */
 
         cptr = cmdline_ptr & 0xf;
         set_fs(cmdline_ptr >> 4);
@@ -99,7 +99,7 @@ int __cmdline_find_option(u32 cmdline_ptr, const char *option, char *buffer, int
  * Returns the position of that option (starts counting with 1)
  * or 0 on not found
  */
-int __cmdline_find_option_bool(u32 cmdline_ptr, const char *option)
+int __cmdline_find_option_bool(unsigned long cmdline_ptr, const char *option)
 {
         addr_t cptr;
         char c;
@@ -111,8 +111,8 @@ int __cmdline_find_option_bool(u32 cmdline_ptr, const char *option)
                 st_wordskip,    /* Miscompare, skip */
         } state = st_wordstart;
 
-        if (!cmdline_ptr || cmdline_ptr >= 0x100000)
-                return -1;      /* No command line, or inaccessible */
+        if (!cmdline_ptr)
+                return -1;      /* No command line */
 
         cptr = cmdline_ptr & 0xf;
         set_fs(cmdline_ptr >> 4);
diff --git a/arch/x86/boot/compressed/cmdline.c b/arch/x86/boot/compressed/cmdline.c
index 10f6b1178c68..bffd73b45b1f 100644
--- a/arch/x86/boot/compressed/cmdline.c
+++ b/arch/x86/boot/compressed/cmdline.c
@@ -13,13 +13,21 @@ static inline char rdfs8(addr_t addr)
         return *((char *)(fs + addr));
 }
 #include "../cmdline.c"
+static unsigned long get_cmd_line_ptr(void)
+{
+        unsigned long cmd_line_ptr = real_mode->hdr.cmd_line_ptr;
+
+        cmd_line_ptr |= (u64)real_mode->ext_cmd_line_ptr << 32;
+
+        return cmd_line_ptr;
+}
 int cmdline_find_option(const char *option, char *buffer, int bufsize)
 {
-        return __cmdline_find_option(real_mode->hdr.cmd_line_ptr, option, buffer, bufsize);
+        return __cmdline_find_option(get_cmd_line_ptr(), option, buffer, bufsize);
 }
 int cmdline_find_option_bool(const char *option)
 {
-        return __cmdline_find_option_bool(real_mode->hdr.cmd_line_ptr, option);
+        return __cmdline_find_option_bool(get_cmd_line_ptr(), option);
 }
 
 #endif
diff --git a/arch/x86/boot/compressed/head_64.S b/arch/x86/boot/compressed/head_64.S
index f5d1aaa0dec8..c1d383d1fb7e 100644
--- a/arch/x86/boot/compressed/head_64.S
+++ b/arch/x86/boot/compressed/head_64.S
@@ -37,6 +37,12 @@
         __HEAD
         .code32
 ENTRY(startup_32)
+        /*
+         * 32bit entry is 0 and it is ABI so immutable!
+         * If we come here directly from a bootloader,
+         * kernel(text+data+bss+brk) ramdisk, zero_page, command line
+         * all need to be under the 4G limit.
+         */
         cld
         /*
          * Test KEEP_SEGMENTS flag to see if the bootloader is asking
@@ -154,6 +160,12 @@ ENTRY(startup_32)
         btsl    $_EFER_LME, %eax
         wrmsr
 
+        /* After gdt is loaded */
+        xorl    %eax, %eax
+        lldt    %ax
+        movl    $0x20, %eax
+        ltr     %ax
+
         /*
          * Setup for the jump to 64bit mode
          *
@@ -176,28 +188,18 @@ ENTRY(startup_32)
         lret
 ENDPROC(startup_32)
 
-no_longmode:
-        /* This isn't an x86-64 CPU so hang */
-1:
-        hlt
-        jmp     1b
-
-#include "../../kernel/verify_cpu.S"
-
-        /*
-         * Be careful here startup_64 needs to be at a predictable
-         * address so I can export it in an ELF header.  Bootloaders
-         * should look at the ELF header to find this address, as
-         * it may change in the future.
-         */
         .code64
         .org 0x200
 ENTRY(startup_64)
         /*
+         * 64bit entry is 0x200 and it is ABI so immutable!
          * We come here either from startup_32 or directly from a
-         * 64bit bootloader.  If we come here from a bootloader we depend on
-         * an identity mapped page table being provied that maps our
-         * entire text+data+bss and hopefully all of memory.
+         * 64bit bootloader.
+         * If we come here from a bootloader, kernel(text+data+bss+brk),
+         * ramdisk, zero_page, command line could be above 4G.
+         * We depend on an identity mapped page table being provided
+         * that maps our entire kernel(text+data+bss+brk), zero page
+         * and command line.
          */
 #ifdef CONFIG_EFI_STUB
         /*
@@ -247,9 +249,6 @@ preferred_addr:
         movl    %eax, %ss
         movl    %eax, %fs
         movl    %eax, %gs
-        lldt    %ax
-        movl    $0x20, %eax
-        ltr     %ax
 
         /*
          * Compute the decompressed kernel start address.  It is where
@@ -349,6 +348,15 @@ relocated:
  */
         jmp     *%rbp
 
+        .code32
+no_longmode:
+        /* This isn't an x86-64 CPU so hang */
+1:
+        hlt
+        jmp     1b
+
+#include "../../kernel/verify_cpu.S"
+
         .data
 gdt:
         .word   gdt_end - gdt
diff --git a/arch/x86/boot/compressed/misc.c b/arch/x86/boot/compressed/misc.c
index 88f7ff6da404..7cb56c6ca351 100644
--- a/arch/x86/boot/compressed/misc.c
+++ b/arch/x86/boot/compressed/misc.c
@@ -325,6 +325,8 @@ asmlinkage void decompress_kernel(void *rmode, memptr heap,
 {
         real_mode = rmode;
 
+        sanitize_boot_params(real_mode);
+
         if (real_mode->screen_info.orig_video_mode == 7) {
                 vidmem = (char *) 0xb0000;
                 vidport = 0x3b4;
diff --git a/arch/x86/boot/compressed/misc.h b/arch/x86/boot/compressed/misc.h
index 0e6dc0ee0eea..674019d8e235 100644
--- a/arch/x86/boot/compressed/misc.h
+++ b/arch/x86/boot/compressed/misc.h
@@ -18,6 +18,7 @@
 #include <asm/page.h>
 #include <asm/boot.h>
 #include <asm/bootparam.h>
+#include <asm/bootparam_utils.h>
 
 #define BOOT_BOOT_H
 #include "../ctype.h"
diff --git a/arch/x86/boot/header.S b/arch/x86/boot/header.S
index 944ce595f767..9ec06a1f6d61 100644
--- a/arch/x86/boot/header.S
+++ b/arch/x86/boot/header.S
@@ -374,6 +374,14 @@ xloadflags:
 #else
 # define XLF0 0
 #endif
+
+#if defined(CONFIG_RELOCATABLE) && defined(CONFIG_X86_64)
+   /* kernel/boot_param/ramdisk could be loaded above 4g */
+# define XLF1 XLF_CAN_BE_LOADED_ABOVE_4G
+#else
+# define XLF1 0
+#endif
+
 #ifdef CONFIG_EFI_STUB
 # ifdef CONFIG_X86_64
 #  define XLF23 XLF_EFI_HANDOVER_64             /* 64-bit EFI handover ok */
@@ -383,7 +391,7 @@ xloadflags:
 #else
 # define XLF23 0
 #endif
-                        .word XLF0 | XLF23
+                        .word XLF0 | XLF1 | XLF23
 
 cmdline_size:   .long   COMMAND_LINE_SIZE-1     #length of the command line,
                                                 #added with boot protocol
diff --git a/arch/x86/configs/i386_defconfig b/arch/x86/configs/i386_defconfig
index 5598547281a7..94447086e551 100644
--- a/arch/x86/configs/i386_defconfig
+++ b/arch/x86/configs/i386_defconfig
@@ -1,3 +1,4 @@
+# CONFIG_64BIT is not set
 CONFIG_EXPERIMENTAL=y
 # CONFIG_LOCALVERSION_AUTO is not set
 CONFIG_SYSVIPC=y
diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile
index e0ca7c9ac383..63947a8f9f0f 100644
--- a/arch/x86/crypto/Makefile
+++ b/arch/x86/crypto/Makefile
@@ -27,6 +27,7 @@ obj-$(CONFIG_CRYPTO_GHASH_CLMUL_NI_INTEL) += ghash-clmulni-intel.o
 
 obj-$(CONFIG_CRYPTO_CRC32C_INTEL) += crc32c-intel.o
 obj-$(CONFIG_CRYPTO_SHA1_SSSE3) += sha1-ssse3.o
+obj-$(CONFIG_CRYPTO_CRC32_PCLMUL) += crc32-pclmul.o
 
 aes-i586-y := aes-i586-asm_32.o aes_glue.o
 twofish-i586-y := twofish-i586-asm_32.o twofish_glue.o
@@ -52,3 +53,4 @@ ghash-clmulni-intel-y := ghash-clmulni-intel_asm.o ghash-clmulni-intel_glue.o
 sha1-ssse3-y := sha1_ssse3_asm.o sha1_ssse3_glue.o
 crc32c-intel-y := crc32c-intel_glue.o
 crc32c-intel-$(CONFIG_CRYPTO_CRC32C_X86_64) += crc32c-pcl-intel-asm_64.o
+crc32-pclmul-y := crc32-pclmul_asm.o crc32-pclmul_glue.o
diff --git a/arch/x86/crypto/aes-i586-asm_32.S b/arch/x86/crypto/aes-i586-asm_32.S
index b949ec2f9af4..2849dbc59e11 100644
--- a/arch/x86/crypto/aes-i586-asm_32.S
+++ b/arch/x86/crypto/aes-i586-asm_32.S
@@ -36,6 +36,7 @@
 .file "aes-i586-asm.S"
 .text
 
+#include <linux/linkage.h>
 #include <asm/asm-offsets.h>
 
 #define tlen 1024   // length of each of 4 'xor' arrays (256 32-bit words)
@@ -219,14 +220,10 @@
 // AES (Rijndael) Encryption Subroutine
 /* void aes_enc_blk(struct crypto_aes_ctx *ctx, u8 *out_blk, const u8 *in_blk) */
 
-.global  aes_enc_blk
-
 .extern  crypto_ft_tab
 .extern  crypto_fl_tab
 
-.align 4
-
-aes_enc_blk:
+ENTRY(aes_enc_blk)
         push    %ebp
         mov     ctx(%esp),%ebp
 
@@ -290,18 +287,15 @@ aes_enc_blk:
         mov     %r0,(%ebp)
         pop     %ebp
         ret
+ENDPROC(aes_enc_blk)
 
 // AES (Rijndael) Decryption Subroutine
 /* void aes_dec_blk(struct crypto_aes_ctx *ctx, u8 *out_blk, const u8 *in_blk) */
 
-.global  aes_dec_blk
-
 .extern  crypto_it_tab
 .extern  crypto_il_tab
 
-.align 4
-
-aes_dec_blk:
+ENTRY(aes_dec_blk)
         push    %ebp
         mov     ctx(%esp),%ebp
 
@@ -365,3 +359,4 @@ aes_dec_blk:
         mov     %r0,(%ebp)
         pop     %ebp
         ret
+ENDPROC(aes_dec_blk)
diff --git a/arch/x86/crypto/aes-x86_64-asm_64.S b/arch/x86/crypto/aes-x86_64-asm_64.S
index 5b577d5a059b..910565547163 100644
--- a/arch/x86/crypto/aes-x86_64-asm_64.S
+++ b/arch/x86/crypto/aes-x86_64-asm_64.S
@@ -15,6 +15,7 @@
 
 .text
 
+#include <linux/linkage.h>
 #include <asm/asm-offsets.h>
 
 #define R1      %rax
@@ -49,10 +50,8 @@
 #define R11     %r11
 
 #define prologue(FUNC,KEY,B128,B192,r1,r2,r3,r4,r5,r6,r7,r8,r9,r10,r11) \
-        .global FUNC;                   \
-        .type   FUNC,@function;         \
-        .align  8;                      \
-FUNC:   movq    r1,r2;                  \
+        ENTRY(FUNC);                    \
+        movq    r1,r2;                  \
         movq    r3,r4;                  \
         leaq    KEY+48(r8),r9;          \
         movq    r10,r11;                \
@@ -71,14 +70,15 @@ FUNC:	movq	r1,r2;			\
         je      B192;                   \
         leaq    32(r9),r9;
 
-#define epilogue(r1,r2,r3,r4,r5,r6,r7,r8,r9) \
+#define epilogue(FUNC,r1,r2,r3,r4,r5,r6,r7,r8,r9) \
         movq    r1,r2;                  \
         movq    r3,r4;                  \
         movl    r5 ## E,(r9);           \
         movl    r6 ## E,4(r9);          \
         movl    r7 ## E,8(r9);          \
         movl    r8 ## E,12(r9);         \
-        ret;
+        ret;                            \
+        ENDPROC(FUNC);
 
 #define round(TAB,OFFSET,r1,r2,r3,r4,r5,r6,r7,r8,ra,rb,rc,rd) \
         movzbl  r2 ## H,r5 ## E;        \
@@ -133,7 +133,7 @@ FUNC:	movq	r1,r2;			\
 #define entry(FUNC,KEY,B128,B192) \
         prologue(FUNC,KEY,B128,B192,R2,R8,R7,R9,R1,R3,R4,R6,R10,R5,R11)
 
-#define return epilogue(R8,R2,R9,R7,R5,R6,R3,R4,R11)
+#define return(FUNC) epilogue(FUNC,R8,R2,R9,R7,R5,R6,R3,R4,R11)
 
 #define encrypt_round(TAB,OFFSET) \
         round(TAB,OFFSET,R1,R2,R3,R4,R5,R6,R7,R10,R5,R6,R3,R4) \
@@ -151,12 +151,12 @@ FUNC:	movq	r1,r2;			\
 
 /* void aes_enc_blk(stuct crypto_tfm *tfm, u8 *out, const u8 *in) */
 
-        entry(aes_enc_blk,0,enc128,enc192)
+        entry(aes_enc_blk,0,.Le128,.Le192)
         encrypt_round(crypto_ft_tab,-96)
         encrypt_round(crypto_ft_tab,-80)
-enc192: encrypt_round(crypto_ft_tab,-64)
+.Le192: encrypt_round(crypto_ft_tab,-64)
         encrypt_round(crypto_ft_tab,-48)
-enc128: encrypt_round(crypto_ft_tab,-32)
+.Le128: encrypt_round(crypto_ft_tab,-32)
         encrypt_round(crypto_ft_tab,-16)
         encrypt_round(crypto_ft_tab,  0)
         encrypt_round(crypto_ft_tab, 16)
@@ -166,16 +166,16 @@ enc128:	encrypt_round(crypto_ft_tab,-32)
         encrypt_round(crypto_ft_tab, 80)
         encrypt_round(crypto_ft_tab, 96)
         encrypt_final(crypto_fl_tab,112)
-        return
+        return(aes_enc_blk)
 
 /* void aes_dec_blk(struct crypto_tfm *tfm, u8 *out, const u8 *in) */
 
-        entry(aes_dec_blk,240,dec128,dec192)
+        entry(aes_dec_blk,240,.Ld128,.Ld192)
         decrypt_round(crypto_it_tab,-96)
         decrypt_round(crypto_it_tab,-80)
-dec192: decrypt_round(crypto_it_tab,-64)
+.Ld192: decrypt_round(crypto_it_tab,-64)
         decrypt_round(crypto_it_tab,-48)
-dec128: decrypt_round(crypto_it_tab,-32)
+.Ld128: decrypt_round(crypto_it_tab,-32)
         decrypt_round(crypto_it_tab,-16)
         decrypt_round(crypto_it_tab,  0)
         decrypt_round(crypto_it_tab, 16)
@@ -185,4 +185,4 @@ dec128:	decrypt_round(crypto_it_tab,-32)
         decrypt_round(crypto_it_tab, 80)
         decrypt_round(crypto_it_tab, 96)
         decrypt_final(crypto_il_tab,112)
-        return
+        return(aes_dec_blk)
diff --git a/arch/x86/crypto/aesni-intel_asm.S b/arch/x86/crypto/aesni-intel_asm.S
index 3470624d7835..04b797767b9e 100644
--- a/arch/x86/crypto/aesni-intel_asm.S
+++ b/arch/x86/crypto/aesni-intel_asm.S
@@ -1262,7 +1262,6 @@ TMP7 XMM1 XMM2 XMM3 XMM4 XMMDst
 * poly = x^128 + x^127 + x^126 + x^121 + 1
 *
 *****************************************************************************/
-
 ENTRY(aesni_gcm_dec)
         push    %r12
         push    %r13
@@ -1437,6 +1436,7 @@ _return_T_done_decrypt:
         pop     %r13
         pop     %r12
         ret
+ENDPROC(aesni_gcm_dec)
 
 
 /*****************************************************************************
@@ -1700,10 +1700,12 @@ _return_T_done_encrypt:
         pop     %r13
         pop     %r12
         ret
+ENDPROC(aesni_gcm_enc)
 
 #endif
 
 
+.align 4
 _key_expansion_128:
 _key_expansion_256a:
         pshufd $0b11111111, %xmm1, %xmm1
@@ -1715,6 +1717,8 @@ _key_expansion_256a:
         movaps %xmm0, (TKEYP)
         add $0x10, TKEYP
         ret
+ENDPROC(_key_expansion_128)
+ENDPROC(_key_expansion_256a)
 
 .align 4
 _key_expansion_192a:
@@ -1739,6 +1743,7 @@ _key_expansion_192a:
         movaps %xmm1, 0x10(TKEYP)
         add $0x20, TKEYP
         ret
+ENDPROC(_key_expansion_192a)
 
 .align 4
 _key_expansion_192b:
@@ -1758,6 +1763,7 @@ _key_expansion_192b:
         movaps %xmm0, (TKEYP)
         add $0x10, TKEYP
         ret
+ENDPROC(_key_expansion_192b)
 
 .align 4
 _key_expansion_256b:
@@ -1770,6 +1776,7 @@ _key_expansion_256b:
         movaps %xmm2, (TKEYP)
         add $0x10, TKEYP
         ret
+ENDPROC(_key_expansion_256b)
 
 /*
  * int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
@@ -1882,6 +1889,7 @@ ENTRY(aesni_set_key)
         popl KEYP
 #endif
         ret
+ENDPROC(aesni_set_key)
 
 /*
  * void aesni_enc(struct crypto_aes_ctx *ctx, u8 *dst, const u8 *src)
@@ -1903,6 +1911,7 @@ ENTRY(aesni_enc)
         popl KEYP
 #endif
         ret
+ENDPROC(aesni_enc)
 
 /*
  * _aesni_enc1:         internal ABI
@@ -1960,6 +1969,7 @@ _aesni_enc1:
         movaps 0x70(TKEYP), KEY
         AESENCLAST KEY STATE
         ret
+ENDPROC(_aesni_enc1)
 
 /*
  * _aesni_enc4: internal ABI
@@ -2068,6 +2078,7 @@ _aesni_enc4:
         AESENCLAST KEY STATE3
         AESENCLAST KEY STATE4
         ret
+ENDPROC(_aesni_enc4)
 
 /*
  * void aesni_dec (struct crypto_aes_ctx *ctx, u8 *dst, const u8 *src)
@@ -2090,6 +2101,7 @@ ENTRY(aesni_dec)
         popl KEYP
 #endif
         ret
+ENDPROC(aesni_dec)
 
 /*
  * _aesni_dec1:         internal ABI
@@ -2147,6 +2159,7 @@ _aesni_dec1:
         movaps 0x70(TKEYP), KEY
         AESDECLAST KEY STATE
         ret
+ENDPROC(_aesni_dec1)
 
 /*
  * _aesni_dec4: internal ABI
@@ -2255,6 +2268,7 @@ _aesni_dec4:
         AESDECLAST KEY STATE3
         AESDECLAST KEY STATE4
         ret
+ENDPROC(_aesni_dec4)
 
 /*
  * void aesni_ecb_enc(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
@@ -2312,6 +2326,7 @@ ENTRY(aesni_ecb_enc)
         popl LEN
 #endif
         ret
+ENDPROC(aesni_ecb_enc)
 
 /*
  * void aesni_ecb_dec(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
@@ -2370,6 +2385,7 @@ ENTRY(aesni_ecb_dec)
         popl LEN
 #endif
         ret
+ENDPROC(aesni_ecb_dec)
 
 /*
  * void aesni_cbc_enc(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
@@ -2411,6 +2427,7 @@ ENTRY(aesni_cbc_enc)
         popl IVP
 #endif
         ret
+ENDPROC(aesni_cbc_enc)
 
 /*
  * void aesni_cbc_dec(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
@@ -2501,6 +2518,7 @@ ENTRY(aesni_cbc_dec)
         popl IVP
 #endif
         ret
+ENDPROC(aesni_cbc_dec)
 
 #ifdef __x86_64__
 .align 16
@@ -2527,6 +2545,7 @@ _aesni_inc_init:
         MOVQ_R64_XMM TCTR_LOW INC
         MOVQ_R64_XMM CTR TCTR_LOW
         ret
+ENDPROC(_aesni_inc_init)
 
 /*
  * _aesni_inc:          internal ABI
@@ -2555,6 +2574,7 @@ _aesni_inc:
         movaps CTR, IV
         PSHUFB_XMM BSWAP_MASK IV
         ret
+ENDPROC(_aesni_inc)
 
 /*
  * void aesni_ctr_enc(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
@@ -2615,4 +2635,5 @@ ENTRY(aesni_ctr_enc)
         movups IV, (IVP)
 .Lctr_enc_just_ret:
         ret
+ENDPROC(aesni_ctr_enc)
 #endif
diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c
index 1b9c22bea8a7..a0795da22c02 100644
--- a/arch/x86/crypto/aesni-intel_glue.c
+++ b/arch/x86/crypto/aesni-intel_glue.c
@@ -40,10 +40,6 @@
 #include <linux/workqueue.h>
 #include <linux/spinlock.h>
 
-#if defined(CONFIG_CRYPTO_CTR) || defined(CONFIG_CRYPTO_CTR_MODULE)
-#define HAS_CTR
-#endif
-
 #if defined(CONFIG_CRYPTO_PCBC) || defined(CONFIG_CRYPTO_PCBC_MODULE)
 #define HAS_PCBC
 #endif
@@ -395,12 +391,6 @@ static int ablk_ctr_init(struct crypto_tfm *tfm)
         return ablk_init_common(tfm, "__driver-ctr-aes-aesni");
 }
 
-#ifdef HAS_CTR
-static int ablk_rfc3686_ctr_init(struct crypto_tfm *tfm)
-{
-        return ablk_init_common(tfm, "rfc3686(__driver-ctr-aes-aesni)");
-}
-#endif
 #endif
 
 #ifdef HAS_PCBC
@@ -1158,33 +1148,6 @@ static struct crypto_alg aesni_algs[] = { {
                         .maxauthsize    = 16,
                 },
         },
-#ifdef HAS_CTR
-}, {
-        .cra_name               = "rfc3686(ctr(aes))",
-        .cra_driver_name        = "rfc3686-ctr-aes-aesni",
-        .cra_priority           = 400,
-        .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
-        .cra_blocksize          = 1,
-        .cra_ctxsize            = sizeof(struct async_helper_ctx),
-        .cra_alignmask          = 0,
-        .cra_type               = &crypto_ablkcipher_type,
-        .cra_module             = THIS_MODULE,
-        .cra_init               = ablk_rfc3686_ctr_init,
-        .cra_exit               = ablk_exit,
-        .cra_u = {
-                .ablkcipher = {
-                        .min_keysize = AES_MIN_KEY_SIZE +
-                                       CTR_RFC3686_NONCE_SIZE,
-                        .max_keysize = AES_MAX_KEY_SIZE +
-                                       CTR_RFC3686_NONCE_SIZE,
-                        .ivsize      = CTR_RFC3686_IV_SIZE,
-                        .setkey      = ablk_set_key,
-                        .encrypt     = ablk_encrypt,
-                        .decrypt     = ablk_decrypt,
-                        .geniv       = "seqiv",
-                },
-        },
-#endif
 #endif
 #ifdef HAS_PCBC
 }, {
diff --git a/arch/x86/crypto/blowfish-x86_64-asm_64.S b/arch/x86/crypto/blowfish-x86_64-asm_64.S
index 391d245dc086..246c67006ed0 100644
--- a/arch/x86/crypto/blowfish-x86_64-asm_64.S
+++ b/arch/x86/crypto/blowfish-x86_64-asm_64.S
@@ -20,6 +20,8 @@
  *
  */
 
+#include <linux/linkage.h>
+
 .file "blowfish-x86_64-asm.S"
 .text
 
@@ -116,11 +118,7 @@
         bswapq                  RX0; \
         xorq RX0,               (RIO);
 
-.align 8
-.global __blowfish_enc_blk
-.type   __blowfish_enc_blk,@function;
-
-__blowfish_enc_blk:
+ENTRY(__blowfish_enc_blk)
         /* input:
          *      %rdi: ctx, CTX
          *      %rsi: dst
@@ -148,19 +146,16 @@ __blowfish_enc_blk:
 
         movq %r10, RIO;
         test %cl, %cl;
-        jnz __enc_xor;
+        jnz .L__enc_xor;
 
         write_block();
         ret;
-__enc_xor:
+.L__enc_xor:
         xor_block();
         ret;
+ENDPROC(__blowfish_enc_blk)
 
-.align 8
-.global blowfish_dec_blk
-.type   blowfish_dec_blk,@function;
-
-blowfish_dec_blk:
+ENTRY(blowfish_dec_blk)
         /* input:
          *      %rdi: ctx, CTX
          *      %rsi: dst
@@ -189,6 +184,7 @@ blowfish_dec_blk:
         movq %r11, %rbp;
 
         ret;
+ENDPROC(blowfish_dec_blk)
 
 /**********************************************************************
   4-way blowfish, four blocks parallel
@@ -300,11 +296,7 @@ blowfish_dec_blk:
         bswapq                  RX3; \
         xorq RX3,               24(RIO);
 
-.align 8
-.global __blowfish_enc_blk_4way
-.type   __blowfish_enc_blk_4way,@function;
-
-__blowfish_enc_blk_4way:
+ENTRY(__blowfish_enc_blk_4way)
         /* input:
          *      %rdi: ctx, CTX
          *      %rsi: dst
@@ -336,7 +328,7 @@ __blowfish_enc_blk_4way:
         movq %r11, RIO;
 
         test %bpl, %bpl;
-        jnz __enc_xor4;
+        jnz .L__enc_xor4;
 
         write_block4();
 
@@ -344,18 +336,15 @@ __blowfish_enc_blk_4way:
         popq %rbp;
         ret;
 
-__enc_xor4:
+.L__enc_xor4:
         xor_block4();
 
         popq %rbx;
         popq %rbp;
         ret;
+ENDPROC(__blowfish_enc_blk_4way)
 
-.align 8
-.global blowfish_dec_blk_4way
-.type   blowfish_dec_blk_4way,@function;
-
-blowfish_dec_blk_4way:
+ENTRY(blowfish_dec_blk_4way)
         /* input:
          *      %rdi: ctx, CTX
          *      %rsi: dst
@@ -387,4 +376,4 @@ blowfish_dec_blk_4way:
         popq %rbp;
 
         ret;
-
+ENDPROC(blowfish_dec_blk_4way)
diff --git a/arch/x86/crypto/camellia-aesni-avx-asm_64.S b/arch/x86/crypto/camellia-aesni-avx-asm_64.S
index 2306d2e4816f..cfc163469c71 100644
--- a/arch/x86/crypto/camellia-aesni-avx-asm_64.S
+++ b/arch/x86/crypto/camellia-aesni-avx-asm_64.S
@@ -15,6 +15,8 @@
  *      http://koti.mbnet.fi/axh/crypto/camellia-BSD-1.2.0-aesni1.tar.xz
  */
 
+#include <linux/linkage.h>
+
 #define CAMELLIA_TABLE_BYTE_LEN 272
 
 /* struct camellia_ctx: */
@@ -190,6 +192,7 @@ roundsm16_x0_x1_x2_x3_x4_x5_x6_x7_y0_y1_y2_y3_y4_y5_y6_y7_cd:
                   %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, %xmm15,
                   %rcx, (%r9));
         ret;
+ENDPROC(roundsm16_x0_x1_x2_x3_x4_x5_x6_x7_y0_y1_y2_y3_y4_y5_y6_y7_cd)
 
 .align 8
 roundsm16_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_ab:
@@ -197,6 +200,7 @@ roundsm16_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_ab:
                   %xmm12, %xmm13, %xmm14, %xmm15, %xmm8, %xmm9, %xmm10, %xmm11,
                   %rax, (%r9));
         ret;
+ENDPROC(roundsm16_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_ab)
 
 /*
  * IN/OUT:
@@ -709,8 +713,6 @@ roundsm16_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_ab:
 .text
 
 .align 8
-.type   __camellia_enc_blk16,@function;
-
 __camellia_enc_blk16:
         /* input:
          *      %rdi: ctx, CTX
@@ -793,10 +795,9 @@ __camellia_enc_blk16:
                      %xmm15, %rax, %rcx, 24);
 
         jmp .Lenc_done;
+ENDPROC(__camellia_enc_blk16)
 
 .align 8
-.type   __camellia_dec_blk16,@function;
-
 __camellia_dec_blk16:
         /* input:
          *      %rdi: ctx, CTX
@@ -877,12 +878,9 @@ __camellia_dec_blk16:
               ((key_table + (24) * 8) + 4)(CTX));
 
         jmp .Ldec_max24;
+ENDPROC(__camellia_dec_blk16)
 
-.align 8
-.global camellia_ecb_enc_16way
-.type   camellia_ecb_enc_16way,@function;
-
-camellia_ecb_enc_16way:
+ENTRY(camellia_ecb_enc_16way)
         /* input:
          *      %rdi: ctx, CTX
          *      %rsi: dst (16 blocks)
@@ -903,12 +901,9 @@ camellia_ecb_enc_16way:
                      %xmm8, %rsi);
 
         ret;
+ENDPROC(camellia_ecb_enc_16way)
 
-.align 8
-.global camellia_ecb_dec_16way
-.type   camellia_ecb_dec_16way,@function;
-
-camellia_ecb_dec_16way:
+ENTRY(camellia_ecb_dec_16way)
         /* input:
          *      %rdi: ctx, CTX
          *      %rsi: dst (16 blocks)
@@ -934,12 +929,9 @@ camellia_ecb_dec_16way:
                      %xmm8, %rsi);
 
         ret;
+ENDPROC(camellia_ecb_dec_16way)
 
-.align 8
-.global camellia_cbc_dec_16way
-.type   camellia_cbc_dec_16way,@function;
-
-camellia_cbc_dec_16way:
+ENTRY(camellia_cbc_dec_16way)
         /* input:
          *      %rdi: ctx, CTX
          *      %rsi: dst (16 blocks)
@@ -986,6 +978,7 @@ camellia_cbc_dec_16way:
                      %xmm8, %rsi);
 
         ret;
+ENDPROC(camellia_cbc_dec_16way)
 
 #define inc_le128(x, minus_one, tmp) \
         vpcmpeqq minus_one, x, tmp; \
@@ -993,11 +986,7 @@ camellia_cbc_dec_16way:
         vpslldq $8, tmp, tmp; \
         vpsubq tmp, x, x;
 
-.align 8
-.global camellia_ctr_16way
-.type   camellia_ctr_16way,@function;
-
-camellia_ctr_16way:
+ENTRY(camellia_ctr_16way)
         /* input:
          *      %rdi: ctx, CTX
          *      %rsi: dst (16 blocks)
@@ -1100,3 +1089,4 @@ camellia_ctr_16way:
                      %xmm8, %rsi);
 
         ret;
+ENDPROC(camellia_ctr_16way)
diff --git a/arch/x86/crypto/camellia-x86_64-asm_64.S b/arch/x86/crypto/camellia-x86_64-asm_64.S
index 0b3374335fdc..310319c601ed 100644
--- a/arch/x86/crypto/camellia-x86_64-asm_64.S
+++ b/arch/x86/crypto/camellia-x86_64-asm_64.S
@@ -20,6 +20,8 @@
  *
  */
 
+#include <linux/linkage.h>
+
 .file "camellia-x86_64-asm_64.S"
 .text
 
@@ -188,10 +190,7 @@
         bswapq                          RAB0; \
         movq RAB0,                      4*2(RIO);
 
-.global __camellia_enc_blk;
-.type   __camellia_enc_blk,@function;
-
-__camellia_enc_blk:
+ENTRY(__camellia_enc_blk)
         /* input:
          *      %rdi: ctx, CTX
          *      %rsi: dst
@@ -214,33 +213,31 @@ __camellia_enc_blk:
         movl $24, RT1d; /* max */
 
         cmpb $16, key_length(CTX);
-        je __enc_done;
+        je .L__enc_done;
 
         enc_fls(24);
         enc_rounds(24);
         movl $32, RT1d; /* max */
 
-__enc_done:
+.L__enc_done:
         testb RXORbl, RXORbl;
         movq RDST, RIO;
 
-        jnz __enc_xor;
+        jnz .L__enc_xor;
 
         enc_outunpack(mov, RT1);
 
         movq RRBP, %rbp;
         ret;
 
-__enc_xor:
+.L__enc_xor:
         enc_outunpack(xor, RT1);
 
         movq RRBP, %rbp;
         ret;
+ENDPROC(__camellia_enc_blk)
 
-.global camellia_dec_blk;
-.type   camellia_dec_blk,@function;
-
-camellia_dec_blk:
+ENTRY(camellia_dec_blk)
         /* input:
          *      %rdi: ctx, CTX
          *      %rsi: dst
@@ -258,12 +255,12 @@ camellia_dec_blk:
         dec_inpack(RT2);
 
         cmpb $24, RT2bl;
-        je __dec_rounds16;
+        je .L__dec_rounds16;
 
         dec_rounds(24);
         dec_fls(24);
 
-__dec_rounds16:
+.L__dec_rounds16:
         dec_rounds(16);
         dec_fls(16);
         dec_rounds(8);
@@ -276,6 +273,7 @@ __dec_rounds16:
 
         movq RRBP, %rbp;
         ret;
+ENDPROC(camellia_dec_blk)
 
 /**********************************************************************
   2-way camellia
@@ -426,10 +424,7 @@ __dec_rounds16:
                 bswapq                          RAB1; \
                 movq RAB1,                      12*2(RIO);
 
-.global __camellia_enc_blk_2way;
-.type   __camellia_enc_blk_2way,@function;
-
-__camellia_enc_blk_2way:
+ENTRY(__camellia_enc_blk_2way)
         /* input:
          *      %rdi: ctx, CTX
          *      %rsi: dst
@@ -453,16 +448,16 @@ __camellia_enc_blk_2way:
         movl $24, RT2d; /* max */
 
         cmpb $16, key_length(CTX);
-        je __enc2_done;
+        je .L__enc2_done;
 
         enc_fls2(24);
         enc_rounds2(24);
         movl $32, RT2d; /* max */
 
-__enc2_done:
+.L__enc2_done:
         test RXORbl, RXORbl;
         movq RDST, RIO;
-        jnz __enc2_xor;
+        jnz .L__enc2_xor;
 
         enc_outunpack2(mov, RT2);
 
@@ -470,17 +465,15 @@ __enc2_done:
         popq %rbx;
         ret;
 
-__enc2_xor:
+.L__enc2_xor:
         enc_outunpack2(xor, RT2);
 
         movq RRBP, %rbp;
         popq %rbx;
         ret;
+ENDPROC(__camellia_enc_blk_2way)
 
-.global camellia_dec_blk_2way;
-.type   camellia_dec_blk_2way,@function;
-
-camellia_dec_blk_2way:
+ENTRY(camellia_dec_blk_2way)
         /* input:
          *      %rdi: ctx, CTX
          *      %rsi: dst
@@ -499,12 +492,12 @@ camellia_dec_blk_2way:
         dec_inpack2(RT2);
 
         cmpb $24, RT2bl;
-        je __dec2_rounds16;
+        je .L__dec2_rounds16;
 
         dec_rounds2(24);
         dec_fls2(24);
 
-__dec2_rounds16:
+.L__dec2_rounds16:
         dec_rounds2(16);
         dec_fls2(16);
         dec_rounds2(8);
@@ -518,3 +511,4 @@ __dec2_rounds16:
         movq RRBP, %rbp;
         movq RXOR, %rbx;
         ret;
+ENDPROC(camellia_dec_blk_2way)
diff --git a/arch/x86/crypto/cast5-avx-x86_64-asm_64.S b/arch/x86/crypto/cast5-avx-x86_64-asm_64.S
index 15b00ac7cbd3..c35fd5d6ecd2 100644
--- a/arch/x86/crypto/cast5-avx-x86_64-asm_64.S
+++ b/arch/x86/crypto/cast5-avx-x86_64-asm_64.S
@@ -23,6 +23,8 @@
  *
  */
 
+#include <linux/linkage.h>
+
 .file "cast5-avx-x86_64-asm_64.S"
 
 .extern cast_s1
@@ -211,8 +213,6 @@
 .text
 
 .align 16
-.type   __cast5_enc_blk16,@function;
-
 __cast5_enc_blk16:
         /* input:
          *      %rdi: ctx, CTX
@@ -263,14 +263,14 @@ __cast5_enc_blk16:
 
         movzbl rr(CTX), %eax;
         testl %eax, %eax;
-        jnz __skip_enc;
+        jnz .L__skip_enc;
 
         round(RL, RR, 12, 1);
         round(RR, RL, 13, 2);
         round(RL, RR, 14, 3);
         round(RR, RL, 15, 1);
 
-__skip_enc:
+.L__skip_enc:
         popq %rbx;
         popq %rbp;
 
@@ -282,10 +282,9 @@ __skip_enc:
         outunpack_blocks(RR4, RL4, RTMP, RX, RKM);
 
         ret;
+ENDPROC(__cast5_enc_blk16)
 
 .align 16
-.type   __cast5_dec_blk16,@function;
-
 __cast5_dec_blk16:
         /* input:
          *      %rdi: ctx, CTX
@@ -323,14 +322,14 @@ __cast5_dec_blk16:
 
         movzbl rr(CTX), %eax;
         testl %eax, %eax;
-        jnz __skip_dec;
+        jnz .L__skip_dec;
 
         round(RL, RR, 15, 1);
         round(RR, RL, 14, 3);
         round(RL, RR, 13, 2);
         round(RR, RL, 12, 1);
 
-__dec_tail:
+.L__dec_tail:
         round(RL, RR, 11, 3);
         round(RR, RL, 10, 2);
         round(RL, RR, 9, 1);
@@ -355,15 +354,12 @@ __dec_tail:
 
         ret;
 
-__skip_dec:
+.L__skip_dec:
         vpsrldq $4, RKR, RKR;
-        jmp __dec_tail;
+        jmp .L__dec_tail;
+ENDPROC(__cast5_dec_blk16)
 
-.align 16
-.global cast5_ecb_enc_16way
-.type   cast5_ecb_enc_16way,@function;
-
-cast5_ecb_enc_16way:
+ENTRY(cast5_ecb_enc_16way)
         /* input:
          *      %rdi: ctx, CTX
          *      %rsi: dst
@@ -393,12 +389,9 @@ cast5_ecb_enc_16way:
         vmovdqu RL4, (7*4*4)(%r11);
 
         ret;
+ENDPROC(cast5_ecb_enc_16way)
 
-.align 16
-.global cast5_ecb_dec_16way
-.type   cast5_ecb_dec_16way,@function;
-
-cast5_ecb_dec_16way:
+ENTRY(cast5_ecb_dec_16way)
         /* input:
          *      %rdi: ctx, CTX
          *      %rsi: dst
@@ -428,12 +421,9 @@ cast5_ecb_dec_16way:
         vmovdqu RL4, (7*4*4)(%r11);
 
         ret;
+ENDPROC(cast5_ecb_dec_16way)
 
-.align 16
-.global cast5_cbc_dec_16way
-.type   cast5_cbc_dec_16way,@function;
-
-cast5_cbc_dec_16way:
+ENTRY(cast5_cbc_dec_16way)
         /* input:
          *      %rdi: ctx, CTX
          *      %rsi: dst
@@ -480,12 +470,9 @@ cast5_cbc_dec_16way:
         popq %r12;
 
         ret;
+ENDPROC(cast5_cbc_dec_16way)
 
-.align 16
-.global cast5_ctr_16way
-.type   cast5_ctr_16way,@function;
-
-cast5_ctr_16way:
+ENTRY(cast5_ctr_16way)
         /* input:
          *      %rdi: ctx, CTX
          *      %rsi: dst
@@ -556,3 +543,4 @@ cast5_ctr_16way:
         popq %r12;
 
         ret;
+ENDPROC(cast5_ctr_16way)
diff --git a/arch/x86/crypto/cast6-avx-x86_64-asm_64.S b/arch/x86/crypto/cast6-avx-x86_64-asm_64.S
index 2569d0da841f..f93b6105a0ce 100644
--- a/arch/x86/crypto/cast6-avx-x86_64-asm_64.S
+++ b/arch/x86/crypto/cast6-avx-x86_64-asm_64.S
@@ -23,6 +23,7 @@
  *
  */
 
+#include <linux/linkage.h>
 #include "glue_helper-asm-avx.S"
 
 .file "cast6-avx-x86_64-asm_64.S"
@@ -250,8 +251,6 @@
 .text
 
 .align 8
-.type   __cast6_enc_blk8,@function;
-
 __cast6_enc_blk8:
         /* input:
          *      %rdi: ctx, CTX
@@ -295,10 +294,9 @@ __cast6_enc_blk8:
         outunpack_blocks(RA2, RB2, RC2, RD2, RTMP, RX, RKRF, RKM);
 
         ret;
+ENDPROC(__cast6_enc_blk8)
 
 .align 8
-.type   __cast6_dec_blk8,@function;
-
 __cast6_dec_blk8:
         /* input:
          *      %rdi: ctx, CTX
@@ -341,12 +339,9 @@ __cast6_dec_blk8:
         outunpack_blocks(RA2, RB2, RC2, RD2, RTMP, RX, RKRF, RKM);
 
         ret;
+ENDPROC(__cast6_dec_blk8)
 
-.align 8
-.global cast6_ecb_enc_8way
-.type   cast6_ecb_enc_8way,@function;
-
-cast6_ecb_enc_8way:
+ENTRY(cast6_ecb_enc_8way)
         /* input:
          *      %rdi: ctx, CTX
          *      %rsi: dst
@@ -362,12 +357,9 @@ cast6_ecb_enc_8way:
         store_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
 
         ret;
+ENDPROC(cast6_ecb_enc_8way)
 
-.align 8
-.global cast6_ecb_dec_8way
-.type   cast6_ecb_dec_8way,@function;
-
-cast6_ecb_dec_8way:
+ENTRY(cast6_ecb_dec_8way)
         /* input:
          *      %rdi: ctx, CTX
          *      %rsi: dst
@@ -383,12 +375,9 @@ cast6_ecb_dec_8way:
         store_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
 
         ret;
+ENDPROC(cast6_ecb_dec_8way)
 
-.align 8
-.global cast6_cbc_dec_8way
-.type   cast6_cbc_dec_8way,@function;
-
-cast6_cbc_dec_8way:
+ENTRY(cast6_cbc_dec_8way)
         /* input:
          *      %rdi: ctx, CTX
          *      %rsi: dst
@@ -409,12 +398,9 @@ cast6_cbc_dec_8way:
         popq %r12;
 
         ret;
+ENDPROC(cast6_cbc_dec_8way)
 
-.align 8
-.global cast6_ctr_8way
-.type   cast6_ctr_8way,@function;
-
-cast6_ctr_8way:
+ENTRY(cast6_ctr_8way)
         /* input:
          *      %rdi: ctx, CTX
          *      %rsi: dst
@@ -437,3 +423,4 @@ cast6_ctr_8way:
         popq %r12;
 
         ret;
+ENDPROC(cast6_ctr_8way)
diff --git a/arch/x86/crypto/crc32-pclmul_asm.S b/arch/x86/crypto/crc32-pclmul_asm.S
new file mode 100644
index 000000000000..c8335014a044
--- /dev/null
+++ b/arch/x86/crypto/crc32-pclmul_asm.S
@@ -0,0 +1,246 @@
+/* GPL HEADER START
+ *
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 only,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License version 2 for more details (a copy is included
+ * in the LICENSE file that accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; If not, see http://www.gnu.org/licenses
+ *
+ * Please  visit http://www.xyratex.com/contact if you need additional
+ * information or have any questions.
+ *
+ * GPL HEADER END
+ */
+
+/*
+ * Copyright 2012 Xyratex Technology Limited
+ *
+ * Using hardware provided PCLMULQDQ instruction to accelerate the CRC32
+ * calculation.
+ * CRC32 polynomial:0x04c11db7(BE)/0xEDB88320(LE)
+ * PCLMULQDQ is a new instruction in Intel SSE4.2, the reference can be found
+ * at:
+ * http://www.intel.com/products/processor/manuals/
+ * Intel(R) 64 and IA-32 Architectures Software Developer's Manual
+ * Volume 2B: Instruction Set Reference, N-Z
+ *
+ * Authors:   Gregory Prestas <Gregory_Prestas@us.xyratex.com>
+ *            Alexander Boyko <Alexander_Boyko@xyratex.com>
+ */
+
+#include <linux/linkage.h>
+#include <asm/inst.h>
+
+
+.align 16
+/*
+ * [x4*128+32 mod P(x) << 32)]'  << 1   = 0x154442bd4
+ * #define CONSTANT_R1  0x154442bd4LL
+ *
+ * [(x4*128-32 mod P(x) << 32)]' << 1   = 0x1c6e41596
+ * #define CONSTANT_R2  0x1c6e41596LL
+ */
+.Lconstant_R2R1:
+        .octa 0x00000001c6e415960000000154442bd4
+/*
+ * [(x128+32 mod P(x) << 32)]'   << 1   = 0x1751997d0
+ * #define CONSTANT_R3  0x1751997d0LL
+ *
+ * [(x128-32 mod P(x) << 32)]'   << 1   = 0x0ccaa009e
+ * #define CONSTANT_R4  0x0ccaa009eLL
+ */
+.Lconstant_R4R3:
+        .octa 0x00000000ccaa009e00000001751997d0
+/*
+ * [(x64 mod P(x) << 32)]'       << 1   = 0x163cd6124
+ * #define CONSTANT_R5  0x163cd6124LL
+ */
+.Lconstant_R5:
+        .octa 0x00000000000000000000000163cd6124
+.Lconstant_mask32:
+        .octa 0x000000000000000000000000FFFFFFFF
+/*
+ * #define CRCPOLY_TRUE_LE_FULL 0x1DB710641LL
+ *
+ * Barrett Reduction constant (u64`) = u` = (x**64 / P(x))` = 0x1F7011641LL
+ * #define CONSTANT_RU  0x1F7011641LL
+ */
+.Lconstant_RUpoly:
+        .octa 0x00000001F701164100000001DB710641
+
+#define CONSTANT %xmm0
+
+#ifdef __x86_64__
+#define BUF     %rdi
+#define LEN     %rsi
+#define CRC     %edx
+#else
+#define BUF     %eax
+#define LEN     %edx
+#define CRC     %ecx
+#endif
+
+
+
+.text
+/**
+ *      Calculate crc32
+ *      BUF - buffer (16 bytes aligned)
+ *      LEN - sizeof buffer (16 bytes aligned), LEN should be grater than 63
+ *      CRC - initial crc32
+ *      return %eax crc32
+ *      uint crc32_pclmul_le_16(unsigned char const *buffer,
+ *                           size_t len, uint crc32)
+ */
+.globl crc32_pclmul_le_16
+.align 4, 0x90
+crc32_pclmul_le_16:/* buffer and buffer size are 16 bytes aligned */
+        movdqa  (BUF), %xmm1
+        movdqa  0x10(BUF), %xmm2
+        movdqa  0x20(BUF), %xmm3
+        movdqa  0x30(BUF), %xmm4
+        movd    CRC, CONSTANT
+        pxor    CONSTANT, %xmm1
+        sub     $0x40, LEN
+        add     $0x40, BUF
+#ifndef __x86_64__
+        /* This is for position independent code(-fPIC) support for 32bit */
+        call    delta
+delta:
+        pop     %ecx
+#endif
+        cmp     $0x40, LEN
+        jb      less_64
+
+#ifdef __x86_64__
+        movdqa .Lconstant_R2R1(%rip), CONSTANT
+#else
+        movdqa .Lconstant_R2R1 - delta(%ecx), CONSTANT
+#endif
+
+loop_64:/*  64 bytes Full cache line folding */
+        prefetchnta    0x40(BUF)
+        movdqa  %xmm1, %xmm5
+        movdqa  %xmm2, %xmm6
+        movdqa  %xmm3, %xmm7
+#ifdef __x86_64__
+        movdqa  %xmm4, %xmm8
+#endif
+        PCLMULQDQ 00, CONSTANT, %xmm1
+        PCLMULQDQ 00, CONSTANT, %xmm2
+        PCLMULQDQ 00, CONSTANT, %xmm3
+#ifdef __x86_64__
+        PCLMULQDQ 00, CONSTANT, %xmm4
+#endif
+        PCLMULQDQ 0x11, CONSTANT, %xmm5
+        PCLMULQDQ 0x11, CONSTANT, %xmm6
+        PCLMULQDQ 0x11, CONSTANT, %xmm7
+#ifdef __x86_64__
+        PCLMULQDQ 0x11, CONSTANT, %xmm8
+#endif
+        pxor    %xmm5, %xmm1
+        pxor    %xmm6, %xmm2
+        pxor    %xmm7, %xmm3
+#ifdef __x86_64__
+        pxor    %xmm8, %xmm4
+#else
+        /* xmm8 unsupported for x32 */
+        movdqa  %xmm4, %xmm5
+        PCLMULQDQ 00, CONSTANT, %xmm4
+        PCLMULQDQ 0x11, CONSTANT, %xmm5
+        pxor    %xmm5, %xmm4
+#endif
+
+        pxor    (BUF), %xmm1
+        pxor    0x10(BUF), %xmm2
+        pxor    0x20(BUF), %xmm3
+        pxor    0x30(BUF), %xmm4
+
+        sub     $0x40, LEN
+        add     $0x40, BUF
+        cmp     $0x40, LEN
+        jge     loop_64
+less_64:/*  Folding cache line into 128bit */
+#ifdef __x86_64__
+        movdqa  .Lconstant_R4R3(%rip), CONSTANT
+#else
+        movdqa  .Lconstant_R4R3 - delta(%ecx), CONSTANT
+#endif
+        prefetchnta     (BUF)
+
+        movdqa  %xmm1, %xmm5
+        PCLMULQDQ 0x00, CONSTANT, %xmm1
+        PCLMULQDQ 0x11, CONSTANT, %xmm5
+        pxor    %xmm5, %xmm1
+        pxor    %xmm2, %xmm1
+
+        movdqa  %xmm1, %xmm5
+        PCLMULQDQ 0x00, CONSTANT, %xmm1
+        PCLMULQDQ 0x11, CONSTANT, %xmm5
+        pxor    %xmm5, %xmm1
+        pxor    %xmm3, %xmm1
+
+        movdqa  %xmm1, %xmm5
+        PCLMULQDQ 0x00, CONSTANT, %xmm1
+        PCLMULQDQ 0x11, CONSTANT, %xmm5
+        pxor    %xmm5, %xmm1
+        pxor    %xmm4, %xmm1
+
+        cmp     $0x10, LEN
+        jb      fold_64
+loop_16:/* Folding rest buffer into 128bit */
+        movdqa  %xmm1, %xmm5
+        PCLMULQDQ 0x00, CONSTANT, %xmm1
+        PCLMULQDQ 0x11, CONSTANT, %xmm5
+        pxor    %xmm5, %xmm1
+        pxor    (BUF), %xmm1
+        sub     $0x10, LEN
+        add     $0x10, BUF
+        cmp     $0x10, LEN
+        jge     loop_16
+
+fold_64:
+        /* perform the last 64 bit fold, also adds 32 zeroes
+         * to the input stream */
+        PCLMULQDQ 0x01, %xmm1, CONSTANT /* R4 * xmm1.low */
+        psrldq  $0x08, %xmm1
+        pxor    CONSTANT, %xmm1
+
+        /* final 32-bit fold */
+        movdqa  %xmm1, %xmm2
+#ifdef __x86_64__
+        movdqa  .Lconstant_R5(%rip), CONSTANT
+        movdqa  .Lconstant_mask32(%rip), %xmm3
+#else
+        movdqa  .Lconstant_R5 - delta(%ecx), CONSTANT
+        movdqa  .Lconstant_mask32 - delta(%ecx), %xmm3
+#endif
+        psrldq  $0x04, %xmm2
+        pand    %xmm3, %xmm1
+        PCLMULQDQ 0x00, CONSTANT, %xmm1
+        pxor    %xmm2, %xmm1
+
+        /* Finish up with the bit-reversed barrett reduction 64 ==> 32 bits */
+#ifdef __x86_64__
+        movdqa  .Lconstant_RUpoly(%rip), CONSTANT
+#else
+        movdqa  .Lconstant_RUpoly - delta(%ecx), CONSTANT
+#endif
+        movdqa  %xmm1, %xmm2
+        pand    %xmm3, %xmm1
+        PCLMULQDQ 0x10, CONSTANT, %xmm1
+        pand    %xmm3, %xmm1
+        PCLMULQDQ 0x00, CONSTANT, %xmm1
+        pxor    %xmm2, %xmm1
+        pextrd  $0x01, %xmm1, %eax
+
+        ret
diff --git a/arch/x86/crypto/crc32-pclmul_glue.c b/arch/x86/crypto/crc32-pclmul_glue.c
new file mode 100644
index 000000000000..9d014a74ef96
--- /dev/null
+++ b/arch/x86/crypto/crc32-pclmul_glue.c
@@ -0,0 +1,201 @@
+/* GPL HEADER START
+ *
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 only,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License version 2 for more details (a copy is included
+ * in the LICENSE file that accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; If not, see http://www.gnu.org/licenses
+ *
+ * Please  visit http://www.xyratex.com/contact if you need additional
+ * information or have any questions.
+ *
+ * GPL HEADER END
+ */
+
+/*
+ * Copyright 2012 Xyratex Technology Limited
+ *
+ * Wrappers for kernel crypto shash api to pclmulqdq crc32 imlementation.
+ */
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/string.h>
+#include <linux/kernel.h>
+#include <linux/crc32.h>
+#include <crypto/internal/hash.h>
+
+#include <asm/cpufeature.h>
+#include <asm/cpu_device_id.h>
+#include <asm/i387.h>
+
+#define CHKSUM_BLOCK_SIZE       1
+#define CHKSUM_DIGEST_SIZE      4
+
+#define PCLMUL_MIN_LEN          64L     /* minimum size of buffer
+                                         * for crc32_pclmul_le_16 */
+#define SCALE_F                 16L     /* size of xmm register */
+#define SCALE_F_MASK            (SCALE_F - 1)
+
+u32 crc32_pclmul_le_16(unsigned char const *buffer, size_t len, u32 crc32);
+
+static u32 __attribute__((pure))
+        crc32_pclmul_le(u32 crc, unsigned char const *p, size_t len)
+{
+        unsigned int iquotient;
+        unsigned int iremainder;
+        unsigned int prealign;
+
+        if (len < PCLMUL_MIN_LEN + SCALE_F_MASK || !irq_fpu_usable())
+                return crc32_le(crc, p, len);
+
+        if ((long)p & SCALE_F_MASK) {
+                /* align p to 16 byte */
+                prealign = SCALE_F - ((long)p & SCALE_F_MASK);
+
+                crc = crc32_le(crc, p, prealign);
+                len -= prealign;
+                p = (unsigned char *)(((unsigned long)p + SCALE_F_MASK) &
+                                     ~SCALE_F_MASK);
+        }
+        iquotient = len & (~SCALE_F_MASK);
+        iremainder = len & SCALE_F_MASK;
+
+        kernel_fpu_begin();
+        crc = crc32_pclmul_le_16(p, iquotient, crc);
+        kernel_fpu_end();
+
+        if (iremainder)
+                crc = crc32_le(crc, p + iquotient, iremainder);
+
+        return crc;
+}
+
+static int crc32_pclmul_cra_init(struct crypto_tfm *tfm)
+{
+        u32 *key = crypto_tfm_ctx(tfm);
+
+        *key = 0;
+
+        return 0;
+}
+
+static int crc32_pclmul_setkey(struct crypto_shash *hash, const u8 *key,
+                        unsigned int keylen)
+{
+        u32 *mctx = crypto_shash_ctx(hash);
+
+        if (keylen != sizeof(u32)) {
+                crypto_shash_set_flags(hash, CRYPTO_TFM_RES_BAD_KEY_LEN);
+                return -EINVAL;
+        }
+        *mctx = le32_to_cpup((__le32 *)key);
+        return 0;
+}
+
+static int crc32_pclmul_init(struct shash_desc *desc)
+{
+        u32 *mctx = crypto_shash_ctx(desc->tfm);
+        u32 *crcp = shash_desc_ctx(desc);
+
+        *crcp = *mctx;
+
+        return 0;
+}
+
+static int crc32_pclmul_update(struct shash_desc *desc, const u8 *data,
+                               unsigned int len)
+{
+        u32 *crcp = shash_desc_ctx(desc);
+
+        *crcp = crc32_pclmul_le(*crcp, data, len);
+        return 0;
+}
+
+/* No final XOR 0xFFFFFFFF, like crc32_le */
+static int __crc32_pclmul_finup(u32 *crcp, const u8 *data, unsigned int len,
+                                u8 *out)
+{
+        *(__le32 *)out = cpu_to_le32(crc32_pclmul_le(*crcp, data, len));
+        return 0;
+}
+
+static int crc32_pclmul_finup(struct shash_desc *desc, const u8 *data,
+                              unsigned int len, u8 *out)
+{
+        return __crc32_pclmul_finup(shash_desc_ctx(desc), data, len, out);
+}
+
+static int crc32_pclmul_final(struct shash_desc *desc, u8 *out)
+{
+        u32 *crcp = shash_desc_ctx(desc);
+
+        *(__le32 *)out = cpu_to_le32p(crcp);
+        return 0;
+}
+
+static int crc32_pclmul_digest(struct shash_desc *desc, const u8 *data,
+                               unsigned int len, u8 *out)
+{
+        return __crc32_pclmul_finup(crypto_shash_ctx(desc->tfm), data, len,
+                                    out);
+}
+
+static struct shash_alg alg = {
+        .setkey         = crc32_pclmul_setkey,
+        .init           = crc32_pclmul_init,
+        .update         = crc32_pclmul_update,
+        .final          = crc32_pclmul_final,
+        .finup          = crc32_pclmul_finup,
+        .digest         = crc32_pclmul_digest,
+        .descsize       = sizeof(u32),
+        .digestsize     = CHKSUM_DIGEST_SIZE,
+        .base           = {
+                        .cra_name               = "crc32",
+                        .cra_driver_name        = "crc32-pclmul",
+                        .cra_priority           = 200,
+                        .cra_blocksize          = CHKSUM_BLOCK_SIZE,
+                        .cra_ctxsize            = sizeof(u32),
+                        .cra_module             = THIS_MODULE,
+                        .cra_init               = crc32_pclmul_cra_init,
+        }
+};
+
+static const struct x86_cpu_id crc32pclmul_cpu_id[] = {
+        X86_FEATURE_MATCH(X86_FEATURE_PCLMULQDQ),
+        {}
+};
+MODULE_DEVICE_TABLE(x86cpu, crc32pclmul_cpu_id);
+
+
+static int __init crc32_pclmul_mod_init(void)
+{
+
+        if (!x86_match_cpu(crc32pclmul_cpu_id)) {
+                pr_info("PCLMULQDQ-NI instructions are not detected.\n");
+                return -ENODEV;
+        }
+        return crypto_register_shash(&alg);
+}
+
+static void __exit crc32_pclmul_mod_fini(void)
+{
+        crypto_unregister_shash(&alg);
+}
+
+module_init(crc32_pclmul_mod_init);
+module_exit(crc32_pclmul_mod_fini);
+
+MODULE_AUTHOR("Alexander Boyko <alexander_boyko@xyratex.com>");
+MODULE_LICENSE("GPL");
+
+MODULE_ALIAS("crc32");
+MODULE_ALIAS("crc32-pclmul");
diff --git a/arch/x86/crypto/crc32c-pcl-intel-asm_64.S b/arch/x86/crypto/crc32c-pcl-intel-asm_64.S
index 93c6d39237ac..cf1a7ec4cc3a 100644
--- a/arch/x86/crypto/crc32c-pcl-intel-asm_64.S
+++ b/arch/x86/crypto/crc32c-pcl-intel-asm_64.S
@@ -42,6 +42,8 @@
  * SOFTWARE.
  */
 
+#include <linux/linkage.h>
+
 ## ISCSI CRC 32 Implementation with crc32 and pclmulqdq Instruction
 
 .macro LABEL prefix n
@@ -68,8 +70,7 @@
 
 # unsigned int crc_pcl(u8 *buffer, int len, unsigned int crc_init);
 
-.global crc_pcl
-crc_pcl:
+ENTRY(crc_pcl)
 #define    bufp         %rdi
 #define    bufp_dw      %edi
 #define    bufp_w       %di
@@ -323,6 +324,9 @@ JMPTBL_ENTRY %i
 .noaltmacro
         i=i+1
 .endr
+
+ENDPROC(crc_pcl)
+
         ################################################################
         ## PCLMULQDQ tables
         ## Table is 128 entries x 2 quad words each
diff --git a/arch/x86/crypto/ghash-clmulni-intel_asm.S b/arch/x86/crypto/ghash-clmulni-intel_asm.S
index 1eb7f90cb7b9..586f41aac361 100644
--- a/arch/x86/crypto/ghash-clmulni-intel_asm.S
+++ b/arch/x86/crypto/ghash-clmulni-intel_asm.S
@@ -94,6 +94,7 @@ __clmul_gf128mul_ble:
         pxor T2, T1
         pxor T1, DATA
         ret
+ENDPROC(__clmul_gf128mul_ble)
 
 /* void clmul_ghash_mul(char *dst, const be128 *shash) */
 ENTRY(clmul_ghash_mul)
@@ -105,6 +106,7 @@ ENTRY(clmul_ghash_mul)
         PSHUFB_XMM BSWAP DATA
         movups DATA, (%rdi)
         ret
+ENDPROC(clmul_ghash_mul)
 
 /*
  * void clmul_ghash_update(char *dst, const char *src, unsigned int srclen,
@@ -131,6 +133,7 @@ ENTRY(clmul_ghash_update)
         movups DATA, (%rdi)
 .Lupdate_just_ret:
         ret
+ENDPROC(clmul_ghash_update)
 
 /*
  * void clmul_ghash_setkey(be128 *shash, const u8 *key);
@@ -155,3 +158,4 @@ ENTRY(clmul_ghash_setkey)
         pxor %xmm1, %xmm0
         movups %xmm0, (%rdi)
         ret
+ENDPROC(clmul_ghash_setkey)
diff --git a/arch/x86/crypto/salsa20-i586-asm_32.S b/arch/x86/crypto/salsa20-i586-asm_32.S
index 72eb306680b2..329452b8f794 100644
--- a/arch/x86/crypto/salsa20-i586-asm_32.S
+++ b/arch/x86/crypto/salsa20-i586-asm_32.S
@@ -2,11 +2,12 @@
 # D. J. Bernstein
 # Public domain.
 
-# enter ECRYPT_encrypt_bytes
+#include <linux/linkage.h>
+
 .text
-.p2align 5
-.globl ECRYPT_encrypt_bytes
-ECRYPT_encrypt_bytes:
+
+# enter salsa20_encrypt_bytes
+ENTRY(salsa20_encrypt_bytes)
         mov     %esp,%eax
         and     $31,%eax
         add     $256,%eax
@@ -933,11 +934,10 @@ ECRYPT_encrypt_bytes:
         add     $64,%esi
         # goto bytesatleast1
         jmp     ._bytesatleast1
-# enter ECRYPT_keysetup
-.text
-.p2align 5
-.globl ECRYPT_keysetup
-ECRYPT_keysetup:
+ENDPROC(salsa20_encrypt_bytes)
+
+# enter salsa20_keysetup
+ENTRY(salsa20_keysetup)
         mov     %esp,%eax
         and     $31,%eax
         add     $256,%eax
@@ -1060,11 +1060,10 @@ ECRYPT_keysetup:
         # leave
         add     %eax,%esp
         ret
-# enter ECRYPT_ivsetup
-.text
-.p2align 5
-.globl ECRYPT_ivsetup
-ECRYPT_ivsetup:
+ENDPROC(salsa20_keysetup)
+
+# enter salsa20_ivsetup
+ENTRY(salsa20_ivsetup)
         mov     %esp,%eax
         and     $31,%eax
         add     $256,%eax
@@ -1112,3 +1111,4 @@ ECRYPT_ivsetup:
         # leave
         add     %eax,%esp
         ret
+ENDPROC(salsa20_ivsetup)
diff --git a/arch/x86/crypto/salsa20-x86_64-asm_64.S b/arch/x86/crypto/salsa20-x86_64-asm_64.S
index 6214a9b09706..9279e0b2d60e 100644
--- a/arch/x86/crypto/salsa20-x86_64-asm_64.S
+++ b/arch/x86/crypto/salsa20-x86_64-asm_64.S
@@ -1,8 +1,7 @@
-# enter ECRYPT_encrypt_bytes
-.text
-.p2align 5
-.globl ECRYPT_encrypt_bytes
-ECRYPT_encrypt_bytes:
+#include <linux/linkage.h>
+
+# enter salsa20_encrypt_bytes
+ENTRY(salsa20_encrypt_bytes)
         mov     %rsp,%r11
         and     $31,%r11
         add     $256,%r11
@@ -802,11 +801,10 @@ ECRYPT_encrypt_bytes:
         # comment:fp stack unchanged by jump
         # goto bytesatleast1
         jmp     ._bytesatleast1
-# enter ECRYPT_keysetup
-.text
-.p2align 5
-.globl ECRYPT_keysetup
-ECRYPT_keysetup:
+ENDPROC(salsa20_encrypt_bytes)
+
+# enter salsa20_keysetup
+ENTRY(salsa20_keysetup)
         mov     %rsp,%r11
         and     $31,%r11
         add     $256,%r11
@@ -892,11 +890,10 @@ ECRYPT_keysetup:
         mov     %rdi,%rax
         mov     %rsi,%rdx
         ret
-# enter ECRYPT_ivsetup
-.text
-.p2align 5
-.globl ECRYPT_ivsetup
-ECRYPT_ivsetup:
+ENDPROC(salsa20_keysetup)
+
+# enter salsa20_ivsetup
+ENTRY(salsa20_ivsetup)
         mov     %rsp,%r11
         and     $31,%r11
         add     $256,%r11
@@ -918,3 +915,4 @@ ECRYPT_ivsetup:
         mov     %rdi,%rax
         mov     %rsi,%rdx
         ret
+ENDPROC(salsa20_ivsetup)
diff --git a/arch/x86/crypto/salsa20_glue.c b/arch/x86/crypto/salsa20_glue.c
index a3a3c0205c16..5e8e67739bb5 100644
--- a/arch/x86/crypto/salsa20_glue.c
+++ b/arch/x86/crypto/salsa20_glue.c
@@ -26,11 +26,6 @@
 #define SALSA20_MIN_KEY_SIZE  16U
 #define SALSA20_MAX_KEY_SIZE  32U
 
-// use the ECRYPT_* function names
-#define salsa20_keysetup        ECRYPT_keysetup
-#define salsa20_ivsetup         ECRYPT_ivsetup
-#define salsa20_encrypt_bytes   ECRYPT_encrypt_bytes
-
 struct salsa20_ctx
 {
         u32 input[16];
diff --git a/arch/x86/crypto/serpent-avx-x86_64-asm_64.S b/arch/x86/crypto/serpent-avx-x86_64-asm_64.S
index 02b0e9fe997c..43c938612b74 100644
--- a/arch/x86/crypto/serpent-avx-x86_64-asm_64.S
+++ b/arch/x86/crypto/serpent-avx-x86_64-asm_64.S
@@ -24,6 +24,7 @@
  *
  */
 
+#include <linux/linkage.h>
 #include "glue_helper-asm-avx.S"
 
 .file "serpent-avx-x86_64-asm_64.S"
@@ -566,8 +567,6 @@
         transpose_4x4(x0, x1, x2, x3, t0, t1, t2)
 
 .align 8
-.type   __serpent_enc_blk8_avx,@function;
-
 __serpent_enc_blk8_avx:
         /* input:
          *      %rdi: ctx, CTX
@@ -619,10 +618,9 @@ __serpent_enc_blk8_avx:
         write_blocks(RA2, RB2, RC2, RD2, RK0, RK1, RK2);
 
         ret;
+ENDPROC(__serpent_enc_blk8_avx)
 
 .align 8
-.type   __serpent_dec_blk8_avx,@function;
-
 __serpent_dec_blk8_avx:
         /* input:
          *      %rdi: ctx, CTX
@@ -674,12 +672,9 @@ __serpent_dec_blk8_avx:
         write_blocks(RC2, RD2, RB2, RE2, RK0, RK1, RK2);
 
         ret;
+ENDPROC(__serpent_dec_blk8_avx)
 
-.align 8
-.global serpent_ecb_enc_8way_avx
-.type   serpent_ecb_enc_8way_avx,@function;
-
-serpent_ecb_enc_8way_avx:
+ENTRY(serpent_ecb_enc_8way_avx)
         /* input:
          *      %rdi: ctx, CTX
          *      %rsi: dst
@@ -693,12 +688,9 @@ serpent_ecb_enc_8way_avx:
         store_8way(%rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
 
         ret;
+ENDPROC(serpent_ecb_enc_8way_avx)
 
-.align 8
-.global serpent_ecb_dec_8way_avx
-.type   serpent_ecb_dec_8way_avx,@function;
-
-serpent_ecb_dec_8way_avx:
+ENTRY(serpent_ecb_dec_8way_avx)
         /* input:
          *      %rdi: ctx, CTX
          *      %rsi: dst
@@ -712,12 +704,9 @@ serpent_ecb_dec_8way_avx:
         store_8way(%rsi, RC1, RD1, RB1, RE1, RC2, RD2, RB2, RE2);
 
         ret;
+ENDPROC(serpent_ecb_dec_8way_avx)
 
-.align 8
-.global serpent_cbc_dec_8way_avx
-.type   serpent_cbc_dec_8way_avx,@function;
-
-serpent_cbc_dec_8way_avx:
+ENTRY(serpent_cbc_dec_8way_avx)
         /* input:
          *      %rdi: ctx, CTX
          *      %rsi: dst
@@ -731,12 +720,9 @@ serpent_cbc_dec_8way_avx:
         store_cbc_8way(%rdx, %rsi, RC1, RD1, RB1, RE1, RC2, RD2, RB2, RE2);
 
         ret;
+ENDPROC(serpent_cbc_dec_8way_avx)
 
-.align 8
-.global serpent_ctr_8way_avx
-.type   serpent_ctr_8way_avx,@function;
-
-serpent_ctr_8way_avx:
+ENTRY(serpent_ctr_8way_avx)
         /* input:
          *      %rdi: ctx, CTX
          *      %rsi: dst
@@ -752,3 +738,4 @@ serpent_ctr_8way_avx:
         store_ctr_8way(%rdx, %rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
 
         ret;
+ENDPROC(serpent_ctr_8way_avx)
diff --git a/arch/x86/crypto/serpent-sse2-i586-asm_32.S b/arch/x86/crypto/serpent-sse2-i586-asm_32.S
index c00053d42f99..d348f1553a79 100644
--- a/arch/x86/crypto/serpent-sse2-i586-asm_32.S
+++ b/arch/x86/crypto/serpent-sse2-i586-asm_32.S
@@ -24,6 +24,8 @@
  *
  */
 
+#include <linux/linkage.h>
+
 .file "serpent-sse2-i586-asm_32.S"
 .text
 
@@ -510,11 +512,7 @@
         pxor t0,                x3; \
         movdqu x3,              (3*4*4)(out);
 
-.align 8
-.global __serpent_enc_blk_4way
-.type   __serpent_enc_blk_4way,@function;
-
-__serpent_enc_blk_4way:
+ENTRY(__serpent_enc_blk_4way)
         /* input:
          *      arg_ctx(%esp): ctx, CTX
          *      arg_dst(%esp): dst
@@ -566,22 +564,19 @@ __serpent_enc_blk_4way:
         movl arg_dst(%esp), %eax;
 
         cmpb $0, arg_xor(%esp);
-        jnz __enc_xor4;
+        jnz .L__enc_xor4;
 
         write_blocks(%eax, RA, RB, RC, RD, RT0, RT1, RE);
 
         ret;
 
-__enc_xor4:
+.L__enc_xor4:
         xor_blocks(%eax, RA, RB, RC, RD, RT0, RT1, RE);
 
         ret;
+ENDPROC(__serpent_enc_blk_4way)
 
-.align 8
-.global serpent_dec_blk_4way
-.type   serpent_dec_blk_4way,@function;
-
-serpent_dec_blk_4way:
+ENTRY(serpent_dec_blk_4way)
         /* input:
          *      arg_ctx(%esp): ctx, CTX
          *      arg_dst(%esp): dst
@@ -633,3 +628,4 @@ serpent_dec_blk_4way:
         write_blocks(%eax, RC, RD, RB, RE, RT0, RT1, RA);
 
         ret;
+ENDPROC(serpent_dec_blk_4way)
diff --git a/arch/x86/crypto/serpent-sse2-x86_64-asm_64.S b/arch/x86/crypto/serpent-sse2-x86_64-asm_64.S
index 3ee1ff04d3e9..acc066c7c6b2 100644
--- a/arch/x86/crypto/serpent-sse2-x86_64-asm_64.S
+++ b/arch/x86/crypto/serpent-sse2-x86_64-asm_64.S
@@ -24,6 +24,8 @@
  *
  */
 
+#include <linux/linkage.h>
+
 .file "serpent-sse2-x86_64-asm_64.S"
 .text
 
@@ -632,11 +634,7 @@
         pxor t0,                x3; \
         movdqu x3,              (3*4*4)(out);
 
-.align 8
-.global __serpent_enc_blk_8way
-.type   __serpent_enc_blk_8way,@function;
-
-__serpent_enc_blk_8way:
+ENTRY(__serpent_enc_blk_8way)
         /* input:
          *      %rdi: ctx, CTX
          *      %rsi: dst
@@ -687,24 +685,21 @@ __serpent_enc_blk_8way:
         leaq (4*4*4)(%rsi), %rax;
 
         testb %cl, %cl;
-        jnz __enc_xor8;
+        jnz .L__enc_xor8;
 
         write_blocks(%rsi, RA1, RB1, RC1, RD1, RK0, RK1, RK2);
         write_blocks(%rax, RA2, RB2, RC2, RD2, RK0, RK1, RK2);
 
         ret;
 
-__enc_xor8:
+.L__enc_xor8:
         xor_blocks(%rsi, RA1, RB1, RC1, RD1, RK0, RK1, RK2);
         xor_blocks(%rax, RA2, RB2, RC2, RD2, RK0, RK1, RK2);
 
         ret;
+ENDPROC(__serpent_enc_blk_8way)
 
-.align 8
-.global serpent_dec_blk_8way
-.type   serpent_dec_blk_8way,@function;
-
-serpent_dec_blk_8way:
+ENTRY(serpent_dec_blk_8way)
         /* input:
          *      %rdi: ctx, CTX
          *      %rsi: dst
@@ -756,3 +751,4 @@ serpent_dec_blk_8way:
         write_blocks(%rax, RC2, RD2, RB2, RE2, RK0, RK1, RK2);
 
         ret;
+ENDPROC(serpent_dec_blk_8way)
diff --git a/arch/x86/crypto/sha1_ssse3_asm.S b/arch/x86/crypto/sha1_ssse3_asm.S
index 49d6987a73d9..a4109506a5e8 100644
--- a/arch/x86/crypto/sha1_ssse3_asm.S
+++ b/arch/x86/crypto/sha1_ssse3_asm.S
@@ -28,6 +28,8 @@
  * (at your option) any later version.
  */
 
+#include <linux/linkage.h>
+
 #define CTX     %rdi    // arg1
 #define BUF     %rsi    // arg2
 #define CNT     %rdx    // arg3
@@ -69,10 +71,8 @@
  * param: function's name
  */
 .macro SHA1_VECTOR_ASM  name
-        .global \name
-        .type   \name, @function
-        .align 32
-\name:
+        ENTRY(\name)
+
         push    %rbx
         push    %rbp
         push    %r12
@@ -106,7 +106,7 @@
         pop     %rbx
         ret
 
-        .size   \name, .-\name
+        ENDPROC(\name)
 .endm
 
 /*
diff --git a/arch/x86/crypto/twofish-avx-x86_64-asm_64.S b/arch/x86/crypto/twofish-avx-x86_64-asm_64.S
index ebac16bfa830..8d3e113b2c95 100644
--- a/arch/x86/crypto/twofish-avx-x86_64-asm_64.S
+++ b/arch/x86/crypto/twofish-avx-x86_64-asm_64.S
@@ -23,6 +23,7 @@
  *
  */
 
+#include <linux/linkage.h>
 #include "glue_helper-asm-avx.S"
 
 .file "twofish-avx-x86_64-asm_64.S"
@@ -243,8 +244,6 @@
         vpxor           x3, wkey, x3;
 
 .align 8
-.type   __twofish_enc_blk8,@function;
-
 __twofish_enc_blk8:
         /* input:
          *      %rdi: ctx, CTX
@@ -284,10 +283,9 @@ __twofish_enc_blk8:
         outunpack_blocks(RC2, RD2, RA2, RB2, RK1, RX0, RY0, RK2);
 
         ret;
+ENDPROC(__twofish_enc_blk8)
 
 .align 8
-.type   __twofish_dec_blk8,@function;
-
 __twofish_dec_blk8:
         /* input:
          *      %rdi: ctx, CTX
@@ -325,12 +323,9 @@ __twofish_dec_blk8:
         outunpack_blocks(RA2, RB2, RC2, RD2, RK1, RX0, RY0, RK2);
 
         ret;
+ENDPROC(__twofish_dec_blk8)
 
-.align 8
-.global twofish_ecb_enc_8way
-.type   twofish_ecb_enc_8way,@function;
-
-twofish_ecb_enc_8way:
+ENTRY(twofish_ecb_enc_8way)
         /* input:
          *      %rdi: ctx, CTX
          *      %rsi: dst
@@ -346,12 +341,9 @@ twofish_ecb_enc_8way:
         store_8way(%r11, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2);
 
         ret;
+ENDPROC(twofish_ecb_enc_8way)
 
-.align 8
-.global twofish_ecb_dec_8way
-.type   twofish_ecb_dec_8way,@function;
-
-twofish_ecb_dec_8way:
+ENTRY(twofish_ecb_dec_8way)
         /* input:
          *      %rdi: ctx, CTX
          *      %rsi: dst
@@ -367,12 +359,9 @@ twofish_ecb_dec_8way:
         store_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
 
         ret;
+ENDPROC(twofish_ecb_dec_8way)
 
-.align 8
-.global twofish_cbc_dec_8way
-.type   twofish_cbc_dec_8way,@function;
-
-twofish_cbc_dec_8way:
+ENTRY(twofish_cbc_dec_8way)
         /* input:
          *      %rdi: ctx, CTX
          *      %rsi: dst
@@ -393,12 +382,9 @@ twofish_cbc_dec_8way:
         popq %r12;
 
         ret;
+ENDPROC(twofish_cbc_dec_8way)
 
-.align 8
-.global twofish_ctr_8way
-.type   twofish_ctr_8way,@function;
-
-twofish_ctr_8way:
+ENTRY(twofish_ctr_8way)
         /* input:
          *      %rdi: ctx, CTX
          *      %rsi: dst
@@ -421,3 +407,4 @@ twofish_ctr_8way:
         popq %r12;
 
         ret;
+ENDPROC(twofish_ctr_8way)
diff --git a/arch/x86/crypto/twofish-i586-asm_32.S b/arch/x86/crypto/twofish-i586-asm_32.S
index 658af4bb35c9..694ea4587ba7 100644
--- a/arch/x86/crypto/twofish-i586-asm_32.S
+++ b/arch/x86/crypto/twofish-i586-asm_32.S
@@ -20,6 +20,7 @@
 .file "twofish-i586-asm.S"
 .text
 
+#include <linux/linkage.h>
 #include <asm/asm-offsets.h>
 
 /* return address at 0 */
@@ -219,11 +220,7 @@
         xor     %esi,           d ## D;\
         ror     $1,             d ## D;
 
-.align 4
-.global twofish_enc_blk
-.global twofish_dec_blk
-
-twofish_enc_blk:
+ENTRY(twofish_enc_blk)
         push    %ebp                    /* save registers according to calling convention*/
         push    %ebx
         push    %esi
@@ -277,8 +274,9 @@ twofish_enc_blk:
         pop     %ebp
         mov     $1,     %eax
         ret
+ENDPROC(twofish_enc_blk)
 
-twofish_dec_blk:
+ENTRY(twofish_dec_blk)
         push    %ebp                    /* save registers according to calling convention*/
         push    %ebx
         push    %esi
@@ -333,3 +331,4 @@ twofish_dec_blk:
         pop     %ebp
         mov     $1,     %eax
         ret
+ENDPROC(twofish_dec_blk)
diff --git a/arch/x86/crypto/twofish-x86_64-asm_64-3way.S b/arch/x86/crypto/twofish-x86_64-asm_64-3way.S
index 5b012a2c5119..1c3b7ceb36d2 100644
--- a/arch/x86/crypto/twofish-x86_64-asm_64-3way.S
+++ b/arch/x86/crypto/twofish-x86_64-asm_64-3way.S
@@ -20,6 +20,8 @@
  *
  */
 
+#include <linux/linkage.h>
+
 .file "twofish-x86_64-asm-3way.S"
 .text
 
@@ -214,11 +216,7 @@
         rorq $32,                       RAB2; \
         outunpack3(mov, RIO, 2, RAB, 2);
 
-.align 8
-.global __twofish_enc_blk_3way
-.type   __twofish_enc_blk_3way,@function;
-
-__twofish_enc_blk_3way:
+ENTRY(__twofish_enc_blk_3way)
         /* input:
          *      %rdi: ctx, CTX
          *      %rsi: dst
@@ -250,7 +248,7 @@ __twofish_enc_blk_3way:
         popq %rbp; /* bool xor */
 
         testb %bpl, %bpl;
-        jnz __enc_xor3;
+        jnz .L__enc_xor3;
 
         outunpack_enc3(mov);
 
@@ -262,7 +260,7 @@ __twofish_enc_blk_3way:
         popq %r15;
         ret;
 
-__enc_xor3:
+.L__enc_xor3:
         outunpack_enc3(xor);
 
         popq %rbx;
@@ -272,11 +270,9 @@ __enc_xor3:
         popq %r14;
         popq %r15;
         ret;
+ENDPROC(__twofish_enc_blk_3way)
 
-.global twofish_dec_blk_3way
-.type   twofish_dec_blk_3way,@function;
-
-twofish_dec_blk_3way:
+ENTRY(twofish_dec_blk_3way)
         /* input:
          *      %rdi: ctx, CTX
          *      %rsi: dst
@@ -313,4 +309,4 @@ twofish_dec_blk_3way:
         popq %r14;
         popq %r15;
         ret;
-
+ENDPROC(twofish_dec_blk_3way)
diff --git a/arch/x86/crypto/twofish-x86_64-asm_64.S b/arch/x86/crypto/twofish-x86_64-asm_64.S
index 7bcf3fcc3668..a039d21986a2 100644
--- a/arch/x86/crypto/twofish-x86_64-asm_64.S
+++ b/arch/x86/crypto/twofish-x86_64-asm_64.S
@@ -20,6 +20,7 @@
 .file "twofish-x86_64-asm.S"
 .text
 
+#include <linux/linkage.h>
 #include <asm/asm-offsets.h>
 
 #define a_offset        0
@@ -214,11 +215,7 @@
         xor     %r8d,           d ## D;\
         ror     $1,             d ## D;
 
-.align 8
-.global twofish_enc_blk
-.global twofish_dec_blk
-
-twofish_enc_blk:
+ENTRY(twofish_enc_blk)
         pushq    R1
 
         /* %rdi contains the ctx address */
@@ -269,8 +266,9 @@ twofish_enc_blk:
         popq    R1
         movq    $1,%rax
         ret
+ENDPROC(twofish_enc_blk)
 
-twofish_dec_blk:
+ENTRY(twofish_dec_blk)
         pushq    R1
 
         /* %rdi contains the ctx address */
@@ -320,3 +318,4 @@ twofish_dec_blk:
         popq    R1
         movq    $1,%rax
         ret
+ENDPROC(twofish_dec_blk)
diff --git a/arch/x86/ia32/ia32_aout.c b/arch/x86/ia32/ia32_aout.c
index a703af19c281..03abf9b70011 100644
--- a/arch/x86/ia32/ia32_aout.c
+++ b/arch/x86/ia32/ia32_aout.c
@@ -271,7 +271,7 @@ static int load_aout_binary(struct linux_binprm *bprm)
         if ((N_MAGIC(ex) != ZMAGIC && N_MAGIC(ex) != OMAGIC &&
              N_MAGIC(ex) != QMAGIC && N_MAGIC(ex) != NMAGIC) ||
             N_TRSIZE(ex) || N_DRSIZE(ex) ||
-            i_size_read(bprm->file->f_path.dentry->d_inode) <
+            i_size_read(file_inode(bprm->file)) <
             ex.a_text+ex.a_data+N_SYMSIZE(ex)+N_TXTOFF(ex)) {
                 return -ENOEXEC;
         }
@@ -425,12 +425,10 @@ beyond_if:
 
 static int load_aout_library(struct file *file)
 {
-        struct inode *inode;
         unsigned long bss, start_addr, len, error;
         int retval;
         struct exec ex;
 
-        inode = file->f_path.dentry->d_inode;
 
         retval = -ENOEXEC;
         error = kernel_read(file, 0, (char *) &ex, sizeof(ex));
@@ -440,7 +438,7 @@ static int load_aout_library(struct file *file)
         /* We come in here for the regular a.out style of shared libraries */
         if ((N_MAGIC(ex) != ZMAGIC && N_MAGIC(ex) != QMAGIC) || N_TRSIZE(ex) ||
             N_DRSIZE(ex) || ((ex.a_entry & 0xfff) && N_MAGIC(ex) == ZMAGIC) ||
-            i_size_read(inode) <
+            i_size_read(file_inode(file)) <
             ex.a_text+ex.a_data+N_SYMSIZE(ex)+N_TXTOFF(ex)) {
                 goto out;
         }
diff --git a/arch/x86/ia32/ia32_signal.c b/arch/x86/ia32/ia32_signal.c
index a1daf4a65009..cf1a471a18a2 100644
--- a/arch/x86/ia32/ia32_signal.c
+++ b/arch/x86/ia32/ia32_signal.c
@@ -129,13 +129,6 @@ int copy_siginfo_from_user32(siginfo_t *to, compat_siginfo_t __user *from)
         return err;
 }
 
-asmlinkage long sys32_sigsuspend(int history0, int history1, old_sigset_t mask)
-{
-        sigset_t blocked;
-        siginitset(&blocked, mask);
-        return sigsuspend(&blocked);
-}
-
 /*
  * Do a signal return; undo the signal stack.
  */
@@ -215,8 +208,9 @@ static int ia32_restore_sigcontext(struct pt_regs *regs,
         return err;
 }
 
-asmlinkage long sys32_sigreturn(struct pt_regs *regs)
+asmlinkage long sys32_sigreturn(void)
 {
+        struct pt_regs *regs = current_pt_regs();
         struct sigframe_ia32 __user *frame = (struct sigframe_ia32 __user *)(regs->sp-8);
         sigset_t set;
         unsigned int ax;
@@ -241,8 +235,9 @@ badframe:
         return 0;
 }
 
-asmlinkage long sys32_rt_sigreturn(struct pt_regs *regs)
+asmlinkage long sys32_rt_sigreturn(void)
 {
+        struct pt_regs *regs = current_pt_regs();
         struct rt_sigframe_ia32 __user *frame;
         sigset_t set;
         unsigned int ax;
@@ -314,7 +309,7 @@ static int ia32_setup_sigcontext(struct sigcontext_ia32 __user *sc,
 /*
  * Determine which stack to use..
  */
-static void __user *get_sigframe(struct k_sigaction *ka, struct pt_regs *regs,
+static void __user *get_sigframe(struct ksignal *ksig, struct pt_regs *regs,
                                  size_t frame_size,
                                  void __user **fpstate)
 {
@@ -324,16 +319,13 @@ static void __user *get_sigframe(struct k_sigaction *ka, struct pt_regs *regs,
         sp = regs->sp;
 
         /* This is the X/Open sanctioned signal stack switching.  */
-        if (ka->sa.sa_flags & SA_ONSTACK) {
-                if (sas_ss_flags(sp) == 0)
-                        sp = current->sas_ss_sp + current->sas_ss_size;
-        }
-
+        if (ksig->ka.sa.sa_flags & SA_ONSTACK)
+                sp = sigsp(sp, ksig);
         /* This is the legacy signal stack switching. */
         else if ((regs->ss & 0xffff) != __USER32_DS &&
-                !(ka->sa.sa_flags & SA_RESTORER) &&
-                 ka->sa.sa_restorer)
-                sp = (unsigned long) ka->sa.sa_restorer;
+                !(ksig->ka.sa.sa_flags & SA_RESTORER) &&
+                 ksig->ka.sa.sa_restorer)
+                sp = (unsigned long) ksig->ka.sa.sa_restorer;
 
         if (used_math()) {
                 unsigned long fx_aligned, math_size;
@@ -352,7 +344,7 @@ static void __user *get_sigframe(struct k_sigaction *ka, struct pt_regs *regs,
         return (void __user *) sp;
 }
 
-int ia32_setup_frame(int sig, struct k_sigaction *ka,
+int ia32_setup_frame(int sig, struct ksignal *ksig,
                      compat_sigset_t *set, struct pt_regs *regs)
 {
         struct sigframe_ia32 __user *frame;
@@ -371,7 +363,7 @@ int ia32_setup_frame(int sig, struct k_sigaction *ka,
                 0x80cd,         /* int $0x80 */
         };
 
-        frame = get_sigframe(ka, regs, sizeof(*frame), &fpstate);
+        frame = get_sigframe(ksig, regs, sizeof(*frame), &fpstate);
 
         if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame)))
                 return -EFAULT;
@@ -388,8 +380,8 @@ int ia32_setup_frame(int sig, struct k_sigaction *ka,
                         return -EFAULT;
         }
 
-        if (ka->sa.sa_flags & SA_RESTORER) {
-                restorer = ka->sa.sa_restorer;
+        if (ksig->ka.sa.sa_flags & SA_RESTORER) {
+                restorer = ksig->ka.sa.sa_restorer;
         } else {
                 /* Return stub is in 32bit vsyscall page */
                 if (current->mm->context.vdso)
@@ -414,7 +406,7 @@ int ia32_setup_frame(int sig, struct k_sigaction *ka,
 
         /* Set up registers for signal handler */
         regs->sp = (unsigned long) frame;
-        regs->ip = (unsigned long) ka->sa.sa_handler;
+        regs->ip = (unsigned long) ksig->ka.sa.sa_handler;
 
         /* Make -mregparm=3 work */
         regs->ax = sig;
@@ -430,7 +422,7 @@ int ia32_setup_frame(int sig, struct k_sigaction *ka,
         return 0;
 }
 
-int ia32_setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
+int ia32_setup_rt_frame(int sig, struct ksignal *ksig,
                         compat_sigset_t *set, struct pt_regs *regs)
 {
         struct rt_sigframe_ia32 __user *frame;
@@ -451,7 +443,7 @@ int ia32_setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
                 0,
         };
 
-        frame = get_sigframe(ka, regs, sizeof(*frame), &fpstate);
+        frame = get_sigframe(ksig, regs, sizeof(*frame), &fpstate);
 
         if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame)))
                 return -EFAULT;
@@ -469,8 +461,8 @@ int ia32_setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
                 put_user_ex(0, &frame->uc.uc_link);
                 err |= __compat_save_altstack(&frame->uc.uc_stack, regs->sp);
 
-                if (ka->sa.sa_flags & SA_RESTORER)
-                        restorer = ka->sa.sa_restorer;
+                if (ksig->ka.sa.sa_flags & SA_RESTORER)
+                        restorer = ksig->ka.sa.sa_restorer;
                 else
                         restorer = VDSO32_SYMBOL(current->mm->context.vdso,
                                                  rt_sigreturn);
@@ -483,7 +475,7 @@ int ia32_setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
                 put_user_ex(*((u64 *)&code), (u64 __user *)frame->retcode);
         } put_user_catch(err);
 
-        err |= copy_siginfo_to_user32(&frame->info, info);
+        err |= copy_siginfo_to_user32(&frame->info, &ksig->info);
         err |= ia32_setup_sigcontext(&frame->uc.uc_mcontext, fpstate,
                                      regs, set->sig[0]);
         err |= __copy_to_user(&frame->uc.uc_sigmask, set, sizeof(*set));
@@ -493,7 +485,7 @@ int ia32_setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
 
         /* Set up registers for signal handler */
         regs->sp = (unsigned long) frame;
-        regs->ip = (unsigned long) ka->sa.sa_handler;
+        regs->ip = (unsigned long) ksig->ka.sa.sa_handler;
 
         /* Make -mregparm=3 work */
         regs->ax = sig;
diff --git a/arch/x86/ia32/ia32entry.S b/arch/x86/ia32/ia32entry.S
index 102ff7cb3e41..474dc1b59f72 100644
--- a/arch/x86/ia32/ia32entry.S
+++ b/arch/x86/ia32/ia32entry.S
@@ -207,7 +207,7 @@ sysexit_from_sys_call:
         testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
         jnz ia32_ret_from_sys_call
         TRACE_IRQS_ON
-        sti
+        ENABLE_INTERRUPTS(CLBR_NONE)
         movl %eax,%esi          /* second arg, syscall return value */
         cmpl $-MAX_ERRNO,%eax   /* is it an error ? */
         jbe 1f
@@ -217,7 +217,7 @@ sysexit_from_sys_call:
         call __audit_syscall_exit
         movq RAX-ARGOFFSET(%rsp),%rax   /* reload syscall return value */
         movl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),%edi
-        cli
+        DISABLE_INTERRUPTS(CLBR_NONE)
         TRACE_IRQS_OFF
         testl %edi,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
         jz \exit
@@ -456,18 +456,16 @@ ia32_badsys:
         ALIGN
 GLOBAL(\label)
         leaq \func(%rip),%rax
-        leaq -ARGOFFSET+8(%rsp),\arg    /* 8 for return address */
         jmp  ia32_ptregs_common 
         .endm
 
         CFI_STARTPROC32
 
-        PTREGSCALL stub32_rt_sigreturn, sys32_rt_sigreturn, %rdi
-        PTREGSCALL stub32_sigreturn, sys32_sigreturn, %rdi
-        PTREGSCALL stub32_execve, compat_sys_execve, %rcx
-        PTREGSCALL stub32_fork, sys_fork, %rdi
-        PTREGSCALL stub32_vfork, sys_vfork, %rdi
-        PTREGSCALL stub32_iopl, sys_iopl, %rsi
+        PTREGSCALL stub32_rt_sigreturn, sys32_rt_sigreturn
+        PTREGSCALL stub32_sigreturn, sys32_sigreturn
+        PTREGSCALL stub32_execve, compat_sys_execve
+        PTREGSCALL stub32_fork, sys_fork
+        PTREGSCALL stub32_vfork, sys_vfork
 
         ALIGN
 GLOBAL(stub32_clone)
diff --git a/arch/x86/ia32/sys_ia32.c b/arch/x86/ia32/sys_ia32.c
index d0b689ba7be2..ad7a20cbc699 100644
--- a/arch/x86/ia32/sys_ia32.c
+++ b/arch/x86/ia32/sys_ia32.c
@@ -172,183 +172,12 @@ asmlinkage long sys32_mprotect(unsigned long start, size_t len,
         return sys_mprotect(start, len, prot);
 }
 
-asmlinkage long sys32_rt_sigaction(int sig, struct sigaction32 __user *act,
-                                   struct sigaction32 __user *oact,
-                                   unsigned int sigsetsize)
-{
-        struct k_sigaction new_ka, old_ka;
-        int ret;
-        compat_sigset_t set32;
-
-        /* XXX: Don't preclude handling different sized sigset_t's.  */
-        if (sigsetsize != sizeof(compat_sigset_t))
-                return -EINVAL;
-
-        if (act) {
-                compat_uptr_t handler, restorer;
-
-                if (!access_ok(VERIFY_READ, act, sizeof(*act)) ||
-                    __get_user(handler, &act->sa_handler) ||
-                    __get_user(new_ka.sa.sa_flags, &act->sa_flags) ||
-                    __get_user(restorer, &act->sa_restorer) ||
-                    __copy_from_user(&set32, &act->sa_mask,
-                                     sizeof(compat_sigset_t)))
-                        return -EFAULT;
-                new_ka.sa.sa_handler = compat_ptr(handler);
-                new_ka.sa.sa_restorer = compat_ptr(restorer);
-
-                /*
-                 * FIXME: here we rely on _COMPAT_NSIG_WORS to be >=
-                 * than _NSIG_WORDS << 1
-                 */
-                switch (_NSIG_WORDS) {
-                case 4: new_ka.sa.sa_mask.sig[3] = set32.sig[6]
-                                | (((long)set32.sig[7]) << 32);
-                case 3: new_ka.sa.sa_mask.sig[2] = set32.sig[4]
-                                | (((long)set32.sig[5]) << 32);
-                case 2: new_ka.sa.sa_mask.sig[1] = set32.sig[2]
-                                | (((long)set32.sig[3]) << 32);
-                case 1: new_ka.sa.sa_mask.sig[0] = set32.sig[0]
-                                | (((long)set32.sig[1]) << 32);
-                }
-        }
-
-        ret = do_sigaction(sig, act ? &new_ka : NULL, oact ? &old_ka : NULL);
-
-        if (!ret && oact) {
-                /*
-                 * FIXME: here we rely on _COMPAT_NSIG_WORS to be >=
-                 * than _NSIG_WORDS << 1
-                 */
-                switch (_NSIG_WORDS) {
-                case 4:
-                        set32.sig[7] = (old_ka.sa.sa_mask.sig[3] >> 32);
-                        set32.sig[6] = old_ka.sa.sa_mask.sig[3];
-                case 3:
-                        set32.sig[5] = (old_ka.sa.sa_mask.sig[2] >> 32);
-                        set32.sig[4] = old_ka.sa.sa_mask.sig[2];
-                case 2:
-                        set32.sig[3] = (old_ka.sa.sa_mask.sig[1] >> 32);
-                        set32.sig[2] = old_ka.sa.sa_mask.sig[1];
-                case 1:
-                        set32.sig[1] = (old_ka.sa.sa_mask.sig[0] >> 32);
-                        set32.sig[0] = old_ka.sa.sa_mask.sig[0];
-                }
-                if (!access_ok(VERIFY_WRITE, oact, sizeof(*oact)) ||
-                    __put_user(ptr_to_compat(old_ka.sa.sa_handler),
-                               &oact->sa_handler) ||
-                    __put_user(ptr_to_compat(old_ka.sa.sa_restorer),
-                               &oact->sa_restorer) ||
-                    __put_user(old_ka.sa.sa_flags, &oact->sa_flags) ||
-                    __copy_to_user(&oact->sa_mask, &set32,
-                                   sizeof(compat_sigset_t)))
-                        return -EFAULT;
-        }
-
-        return ret;
-}
-
-asmlinkage long sys32_sigaction(int sig, struct old_sigaction32 __user *act,
-                                struct old_sigaction32 __user *oact)
-{
-        struct k_sigaction new_ka, old_ka;
-        int ret;
-
-        if (act) {
-                compat_old_sigset_t mask;
-                compat_uptr_t handler, restorer;
-
-                if (!access_ok(VERIFY_READ, act, sizeof(*act)) ||
-                    __get_user(handler, &act->sa_handler) ||
-                    __get_user(new_ka.sa.sa_flags, &act->sa_flags) ||
-                    __get_user(restorer, &act->sa_restorer) ||
-                    __get_user(mask, &act->sa_mask))
-                        return -EFAULT;
-
-                new_ka.sa.sa_handler = compat_ptr(handler);
-                new_ka.sa.sa_restorer = compat_ptr(restorer);
-
-                siginitset(&new_ka.sa.sa_mask, mask);
-        }
-
-        ret = do_sigaction(sig, act ? &new_ka : NULL, oact ? &old_ka : NULL);
-
-        if (!ret && oact) {
-                if (!access_ok(VERIFY_WRITE, oact, sizeof(*oact)) ||
-                    __put_user(ptr_to_compat(old_ka.sa.sa_handler),
-                               &oact->sa_handler) ||
-                    __put_user(ptr_to_compat(old_ka.sa.sa_restorer),
-                               &oact->sa_restorer) ||
-                    __put_user(old_ka.sa.sa_flags, &oact->sa_flags) ||
-                    __put_user(old_ka.sa.sa_mask.sig[0], &oact->sa_mask))
-                        return -EFAULT;
-        }
-
-        return ret;
-}
-
 asmlinkage long sys32_waitpid(compat_pid_t pid, unsigned int __user *stat_addr,
                               int options)
 {
         return compat_sys_wait4(pid, stat_addr, options, NULL);
 }
 
-/* 32-bit timeval and related flotsam.  */
-
-asmlinkage long sys32_sched_rr_get_interval(compat_pid_t pid,
-                                    struct compat_timespec __user *interval)
-{
-        struct timespec t;
-        int ret;
-        mm_segment_t old_fs = get_fs();
-
-        set_fs(KERNEL_DS);
-        ret = sys_sched_rr_get_interval(pid, (struct timespec __user *)&t);
-        set_fs(old_fs);
-        if (put_compat_timespec(&t, interval))
-                return -EFAULT;
-        return ret;
-}
-
-asmlinkage long sys32_rt_sigpending(compat_sigset_t __user *set,
-                                    compat_size_t sigsetsize)
-{
-        sigset_t s;
-        compat_sigset_t s32;
-        int ret;
-        mm_segment_t old_fs = get_fs();
-
-        set_fs(KERNEL_DS);
-        ret = sys_rt_sigpending((sigset_t __user *)&s, sigsetsize);
-        set_fs(old_fs);
-        if (!ret) {
-                switch (_NSIG_WORDS) {
-                case 4: s32.sig[7] = (s.sig[3] >> 32); s32.sig[6] = s.sig[3];
-                case 3: s32.sig[5] = (s.sig[2] >> 32); s32.sig[4] = s.sig[2];
-                case 2: s32.sig[3] = (s.sig[1] >> 32); s32.sig[2] = s.sig[1];
-                case 1: s32.sig[1] = (s.sig[0] >> 32); s32.sig[0] = s.sig[0];
-                }
-                if (copy_to_user(set, &s32, sizeof(compat_sigset_t)))
-                        return -EFAULT;
-        }
-        return ret;
-}
-
-asmlinkage long sys32_rt_sigqueueinfo(int pid, int sig,
-                                      compat_siginfo_t __user *uinfo)
-{
-        siginfo_t info;
-        int ret;
-        mm_segment_t old_fs = get_fs();
-
-        if (copy_siginfo_from_user32(&info, uinfo))
-                return -EFAULT;
-        set_fs(KERNEL_DS);
-        ret = sys_rt_sigqueueinfo(pid, sig, (siginfo_t __user *)&info);
-        set_fs(old_fs);
-        return ret;
-}
-
 /* warning: next two assume little endian */
 asmlinkage long sys32_pread(unsigned int fd, char __user *ubuf, u32 count,
                             u32 poslo, u32 poshi)
@@ -389,11 +218,6 @@ asmlinkage long sys32_sendfile(int out_fd, int in_fd,
  * Some system calls that need sign extended arguments. This could be
  * done by a generic wrapper.
  */
-long sys32_lseek(unsigned int fd, int offset, unsigned int whence)
-{
-        return sys_lseek(fd, offset, whence);
-}
-
 long sys32_kill(int pid, int sig)
 {
         return sys_kill(pid, sig);
diff --git a/arch/x86/include/asm/acpi.h b/arch/x86/include/asm/acpi.h
index 0c44630d1789..b31bf97775fc 100644
--- a/arch/x86/include/asm/acpi.h
+++ b/arch/x86/include/asm/acpi.h
@@ -49,10 +49,6 @@
 
 /* Asm macros */
 
-#define ACPI_ASM_MACROS
-#define BREAKPOINT3
-#define ACPI_DISABLE_IRQS() local_irq_disable()
-#define ACPI_ENABLE_IRQS()  local_irq_enable()
 #define ACPI_FLUSH_CPU_CACHE()  wbinvd()
 
 int __acpi_acquire_global_lock(unsigned int *lock);
diff --git a/arch/x86/include/asm/amd_nb.h b/arch/x86/include/asm/amd_nb.h
index b3341e9cd8fd..a54ee1d054d9 100644
--- a/arch/x86/include/asm/amd_nb.h
+++ b/arch/x86/include/asm/amd_nb.h
@@ -81,6 +81,23 @@ static inline struct amd_northbridge *node_to_amd_nb(int node)
         return (node < amd_northbridges.num) ? &amd_northbridges.nb[node] : NULL;
 }
 
+static inline u16 amd_get_node_id(struct pci_dev *pdev)
+{
+        struct pci_dev *misc;
+        int i;
+
+        for (i = 0; i != amd_nb_num(); i++) {
+                misc = node_to_amd_nb(i)->misc;
+
+                if (pci_domain_nr(misc->bus) == pci_domain_nr(pdev->bus) &&
+                    PCI_SLOT(misc->devfn) == PCI_SLOT(pdev->devfn))
+                        return i;
+        }
+
+        WARN(1, "Unable to find AMD Northbridge id for %s\n", pci_name(pdev));
+        return 0;
+}
+
 #else
 
 #define amd_nb_num(x)           0
diff --git a/arch/x86/include/asm/bootparam_utils.h b/arch/x86/include/asm/bootparam_utils.h
new file mode 100644
index 000000000000..653668d140f9
--- /dev/null
+++ b/arch/x86/include/asm/bootparam_utils.h
@@ -0,0 +1,54 @@
+#ifndef _ASM_X86_BOOTPARAM_UTILS_H
+#define _ASM_X86_BOOTPARAM_UTILS_H
+
+#include <asm/bootparam.h>
+
+/*
+ * This file is included from multiple environments.  Do not
+ * add completing #includes to make it standalone.
+ */
+
+/*
+ * Deal with bootloaders which fail to initialize unknown fields in
+ * boot_params to zero.  The list fields in this list are taken from
+ * analysis of kexec-tools; if other broken bootloaders initialize a
+ * different set of fields we will need to figure out how to disambiguate.
+ *
+ * Note: efi_info is commonly left uninitialized, but that field has a
+ * private magic, so it is better to leave it unchanged.
+ */
+static void sanitize_boot_params(struct boot_params *boot_params)
+{
+        /* 
+         * IMPORTANT NOTE TO BOOTLOADER AUTHORS: do not simply clear
+         * this field.  The purpose of this field is to guarantee
+         * compliance with the x86 boot spec located in
+         * Documentation/x86/boot.txt .  That spec says that the
+         * *whole* structure should be cleared, after which only the
+         * portion defined by struct setup_header (boot_params->hdr)
+         * should be copied in.
+         *
+         * If you're having an issue because the sentinel is set, you
+         * need to change the whole structure to be cleared, not this
+         * (or any other) individual field, or you will soon have
+         * problems again.
+         */
+        if (boot_params->sentinel) {
+                /* fields in boot_params are left uninitialized, clear them */
+                memset(&boot_params->olpc_ofw_header, 0,
+                       (char *)&boot_params->efi_info -
+                        (char *)&boot_params->olpc_ofw_header);
+                memset(&boot_params->kbd_status, 0,
+                       (char *)&boot_params->hdr -
+                       (char *)&boot_params->kbd_status);
+                memset(&boot_params->_pad7[0], 0,
+                       (char *)&boot_params->edd_mbr_sig_buffer[0] -
+                        (char *)&boot_params->_pad7[0]);
+                memset(&boot_params->_pad8[0], 0,
+                       (char *)&boot_params->eddbuf[0] -
+                        (char *)&boot_params->_pad8[0]);
+                memset(&boot_params->_pad9[0], 0, sizeof(boot_params->_pad9));
+        }
+}
+
+#endif /* _ASM_X86_BOOTPARAM_UTILS_H */
diff --git a/arch/x86/include/asm/cpufeature.h b/arch/x86/include/asm/cpufeature.h
index 2d9075e863a0..93fe929d1cee 100644
--- a/arch/x86/include/asm/cpufeature.h
+++ b/arch/x86/include/asm/cpufeature.h
@@ -167,6 +167,7 @@
 #define X86_FEATURE_TBM         (6*32+21) /* trailing bit manipulations */
 #define X86_FEATURE_TOPOEXT     (6*32+22) /* topology extensions CPUID leafs */
 #define X86_FEATURE_PERFCTR_CORE (6*32+23) /* core performance counter extensions */
+#define X86_FEATURE_PERFCTR_NB  (6*32+24) /* NB performance counter extensions */
 
 /*
  * Auxiliary flags: Linux defined - For features scattered in various
@@ -309,6 +310,7 @@ extern const char * const x86_power_flags[32];
 #define cpu_has_hypervisor      boot_cpu_has(X86_FEATURE_HYPERVISOR)
 #define cpu_has_pclmulqdq       boot_cpu_has(X86_FEATURE_PCLMULQDQ)
 #define cpu_has_perfctr_core    boot_cpu_has(X86_FEATURE_PERFCTR_CORE)
+#define cpu_has_perfctr_nb      boot_cpu_has(X86_FEATURE_PERFCTR_NB)
 #define cpu_has_cx8             boot_cpu_has(X86_FEATURE_CX8)
 #define cpu_has_cx16            boot_cpu_has(X86_FEATURE_CX16)
 #define cpu_has_eager_fpu       boot_cpu_has(X86_FEATURE_EAGER_FPU)
diff --git a/arch/x86/include/asm/fpu-internal.h b/arch/x86/include/asm/fpu-internal.h
index 41ab26ea6564..e25cc33ec54d 100644
--- a/arch/x86/include/asm/fpu-internal.h
+++ b/arch/x86/include/asm/fpu-internal.h
@@ -26,9 +26,10 @@
 #ifdef CONFIG_X86_64
 # include <asm/sigcontext32.h>
 # include <asm/user32.h>
-int ia32_setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
+struct ksignal;
+int ia32_setup_rt_frame(int sig, struct ksignal *ksig,
                         compat_sigset_t *set, struct pt_regs *regs);
-int ia32_setup_frame(int sig, struct k_sigaction *ka,
+int ia32_setup_frame(int sig, struct ksignal *ksig,
                      compat_sigset_t *set, struct pt_regs *regs);
 #else
 # define user_i387_ia32_struct  user_i387_struct
diff --git a/arch/x86/include/asm/ftrace.h b/arch/x86/include/asm/ftrace.h
index 9a25b522d377..0525a8bdf65d 100644
--- a/arch/x86/include/asm/ftrace.h
+++ b/arch/x86/include/asm/ftrace.h
@@ -44,7 +44,6 @@
 
 #ifdef CONFIG_DYNAMIC_FTRACE
 #define ARCH_SUPPORTS_FTRACE_OPS 1
-#define ARCH_SUPPORTS_FTRACE_SAVE_REGS
 #endif
 
 #ifndef __ASSEMBLY__
@@ -73,4 +72,28 @@ int ftrace_int3_handler(struct pt_regs *regs);
 #endif /* __ASSEMBLY__ */
 #endif /* CONFIG_FUNCTION_TRACER */
 
+
+#if !defined(__ASSEMBLY__) && !defined(COMPILE_OFFSETS)
+
+#if defined(CONFIG_FTRACE_SYSCALLS) && defined(CONFIG_IA32_EMULATION)
+#include <asm/compat.h>
+
+/*
+ * Because ia32 syscalls do not map to x86_64 syscall numbers
+ * this screws up the trace output when tracing a ia32 task.
+ * Instead of reporting bogus syscalls, just do not trace them.
+ *
+ * If the user realy wants these, then they should use the
+ * raw syscall tracepoints with filtering.
+ */
+#define ARCH_TRACE_IGNORE_COMPAT_SYSCALLS 1
+static inline bool arch_trace_is_compat_syscall(struct pt_regs *regs)
+{
+        if (is_compat_task())
+                return true;
+        return false;
+}
+#endif /* CONFIG_FTRACE_SYSCALLS && CONFIG_IA32_EMULATION */
+#endif /* !__ASSEMBLY__  && !COMPILE_OFFSETS */
+
 #endif /* _ASM_X86_FTRACE_H */
diff --git a/arch/x86/include/asm/hpet.h b/arch/x86/include/asm/hpet.h
index 434e2106cc87..b18df579c0e9 100644
--- a/arch/x86/include/asm/hpet.h
+++ b/arch/x86/include/asm/hpet.h
@@ -80,9 +80,9 @@ extern void hpet_msi_write(struct hpet_dev *hdev, struct msi_msg *msg);
 extern void hpet_msi_read(struct hpet_dev *hdev, struct msi_msg *msg);
 
 #ifdef CONFIG_PCI_MSI
-extern int arch_setup_hpet_msi(unsigned int irq, unsigned int id);
+extern int default_setup_hpet_msi(unsigned int irq, unsigned int id);
 #else
-static inline int arch_setup_hpet_msi(unsigned int irq, unsigned int id)
+static inline int default_setup_hpet_msi(unsigned int irq, unsigned int id)
 {
         return -EINVAL;
 }
@@ -111,6 +111,7 @@ extern void hpet_unregister_irq_handler(rtc_irq_handler handler);
 static inline int hpet_enable(void) { return 0; }
 static inline int is_hpet_enabled(void) { return 0; }
 #define hpet_readl(a) 0
+#define default_setup_hpet_msi  NULL
 
 #endif
 #endif /* _ASM_X86_HPET_H */
diff --git a/arch/x86/include/asm/hw_irq.h b/arch/x86/include/asm/hw_irq.h
index eb92a6ed2be7..10a78c3d3d5a 100644
--- a/arch/x86/include/asm/hw_irq.h
+++ b/arch/x86/include/asm/hw_irq.h
@@ -101,6 +101,7 @@ static inline void set_io_apic_irq_attr(struct io_apic_irq_attr *irq_attr,
         irq_attr->polarity      = polarity;
 }
 
+/* Intel specific interrupt remapping information */
 struct irq_2_iommu {
         struct intel_iommu *iommu;
         u16 irte_index;
@@ -108,6 +109,12 @@ struct irq_2_iommu {
         u8  irte_mask;
 };
 
+/* AMD specific interrupt remapping information */
+struct irq_2_irte {
+        u16 devid; /* Device ID for IRTE table */
+        u16 index; /* Index into IRTE table*/
+};
+
 /*
  * This is performance-critical, we want to do it O(1)
  *
@@ -120,7 +127,11 @@ struct irq_cfg {
         u8                      vector;
         u8                      move_in_progress : 1;
 #ifdef CONFIG_IRQ_REMAP
-        struct irq_2_iommu      irq_2_iommu;
+        u8                      remapped : 1;
+        union {
+                struct irq_2_iommu irq_2_iommu;
+                struct irq_2_irte  irq_2_irte;
+        };
 #endif
 };
 
diff --git a/arch/x86/include/asm/hypervisor.h b/arch/x86/include/asm/hypervisor.h
index b518c7509933..86095ed14135 100644
--- a/arch/x86/include/asm/hypervisor.h
+++ b/arch/x86/include/asm/hypervisor.h
@@ -25,6 +25,7 @@
 
 extern void init_hypervisor(struct cpuinfo_x86 *c);
 extern void init_hypervisor_platform(void);
+extern bool hypervisor_x2apic_available(void);
 
 /*
  * x86 hypervisor information
@@ -41,6 +42,9 @@ struct hypervisor_x86 {
 
         /* Platform setup (run once per boot) */
         void            (*init_platform)(void);
+
+        /* X2APIC detection (run once per boot) */
+        bool            (*x2apic_available)(void);
 };
 
 extern const struct hypervisor_x86 *x86_hyper;
@@ -51,13 +55,4 @@ extern const struct hypervisor_x86 x86_hyper_ms_hyperv;
 extern const struct hypervisor_x86 x86_hyper_xen_hvm;
 extern const struct hypervisor_x86 x86_hyper_kvm;
 
-static inline bool hypervisor_x2apic_available(void)
-{
-        if (kvm_para_available())
-                return true;
-        if (xen_x2apic_para_available())
-                return true;
-        return false;
-}
-
 #endif
diff --git a/arch/x86/include/asm/ia32.h b/arch/x86/include/asm/ia32.h
index 4c6da2e4bb1d..d0e8e0141041 100644
--- a/arch/x86/include/asm/ia32.h
+++ b/arch/x86/include/asm/ia32.h
@@ -13,21 +13,6 @@
 #include <asm/sigcontext32.h>
 
 /* signal.h */
-struct sigaction32 {
-        unsigned int  sa_handler;       /* Really a pointer, but need to deal
-                                           with 32 bits */
-        unsigned int sa_flags;
-        unsigned int sa_restorer;       /* Another 32 bit pointer */
-        compat_sigset_t sa_mask;        /* A 32 bit mask */
-};
-
-struct old_sigaction32 {
-        unsigned int  sa_handler;       /* Really a pointer, but need to deal
-                                           with 32 bits */
-        compat_old_sigset_t sa_mask;    /* A 32 bit mask */
-        unsigned int sa_flags;
-        unsigned int sa_restorer;       /* Another 32 bit pointer */
-};
 
 struct ucontext_ia32 {
         unsigned int      uc_flags;
diff --git a/arch/x86/include/asm/init.h b/arch/x86/include/asm/init.h
index adcc0ae73d09..223042086f4e 100644
--- a/arch/x86/include/asm/init.h
+++ b/arch/x86/include/asm/init.h
@@ -1,20 +1,14 @@
-#ifndef _ASM_X86_INIT_32_H
-#define _ASM_X86_INIT_32_H
+#ifndef _ASM_X86_INIT_H
+#define _ASM_X86_INIT_H
 
-#ifdef CONFIG_X86_32
-extern void __init early_ioremap_page_table_range_init(void);
-#endif
+struct x86_mapping_info {
+        void *(*alloc_pgt_page)(void *); /* allocate buf for page table */
+        void *context;                   /* context for alloc_pgt_page */
+        unsigned long pmd_flag;          /* page flag for PMD entry */
+        bool kernel_mapping;             /* kernel mapping or ident mapping */
+};
 
-extern void __init zone_sizes_init(void);
+int kernel_ident_mapping_init(struct x86_mapping_info *info, pgd_t *pgd_page,
+                                unsigned long addr, unsigned long end);
 
-extern unsigned long __init
-kernel_physical_mapping_init(unsigned long start,
-                             unsigned long end,
-                             unsigned long page_size_mask);
-
-
-extern unsigned long __initdata pgt_buf_start;
-extern unsigned long __meminitdata pgt_buf_end;
-extern unsigned long __meminitdata pgt_buf_top;
-
-#endif /* _ASM_X86_INIT_32_H */
+#endif /* _ASM_X86_INIT_H */
diff --git a/arch/x86/include/asm/io_apic.h b/arch/x86/include/asm/io_apic.h
index 73d8c5398ea9..459e50a424d1 100644
--- a/arch/x86/include/asm/io_apic.h
+++ b/arch/x86/include/asm/io_apic.h
@@ -144,11 +144,24 @@ extern int timer_through_8259;
         (mp_irq_entries && !skip_ioapic_setup && io_apic_irqs)
 
 struct io_apic_irq_attr;
+struct irq_cfg;
 extern int io_apic_set_pci_routing(struct device *dev, int irq,
                  struct io_apic_irq_attr *irq_attr);
 void setup_IO_APIC_irq_extra(u32 gsi);
 extern void ioapic_insert_resources(void);
 
+extern int native_setup_ioapic_entry(int, struct IO_APIC_route_entry *,
+                                     unsigned int, int,
+                                     struct io_apic_irq_attr *);
+extern int native_setup_ioapic_entry(int, struct IO_APIC_route_entry *,
+                                     unsigned int, int,
+                                     struct io_apic_irq_attr *);
+extern void eoi_ioapic_irq(unsigned int irq, struct irq_cfg *cfg);
+
+extern void native_compose_msi_msg(struct pci_dev *pdev,
+                                   unsigned int irq, unsigned int dest,
+                                   struct msi_msg *msg, u8 hpet_id);
+extern void native_eoi_ioapic_pin(int apic, int pin, int vector);
 int io_apic_setup_irq_pin_once(unsigned int irq, int node, struct io_apic_irq_attr *attr);
 
 extern int save_ioapic_entries(void);
@@ -179,6 +192,12 @@ extern void __init native_io_apic_init_mappings(void);
 extern unsigned int native_io_apic_read(unsigned int apic, unsigned int reg);
 extern void native_io_apic_write(unsigned int apic, unsigned int reg, unsigned int val);
 extern void native_io_apic_modify(unsigned int apic, unsigned int reg, unsigned int val);
+extern void native_disable_io_apic(void);
+extern void native_io_apic_print_entries(unsigned int apic, unsigned int nr_entries);
+extern void intel_ir_io_apic_print_entries(unsigned int apic, unsigned int nr_entries);
+extern int native_ioapic_set_affinity(struct irq_data *,
+                                      const struct cpumask *,
+                                      bool);
 
 static inline unsigned int io_apic_read(unsigned int apic, unsigned int reg)
 {
@@ -193,6 +212,9 @@ static inline void io_apic_modify(unsigned int apic, unsigned int reg, unsigned
 {
         x86_io_apic_ops.modify(apic, reg, value);
 }
+
+extern void io_apic_eoi(unsigned int apic, unsigned int vector);
+
 #else  /* !CONFIG_X86_IO_APIC */
 
 #define io_apic_assign_pci_irqs 0
@@ -223,6 +245,12 @@ static inline void disable_ioapic_support(void) { }
 #define native_io_apic_read             NULL
 #define native_io_apic_write            NULL
 #define native_io_apic_modify           NULL
+#define native_disable_io_apic          NULL
+#define native_io_apic_print_entries    NULL
+#define native_ioapic_set_affinity      NULL
+#define native_setup_ioapic_entry       NULL
+#define native_compose_msi_msg          NULL
+#define native_eoi_ioapic_pin           NULL
 #endif
 
 #endif /* _ASM_X86_IO_APIC_H */
diff --git a/arch/x86/include/asm/irq_remapping.h b/arch/x86/include/asm/irq_remapping.h
index 5fb9bbbd2f14..95fd3527f632 100644
--- a/arch/x86/include/asm/irq_remapping.h
+++ b/arch/x86/include/asm/irq_remapping.h
@@ -26,8 +26,6 @@
 
 #ifdef CONFIG_IRQ_REMAP
 
-extern int irq_remapping_enabled;
-
 extern void setup_irq_remapping_ops(void);
 extern int irq_remapping_supported(void);
 extern int irq_remapping_prepare(void);
@@ -40,21 +38,19 @@ extern int setup_ioapic_remapped_entry(int irq,
                                        unsigned int destination,
                                        int vector,
                                        struct io_apic_irq_attr *attr);
-extern int set_remapped_irq_affinity(struct irq_data *data,
-                                     const struct cpumask *mask,
-                                     bool force);
 extern void free_remapped_irq(int irq);
 extern void compose_remapped_msi_msg(struct pci_dev *pdev,
                                      unsigned int irq, unsigned int dest,
                                      struct msi_msg *msg, u8 hpet_id);
-extern int msi_alloc_remapped_irq(struct pci_dev *pdev, int irq, int nvec);
-extern int msi_setup_remapped_irq(struct pci_dev *pdev, unsigned int irq,
-                                  int index, int sub_handle);
 extern int setup_hpet_msi_remapped(unsigned int irq, unsigned int id);
+extern void panic_if_irq_remap(const char *msg);
+extern bool setup_remapped_irq(int irq,
+                               struct irq_cfg *cfg,
+                               struct irq_chip *chip);
 
-#else  /* CONFIG_IRQ_REMAP */
+void irq_remap_modify_chip_defaults(struct irq_chip *chip);
 
-#define irq_remapping_enabled   0
+#else  /* CONFIG_IRQ_REMAP */
 
 static inline void setup_irq_remapping_ops(void) { }
 static inline int irq_remapping_supported(void) { return 0; }
@@ -71,30 +67,30 @@ static inline int setup_ioapic_remapped_entry(int irq,
 {
         return -ENODEV;
 }
-static inline int set_remapped_irq_affinity(struct irq_data *data,
-                                            const struct cpumask *mask,
-                                            bool force)
-{
-        return 0;
-}
 static inline void free_remapped_irq(int irq) { }
 static inline void compose_remapped_msi_msg(struct pci_dev *pdev,
                                             unsigned int irq, unsigned int dest,
                                             struct msi_msg *msg, u8 hpet_id)
 {
 }
-static inline int msi_alloc_remapped_irq(struct pci_dev *pdev, int irq, int nvec)
+static inline int setup_hpet_msi_remapped(unsigned int irq, unsigned int id)
 {
         return -ENODEV;
 }
-static inline int msi_setup_remapped_irq(struct pci_dev *pdev, unsigned int irq,
-                                         int index, int sub_handle)
+
+static inline void panic_if_irq_remap(const char *msg)
+{
+}
+
+static inline void irq_remap_modify_chip_defaults(struct irq_chip *chip)
 {
-        return -ENODEV;
 }
-static inline int setup_hpet_msi_remapped(unsigned int irq, unsigned int id)
+
+static inline bool setup_remapped_irq(int irq,
+                                      struct irq_cfg *cfg,
+                                      struct irq_chip *chip)
 {
-        return -ENODEV;
+        return false;
 }
 #endif /* CONFIG_IRQ_REMAP */
 
diff --git a/arch/x86/include/asm/irq_vectors.h b/arch/x86/include/asm/irq_vectors.h
index 1508e518c7e3..aac5fa62a86c 100644
--- a/arch/x86/include/asm/irq_vectors.h
+++ b/arch/x86/include/asm/irq_vectors.h
@@ -109,8 +109,8 @@
 
 #define UV_BAU_MESSAGE                  0xf5
 
-/* Xen vector callback to receive events in a HVM domain */
-#define XEN_HVM_EVTCHN_CALLBACK         0xf3
+/* Vector on which hypervisor callbacks will be delivered */
+#define HYPERVISOR_CALLBACK_VECTOR      0xf3
 
 /*
  * Local APIC timer IRQ vector is on a different priority level,
diff --git a/arch/x86/include/asm/kexec.h b/arch/x86/include/asm/kexec.h
index 6080d2694bad..17483a492f18 100644
--- a/arch/x86/include/asm/kexec.h
+++ b/arch/x86/include/asm/kexec.h
@@ -48,11 +48,11 @@
 # define vmcore_elf_check_arch_cross(x) ((x)->e_machine == EM_X86_64)
 #else
 /* Maximum physical address we can use pages from */
-# define KEXEC_SOURCE_MEMORY_LIMIT      (0xFFFFFFFFFFUL)
+# define KEXEC_SOURCE_MEMORY_LIMIT      (MAXMEM-1)
 /* Maximum address we can reach in physical address mode */
-# define KEXEC_DESTINATION_MEMORY_LIMIT (0xFFFFFFFFFFUL)
+# define KEXEC_DESTINATION_MEMORY_LIMIT (MAXMEM-1)
 /* Maximum address we can use for the control pages */
-# define KEXEC_CONTROL_MEMORY_LIMIT     (0xFFFFFFFFFFUL)
+# define KEXEC_CONTROL_MEMORY_LIMIT     (MAXMEM-1)
 
 /* Allocate one page for the pdp and the second for the code */
 # define KEXEC_CONTROL_PAGE_SIZE  (4096UL + 4096UL)
diff --git a/arch/x86/include/asm/kprobes.h b/arch/x86/include/asm/kprobes.h
index d3ddd17405d0..5a6d2873f80e 100644
--- a/arch/x86/include/asm/kprobes.h
+++ b/arch/x86/include/asm/kprobes.h
@@ -77,6 +77,7 @@ struct arch_specific_insn {
          * a post_handler or break_handler).
          */
         int boostable;
+        bool if_modifier;
 };
 
 struct arch_optimized_insn {
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index dc87b65e9c3a..4979778cc7fb 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -33,10 +33,10 @@
 
 #define KVM_MAX_VCPUS 254
 #define KVM_SOFT_MAX_VCPUS 160
-#define KVM_MEMORY_SLOTS 32
-/* memory slots that does not exposed to userspace */
-#define KVM_PRIVATE_MEM_SLOTS 4
-#define KVM_MEM_SLOTS_NUM (KVM_MEMORY_SLOTS + KVM_PRIVATE_MEM_SLOTS)
+#define KVM_USER_MEM_SLOTS 125
+/* memory slots that are not exposed to userspace */
+#define KVM_PRIVATE_MEM_SLOTS 3
+#define KVM_MEM_SLOTS_NUM (KVM_USER_MEM_SLOTS + KVM_PRIVATE_MEM_SLOTS)
 
 #define KVM_MMIO_SIZE 16
 
@@ -219,11 +219,6 @@ struct kvm_mmu_page {
         u64 *spt;
         /* hold the gfn of each spte inside spt */
         gfn_t *gfns;
-        /*
-         * One bit set per slot which has memory
-         * in this shadow page.
-         */
-        DECLARE_BITMAP(slot_bitmap, KVM_MEM_SLOTS_NUM);
         bool unsync;
         int root_count;          /* Currently serving as active root */
         unsigned int unsync_children;
@@ -419,8 +414,8 @@ struct kvm_vcpu_arch {
         gpa_t time;
         struct pvclock_vcpu_time_info hv_clock;
         unsigned int hw_tsc_khz;
-        unsigned int time_offset;
-        struct page *time_page;
+        struct gfn_to_hva_cache pv_time;
+        bool pv_time_enabled;
         /* set guest stopped flag in pvclock flags field */
         bool pvclock_set_guest_stopped_request;
 
@@ -502,6 +497,13 @@ struct kvm_vcpu_arch {
                 u64 msr_val;
                 struct gfn_to_hva_cache data;
         } pv_eoi;
+
+        /*
+         * Indicate whether the access faults on its page table in guest
+         * which is set when fix page fault and used to detect unhandeable
+         * instruction.
+         */
+        bool write_fault_to_shadow_pgtable;
 };
 
 struct kvm_lpage_info {
@@ -697,6 +699,11 @@ struct kvm_x86_ops {
         void (*enable_nmi_window)(struct kvm_vcpu *vcpu);
         void (*enable_irq_window)(struct kvm_vcpu *vcpu);
         void (*update_cr8_intercept)(struct kvm_vcpu *vcpu, int tpr, int irr);
+        int (*vm_has_apicv)(struct kvm *kvm);
+        void (*hwapic_irr_update)(struct kvm_vcpu *vcpu, int max_irr);
+        void (*hwapic_isr_update)(struct kvm *kvm, int isr);
+        void (*load_eoi_exitmap)(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap);
+        void (*set_virtual_x2apic_mode)(struct kvm_vcpu *vcpu, bool set);
         int (*set_tss_addr)(struct kvm *kvm, unsigned int addr);
         int (*get_tdp_level)(void);
         u64 (*get_mt_mask)(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio);
@@ -991,6 +998,7 @@ int kvm_age_hva(struct kvm *kvm, unsigned long hva);
 int kvm_test_age_hva(struct kvm *kvm, unsigned long hva);
 void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte);
 int cpuid_maxphyaddr(struct kvm_vcpu *vcpu);
+int kvm_cpu_has_injectable_intr(struct kvm_vcpu *v);
 int kvm_cpu_has_interrupt(struct kvm_vcpu *vcpu);
 int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu);
 int kvm_cpu_get_interrupt(struct kvm_vcpu *v);
diff --git a/arch/x86/include/asm/kvm_para.h b/arch/x86/include/asm/kvm_para.h
index 5ed1f16187be..695399f2d5eb 100644
--- a/arch/x86/include/asm/kvm_para.h
+++ b/arch/x86/include/asm/kvm_para.h
@@ -27,7 +27,7 @@ static inline bool kvm_check_and_clear_guest_paused(void)
  *
  * Up to four arguments may be passed in rbx, rcx, rdx, and rsi respectively.
  * The hypercall number should be placed in rax and the return value will be
- * placed in rax.  No other registers will be clobbered unless explicited
+ * placed in rax.  No other registers will be clobbered unless explicitly
  * noted by the particular hypercall.
  */
 
@@ -85,13 +85,13 @@ static inline long kvm_hypercall4(unsigned int nr, unsigned long p1,
         return ret;
 }
 
-static inline int kvm_para_available(void)
+static inline bool kvm_para_available(void)
 {
         unsigned int eax, ebx, ecx, edx;
         char signature[13];
 
         if (boot_cpu_data.cpuid_level < 0)
-                return 0;       /* So we don't blow up on old processors */
+                return false;   /* So we don't blow up on old processors */
 
         if (cpu_has_hypervisor) {
                 cpuid(KVM_CPUID_SIGNATURE, &eax, &ebx, &ecx, &edx);
@@ -101,10 +101,10 @@ static inline int kvm_para_available(void)
                 signature[12] = 0;
 
                 if (strcmp(signature, "KVMKVMKVM") == 0)
-                        return 1;
+                        return true;
         }
 
-        return 0;
+        return false;
 }
 
 static inline unsigned int kvm_arch_para_features(void)
diff --git a/arch/x86/include/asm/linkage.h b/arch/x86/include/asm/linkage.h
index 48142971b25d..79327e9483a3 100644
--- a/arch/x86/include/asm/linkage.h
+++ b/arch/x86/include/asm/linkage.h
@@ -27,20 +27,20 @@
 #define __asmlinkage_protect0(ret) \
         __asmlinkage_protect_n(ret)
 #define __asmlinkage_protect1(ret, arg1) \
-        __asmlinkage_protect_n(ret, "g" (arg1))
+        __asmlinkage_protect_n(ret, "m" (arg1))
 #define __asmlinkage_protect2(ret, arg1, arg2) \
-        __asmlinkage_protect_n(ret, "g" (arg1), "g" (arg2))
+        __asmlinkage_protect_n(ret, "m" (arg1), "m" (arg2))
 #define __asmlinkage_protect3(ret, arg1, arg2, arg3) \
-        __asmlinkage_protect_n(ret, "g" (arg1), "g" (arg2), "g" (arg3))
+        __asmlinkage_protect_n(ret, "m" (arg1), "m" (arg2), "m" (arg3))
 #define __asmlinkage_protect4(ret, arg1, arg2, arg3, arg4) \
-        __asmlinkage_protect_n(ret, "g" (arg1), "g" (arg2), "g" (arg3), \
-                              "g" (arg4))
+        __asmlinkage_protect_n(ret, "m" (arg1), "m" (arg2), "m" (arg3), \
+                              "m" (arg4))
 #define __asmlinkage_protect5(ret, arg1, arg2, arg3, arg4, arg5) \
-        __asmlinkage_protect_n(ret, "g" (arg1), "g" (arg2), "g" (arg3), \
-                              "g" (arg4), "g" (arg5))
+        __asmlinkage_protect_n(ret, "m" (arg1), "m" (arg2), "m" (arg3), \
+                              "m" (arg4), "m" (arg5))
 #define __asmlinkage_protect6(ret, arg1, arg2, arg3, arg4, arg5, arg6) \
-        __asmlinkage_protect_n(ret, "g" (arg1), "g" (arg2), "g" (arg3), \
-                              "g" (arg4), "g" (arg5), "g" (arg6))
+        __asmlinkage_protect_n(ret, "m" (arg1), "m" (arg2), "m" (arg3), \
+                              "m" (arg4), "m" (arg5), "m" (arg6))
 
 #endif /* CONFIG_X86_32 */
 
diff --git a/arch/x86/include/asm/mce.h b/arch/x86/include/asm/mce.h
index ecdfee60ee4a..f4076af1f4ed 100644
--- a/arch/x86/include/asm/mce.h
+++ b/arch/x86/include/asm/mce.h
@@ -3,6 +3,90 @@
 
 #include <uapi/asm/mce.h>
 
+/*
+ * Machine Check support for x86
+ */
+
+/* MCG_CAP register defines */
+#define MCG_BANKCNT_MASK        0xff         /* Number of Banks */
+#define MCG_CTL_P               (1ULL<<8)    /* MCG_CTL register available */
+#define MCG_EXT_P               (1ULL<<9)    /* Extended registers available */
+#define MCG_CMCI_P              (1ULL<<10)   /* CMCI supported */
+#define MCG_EXT_CNT_MASK        0xff0000     /* Number of Extended registers */
+#define MCG_EXT_CNT_SHIFT       16
+#define MCG_EXT_CNT(c)          (((c) & MCG_EXT_CNT_MASK) >> MCG_EXT_CNT_SHIFT)
+#define MCG_SER_P               (1ULL<<24)   /* MCA recovery/new status bits */
+
+/* MCG_STATUS register defines */
+#define MCG_STATUS_RIPV  (1ULL<<0)   /* restart ip valid */
+#define MCG_STATUS_EIPV  (1ULL<<1)   /* ip points to correct instruction */
+#define MCG_STATUS_MCIP  (1ULL<<2)   /* machine check in progress */
+
+/* MCi_STATUS register defines */
+#define MCI_STATUS_VAL   (1ULL<<63)  /* valid error */
+#define MCI_STATUS_OVER  (1ULL<<62)  /* previous errors lost */
+#define MCI_STATUS_UC    (1ULL<<61)  /* uncorrected error */
+#define MCI_STATUS_EN    (1ULL<<60)  /* error enabled */
+#define MCI_STATUS_MISCV (1ULL<<59)  /* misc error reg. valid */
+#define MCI_STATUS_ADDRV (1ULL<<58)  /* addr reg. valid */
+#define MCI_STATUS_PCC   (1ULL<<57)  /* processor context corrupt */
+#define MCI_STATUS_S     (1ULL<<56)  /* Signaled machine check */
+#define MCI_STATUS_AR    (1ULL<<55)  /* Action required */
+#define MCACOD            0xffff     /* MCA Error Code */
+
+/* Architecturally defined codes from SDM Vol. 3B Chapter 15 */
+#define MCACOD_SCRUB    0x00C0  /* 0xC0-0xCF Memory Scrubbing */
+#define MCACOD_SCRUBMSK 0xfff0
+#define MCACOD_L3WB     0x017A  /* L3 Explicit Writeback */
+#define MCACOD_DATA     0x0134  /* Data Load */
+#define MCACOD_INSTR    0x0150  /* Instruction Fetch */
+
+/* MCi_MISC register defines */
+#define MCI_MISC_ADDR_LSB(m)    ((m) & 0x3f)
+#define MCI_MISC_ADDR_MODE(m)   (((m) >> 6) & 7)
+#define  MCI_MISC_ADDR_SEGOFF   0       /* segment offset */
+#define  MCI_MISC_ADDR_LINEAR   1       /* linear address */
+#define  MCI_MISC_ADDR_PHYS     2       /* physical address */
+#define  MCI_MISC_ADDR_MEM      3       /* memory address */
+#define  MCI_MISC_ADDR_GENERIC  7       /* generic */
+
+/* CTL2 register defines */
+#define MCI_CTL2_CMCI_EN                (1ULL << 30)
+#define MCI_CTL2_CMCI_THRESHOLD_MASK    0x7fffULL
+
+#define MCJ_CTX_MASK            3
+#define MCJ_CTX(flags)          ((flags) & MCJ_CTX_MASK)
+#define MCJ_CTX_RANDOM          0    /* inject context: random */
+#define MCJ_CTX_PROCESS         0x1  /* inject context: process */
+#define MCJ_CTX_IRQ             0x2  /* inject context: IRQ */
+#define MCJ_NMI_BROADCAST       0x4  /* do NMI broadcasting */
+#define MCJ_EXCEPTION           0x8  /* raise as exception */
+#define MCJ_IRQ_BRAODCAST       0x10 /* do IRQ broadcasting */
+
+#define MCE_OVERFLOW 0          /* bit 0 in flags means overflow */
+
+/* Software defined banks */
+#define MCE_EXTENDED_BANK       128
+#define MCE_THERMAL_BANK        (MCE_EXTENDED_BANK + 0)
+#define K8_MCE_THRESHOLD_BASE   (MCE_EXTENDED_BANK + 1)
+
+#define MCE_LOG_LEN 32
+#define MCE_LOG_SIGNATURE       "MACHINECHECK"
+
+/*
+ * This structure contains all data related to the MCE log.  Also
+ * carries a signature to make it easier to find from external
+ * debugging tools.  Each entry is only valid when its finished flag
+ * is set.
+ */
+struct mce_log {
+        char signature[12]; /* "MACHINECHECK" */
+        unsigned len;       /* = MCE_LOG_LEN */
+        unsigned next;
+        unsigned flags;
+        unsigned recordlen;     /* length of struct mce */
+        struct mce entry[MCE_LOG_LEN];
+};
 
 struct mca_config {
         bool dont_log_ce;
diff --git a/arch/x86/include/asm/microcode.h b/arch/x86/include/asm/microcode.h
index 43d921b4752c..6825e2efd1b4 100644
--- a/arch/x86/include/asm/microcode.h
+++ b/arch/x86/include/asm/microcode.h
@@ -57,4 +57,18 @@ static inline struct microcode_ops * __init init_amd_microcode(void)
 static inline void __exit exit_amd_microcode(void) {}
 #endif
 
+#ifdef CONFIG_MICROCODE_EARLY
+#define MAX_UCODE_COUNT 128
+extern void __init load_ucode_bsp(void);
+extern __init void load_ucode_ap(void);
+extern int __init save_microcode_in_initrd(void);
+#else
+static inline void __init load_ucode_bsp(void) {}
+static inline __init void load_ucode_ap(void) {}
+static inline int __init save_microcode_in_initrd(void)
+{
+        return 0;
+}
+#endif
+
 #endif /* _ASM_X86_MICROCODE_H */
diff --git a/arch/x86/include/asm/microcode_intel.h b/arch/x86/include/asm/microcode_intel.h
new file mode 100644
index 000000000000..5356f927d411
--- /dev/null
+++ b/arch/x86/include/asm/microcode_intel.h
@@ -0,0 +1,85 @@
+#ifndef _ASM_X86_MICROCODE_INTEL_H
+#define _ASM_X86_MICROCODE_INTEL_H
+
+#include <asm/microcode.h>
+
+struct microcode_header_intel {
+        unsigned int            hdrver;
+        unsigned int            rev;
+        unsigned int            date;
+        unsigned int            sig;
+        unsigned int            cksum;
+        unsigned int            ldrver;
+        unsigned int            pf;
+        unsigned int            datasize;
+        unsigned int            totalsize;
+        unsigned int            reserved[3];
+};
+
+struct microcode_intel {
+        struct microcode_header_intel hdr;
+        unsigned int            bits[0];
+};
+
+/* microcode format is extended from prescott processors */
+struct extended_signature {
+        unsigned int            sig;
+        unsigned int            pf;
+        unsigned int            cksum;
+};
+
+struct extended_sigtable {
+        unsigned int            count;
+        unsigned int            cksum;
+        unsigned int            reserved[3];
+        struct extended_signature sigs[0];
+};
+
+#define DEFAULT_UCODE_DATASIZE  (2000)
+#define MC_HEADER_SIZE          (sizeof(struct microcode_header_intel))
+#define DEFAULT_UCODE_TOTALSIZE (DEFAULT_UCODE_DATASIZE + MC_HEADER_SIZE)
+#define EXT_HEADER_SIZE         (sizeof(struct extended_sigtable))
+#define EXT_SIGNATURE_SIZE      (sizeof(struct extended_signature))
+#define DWSIZE                  (sizeof(u32))
+
+#define get_totalsize(mc) \
+        (((struct microcode_intel *)mc)->hdr.totalsize ? \
+         ((struct microcode_intel *)mc)->hdr.totalsize : \
+         DEFAULT_UCODE_TOTALSIZE)
+
+#define get_datasize(mc) \
+        (((struct microcode_intel *)mc)->hdr.datasize ? \
+         ((struct microcode_intel *)mc)->hdr.datasize : DEFAULT_UCODE_DATASIZE)
+
+#define sigmatch(s1, s2, p1, p2) \
+        (((s1) == (s2)) && (((p1) & (p2)) || (((p1) == 0) && ((p2) == 0))))
+
+#define exttable_size(et) ((et)->count * EXT_SIGNATURE_SIZE + EXT_HEADER_SIZE)
+
+extern int
+get_matching_microcode(unsigned int csig, int cpf, void *mc, int rev);
+extern int microcode_sanity_check(void *mc, int print_err);
+extern int get_matching_sig(unsigned int csig, int cpf, void *mc, int rev);
+extern int
+update_match_revision(struct microcode_header_intel *mc_header, int rev);
+
+#ifdef CONFIG_MICROCODE_INTEL_EARLY
+extern void __init load_ucode_intel_bsp(void);
+extern void __cpuinit load_ucode_intel_ap(void);
+extern void show_ucode_info_early(void);
+#else
+static inline __init void load_ucode_intel_bsp(void) {}
+static inline __cpuinit void load_ucode_intel_ap(void) {}
+static inline void show_ucode_info_early(void) {}
+#endif
+
+#if defined(CONFIG_MICROCODE_INTEL_EARLY) && defined(CONFIG_HOTPLUG_CPU)
+extern int save_mc_for_early(u8 *mc);
+#else
+static inline int save_mc_for_early(u8 *mc)
+{
+        return 0;
+}
+#endif
+
+#endif /* _ASM_X86_MICROCODE_INTEL_H */
diff --git a/arch/x86/include/asm/mmzone_32.h b/arch/x86/include/asm/mmzone_32.h
index eb05fb3b02fb..8a9b3e288cb4 100644
--- a/arch/x86/include/asm/mmzone_32.h
+++ b/arch/x86/include/asm/mmzone_32.h
@@ -14,12 +14,6 @@ extern struct pglist_data *node_data[];
 
 #include <asm/numaq.h>
 
-extern void resume_map_numa_kva(pgd_t *pgd);
-
-#else /* !CONFIG_NUMA */
-
-static inline void resume_map_numa_kva(pgd_t *pgd) {}
-
 #endif /* CONFIG_NUMA */
 
 #ifdef CONFIG_DISCONTIGMEM
diff --git a/arch/x86/include/asm/mshyperv.h b/arch/x86/include/asm/mshyperv.h
index 79ce5685ab64..c2934be2446a 100644
--- a/arch/x86/include/asm/mshyperv.h
+++ b/arch/x86/include/asm/mshyperv.h
@@ -11,4 +11,8 @@ struct ms_hyperv_info {
 
 extern struct ms_hyperv_info ms_hyperv;
 
+void hyperv_callback_vector(void);
+void hyperv_vector_handler(struct pt_regs *regs);
+void hv_register_vmbus_handler(int irq, irq_handler_t handler);
+
 #endif
diff --git a/arch/x86/include/asm/mwait.h b/arch/x86/include/asm/mwait.h
index bcdff997668c..2f366d0ac6b4 100644
--- a/arch/x86/include/asm/mwait.h
+++ b/arch/x86/include/asm/mwait.h
@@ -4,7 +4,8 @@
 #define MWAIT_SUBSTATE_MASK             0xf
 #define MWAIT_CSTATE_MASK               0xf
 #define MWAIT_SUBSTATE_SIZE             4
-#define MWAIT_MAX_NUM_CSTATES           8
+#define MWAIT_HINT2CSTATE(hint)         (((hint) >> MWAIT_SUBSTATE_SIZE) & MWAIT_CSTATE_MASK)
+#define MWAIT_HINT2SUBSTATE(hint)       ((hint) & MWAIT_CSTATE_MASK)
 
 #define CPUID_MWAIT_LEAF                5
 #define CPUID5_ECX_EXTENSIONS_SUPPORTED 0x1
diff --git a/arch/x86/include/asm/numa.h b/arch/x86/include/asm/numa.h
index 49119fcea2dc..1b99ee5c9f00 100644
--- a/arch/x86/include/asm/numa.h
+++ b/arch/x86/include/asm/numa.h
@@ -54,13 +54,11 @@ static inline int numa_cpu_node(int cpu)
 
 #ifdef CONFIG_X86_32
 # include <asm/numa_32.h>
-#else
-# include <asm/numa_64.h>
 #endif
 
 #ifdef CONFIG_NUMA
-extern void __cpuinit numa_set_node(int cpu, int node);
-extern void __cpuinit numa_clear_node(int cpu);
+extern void numa_set_node(int cpu, int node);
+extern void numa_clear_node(int cpu);
 extern void __init init_cpu_to_node(void);
 extern void __cpuinit numa_add_cpu(int cpu);
 extern void __cpuinit numa_remove_cpu(int cpu);
diff --git a/arch/x86/include/asm/numa_64.h b/arch/x86/include/asm/numa_64.h
deleted file mode 100644
index 0c05f7ae46e8..000000000000
--- a/arch/x86/include/asm/numa_64.h
+++ /dev/null
@@ -1,6 +0,0 @@
-#ifndef _ASM_X86_NUMA_64_H
-#define _ASM_X86_NUMA_64_H
-
-extern unsigned long numa_free_all_bootmem(void);
-
-#endif /* _ASM_X86_NUMA_64_H */
diff --git a/arch/x86/include/asm/page.h b/arch/x86/include/asm/page.h
index 8ca82839288a..c87892442e53 100644
--- a/arch/x86/include/asm/page.h
+++ b/arch/x86/include/asm/page.h
@@ -17,6 +17,10 @@
 
 struct page;
 
+#include <linux/range.h>
+extern struct range pfn_mapped[];
+extern int nr_pfn_mapped;
+
 static inline void clear_user_page(void *page, unsigned long vaddr,
                                    struct page *pg)
 {
@@ -44,7 +48,8 @@ static inline void copy_user_page(void *to, void *from, unsigned long vaddr,
  * case properly. Once all supported versions of gcc understand it, we can
  * remove this Voodoo magic stuff. (i.e. once gcc3.x is deprecated)
  */
-#define __pa_symbol(x)  __pa(__phys_reloc_hide((unsigned long)(x)))
+#define __pa_symbol(x) \
+        __phys_addr_symbol(__phys_reloc_hide((unsigned long)(x)))
 
 #define __va(x)                 ((void *)((unsigned long)(x)+PAGE_OFFSET))
 
diff --git a/arch/x86/include/asm/page_32.h b/arch/x86/include/asm/page_32.h
index da4e762406f7..4d550d04b609 100644
--- a/arch/x86/include/asm/page_32.h
+++ b/arch/x86/include/asm/page_32.h
@@ -15,6 +15,7 @@ extern unsigned long __phys_addr(unsigned long);
 #else
 #define __phys_addr(x)          __phys_addr_nodebug(x)
 #endif
+#define __phys_addr_symbol(x)   __phys_addr(x)
 #define __phys_reloc_hide(x)    RELOC_HIDE((x), 0)
 
 #ifdef CONFIG_FLATMEM
diff --git a/arch/x86/include/asm/page_64.h b/arch/x86/include/asm/page_64.h
index 072694ed81a5..0f1ddee6a0ce 100644
--- a/arch/x86/include/asm/page_64.h
+++ b/arch/x86/include/asm/page_64.h
@@ -3,4 +3,40 @@
 
 #include <asm/page_64_types.h>
 
+#ifndef __ASSEMBLY__
+
+/* duplicated to the one in bootmem.h */
+extern unsigned long max_pfn;
+extern unsigned long phys_base;
+
+static inline unsigned long __phys_addr_nodebug(unsigned long x)
+{
+        unsigned long y = x - __START_KERNEL_map;
+
+        /* use the carry flag to determine if x was < __START_KERNEL_map */
+        x = y + ((x > y) ? phys_base : (__START_KERNEL_map - PAGE_OFFSET));
+
+        return x;
+}
+
+#ifdef CONFIG_DEBUG_VIRTUAL
+extern unsigned long __phys_addr(unsigned long);
+extern unsigned long __phys_addr_symbol(unsigned long);
+#else
+#define __phys_addr(x)          __phys_addr_nodebug(x)
+#define __phys_addr_symbol(x) \
+        ((unsigned long)(x) - __START_KERNEL_map + phys_base)
+#endif
+
+#define __phys_reloc_hide(x)    (x)
+
+#ifdef CONFIG_FLATMEM
+#define pfn_valid(pfn)          ((pfn) < max_pfn)
+#endif
+
+void clear_page(void *page);
+void copy_page(void *to, void *from);
+
+#endif  /* !__ASSEMBLY__ */
+
 #endif /* _ASM_X86_PAGE_64_H */
diff --git a/arch/x86/include/asm/page_64_types.h b/arch/x86/include/asm/page_64_types.h
index 320f7bb95f76..8b491e66eaa8 100644
--- a/arch/x86/include/asm/page_64_types.h
+++ b/arch/x86/include/asm/page_64_types.h
@@ -50,26 +50,4 @@
 #define KERNEL_IMAGE_SIZE       (512 * 1024 * 1024)
 #define KERNEL_IMAGE_START      _AC(0xffffffff80000000, UL)
 
-#ifndef __ASSEMBLY__
-void clear_page(void *page);
-void copy_page(void *to, void *from);
-
-/* duplicated to the one in bootmem.h */
-extern unsigned long max_pfn;
-extern unsigned long phys_base;
-
-extern unsigned long __phys_addr(unsigned long);
-#define __phys_reloc_hide(x)    (x)
-
-#define vmemmap ((struct page *)VMEMMAP_START)
-
-extern void init_extra_mapping_uc(unsigned long phys, unsigned long size);
-extern void init_extra_mapping_wb(unsigned long phys, unsigned long size);
-
-#endif  /* !__ASSEMBLY__ */
-
-#ifdef CONFIG_FLATMEM
-#define pfn_valid(pfn)          ((pfn) < max_pfn)
-#endif
-
 #endif /* _ASM_X86_PAGE_64_DEFS_H */
diff --git a/arch/x86/include/asm/page_types.h b/arch/x86/include/asm/page_types.h
index e21fdd10479f..54c97879195e 100644
--- a/arch/x86/include/asm/page_types.h
+++ b/arch/x86/include/asm/page_types.h
@@ -51,6 +51,8 @@ static inline phys_addr_t get_max_mapped(void)
         return (phys_addr_t)max_pfn_mapped << PAGE_SHIFT;
 }
 
+bool pfn_range_is_mapped(unsigned long start_pfn, unsigned long end_pfn);
+
 extern unsigned long init_memory_mapping(unsigned long start,
                                          unsigned long end);
 
diff --git a/arch/x86/include/asm/pci.h b/arch/x86/include/asm/pci.h
index dba7805176bf..d9e9e6c7ed32 100644
--- a/arch/x86/include/asm/pci.h
+++ b/arch/x86/include/asm/pci.h
@@ -14,6 +14,9 @@
 struct pci_sysdata {
         int             domain;         /* PCI domain */
         int             node;           /* NUMA node */
+#ifdef CONFIG_ACPI
+        void            *acpi;          /* ACPI-specific data */
+#endif
 #ifdef CONFIG_X86_64
         void            *iommu;         /* IOMMU private data */
 #endif
@@ -121,9 +124,12 @@ static inline void x86_restore_msi_irqs(struct pci_dev *dev, int irq)
 #define arch_teardown_msi_irq x86_teardown_msi_irq
 #define arch_restore_msi_irqs x86_restore_msi_irqs
 /* implemented in arch/x86/kernel/apic/io_apic. */
+struct msi_desc;
 int native_setup_msi_irqs(struct pci_dev *dev, int nvec, int type);
 void native_teardown_msi_irq(unsigned int irq);
 void native_restore_msi_irqs(struct pci_dev *dev, int irq);
+int setup_msi_irq(struct pci_dev *dev, struct msi_desc *msidesc,
+                  unsigned int irq_base, unsigned int irq_offset);
 /* default to the implementation in drivers/lib/msi.c */
 #define HAVE_DEFAULT_MSI_TEARDOWN_IRQS
 #define HAVE_DEFAULT_MSI_RESTORE_IRQS
diff --git a/arch/x86/include/asm/pci_x86.h b/arch/x86/include/asm/pci_x86.h
index 747e5a38b590..fa1195dae425 100644
--- a/arch/x86/include/asm/pci_x86.h
+++ b/arch/x86/include/asm/pci_x86.h
@@ -54,7 +54,6 @@ void pcibios_set_cache_line_size(void);
 /* pci-pc.c */
 
 extern int pcibios_last_bus;
-extern struct pci_bus *pci_root_bus;
 extern struct pci_ops pci_root_ops;
 
 void pcibios_scan_specific_bus(int busn);
diff --git a/arch/x86/include/asm/perf_event.h b/arch/x86/include/asm/perf_event.h
index 4fabcdf1cfa7..57cb63402213 100644
--- a/arch/x86/include/asm/perf_event.h
+++ b/arch/x86/include/asm/perf_event.h
@@ -29,8 +29,13 @@
 #define ARCH_PERFMON_EVENTSEL_INV                       (1ULL << 23)
 #define ARCH_PERFMON_EVENTSEL_CMASK                     0xFF000000ULL
 
-#define AMD_PERFMON_EVENTSEL_GUESTONLY                  (1ULL << 40)
-#define AMD_PERFMON_EVENTSEL_HOSTONLY                   (1ULL << 41)
+#define AMD64_EVENTSEL_INT_CORE_ENABLE                  (1ULL << 36)
+#define AMD64_EVENTSEL_GUESTONLY                        (1ULL << 40)
+#define AMD64_EVENTSEL_HOSTONLY                         (1ULL << 41)
+
+#define AMD64_EVENTSEL_INT_CORE_SEL_SHIFT               37
+#define AMD64_EVENTSEL_INT_CORE_SEL_MASK                \
+        (0xFULL << AMD64_EVENTSEL_INT_CORE_SEL_SHIFT)
 
 #define AMD64_EVENTSEL_EVENT    \
         (ARCH_PERFMON_EVENTSEL_EVENT | (0x0FULL << 32))
@@ -46,8 +51,12 @@
 #define AMD64_RAW_EVENT_MASK            \
         (X86_RAW_EVENT_MASK          |  \
          AMD64_EVENTSEL_EVENT)
+#define AMD64_RAW_EVENT_MASK_NB         \
+        (AMD64_EVENTSEL_EVENT        |  \
+         ARCH_PERFMON_EVENTSEL_UMASK)
 #define AMD64_NUM_COUNTERS                              4
 #define AMD64_NUM_COUNTERS_CORE                         6
+#define AMD64_NUM_COUNTERS_NB                           4
 
 #define ARCH_PERFMON_UNHALTED_CORE_CYCLES_SEL           0x3c
 #define ARCH_PERFMON_UNHALTED_CORE_CYCLES_UMASK         (0x00 << 8)
diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h
index 5199db2923d3..1e672234c4ff 100644
--- a/arch/x86/include/asm/pgtable.h
+++ b/arch/x86/include/asm/pgtable.h
@@ -142,6 +142,11 @@ static inline unsigned long pmd_pfn(pmd_t pmd)
         return (pmd_val(pmd) & PTE_PFN_MASK) >> PAGE_SHIFT;
 }
 
+static inline unsigned long pud_pfn(pud_t pud)
+{
+        return (pud_val(pud) & PTE_PFN_MASK) >> PAGE_SHIFT;
+}
+
 #define pte_page(pte)   pfn_to_page(pte_pfn(pte))
 
 static inline int pmd_large(pmd_t pte)
@@ -390,6 +395,7 @@ pte_t *populate_extra_pte(unsigned long vaddr);
 
 #ifndef __ASSEMBLY__
 #include <linux/mm_types.h>
+#include <linux/log2.h>
 
 static inline int pte_none(pte_t pte)
 {
@@ -615,6 +621,8 @@ static inline int pgd_none(pgd_t pgd)
 #ifndef __ASSEMBLY__
 
 extern int direct_gbpages;
+void init_mem_mapping(void);
+void early_alloc_pgt_buf(void);
 
 /* local pte updates need not use xchg for locking */
 static inline pte_t native_local_ptep_get_and_clear(pte_t *ptep)
@@ -781,6 +789,32 @@ static inline void clone_pgd_range(pgd_t *dst, pgd_t *src, int count)
        memcpy(dst, src, count * sizeof(pgd_t));
 }
 
+#define PTE_SHIFT ilog2(PTRS_PER_PTE)
+static inline int page_level_shift(enum pg_level level)
+{
+        return (PAGE_SHIFT - PTE_SHIFT) + level * PTE_SHIFT;
+}
+static inline unsigned long page_level_size(enum pg_level level)
+{
+        return 1UL << page_level_shift(level);
+}
+static inline unsigned long page_level_mask(enum pg_level level)
+{
+        return ~(page_level_size(level) - 1);
+}
+
+/*
+ * The x86 doesn't have any external MMU info: the kernel page
+ * tables contain all the necessary information.
+ */
+static inline void update_mmu_cache(struct vm_area_struct *vma,
+                unsigned long addr, pte_t *ptep)
+{
+}
+static inline void update_mmu_cache_pmd(struct vm_area_struct *vma,
+                unsigned long addr, pmd_t *pmd)
+{
+}
 
 #include <asm-generic/pgtable.h>
 #endif  /* __ASSEMBLY__ */
diff --git a/arch/x86/include/asm/pgtable_32.h b/arch/x86/include/asm/pgtable_32.h
index 8faa215a503e..9ee322103c6d 100644
--- a/arch/x86/include/asm/pgtable_32.h
+++ b/arch/x86/include/asm/pgtable_32.h
@@ -66,13 +66,6 @@ do {						\
         __flush_tlb_one((vaddr));               \
 } while (0)
 
-/*
- * The i386 doesn't have any external MMU info: the kernel page
- * tables contain all the necessary information.
- */
-#define update_mmu_cache(vma, address, ptep) do { } while (0)
-#define update_mmu_cache_pmd(vma, address, pmd) do { } while (0)
-
 #endif /* !__ASSEMBLY__ */
 
 /*
diff --git a/arch/x86/include/asm/pgtable_64.h b/arch/x86/include/asm/pgtable_64.h
index 47356f9df82e..e22c1dbf7feb 100644
--- a/arch/x86/include/asm/pgtable_64.h
+++ b/arch/x86/include/asm/pgtable_64.h
@@ -142,9 +142,6 @@ static inline int pgd_large(pgd_t pgd) { return 0; }
 #define pte_offset_map(dir, address) pte_offset_kernel((dir), (address))
 #define pte_unmap(pte) ((void)(pte))/* NOP */
 
-#define update_mmu_cache(vma, address, ptep) do { } while (0)
-#define update_mmu_cache_pmd(vma, address, pmd) do { } while (0)
-
 /* Encode and de-code a swap entry */
 #if _PAGE_BIT_FILE < _PAGE_BIT_PROTNONE
 #define SWP_TYPE_BITS (_PAGE_BIT_FILE - _PAGE_BIT_PRESENT - 1)
@@ -183,6 +180,11 @@ extern void cleanup_highmap(void);
 
 #define __HAVE_ARCH_PTE_SAME
 
+#define vmemmap ((struct page *)VMEMMAP_START)
+
+extern void init_extra_mapping_uc(unsigned long phys, unsigned long size);
+extern void init_extra_mapping_wb(unsigned long phys, unsigned long size);
+
 #endif /* !__ASSEMBLY__ */
 
 #endif /* _ASM_X86_PGTABLE_64_H */
diff --git a/arch/x86/include/asm/pgtable_64_types.h b/arch/x86/include/asm/pgtable_64_types.h
index 766ea16fbbbd..2d883440cb9a 100644
--- a/arch/x86/include/asm/pgtable_64_types.h
+++ b/arch/x86/include/asm/pgtable_64_types.h
@@ -1,6 +1,8 @@
 #ifndef _ASM_X86_PGTABLE_64_DEFS_H
 #define _ASM_X86_PGTABLE_64_DEFS_H
 
+#include <asm/sparsemem.h>
+
 #ifndef __ASSEMBLY__
 #include <linux/types.h>
 
@@ -60,4 +62,6 @@ typedef struct { pteval_t pte; } pte_t;
 #define MODULES_END      _AC(0xffffffffff000000, UL)
 #define MODULES_LEN   (MODULES_END - MODULES_VADDR)
 
+#define EARLY_DYNAMIC_PAGE_TABLES       64
+
 #endif /* _ASM_X86_PGTABLE_64_DEFS_H */
diff --git a/arch/x86/include/asm/pgtable_types.h b/arch/x86/include/asm/pgtable_types.h
index 3c32db8c539d..567b5d0632b2 100644
--- a/arch/x86/include/asm/pgtable_types.h
+++ b/arch/x86/include/asm/pgtable_types.h
@@ -321,7 +321,6 @@ int phys_mem_access_prot_allowed(struct file *file, unsigned long pfn,
 /* Install a pte for a particular vaddr in kernel space. */
 void set_pte_vaddr(unsigned long vaddr, pte_t pte);
 
-extern void native_pagetable_reserve(u64 start, u64 end);
 #ifdef CONFIG_X86_32
 extern void native_pagetable_init(void);
 #else
@@ -331,7 +330,7 @@ extern void native_pagetable_init(void);
 struct seq_file;
 extern void arch_report_meminfo(struct seq_file *m);
 
-enum {
+enum pg_level {
         PG_LEVEL_NONE,
         PG_LEVEL_4K,
         PG_LEVEL_2M,
@@ -352,6 +351,8 @@ static inline void update_page_count(int level, unsigned long pages) { }
  * as a pte too.
  */
 extern pte_t *lookup_address(unsigned long address, unsigned int *level);
+extern int __split_large_page(pte_t *kpte, unsigned long address, pte_t *pbase);
+extern phys_addr_t slow_virt_to_phys(void *__address);
 
 #endif  /* !__ASSEMBLY__ */
 
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index 888184b2fc85..3270116b1488 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -89,7 +89,6 @@ struct cpuinfo_x86 {
         char                    wp_works_ok;    /* It doesn't on 386's */
 
         /* Problems on some 486Dx4's and old 386's: */
-        char                    hlt_works_ok;
         char                    hard_math;
         char                    rfu;
         char                    fdiv_bug;
@@ -165,15 +164,6 @@ DECLARE_PER_CPU_SHARED_ALIGNED(struct cpuinfo_x86, cpu_info);
 
 extern const struct seq_operations cpuinfo_op;
 
-static inline int hlt_works(int cpu)
-{
-#ifdef CONFIG_X86_32
-        return cpu_data(cpu).hlt_works_ok;
-#else
-        return 1;
-#endif
-}
-
 #define cache_line_size()       (boot_cpu_data.x86_cache_alignment)
 
 extern void cpu_detect(struct cpuinfo_x86 *c);
@@ -190,6 +180,14 @@ extern void init_amd_cacheinfo(struct cpuinfo_x86 *c);
 extern void detect_extended_topology(struct cpuinfo_x86 *c);
 extern void detect_ht(struct cpuinfo_x86 *c);
 
+#ifdef CONFIG_X86_32
+extern int have_cpuid_p(void);
+#else
+static inline int have_cpuid_p(void)
+{
+        return 1;
+}
+#endif
 static inline void native_cpuid(unsigned int *eax, unsigned int *ebx,
                                 unsigned int *ecx, unsigned int *edx)
 {
@@ -725,12 +723,13 @@ extern unsigned long		boot_option_idle_override;
 extern bool                     amd_e400_c1e_detected;
 
 enum idle_boot_override {IDLE_NO_OVERRIDE=0, IDLE_HALT, IDLE_NOMWAIT,
-                         IDLE_POLL, IDLE_FORCE_MWAIT};
+                         IDLE_POLL};
 
 extern void enable_sep_cpu(void);
 extern int sysenter_setup(void);
 
 extern void early_trap_init(void);
+void early_trap_pf_init(void);
 
 /* Defined in head.S */
 extern struct desc_ptr          early_gdt_descr;
@@ -943,7 +942,7 @@ extern void start_thread(struct pt_regs *regs, unsigned long new_ip,
 extern int get_tsc_mode(unsigned long adr);
 extern int set_tsc_mode(unsigned int val);
 
-extern int amd_get_nb_id(int cpu);
+extern u16 amd_get_nb_id(int cpu);
 
 struct aperfmperf {
         u64 aperf, mperf;
@@ -998,7 +997,11 @@ extern unsigned long arch_align_stack(unsigned long sp);
 extern void free_init_pages(char *what, unsigned long begin, unsigned long end);
 
 void default_idle(void);
-bool set_pm_idle_to_default(void);
+#ifdef  CONFIG_XEN
+bool xen_set_default_idle(void);
+#else
+#define xen_set_default_idle 0
+#endif
 
 void stop_this_cpu(void *dummy);
 
diff --git a/arch/x86/include/asm/proto.h b/arch/x86/include/asm/proto.h
index 6f414ed88620..6fd3fd769796 100644
--- a/arch/x86/include/asm/proto.h
+++ b/arch/x86/include/asm/proto.h
@@ -5,8 +5,6 @@
 
 /* misc architecture specific prototypes */
 
-void early_idt_handler(void);
-
 void system_call(void);
 void syscall_init(void);
 
diff --git a/arch/x86/include/asm/realmode.h b/arch/x86/include/asm/realmode.h
index fe1ec5bcd846..9c6b890d5e7a 100644
--- a/arch/x86/include/asm/realmode.h
+++ b/arch/x86/include/asm/realmode.h
@@ -58,6 +58,7 @@ extern unsigned char boot_gdt[];
 extern unsigned char secondary_startup_64[];
 #endif
 
-extern void __init setup_real_mode(void);
+void reserve_real_mode(void);
+void setup_real_mode(void);
 
 #endif /* _ARCH_X86_REALMODE_H */
diff --git a/arch/x86/include/asm/required-features.h b/arch/x86/include/asm/required-features.h
index 6c7fc25f2c34..5c6e4fb370f5 100644
--- a/arch/x86/include/asm/required-features.h
+++ b/arch/x86/include/asm/required-features.h
@@ -47,6 +47,12 @@
 # define NEED_NOPL      0
 #endif
 
+#ifdef CONFIG_MATOM
+# define NEED_MOVBE     (1<<(X86_FEATURE_MOVBE & 31))
+#else
+# define NEED_MOVBE     0
+#endif
+
 #ifdef CONFIG_X86_64
 #ifdef CONFIG_PARAVIRT
 /* Paravirtualized systems may not have PSE or PGE available */
@@ -80,7 +86,7 @@
 
 #define REQUIRED_MASK2  0
 #define REQUIRED_MASK3  (NEED_NOPL)
-#define REQUIRED_MASK4  0
+#define REQUIRED_MASK4  (NEED_MOVBE)
 #define REQUIRED_MASK5  0
 #define REQUIRED_MASK6  0
 #define REQUIRED_MASK7  0
diff --git a/arch/x86/include/asm/signal.h b/arch/x86/include/asm/signal.h
index 216bf364a7e7..35e67a457182 100644
--- a/arch/x86/include/asm/signal.h
+++ b/arch/x86/include/asm/signal.h
@@ -31,27 +31,9 @@ typedef sigset_t compat_sigset_t;
 #include <uapi/asm/signal.h>
 #ifndef __ASSEMBLY__
 extern void do_notify_resume(struct pt_regs *, void *, __u32);
-#ifdef __i386__
-struct old_sigaction {
-        __sighandler_t sa_handler;
-        old_sigset_t sa_mask;
-        unsigned long sa_flags;
-        __sigrestore_t sa_restorer;
-};
-
-struct sigaction {
-        __sighandler_t sa_handler;
-        unsigned long sa_flags;
-        __sigrestore_t sa_restorer;
-        sigset_t sa_mask;               /* mask last for extensibility */
-};
-
-struct k_sigaction {
-        struct sigaction sa;
-};
 
-#else /* __i386__ */
-#endif /* !__i386__ */
+#define __ARCH_HAS_SA_RESTORER
+
 #include <asm/sigcontext.h>
 
 #ifdef __i386__
diff --git a/arch/x86/include/asm/sys_ia32.h b/arch/x86/include/asm/sys_ia32.h
index 31f61f96e0fb..8459efc39686 100644
--- a/arch/x86/include/asm/sys_ia32.h
+++ b/arch/x86/include/asm/sys_ia32.h
@@ -32,29 +32,17 @@ struct mmap_arg_struct32;
 asmlinkage long sys32_mmap(struct mmap_arg_struct32 __user *);
 asmlinkage long sys32_mprotect(unsigned long, size_t, unsigned long);
 
-struct sigaction32;
-struct old_sigaction32;
-asmlinkage long sys32_rt_sigaction(int, struct sigaction32 __user *,
-                                   struct sigaction32 __user *, unsigned int);
-asmlinkage long sys32_sigaction(int, struct old_sigaction32 __user *,
-                                struct old_sigaction32 __user *);
 asmlinkage long sys32_alarm(unsigned int);
 
 asmlinkage long sys32_waitpid(compat_pid_t, unsigned int __user *, int);
 asmlinkage long sys32_sysfs(int, u32, u32);
 
-asmlinkage long sys32_sched_rr_get_interval(compat_pid_t,
-                                            struct compat_timespec __user *);
-asmlinkage long sys32_rt_sigpending(compat_sigset_t __user *, compat_size_t);
-asmlinkage long sys32_rt_sigqueueinfo(int, int, compat_siginfo_t __user *);
-
 asmlinkage long sys32_pread(unsigned int, char __user *, u32, u32, u32);
 asmlinkage long sys32_pwrite(unsigned int, const char __user *, u32, u32, u32);
 
 asmlinkage long sys32_personality(unsigned long);
 asmlinkage long sys32_sendfile(int, int, compat_off_t __user *, s32);
 
-long sys32_lseek(unsigned int, int, unsigned int);
 long sys32_kill(int, int);
 long sys32_fadvise64_64(int, __u32, __u32, __u32, __u32, int);
 long sys32_vm86_warning(void);
@@ -68,9 +56,8 @@ asmlinkage long sys32_fallocate(int, int, unsigned,
                                 unsigned, unsigned, unsigned);
 
 /* ia32/ia32_signal.c */
-asmlinkage long sys32_sigsuspend(int, int, old_sigset_t);
-asmlinkage long sys32_sigreturn(struct pt_regs *);
-asmlinkage long sys32_rt_sigreturn(struct pt_regs *);
+asmlinkage long sys32_sigreturn(void);
+asmlinkage long sys32_rt_sigreturn(void);
 
 /* ia32/ipc32.c */
 asmlinkage long sys32_ipc(u32, int, int, int, compat_uptr_t, u32);
diff --git a/arch/x86/include/asm/syscalls.h b/arch/x86/include/asm/syscalls.h
index 58b7e3eac0ae..6cf0a9cc60cd 100644
--- a/arch/x86/include/asm/syscalls.h
+++ b/arch/x86/include/asm/syscalls.h
@@ -18,13 +18,13 @@
 /* Common in X86_32 and X86_64 */
 /* kernel/ioport.c */
 asmlinkage long sys_ioperm(unsigned long, unsigned long, int);
-long sys_iopl(unsigned int, struct pt_regs *);
+asmlinkage long sys_iopl(unsigned int);
 
 /* kernel/ldt.c */
 asmlinkage int sys_modify_ldt(int, void __user *, unsigned long);
 
 /* kernel/signal.c */
-long sys_rt_sigreturn(struct pt_regs *);
+long sys_rt_sigreturn(void);
 
 /* kernel/tls.c */
 asmlinkage int sys_set_thread_area(struct user_desc __user *);
@@ -34,14 +34,11 @@ asmlinkage int sys_get_thread_area(struct user_desc __user *);
 #ifdef CONFIG_X86_32
 
 /* kernel/signal.c */
-asmlinkage int sys_sigsuspend(int, int, old_sigset_t);
-asmlinkage int sys_sigaction(int, const struct old_sigaction __user *,
-                             struct old_sigaction __user *);
-unsigned long sys_sigreturn(struct pt_regs *);
+unsigned long sys_sigreturn(void);
 
 /* kernel/vm86_32.c */
-int sys_vm86old(struct vm86_struct __user *, struct pt_regs *);
-int sys_vm86(unsigned long, unsigned long, struct pt_regs *);
+int sys_vm86old(struct vm86_struct __user *);
+int sys_vm86(unsigned long, unsigned long);
 
 #else /* CONFIG_X86_32 */
 
diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h
index 2d946e63ee82..2cd056e3ada3 100644
--- a/arch/x86/include/asm/thread_info.h
+++ b/arch/x86/include/asm/thread_info.h
@@ -20,7 +20,6 @@
 struct task_struct;
 struct exec_domain;
 #include <asm/processor.h>
-#include <asm/ftrace.h>
 #include <linux/atomic.h>
 
 struct thread_info {
diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h
index 0fee48e279cc..50a7fc0f824a 100644
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -20,10 +20,20 @@ static inline void __native_flush_tlb(void)
         native_write_cr3(native_read_cr3());
 }
 
+static inline void __native_flush_tlb_global_irq_disabled(void)
+{
+        unsigned long cr4;
+
+        cr4 = native_read_cr4();
+        /* clear PGE */
+        native_write_cr4(cr4 & ~X86_CR4_PGE);
+        /* write old PGE again and flush TLBs */
+        native_write_cr4(cr4);
+}
+
 static inline void __native_flush_tlb_global(void)
 {
         unsigned long flags;
-        unsigned long cr4;
 
         /*
          * Read-modify-write to CR4 - protect it from preemption and
@@ -32,11 +42,7 @@ static inline void __native_flush_tlb_global(void)
          */
         raw_local_irq_save(flags);
 
-        cr4 = native_read_cr4();
-        /* clear PGE */
-        native_write_cr4(cr4 & ~X86_CR4_PGE);
-        /* write old PGE again and flush TLBs */
-        native_write_cr4(cr4);
+        __native_flush_tlb_global_irq_disabled();
 
         raw_local_irq_restore(flags);
 }
diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h
index 1709801d18ec..5ee26875baea 100644
--- a/arch/x86/include/asm/uaccess.h
+++ b/arch/x86/include/asm/uaccess.h
@@ -125,13 +125,12 @@ extern int __get_user_4(void);
 extern int __get_user_8(void);
 extern int __get_user_bad(void);
 
-#define __get_user_x(size, ret, x, ptr)               \
-        asm volatile("call __get_user_" #size         \
-                     : "=a" (ret), "=d" (x)           \
-                     : "0" (ptr))                     \
-
-/* Careful: we have to cast the result to the type of the pointer
- * for sign reasons */
+/*
+ * This is a type: either unsigned long, if the argument fits into
+ * that type, or otherwise unsigned long long.
+ */
+#define __inttype(x) \
+__typeof__(__builtin_choose_expr(sizeof(x) > sizeof(0UL), 0ULL, 0UL))
 
 /**
  * get_user: - Get a simple variable from user space.
@@ -150,38 +149,26 @@ extern int __get_user_bad(void);
  * Returns zero on success, or -EFAULT on error.
  * On error, the variable @x is set to zero.
  */
-#ifdef CONFIG_X86_32
-#define __get_user_8(__ret_gu, __val_gu, ptr)                           \
-                __get_user_x(X, __ret_gu, __val_gu, ptr)
-#else
-#define __get_user_8(__ret_gu, __val_gu, ptr)                           \
-                __get_user_x(8, __ret_gu, __val_gu, ptr)
-#endif
-
+/*
+ * Careful: we have to cast the result to the type of the pointer
+ * for sign reasons.
+ *
+ * The use of %edx as the register specifier is a bit of a
+ * simplification, as gcc only cares about it as the starting point
+ * and not size: for a 64-bit value it will use %ecx:%edx on 32 bits
+ * (%ecx being the next register in gcc's x86 register sequence), and
+ * %rdx on 64 bits.
+ */
 #define get_user(x, ptr)                                                \
 ({                                                                      \
         int __ret_gu;                                                   \
-        unsigned long __val_gu;                                         \
+        register __inttype(*(ptr)) __val_gu asm("%edx");                \
         __chk_user_ptr(ptr);                                            \
         might_fault();                                                  \
-        switch (sizeof(*(ptr))) {                                       \
-        case 1:                                                         \
-                __get_user_x(1, __ret_gu, __val_gu, ptr);               \
-                break;                                                  \
-        case 2:                                                         \
-                __get_user_x(2, __ret_gu, __val_gu, ptr);               \
-                break;                                                  \
-        case 4:                                                         \
-                __get_user_x(4, __ret_gu, __val_gu, ptr);               \
-                break;                                                  \
-        case 8:                                                         \
-                __get_user_8(__ret_gu, __val_gu, ptr);                  \
-                break;                                                  \
-        default:                                                        \
-                __get_user_x(X, __ret_gu, __val_gu, ptr);               \
-                break;                                                  \
-        }                                                               \
-        (x) = (__typeof__(*(ptr)))__val_gu;                             \
+        asm volatile("call __get_user_%P3"                              \
+                     : "=a" (__ret_gu), "=r" (__val_gu)                 \
+                     : "0" (ptr), "i" (sizeof(*(ptr))));                \
+        (x) = (__typeof__(*(ptr))) __val_gu;                            \
         __ret_gu;                                                       \
 })
 
diff --git a/arch/x86/include/asm/unistd.h b/arch/x86/include/asm/unistd.h
index a0790e07ba65..3d5df1c4447f 100644
--- a/arch/x86/include/asm/unistd.h
+++ b/arch/x86/include/asm/unistd.h
@@ -38,8 +38,6 @@
 # define __ARCH_WANT_SYS_OLD_GETRLIMIT
 # define __ARCH_WANT_SYS_OLD_UNAME
 # define __ARCH_WANT_SYS_PAUSE
-# define __ARCH_WANT_SYS_RT_SIGACTION
-# define __ARCH_WANT_SYS_RT_SIGSUSPEND
 # define __ARCH_WANT_SYS_SGETMASK
 # define __ARCH_WANT_SYS_SIGNAL
 # define __ARCH_WANT_SYS_SIGPENDING
diff --git a/arch/x86/include/asm/uv/uv_hub.h b/arch/x86/include/asm/uv/uv_hub.h
index 21f7385badb8..2c32df95bb78 100644
--- a/arch/x86/include/asm/uv/uv_hub.h
+++ b/arch/x86/include/asm/uv/uv_hub.h
@@ -5,7 +5,7 @@
  *
  * SGI UV architectural definitions
  *
- * Copyright (C) 2007-2010 Silicon Graphics, Inc. All rights reserved.
+ * Copyright (C) 2007-2013 Silicon Graphics, Inc. All rights reserved.
  */
 
 #ifndef _ASM_X86_UV_UV_HUB_H
@@ -175,6 +175,7 @@ DECLARE_PER_CPU(struct uv_hub_info_s, __uv_hub_info);
  */
 #define UV1_HUB_REVISION_BASE           1
 #define UV2_HUB_REVISION_BASE           3
+#define UV3_HUB_REVISION_BASE           5
 
 static inline int is_uv1_hub(void)
 {
@@ -183,6 +184,23 @@ static inline int is_uv1_hub(void)
 
 static inline int is_uv2_hub(void)
 {
+        return ((uv_hub_info->hub_revision >= UV2_HUB_REVISION_BASE) &&
+                (uv_hub_info->hub_revision < UV3_HUB_REVISION_BASE));
+}
+
+static inline int is_uv3_hub(void)
+{
+        return uv_hub_info->hub_revision >= UV3_HUB_REVISION_BASE;
+}
+
+static inline int is_uv_hub(void)
+{
+        return uv_hub_info->hub_revision;
+}
+
+/* code common to uv2 and uv3 only */
+static inline int is_uvx_hub(void)
+{
         return uv_hub_info->hub_revision >= UV2_HUB_REVISION_BASE;
 }
 
@@ -230,14 +248,23 @@ union uvh_apicid {
 #define UV2_LOCAL_MMR_SIZE              (32UL * 1024 * 1024)
 #define UV2_GLOBAL_MMR32_SIZE           (32UL * 1024 * 1024)
 
-#define UV_LOCAL_MMR_BASE               (is_uv1_hub() ? UV1_LOCAL_MMR_BASE     \
-                                                : UV2_LOCAL_MMR_BASE)
-#define UV_GLOBAL_MMR32_BASE            (is_uv1_hub() ? UV1_GLOBAL_MMR32_BASE  \
-                                                : UV2_GLOBAL_MMR32_BASE)
-#define UV_LOCAL_MMR_SIZE               (is_uv1_hub() ? UV1_LOCAL_MMR_SIZE :   \
-                                                UV2_LOCAL_MMR_SIZE)
+#define UV3_LOCAL_MMR_BASE              0xfa000000UL
+#define UV3_GLOBAL_MMR32_BASE           0xfc000000UL
+#define UV3_LOCAL_MMR_SIZE              (32UL * 1024 * 1024)
+#define UV3_GLOBAL_MMR32_SIZE           (32UL * 1024 * 1024)
+
+#define UV_LOCAL_MMR_BASE               (is_uv1_hub() ? UV1_LOCAL_MMR_BASE : \
+                                        (is_uv2_hub() ? UV2_LOCAL_MMR_BASE : \
+                                                        UV3_LOCAL_MMR_BASE))
+#define UV_GLOBAL_MMR32_BASE            (is_uv1_hub() ? UV1_GLOBAL_MMR32_BASE :\
+                                        (is_uv2_hub() ? UV2_GLOBAL_MMR32_BASE :\
+                                                        UV3_GLOBAL_MMR32_BASE))
+#define UV_LOCAL_MMR_SIZE               (is_uv1_hub() ? UV1_LOCAL_MMR_SIZE : \
+                                        (is_uv2_hub() ? UV2_LOCAL_MMR_SIZE : \
+                                                        UV3_LOCAL_MMR_SIZE))
 #define UV_GLOBAL_MMR32_SIZE            (is_uv1_hub() ? UV1_GLOBAL_MMR32_SIZE :\
-                                                UV2_GLOBAL_MMR32_SIZE)
+                                        (is_uv2_hub() ? UV2_GLOBAL_MMR32_SIZE :\
+                                                        UV3_GLOBAL_MMR32_SIZE))
 #define UV_GLOBAL_MMR64_BASE            (uv_hub_info->global_mmr_base)
 
 #define UV_GLOBAL_GRU_MMR_BASE          0x4000000
@@ -599,6 +626,7 @@ static inline void uv_hub_send_ipi(int pnode, int apicid, int vector)
  *     1 - UV1 rev 1.0 initial silicon
  *     2 - UV1 rev 2.0 production silicon
  *     3 - UV2 rev 1.0 initial silicon
+ *     5 - UV3 rev 1.0 initial silicon
  */
 static inline int uv_get_min_hub_revision_id(void)
 {
diff --git a/arch/x86/include/asm/uv/uv_mmrs.h b/arch/x86/include/asm/uv/uv_mmrs.h
index cf1d73643f60..bd5f80e58a23 100644
--- a/arch/x86/include/asm/uv/uv_mmrs.h
+++ b/arch/x86/include/asm/uv/uv_mmrs.h
@@ -5,16 +5,25 @@
  *
  * SGI UV MMR definitions
  *
- * Copyright (C) 2007-2011 Silicon Graphics, Inc. All rights reserved.
+ * Copyright (C) 2007-2013 Silicon Graphics, Inc. All rights reserved.
  */
 
 #ifndef _ASM_X86_UV_UV_MMRS_H
 #define _ASM_X86_UV_UV_MMRS_H
 
 /*
- * This file contains MMR definitions for both UV1 & UV2 hubs.
+ * This file contains MMR definitions for all UV hubs types.
  *
- * In general, MMR addresses and structures are identical on both hubs.
+ * To minimize coding differences between hub types, the symbols are
+ * grouped by architecture types.
+ *
+ * UVH  - definitions common to all UV hub types.
+ * UVXH - definitions common to all UV eXtended hub types (currently 2 & 3).
+ * UV1H - definitions specific to UV type 1 hub.
+ * UV2H - definitions specific to UV type 2 hub.
+ * UV3H - definitions specific to UV type 3 hub.
+ *
+ * So in general, MMR addresses and structures are identical on all hubs types.
  * These MMRs are identified as:
  *      #define UVH_xxx         <address>
  *      union uvh_xxx {
@@ -23,24 +32,36 @@
  *              } s;
  *      };
  *
- * If the MMR exists on both hub type but has different addresses or
- * contents, the MMR definition is similar to:
- *      #define UV1H_xxx        <uv1 address>
- *      #define UV2H_xxx        <uv2address>
- *      #define UVH_xxx         (is_uv1_hub() ? UV1H_xxx : UV2H_xxx)
+ * If the MMR exists on all hub types but have different addresses:
+ *      #define UV1Hxxx a
+ *      #define UV2Hxxx b
+ *      #define UV3Hxxx c
+ *      #define UVHxxx  (is_uv1_hub() ? UV1Hxxx :
+ *                      (is_uv2_hub() ? UV2Hxxx :
+ *                                      UV3Hxxx))
+ *
+ * If the MMR exists on all hub types > 1 but have different addresses:
+ *      #define UV2Hxxx b
+ *      #define UV3Hxxx c
+ *      #define UVXHxxx (is_uv2_hub() ? UV2Hxxx :
+ *                                      UV3Hxxx))
+ *
  *      union uvh_xxx {
  *              unsigned long       v;
- *              struct uv1h_int_cmpd_s {         (Common fields only)
+ *              struct uvh_xxx_s {       # Common fields only
  *              } s;
- *              struct uv1h_int_cmpd_s {         (Full UV1 definition)
+ *              struct uv1h_xxx_s {      # Full UV1 definition (*)
  *              } s1;
- *              struct uv2h_int_cmpd_s {         (Full UV2 definition)
+ *              struct uv2h_xxx_s {      # Full UV2 definition (*)
  *              } s2;
+ *              struct uv3h_xxx_s {      # Full UV3 definition (*)
+ *              } s3;
  *      };
+ *              (* - if present and different than the common struct)
  *
- * Only essential difference are enumerated. For example, if the address is
- * the same for both UV1 & UV2, only a single #define is generated. Likewise,
- * if the contents is the same for both hubs, only the "s" structure is
+ * Only essential differences are enumerated. For example, if the address is
+ * the same for all UV's, only a single #define is generated. Likewise,
+ * if the contents is the same for all hubs, only the "s" structure is
  * generated.
  *
  * If the MMR exists on ONLY 1 type of hub, no generic definition is
@@ -51,6 +72,8 @@
  *              struct uvh_int_cmpd_s {
  *              } sn;
  *      };
+ *
+ * (GEN Flags: mflags_opt= undefs=0 UV23=UVXH)
  */
 
 #define UV_MMR_ENABLE           (1UL << 63)
@@ -58,15 +81,18 @@
 #define UV1_HUB_PART_NUMBER     0x88a5
 #define UV2_HUB_PART_NUMBER     0x8eb8
 #define UV2_HUB_PART_NUMBER_X   0x1111
+#define UV3_HUB_PART_NUMBER     0x9578
+#define UV3_HUB_PART_NUMBER_X   0x4321
 
-/* Compat: if this #define is present, UV headers support UV2 */
+/* Compat: Indicate which UV Hubs are supported. */
 #define UV2_HUB_IS_SUPPORTED    1
+#define UV3_HUB_IS_SUPPORTED    1
 
 /* ========================================================================= */
 /*                          UVH_BAU_DATA_BROADCAST                           */
 /* ========================================================================= */
-#define UVH_BAU_DATA_BROADCAST                          0x61688UL
-#define UVH_BAU_DATA_BROADCAST_32                       0x440
+#define UVH_BAU_DATA_BROADCAST 0x61688UL
+#define UVH_BAU_DATA_BROADCAST_32 0x440
 
 #define UVH_BAU_DATA_BROADCAST_ENABLE_SHFT              0
 #define UVH_BAU_DATA_BROADCAST_ENABLE_MASK              0x0000000000000001UL
@@ -82,8 +108,8 @@ union uvh_bau_data_broadcast_u {
 /* ========================================================================= */
 /*                           UVH_BAU_DATA_CONFIG                             */
 /* ========================================================================= */
-#define UVH_BAU_DATA_CONFIG                             0x61680UL
-#define UVH_BAU_DATA_CONFIG_32                          0x438
+#define UVH_BAU_DATA_CONFIG 0x61680UL
+#define UVH_BAU_DATA_CONFIG_32 0x438
 
 #define UVH_BAU_DATA_CONFIG_VECTOR_SHFT                 0
 #define UVH_BAU_DATA_CONFIG_DM_SHFT                     8
@@ -121,10 +147,14 @@ union uvh_bau_data_config_u {
 /* ========================================================================= */
 /*                           UVH_EVENT_OCCURRED0                             */
 /* ========================================================================= */
-#define UVH_EVENT_OCCURRED0                             0x70000UL
-#define UVH_EVENT_OCCURRED0_32                          0x5e8
+#define UVH_EVENT_OCCURRED0 0x70000UL
+#define UVH_EVENT_OCCURRED0_32 0x5e8
+
+#define UVH_EVENT_OCCURRED0_LB_HCERR_SHFT               0
+#define UVH_EVENT_OCCURRED0_RH_AOERR0_SHFT              11
+#define UVH_EVENT_OCCURRED0_LB_HCERR_MASK               0x0000000000000001UL
+#define UVH_EVENT_OCCURRED0_RH_AOERR0_MASK              0x0000000000000800UL
 
-#define UV1H_EVENT_OCCURRED0_LB_HCERR_SHFT              0
 #define UV1H_EVENT_OCCURRED0_GR0_HCERR_SHFT             1
 #define UV1H_EVENT_OCCURRED0_GR1_HCERR_SHFT             2
 #define UV1H_EVENT_OCCURRED0_LH_HCERR_SHFT              3
@@ -135,7 +165,6 @@ union uvh_bau_data_config_u {
 #define UV1H_EVENT_OCCURRED0_GR0_AOERR0_SHFT            8
 #define UV1H_EVENT_OCCURRED0_GR1_AOERR0_SHFT            9
 #define UV1H_EVENT_OCCURRED0_LH_AOERR0_SHFT             10
-#define UV1H_EVENT_OCCURRED0_RH_AOERR0_SHFT             11
 #define UV1H_EVENT_OCCURRED0_XN_AOERR0_SHFT             12
 #define UV1H_EVENT_OCCURRED0_SI_AOERR0_SHFT             13
 #define UV1H_EVENT_OCCURRED0_LB_AOERR1_SHFT             14
@@ -181,7 +210,6 @@ union uvh_bau_data_config_u {
 #define UV1H_EVENT_OCCURRED0_RTC3_SHFT                  54
 #define UV1H_EVENT_OCCURRED0_BAU_DATA_SHFT              55
 #define UV1H_EVENT_OCCURRED0_POWER_MANAGEMENT_REQ_SHFT  56
-#define UV1H_EVENT_OCCURRED0_LB_HCERR_MASK              0x0000000000000001UL
 #define UV1H_EVENT_OCCURRED0_GR0_HCERR_MASK             0x0000000000000002UL
 #define UV1H_EVENT_OCCURRED0_GR1_HCERR_MASK             0x0000000000000004UL
 #define UV1H_EVENT_OCCURRED0_LH_HCERR_MASK              0x0000000000000008UL
@@ -192,7 +220,6 @@ union uvh_bau_data_config_u {
 #define UV1H_EVENT_OCCURRED0_GR0_AOERR0_MASK            0x0000000000000100UL
 #define UV1H_EVENT_OCCURRED0_GR1_AOERR0_MASK            0x0000000000000200UL
 #define UV1H_EVENT_OCCURRED0_LH_AOERR0_MASK             0x0000000000000400UL
-#define UV1H_EVENT_OCCURRED0_RH_AOERR0_MASK             0x0000000000000800UL
 #define UV1H_EVENT_OCCURRED0_XN_AOERR0_MASK             0x0000000000001000UL
 #define UV1H_EVENT_OCCURRED0_SI_AOERR0_MASK             0x0000000000002000UL
 #define UV1H_EVENT_OCCURRED0_LB_AOERR1_MASK             0x0000000000004000UL
@@ -239,188 +266,130 @@ union uvh_bau_data_config_u {
 #define UV1H_EVENT_OCCURRED0_BAU_DATA_MASK              0x0080000000000000UL
 #define UV1H_EVENT_OCCURRED0_POWER_MANAGEMENT_REQ_MASK  0x0100000000000000UL
 
-#define UV2H_EVENT_OCCURRED0_LB_HCERR_SHFT              0
-#define UV2H_EVENT_OCCURRED0_QP_HCERR_SHFT              1
-#define UV2H_EVENT_OCCURRED0_RH_HCERR_SHFT              2
-#define UV2H_EVENT_OCCURRED0_LH0_HCERR_SHFT             3
-#define UV2H_EVENT_OCCURRED0_LH1_HCERR_SHFT             4
-#define UV2H_EVENT_OCCURRED0_GR0_HCERR_SHFT             5
-#define UV2H_EVENT_OCCURRED0_GR1_HCERR_SHFT             6
-#define UV2H_EVENT_OCCURRED0_NI0_HCERR_SHFT             7
-#define UV2H_EVENT_OCCURRED0_NI1_HCERR_SHFT             8
-#define UV2H_EVENT_OCCURRED0_LB_AOERR0_SHFT             9
-#define UV2H_EVENT_OCCURRED0_QP_AOERR0_SHFT             10
-#define UV2H_EVENT_OCCURRED0_RH_AOERR0_SHFT             11
-#define UV2H_EVENT_OCCURRED0_LH0_AOERR0_SHFT            12
-#define UV2H_EVENT_OCCURRED0_LH1_AOERR0_SHFT            13
-#define UV2H_EVENT_OCCURRED0_GR0_AOERR0_SHFT            14
-#define UV2H_EVENT_OCCURRED0_GR1_AOERR0_SHFT            15
-#define UV2H_EVENT_OCCURRED0_XB_AOERR0_SHFT             16
-#define UV2H_EVENT_OCCURRED0_RT_AOERR0_SHFT             17
-#define UV2H_EVENT_OCCURRED0_NI0_AOERR0_SHFT            18
-#define UV2H_EVENT_OCCURRED0_NI1_AOERR0_SHFT            19
-#define UV2H_EVENT_OCCURRED0_LB_AOERR1_SHFT             20
-#define UV2H_EVENT_OCCURRED0_QP_AOERR1_SHFT             21
-#define UV2H_EVENT_OCCURRED0_RH_AOERR1_SHFT             22
-#define UV2H_EVENT_OCCURRED0_LH0_AOERR1_SHFT            23
-#define UV2H_EVENT_OCCURRED0_LH1_AOERR1_SHFT            24
-#define UV2H_EVENT_OCCURRED0_GR0_AOERR1_SHFT            25
-#define UV2H_EVENT_OCCURRED0_GR1_AOERR1_SHFT            26
-#define UV2H_EVENT_OCCURRED0_XB_AOERR1_SHFT             27
-#define UV2H_EVENT_OCCURRED0_RT_AOERR1_SHFT             28
-#define UV2H_EVENT_OCCURRED0_NI0_AOERR1_SHFT            29
-#define UV2H_EVENT_OCCURRED0_NI1_AOERR1_SHFT            30
-#define UV2H_EVENT_OCCURRED0_SYSTEM_SHUTDOWN_INT_SHFT   31
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_0_SHFT          32
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_1_SHFT          33
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_2_SHFT          34
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_3_SHFT          35
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_4_SHFT          36
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_5_SHFT          37
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_6_SHFT          38
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_7_SHFT          39
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_8_SHFT          40
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_9_SHFT          41
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_10_SHFT         42
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_11_SHFT         43
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_12_SHFT         44
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_13_SHFT         45
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_14_SHFT         46
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_15_SHFT         47
-#define UV2H_EVENT_OCCURRED0_L1_NMI_INT_SHFT            48
-#define UV2H_EVENT_OCCURRED0_STOP_CLOCK_SHFT            49
-#define UV2H_EVENT_OCCURRED0_ASIC_TO_L1_SHFT            50
-#define UV2H_EVENT_OCCURRED0_L1_TO_ASIC_SHFT            51
-#define UV2H_EVENT_OCCURRED0_LA_SEQ_TRIGGER_SHFT        52
-#define UV2H_EVENT_OCCURRED0_IPI_INT_SHFT               53
-#define UV2H_EVENT_OCCURRED0_EXTIO_INT0_SHFT            54
-#define UV2H_EVENT_OCCURRED0_EXTIO_INT1_SHFT            55
-#define UV2H_EVENT_OCCURRED0_EXTIO_INT2_SHFT            56
-#define UV2H_EVENT_OCCURRED0_EXTIO_INT3_SHFT            57
-#define UV2H_EVENT_OCCURRED0_PROFILE_INT_SHFT           58
-#define UV2H_EVENT_OCCURRED0_LB_HCERR_MASK              0x0000000000000001UL
-#define UV2H_EVENT_OCCURRED0_QP_HCERR_MASK              0x0000000000000002UL
-#define UV2H_EVENT_OCCURRED0_RH_HCERR_MASK              0x0000000000000004UL
-#define UV2H_EVENT_OCCURRED0_LH0_HCERR_MASK             0x0000000000000008UL
-#define UV2H_EVENT_OCCURRED0_LH1_HCERR_MASK             0x0000000000000010UL
-#define UV2H_EVENT_OCCURRED0_GR0_HCERR_MASK             0x0000000000000020UL
-#define UV2H_EVENT_OCCURRED0_GR1_HCERR_MASK             0x0000000000000040UL
-#define UV2H_EVENT_OCCURRED0_NI0_HCERR_MASK             0x0000000000000080UL
-#define UV2H_EVENT_OCCURRED0_NI1_HCERR_MASK             0x0000000000000100UL
-#define UV2H_EVENT_OCCURRED0_LB_AOERR0_MASK             0x0000000000000200UL
-#define UV2H_EVENT_OCCURRED0_QP_AOERR0_MASK             0x0000000000000400UL
-#define UV2H_EVENT_OCCURRED0_RH_AOERR0_MASK             0x0000000000000800UL
-#define UV2H_EVENT_OCCURRED0_LH0_AOERR0_MASK            0x0000000000001000UL
-#define UV2H_EVENT_OCCURRED0_LH1_AOERR0_MASK            0x0000000000002000UL
-#define UV2H_EVENT_OCCURRED0_GR0_AOERR0_MASK            0x0000000000004000UL
-#define UV2H_EVENT_OCCURRED0_GR1_AOERR0_MASK            0x0000000000008000UL
-#define UV2H_EVENT_OCCURRED0_XB_AOERR0_MASK             0x0000000000010000UL
-#define UV2H_EVENT_OCCURRED0_RT_AOERR0_MASK             0x0000000000020000UL
-#define UV2H_EVENT_OCCURRED0_NI0_AOERR0_MASK            0x0000000000040000UL
-#define UV2H_EVENT_OCCURRED0_NI1_AOERR0_MASK            0x0000000000080000UL
-#define UV2H_EVENT_OCCURRED0_LB_AOERR1_MASK             0x0000000000100000UL
-#define UV2H_EVENT_OCCURRED0_QP_AOERR1_MASK             0x0000000000200000UL
-#define UV2H_EVENT_OCCURRED0_RH_AOERR1_MASK             0x0000000000400000UL
-#define UV2H_EVENT_OCCURRED0_LH0_AOERR1_MASK            0x0000000000800000UL
-#define UV2H_EVENT_OCCURRED0_LH1_AOERR1_MASK            0x0000000001000000UL
-#define UV2H_EVENT_OCCURRED0_GR0_AOERR1_MASK            0x0000000002000000UL
-#define UV2H_EVENT_OCCURRED0_GR1_AOERR1_MASK            0x0000000004000000UL
-#define UV2H_EVENT_OCCURRED0_XB_AOERR1_MASK             0x0000000008000000UL
-#define UV2H_EVENT_OCCURRED0_RT_AOERR1_MASK             0x0000000010000000UL
-#define UV2H_EVENT_OCCURRED0_NI0_AOERR1_MASK            0x0000000020000000UL
-#define UV2H_EVENT_OCCURRED0_NI1_AOERR1_MASK            0x0000000040000000UL
-#define UV2H_EVENT_OCCURRED0_SYSTEM_SHUTDOWN_INT_MASK   0x0000000080000000UL
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_0_MASK          0x0000000100000000UL
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_1_MASK          0x0000000200000000UL
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_2_MASK          0x0000000400000000UL
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_3_MASK          0x0000000800000000UL
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_4_MASK          0x0000001000000000UL
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_5_MASK          0x0000002000000000UL
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_6_MASK          0x0000004000000000UL
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_7_MASK          0x0000008000000000UL
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_8_MASK          0x0000010000000000UL
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_9_MASK          0x0000020000000000UL
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_10_MASK         0x0000040000000000UL
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_11_MASK         0x0000080000000000UL
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_12_MASK         0x0000100000000000UL
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_13_MASK         0x0000200000000000UL
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_14_MASK         0x0000400000000000UL
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_15_MASK         0x0000800000000000UL
-#define UV2H_EVENT_OCCURRED0_L1_NMI_INT_MASK            0x0001000000000000UL
-#define UV2H_EVENT_OCCURRED0_STOP_CLOCK_MASK            0x0002000000000000UL
-#define UV2H_EVENT_OCCURRED0_ASIC_TO_L1_MASK            0x0004000000000000UL
-#define UV2H_EVENT_OCCURRED0_L1_TO_ASIC_MASK            0x0008000000000000UL
-#define UV2H_EVENT_OCCURRED0_LA_SEQ_TRIGGER_MASK        0x0010000000000000UL
-#define UV2H_EVENT_OCCURRED0_IPI_INT_MASK               0x0020000000000000UL
-#define UV2H_EVENT_OCCURRED0_EXTIO_INT0_MASK            0x0040000000000000UL
-#define UV2H_EVENT_OCCURRED0_EXTIO_INT1_MASK            0x0080000000000000UL
-#define UV2H_EVENT_OCCURRED0_EXTIO_INT2_MASK            0x0100000000000000UL
-#define UV2H_EVENT_OCCURRED0_EXTIO_INT3_MASK            0x0200000000000000UL
-#define UV2H_EVENT_OCCURRED0_PROFILE_INT_MASK           0x0400000000000000UL
+#define UVXH_EVENT_OCCURRED0_QP_HCERR_SHFT              1
+#define UVXH_EVENT_OCCURRED0_RH_HCERR_SHFT              2
+#define UVXH_EVENT_OCCURRED0_LH0_HCERR_SHFT             3
+#define UVXH_EVENT_OCCURRED0_LH1_HCERR_SHFT             4
+#define UVXH_EVENT_OCCURRED0_GR0_HCERR_SHFT             5
+#define UVXH_EVENT_OCCURRED0_GR1_HCERR_SHFT             6
+#define UVXH_EVENT_OCCURRED0_NI0_HCERR_SHFT             7
+#define UVXH_EVENT_OCCURRED0_NI1_HCERR_SHFT             8
+#define UVXH_EVENT_OCCURRED0_LB_AOERR0_SHFT             9
+#define UVXH_EVENT_OCCURRED0_QP_AOERR0_SHFT             10
+#define UVXH_EVENT_OCCURRED0_LH0_AOERR0_SHFT            12
+#define UVXH_EVENT_OCCURRED0_LH1_AOERR0_SHFT            13
+#define UVXH_EVENT_OCCURRED0_GR0_AOERR0_SHFT            14
+#define UVXH_EVENT_OCCURRED0_GR1_AOERR0_SHFT            15
+#define UVXH_EVENT_OCCURRED0_XB_AOERR0_SHFT             16
+#define UVXH_EVENT_OCCURRED0_RT_AOERR0_SHFT             17
+#define UVXH_EVENT_OCCURRED0_NI0_AOERR0_SHFT            18
+#define UVXH_EVENT_OCCURRED0_NI1_AOERR0_SHFT            19
+#define UVXH_EVENT_OCCURRED0_LB_AOERR1_SHFT             20
+#define UVXH_EVENT_OCCURRED0_QP_AOERR1_SHFT             21
+#define UVXH_EVENT_OCCURRED0_RH_AOERR1_SHFT             22
+#define UVXH_EVENT_OCCURRED0_LH0_AOERR1_SHFT            23
+#define UVXH_EVENT_OCCURRED0_LH1_AOERR1_SHFT            24
+#define UVXH_EVENT_OCCURRED0_GR0_AOERR1_SHFT            25
+#define UVXH_EVENT_OCCURRED0_GR1_AOERR1_SHFT            26
+#define UVXH_EVENT_OCCURRED0_XB_AOERR1_SHFT             27
+#define UVXH_EVENT_OCCURRED0_RT_AOERR1_SHFT             28
+#define UVXH_EVENT_OCCURRED0_NI0_AOERR1_SHFT            29
+#define UVXH_EVENT_OCCURRED0_NI1_AOERR1_SHFT            30
+#define UVXH_EVENT_OCCURRED0_SYSTEM_SHUTDOWN_INT_SHFT   31
+#define UVXH_EVENT_OCCURRED0_LB_IRQ_INT_0_SHFT          32
+#define UVXH_EVENT_OCCURRED0_LB_IRQ_INT_1_SHFT          33
+#define UVXH_EVENT_OCCURRED0_LB_IRQ_INT_2_SHFT          34
+#define UVXH_EVENT_OCCURRED0_LB_IRQ_INT_3_SHFT          35
+#define UVXH_EVENT_OCCURRED0_LB_IRQ_INT_4_SHFT          36
+#define UVXH_EVENT_OCCURRED0_LB_IRQ_INT_5_SHFT          37
+#define UVXH_EVENT_OCCURRED0_LB_IRQ_INT_6_SHFT          38
+#define UVXH_EVENT_OCCURRED0_LB_IRQ_INT_7_SHFT          39
+#define UVXH_EVENT_OCCURRED0_LB_IRQ_INT_8_SHFT          40
+#define UVXH_EVENT_OCCURRED0_LB_IRQ_INT_9_SHFT          41
+#define UVXH_EVENT_OCCURRED0_LB_IRQ_INT_10_SHFT         42
+#define UVXH_EVENT_OCCURRED0_LB_IRQ_INT_11_SHFT         43
+#define UVXH_EVENT_OCCURRED0_LB_IRQ_INT_12_SHFT         44
+#define UVXH_EVENT_OCCURRED0_LB_IRQ_INT_13_SHFT         45
+#define UVXH_EVENT_OCCURRED0_LB_IRQ_INT_14_SHFT         46
+#define UVXH_EVENT_OCCURRED0_LB_IRQ_INT_15_SHFT         47
+#define UVXH_EVENT_OCCURRED0_L1_NMI_INT_SHFT            48
+#define UVXH_EVENT_OCCURRED0_STOP_CLOCK_SHFT            49
+#define UVXH_EVENT_OCCURRED0_ASIC_TO_L1_SHFT            50
+#define UVXH_EVENT_OCCURRED0_L1_TO_ASIC_SHFT            51
+#define UVXH_EVENT_OCCURRED0_LA_SEQ_TRIGGER_SHFT        52
+#define UVXH_EVENT_OCCURRED0_IPI_INT_SHFT               53
+#define UVXH_EVENT_OCCURRED0_EXTIO_INT0_SHFT            54
+#define UVXH_EVENT_OCCURRED0_EXTIO_INT1_SHFT            55
+#define UVXH_EVENT_OCCURRED0_EXTIO_INT2_SHFT            56
+#define UVXH_EVENT_OCCURRED0_EXTIO_INT3_SHFT            57
+#define UVXH_EVENT_OCCURRED0_PROFILE_INT_SHFT           58
+#define UVXH_EVENT_OCCURRED0_QP_HCERR_MASK              0x0000000000000002UL
+#define UVXH_EVENT_OCCURRED0_RH_HCERR_MASK              0x0000000000000004UL
+#define UVXH_EVENT_OCCURRED0_LH0_HCERR_MASK             0x0000000000000008UL
+#define UVXH_EVENT_OCCURRED0_LH1_HCERR_MASK             0x0000000000000010UL
+#define UVXH_EVENT_OCCURRED0_GR0_HCERR_MASK             0x0000000000000020UL
+#define UVXH_EVENT_OCCURRED0_GR1_HCERR_MASK             0x0000000000000040UL
+#define UVXH_EVENT_OCCURRED0_NI0_HCERR_MASK             0x0000000000000080UL
+#define UVXH_EVENT_OCCURRED0_NI1_HCERR_MASK             0x0000000000000100UL
+#define UVXH_EVENT_OCCURRED0_LB_AOERR0_MASK             0x0000000000000200UL
+#define UVXH_EVENT_OCCURRED0_QP_AOERR0_MASK             0x0000000000000400UL
+#define UVXH_EVENT_OCCURRED0_LH0_AOERR0_MASK            0x0000000000001000UL
+#define UVXH_EVENT_OCCURRED0_LH1_AOERR0_MASK            0x0000000000002000UL
+#define UVXH_EVENT_OCCURRED0_GR0_AOERR0_MASK            0x0000000000004000UL
+#define UVXH_EVENT_OCCURRED0_GR1_AOERR0_MASK            0x0000000000008000UL
+#define UVXH_EVENT_OCCURRED0_XB_AOERR0_MASK             0x0000000000010000UL
+#define UVXH_EVENT_OCCURRED0_RT_AOERR0_MASK             0x0000000000020000UL
+#define UVXH_EVENT_OCCURRED0_NI0_AOERR0_MASK            0x0000000000040000UL
+#define UVXH_EVENT_OCCURRED0_NI1_AOERR0_MASK            0x0000000000080000UL
+#define UVXH_EVENT_OCCURRED0_LB_AOERR1_MASK             0x0000000000100000UL
+#define UVXH_EVENT_OCCURRED0_QP_AOERR1_MASK             0x0000000000200000UL
+#define UVXH_EVENT_OCCURRED0_RH_AOERR1_MASK             0x0000000000400000UL
+#define UVXH_EVENT_OCCURRED0_LH0_AOERR1_MASK            0x0000000000800000UL
+#define UVXH_EVENT_OCCURRED0_LH1_AOERR1_MASK            0x0000000001000000UL
+#define UVXH_EVENT_OCCURRED0_GR0_AOERR1_MASK            0x0000000002000000UL
+#define UVXH_EVENT_OCCURRED0_GR1_AOERR1_MASK            0x0000000004000000UL
+#define UVXH_EVENT_OCCURRED0_XB_AOERR1_MASK             0x0000000008000000UL
+#define UVXH_EVENT_OCCURRED0_RT_AOERR1_MASK             0x0000000010000000UL
+#define UVXH_EVENT_OCCURRED0_NI0_AOERR1_MASK            0x0000000020000000UL
+#define UVXH_EVENT_OCCURRED0_NI1_AOERR1_MASK            0x0000000040000000UL
+#define UVXH_EVENT_OCCURRED0_SYSTEM_SHUTDOWN_INT_MASK   0x0000000080000000UL
+#define UVXH_EVENT_OCCURRED0_LB_IRQ_INT_0_MASK          0x0000000100000000UL
+#define UVXH_EVENT_OCCURRED0_LB_IRQ_INT_1_MASK          0x0000000200000000UL
+#define UVXH_EVENT_OCCURRED0_LB_IRQ_INT_2_MASK          0x0000000400000000UL
+#define UVXH_EVENT_OCCURRED0_LB_IRQ_INT_3_MASK          0x0000000800000000UL
+#define UVXH_EVENT_OCCURRED0_LB_IRQ_INT_4_MASK          0x0000001000000000UL
+#define UVXH_EVENT_OCCURRED0_LB_IRQ_INT_5_MASK          0x0000002000000000UL
+#define UVXH_EVENT_OCCURRED0_LB_IRQ_INT_6_MASK          0x0000004000000000UL
+#define UVXH_EVENT_OCCURRED0_LB_IRQ_INT_7_MASK          0x0000008000000000UL
+#define UVXH_EVENT_OCCURRED0_LB_IRQ_INT_8_MASK          0x0000010000000000UL
+#define UVXH_EVENT_OCCURRED0_LB_IRQ_INT_9_MASK          0x0000020000000000UL
+#define UVXH_EVENT_OCCURRED0_LB_IRQ_INT_10_MASK         0x0000040000000000UL
+#define UVXH_EVENT_OCCURRED0_LB_IRQ_INT_11_MASK         0x0000080000000000UL
+#define UVXH_EVENT_OCCURRED0_LB_IRQ_INT_12_MASK         0x0000100000000000UL
+#define UVXH_EVENT_OCCURRED0_LB_IRQ_INT_13_MASK         0x0000200000000000UL
+#define UVXH_EVENT_OCCURRED0_LB_IRQ_INT_14_MASK         0x0000400000000000UL
+#define UVXH_EVENT_OCCURRED0_LB_IRQ_INT_15_MASK         0x0000800000000000UL
+#define UVXH_EVENT_OCCURRED0_L1_NMI_INT_MASK            0x0001000000000000UL
+#define UVXH_EVENT_OCCURRED0_STOP_CLOCK_MASK            0x0002000000000000UL
+#define UVXH_EVENT_OCCURRED0_ASIC_TO_L1_MASK            0x0004000000000000UL
+#define UVXH_EVENT_OCCURRED0_L1_TO_ASIC_MASK            0x0008000000000000UL
+#define UVXH_EVENT_OCCURRED0_LA_SEQ_TRIGGER_MASK        0x0010000000000000UL
+#define UVXH_EVENT_OCCURRED0_IPI_INT_MASK               0x0020000000000000UL
+#define UVXH_EVENT_OCCURRED0_EXTIO_INT0_MASK            0x0040000000000000UL
+#define UVXH_EVENT_OCCURRED0_EXTIO_INT1_MASK            0x0080000000000000UL
+#define UVXH_EVENT_OCCURRED0_EXTIO_INT2_MASK            0x0100000000000000UL
+#define UVXH_EVENT_OCCURRED0_EXTIO_INT3_MASK            0x0200000000000000UL
+#define UVXH_EVENT_OCCURRED0_PROFILE_INT_MASK           0x0400000000000000UL
 
 union uvh_event_occurred0_u {
         unsigned long   v;
-        struct uv1h_event_occurred0_s {
+        struct uvh_event_occurred0_s {
                 unsigned long   lb_hcerr:1;                     /* RW, W1C */
-                unsigned long   gr0_hcerr:1;                    /* RW, W1C */
-                unsigned long   gr1_hcerr:1;                    /* RW, W1C */
-                unsigned long   lh_hcerr:1;                     /* RW, W1C */
-                unsigned long   rh_hcerr:1;                     /* RW, W1C */
-                unsigned long   xn_hcerr:1;                     /* RW, W1C */
-                unsigned long   si_hcerr:1;                     /* RW, W1C */
-                unsigned long   lb_aoerr0:1;                    /* RW, W1C */
-                unsigned long   gr0_aoerr0:1;                   /* RW, W1C */
-                unsigned long   gr1_aoerr0:1;                   /* RW, W1C */
-                unsigned long   lh_aoerr0:1;                    /* RW, W1C */
+                unsigned long   rsvd_1_10:10;
                 unsigned long   rh_aoerr0:1;                    /* RW, W1C */
-                unsigned long   xn_aoerr0:1;                    /* RW, W1C */
-                unsigned long   si_aoerr0:1;                    /* RW, W1C */
-                unsigned long   lb_aoerr1:1;                    /* RW, W1C */
-                unsigned long   gr0_aoerr1:1;                   /* RW, W1C */
-                unsigned long   gr1_aoerr1:1;                   /* RW, W1C */
-                unsigned long   lh_aoerr1:1;                    /* RW, W1C */
-                unsigned long   rh_aoerr1:1;                    /* RW, W1C */
-                unsigned long   xn_aoerr1:1;                    /* RW, W1C */
-                unsigned long   si_aoerr1:1;                    /* RW, W1C */
-                unsigned long   rh_vpi_int:1;                   /* RW, W1C */
-                unsigned long   system_shutdown_int:1;          /* RW, W1C */
-                unsigned long   lb_irq_int_0:1;                 /* RW, W1C */
-                unsigned long   lb_irq_int_1:1;                 /* RW, W1C */
-                unsigned long   lb_irq_int_2:1;                 /* RW, W1C */
-                unsigned long   lb_irq_int_3:1;                 /* RW, W1C */
-                unsigned long   lb_irq_int_4:1;                 /* RW, W1C */
-                unsigned long   lb_irq_int_5:1;                 /* RW, W1C */
-                unsigned long   lb_irq_int_6:1;                 /* RW, W1C */
-                unsigned long   lb_irq_int_7:1;                 /* RW, W1C */
-                unsigned long   lb_irq_int_8:1;                 /* RW, W1C */
-                unsigned long   lb_irq_int_9:1;                 /* RW, W1C */
-                unsigned long   lb_irq_int_10:1;                /* RW, W1C */
-                unsigned long   lb_irq_int_11:1;                /* RW, W1C */
-                unsigned long   lb_irq_int_12:1;                /* RW, W1C */
-                unsigned long   lb_irq_int_13:1;                /* RW, W1C */
-                unsigned long   lb_irq_int_14:1;                /* RW, W1C */
-                unsigned long   lb_irq_int_15:1;                /* RW, W1C */
-                unsigned long   l1_nmi_int:1;                   /* RW, W1C */
-                unsigned long   stop_clock:1;                   /* RW, W1C */
-                unsigned long   asic_to_l1:1;                   /* RW, W1C */
-                unsigned long   l1_to_asic:1;                   /* RW, W1C */
-                unsigned long   ltc_int:1;                      /* RW, W1C */
-                unsigned long   la_seq_trigger:1;               /* RW, W1C */
-                unsigned long   ipi_int:1;                      /* RW, W1C */
-                unsigned long   extio_int0:1;                   /* RW, W1C */
-                unsigned long   extio_int1:1;                   /* RW, W1C */
-                unsigned long   extio_int2:1;                   /* RW, W1C */
-                unsigned long   extio_int3:1;                   /* RW, W1C */
-                unsigned long   profile_int:1;                  /* RW, W1C */
-                unsigned long   rtc0:1;                         /* RW, W1C */
-                unsigned long   rtc1:1;                         /* RW, W1C */
-                unsigned long   rtc2:1;                         /* RW, W1C */
-                unsigned long   rtc3:1;                         /* RW, W1C */
-                unsigned long   bau_data:1;                     /* RW, W1C */
-                unsigned long   power_management_req:1;         /* RW, W1C */
-                unsigned long   rsvd_57_63:7;
-        } s1;
-        struct uv2h_event_occurred0_s {
+                unsigned long   rsvd_12_63:52;
+        } s;
+        struct uvxh_event_occurred0_s {
                 unsigned long   lb_hcerr:1;                     /* RW */
                 unsigned long   qp_hcerr:1;                     /* RW */
                 unsigned long   rh_hcerr:1;                     /* RW */
@@ -481,19 +450,20 @@ union uvh_event_occurred0_u {
                 unsigned long   extio_int3:1;                   /* RW */
                 unsigned long   profile_int:1;                  /* RW */
                 unsigned long   rsvd_59_63:5;
-        } s2;
+        } sx;
 };
 
 /* ========================================================================= */
 /*                        UVH_EVENT_OCCURRED0_ALIAS                          */
 /* ========================================================================= */
-#define UVH_EVENT_OCCURRED0_ALIAS                       0x0000000000070008UL
-#define UVH_EVENT_OCCURRED0_ALIAS_32                    0x5f0
+#define UVH_EVENT_OCCURRED0_ALIAS 0x70008UL
+#define UVH_EVENT_OCCURRED0_ALIAS_32 0x5f0
+
 
 /* ========================================================================= */
 /*                         UVH_GR0_TLB_INT0_CONFIG                           */
 /* ========================================================================= */
-#define UVH_GR0_TLB_INT0_CONFIG                         0x61b00UL
+#define UVH_GR0_TLB_INT0_CONFIG 0x61b00UL
 
 #define UVH_GR0_TLB_INT0_CONFIG_VECTOR_SHFT             0
 #define UVH_GR0_TLB_INT0_CONFIG_DM_SHFT                 8
@@ -531,7 +501,7 @@ union uvh_gr0_tlb_int0_config_u {
 /* ========================================================================= */
 /*                         UVH_GR0_TLB_INT1_CONFIG                           */
 /* ========================================================================= */
-#define UVH_GR0_TLB_INT1_CONFIG                         0x61b40UL
+#define UVH_GR0_TLB_INT1_CONFIG 0x61b40UL
 
 #define UVH_GR0_TLB_INT1_CONFIG_VECTOR_SHFT             0
 #define UVH_GR0_TLB_INT1_CONFIG_DM_SHFT                 8
@@ -571,9 +541,11 @@ union uvh_gr0_tlb_int1_config_u {
 /* ========================================================================= */
 #define UV1H_GR0_TLB_MMR_CONTROL 0x401080UL
 #define UV2H_GR0_TLB_MMR_CONTROL 0xc01080UL
-#define UVH_GR0_TLB_MMR_CONTROL (is_uv1_hub() ?                         \
-                        UV1H_GR0_TLB_MMR_CONTROL :                      \
-                        UV2H_GR0_TLB_MMR_CONTROL)
+#define UV3H_GR0_TLB_MMR_CONTROL 0xc01080UL
+#define UVH_GR0_TLB_MMR_CONTROL                                         \
+                (is_uv1_hub() ? UV1H_GR0_TLB_MMR_CONTROL :              \
+                (is_uv2_hub() ? UV2H_GR0_TLB_MMR_CONTROL :              \
+                                UV3H_GR0_TLB_MMR_CONTROL))
 
 #define UVH_GR0_TLB_MMR_CONTROL_INDEX_SHFT              0
 #define UVH_GR0_TLB_MMR_CONTROL_MEM_SEL_SHFT            12
@@ -611,6 +583,21 @@ union uvh_gr0_tlb_int1_config_u {
 #define UV1H_GR0_TLB_MMR_CONTROL_MMR_INJ_TLBRREG_MASK   0x0100000000000000UL
 #define UV1H_GR0_TLB_MMR_CONTROL_MMR_INJ_TLBLRUV_MASK   0x1000000000000000UL
 
+#define UVXH_GR0_TLB_MMR_CONTROL_INDEX_SHFT             0
+#define UVXH_GR0_TLB_MMR_CONTROL_MEM_SEL_SHFT           12
+#define UVXH_GR0_TLB_MMR_CONTROL_AUTO_VALID_EN_SHFT     16
+#define UVXH_GR0_TLB_MMR_CONTROL_MMR_HASH_INDEX_EN_SHFT 20
+#define UVXH_GR0_TLB_MMR_CONTROL_MMR_WRITE_SHFT         30
+#define UVXH_GR0_TLB_MMR_CONTROL_MMR_READ_SHFT          31
+#define UVXH_GR0_TLB_MMR_CONTROL_MMR_OP_DONE_SHFT       32
+#define UVXH_GR0_TLB_MMR_CONTROL_INDEX_MASK             0x0000000000000fffUL
+#define UVXH_GR0_TLB_MMR_CONTROL_MEM_SEL_MASK           0x0000000000003000UL
+#define UVXH_GR0_TLB_MMR_CONTROL_AUTO_VALID_EN_MASK     0x0000000000010000UL
+#define UVXH_GR0_TLB_MMR_CONTROL_MMR_HASH_INDEX_EN_MASK 0x0000000000100000UL
+#define UVXH_GR0_TLB_MMR_CONTROL_MMR_WRITE_MASK         0x0000000040000000UL
+#define UVXH_GR0_TLB_MMR_CONTROL_MMR_READ_MASK          0x0000000080000000UL
+#define UVXH_GR0_TLB_MMR_CONTROL_MMR_OP_DONE_MASK       0x0000000100000000UL
+
 #define UV2H_GR0_TLB_MMR_CONTROL_INDEX_SHFT             0
 #define UV2H_GR0_TLB_MMR_CONTROL_MEM_SEL_SHFT           12
 #define UV2H_GR0_TLB_MMR_CONTROL_AUTO_VALID_EN_SHFT     16
@@ -630,6 +617,23 @@ union uvh_gr0_tlb_int1_config_u {
 #define UV2H_GR0_TLB_MMR_CONTROL_MMR_INJ_CON_MASK       0x0001000000000000UL
 #define UV2H_GR0_TLB_MMR_CONTROL_MMR_INJ_TLBRAM_MASK    0x0010000000000000UL
 
+#define UV3H_GR0_TLB_MMR_CONTROL_INDEX_SHFT             0
+#define UV3H_GR0_TLB_MMR_CONTROL_MEM_SEL_SHFT           12
+#define UV3H_GR0_TLB_MMR_CONTROL_AUTO_VALID_EN_SHFT     16
+#define UV3H_GR0_TLB_MMR_CONTROL_MMR_HASH_INDEX_EN_SHFT 20
+#define UV3H_GR0_TLB_MMR_CONTROL_ECC_SEL_SHFT           21
+#define UV3H_GR0_TLB_MMR_CONTROL_MMR_WRITE_SHFT         30
+#define UV3H_GR0_TLB_MMR_CONTROL_MMR_READ_SHFT          31
+#define UV3H_GR0_TLB_MMR_CONTROL_MMR_OP_DONE_SHFT       32
+#define UV3H_GR0_TLB_MMR_CONTROL_INDEX_MASK             0x0000000000000fffUL
+#define UV3H_GR0_TLB_MMR_CONTROL_MEM_SEL_MASK           0x0000000000003000UL
+#define UV3H_GR0_TLB_MMR_CONTROL_AUTO_VALID_EN_MASK     0x0000000000010000UL
+#define UV3H_GR0_TLB_MMR_CONTROL_MMR_HASH_INDEX_EN_MASK 0x0000000000100000UL
+#define UV3H_GR0_TLB_MMR_CONTROL_ECC_SEL_MASK           0x0000000000200000UL
+#define UV3H_GR0_TLB_MMR_CONTROL_MMR_WRITE_MASK         0x0000000040000000UL
+#define UV3H_GR0_TLB_MMR_CONTROL_MMR_READ_MASK          0x0000000080000000UL
+#define UV3H_GR0_TLB_MMR_CONTROL_MMR_OP_DONE_MASK       0x0000000100000000UL
+
 union uvh_gr0_tlb_mmr_control_u {
         unsigned long   v;
         struct uvh_gr0_tlb_mmr_control_s {
@@ -642,7 +646,9 @@ union uvh_gr0_tlb_mmr_control_u {
                 unsigned long   rsvd_21_29:9;
                 unsigned long   mmr_write:1;                    /* WP */
                 unsigned long   mmr_read:1;                     /* WP */
-                unsigned long   rsvd_32_63:32;
+                unsigned long   rsvd_32_48:17;
+                unsigned long   rsvd_49_51:3;
+                unsigned long   rsvd_52_63:12;
         } s;
         struct uv1h_gr0_tlb_mmr_control_s {
                 unsigned long   index:12;                       /* RW */
@@ -666,6 +672,23 @@ union uvh_gr0_tlb_mmr_control_u {
                 unsigned long   mmr_inj_tlblruv:1;              /* RW */
                 unsigned long   rsvd_61_63:3;
         } s1;
+        struct uvxh_gr0_tlb_mmr_control_s {
+                unsigned long   index:12;                       /* RW */
+                unsigned long   mem_sel:2;                      /* RW */
+                unsigned long   rsvd_14_15:2;
+                unsigned long   auto_valid_en:1;                /* RW */
+                unsigned long   rsvd_17_19:3;
+                unsigned long   mmr_hash_index_en:1;            /* RW */
+                unsigned long   rsvd_21_29:9;
+                unsigned long   mmr_write:1;                    /* WP */
+                unsigned long   mmr_read:1;                     /* WP */
+                unsigned long   mmr_op_done:1;                  /* RW */
+                unsigned long   rsvd_33_47:15;
+                unsigned long   rsvd_48:1;
+                unsigned long   rsvd_49_51:3;
+                unsigned long   rsvd_52:1;
+                unsigned long   rsvd_53_63:11;
+        } sx;
         struct uv2h_gr0_tlb_mmr_control_s {
                 unsigned long   index:12;                       /* RW */
                 unsigned long   mem_sel:2;                      /* RW */
@@ -683,6 +706,24 @@ union uvh_gr0_tlb_mmr_control_u {
                 unsigned long   mmr_inj_tlbram:1;               /* RW */
                 unsigned long   rsvd_53_63:11;
         } s2;
+        struct uv3h_gr0_tlb_mmr_control_s {
+                unsigned long   index:12;                       /* RW */
+                unsigned long   mem_sel:2;                      /* RW */
+                unsigned long   rsvd_14_15:2;
+                unsigned long   auto_valid_en:1;                /* RW */
+                unsigned long   rsvd_17_19:3;
+                unsigned long   mmr_hash_index_en:1;            /* RW */
+                unsigned long   ecc_sel:1;                      /* RW */
+                unsigned long   rsvd_22_29:8;
+                unsigned long   mmr_write:1;                    /* WP */
+                unsigned long   mmr_read:1;                     /* WP */
+                unsigned long   mmr_op_done:1;                  /* RW */
+                unsigned long   rsvd_33_47:15;
+                unsigned long   undef_48:1;                     /* Undefined */
+                unsigned long   rsvd_49_51:3;
+                unsigned long   undef_52:1;                     /* Undefined */
+                unsigned long   rsvd_53_63:11;
+        } s3;
 };
 
 /* ========================================================================= */
@@ -690,9 +731,11 @@ union uvh_gr0_tlb_mmr_control_u {
 /* ========================================================================= */
 #define UV1H_GR0_TLB_MMR_READ_DATA_HI 0x4010a0UL
 #define UV2H_GR0_TLB_MMR_READ_DATA_HI 0xc010a0UL
-#define UVH_GR0_TLB_MMR_READ_DATA_HI (is_uv1_hub() ?                    \
-                        UV1H_GR0_TLB_MMR_READ_DATA_HI :                 \
-                        UV2H_GR0_TLB_MMR_READ_DATA_HI)
+#define UV3H_GR0_TLB_MMR_READ_DATA_HI 0xc010a0UL
+#define UVH_GR0_TLB_MMR_READ_DATA_HI                                    \
+                (is_uv1_hub() ? UV1H_GR0_TLB_MMR_READ_DATA_HI :         \
+                (is_uv2_hub() ? UV2H_GR0_TLB_MMR_READ_DATA_HI :         \
+                                UV3H_GR0_TLB_MMR_READ_DATA_HI))
 
 #define UVH_GR0_TLB_MMR_READ_DATA_HI_PFN_SHFT           0
 #define UVH_GR0_TLB_MMR_READ_DATA_HI_GAA_SHFT           41
@@ -703,6 +746,46 @@ union uvh_gr0_tlb_mmr_control_u {
 #define UVH_GR0_TLB_MMR_READ_DATA_HI_DIRTY_MASK         0x0000080000000000UL
 #define UVH_GR0_TLB_MMR_READ_DATA_HI_LARGER_MASK        0x0000100000000000UL
 
+#define UV1H_GR0_TLB_MMR_READ_DATA_HI_PFN_SHFT          0
+#define UV1H_GR0_TLB_MMR_READ_DATA_HI_GAA_SHFT          41
+#define UV1H_GR0_TLB_MMR_READ_DATA_HI_DIRTY_SHFT        43
+#define UV1H_GR0_TLB_MMR_READ_DATA_HI_LARGER_SHFT       44
+#define UV1H_GR0_TLB_MMR_READ_DATA_HI_PFN_MASK          0x000001ffffffffffUL
+#define UV1H_GR0_TLB_MMR_READ_DATA_HI_GAA_MASK          0x0000060000000000UL
+#define UV1H_GR0_TLB_MMR_READ_DATA_HI_DIRTY_MASK        0x0000080000000000UL
+#define UV1H_GR0_TLB_MMR_READ_DATA_HI_LARGER_MASK       0x0000100000000000UL
+
+#define UVXH_GR0_TLB_MMR_READ_DATA_HI_PFN_SHFT          0
+#define UVXH_GR0_TLB_MMR_READ_DATA_HI_GAA_SHFT          41
+#define UVXH_GR0_TLB_MMR_READ_DATA_HI_DIRTY_SHFT        43
+#define UVXH_GR0_TLB_MMR_READ_DATA_HI_LARGER_SHFT       44
+#define UVXH_GR0_TLB_MMR_READ_DATA_HI_PFN_MASK          0x000001ffffffffffUL
+#define UVXH_GR0_TLB_MMR_READ_DATA_HI_GAA_MASK          0x0000060000000000UL
+#define UVXH_GR0_TLB_MMR_READ_DATA_HI_DIRTY_MASK        0x0000080000000000UL
+#define UVXH_GR0_TLB_MMR_READ_DATA_HI_LARGER_MASK       0x0000100000000000UL
+
+#define UV2H_GR0_TLB_MMR_READ_DATA_HI_PFN_SHFT          0
+#define UV2H_GR0_TLB_MMR_READ_DATA_HI_GAA_SHFT          41
+#define UV2H_GR0_TLB_MMR_READ_DATA_HI_DIRTY_SHFT        43
+#define UV2H_GR0_TLB_MMR_READ_DATA_HI_LARGER_SHFT       44
+#define UV2H_GR0_TLB_MMR_READ_DATA_HI_PFN_MASK          0x000001ffffffffffUL
+#define UV2H_GR0_TLB_MMR_READ_DATA_HI_GAA_MASK          0x0000060000000000UL
+#define UV2H_GR0_TLB_MMR_READ_DATA_HI_DIRTY_MASK        0x0000080000000000UL
+#define UV2H_GR0_TLB_MMR_READ_DATA_HI_LARGER_MASK       0x0000100000000000UL
+
+#define UV3H_GR0_TLB_MMR_READ_DATA_HI_PFN_SHFT          0
+#define UV3H_GR0_TLB_MMR_READ_DATA_HI_GAA_SHFT          41
+#define UV3H_GR0_TLB_MMR_READ_DATA_HI_DIRTY_SHFT        43
+#define UV3H_GR0_TLB_MMR_READ_DATA_HI_LARGER_SHFT       44
+#define UV3H_GR0_TLB_MMR_READ_DATA_HI_AA_EXT_SHFT       45
+#define UV3H_GR0_TLB_MMR_READ_DATA_HI_WAY_ECC_SHFT      55
+#define UV3H_GR0_TLB_MMR_READ_DATA_HI_PFN_MASK          0x000001ffffffffffUL
+#define UV3H_GR0_TLB_MMR_READ_DATA_HI_GAA_MASK          0x0000060000000000UL
+#define UV3H_GR0_TLB_MMR_READ_DATA_HI_DIRTY_MASK        0x0000080000000000UL
+#define UV3H_GR0_TLB_MMR_READ_DATA_HI_LARGER_MASK       0x0000100000000000UL
+#define UV3H_GR0_TLB_MMR_READ_DATA_HI_AA_EXT_MASK       0x0000200000000000UL
+#define UV3H_GR0_TLB_MMR_READ_DATA_HI_WAY_ECC_MASK      0xff80000000000000UL
+
 union uvh_gr0_tlb_mmr_read_data_hi_u {
         unsigned long   v;
         struct uvh_gr0_tlb_mmr_read_data_hi_s {
@@ -712,6 +795,36 @@ union uvh_gr0_tlb_mmr_read_data_hi_u {
                 unsigned long   larger:1;                       /* RO */
                 unsigned long   rsvd_45_63:19;
         } s;
+        struct uv1h_gr0_tlb_mmr_read_data_hi_s {
+                unsigned long   pfn:41;                         /* RO */
+                unsigned long   gaa:2;                          /* RO */
+                unsigned long   dirty:1;                        /* RO */
+                unsigned long   larger:1;                       /* RO */
+                unsigned long   rsvd_45_63:19;
+        } s1;
+        struct uvxh_gr0_tlb_mmr_read_data_hi_s {
+                unsigned long   pfn:41;                         /* RO */
+                unsigned long   gaa:2;                          /* RO */
+                unsigned long   dirty:1;                        /* RO */
+                unsigned long   larger:1;                       /* RO */
+                unsigned long   rsvd_45_63:19;
+        } sx;
+        struct uv2h_gr0_tlb_mmr_read_data_hi_s {
+                unsigned long   pfn:41;                         /* RO */
+                unsigned long   gaa:2;                          /* RO */
+                unsigned long   dirty:1;                        /* RO */
+                unsigned long   larger:1;                       /* RO */
+                unsigned long   rsvd_45_63:19;
+        } s2;
+        struct uv3h_gr0_tlb_mmr_read_data_hi_s {
+                unsigned long   pfn:41;                         /* RO */
+                unsigned long   gaa:2;                          /* RO */
+                unsigned long   dirty:1;                        /* RO */
+                unsigned long   larger:1;                       /* RO */
+                unsigned long   aa_ext:1;                       /* RO */
+                unsigned long   undef_46_54:9;                  /* Undefined */
+                unsigned long   way_ecc:9;                      /* RO */
+        } s3;
 };
 
 /* ========================================================================= */
@@ -719,9 +832,11 @@ union uvh_gr0_tlb_mmr_read_data_hi_u {
 /* ========================================================================= */
 #define UV1H_GR0_TLB_MMR_READ_DATA_LO 0x4010a8UL
 #define UV2H_GR0_TLB_MMR_READ_DATA_LO 0xc010a8UL
-#define UVH_GR0_TLB_MMR_READ_DATA_LO (is_uv1_hub() ?                    \
-                        UV1H_GR0_TLB_MMR_READ_DATA_LO :                 \
-                        UV2H_GR0_TLB_MMR_READ_DATA_LO)
+#define UV3H_GR0_TLB_MMR_READ_DATA_LO 0xc010a8UL
+#define UVH_GR0_TLB_MMR_READ_DATA_LO                                    \
+                (is_uv1_hub() ? UV1H_GR0_TLB_MMR_READ_DATA_LO :         \
+                (is_uv2_hub() ? UV2H_GR0_TLB_MMR_READ_DATA_LO :         \
+                                UV3H_GR0_TLB_MMR_READ_DATA_LO))
 
 #define UVH_GR0_TLB_MMR_READ_DATA_LO_VPN_SHFT           0
 #define UVH_GR0_TLB_MMR_READ_DATA_LO_ASID_SHFT          39
@@ -730,6 +845,34 @@ union uvh_gr0_tlb_mmr_read_data_hi_u {
 #define UVH_GR0_TLB_MMR_READ_DATA_LO_ASID_MASK          0x7fffff8000000000UL
 #define UVH_GR0_TLB_MMR_READ_DATA_LO_VALID_MASK         0x8000000000000000UL
 
+#define UV1H_GR0_TLB_MMR_READ_DATA_LO_VPN_SHFT          0
+#define UV1H_GR0_TLB_MMR_READ_DATA_LO_ASID_SHFT         39
+#define UV1H_GR0_TLB_MMR_READ_DATA_LO_VALID_SHFT        63
+#define UV1H_GR0_TLB_MMR_READ_DATA_LO_VPN_MASK          0x0000007fffffffffUL
+#define UV1H_GR0_TLB_MMR_READ_DATA_LO_ASID_MASK         0x7fffff8000000000UL
+#define UV1H_GR0_TLB_MMR_READ_DATA_LO_VALID_MASK        0x8000000000000000UL
+
+#define UVXH_GR0_TLB_MMR_READ_DATA_LO_VPN_SHFT          0
+#define UVXH_GR0_TLB_MMR_READ_DATA_LO_ASID_SHFT         39
+#define UVXH_GR0_TLB_MMR_READ_DATA_LO_VALID_SHFT        63
+#define UVXH_GR0_TLB_MMR_READ_DATA_LO_VPN_MASK          0x0000007fffffffffUL
+#define UVXH_GR0_TLB_MMR_READ_DATA_LO_ASID_MASK         0x7fffff8000000000UL
+#define UVXH_GR0_TLB_MMR_READ_DATA_LO_VALID_MASK        0x8000000000000000UL
+
+#define UV2H_GR0_TLB_MMR_READ_DATA_LO_VPN_SHFT          0
+#define UV2H_GR0_TLB_MMR_READ_DATA_LO_ASID_SHFT         39
+#define UV2H_GR0_TLB_MMR_READ_DATA_LO_VALID_SHFT        63
+#define UV2H_GR0_TLB_MMR_READ_DATA_LO_VPN_MASK          0x0000007fffffffffUL
+#define UV2H_GR0_TLB_MMR_READ_DATA_LO_ASID_MASK         0x7fffff8000000000UL
+#define UV2H_GR0_TLB_MMR_READ_DATA_LO_VALID_MASK        0x8000000000000000UL
+
+#define UV3H_GR0_TLB_MMR_READ_DATA_LO_VPN_SHFT          0
+#define UV3H_GR0_TLB_MMR_READ_DATA_LO_ASID_SHFT         39
+#define UV3H_GR0_TLB_MMR_READ_DATA_LO_VALID_SHFT        63
+#define UV3H_GR0_TLB_MMR_READ_DATA_LO_VPN_MASK          0x0000007fffffffffUL
+#define UV3H_GR0_TLB_MMR_READ_DATA_LO_ASID_MASK         0x7fffff8000000000UL
+#define UV3H_GR0_TLB_MMR_READ_DATA_LO_VALID_MASK        0x8000000000000000UL
+
 union uvh_gr0_tlb_mmr_read_data_lo_u {
         unsigned long   v;
         struct uvh_gr0_tlb_mmr_read_data_lo_s {
@@ -737,12 +880,32 @@ union uvh_gr0_tlb_mmr_read_data_lo_u {
                 unsigned long   asid:24;                        /* RO */
                 unsigned long   valid:1;                        /* RO */
         } s;
+        struct uv1h_gr0_tlb_mmr_read_data_lo_s {
+                unsigned long   vpn:39;                         /* RO */
+                unsigned long   asid:24;                        /* RO */
+                unsigned long   valid:1;                        /* RO */
+        } s1;
+        struct uvxh_gr0_tlb_mmr_read_data_lo_s {
+                unsigned long   vpn:39;                         /* RO */
+                unsigned long   asid:24;                        /* RO */
+                unsigned long   valid:1;                        /* RO */
+        } sx;
+        struct uv2h_gr0_tlb_mmr_read_data_lo_s {
+                unsigned long   vpn:39;                         /* RO */
+                unsigned long   asid:24;                        /* RO */
+                unsigned long   valid:1;                        /* RO */
+        } s2;
+        struct uv3h_gr0_tlb_mmr_read_data_lo_s {
+                unsigned long   vpn:39;                         /* RO */
+                unsigned long   asid:24;                        /* RO */
+                unsigned long   valid:1;                        /* RO */
+        } s3;
 };
 
 /* ========================================================================= */
 /*                         UVH_GR1_TLB_INT0_CONFIG                           */
 /* ========================================================================= */
-#define UVH_GR1_TLB_INT0_CONFIG                         0x61f00UL
+#define UVH_GR1_TLB_INT0_CONFIG 0x61f00UL
 
 #define UVH_GR1_TLB_INT0_CONFIG_VECTOR_SHFT             0
 #define UVH_GR1_TLB_INT0_CONFIG_DM_SHFT                 8
@@ -780,7 +943,7 @@ union uvh_gr1_tlb_int0_config_u {
 /* ========================================================================= */
 /*                         UVH_GR1_TLB_INT1_CONFIG                           */
 /* ========================================================================= */
-#define UVH_GR1_TLB_INT1_CONFIG                         0x61f40UL
+#define UVH_GR1_TLB_INT1_CONFIG 0x61f40UL
 
 #define UVH_GR1_TLB_INT1_CONFIG_VECTOR_SHFT             0
 #define UVH_GR1_TLB_INT1_CONFIG_DM_SHFT                 8
@@ -820,9 +983,11 @@ union uvh_gr1_tlb_int1_config_u {
 /* ========================================================================= */
 #define UV1H_GR1_TLB_MMR_CONTROL 0x801080UL
 #define UV2H_GR1_TLB_MMR_CONTROL 0x1001080UL
-#define UVH_GR1_TLB_MMR_CONTROL (is_uv1_hub() ?                         \
-                        UV1H_GR1_TLB_MMR_CONTROL :                      \
-                        UV2H_GR1_TLB_MMR_CONTROL)
+#define UV3H_GR1_TLB_MMR_CONTROL 0x1001080UL
+#define UVH_GR1_TLB_MMR_CONTROL                                         \
+                (is_uv1_hub() ? UV1H_GR1_TLB_MMR_CONTROL :              \
+                (is_uv2_hub() ? UV2H_GR1_TLB_MMR_CONTROL :              \
+                                UV3H_GR1_TLB_MMR_CONTROL))
 
 #define UVH_GR1_TLB_MMR_CONTROL_INDEX_SHFT              0
 #define UVH_GR1_TLB_MMR_CONTROL_MEM_SEL_SHFT            12
@@ -860,6 +1025,21 @@ union uvh_gr1_tlb_int1_config_u {
 #define UV1H_GR1_TLB_MMR_CONTROL_MMR_INJ_TLBRREG_MASK   0x0100000000000000UL
 #define UV1H_GR1_TLB_MMR_CONTROL_MMR_INJ_TLBLRUV_MASK   0x1000000000000000UL
 
+#define UVXH_GR1_TLB_MMR_CONTROL_INDEX_SHFT             0
+#define UVXH_GR1_TLB_MMR_CONTROL_MEM_SEL_SHFT           12
+#define UVXH_GR1_TLB_MMR_CONTROL_AUTO_VALID_EN_SHFT     16
+#define UVXH_GR1_TLB_MMR_CONTROL_MMR_HASH_INDEX_EN_SHFT 20
+#define UVXH_GR1_TLB_MMR_CONTROL_MMR_WRITE_SHFT         30
+#define UVXH_GR1_TLB_MMR_CONTROL_MMR_READ_SHFT          31
+#define UVXH_GR1_TLB_MMR_CONTROL_MMR_OP_DONE_SHFT       32
+#define UVXH_GR1_TLB_MMR_CONTROL_INDEX_MASK             0x0000000000000fffUL
+#define UVXH_GR1_TLB_MMR_CONTROL_MEM_SEL_MASK           0x0000000000003000UL
+#define UVXH_GR1_TLB_MMR_CONTROL_AUTO_VALID_EN_MASK     0x0000000000010000UL
+#define UVXH_GR1_TLB_MMR_CONTROL_MMR_HASH_INDEX_EN_MASK 0x0000000000100000UL
+#define UVXH_GR1_TLB_MMR_CONTROL_MMR_WRITE_MASK         0x0000000040000000UL
+#define UVXH_GR1_TLB_MMR_CONTROL_MMR_READ_MASK          0x0000000080000000UL
+#define UVXH_GR1_TLB_MMR_CONTROL_MMR_OP_DONE_MASK       0x0000000100000000UL
+
 #define UV2H_GR1_TLB_MMR_CONTROL_INDEX_SHFT             0
 #define UV2H_GR1_TLB_MMR_CONTROL_MEM_SEL_SHFT           12
 #define UV2H_GR1_TLB_MMR_CONTROL_AUTO_VALID_EN_SHFT     16
@@ -879,6 +1059,23 @@ union uvh_gr1_tlb_int1_config_u {
 #define UV2H_GR1_TLB_MMR_CONTROL_MMR_INJ_CON_MASK       0x0001000000000000UL
 #define UV2H_GR1_TLB_MMR_CONTROL_MMR_INJ_TLBRAM_MASK    0x0010000000000000UL
 
+#define UV3H_GR1_TLB_MMR_CONTROL_INDEX_SHFT             0
+#define UV3H_GR1_TLB_MMR_CONTROL_MEM_SEL_SHFT           12
+#define UV3H_GR1_TLB_MMR_CONTROL_AUTO_VALID_EN_SHFT     16
+#define UV3H_GR1_TLB_MMR_CONTROL_MMR_HASH_INDEX_EN_SHFT 20
+#define UV3H_GR1_TLB_MMR_CONTROL_ECC_SEL_SHFT           21
+#define UV3H_GR1_TLB_MMR_CONTROL_MMR_WRITE_SHFT         30
+#define UV3H_GR1_TLB_MMR_CONTROL_MMR_READ_SHFT          31
+#define UV3H_GR1_TLB_MMR_CONTROL_MMR_OP_DONE_SHFT       32
+#define UV3H_GR1_TLB_MMR_CONTROL_INDEX_MASK             0x0000000000000fffUL
+#define UV3H_GR1_TLB_MMR_CONTROL_MEM_SEL_MASK           0x0000000000003000UL
+#define UV3H_GR1_TLB_MMR_CONTROL_AUTO_VALID_EN_MASK     0x0000000000010000UL
+#define UV3H_GR1_TLB_MMR_CONTROL_MMR_HASH_INDEX_EN_MASK 0x0000000000100000UL
+#define UV3H_GR1_TLB_MMR_CONTROL_ECC_SEL_MASK           0x0000000000200000UL
+#define UV3H_GR1_TLB_MMR_CONTROL_MMR_WRITE_MASK         0x0000000040000000UL
+#define UV3H_GR1_TLB_MMR_CONTROL_MMR_READ_MASK          0x0000000080000000UL
+#define UV3H_GR1_TLB_MMR_CONTROL_MMR_OP_DONE_MASK       0x0000000100000000UL
+
 union uvh_gr1_tlb_mmr_control_u {
         unsigned long   v;
         struct uvh_gr1_tlb_mmr_control_s {
@@ -891,7 +1088,9 @@ union uvh_gr1_tlb_mmr_control_u {
                 unsigned long   rsvd_21_29:9;
                 unsigned long   mmr_write:1;                    /* WP */
                 unsigned long   mmr_read:1;                     /* WP */
-                unsigned long   rsvd_32_63:32;
+                unsigned long   rsvd_32_48:17;
+                unsigned long   rsvd_49_51:3;
+                unsigned long   rsvd_52_63:12;
         } s;
         struct uv1h_gr1_tlb_mmr_control_s {
                 unsigned long   index:12;                       /* RW */
@@ -915,6 +1114,23 @@ union uvh_gr1_tlb_mmr_control_u {
                 unsigned long   mmr_inj_tlblruv:1;              /* RW */
                 unsigned long   rsvd_61_63:3;
         } s1;
+        struct uvxh_gr1_tlb_mmr_control_s {
+                unsigned long   index:12;                       /* RW */
+                unsigned long   mem_sel:2;                      /* RW */
+                unsigned long   rsvd_14_15:2;
+                unsigned long   auto_valid_en:1;                /* RW */
+                unsigned long   rsvd_17_19:3;
+                unsigned long   mmr_hash_index_en:1;            /* RW */
+                unsigned long   rsvd_21_29:9;
+                unsigned long   mmr_write:1;                    /* WP */
+                unsigned long   mmr_read:1;                     /* WP */
+                unsigned long   mmr_op_done:1;                  /* RW */
+                unsigned long   rsvd_33_47:15;
+                unsigned long   rsvd_48:1;
+                unsigned long   rsvd_49_51:3;
+                unsigned long   rsvd_52:1;
+                unsigned long   rsvd_53_63:11;
+        } sx;
         struct uv2h_gr1_tlb_mmr_control_s {
                 unsigned long   index:12;                       /* RW */
                 unsigned long   mem_sel:2;                      /* RW */
@@ -932,6 +1148,24 @@ union uvh_gr1_tlb_mmr_control_u {
                 unsigned long   mmr_inj_tlbram:1;               /* RW */
                 unsigned long   rsvd_53_63:11;
         } s2;
+        struct uv3h_gr1_tlb_mmr_control_s {
+                unsigned long   index:12;                       /* RW */
+                unsigned long   mem_sel:2;                      /* RW */
+                unsigned long   rsvd_14_15:2;
+                unsigned long   auto_valid_en:1;                /* RW */
+                unsigned long   rsvd_17_19:3;
+                unsigned long   mmr_hash_index_en:1;            /* RW */
+                unsigned long   ecc_sel:1;                      /* RW */
+                unsigned long   rsvd_22_29:8;
+                unsigned long   mmr_write:1;                    /* WP */
+                unsigned long   mmr_read:1;                     /* WP */
+                unsigned long   mmr_op_done:1;                  /* RW */
+                unsigned long   rsvd_33_47:15;
+                unsigned long   undef_48:1;                     /* Undefined */
+                unsigned long   rsvd_49_51:3;
+                unsigned long   undef_52:1;                     /* Undefined */
+                unsigned long   rsvd_53_63:11;
+        } s3;
 };
 
 /* ========================================================================= */
@@ -939,9 +1173,11 @@ union uvh_gr1_tlb_mmr_control_u {
 /* ========================================================================= */
 #define UV1H_GR1_TLB_MMR_READ_DATA_HI 0x8010a0UL
 #define UV2H_GR1_TLB_MMR_READ_DATA_HI 0x10010a0UL
-#define UVH_GR1_TLB_MMR_READ_DATA_HI (is_uv1_hub() ?                    \
-                        UV1H_GR1_TLB_MMR_READ_DATA_HI :                 \
-                        UV2H_GR1_TLB_MMR_READ_DATA_HI)
+#define UV3H_GR1_TLB_MMR_READ_DATA_HI 0x10010a0UL
+#define UVH_GR1_TLB_MMR_READ_DATA_HI                                    \
+                (is_uv1_hub() ? UV1H_GR1_TLB_MMR_READ_DATA_HI :         \
+                (is_uv2_hub() ? UV2H_GR1_TLB_MMR_READ_DATA_HI :         \
+                                UV3H_GR1_TLB_MMR_READ_DATA_HI))
 
 #define UVH_GR1_TLB_MMR_READ_DATA_HI_PFN_SHFT           0
 #define UVH_GR1_TLB_MMR_READ_DATA_HI_GAA_SHFT           41
@@ -952,6 +1188,46 @@ union uvh_gr1_tlb_mmr_control_u {
 #define UVH_GR1_TLB_MMR_READ_DATA_HI_DIRTY_MASK         0x0000080000000000UL
 #define UVH_GR1_TLB_MMR_READ_DATA_HI_LARGER_MASK        0x0000100000000000UL
 
+#define UV1H_GR1_TLB_MMR_READ_DATA_HI_PFN_SHFT          0
+#define UV1H_GR1_TLB_MMR_READ_DATA_HI_GAA_SHFT          41
+#define UV1H_GR1_TLB_MMR_READ_DATA_HI_DIRTY_SHFT        43
+#define UV1H_GR1_TLB_MMR_READ_DATA_HI_LARGER_SHFT       44
+#define UV1H_GR1_TLB_MMR_READ_DATA_HI_PFN_MASK          0x000001ffffffffffUL
+#define UV1H_GR1_TLB_MMR_READ_DATA_HI_GAA_MASK          0x0000060000000000UL
+#define UV1H_GR1_TLB_MMR_READ_DATA_HI_DIRTY_MASK        0x0000080000000000UL
+#define UV1H_GR1_TLB_MMR_READ_DATA_HI_LARGER_MASK       0x0000100000000000UL
+
+#define UVXH_GR1_TLB_MMR_READ_DATA_HI_PFN_SHFT          0
+#define UVXH_GR1_TLB_MMR_READ_DATA_HI_GAA_SHFT          41
+#define UVXH_GR1_TLB_MMR_READ_DATA_HI_DIRTY_SHFT        43
+#define UVXH_GR1_TLB_MMR_READ_DATA_HI_LARGER_SHFT       44
+#define UVXH_GR1_TLB_MMR_READ_DATA_HI_PFN_MASK          0x000001ffffffffffUL
+#define UVXH_GR1_TLB_MMR_READ_DATA_HI_GAA_MASK          0x0000060000000000UL
+#define UVXH_GR1_TLB_MMR_READ_DATA_HI_DIRTY_MASK        0x0000080000000000UL
+#define UVXH_GR1_TLB_MMR_READ_DATA_HI_LARGER_MASK       0x0000100000000000UL
+
+#define UV2H_GR1_TLB_MMR_READ_DATA_HI_PFN_SHFT          0
+#define UV2H_GR1_TLB_MMR_READ_DATA_HI_GAA_SHFT          41
+#define UV2H_GR1_TLB_MMR_READ_DATA_HI_DIRTY_SHFT        43
+#define UV2H_GR1_TLB_MMR_READ_DATA_HI_LARGER_SHFT       44
+#define UV2H_GR1_TLB_MMR_READ_DATA_HI_PFN_MASK          0x000001ffffffffffUL
+#define UV2H_GR1_TLB_MMR_READ_DATA_HI_GAA_MASK          0x0000060000000000UL
+#define UV2H_GR1_TLB_MMR_READ_DATA_HI_DIRTY_MASK        0x0000080000000000UL
+#define UV2H_GR1_TLB_MMR_READ_DATA_HI_LARGER_MASK       0x0000100000000000UL
+
+#define UV3H_GR1_TLB_MMR_READ_DATA_HI_PFN_SHFT          0
+#define UV3H_GR1_TLB_MMR_READ_DATA_HI_GAA_SHFT          41
+#define UV3H_GR1_TLB_MMR_READ_DATA_HI_DIRTY_SHFT        43
+#define UV3H_GR1_TLB_MMR_READ_DATA_HI_LARGER_SHFT       44
+#define UV3H_GR1_TLB_MMR_READ_DATA_HI_AA_EXT_SHFT       45
+#define UV3H_GR1_TLB_MMR_READ_DATA_HI_WAY_ECC_SHFT      55
+#define UV3H_GR1_TLB_MMR_READ_DATA_HI_PFN_MASK          0x000001ffffffffffUL
+#define UV3H_GR1_TLB_MMR_READ_DATA_HI_GAA_MASK          0x0000060000000000UL
+#define UV3H_GR1_TLB_MMR_READ_DATA_HI_DIRTY_MASK        0x0000080000000000UL
+#define UV3H_GR1_TLB_MMR_READ_DATA_HI_LARGER_MASK       0x0000100000000000UL
+#define UV3H_GR1_TLB_MMR_READ_DATA_HI_AA_EXT_MASK       0x0000200000000000UL
+#define UV3H_GR1_TLB_MMR_READ_DATA_HI_WAY_ECC_MASK      0xff80000000000000UL
+
 union uvh_gr1_tlb_mmr_read_data_hi_u {
         unsigned long   v;
         struct uvh_gr1_tlb_mmr_read_data_hi_s {
@@ -961,6 +1237,36 @@ union uvh_gr1_tlb_mmr_read_data_hi_u {
                 unsigned long   larger:1;                       /* RO */
                 unsigned long   rsvd_45_63:19;
         } s;
+        struct uv1h_gr1_tlb_mmr_read_data_hi_s {
+                unsigned long   pfn:41;                         /* RO */
+                unsigned long   gaa:2;                          /* RO */
+                unsigned long   dirty:1;                        /* RO */
+                unsigned long   larger:1;                       /* RO */
+                unsigned long   rsvd_45_63:19;
+        } s1;
+        struct uvxh_gr1_tlb_mmr_read_data_hi_s {
+                unsigned long   pfn:41;                         /* RO */
+                unsigned long   gaa:2;                          /* RO */
+                unsigned long   dirty:1;                        /* RO */
+                unsigned long   larger:1;                       /* RO */
+                unsigned long   rsvd_45_63:19;
+        } sx;
+        struct uv2h_gr1_tlb_mmr_read_data_hi_s {
+                unsigned long   pfn:41;                         /* RO */
+                unsigned long   gaa:2;                          /* RO */
+                unsigned long   dirty:1;                        /* RO */
+                unsigned long   larger:1;                       /* RO */
+                unsigned long   rsvd_45_63:19;
+        } s2;
+        struct uv3h_gr1_tlb_mmr_read_data_hi_s {
+                unsigned long   pfn:41;                         /* RO */
+                unsigned long   gaa:2;                          /* RO */
+                unsigned long   dirty:1;                        /* RO */
+                unsigned long   larger:1;                       /* RO */
+                unsigned long   aa_ext:1;                       /* RO */
+                unsigned long   undef_46_54:9;                  /* Undefined */
+                unsigned long   way_ecc:9;                      /* RO */
+        } s3;
 };
 
 /* ========================================================================= */
@@ -968,9 +1274,11 @@ union uvh_gr1_tlb_mmr_read_data_hi_u {
 /* ========================================================================= */
 #define UV1H_GR1_TLB_MMR_READ_DATA_LO 0x8010a8UL
 #define UV2H_GR1_TLB_MMR_READ_DATA_LO 0x10010a8UL
-#define UVH_GR1_TLB_MMR_READ_DATA_LO (is_uv1_hub() ?                    \
-                        UV1H_GR1_TLB_MMR_READ_DATA_LO :                 \
-                        UV2H_GR1_TLB_MMR_READ_DATA_LO)
+#define UV3H_GR1_TLB_MMR_READ_DATA_LO 0x10010a8UL
+#define UVH_GR1_TLB_MMR_READ_DATA_LO                                    \
+                (is_uv1_hub() ? UV1H_GR1_TLB_MMR_READ_DATA_LO :         \
+                (is_uv2_hub() ? UV2H_GR1_TLB_MMR_READ_DATA_LO :         \
+                                UV3H_GR1_TLB_MMR_READ_DATA_LO))
 
 #define UVH_GR1_TLB_MMR_READ_DATA_LO_VPN_SHFT           0
 #define UVH_GR1_TLB_MMR_READ_DATA_LO_ASID_SHFT          39
@@ -979,6 +1287,34 @@ union uvh_gr1_tlb_mmr_read_data_hi_u {
 #define UVH_GR1_TLB_MMR_READ_DATA_LO_ASID_MASK          0x7fffff8000000000UL
 #define UVH_GR1_TLB_MMR_READ_DATA_LO_VALID_MASK         0x8000000000000000UL
 
+#define UV1H_GR1_TLB_MMR_READ_DATA_LO_VPN_SHFT          0
+#define UV1H_GR1_TLB_MMR_READ_DATA_LO_ASID_SHFT         39
+#define UV1H_GR1_TLB_MMR_READ_DATA_LO_VALID_SHFT        63
+#define UV1H_GR1_TLB_MMR_READ_DATA_LO_VPN_MASK          0x0000007fffffffffUL
+#define UV1H_GR1_TLB_MMR_READ_DATA_LO_ASID_MASK         0x7fffff8000000000UL
+#define UV1H_GR1_TLB_MMR_READ_DATA_LO_VALID_MASK        0x8000000000000000UL
+
+#define UVXH_GR1_TLB_MMR_READ_DATA_LO_VPN_SHFT          0
+#define UVXH_GR1_TLB_MMR_READ_DATA_LO_ASID_SHFT         39
+#define UVXH_GR1_TLB_MMR_READ_DATA_LO_VALID_SHFT        63
+#define UVXH_GR1_TLB_MMR_READ_DATA_LO_VPN_MASK          0x0000007fffffffffUL
+#define UVXH_GR1_TLB_MMR_READ_DATA_LO_ASID_MASK         0x7fffff8000000000UL
+#define UVXH_GR1_TLB_MMR_READ_DATA_LO_VALID_MASK        0x8000000000000000UL
+
+#define UV2H_GR1_TLB_MMR_READ_DATA_LO_VPN_SHFT          0
+#define UV2H_GR1_TLB_MMR_READ_DATA_LO_ASID_SHFT         39
+#define UV2H_GR1_TLB_MMR_READ_DATA_LO_VALID_SHFT        63
+#define UV2H_GR1_TLB_MMR_READ_DATA_LO_VPN_MASK          0x0000007fffffffffUL
+#define UV2H_GR1_TLB_MMR_READ_DATA_LO_ASID_MASK         0x7fffff8000000000UL
+#define UV2H_GR1_TLB_MMR_READ_DATA_LO_VALID_MASK        0x8000000000000000UL
+
+#define UV3H_GR1_TLB_MMR_READ_DATA_LO_VPN_SHFT          0
+#define UV3H_GR1_TLB_MMR_READ_DATA_LO_ASID_SHFT         39
+#define UV3H_GR1_TLB_MMR_READ_DATA_LO_VALID_SHFT        63
+#define UV3H_GR1_TLB_MMR_READ_DATA_LO_VPN_MASK          0x0000007fffffffffUL
+#define UV3H_GR1_TLB_MMR_READ_DATA_LO_ASID_MASK         0x7fffff8000000000UL
+#define UV3H_GR1_TLB_MMR_READ_DATA_LO_VALID_MASK        0x8000000000000000UL
+
 union uvh_gr1_tlb_mmr_read_data_lo_u {
         unsigned long   v;
         struct uvh_gr1_tlb_mmr_read_data_lo_s {
@@ -986,12 +1322,32 @@ union uvh_gr1_tlb_mmr_read_data_lo_u {
                 unsigned long   asid:24;                        /* RO */
                 unsigned long   valid:1;                        /* RO */
         } s;
+        struct uv1h_gr1_tlb_mmr_read_data_lo_s {
+                unsigned long   vpn:39;                         /* RO */
+                unsigned long   asid:24;                        /* RO */
+                unsigned long   valid:1;                        /* RO */
+        } s1;
+        struct uvxh_gr1_tlb_mmr_read_data_lo_s {
+                unsigned long   vpn:39;                         /* RO */
+                unsigned long   asid:24;                        /* RO */
+                unsigned long   valid:1;                        /* RO */
+        } sx;
+        struct uv2h_gr1_tlb_mmr_read_data_lo_s {
+                unsigned long   vpn:39;                         /* RO */
+                unsigned long   asid:24;                        /* RO */
+                unsigned long   valid:1;                        /* RO */
+        } s2;
+        struct uv3h_gr1_tlb_mmr_read_data_lo_s {
+                unsigned long   vpn:39;                         /* RO */
+                unsigned long   asid:24;                        /* RO */
+                unsigned long   valid:1;                        /* RO */
+        } s3;
 };
 
 /* ========================================================================= */
 /*                               UVH_INT_CMPB                                */
 /* ========================================================================= */
-#define UVH_INT_CMPB                                    0x22080UL
+#define UVH_INT_CMPB 0x22080UL
 
 #define UVH_INT_CMPB_REAL_TIME_CMPB_SHFT                0
 #define UVH_INT_CMPB_REAL_TIME_CMPB_MASK                0x00ffffffffffffffUL
@@ -1007,10 +1363,13 @@ union uvh_int_cmpb_u {
 /* ========================================================================= */
 /*                               UVH_INT_CMPC                                */
 /* ========================================================================= */
-#define UVH_INT_CMPC                                    0x22100UL
+#define UVH_INT_CMPC 0x22100UL
+
+#define UV1H_INT_CMPC_REAL_TIME_CMPC_SHFT               0
+#define UV1H_INT_CMPC_REAL_TIME_CMPC_MASK               0x00ffffffffffffffUL
 
-#define UVH_INT_CMPC_REAL_TIME_CMPC_SHFT                0
-#define UVH_INT_CMPC_REAL_TIME_CMPC_MASK                0xffffffffffffffUL
+#define UVXH_INT_CMPC_REAL_TIME_CMP_2_SHFT              0
+#define UVXH_INT_CMPC_REAL_TIME_CMP_2_MASK              0x00ffffffffffffffUL
 
 union uvh_int_cmpc_u {
         unsigned long   v;
@@ -1023,10 +1382,13 @@ union uvh_int_cmpc_u {
 /* ========================================================================= */
 /*                               UVH_INT_CMPD                                */
 /* ========================================================================= */
-#define UVH_INT_CMPD                                    0x22180UL
+#define UVH_INT_CMPD 0x22180UL
 
-#define UVH_INT_CMPD_REAL_TIME_CMPD_SHFT                0
-#define UVH_INT_CMPD_REAL_TIME_CMPD_MASK                0xffffffffffffffUL
+#define UV1H_INT_CMPD_REAL_TIME_CMPD_SHFT               0
+#define UV1H_INT_CMPD_REAL_TIME_CMPD_MASK               0x00ffffffffffffffUL
+
+#define UVXH_INT_CMPD_REAL_TIME_CMP_3_SHFT              0
+#define UVXH_INT_CMPD_REAL_TIME_CMP_3_MASK              0x00ffffffffffffffUL
 
 union uvh_int_cmpd_u {
         unsigned long   v;
@@ -1039,8 +1401,8 @@ union uvh_int_cmpd_u {
 /* ========================================================================= */
 /*                               UVH_IPI_INT                                 */
 /* ========================================================================= */
-#define UVH_IPI_INT                                     0x60500UL
-#define UVH_IPI_INT_32                                  0x348
+#define UVH_IPI_INT 0x60500UL
+#define UVH_IPI_INT_32 0x348
 
 #define UVH_IPI_INT_VECTOR_SHFT                         0
 #define UVH_IPI_INT_DELIVERY_MODE_SHFT                  8
@@ -1069,8 +1431,8 @@ union uvh_ipi_int_u {
 /* ========================================================================= */
 /*                   UVH_LB_BAU_INTD_PAYLOAD_QUEUE_FIRST                     */
 /* ========================================================================= */
-#define UVH_LB_BAU_INTD_PAYLOAD_QUEUE_FIRST             0x320050UL
-#define UVH_LB_BAU_INTD_PAYLOAD_QUEUE_FIRST_32          0x9c0
+#define UVH_LB_BAU_INTD_PAYLOAD_QUEUE_FIRST 0x320050UL
+#define UVH_LB_BAU_INTD_PAYLOAD_QUEUE_FIRST_32 0x9c0
 
 #define UVH_LB_BAU_INTD_PAYLOAD_QUEUE_FIRST_ADDRESS_SHFT 4
 #define UVH_LB_BAU_INTD_PAYLOAD_QUEUE_FIRST_NODE_ID_SHFT 49
@@ -1091,8 +1453,8 @@ union uvh_lb_bau_intd_payload_queue_first_u {
 /* ========================================================================= */
 /*                    UVH_LB_BAU_INTD_PAYLOAD_QUEUE_LAST                     */
 /* ========================================================================= */
-#define UVH_LB_BAU_INTD_PAYLOAD_QUEUE_LAST              0x320060UL
-#define UVH_LB_BAU_INTD_PAYLOAD_QUEUE_LAST_32           0x9c8
+#define UVH_LB_BAU_INTD_PAYLOAD_QUEUE_LAST 0x320060UL
+#define UVH_LB_BAU_INTD_PAYLOAD_QUEUE_LAST_32 0x9c8
 
 #define UVH_LB_BAU_INTD_PAYLOAD_QUEUE_LAST_ADDRESS_SHFT 4
 #define UVH_LB_BAU_INTD_PAYLOAD_QUEUE_LAST_ADDRESS_MASK 0x000007fffffffff0UL
@@ -1109,8 +1471,8 @@ union uvh_lb_bau_intd_payload_queue_last_u {
 /* ========================================================================= */
 /*                    UVH_LB_BAU_INTD_PAYLOAD_QUEUE_TAIL                     */
 /* ========================================================================= */
-#define UVH_LB_BAU_INTD_PAYLOAD_QUEUE_TAIL              0x320070UL
-#define UVH_LB_BAU_INTD_PAYLOAD_QUEUE_TAIL_32           0x9d0
+#define UVH_LB_BAU_INTD_PAYLOAD_QUEUE_TAIL 0x320070UL
+#define UVH_LB_BAU_INTD_PAYLOAD_QUEUE_TAIL_32 0x9d0
 
 #define UVH_LB_BAU_INTD_PAYLOAD_QUEUE_TAIL_ADDRESS_SHFT 4
 #define UVH_LB_BAU_INTD_PAYLOAD_QUEUE_TAIL_ADDRESS_MASK 0x000007fffffffff0UL
@@ -1127,8 +1489,8 @@ union uvh_lb_bau_intd_payload_queue_tail_u {
 /* ========================================================================= */
 /*                   UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE                    */
 /* ========================================================================= */
-#define UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE            0x320080UL
-#define UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_32         0xa68
+#define UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE 0x320080UL
+#define UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_32 0xa68
 
 #define UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_PENDING_0_SHFT 0
 #define UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_PENDING_1_SHFT 1
@@ -1189,14 +1551,21 @@ union uvh_lb_bau_intd_software_acknowledge_u {
 /* ========================================================================= */
 /*                UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_ALIAS                 */
 /* ========================================================================= */
-#define UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_ALIAS      0x0000000000320088UL
-#define UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_ALIAS_32   0xa70
+#define UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_ALIAS 0x320088UL
+#define UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_ALIAS_32 0xa70
+
 
 /* ========================================================================= */
 /*                         UVH_LB_BAU_MISC_CONTROL                           */
 /* ========================================================================= */
-#define UVH_LB_BAU_MISC_CONTROL                         0x320170UL
-#define UVH_LB_BAU_MISC_CONTROL_32                      0xa10
+#define UVH_LB_BAU_MISC_CONTROL 0x320170UL
+#define UV1H_LB_BAU_MISC_CONTROL 0x320170UL
+#define UV2H_LB_BAU_MISC_CONTROL 0x320170UL
+#define UV3H_LB_BAU_MISC_CONTROL 0x320170UL
+#define UVH_LB_BAU_MISC_CONTROL_32 0xa10
+#define UV1H_LB_BAU_MISC_CONTROL_32 0x320170UL
+#define UV2H_LB_BAU_MISC_CONTROL_32 0x320170UL
+#define UV3H_LB_BAU_MISC_CONTROL_32 0x320170UL
 
 #define UVH_LB_BAU_MISC_CONTROL_REJECTION_DELAY_SHFT    0
 #define UVH_LB_BAU_MISC_CONTROL_APIC_MODE_SHFT          8
@@ -1213,6 +1582,7 @@ union uvh_lb_bau_intd_software_acknowledge_u {
 #define UVH_LB_BAU_MISC_CONTROL_PROGRAMMED_INITIAL_PRIORITY_SHFT 24
 #define UVH_LB_BAU_MISC_CONTROL_USE_INCOMING_PRIORITY_SHFT 27
 #define UVH_LB_BAU_MISC_CONTROL_ENABLE_PROGRAMMED_INITIAL_PRIORITY_SHFT 28
+#define UVH_LB_BAU_MISC_CONTROL_FUN_SHFT                48
 #define UVH_LB_BAU_MISC_CONTROL_REJECTION_DELAY_MASK    0x00000000000000ffUL
 #define UVH_LB_BAU_MISC_CONTROL_APIC_MODE_MASK          0x0000000000000100UL
 #define UVH_LB_BAU_MISC_CONTROL_FORCE_BROADCAST_MASK    0x0000000000000200UL
@@ -1228,6 +1598,7 @@ union uvh_lb_bau_intd_software_acknowledge_u {
 #define UVH_LB_BAU_MISC_CONTROL_PROGRAMMED_INITIAL_PRIORITY_MASK 0x0000000007000000UL
 #define UVH_LB_BAU_MISC_CONTROL_USE_INCOMING_PRIORITY_MASK 0x0000000008000000UL
 #define UVH_LB_BAU_MISC_CONTROL_ENABLE_PROGRAMMED_INITIAL_PRIORITY_MASK 0x0000000010000000UL
+#define UVH_LB_BAU_MISC_CONTROL_FUN_MASK                0xffff000000000000UL
 
 #define UV1H_LB_BAU_MISC_CONTROL_REJECTION_DELAY_SHFT   0
 #define UV1H_LB_BAU_MISC_CONTROL_APIC_MODE_SHFT         8
@@ -1262,6 +1633,53 @@ union uvh_lb_bau_intd_software_acknowledge_u {
 #define UV1H_LB_BAU_MISC_CONTROL_ENABLE_PROGRAMMED_INITIAL_PRIORITY_MASK 0x0000000010000000UL
 #define UV1H_LB_BAU_MISC_CONTROL_FUN_MASK               0xffff000000000000UL
 
+#define UVXH_LB_BAU_MISC_CONTROL_REJECTION_DELAY_SHFT   0
+#define UVXH_LB_BAU_MISC_CONTROL_APIC_MODE_SHFT         8
+#define UVXH_LB_BAU_MISC_CONTROL_FORCE_BROADCAST_SHFT   9
+#define UVXH_LB_BAU_MISC_CONTROL_FORCE_LOCK_NOP_SHFT    10
+#define UVXH_LB_BAU_MISC_CONTROL_QPI_AGENT_PRESENCE_VECTOR_SHFT 11
+#define UVXH_LB_BAU_MISC_CONTROL_DESCRIPTOR_FETCH_MODE_SHFT 14
+#define UVXH_LB_BAU_MISC_CONTROL_ENABLE_INTD_SOFT_ACK_MODE_SHFT 15
+#define UVXH_LB_BAU_MISC_CONTROL_INTD_SOFT_ACK_TIMEOUT_PERIOD_SHFT 16
+#define UVXH_LB_BAU_MISC_CONTROL_ENABLE_DUAL_MAPPING_MODE_SHFT 20
+#define UVXH_LB_BAU_MISC_CONTROL_VGA_IO_PORT_DECODE_ENABLE_SHFT 21
+#define UVXH_LB_BAU_MISC_CONTROL_VGA_IO_PORT_16_BIT_DECODE_SHFT 22
+#define UVXH_LB_BAU_MISC_CONTROL_SUPPRESS_DEST_REGISTRATION_SHFT 23
+#define UVXH_LB_BAU_MISC_CONTROL_PROGRAMMED_INITIAL_PRIORITY_SHFT 24
+#define UVXH_LB_BAU_MISC_CONTROL_USE_INCOMING_PRIORITY_SHFT 27
+#define UVXH_LB_BAU_MISC_CONTROL_ENABLE_PROGRAMMED_INITIAL_PRIORITY_SHFT 28
+#define UVXH_LB_BAU_MISC_CONTROL_ENABLE_AUTOMATIC_APIC_MODE_SELECTION_SHFT 29
+#define UVXH_LB_BAU_MISC_CONTROL_APIC_MODE_STATUS_SHFT  30
+#define UVXH_LB_BAU_MISC_CONTROL_SUPPRESS_INTERRUPTS_TO_SELF_SHFT 31
+#define UVXH_LB_BAU_MISC_CONTROL_ENABLE_LOCK_BASED_SYSTEM_FLUSH_SHFT 32
+#define UVXH_LB_BAU_MISC_CONTROL_ENABLE_EXTENDED_SB_STATUS_SHFT 33
+#define UVXH_LB_BAU_MISC_CONTROL_SUPPRESS_INT_PRIO_UDT_TO_SELF_SHFT 34
+#define UVXH_LB_BAU_MISC_CONTROL_USE_LEGACY_DESCRIPTOR_FORMATS_SHFT 35
+#define UVXH_LB_BAU_MISC_CONTROL_FUN_SHFT               48
+#define UVXH_LB_BAU_MISC_CONTROL_REJECTION_DELAY_MASK   0x00000000000000ffUL
+#define UVXH_LB_BAU_MISC_CONTROL_APIC_MODE_MASK         0x0000000000000100UL
+#define UVXH_LB_BAU_MISC_CONTROL_FORCE_BROADCAST_MASK   0x0000000000000200UL
+#define UVXH_LB_BAU_MISC_CONTROL_FORCE_LOCK_NOP_MASK    0x0000000000000400UL
+#define UVXH_LB_BAU_MISC_CONTROL_QPI_AGENT_PRESENCE_VECTOR_MASK 0x0000000000003800UL
+#define UVXH_LB_BAU_MISC_CONTROL_DESCRIPTOR_FETCH_MODE_MASK 0x0000000000004000UL
+#define UVXH_LB_BAU_MISC_CONTROL_ENABLE_INTD_SOFT_ACK_MODE_MASK 0x0000000000008000UL
+#define UVXH_LB_BAU_MISC_CONTROL_INTD_SOFT_ACK_TIMEOUT_PERIOD_MASK 0x00000000000f0000UL
+#define UVXH_LB_BAU_MISC_CONTROL_ENABLE_DUAL_MAPPING_MODE_MASK 0x0000000000100000UL
+#define UVXH_LB_BAU_MISC_CONTROL_VGA_IO_PORT_DECODE_ENABLE_MASK 0x0000000000200000UL
+#define UVXH_LB_BAU_MISC_CONTROL_VGA_IO_PORT_16_BIT_DECODE_MASK 0x0000000000400000UL
+#define UVXH_LB_BAU_MISC_CONTROL_SUPPRESS_DEST_REGISTRATION_MASK 0x0000000000800000UL
+#define UVXH_LB_BAU_MISC_CONTROL_PROGRAMMED_INITIAL_PRIORITY_MASK 0x0000000007000000UL
+#define UVXH_LB_BAU_MISC_CONTROL_USE_INCOMING_PRIORITY_MASK 0x0000000008000000UL
+#define UVXH_LB_BAU_MISC_CONTROL_ENABLE_PROGRAMMED_INITIAL_PRIORITY_MASK 0x0000000010000000UL
+#define UVXH_LB_BAU_MISC_CONTROL_ENABLE_AUTOMATIC_APIC_MODE_SELECTION_MASK 0x0000000020000000UL
+#define UVXH_LB_BAU_MISC_CONTROL_APIC_MODE_STATUS_MASK  0x0000000040000000UL
+#define UVXH_LB_BAU_MISC_CONTROL_SUPPRESS_INTERRUPTS_TO_SELF_MASK 0x0000000080000000UL
+#define UVXH_LB_BAU_MISC_CONTROL_ENABLE_LOCK_BASED_SYSTEM_FLUSH_MASK 0x0000000100000000UL
+#define UVXH_LB_BAU_MISC_CONTROL_ENABLE_EXTENDED_SB_STATUS_MASK 0x0000000200000000UL
+#define UVXH_LB_BAU_MISC_CONTROL_SUPPRESS_INT_PRIO_UDT_TO_SELF_MASK 0x0000000400000000UL
+#define UVXH_LB_BAU_MISC_CONTROL_USE_LEGACY_DESCRIPTOR_FORMATS_MASK 0x0000000800000000UL
+#define UVXH_LB_BAU_MISC_CONTROL_FUN_MASK               0xffff000000000000UL
+
 #define UV2H_LB_BAU_MISC_CONTROL_REJECTION_DELAY_SHFT   0
 #define UV2H_LB_BAU_MISC_CONTROL_APIC_MODE_SHFT         8
 #define UV2H_LB_BAU_MISC_CONTROL_FORCE_BROADCAST_SHFT   9
@@ -1309,6 +1727,59 @@ union uvh_lb_bau_intd_software_acknowledge_u {
 #define UV2H_LB_BAU_MISC_CONTROL_USE_LEGACY_DESCRIPTOR_FORMATS_MASK 0x0000000800000000UL
 #define UV2H_LB_BAU_MISC_CONTROL_FUN_MASK               0xffff000000000000UL
 
+#define UV3H_LB_BAU_MISC_CONTROL_REJECTION_DELAY_SHFT   0
+#define UV3H_LB_BAU_MISC_CONTROL_APIC_MODE_SHFT         8
+#define UV3H_LB_BAU_MISC_CONTROL_FORCE_BROADCAST_SHFT   9
+#define UV3H_LB_BAU_MISC_CONTROL_FORCE_LOCK_NOP_SHFT    10
+#define UV3H_LB_BAU_MISC_CONTROL_QPI_AGENT_PRESENCE_VECTOR_SHFT 11
+#define UV3H_LB_BAU_MISC_CONTROL_DESCRIPTOR_FETCH_MODE_SHFT 14
+#define UV3H_LB_BAU_MISC_CONTROL_ENABLE_INTD_SOFT_ACK_MODE_SHFT 15
+#define UV3H_LB_BAU_MISC_CONTROL_INTD_SOFT_ACK_TIMEOUT_PERIOD_SHFT 16
+#define UV3H_LB_BAU_MISC_CONTROL_ENABLE_DUAL_MAPPING_MODE_SHFT 20
+#define UV3H_LB_BAU_MISC_CONTROL_VGA_IO_PORT_DECODE_ENABLE_SHFT 21
+#define UV3H_LB_BAU_MISC_CONTROL_VGA_IO_PORT_16_BIT_DECODE_SHFT 22
+#define UV3H_LB_BAU_MISC_CONTROL_SUPPRESS_DEST_REGISTRATION_SHFT 23
+#define UV3H_LB_BAU_MISC_CONTROL_PROGRAMMED_INITIAL_PRIORITY_SHFT 24
+#define UV3H_LB_BAU_MISC_CONTROL_USE_INCOMING_PRIORITY_SHFT 27
+#define UV3H_LB_BAU_MISC_CONTROL_ENABLE_PROGRAMMED_INITIAL_PRIORITY_SHFT 28
+#define UV3H_LB_BAU_MISC_CONTROL_ENABLE_AUTOMATIC_APIC_MODE_SELECTION_SHFT 29
+#define UV3H_LB_BAU_MISC_CONTROL_APIC_MODE_STATUS_SHFT  30
+#define UV3H_LB_BAU_MISC_CONTROL_SUPPRESS_INTERRUPTS_TO_SELF_SHFT 31
+#define UV3H_LB_BAU_MISC_CONTROL_ENABLE_LOCK_BASED_SYSTEM_FLUSH_SHFT 32
+#define UV3H_LB_BAU_MISC_CONTROL_ENABLE_EXTENDED_SB_STATUS_SHFT 33
+#define UV3H_LB_BAU_MISC_CONTROL_SUPPRESS_INT_PRIO_UDT_TO_SELF_SHFT 34
+#define UV3H_LB_BAU_MISC_CONTROL_USE_LEGACY_DESCRIPTOR_FORMATS_SHFT 35
+#define UV3H_LB_BAU_MISC_CONTROL_SUPPRESS_QUIESCE_MSGS_TO_QPI_SHFT 36
+#define UV3H_LB_BAU_MISC_CONTROL_ENABLE_INTD_PREFETCH_HINT_SHFT 37
+#define UV3H_LB_BAU_MISC_CONTROL_THREAD_KILL_TIMEBASE_SHFT 38
+#define UV3H_LB_BAU_MISC_CONTROL_FUN_SHFT               48
+#define UV3H_LB_BAU_MISC_CONTROL_REJECTION_DELAY_MASK   0x00000000000000ffUL
+#define UV3H_LB_BAU_MISC_CONTROL_APIC_MODE_MASK         0x0000000000000100UL
+#define UV3H_LB_BAU_MISC_CONTROL_FORCE_BROADCAST_MASK   0x0000000000000200UL
+#define UV3H_LB_BAU_MISC_CONTROL_FORCE_LOCK_NOP_MASK    0x0000000000000400UL
+#define UV3H_LB_BAU_MISC_CONTROL_QPI_AGENT_PRESENCE_VECTOR_MASK 0x0000000000003800UL
+#define UV3H_LB_BAU_MISC_CONTROL_DESCRIPTOR_FETCH_MODE_MASK 0x0000000000004000UL
+#define UV3H_LB_BAU_MISC_CONTROL_ENABLE_INTD_SOFT_ACK_MODE_MASK 0x0000000000008000UL
+#define UV3H_LB_BAU_MISC_CONTROL_INTD_SOFT_ACK_TIMEOUT_PERIOD_MASK 0x00000000000f0000UL
+#define UV3H_LB_BAU_MISC_CONTROL_ENABLE_DUAL_MAPPING_MODE_MASK 0x0000000000100000UL
+#define UV3H_LB_BAU_MISC_CONTROL_VGA_IO_PORT_DECODE_ENABLE_MASK 0x0000000000200000UL
+#define UV3H_LB_BAU_MISC_CONTROL_VGA_IO_PORT_16_BIT_DECODE_MASK 0x0000000000400000UL
+#define UV3H_LB_BAU_MISC_CONTROL_SUPPRESS_DEST_REGISTRATION_MASK 0x0000000000800000UL
+#define UV3H_LB_BAU_MISC_CONTROL_PROGRAMMED_INITIAL_PRIORITY_MASK 0x0000000007000000UL
+#define UV3H_LB_BAU_MISC_CONTROL_USE_INCOMING_PRIORITY_MASK 0x0000000008000000UL
+#define UV3H_LB_BAU_MISC_CONTROL_ENABLE_PROGRAMMED_INITIAL_PRIORITY_MASK 0x0000000010000000UL
+#define UV3H_LB_BAU_MISC_CONTROL_ENABLE_AUTOMATIC_APIC_MODE_SELECTION_MASK 0x0000000020000000UL
+#define UV3H_LB_BAU_MISC_CONTROL_APIC_MODE_STATUS_MASK  0x0000000040000000UL
+#define UV3H_LB_BAU_MISC_CONTROL_SUPPRESS_INTERRUPTS_TO_SELF_MASK 0x0000000080000000UL
+#define UV3H_LB_BAU_MISC_CONTROL_ENABLE_LOCK_BASED_SYSTEM_FLUSH_MASK 0x0000000100000000UL
+#define UV3H_LB_BAU_MISC_CONTROL_ENABLE_EXTENDED_SB_STATUS_MASK 0x0000000200000000UL
+#define UV3H_LB_BAU_MISC_CONTROL_SUPPRESS_INT_PRIO_UDT_TO_SELF_MASK 0x0000000400000000UL
+#define UV3H_LB_BAU_MISC_CONTROL_USE_LEGACY_DESCRIPTOR_FORMATS_MASK 0x0000000800000000UL
+#define UV3H_LB_BAU_MISC_CONTROL_SUPPRESS_QUIESCE_MSGS_TO_QPI_MASK 0x0000001000000000UL
+#define UV3H_LB_BAU_MISC_CONTROL_ENABLE_INTD_PREFETCH_HINT_MASK 0x0000002000000000UL
+#define UV3H_LB_BAU_MISC_CONTROL_THREAD_KILL_TIMEBASE_MASK 0x00003fc000000000UL
+#define UV3H_LB_BAU_MISC_CONTROL_FUN_MASK               0xffff000000000000UL
+
 union uvh_lb_bau_misc_control_u {
         unsigned long   v;
         struct uvh_lb_bau_misc_control_s {
@@ -1327,7 +1798,8 @@ union uvh_lb_bau_misc_control_u {
                 unsigned long   programmed_initial_priority:3;  /* RW */
                 unsigned long   use_incoming_priority:1;        /* RW */
                 unsigned long   enable_programmed_initial_priority:1;/* RW */
-                unsigned long   rsvd_29_63:35;
+                unsigned long   rsvd_29_47:19;
+                unsigned long   fun:16;                         /* RW */
         } s;
         struct uv1h_lb_bau_misc_control_s {
                 unsigned long   rejection_delay:8;              /* RW */
@@ -1348,6 +1820,32 @@ union uvh_lb_bau_misc_control_u {
                 unsigned long   rsvd_29_47:19;
                 unsigned long   fun:16;                         /* RW */
         } s1;
+        struct uvxh_lb_bau_misc_control_s {
+                unsigned long   rejection_delay:8;              /* RW */
+                unsigned long   apic_mode:1;                    /* RW */
+                unsigned long   force_broadcast:1;              /* RW */
+                unsigned long   force_lock_nop:1;               /* RW */
+                unsigned long   qpi_agent_presence_vector:3;    /* RW */
+                unsigned long   descriptor_fetch_mode:1;        /* RW */
+                unsigned long   enable_intd_soft_ack_mode:1;    /* RW */
+                unsigned long   intd_soft_ack_timeout_period:4; /* RW */
+                unsigned long   enable_dual_mapping_mode:1;     /* RW */
+                unsigned long   vga_io_port_decode_enable:1;    /* RW */
+                unsigned long   vga_io_port_16_bit_decode:1;    /* RW */
+                unsigned long   suppress_dest_registration:1;   /* RW */
+                unsigned long   programmed_initial_priority:3;  /* RW */
+                unsigned long   use_incoming_priority:1;        /* RW */
+                unsigned long   enable_programmed_initial_priority:1;/* RW */
+                unsigned long   enable_automatic_apic_mode_selection:1;/* RW */
+                unsigned long   apic_mode_status:1;             /* RO */
+                unsigned long   suppress_interrupts_to_self:1;  /* RW */
+                unsigned long   enable_lock_based_system_flush:1;/* RW */
+                unsigned long   enable_extended_sb_status:1;    /* RW */
+                unsigned long   suppress_int_prio_udt_to_self:1;/* RW */
+                unsigned long   use_legacy_descriptor_formats:1;/* RW */
+                unsigned long   rsvd_36_47:12;
+                unsigned long   fun:16;                         /* RW */
+        } sx;
         struct uv2h_lb_bau_misc_control_s {
                 unsigned long   rejection_delay:8;              /* RW */
                 unsigned long   apic_mode:1;                    /* RW */
@@ -1374,13 +1872,42 @@ union uvh_lb_bau_misc_control_u {
                 unsigned long   rsvd_36_47:12;
                 unsigned long   fun:16;                         /* RW */
         } s2;
+        struct uv3h_lb_bau_misc_control_s {
+                unsigned long   rejection_delay:8;              /* RW */
+                unsigned long   apic_mode:1;                    /* RW */
+                unsigned long   force_broadcast:1;              /* RW */
+                unsigned long   force_lock_nop:1;               /* RW */
+                unsigned long   qpi_agent_presence_vector:3;    /* RW */
+                unsigned long   descriptor_fetch_mode:1;        /* RW */
+                unsigned long   enable_intd_soft_ack_mode:1;    /* RW */
+                unsigned long   intd_soft_ack_timeout_period:4; /* RW */
+                unsigned long   enable_dual_mapping_mode:1;     /* RW */
+                unsigned long   vga_io_port_decode_enable:1;    /* RW */
+                unsigned long   vga_io_port_16_bit_decode:1;    /* RW */
+                unsigned long   suppress_dest_registration:1;   /* RW */
+                unsigned long   programmed_initial_priority:3;  /* RW */
+                unsigned long   use_incoming_priority:1;        /* RW */
+                unsigned long   enable_programmed_initial_priority:1;/* RW */
+                unsigned long   enable_automatic_apic_mode_selection:1;/* RW */
+                unsigned long   apic_mode_status:1;             /* RO */
+                unsigned long   suppress_interrupts_to_self:1;  /* RW */
+                unsigned long   enable_lock_based_system_flush:1;/* RW */
+                unsigned long   enable_extended_sb_status:1;    /* RW */
+                unsigned long   suppress_int_prio_udt_to_self:1;/* RW */
+                unsigned long   use_legacy_descriptor_formats:1;/* RW */
+                unsigned long   suppress_quiesce_msgs_to_qpi:1; /* RW */
+                unsigned long   enable_intd_prefetch_hint:1;    /* RW */
+                unsigned long   thread_kill_timebase:8;         /* RW */
+                unsigned long   rsvd_46_47:2;
+                unsigned long   fun:16;                         /* RW */
+        } s3;
 };
 
 /* ========================================================================= */
 /*                     UVH_LB_BAU_SB_ACTIVATION_CONTROL                      */
 /* ========================================================================= */
-#define UVH_LB_BAU_SB_ACTIVATION_CONTROL                0x320020UL
-#define UVH_LB_BAU_SB_ACTIVATION_CONTROL_32             0x9a8
+#define UVH_LB_BAU_SB_ACTIVATION_CONTROL 0x320020UL
+#define UVH_LB_BAU_SB_ACTIVATION_CONTROL_32 0x9a8
 
 #define UVH_LB_BAU_SB_ACTIVATION_CONTROL_INDEX_SHFT     0
 #define UVH_LB_BAU_SB_ACTIVATION_CONTROL_PUSH_SHFT      62
@@ -1402,8 +1929,8 @@ union uvh_lb_bau_sb_activation_control_u {
 /* ========================================================================= */
 /*                    UVH_LB_BAU_SB_ACTIVATION_STATUS_0                      */
 /* ========================================================================= */
-#define UVH_LB_BAU_SB_ACTIVATION_STATUS_0               0x320030UL
-#define UVH_LB_BAU_SB_ACTIVATION_STATUS_0_32            0x9b0
+#define UVH_LB_BAU_SB_ACTIVATION_STATUS_0 0x320030UL
+#define UVH_LB_BAU_SB_ACTIVATION_STATUS_0_32 0x9b0
 
 #define UVH_LB_BAU_SB_ACTIVATION_STATUS_0_STATUS_SHFT   0
 #define UVH_LB_BAU_SB_ACTIVATION_STATUS_0_STATUS_MASK   0xffffffffffffffffUL
@@ -1418,8 +1945,8 @@ union uvh_lb_bau_sb_activation_status_0_u {
 /* ========================================================================= */
 /*                    UVH_LB_BAU_SB_ACTIVATION_STATUS_1                      */
 /* ========================================================================= */
-#define UVH_LB_BAU_SB_ACTIVATION_STATUS_1               0x320040UL
-#define UVH_LB_BAU_SB_ACTIVATION_STATUS_1_32            0x9b8
+#define UVH_LB_BAU_SB_ACTIVATION_STATUS_1 0x320040UL
+#define UVH_LB_BAU_SB_ACTIVATION_STATUS_1_32 0x9b8
 
 #define UVH_LB_BAU_SB_ACTIVATION_STATUS_1_STATUS_SHFT   0
 #define UVH_LB_BAU_SB_ACTIVATION_STATUS_1_STATUS_MASK   0xffffffffffffffffUL
@@ -1434,8 +1961,8 @@ union uvh_lb_bau_sb_activation_status_1_u {
 /* ========================================================================= */
 /*                      UVH_LB_BAU_SB_DESCRIPTOR_BASE                        */
 /* ========================================================================= */
-#define UVH_LB_BAU_SB_DESCRIPTOR_BASE                   0x320010UL
-#define UVH_LB_BAU_SB_DESCRIPTOR_BASE_32                0x9a0
+#define UVH_LB_BAU_SB_DESCRIPTOR_BASE 0x320010UL
+#define UVH_LB_BAU_SB_DESCRIPTOR_BASE_32 0x9a0
 
 #define UVH_LB_BAU_SB_DESCRIPTOR_BASE_PAGE_ADDRESS_SHFT 12
 #define UVH_LB_BAU_SB_DESCRIPTOR_BASE_NODE_ID_SHFT      49
@@ -1456,7 +1983,10 @@ union uvh_lb_bau_sb_descriptor_base_u {
 /* ========================================================================= */
 /*                               UVH_NODE_ID                                 */
 /* ========================================================================= */
-#define UVH_NODE_ID                                     0x0UL
+#define UVH_NODE_ID 0x0UL
+#define UV1H_NODE_ID 0x0UL
+#define UV2H_NODE_ID 0x0UL
+#define UV3H_NODE_ID 0x0UL
 
 #define UVH_NODE_ID_FORCE1_SHFT                         0
 #define UVH_NODE_ID_MANUFACTURER_SHFT                   1
@@ -1484,6 +2014,21 @@ union uvh_lb_bau_sb_descriptor_base_u {
 #define UV1H_NODE_ID_NODES_PER_BIT_MASK                 0x007f000000000000UL
 #define UV1H_NODE_ID_NI_PORT_MASK                       0x0f00000000000000UL
 
+#define UVXH_NODE_ID_FORCE1_SHFT                        0
+#define UVXH_NODE_ID_MANUFACTURER_SHFT                  1
+#define UVXH_NODE_ID_PART_NUMBER_SHFT                   12
+#define UVXH_NODE_ID_REVISION_SHFT                      28
+#define UVXH_NODE_ID_NODE_ID_SHFT                       32
+#define UVXH_NODE_ID_NODES_PER_BIT_SHFT                 50
+#define UVXH_NODE_ID_NI_PORT_SHFT                       57
+#define UVXH_NODE_ID_FORCE1_MASK                        0x0000000000000001UL
+#define UVXH_NODE_ID_MANUFACTURER_MASK                  0x0000000000000ffeUL
+#define UVXH_NODE_ID_PART_NUMBER_MASK                   0x000000000ffff000UL
+#define UVXH_NODE_ID_REVISION_MASK                      0x00000000f0000000UL
+#define UVXH_NODE_ID_NODE_ID_MASK                       0x00007fff00000000UL
+#define UVXH_NODE_ID_NODES_PER_BIT_MASK                 0x01fc000000000000UL
+#define UVXH_NODE_ID_NI_PORT_MASK                       0x3e00000000000000UL
+
 #define UV2H_NODE_ID_FORCE1_SHFT                        0
 #define UV2H_NODE_ID_MANUFACTURER_SHFT                  1
 #define UV2H_NODE_ID_PART_NUMBER_SHFT                   12
@@ -1499,6 +2044,25 @@ union uvh_lb_bau_sb_descriptor_base_u {
 #define UV2H_NODE_ID_NODES_PER_BIT_MASK                 0x01fc000000000000UL
 #define UV2H_NODE_ID_NI_PORT_MASK                       0x3e00000000000000UL
 
+#define UV3H_NODE_ID_FORCE1_SHFT                        0
+#define UV3H_NODE_ID_MANUFACTURER_SHFT                  1
+#define UV3H_NODE_ID_PART_NUMBER_SHFT                   12
+#define UV3H_NODE_ID_REVISION_SHFT                      28
+#define UV3H_NODE_ID_NODE_ID_SHFT                       32
+#define UV3H_NODE_ID_ROUTER_SELECT_SHFT                 48
+#define UV3H_NODE_ID_RESERVED_2_SHFT                    49
+#define UV3H_NODE_ID_NODES_PER_BIT_SHFT                 50
+#define UV3H_NODE_ID_NI_PORT_SHFT                       57
+#define UV3H_NODE_ID_FORCE1_MASK                        0x0000000000000001UL
+#define UV3H_NODE_ID_MANUFACTURER_MASK                  0x0000000000000ffeUL
+#define UV3H_NODE_ID_PART_NUMBER_MASK                   0x000000000ffff000UL
+#define UV3H_NODE_ID_REVISION_MASK                      0x00000000f0000000UL
+#define UV3H_NODE_ID_NODE_ID_MASK                       0x00007fff00000000UL
+#define UV3H_NODE_ID_ROUTER_SELECT_MASK                 0x0001000000000000UL
+#define UV3H_NODE_ID_RESERVED_2_MASK                    0x0002000000000000UL
+#define UV3H_NODE_ID_NODES_PER_BIT_MASK                 0x01fc000000000000UL
+#define UV3H_NODE_ID_NI_PORT_MASK                       0x3e00000000000000UL
+
 union uvh_node_id_u {
         unsigned long   v;
         struct uvh_node_id_s {
@@ -1521,6 +2085,17 @@ union uvh_node_id_u {
                 unsigned long   ni_port:4;                      /* RO */
                 unsigned long   rsvd_60_63:4;
         } s1;
+        struct uvxh_node_id_s {
+                unsigned long   force1:1;                       /* RO */
+                unsigned long   manufacturer:11;                /* RO */
+                unsigned long   part_number:16;                 /* RO */
+                unsigned long   revision:4;                     /* RO */
+                unsigned long   node_id:15;                     /* RW */
+                unsigned long   rsvd_47_49:3;
+                unsigned long   nodes_per_bit:7;                /* RO */
+                unsigned long   ni_port:5;                      /* RO */
+                unsigned long   rsvd_62_63:2;
+        } sx;
         struct uv2h_node_id_s {
                 unsigned long   force1:1;                       /* RO */
                 unsigned long   manufacturer:11;                /* RO */
@@ -1532,13 +2107,26 @@ union uvh_node_id_u {
                 unsigned long   ni_port:5;                      /* RO */
                 unsigned long   rsvd_62_63:2;
         } s2;
+        struct uv3h_node_id_s {
+                unsigned long   force1:1;                       /* RO */
+                unsigned long   manufacturer:11;                /* RO */
+                unsigned long   part_number:16;                 /* RO */
+                unsigned long   revision:4;                     /* RO */
+                unsigned long   node_id:15;                     /* RW */
+                unsigned long   rsvd_47:1;
+                unsigned long   router_select:1;                /* RO */
+                unsigned long   rsvd_49:1;
+                unsigned long   nodes_per_bit:7;                /* RO */
+                unsigned long   ni_port:5;                      /* RO */
+                unsigned long   rsvd_62_63:2;
+        } s3;
 };
 
 /* ========================================================================= */
 /*                          UVH_NODE_PRESENT_TABLE                           */
 /* ========================================================================= */
-#define UVH_NODE_PRESENT_TABLE                          0x1400UL
-#define UVH_NODE_PRESENT_TABLE_DEPTH                    16
+#define UVH_NODE_PRESENT_TABLE 0x1400UL
+#define UVH_NODE_PRESENT_TABLE_DEPTH 16
 
 #define UVH_NODE_PRESENT_TABLE_NODES_SHFT               0
 #define UVH_NODE_PRESENT_TABLE_NODES_MASK               0xffffffffffffffffUL
@@ -1553,7 +2141,7 @@ union uvh_node_present_table_u {
 /* ========================================================================= */
 /*                 UVH_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR                  */
 /* ========================================================================= */
-#define UVH_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR        0x16000c8UL
+#define UVH_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR 0x16000c8UL
 
 #define UVH_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_BASE_SHFT 24
 #define UVH_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_M_ALIAS_SHFT 48
@@ -1577,7 +2165,7 @@ union uvh_rh_gam_alias210_overlay_config_0_mmr_u {
 /* ========================================================================= */
 /*                 UVH_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR                  */
 /* ========================================================================= */
-#define UVH_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR        0x16000d8UL
+#define UVH_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR 0x16000d8UL
 
 #define UVH_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_BASE_SHFT 24
 #define UVH_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_M_ALIAS_SHFT 48
@@ -1601,7 +2189,7 @@ union uvh_rh_gam_alias210_overlay_config_1_mmr_u {
 /* ========================================================================= */
 /*                 UVH_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR                  */
 /* ========================================================================= */
-#define UVH_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR        0x16000e8UL
+#define UVH_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR 0x16000e8UL
 
 #define UVH_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_BASE_SHFT 24
 #define UVH_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_M_ALIAS_SHFT 48
@@ -1625,7 +2213,7 @@ union uvh_rh_gam_alias210_overlay_config_2_mmr_u {
 /* ========================================================================= */
 /*                UVH_RH_GAM_ALIAS210_REDIRECT_CONFIG_0_MMR                  */
 /* ========================================================================= */
-#define UVH_RH_GAM_ALIAS210_REDIRECT_CONFIG_0_MMR       0x16000d0UL
+#define UVH_RH_GAM_ALIAS210_REDIRECT_CONFIG_0_MMR 0x16000d0UL
 
 #define UVH_RH_GAM_ALIAS210_REDIRECT_CONFIG_0_MMR_DEST_BASE_SHFT 24
 #define UVH_RH_GAM_ALIAS210_REDIRECT_CONFIG_0_MMR_DEST_BASE_MASK 0x00003fffff000000UL
@@ -1642,7 +2230,7 @@ union uvh_rh_gam_alias210_redirect_config_0_mmr_u {
 /* ========================================================================= */
 /*                UVH_RH_GAM_ALIAS210_REDIRECT_CONFIG_1_MMR                  */
 /* ========================================================================= */
-#define UVH_RH_GAM_ALIAS210_REDIRECT_CONFIG_1_MMR       0x16000e0UL
+#define UVH_RH_GAM_ALIAS210_REDIRECT_CONFIG_1_MMR 0x16000e0UL
 
 #define UVH_RH_GAM_ALIAS210_REDIRECT_CONFIG_1_MMR_DEST_BASE_SHFT 24
 #define UVH_RH_GAM_ALIAS210_REDIRECT_CONFIG_1_MMR_DEST_BASE_MASK 0x00003fffff000000UL
@@ -1659,7 +2247,7 @@ union uvh_rh_gam_alias210_redirect_config_1_mmr_u {
 /* ========================================================================= */
 /*                UVH_RH_GAM_ALIAS210_REDIRECT_CONFIG_2_MMR                  */
 /* ========================================================================= */
-#define UVH_RH_GAM_ALIAS210_REDIRECT_CONFIG_2_MMR       0x16000f0UL
+#define UVH_RH_GAM_ALIAS210_REDIRECT_CONFIG_2_MMR 0x16000f0UL
 
 #define UVH_RH_GAM_ALIAS210_REDIRECT_CONFIG_2_MMR_DEST_BASE_SHFT 24
 #define UVH_RH_GAM_ALIAS210_REDIRECT_CONFIG_2_MMR_DEST_BASE_MASK 0x00003fffff000000UL
@@ -1676,7 +2264,10 @@ union uvh_rh_gam_alias210_redirect_config_2_mmr_u {
 /* ========================================================================= */
 /*                          UVH_RH_GAM_CONFIG_MMR                            */
 /* ========================================================================= */
-#define UVH_RH_GAM_CONFIG_MMR                           0x1600000UL
+#define UVH_RH_GAM_CONFIG_MMR 0x1600000UL
+#define UV1H_RH_GAM_CONFIG_MMR 0x1600000UL
+#define UV2H_RH_GAM_CONFIG_MMR 0x1600000UL
+#define UV3H_RH_GAM_CONFIG_MMR 0x1600000UL
 
 #define UVH_RH_GAM_CONFIG_MMR_M_SKT_SHFT                0
 #define UVH_RH_GAM_CONFIG_MMR_N_SKT_SHFT                6
@@ -1690,11 +2281,21 @@ union uvh_rh_gam_alias210_redirect_config_2_mmr_u {
 #define UV1H_RH_GAM_CONFIG_MMR_N_SKT_MASK               0x00000000000003c0UL
 #define UV1H_RH_GAM_CONFIG_MMR_MMIOL_CFG_MASK           0x0000000000001000UL
 
+#define UVXH_RH_GAM_CONFIG_MMR_M_SKT_SHFT               0
+#define UVXH_RH_GAM_CONFIG_MMR_N_SKT_SHFT               6
+#define UVXH_RH_GAM_CONFIG_MMR_M_SKT_MASK               0x000000000000003fUL
+#define UVXH_RH_GAM_CONFIG_MMR_N_SKT_MASK               0x00000000000003c0UL
+
 #define UV2H_RH_GAM_CONFIG_MMR_M_SKT_SHFT               0
 #define UV2H_RH_GAM_CONFIG_MMR_N_SKT_SHFT               6
 #define UV2H_RH_GAM_CONFIG_MMR_M_SKT_MASK               0x000000000000003fUL
 #define UV2H_RH_GAM_CONFIG_MMR_N_SKT_MASK               0x00000000000003c0UL
 
+#define UV3H_RH_GAM_CONFIG_MMR_M_SKT_SHFT               0
+#define UV3H_RH_GAM_CONFIG_MMR_N_SKT_SHFT               6
+#define UV3H_RH_GAM_CONFIG_MMR_M_SKT_MASK               0x000000000000003fUL
+#define UV3H_RH_GAM_CONFIG_MMR_N_SKT_MASK               0x00000000000003c0UL
+
 union uvh_rh_gam_config_mmr_u {
         unsigned long   v;
         struct uvh_rh_gam_config_mmr_s {
@@ -1709,20 +2310,37 @@ union uvh_rh_gam_config_mmr_u {
                 unsigned long   mmiol_cfg:1;                    /* RW */
                 unsigned long   rsvd_13_63:51;
         } s1;
+        struct uvxh_rh_gam_config_mmr_s {
+                unsigned long   m_skt:6;                        /* RW */
+                unsigned long   n_skt:4;                        /* RW */
+                unsigned long   rsvd_10_63:54;
+        } sx;
         struct uv2h_rh_gam_config_mmr_s {
                 unsigned long   m_skt:6;                        /* RW */
                 unsigned long   n_skt:4;                        /* RW */
                 unsigned long   rsvd_10_63:54;
         } s2;
+        struct uv3h_rh_gam_config_mmr_s {
+                unsigned long   m_skt:6;                        /* RW */
+                unsigned long   n_skt:4;                        /* RW */
+                unsigned long   rsvd_10_63:54;
+        } s3;
 };
 
 /* ========================================================================= */
 /*                    UVH_RH_GAM_GRU_OVERLAY_CONFIG_MMR                      */
 /* ========================================================================= */
-#define UVH_RH_GAM_GRU_OVERLAY_CONFIG_MMR               0x1600010UL
+#define UVH_RH_GAM_GRU_OVERLAY_CONFIG_MMR 0x1600010UL
+#define UV1H_RH_GAM_GRU_OVERLAY_CONFIG_MMR 0x1600010UL
+#define UV2H_RH_GAM_GRU_OVERLAY_CONFIG_MMR 0x1600010UL
+#define UV3H_RH_GAM_GRU_OVERLAY_CONFIG_MMR 0x1600010UL
 
 #define UVH_RH_GAM_GRU_OVERLAY_CONFIG_MMR_BASE_SHFT     28
+#define UVH_RH_GAM_GRU_OVERLAY_CONFIG_MMR_N_GRU_SHFT    52
+#define UVH_RH_GAM_GRU_OVERLAY_CONFIG_MMR_ENABLE_SHFT   63
 #define UVH_RH_GAM_GRU_OVERLAY_CONFIG_MMR_BASE_MASK     0x00003ffff0000000UL
+#define UVH_RH_GAM_GRU_OVERLAY_CONFIG_MMR_N_GRU_MASK    0x00f0000000000000UL
+#define UVH_RH_GAM_GRU_OVERLAY_CONFIG_MMR_ENABLE_MASK   0x8000000000000000UL
 
 #define UV1H_RH_GAM_GRU_OVERLAY_CONFIG_MMR_BASE_SHFT    28
 #define UV1H_RH_GAM_GRU_OVERLAY_CONFIG_MMR_GR4_SHFT     48
@@ -1733,6 +2351,13 @@ union uvh_rh_gam_config_mmr_u {
 #define UV1H_RH_GAM_GRU_OVERLAY_CONFIG_MMR_N_GRU_MASK   0x00f0000000000000UL
 #define UV1H_RH_GAM_GRU_OVERLAY_CONFIG_MMR_ENABLE_MASK  0x8000000000000000UL
 
+#define UVXH_RH_GAM_GRU_OVERLAY_CONFIG_MMR_BASE_SHFT    28
+#define UVXH_RH_GAM_GRU_OVERLAY_CONFIG_MMR_N_GRU_SHFT   52
+#define UVXH_RH_GAM_GRU_OVERLAY_CONFIG_MMR_ENABLE_SHFT  63
+#define UVXH_RH_GAM_GRU_OVERLAY_CONFIG_MMR_BASE_MASK    0x00003ffff0000000UL
+#define UVXH_RH_GAM_GRU_OVERLAY_CONFIG_MMR_N_GRU_MASK   0x00f0000000000000UL
+#define UVXH_RH_GAM_GRU_OVERLAY_CONFIG_MMR_ENABLE_MASK  0x8000000000000000UL
+
 #define UV2H_RH_GAM_GRU_OVERLAY_CONFIG_MMR_BASE_SHFT    28
 #define UV2H_RH_GAM_GRU_OVERLAY_CONFIG_MMR_N_GRU_SHFT   52
 #define UV2H_RH_GAM_GRU_OVERLAY_CONFIG_MMR_ENABLE_SHFT  63
@@ -1740,12 +2365,23 @@ union uvh_rh_gam_config_mmr_u {
 #define UV2H_RH_GAM_GRU_OVERLAY_CONFIG_MMR_N_GRU_MASK   0x00f0000000000000UL
 #define UV2H_RH_GAM_GRU_OVERLAY_CONFIG_MMR_ENABLE_MASK  0x8000000000000000UL
 
+#define UV3H_RH_GAM_GRU_OVERLAY_CONFIG_MMR_BASE_SHFT    28
+#define UV3H_RH_GAM_GRU_OVERLAY_CONFIG_MMR_N_GRU_SHFT   52
+#define UV3H_RH_GAM_GRU_OVERLAY_CONFIG_MMR_MODE_SHFT    62
+#define UV3H_RH_GAM_GRU_OVERLAY_CONFIG_MMR_ENABLE_SHFT  63
+#define UV3H_RH_GAM_GRU_OVERLAY_CONFIG_MMR_BASE_MASK    0x00003ffff0000000UL
+#define UV3H_RH_GAM_GRU_OVERLAY_CONFIG_MMR_N_GRU_MASK   0x00f0000000000000UL
+#define UV3H_RH_GAM_GRU_OVERLAY_CONFIG_MMR_MODE_MASK    0x4000000000000000UL
+#define UV3H_RH_GAM_GRU_OVERLAY_CONFIG_MMR_ENABLE_MASK  0x8000000000000000UL
+
 union uvh_rh_gam_gru_overlay_config_mmr_u {
         unsigned long   v;
         struct uvh_rh_gam_gru_overlay_config_mmr_s {
                 unsigned long   rsvd_0_27:28;
                 unsigned long   base:18;                        /* RW */
-                unsigned long   rsvd_46_62:17;
+                unsigned long   rsvd_46_51:6;
+                unsigned long   n_gru:4;                        /* RW */
+                unsigned long   rsvd_56_62:7;
                 unsigned long   enable:1;                       /* RW */
         } s;
         struct uv1h_rh_gam_gru_overlay_config_mmr_s {
@@ -1758,6 +2394,14 @@ union uvh_rh_gam_gru_overlay_config_mmr_u {
                 unsigned long   rsvd_56_62:7;
                 unsigned long   enable:1;                       /* RW */
         } s1;
+        struct uvxh_rh_gam_gru_overlay_config_mmr_s {
+                unsigned long   rsvd_0_27:28;
+                unsigned long   base:18;                        /* RW */
+                unsigned long   rsvd_46_51:6;
+                unsigned long   n_gru:4;                        /* RW */
+                unsigned long   rsvd_56_62:7;
+                unsigned long   enable:1;                       /* RW */
+        } sx;
         struct uv2h_rh_gam_gru_overlay_config_mmr_s {
                 unsigned long   rsvd_0_27:28;
                 unsigned long   base:18;                        /* RW */
@@ -1766,12 +2410,22 @@ union uvh_rh_gam_gru_overlay_config_mmr_u {
                 unsigned long   rsvd_56_62:7;
                 unsigned long   enable:1;                       /* RW */
         } s2;
+        struct uv3h_rh_gam_gru_overlay_config_mmr_s {
+                unsigned long   rsvd_0_27:28;
+                unsigned long   base:18;                        /* RW */
+                unsigned long   rsvd_46_51:6;
+                unsigned long   n_gru:4;                        /* RW */
+                unsigned long   rsvd_56_61:6;
+                unsigned long   mode:1;                         /* RW */
+                unsigned long   enable:1;                       /* RW */
+        } s3;
 };
 
 /* ========================================================================= */
 /*                   UVH_RH_GAM_MMIOH_OVERLAY_CONFIG_MMR                     */
 /* ========================================================================= */
-#define UVH_RH_GAM_MMIOH_OVERLAY_CONFIG_MMR             0x1600030UL
+#define UV1H_RH_GAM_MMIOH_OVERLAY_CONFIG_MMR 0x1600030UL
+#define UV2H_RH_GAM_MMIOH_OVERLAY_CONFIG_MMR 0x1600030UL
 
 #define UV1H_RH_GAM_MMIOH_OVERLAY_CONFIG_MMR_BASE_SHFT  30
 #define UV1H_RH_GAM_MMIOH_OVERLAY_CONFIG_MMR_M_IO_SHFT  46
@@ -1814,10 +2468,15 @@ union uvh_rh_gam_mmioh_overlay_config_mmr_u {
 /* ========================================================================= */
 /*                    UVH_RH_GAM_MMR_OVERLAY_CONFIG_MMR                      */
 /* ========================================================================= */
-#define UVH_RH_GAM_MMR_OVERLAY_CONFIG_MMR               0x1600028UL
+#define UVH_RH_GAM_MMR_OVERLAY_CONFIG_MMR 0x1600028UL
+#define UV1H_RH_GAM_MMR_OVERLAY_CONFIG_MMR 0x1600028UL
+#define UV2H_RH_GAM_MMR_OVERLAY_CONFIG_MMR 0x1600028UL
+#define UV3H_RH_GAM_MMR_OVERLAY_CONFIG_MMR 0x1600028UL
 
 #define UVH_RH_GAM_MMR_OVERLAY_CONFIG_MMR_BASE_SHFT     26
+#define UVH_RH_GAM_MMR_OVERLAY_CONFIG_MMR_ENABLE_SHFT   63
 #define UVH_RH_GAM_MMR_OVERLAY_CONFIG_MMR_BASE_MASK     0x00003ffffc000000UL
+#define UVH_RH_GAM_MMR_OVERLAY_CONFIG_MMR_ENABLE_MASK   0x8000000000000000UL
 
 #define UV1H_RH_GAM_MMR_OVERLAY_CONFIG_MMR_BASE_SHFT    26
 #define UV1H_RH_GAM_MMR_OVERLAY_CONFIG_MMR_DUAL_HUB_SHFT 46
@@ -1826,11 +2485,21 @@ union uvh_rh_gam_mmioh_overlay_config_mmr_u {
 #define UV1H_RH_GAM_MMR_OVERLAY_CONFIG_MMR_DUAL_HUB_MASK 0x0000400000000000UL
 #define UV1H_RH_GAM_MMR_OVERLAY_CONFIG_MMR_ENABLE_MASK  0x8000000000000000UL
 
+#define UVXH_RH_GAM_MMR_OVERLAY_CONFIG_MMR_BASE_SHFT    26
+#define UVXH_RH_GAM_MMR_OVERLAY_CONFIG_MMR_ENABLE_SHFT  63
+#define UVXH_RH_GAM_MMR_OVERLAY_CONFIG_MMR_BASE_MASK    0x00003ffffc000000UL
+#define UVXH_RH_GAM_MMR_OVERLAY_CONFIG_MMR_ENABLE_MASK  0x8000000000000000UL
+
 #define UV2H_RH_GAM_MMR_OVERLAY_CONFIG_MMR_BASE_SHFT    26
 #define UV2H_RH_GAM_MMR_OVERLAY_CONFIG_MMR_ENABLE_SHFT  63
 #define UV2H_RH_GAM_MMR_OVERLAY_CONFIG_MMR_BASE_MASK    0x00003ffffc000000UL
 #define UV2H_RH_GAM_MMR_OVERLAY_CONFIG_MMR_ENABLE_MASK  0x8000000000000000UL
 
+#define UV3H_RH_GAM_MMR_OVERLAY_CONFIG_MMR_BASE_SHFT    26
+#define UV3H_RH_GAM_MMR_OVERLAY_CONFIG_MMR_ENABLE_SHFT  63
+#define UV3H_RH_GAM_MMR_OVERLAY_CONFIG_MMR_BASE_MASK    0x00003ffffc000000UL
+#define UV3H_RH_GAM_MMR_OVERLAY_CONFIG_MMR_ENABLE_MASK  0x8000000000000000UL
+
 union uvh_rh_gam_mmr_overlay_config_mmr_u {
         unsigned long   v;
         struct uvh_rh_gam_mmr_overlay_config_mmr_s {
@@ -1846,18 +2515,30 @@ union uvh_rh_gam_mmr_overlay_config_mmr_u {
                 unsigned long   rsvd_47_62:16;
                 unsigned long   enable:1;                       /* RW */
         } s1;
+        struct uvxh_rh_gam_mmr_overlay_config_mmr_s {
+                unsigned long   rsvd_0_25:26;
+                unsigned long   base:20;                        /* RW */
+                unsigned long   rsvd_46_62:17;
+                unsigned long   enable:1;                       /* RW */
+        } sx;
         struct uv2h_rh_gam_mmr_overlay_config_mmr_s {
                 unsigned long   rsvd_0_25:26;
                 unsigned long   base:20;                        /* RW */
                 unsigned long   rsvd_46_62:17;
                 unsigned long   enable:1;                       /* RW */
         } s2;
+        struct uv3h_rh_gam_mmr_overlay_config_mmr_s {
+                unsigned long   rsvd_0_25:26;
+                unsigned long   base:20;                        /* RW */
+                unsigned long   rsvd_46_62:17;
+                unsigned long   enable:1;                       /* RW */
+        } s3;
 };
 
 /* ========================================================================= */
 /*                                 UVH_RTC                                   */
 /* ========================================================================= */
-#define UVH_RTC                                         0x340000UL
+#define UVH_RTC 0x340000UL
 
 #define UVH_RTC_REAL_TIME_CLOCK_SHFT                    0
 #define UVH_RTC_REAL_TIME_CLOCK_MASK                    0x00ffffffffffffffUL
@@ -1873,7 +2554,7 @@ union uvh_rtc_u {
 /* ========================================================================= */
 /*                           UVH_RTC1_INT_CONFIG                             */
 /* ========================================================================= */
-#define UVH_RTC1_INT_CONFIG                             0x615c0UL
+#define UVH_RTC1_INT_CONFIG 0x615c0UL
 
 #define UVH_RTC1_INT_CONFIG_VECTOR_SHFT                 0
 #define UVH_RTC1_INT_CONFIG_DM_SHFT                     8
@@ -1911,8 +2592,8 @@ union uvh_rtc1_int_config_u {
 /* ========================================================================= */
 /*                               UVH_SCRATCH5                                */
 /* ========================================================================= */
-#define UVH_SCRATCH5                                    0x2d0200UL
-#define UVH_SCRATCH5_32                                 0x778
+#define UVH_SCRATCH5 0x2d0200UL
+#define UVH_SCRATCH5_32 0x778
 
 #define UVH_SCRATCH5_SCRATCH5_SHFT                      0
 #define UVH_SCRATCH5_SCRATCH5_MASK                      0xffffffffffffffffUL
@@ -1925,79 +2606,79 @@ union uvh_scratch5_u {
 };
 
 /* ========================================================================= */
-/*                           UV2H_EVENT_OCCURRED2                            */
-/* ========================================================================= */
-#define UV2H_EVENT_OCCURRED2                            0x70100UL
-#define UV2H_EVENT_OCCURRED2_32                         0xb68
-
-#define UV2H_EVENT_OCCURRED2_RTC_0_SHFT                 0
-#define UV2H_EVENT_OCCURRED2_RTC_1_SHFT                 1
-#define UV2H_EVENT_OCCURRED2_RTC_2_SHFT                 2
-#define UV2H_EVENT_OCCURRED2_RTC_3_SHFT                 3
-#define UV2H_EVENT_OCCURRED2_RTC_4_SHFT                 4
-#define UV2H_EVENT_OCCURRED2_RTC_5_SHFT                 5
-#define UV2H_EVENT_OCCURRED2_RTC_6_SHFT                 6
-#define UV2H_EVENT_OCCURRED2_RTC_7_SHFT                 7
-#define UV2H_EVENT_OCCURRED2_RTC_8_SHFT                 8
-#define UV2H_EVENT_OCCURRED2_RTC_9_SHFT                 9
-#define UV2H_EVENT_OCCURRED2_RTC_10_SHFT                10
-#define UV2H_EVENT_OCCURRED2_RTC_11_SHFT                11
-#define UV2H_EVENT_OCCURRED2_RTC_12_SHFT                12
-#define UV2H_EVENT_OCCURRED2_RTC_13_SHFT                13
-#define UV2H_EVENT_OCCURRED2_RTC_14_SHFT                14
-#define UV2H_EVENT_OCCURRED2_RTC_15_SHFT                15
-#define UV2H_EVENT_OCCURRED2_RTC_16_SHFT                16
-#define UV2H_EVENT_OCCURRED2_RTC_17_SHFT                17
-#define UV2H_EVENT_OCCURRED2_RTC_18_SHFT                18
-#define UV2H_EVENT_OCCURRED2_RTC_19_SHFT                19
-#define UV2H_EVENT_OCCURRED2_RTC_20_SHFT                20
-#define UV2H_EVENT_OCCURRED2_RTC_21_SHFT                21
-#define UV2H_EVENT_OCCURRED2_RTC_22_SHFT                22
-#define UV2H_EVENT_OCCURRED2_RTC_23_SHFT                23
-#define UV2H_EVENT_OCCURRED2_RTC_24_SHFT                24
-#define UV2H_EVENT_OCCURRED2_RTC_25_SHFT                25
-#define UV2H_EVENT_OCCURRED2_RTC_26_SHFT                26
-#define UV2H_EVENT_OCCURRED2_RTC_27_SHFT                27
-#define UV2H_EVENT_OCCURRED2_RTC_28_SHFT                28
-#define UV2H_EVENT_OCCURRED2_RTC_29_SHFT                29
-#define UV2H_EVENT_OCCURRED2_RTC_30_SHFT                30
-#define UV2H_EVENT_OCCURRED2_RTC_31_SHFT                31
-#define UV2H_EVENT_OCCURRED2_RTC_0_MASK                 0x0000000000000001UL
-#define UV2H_EVENT_OCCURRED2_RTC_1_MASK                 0x0000000000000002UL
-#define UV2H_EVENT_OCCURRED2_RTC_2_MASK                 0x0000000000000004UL
-#define UV2H_EVENT_OCCURRED2_RTC_3_MASK                 0x0000000000000008UL
-#define UV2H_EVENT_OCCURRED2_RTC_4_MASK                 0x0000000000000010UL
-#define UV2H_EVENT_OCCURRED2_RTC_5_MASK                 0x0000000000000020UL
-#define UV2H_EVENT_OCCURRED2_RTC_6_MASK                 0x0000000000000040UL
-#define UV2H_EVENT_OCCURRED2_RTC_7_MASK                 0x0000000000000080UL
-#define UV2H_EVENT_OCCURRED2_RTC_8_MASK                 0x0000000000000100UL
-#define UV2H_EVENT_OCCURRED2_RTC_9_MASK                 0x0000000000000200UL
-#define UV2H_EVENT_OCCURRED2_RTC_10_MASK                0x0000000000000400UL
-#define UV2H_EVENT_OCCURRED2_RTC_11_MASK                0x0000000000000800UL
-#define UV2H_EVENT_OCCURRED2_RTC_12_MASK                0x0000000000001000UL
-#define UV2H_EVENT_OCCURRED2_RTC_13_MASK                0x0000000000002000UL
-#define UV2H_EVENT_OCCURRED2_RTC_14_MASK                0x0000000000004000UL
-#define UV2H_EVENT_OCCURRED2_RTC_15_MASK                0x0000000000008000UL
-#define UV2H_EVENT_OCCURRED2_RTC_16_MASK                0x0000000000010000UL
-#define UV2H_EVENT_OCCURRED2_RTC_17_MASK                0x0000000000020000UL
-#define UV2H_EVENT_OCCURRED2_RTC_18_MASK                0x0000000000040000UL
-#define UV2H_EVENT_OCCURRED2_RTC_19_MASK                0x0000000000080000UL
-#define UV2H_EVENT_OCCURRED2_RTC_20_MASK                0x0000000000100000UL
-#define UV2H_EVENT_OCCURRED2_RTC_21_MASK                0x0000000000200000UL
-#define UV2H_EVENT_OCCURRED2_RTC_22_MASK                0x0000000000400000UL
-#define UV2H_EVENT_OCCURRED2_RTC_23_MASK                0x0000000000800000UL
-#define UV2H_EVENT_OCCURRED2_RTC_24_MASK                0x0000000001000000UL
-#define UV2H_EVENT_OCCURRED2_RTC_25_MASK                0x0000000002000000UL
-#define UV2H_EVENT_OCCURRED2_RTC_26_MASK                0x0000000004000000UL
-#define UV2H_EVENT_OCCURRED2_RTC_27_MASK                0x0000000008000000UL
-#define UV2H_EVENT_OCCURRED2_RTC_28_MASK                0x0000000010000000UL
-#define UV2H_EVENT_OCCURRED2_RTC_29_MASK                0x0000000020000000UL
-#define UV2H_EVENT_OCCURRED2_RTC_30_MASK                0x0000000040000000UL
-#define UV2H_EVENT_OCCURRED2_RTC_31_MASK                0x0000000080000000UL
-
-union uv2h_event_occurred2_u {
+/*                          UVXH_EVENT_OCCURRED2                             */
+/* ========================================================================= */
+#define UVXH_EVENT_OCCURRED2 0x70100UL
+#define UVXH_EVENT_OCCURRED2_32 0xb68
+
+#define UVXH_EVENT_OCCURRED2_RTC_0_SHFT                 0
+#define UVXH_EVENT_OCCURRED2_RTC_1_SHFT                 1
+#define UVXH_EVENT_OCCURRED2_RTC_2_SHFT                 2
+#define UVXH_EVENT_OCCURRED2_RTC_3_SHFT                 3
+#define UVXH_EVENT_OCCURRED2_RTC_4_SHFT                 4
+#define UVXH_EVENT_OCCURRED2_RTC_5_SHFT                 5
+#define UVXH_EVENT_OCCURRED2_RTC_6_SHFT                 6
+#define UVXH_EVENT_OCCURRED2_RTC_7_SHFT                 7
+#define UVXH_EVENT_OCCURRED2_RTC_8_SHFT                 8
+#define UVXH_EVENT_OCCURRED2_RTC_9_SHFT                 9
+#define UVXH_EVENT_OCCURRED2_RTC_10_SHFT                10
+#define UVXH_EVENT_OCCURRED2_RTC_11_SHFT                11
+#define UVXH_EVENT_OCCURRED2_RTC_12_SHFT                12
+#define UVXH_EVENT_OCCURRED2_RTC_13_SHFT                13
+#define UVXH_EVENT_OCCURRED2_RTC_14_SHFT                14
+#define UVXH_EVENT_OCCURRED2_RTC_15_SHFT                15
+#define UVXH_EVENT_OCCURRED2_RTC_16_SHFT                16
+#define UVXH_EVENT_OCCURRED2_RTC_17_SHFT                17
+#define UVXH_EVENT_OCCURRED2_RTC_18_SHFT                18
+#define UVXH_EVENT_OCCURRED2_RTC_19_SHFT                19
+#define UVXH_EVENT_OCCURRED2_RTC_20_SHFT                20
+#define UVXH_EVENT_OCCURRED2_RTC_21_SHFT                21
+#define UVXH_EVENT_OCCURRED2_RTC_22_SHFT                22
+#define UVXH_EVENT_OCCURRED2_RTC_23_SHFT                23
+#define UVXH_EVENT_OCCURRED2_RTC_24_SHFT                24
+#define UVXH_EVENT_OCCURRED2_RTC_25_SHFT                25
+#define UVXH_EVENT_OCCURRED2_RTC_26_SHFT                26
+#define UVXH_EVENT_OCCURRED2_RTC_27_SHFT                27
+#define UVXH_EVENT_OCCURRED2_RTC_28_SHFT                28
+#define UVXH_EVENT_OCCURRED2_RTC_29_SHFT                29
+#define UVXH_EVENT_OCCURRED2_RTC_30_SHFT                30
+#define UVXH_EVENT_OCCURRED2_RTC_31_SHFT                31
+#define UVXH_EVENT_OCCURRED2_RTC_0_MASK                 0x0000000000000001UL
+#define UVXH_EVENT_OCCURRED2_RTC_1_MASK                 0x0000000000000002UL
+#define UVXH_EVENT_OCCURRED2_RTC_2_MASK                 0x0000000000000004UL
+#define UVXH_EVENT_OCCURRED2_RTC_3_MASK                 0x0000000000000008UL
+#define UVXH_EVENT_OCCURRED2_RTC_4_MASK                 0x0000000000000010UL
+#define UVXH_EVENT_OCCURRED2_RTC_5_MASK                 0x0000000000000020UL
+#define UVXH_EVENT_OCCURRED2_RTC_6_MASK                 0x0000000000000040UL
+#define UVXH_EVENT_OCCURRED2_RTC_7_MASK                 0x0000000000000080UL
+#define UVXH_EVENT_OCCURRED2_RTC_8_MASK                 0x0000000000000100UL
+#define UVXH_EVENT_OCCURRED2_RTC_9_MASK                 0x0000000000000200UL
+#define UVXH_EVENT_OCCURRED2_RTC_10_MASK                0x0000000000000400UL
+#define UVXH_EVENT_OCCURRED2_RTC_11_MASK                0x0000000000000800UL
+#define UVXH_EVENT_OCCURRED2_RTC_12_MASK                0x0000000000001000UL
+#define UVXH_EVENT_OCCURRED2_RTC_13_MASK                0x0000000000002000UL
+#define UVXH_EVENT_OCCURRED2_RTC_14_MASK                0x0000000000004000UL
+#define UVXH_EVENT_OCCURRED2_RTC_15_MASK                0x0000000000008000UL
+#define UVXH_EVENT_OCCURRED2_RTC_16_MASK                0x0000000000010000UL
+#define UVXH_EVENT_OCCURRED2_RTC_17_MASK                0x0000000000020000UL
+#define UVXH_EVENT_OCCURRED2_RTC_18_MASK                0x0000000000040000UL
+#define UVXH_EVENT_OCCURRED2_RTC_19_MASK                0x0000000000080000UL
+#define UVXH_EVENT_OCCURRED2_RTC_20_MASK                0x0000000000100000UL
+#define UVXH_EVENT_OCCURRED2_RTC_21_MASK                0x0000000000200000UL
+#define UVXH_EVENT_OCCURRED2_RTC_22_MASK                0x0000000000400000UL
+#define UVXH_EVENT_OCCURRED2_RTC_23_MASK                0x0000000000800000UL
+#define UVXH_EVENT_OCCURRED2_RTC_24_MASK                0x0000000001000000UL
+#define UVXH_EVENT_OCCURRED2_RTC_25_MASK                0x0000000002000000UL
+#define UVXH_EVENT_OCCURRED2_RTC_26_MASK                0x0000000004000000UL
+#define UVXH_EVENT_OCCURRED2_RTC_27_MASK                0x0000000008000000UL
+#define UVXH_EVENT_OCCURRED2_RTC_28_MASK                0x0000000010000000UL
+#define UVXH_EVENT_OCCURRED2_RTC_29_MASK                0x0000000020000000UL
+#define UVXH_EVENT_OCCURRED2_RTC_30_MASK                0x0000000040000000UL
+#define UVXH_EVENT_OCCURRED2_RTC_31_MASK                0x0000000080000000UL
+
+union uvxh_event_occurred2_u {
         unsigned long   v;
-        struct uv2h_event_occurred2_s {
+        struct uvxh_event_occurred2_s {
                 unsigned long   rtc_0:1;                        /* RW */
                 unsigned long   rtc_1:1;                        /* RW */
                 unsigned long   rtc_2:1;                        /* RW */
@@ -2031,29 +2712,46 @@ union uv2h_event_occurred2_u {
                 unsigned long   rtc_30:1;                       /* RW */
                 unsigned long   rtc_31:1;                       /* RW */
                 unsigned long   rsvd_32_63:32;
-        } s1;
+        } sx;
 };
 
 /* ========================================================================= */
-/*                        UV2H_EVENT_OCCURRED2_ALIAS                         */
+/*                       UVXH_EVENT_OCCURRED2_ALIAS                          */
 /* ========================================================================= */
-#define UV2H_EVENT_OCCURRED2_ALIAS                      0x70108UL
-#define UV2H_EVENT_OCCURRED2_ALIAS_32                   0xb70
+#define UVXH_EVENT_OCCURRED2_ALIAS 0x70108UL
+#define UVXH_EVENT_OCCURRED2_ALIAS_32 0xb70
+
 
 /* ========================================================================= */
-/*                    UV2H_LB_BAU_SB_ACTIVATION_STATUS_2                     */
+/*                   UVXH_LB_BAU_SB_ACTIVATION_STATUS_2                      */
 /* ========================================================================= */
-#define UV2H_LB_BAU_SB_ACTIVATION_STATUS_2              0x320130UL
-#define UV2H_LB_BAU_SB_ACTIVATION_STATUS_2_32           0x9f0
+#define UVXH_LB_BAU_SB_ACTIVATION_STATUS_2 0x320130UL
+#define UV2H_LB_BAU_SB_ACTIVATION_STATUS_2 0x320130UL
+#define UV3H_LB_BAU_SB_ACTIVATION_STATUS_2 0x320130UL
+#define UVXH_LB_BAU_SB_ACTIVATION_STATUS_2_32 0x9f0
+#define UV2H_LB_BAU_SB_ACTIVATION_STATUS_2_32 0x320130UL
+#define UV3H_LB_BAU_SB_ACTIVATION_STATUS_2_32 0x320130UL
+
+#define UVXH_LB_BAU_SB_ACTIVATION_STATUS_2_AUX_ERROR_SHFT 0
+#define UVXH_LB_BAU_SB_ACTIVATION_STATUS_2_AUX_ERROR_MASK 0xffffffffffffffffUL
 
 #define UV2H_LB_BAU_SB_ACTIVATION_STATUS_2_AUX_ERROR_SHFT 0
 #define UV2H_LB_BAU_SB_ACTIVATION_STATUS_2_AUX_ERROR_MASK 0xffffffffffffffffUL
 
-union uv2h_lb_bau_sb_activation_status_2_u {
+#define UV3H_LB_BAU_SB_ACTIVATION_STATUS_2_AUX_ERROR_SHFT 0
+#define UV3H_LB_BAU_SB_ACTIVATION_STATUS_2_AUX_ERROR_MASK 0xffffffffffffffffUL
+
+union uvxh_lb_bau_sb_activation_status_2_u {
         unsigned long   v;
+        struct uvxh_lb_bau_sb_activation_status_2_s {
+                unsigned long   aux_error:64;                   /* RW */
+        } sx;
         struct uv2h_lb_bau_sb_activation_status_2_s {
                 unsigned long   aux_error:64;                   /* RW */
-        } s1;
+        } s2;
+        struct uv3h_lb_bau_sb_activation_status_2_s {
+                unsigned long   aux_error:64;                   /* RW */
+        } s3;
 };
 
 /* ========================================================================= */
@@ -2073,5 +2771,87 @@ union uv1h_lb_target_physical_apic_id_mask_u {
         } s1;
 };
 
+/* ========================================================================= */
+/*                   UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR                   */
+/* ========================================================================= */
+#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR           0x1603000UL
+
+#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_BASE_SHFT 26
+#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_M_IO_SHFT 46
+#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_ENABLE_SHFT 63
+#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_BASE_MASK 0x00003ffffc000000UL
+#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_M_IO_MASK 0x000fc00000000000UL
+#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_ENABLE_MASK 0x8000000000000000UL
+
+union uv3h_rh_gam_mmioh_overlay_config0_mmr_u {
+        unsigned long   v;
+        struct uv3h_rh_gam_mmioh_overlay_config0_mmr_s {
+                unsigned long   rsvd_0_25:26;
+                unsigned long   base:20;                        /* RW */
+                unsigned long   m_io:6;                         /* RW */
+                unsigned long   n_io:4;
+                unsigned long   rsvd_56_62:7;
+                unsigned long   enable:1;                       /* RW */
+        } s3;
+};
+
+/* ========================================================================= */
+/*                   UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR                   */
+/* ========================================================================= */
+#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR           0x1604000UL
+
+#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_BASE_SHFT 26
+#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_M_IO_SHFT 46
+#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_ENABLE_SHFT 63
+#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_BASE_MASK 0x00003ffffc000000UL
+#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_M_IO_MASK 0x000fc00000000000UL
+#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_ENABLE_MASK 0x8000000000000000UL
+
+union uv3h_rh_gam_mmioh_overlay_config1_mmr_u {
+        unsigned long   v;
+        struct uv3h_rh_gam_mmioh_overlay_config1_mmr_s {
+                unsigned long   rsvd_0_25:26;
+                unsigned long   base:20;                        /* RW */
+                unsigned long   m_io:6;                         /* RW */
+                unsigned long   n_io:4;
+                unsigned long   rsvd_56_62:7;
+                unsigned long   enable:1;                       /* RW */
+        } s3;
+};
+
+/* ========================================================================= */
+/*                  UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR                   */
+/* ========================================================================= */
+#define UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR          0x1603800UL
+#define UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR_DEPTH    128
+
+#define UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR_NASID_SHFT 0
+#define UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR_NASID_MASK 0x0000000000007fffUL
+
+union uv3h_rh_gam_mmioh_redirect_config0_mmr_u {
+        unsigned long   v;
+        struct uv3h_rh_gam_mmioh_redirect_config0_mmr_s {
+                unsigned long   nasid:15;                       /* RW */
+                unsigned long   rsvd_15_63:49;
+        } s3;
+};
+
+/* ========================================================================= */
+/*                  UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR                   */
+/* ========================================================================= */
+#define UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR          0x1604800UL
+#define UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR_DEPTH    128
+
+#define UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR_NASID_SHFT 0
+#define UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR_NASID_MASK 0x0000000000007fffUL
+
+union uv3h_rh_gam_mmioh_redirect_config1_mmr_u {
+        unsigned long   v;
+        struct uv3h_rh_gam_mmioh_redirect_config1_mmr_s {
+                unsigned long   nasid:15;                       /* RW */
+                unsigned long   rsvd_15_63:49;
+        } s3;
+};
+
 
 #endif /* _ASM_X86_UV_UV_MMRS_H */
diff --git a/arch/x86/include/asm/vmx.h b/arch/x86/include/asm/vmx.h
index 235b49fa554b..b6fbf860e398 100644
--- a/arch/x86/include/asm/vmx.h
+++ b/arch/x86/include/asm/vmx.h
@@ -57,9 +57,12 @@
 #define SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES 0x00000001
 #define SECONDARY_EXEC_ENABLE_EPT               0x00000002
 #define SECONDARY_EXEC_RDTSCP                   0x00000008
+#define SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE   0x00000010
 #define SECONDARY_EXEC_ENABLE_VPID              0x00000020
 #define SECONDARY_EXEC_WBINVD_EXITING           0x00000040
 #define SECONDARY_EXEC_UNRESTRICTED_GUEST       0x00000080
+#define SECONDARY_EXEC_APIC_REGISTER_VIRT       0x00000100
+#define SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY    0x00000200
 #define SECONDARY_EXEC_PAUSE_LOOP_EXITING       0x00000400
 #define SECONDARY_EXEC_ENABLE_INVPCID           0x00001000
 
@@ -97,6 +100,7 @@ enum vmcs_field {
         GUEST_GS_SELECTOR               = 0x0000080a,
         GUEST_LDTR_SELECTOR             = 0x0000080c,
         GUEST_TR_SELECTOR               = 0x0000080e,
+        GUEST_INTR_STATUS               = 0x00000810,
         HOST_ES_SELECTOR                = 0x00000c00,
         HOST_CS_SELECTOR                = 0x00000c02,
         HOST_SS_SELECTOR                = 0x00000c04,
@@ -124,6 +128,14 @@ enum vmcs_field {
         APIC_ACCESS_ADDR_HIGH           = 0x00002015,
         EPT_POINTER                     = 0x0000201a,
         EPT_POINTER_HIGH                = 0x0000201b,
+        EOI_EXIT_BITMAP0                = 0x0000201c,
+        EOI_EXIT_BITMAP0_HIGH           = 0x0000201d,
+        EOI_EXIT_BITMAP1                = 0x0000201e,
+        EOI_EXIT_BITMAP1_HIGH           = 0x0000201f,
+        EOI_EXIT_BITMAP2                = 0x00002020,
+        EOI_EXIT_BITMAP2_HIGH           = 0x00002021,
+        EOI_EXIT_BITMAP3                = 0x00002022,
+        EOI_EXIT_BITMAP3_HIGH           = 0x00002023,
         GUEST_PHYSICAL_ADDRESS          = 0x00002400,
         GUEST_PHYSICAL_ADDRESS_HIGH     = 0x00002401,
         VMCS_LINK_POINTER               = 0x00002800,
@@ -346,9 +358,9 @@ enum vmcs_field {
 
 #define AR_RESERVD_MASK 0xfffe0f00
 
-#define TSS_PRIVATE_MEMSLOT                     (KVM_MEMORY_SLOTS + 0)
-#define APIC_ACCESS_PAGE_PRIVATE_MEMSLOT        (KVM_MEMORY_SLOTS + 1)
-#define IDENTITY_PAGETABLE_PRIVATE_MEMSLOT      (KVM_MEMORY_SLOTS + 2)
+#define TSS_PRIVATE_MEMSLOT                     (KVM_USER_MEM_SLOTS + 0)
+#define APIC_ACCESS_PAGE_PRIVATE_MEMSLOT        (KVM_USER_MEM_SLOTS + 1)
+#define IDENTITY_PAGETABLE_PRIVATE_MEMSLOT      (KVM_USER_MEM_SLOTS + 2)
 
 #define VMX_NR_VPIDS                            (1 << 16)
 #define VMX_VPID_EXTENT_SINGLE_CONTEXT          1
diff --git a/arch/x86/include/asm/x86_init.h b/arch/x86/include/asm/x86_init.h
index 57693498519c..d8d99222b36a 100644
--- a/arch/x86/include/asm/x86_init.h
+++ b/arch/x86/include/asm/x86_init.h
@@ -69,17 +69,6 @@ struct x86_init_oem {
 };
 
 /**
- * struct x86_init_mapping - platform specific initial kernel pagetable setup
- * @pagetable_reserve:  reserve a range of addresses for kernel pagetable usage
- *
- * For more details on the purpose of this hook, look in
- * init_memory_mapping and the commit that added it.
- */
-struct x86_init_mapping {
-        void (*pagetable_reserve)(u64 start, u64 end);
-};
-
-/**
  * struct x86_init_paging - platform specific paging functions
  * @pagetable_init:     platform specific paging initialization call to setup
  *                      the kernel pagetables and prepare accessors functions.
@@ -136,7 +125,6 @@ struct x86_init_ops {
         struct x86_init_mpparse         mpparse;
         struct x86_init_irqs            irqs;
         struct x86_init_oem             oem;
-        struct x86_init_mapping         mapping;
         struct x86_init_paging          paging;
         struct x86_init_timers          timers;
         struct x86_init_iommu           iommu;
@@ -181,19 +169,38 @@ struct x86_platform_ops {
 };
 
 struct pci_dev;
+struct msi_msg;
 
 struct x86_msi_ops {
         int (*setup_msi_irqs)(struct pci_dev *dev, int nvec, int type);
+        void (*compose_msi_msg)(struct pci_dev *dev, unsigned int irq,
+                                unsigned int dest, struct msi_msg *msg,
+                               u8 hpet_id);
         void (*teardown_msi_irq)(unsigned int irq);
         void (*teardown_msi_irqs)(struct pci_dev *dev);
         void (*restore_msi_irqs)(struct pci_dev *dev, int irq);
+        int  (*setup_hpet_msi)(unsigned int irq, unsigned int id);
 };
 
+struct IO_APIC_route_entry;
+struct io_apic_irq_attr;
+struct irq_data;
+struct cpumask;
+
 struct x86_io_apic_ops {
-        void            (*init)  (void);
-        unsigned int    (*read)  (unsigned int apic, unsigned int reg);
-        void            (*write) (unsigned int apic, unsigned int reg, unsigned int value);
-        void            (*modify)(unsigned int apic, unsigned int reg, unsigned int value);
+        void            (*init)   (void);
+        unsigned int    (*read)   (unsigned int apic, unsigned int reg);
+        void            (*write)  (unsigned int apic, unsigned int reg, unsigned int value);
+        void            (*modify) (unsigned int apic, unsigned int reg, unsigned int value);
+        void            (*disable)(void);
+        void            (*print_entries)(unsigned int apic, unsigned int nr_entries);
+        int             (*set_affinity)(struct irq_data *data,
+                                        const struct cpumask *mask,
+                                        bool force);
+        int             (*setup_entry)(int irq, struct IO_APIC_route_entry *entry,
+                                       unsigned int destination, int vector,
+                                       struct io_apic_irq_attr *attr);
+        void            (*eoi_ioapic_pin)(int apic, int pin, int vector);
 };
 
 extern struct x86_init_ops x86_init;
diff --git a/arch/x86/include/asm/xen/events.h b/arch/x86/include/asm/xen/events.h
index cc146d51449e..ca842f2769ef 100644
--- a/arch/x86/include/asm/xen/events.h
+++ b/arch/x86/include/asm/xen/events.h
@@ -16,4 +16,7 @@ static inline int xen_irqs_disabled(struct pt_regs *regs)
         return raw_irqs_disabled_flags(regs->flags);
 }
 
+/* No need for a barrier -- XCHG is a barrier on x86. */
+#define xchg_xen_ulong(ptr, val) xchg((ptr), (val))
+
 #endif /* _ASM_X86_XEN_EVENTS_H */
diff --git a/arch/x86/include/asm/xen/hypercall.h b/arch/x86/include/asm/xen/hypercall.h
index c20d1ce62dc6..e709884d0ef9 100644
--- a/arch/x86/include/asm/xen/hypercall.h
+++ b/arch/x86/include/asm/xen/hypercall.h
@@ -382,14 +382,14 @@ HYPERVISOR_console_io(int cmd, int count, char *str)
         return _hypercall3(int, console_io, cmd, count, str);
 }
 
-extern int __must_check HYPERVISOR_physdev_op_compat(int, void *);
+extern int __must_check xen_physdev_op_compat(int, void *);
 
 static inline int
 HYPERVISOR_physdev_op(int cmd, void *arg)
 {
         int rc = _hypercall2(int, physdev_op, cmd, arg);
         if (unlikely(rc == -ENOSYS))
-                rc = HYPERVISOR_physdev_op_compat(cmd, arg);
+                rc = xen_physdev_op_compat(cmd, arg);
         return rc;
 }
 
diff --git a/arch/x86/include/asm/xen/page.h b/arch/x86/include/asm/xen/page.h
index 472b9b783019..6aef9fbc09b7 100644
--- a/arch/x86/include/asm/xen/page.h
+++ b/arch/x86/include/asm/xen/page.h
@@ -212,4 +212,6 @@ unsigned long arbitrary_virt_to_mfn(void *vaddr);
 void make_lowmem_page_readonly(void *vaddr);
 void make_lowmem_page_readwrite(void *vaddr);
 
+#define xen_remap(cookie, size) ioremap((cookie), (size));
+
 #endif /* _ASM_X86_XEN_PAGE_H */
diff --git a/arch/x86/include/asm/xor.h b/arch/x86/include/asm/xor.h
index f8fde90bc45e..d8829751b3f8 100644
--- a/arch/x86/include/asm/xor.h
+++ b/arch/x86/include/asm/xor.h
@@ -1,10 +1,499 @@
 #ifdef CONFIG_KMEMCHECK
 /* kmemcheck doesn't handle MMX/SSE/SSE2 instructions */
 # include <asm-generic/xor.h>
+#elif !defined(_ASM_X86_XOR_H)
+#define _ASM_X86_XOR_H
+
+/*
+ * Optimized RAID-5 checksumming functions for SSE.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * You should have received a copy of the GNU General Public License
+ * (for example /usr/src/linux/COPYING); if not, write to the Free
+ * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+/*
+ * Cache avoiding checksumming functions utilizing KNI instructions
+ * Copyright (C) 1999 Zach Brown (with obvious credit due Ingo)
+ */
+
+/*
+ * Based on
+ * High-speed RAID5 checksumming functions utilizing SSE instructions.
+ * Copyright (C) 1998 Ingo Molnar.
+ */
+
+/*
+ * x86-64 changes / gcc fixes from Andi Kleen.
+ * Copyright 2002 Andi Kleen, SuSE Labs.
+ *
+ * This hasn't been optimized for the hammer yet, but there are likely
+ * no advantages to be gotten from x86-64 here anyways.
+ */
+
+#include <asm/i387.h>
+
+#ifdef CONFIG_X86_32
+/* reduce register pressure */
+# define XOR_CONSTANT_CONSTRAINT "i"
 #else
+# define XOR_CONSTANT_CONSTRAINT "re"
+#endif
+
+#define OFFS(x)         "16*("#x")"
+#define PF_OFFS(x)      "256+16*("#x")"
+#define PF0(x)          "       prefetchnta "PF_OFFS(x)"(%[p1])         ;\n"
+#define LD(x, y)        "       movaps "OFFS(x)"(%[p1]), %%xmm"#y"      ;\n"
+#define ST(x, y)        "       movaps %%xmm"#y", "OFFS(x)"(%[p1])      ;\n"
+#define PF1(x)          "       prefetchnta "PF_OFFS(x)"(%[p2])         ;\n"
+#define PF2(x)          "       prefetchnta "PF_OFFS(x)"(%[p3])         ;\n"
+#define PF3(x)          "       prefetchnta "PF_OFFS(x)"(%[p4])         ;\n"
+#define PF4(x)          "       prefetchnta "PF_OFFS(x)"(%[p5])         ;\n"
+#define XO1(x, y)       "       xorps "OFFS(x)"(%[p2]), %%xmm"#y"       ;\n"
+#define XO2(x, y)       "       xorps "OFFS(x)"(%[p3]), %%xmm"#y"       ;\n"
+#define XO3(x, y)       "       xorps "OFFS(x)"(%[p4]), %%xmm"#y"       ;\n"
+#define XO4(x, y)       "       xorps "OFFS(x)"(%[p5]), %%xmm"#y"       ;\n"
+#define NOP(x)
+
+#define BLK64(pf, op, i)                                \
+                pf(i)                                   \
+                op(i, 0)                                \
+                        op(i + 1, 1)                    \
+                                op(i + 2, 2)            \
+                                        op(i + 3, 3)
+
+static void
+xor_sse_2(unsigned long bytes, unsigned long *p1, unsigned long *p2)
+{
+        unsigned long lines = bytes >> 8;
+
+        kernel_fpu_begin();
+
+        asm volatile(
+#undef BLOCK
+#define BLOCK(i)                                        \
+                LD(i, 0)                                \
+                        LD(i + 1, 1)                    \
+                PF1(i)                                  \
+                                PF1(i + 2)              \
+                                LD(i + 2, 2)            \
+                                        LD(i + 3, 3)    \
+                PF0(i + 4)                              \
+                                PF0(i + 6)              \
+                XO1(i, 0)                               \
+                        XO1(i + 1, 1)                   \
+                                XO1(i + 2, 2)           \
+                                        XO1(i + 3, 3)   \
+                ST(i, 0)                                \
+                        ST(i + 1, 1)                    \
+                                ST(i + 2, 2)            \
+                                        ST(i + 3, 3)    \
+
+
+                PF0(0)
+                                PF0(2)
+
+        " .align 32                     ;\n"
+        " 1:                            ;\n"
+
+                BLOCK(0)
+                BLOCK(4)
+                BLOCK(8)
+                BLOCK(12)
+
+        "       add %[inc], %[p1]       ;\n"
+        "       add %[inc], %[p2]       ;\n"
+        "       dec %[cnt]              ;\n"
+        "       jnz 1b                  ;\n"
+        : [cnt] "+r" (lines),
+          [p1] "+r" (p1), [p2] "+r" (p2)
+        : [inc] XOR_CONSTANT_CONSTRAINT (256UL)
+        : "memory");
+
+        kernel_fpu_end();
+}
+
+static void
+xor_sse_2_pf64(unsigned long bytes, unsigned long *p1, unsigned long *p2)
+{
+        unsigned long lines = bytes >> 8;
+
+        kernel_fpu_begin();
+
+        asm volatile(
+#undef BLOCK
+#define BLOCK(i)                        \
+                BLK64(PF0, LD, i)       \
+                BLK64(PF1, XO1, i)      \
+                BLK64(NOP, ST, i)       \
+
+        " .align 32                     ;\n"
+        " 1:                            ;\n"
+
+                BLOCK(0)
+                BLOCK(4)
+                BLOCK(8)
+                BLOCK(12)
+
+        "       add %[inc], %[p1]       ;\n"
+        "       add %[inc], %[p2]       ;\n"
+        "       dec %[cnt]              ;\n"
+        "       jnz 1b                  ;\n"
+        : [cnt] "+r" (lines),
+          [p1] "+r" (p1), [p2] "+r" (p2)
+        : [inc] XOR_CONSTANT_CONSTRAINT (256UL)
+        : "memory");
+
+        kernel_fpu_end();
+}
+
+static void
+xor_sse_3(unsigned long bytes, unsigned long *p1, unsigned long *p2,
+          unsigned long *p3)
+{
+        unsigned long lines = bytes >> 8;
+
+        kernel_fpu_begin();
+
+        asm volatile(
+#undef BLOCK
+#define BLOCK(i) \
+                PF1(i)                                  \
+                                PF1(i + 2)              \
+                LD(i, 0)                                \
+                        LD(i + 1, 1)                    \
+                                LD(i + 2, 2)            \
+                                        LD(i + 3, 3)    \
+                PF2(i)                                  \
+                                PF2(i + 2)              \
+                PF0(i + 4)                              \
+                                PF0(i + 6)              \
+                XO1(i, 0)                               \
+                        XO1(i + 1, 1)                   \
+                                XO1(i + 2, 2)           \
+                                        XO1(i + 3, 3)   \
+                XO2(i, 0)                               \
+                        XO2(i + 1, 1)                   \
+                                XO2(i + 2, 2)           \
+                                        XO2(i + 3, 3)   \
+                ST(i, 0)                                \
+                        ST(i + 1, 1)                    \
+                                ST(i + 2, 2)            \
+                                        ST(i + 3, 3)    \
+
+
+                PF0(0)
+                                PF0(2)
+
+        " .align 32                     ;\n"
+        " 1:                            ;\n"
+
+                BLOCK(0)
+                BLOCK(4)
+                BLOCK(8)
+                BLOCK(12)
+
+        "       add %[inc], %[p1]       ;\n"
+        "       add %[inc], %[p2]       ;\n"
+        "       add %[inc], %[p3]       ;\n"
+        "       dec %[cnt]              ;\n"
+        "       jnz 1b                  ;\n"
+        : [cnt] "+r" (lines),
+          [p1] "+r" (p1), [p2] "+r" (p2), [p3] "+r" (p3)
+        : [inc] XOR_CONSTANT_CONSTRAINT (256UL)
+        : "memory");
+
+        kernel_fpu_end();
+}
+
+static void
+xor_sse_3_pf64(unsigned long bytes, unsigned long *p1, unsigned long *p2,
+               unsigned long *p3)
+{
+        unsigned long lines = bytes >> 8;
+
+        kernel_fpu_begin();
+
+        asm volatile(
+#undef BLOCK
+#define BLOCK(i)                        \
+                BLK64(PF0, LD, i)       \
+                BLK64(PF1, XO1, i)      \
+                BLK64(PF2, XO2, i)      \
+                BLK64(NOP, ST, i)       \
+
+        " .align 32                     ;\n"
+        " 1:                            ;\n"
+
+                BLOCK(0)
+                BLOCK(4)
+                BLOCK(8)
+                BLOCK(12)
+
+        "       add %[inc], %[p1]       ;\n"
+        "       add %[inc], %[p2]       ;\n"
+        "       add %[inc], %[p3]       ;\n"
+        "       dec %[cnt]              ;\n"
+        "       jnz 1b                  ;\n"
+        : [cnt] "+r" (lines),
+          [p1] "+r" (p1), [p2] "+r" (p2), [p3] "+r" (p3)
+        : [inc] XOR_CONSTANT_CONSTRAINT (256UL)
+        : "memory");
+
+        kernel_fpu_end();
+}
+
+static void
+xor_sse_4(unsigned long bytes, unsigned long *p1, unsigned long *p2,
+          unsigned long *p3, unsigned long *p4)
+{
+        unsigned long lines = bytes >> 8;
+
+        kernel_fpu_begin();
+
+        asm volatile(
+#undef BLOCK
+#define BLOCK(i) \
+                PF1(i)                                  \
+                                PF1(i + 2)              \
+                LD(i, 0)                                \
+                        LD(i + 1, 1)                    \
+                                LD(i + 2, 2)            \
+                                        LD(i + 3, 3)    \
+                PF2(i)                                  \
+                                PF2(i + 2)              \
+                XO1(i, 0)                               \
+                        XO1(i + 1, 1)                   \
+                                XO1(i + 2, 2)           \
+                                        XO1(i + 3, 3)   \
+                PF3(i)                                  \
+                                PF3(i + 2)              \
+                PF0(i + 4)                              \
+                                PF0(i + 6)              \
+                XO2(i, 0)                               \
+                        XO2(i + 1, 1)                   \
+                                XO2(i + 2, 2)           \
+                                        XO2(i + 3, 3)   \
+                XO3(i, 0)                               \
+                        XO3(i + 1, 1)                   \
+                                XO3(i + 2, 2)           \
+                                        XO3(i + 3, 3)   \
+                ST(i, 0)                                \
+                        ST(i + 1, 1)                    \
+                                ST(i + 2, 2)            \
+                                        ST(i + 3, 3)    \
+
+
+                PF0(0)
+                                PF0(2)
+
+        " .align 32                     ;\n"
+        " 1:                            ;\n"
+
+                BLOCK(0)
+                BLOCK(4)
+                BLOCK(8)
+                BLOCK(12)
+
+        "       add %[inc], %[p1]       ;\n"
+        "       add %[inc], %[p2]       ;\n"
+        "       add %[inc], %[p3]       ;\n"
+        "       add %[inc], %[p4]       ;\n"
+        "       dec %[cnt]              ;\n"
+        "       jnz 1b                  ;\n"
+        : [cnt] "+r" (lines), [p1] "+r" (p1),
+          [p2] "+r" (p2), [p3] "+r" (p3), [p4] "+r" (p4)
+        : [inc] XOR_CONSTANT_CONSTRAINT (256UL)
+        : "memory");
+
+        kernel_fpu_end();
+}
+
+static void
+xor_sse_4_pf64(unsigned long bytes, unsigned long *p1, unsigned long *p2,
+               unsigned long *p3, unsigned long *p4)
+{
+        unsigned long lines = bytes >> 8;
+
+        kernel_fpu_begin();
+
+        asm volatile(
+#undef BLOCK
+#define BLOCK(i)                        \
+                BLK64(PF0, LD, i)       \
+                BLK64(PF1, XO1, i)      \
+                BLK64(PF2, XO2, i)      \
+                BLK64(PF3, XO3, i)      \
+                BLK64(NOP, ST, i)       \
+
+        " .align 32                     ;\n"
+        " 1:                            ;\n"
+
+                BLOCK(0)
+                BLOCK(4)
+                BLOCK(8)
+                BLOCK(12)
+
+        "       add %[inc], %[p1]       ;\n"
+        "       add %[inc], %[p2]       ;\n"
+        "       add %[inc], %[p3]       ;\n"
+        "       add %[inc], %[p4]       ;\n"
+        "       dec %[cnt]              ;\n"
+        "       jnz 1b                  ;\n"
+        : [cnt] "+r" (lines), [p1] "+r" (p1),
+          [p2] "+r" (p2), [p3] "+r" (p3), [p4] "+r" (p4)
+        : [inc] XOR_CONSTANT_CONSTRAINT (256UL)
+        : "memory");
+
+        kernel_fpu_end();
+}
+
+static void
+xor_sse_5(unsigned long bytes, unsigned long *p1, unsigned long *p2,
+          unsigned long *p3, unsigned long *p4, unsigned long *p5)
+{
+        unsigned long lines = bytes >> 8;
+
+        kernel_fpu_begin();
+
+        asm volatile(
+#undef BLOCK
+#define BLOCK(i) \
+                PF1(i)                                  \
+                                PF1(i + 2)              \
+                LD(i, 0)                                \
+                        LD(i + 1, 1)                    \
+                                LD(i + 2, 2)            \
+                                        LD(i + 3, 3)    \
+                PF2(i)                                  \
+                                PF2(i + 2)              \
+                XO1(i, 0)                               \
+                        XO1(i + 1, 1)                   \
+                                XO1(i + 2, 2)           \
+                                        XO1(i + 3, 3)   \
+                PF3(i)                                  \
+                                PF3(i + 2)              \
+                XO2(i, 0)                               \
+                        XO2(i + 1, 1)                   \
+                                XO2(i + 2, 2)           \
+                                        XO2(i + 3, 3)   \
+                PF4(i)                                  \
+                                PF4(i + 2)              \
+                PF0(i + 4)                              \
+                                PF0(i + 6)              \
+                XO3(i, 0)                               \
+                        XO3(i + 1, 1)                   \
+                                XO3(i + 2, 2)           \
+                                        XO3(i + 3, 3)   \
+                XO4(i, 0)                               \
+                        XO4(i + 1, 1)                   \
+                                XO4(i + 2, 2)           \
+                                        XO4(i + 3, 3)   \
+                ST(i, 0)                                \
+                        ST(i + 1, 1)                    \
+                                ST(i + 2, 2)            \
+                                        ST(i + 3, 3)    \
+
+
+                PF0(0)
+                                PF0(2)
+
+        " .align 32                     ;\n"
+        " 1:                            ;\n"
+
+                BLOCK(0)
+                BLOCK(4)
+                BLOCK(8)
+                BLOCK(12)
+
+        "       add %[inc], %[p1]       ;\n"
+        "       add %[inc], %[p2]       ;\n"
+        "       add %[inc], %[p3]       ;\n"
+        "       add %[inc], %[p4]       ;\n"
+        "       add %[inc], %[p5]       ;\n"
+        "       dec %[cnt]              ;\n"
+        "       jnz 1b                  ;\n"
+        : [cnt] "+r" (lines), [p1] "+r" (p1), [p2] "+r" (p2),
+          [p3] "+r" (p3), [p4] "+r" (p4), [p5] "+r" (p5)
+        : [inc] XOR_CONSTANT_CONSTRAINT (256UL)
+        : "memory");
+
+        kernel_fpu_end();
+}
+
+static void
+xor_sse_5_pf64(unsigned long bytes, unsigned long *p1, unsigned long *p2,
+               unsigned long *p3, unsigned long *p4, unsigned long *p5)
+{
+        unsigned long lines = bytes >> 8;
+
+        kernel_fpu_begin();
+
+        asm volatile(
+#undef BLOCK
+#define BLOCK(i)                        \
+                BLK64(PF0, LD, i)       \
+                BLK64(PF1, XO1, i)      \
+                BLK64(PF2, XO2, i)      \
+                BLK64(PF3, XO3, i)      \
+                BLK64(PF4, XO4, i)      \
+                BLK64(NOP, ST, i)       \
+
+        " .align 32                     ;\n"
+        " 1:                            ;\n"
+
+                BLOCK(0)
+                BLOCK(4)
+                BLOCK(8)
+                BLOCK(12)
+
+        "       add %[inc], %[p1]       ;\n"
+        "       add %[inc], %[p2]       ;\n"
+        "       add %[inc], %[p3]       ;\n"
+        "       add %[inc], %[p4]       ;\n"
+        "       add %[inc], %[p5]       ;\n"
+        "       dec %[cnt]              ;\n"
+        "       jnz 1b                  ;\n"
+        : [cnt] "+r" (lines), [p1] "+r" (p1), [p2] "+r" (p2),
+          [p3] "+r" (p3), [p4] "+r" (p4), [p5] "+r" (p5)
+        : [inc] XOR_CONSTANT_CONSTRAINT (256UL)
+        : "memory");
+
+        kernel_fpu_end();
+}
+
+static struct xor_block_template xor_block_sse_pf64 = {
+        .name = "prefetch64-sse",
+        .do_2 = xor_sse_2_pf64,
+        .do_3 = xor_sse_3_pf64,
+        .do_4 = xor_sse_4_pf64,
+        .do_5 = xor_sse_5_pf64,
+};
+
+#undef LD
+#undef XO1
+#undef XO2
+#undef XO3
+#undef XO4
+#undef ST
+#undef NOP
+#undef BLK64
+#undef BLOCK
+
+#undef XOR_CONSTANT_CONSTRAINT
+
 #ifdef CONFIG_X86_32
 # include <asm/xor_32.h>
 #else
 # include <asm/xor_64.h>
 #endif
-#endif
+
+#define XOR_SELECT_TEMPLATE(FASTEST) \
+        AVX_SELECT(FASTEST)
+
+#endif /* _ASM_X86_XOR_H */
diff --git a/arch/x86/include/asm/xor_32.h b/arch/x86/include/asm/xor_32.h
index f79cb7ec0e06..ce05722e3c68 100644
--- a/arch/x86/include/asm/xor_32.h
+++ b/arch/x86/include/asm/xor_32.h
@@ -2,7 +2,7 @@
 #define _ASM_X86_XOR_32_H
 
 /*
- * Optimized RAID-5 checksumming functions for MMX and SSE.
+ * Optimized RAID-5 checksumming functions for MMX.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -529,290 +529,6 @@ static struct xor_block_template xor_block_p5_mmx = {
         .do_5 = xor_p5_mmx_5,
 };
 
-/*
- * Cache avoiding checksumming functions utilizing KNI instructions
- * Copyright (C) 1999 Zach Brown (with obvious credit due Ingo)
- */
-
-#define OFFS(x)         "16*("#x")"
-#define PF_OFFS(x)      "256+16*("#x")"
-#define PF0(x)          "       prefetchnta "PF_OFFS(x)"(%1)            ;\n"
-#define LD(x, y)        "       movaps   "OFFS(x)"(%1), %%xmm"#y"       ;\n"
-#define ST(x, y)        "       movaps %%xmm"#y",   "OFFS(x)"(%1)       ;\n"
-#define PF1(x)          "       prefetchnta "PF_OFFS(x)"(%2)            ;\n"
-#define PF2(x)          "       prefetchnta "PF_OFFS(x)"(%3)            ;\n"
-#define PF3(x)          "       prefetchnta "PF_OFFS(x)"(%4)            ;\n"
-#define PF4(x)          "       prefetchnta "PF_OFFS(x)"(%5)            ;\n"
-#define PF5(x)          "       prefetchnta "PF_OFFS(x)"(%6)            ;\n"
-#define XO1(x, y)       "       xorps   "OFFS(x)"(%2), %%xmm"#y"        ;\n"
-#define XO2(x, y)       "       xorps   "OFFS(x)"(%3), %%xmm"#y"        ;\n"
-#define XO3(x, y)       "       xorps   "OFFS(x)"(%4), %%xmm"#y"        ;\n"
-#define XO4(x, y)       "       xorps   "OFFS(x)"(%5), %%xmm"#y"        ;\n"
-#define XO5(x, y)       "       xorps   "OFFS(x)"(%6), %%xmm"#y"        ;\n"
-
-
-static void
-xor_sse_2(unsigned long bytes, unsigned long *p1, unsigned long *p2)
-{
-        unsigned long lines = bytes >> 8;
-
-        kernel_fpu_begin();
-
-        asm volatile(
-#undef BLOCK
-#define BLOCK(i)                                        \
-                LD(i, 0)                                \
-                        LD(i + 1, 1)                    \
-                PF1(i)                                  \
-                                PF1(i + 2)              \
-                                LD(i + 2, 2)            \
-                                        LD(i + 3, 3)    \
-                PF0(i + 4)                              \
-                                PF0(i + 6)              \
-                XO1(i, 0)                               \
-                        XO1(i + 1, 1)                   \
-                                XO1(i + 2, 2)           \
-                                        XO1(i + 3, 3)   \
-                ST(i, 0)                                \
-                        ST(i + 1, 1)                    \
-                                ST(i + 2, 2)            \
-                                        ST(i + 3, 3)    \
-
-
-                PF0(0)
-                                PF0(2)
-
-        " .align 32                     ;\n"
-        " 1:                            ;\n"
-
-                BLOCK(0)
-                BLOCK(4)
-                BLOCK(8)
-                BLOCK(12)
-
-        "       addl $256, %1           ;\n"
-        "       addl $256, %2           ;\n"
-        "       decl %0                 ;\n"
-        "       jnz 1b                  ;\n"
-        : "+r" (lines),
-          "+r" (p1), "+r" (p2)
-        :
-        : "memory");
-
-        kernel_fpu_end();
-}
-
-static void
-xor_sse_3(unsigned long bytes, unsigned long *p1, unsigned long *p2,
-          unsigned long *p3)
-{
-        unsigned long lines = bytes >> 8;
-
-        kernel_fpu_begin();
-
-        asm volatile(
-#undef BLOCK
-#define BLOCK(i) \
-                PF1(i)                                  \
-                                PF1(i + 2)              \
-                LD(i,0)                                 \
-                        LD(i + 1, 1)                    \
-                                LD(i + 2, 2)            \
-                                        LD(i + 3, 3)    \
-                PF2(i)                                  \
-                                PF2(i + 2)              \
-                PF0(i + 4)                              \
-                                PF0(i + 6)              \
-                XO1(i,0)                                \
-                        XO1(i + 1, 1)                   \
-                                XO1(i + 2, 2)           \
-                                        XO1(i + 3, 3)   \
-                XO2(i,0)                                \
-                        XO2(i + 1, 1)                   \
-                                XO2(i + 2, 2)           \
-                                        XO2(i + 3, 3)   \
-                ST(i,0)                                 \
-                        ST(i + 1, 1)                    \
-                                ST(i + 2, 2)            \
-                                        ST(i + 3, 3)    \
-
-
-                PF0(0)
-                                PF0(2)
-
-        " .align 32                     ;\n"
-        " 1:                            ;\n"
-
-                BLOCK(0)
-                BLOCK(4)
-                BLOCK(8)
-                BLOCK(12)
-
-        "       addl $256, %1           ;\n"
-        "       addl $256, %2           ;\n"
-        "       addl $256, %3           ;\n"
-        "       decl %0                 ;\n"
-        "       jnz 1b                  ;\n"
-        : "+r" (lines),
-          "+r" (p1), "+r"(p2), "+r"(p3)
-        :
-        : "memory" );
-
-        kernel_fpu_end();
-}
-
-static void
-xor_sse_4(unsigned long bytes, unsigned long *p1, unsigned long *p2,
-          unsigned long *p3, unsigned long *p4)
-{
-        unsigned long lines = bytes >> 8;
-
-        kernel_fpu_begin();
-
-        asm volatile(
-#undef BLOCK
-#define BLOCK(i) \
-                PF1(i)                                  \
-                                PF1(i + 2)              \
-                LD(i,0)                                 \
-                        LD(i + 1, 1)                    \
-                                LD(i + 2, 2)            \
-                                        LD(i + 3, 3)    \
-                PF2(i)                                  \
-                                PF2(i + 2)              \
-                XO1(i,0)                                \
-                        XO1(i + 1, 1)                   \
-                                XO1(i + 2, 2)           \
-                                        XO1(i + 3, 3)   \
-                PF3(i)                                  \
-                                PF3(i + 2)              \
-                PF0(i + 4)                              \
-                                PF0(i + 6)              \
-                XO2(i,0)                                \
-                        XO2(i + 1, 1)                   \
-                                XO2(i + 2, 2)           \
-                                        XO2(i + 3, 3)   \
-                XO3(i,0)                                \
-                        XO3(i + 1, 1)                   \
-                                XO3(i + 2, 2)           \
-                                        XO3(i + 3, 3)   \
-                ST(i,0)                                 \
-                        ST(i + 1, 1)                    \
-                                ST(i + 2, 2)            \
-                                        ST(i + 3, 3)    \
-
-
-                PF0(0)
-                                PF0(2)
-
-        " .align 32                     ;\n"
-        " 1:                            ;\n"
-
-                BLOCK(0)
-                BLOCK(4)
-                BLOCK(8)
-                BLOCK(12)
-
-        "       addl $256, %1           ;\n"
-        "       addl $256, %2           ;\n"
-        "       addl $256, %3           ;\n"
-        "       addl $256, %4           ;\n"
-        "       decl %0                 ;\n"
-        "       jnz 1b                  ;\n"
-        : "+r" (lines),
-          "+r" (p1), "+r" (p2), "+r" (p3), "+r" (p4)
-        :
-        : "memory" );
-
-        kernel_fpu_end();
-}
-
-static void
-xor_sse_5(unsigned long bytes, unsigned long *p1, unsigned long *p2,
-          unsigned long *p3, unsigned long *p4, unsigned long *p5)
-{
-        unsigned long lines = bytes >> 8;
-
-        kernel_fpu_begin();
-
-        /* Make sure GCC forgets anything it knows about p4 or p5,
-           such that it won't pass to the asm volatile below a
-           register that is shared with any other variable.  That's
-           because we modify p4 and p5 there, but we can't mark them
-           as read/write, otherwise we'd overflow the 10-asm-operands
-           limit of GCC < 3.1.  */
-        asm("" : "+r" (p4), "+r" (p5));
-
-        asm volatile(
-#undef BLOCK
-#define BLOCK(i) \
-                PF1(i)                                  \
-                                PF1(i + 2)              \
-                LD(i,0)                                 \
-                        LD(i + 1, 1)                    \
-                                LD(i + 2, 2)            \
-                                        LD(i + 3, 3)    \
-                PF2(i)                                  \
-                                PF2(i + 2)              \
-                XO1(i,0)                                \
-                        XO1(i + 1, 1)                   \
-                                XO1(i + 2, 2)           \
-                                        XO1(i + 3, 3)   \
-                PF3(i)                                  \
-                                PF3(i + 2)              \
-                XO2(i,0)                                \
-                        XO2(i + 1, 1)                   \
-                                XO2(i + 2, 2)           \
-                                        XO2(i + 3, 3)   \
-                PF4(i)                                  \
-                                PF4(i + 2)              \
-                PF0(i + 4)                              \
-                                PF0(i + 6)              \
-                XO3(i,0)                                \
-                        XO3(i + 1, 1)                   \
-                                XO3(i + 2, 2)           \
-                                        XO3(i + 3, 3)   \
-                XO4(i,0)                                \
-                        XO4(i + 1, 1)                   \
-                                XO4(i + 2, 2)           \
-                                        XO4(i + 3, 3)   \
-                ST(i,0)                                 \
-                        ST(i + 1, 1)                    \
-                                ST(i + 2, 2)            \
-                                        ST(i + 3, 3)    \
-
-
-                PF0(0)
-                                PF0(2)
-
-        " .align 32                     ;\n"
-        " 1:                            ;\n"
-
-                BLOCK(0)
-                BLOCK(4)
-                BLOCK(8)
-                BLOCK(12)
-
-        "       addl $256, %1           ;\n"
-        "       addl $256, %2           ;\n"
-        "       addl $256, %3           ;\n"
-        "       addl $256, %4           ;\n"
-        "       addl $256, %5           ;\n"
-        "       decl %0                 ;\n"
-        "       jnz 1b                  ;\n"
-        : "+r" (lines),
-          "+r" (p1), "+r" (p2), "+r" (p3)
-        : "r" (p4), "r" (p5)
-        : "memory");
-
-        /* p4 and p5 were modified, and now the variables are dead.
-           Clobber them just to be sure nobody does something stupid
-           like assuming they have some legal value.  */
-        asm("" : "=r" (p4), "=r" (p5));
-
-        kernel_fpu_end();
-}
-
 static struct xor_block_template xor_block_pIII_sse = {
         .name = "pIII_sse",
         .do_2 = xor_sse_2,
@@ -827,26 +543,25 @@ static struct xor_block_template xor_block_pIII_sse = {
 /* Also try the generic routines.  */
 #include <asm-generic/xor.h>
 
+/* We force the use of the SSE xor block because it can write around L2.
+   We may also be able to load into the L1 only depending on how the cpu
+   deals with a load to a line that is being prefetched.  */
 #undef XOR_TRY_TEMPLATES
 #define XOR_TRY_TEMPLATES                               \
 do {                                                    \
-        xor_speed(&xor_block_8regs);                    \
-        xor_speed(&xor_block_8regs_p);                  \
-        xor_speed(&xor_block_32regs);                   \
-        xor_speed(&xor_block_32regs_p);                 \
         AVX_XOR_SPEED;                                  \
-        if (cpu_has_xmm)                                \
+        if (cpu_has_xmm) {                              \
                 xor_speed(&xor_block_pIII_sse);         \
-        if (cpu_has_mmx) {                              \
+                xor_speed(&xor_block_sse_pf64);         \
+        } else if (cpu_has_mmx) {                       \
                 xor_speed(&xor_block_pII_mmx);          \
                 xor_speed(&xor_block_p5_mmx);           \
+        } else {                                        \
+                xor_speed(&xor_block_8regs);            \
+                xor_speed(&xor_block_8regs_p);          \
+                xor_speed(&xor_block_32regs);           \
+                xor_speed(&xor_block_32regs_p);         \
         }                                               \
 } while (0)
 
-/* We force the use of the SSE xor block because it can write around L2.
-   We may also be able to load into the L1 only depending on how the cpu
-   deals with a load to a line that is being prefetched.  */
-#define XOR_SELECT_TEMPLATE(FASTEST)                    \
-        AVX_SELECT(cpu_has_xmm ? &xor_block_pIII_sse : FASTEST)
-
 #endif /* _ASM_X86_XOR_32_H */
diff --git a/arch/x86/include/asm/xor_64.h b/arch/x86/include/asm/xor_64.h
index 87ac522c4af5..546f1e3b87cc 100644
--- a/arch/x86/include/asm/xor_64.h
+++ b/arch/x86/include/asm/xor_64.h
@@ -1,301 +1,6 @@
 #ifndef _ASM_X86_XOR_64_H
 #define _ASM_X86_XOR_64_H
 
-/*
- * Optimized RAID-5 checksumming functions for MMX and SSE.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2, or (at your option)
- * any later version.
- *
- * You should have received a copy of the GNU General Public License
- * (for example /usr/src/linux/COPYING); if not, write to the Free
- * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
- */
-
-
-/*
- * Cache avoiding checksumming functions utilizing KNI instructions
- * Copyright (C) 1999 Zach Brown (with obvious credit due Ingo)
- */
-
-/*
- * Based on
- * High-speed RAID5 checksumming functions utilizing SSE instructions.
- * Copyright (C) 1998 Ingo Molnar.
- */
-
-/*
- * x86-64 changes / gcc fixes from Andi Kleen.
- * Copyright 2002 Andi Kleen, SuSE Labs.
- *
- * This hasn't been optimized for the hammer yet, but there are likely
- * no advantages to be gotten from x86-64 here anyways.
- */
-
-#include <asm/i387.h>
-
-#define OFFS(x)         "16*("#x")"
-#define PF_OFFS(x)      "256+16*("#x")"
-#define PF0(x)          "       prefetchnta "PF_OFFS(x)"(%[p1])         ;\n"
-#define LD(x, y)        "       movaps   "OFFS(x)"(%[p1]), %%xmm"#y"    ;\n"
-#define ST(x, y)        "       movaps %%xmm"#y",   "OFFS(x)"(%[p1])    ;\n"
-#define PF1(x)          "       prefetchnta "PF_OFFS(x)"(%[p2])         ;\n"
-#define PF2(x)          "       prefetchnta "PF_OFFS(x)"(%[p3])         ;\n"
-#define PF3(x)          "       prefetchnta "PF_OFFS(x)"(%[p4])         ;\n"
-#define PF4(x)          "       prefetchnta "PF_OFFS(x)"(%[p5])         ;\n"
-#define PF5(x)          "       prefetchnta "PF_OFFS(x)"(%[p6])         ;\n"
-#define XO1(x, y)       "       xorps   "OFFS(x)"(%[p2]), %%xmm"#y"     ;\n"
-#define XO2(x, y)       "       xorps   "OFFS(x)"(%[p3]), %%xmm"#y"     ;\n"
-#define XO3(x, y)       "       xorps   "OFFS(x)"(%[p4]), %%xmm"#y"     ;\n"
-#define XO4(x, y)       "       xorps   "OFFS(x)"(%[p5]), %%xmm"#y"     ;\n"
-#define XO5(x, y)       "       xorps   "OFFS(x)"(%[p6]), %%xmm"#y"     ;\n"
-
-
-static void
-xor_sse_2(unsigned long bytes, unsigned long *p1, unsigned long *p2)
-{
-        unsigned int lines = bytes >> 8;
-
-        kernel_fpu_begin();
-
-        asm volatile(
-#undef BLOCK
-#define BLOCK(i) \
-                LD(i, 0)                                \
-                        LD(i + 1, 1)                    \
-                PF1(i)                                  \
-                                PF1(i + 2)              \
-                                LD(i + 2, 2)            \
-                                        LD(i + 3, 3)    \
-                PF0(i + 4)                              \
-                                PF0(i + 6)              \
-                XO1(i, 0)                               \
-                        XO1(i + 1, 1)                   \
-                                XO1(i + 2, 2)           \
-                                        XO1(i + 3, 3)   \
-                ST(i, 0)                                \
-                        ST(i + 1, 1)                    \
-                                ST(i + 2, 2)            \
-                                        ST(i + 3, 3)    \
-
-
-                PF0(0)
-                                PF0(2)
-
-        " .align 32                     ;\n"
-        " 1:                            ;\n"
-
-                BLOCK(0)
-                BLOCK(4)
-                BLOCK(8)
-                BLOCK(12)
-
-        "       addq %[inc], %[p1]           ;\n"
-        "       addq %[inc], %[p2]           ;\n"
-                "               decl %[cnt] ; jnz 1b"
-        : [p1] "+r" (p1), [p2] "+r" (p2), [cnt] "+r" (lines)
-        : [inc] "r" (256UL)
-        : "memory");
-
-        kernel_fpu_end();
-}
-
-static void
-xor_sse_3(unsigned long bytes, unsigned long *p1, unsigned long *p2,
-          unsigned long *p3)
-{
-        unsigned int lines = bytes >> 8;
-
-        kernel_fpu_begin();
-        asm volatile(
-#undef BLOCK
-#define BLOCK(i) \
-                PF1(i)                                  \
-                                PF1(i + 2)              \
-                LD(i, 0)                                        \
-                        LD(i + 1, 1)                    \
-                                LD(i + 2, 2)            \
-                                        LD(i + 3, 3)    \
-                PF2(i)                                  \
-                                PF2(i + 2)              \
-                PF0(i + 4)                              \
-                                PF0(i + 6)              \
-                XO1(i, 0)                               \
-                        XO1(i + 1, 1)                   \
-                                XO1(i + 2, 2)           \
-                                        XO1(i + 3, 3)   \
-                XO2(i, 0)                               \
-                        XO2(i + 1, 1)                   \
-                                XO2(i + 2, 2)           \
-                                        XO2(i + 3, 3)   \
-                ST(i, 0)                                \
-                        ST(i + 1, 1)                    \
-                                ST(i + 2, 2)            \
-                                        ST(i + 3, 3)    \
-
-
-                PF0(0)
-                                PF0(2)
-
-        " .align 32                     ;\n"
-        " 1:                            ;\n"
-
-                BLOCK(0)
-                BLOCK(4)
-                BLOCK(8)
-                BLOCK(12)
-
-        "       addq %[inc], %[p1]           ;\n"
-        "       addq %[inc], %[p2]          ;\n"
-        "       addq %[inc], %[p3]           ;\n"
-                "               decl %[cnt] ; jnz 1b"
-        : [cnt] "+r" (lines),
-          [p1] "+r" (p1), [p2] "+r" (p2), [p3] "+r" (p3)
-        : [inc] "r" (256UL)
-        : "memory");
-        kernel_fpu_end();
-}
-
-static void
-xor_sse_4(unsigned long bytes, unsigned long *p1, unsigned long *p2,
-          unsigned long *p3, unsigned long *p4)
-{
-        unsigned int lines = bytes >> 8;
-
-        kernel_fpu_begin();
-
-        asm volatile(
-#undef BLOCK
-#define BLOCK(i) \
-                PF1(i)                                  \
-                                PF1(i + 2)              \
-                LD(i, 0)                                \
-                        LD(i + 1, 1)                    \
-                                LD(i + 2, 2)            \
-                                        LD(i + 3, 3)    \
-                PF2(i)                                  \
-                                PF2(i + 2)              \
-                XO1(i, 0)                               \
-                        XO1(i + 1, 1)                   \
-                                XO1(i + 2, 2)           \
-                                        XO1(i + 3, 3)   \
-                PF3(i)                                  \
-                                PF3(i + 2)              \
-                PF0(i + 4)                              \
-                                PF0(i + 6)              \
-                XO2(i, 0)                               \
-                        XO2(i + 1, 1)                   \
-                                XO2(i + 2, 2)           \
-                                        XO2(i + 3, 3)   \
-                XO3(i, 0)                               \
-                        XO3(i + 1, 1)                   \
-                                XO3(i + 2, 2)           \
-                                        XO3(i + 3, 3)   \
-                ST(i, 0)                                \
-                        ST(i + 1, 1)                    \
-                                ST(i + 2, 2)            \
-                                        ST(i + 3, 3)    \
-
-
-                PF0(0)
-                                PF0(2)
-
-        " .align 32                     ;\n"
-        " 1:                            ;\n"
-
-                BLOCK(0)
-                BLOCK(4)
-                BLOCK(8)
-                BLOCK(12)
-
-        "       addq %[inc], %[p1]           ;\n"
-        "       addq %[inc], %[p2]           ;\n"
-        "       addq %[inc], %[p3]           ;\n"
-        "       addq %[inc], %[p4]           ;\n"
-        "       decl %[cnt] ; jnz 1b"
-        : [cnt] "+c" (lines),
-          [p1] "+r" (p1), [p2] "+r" (p2), [p3] "+r" (p3), [p4] "+r" (p4)
-        : [inc] "r" (256UL)
-        : "memory" );
-
-        kernel_fpu_end();
-}
-
-static void
-xor_sse_5(unsigned long bytes, unsigned long *p1, unsigned long *p2,
-          unsigned long *p3, unsigned long *p4, unsigned long *p5)
-{
-        unsigned int lines = bytes >> 8;
-
-        kernel_fpu_begin();
-
-        asm volatile(
-#undef BLOCK
-#define BLOCK(i) \
-                PF1(i)                                  \
-                                PF1(i + 2)              \
-                LD(i, 0)                                \
-                        LD(i + 1, 1)                    \
-                                LD(i + 2, 2)            \
-                                        LD(i + 3, 3)    \
-                PF2(i)                                  \
-                                PF2(i + 2)              \
-                XO1(i, 0)                               \
-                        XO1(i + 1, 1)                   \
-                                XO1(i + 2, 2)           \
-                                        XO1(i + 3, 3)   \
-                PF3(i)                                  \
-                                PF3(i + 2)              \
-                XO2(i, 0)                               \
-                        XO2(i + 1, 1)                   \
-                                XO2(i + 2, 2)           \
-                                        XO2(i + 3, 3)   \
-                PF4(i)                                  \
-                                PF4(i + 2)              \
-                PF0(i + 4)                              \
-                                PF0(i + 6)              \
-                XO3(i, 0)                               \
-                        XO3(i + 1, 1)                   \
-                                XO3(i + 2, 2)           \
-                                        XO3(i + 3, 3)   \
-                XO4(i, 0)                               \
-                        XO4(i + 1, 1)                   \
-                                XO4(i + 2, 2)           \
-                                        XO4(i + 3, 3)   \
-                ST(i, 0)                                \
-                        ST(i + 1, 1)                    \
-                                ST(i + 2, 2)            \
-                                        ST(i + 3, 3)    \
-
-
-                PF0(0)
-                                PF0(2)
-
-        " .align 32                     ;\n"
-        " 1:                            ;\n"
-
-                BLOCK(0)
-                BLOCK(4)
-                BLOCK(8)
-                BLOCK(12)
-
-        "       addq %[inc], %[p1]           ;\n"
-        "       addq %[inc], %[p2]           ;\n"
-        "       addq %[inc], %[p3]           ;\n"
-        "       addq %[inc], %[p4]           ;\n"
-        "       addq %[inc], %[p5]           ;\n"
-        "       decl %[cnt] ; jnz 1b"
-        : [cnt] "+c" (lines),
-          [p1] "+r" (p1), [p2] "+r" (p2), [p3] "+r" (p3), [p4] "+r" (p4),
-          [p5] "+r" (p5)
-        : [inc] "r" (256UL)
-        : "memory");
-
-        kernel_fpu_end();
-}
-
 static struct xor_block_template xor_block_sse = {
         .name = "generic_sse",
         .do_2 = xor_sse_2,
@@ -308,17 +13,15 @@ static struct xor_block_template xor_block_sse = {
 /* Also try the AVX routines */
 #include <asm/xor_avx.h>
 
+/* We force the use of the SSE xor block because it can write around L2.
+   We may also be able to load into the L1 only depending on how the cpu
+   deals with a load to a line that is being prefetched.  */
 #undef XOR_TRY_TEMPLATES
 #define XOR_TRY_TEMPLATES                       \
 do {                                            \
         AVX_XOR_SPEED;                          \
+        xor_speed(&xor_block_sse_pf64);         \
         xor_speed(&xor_block_sse);              \
 } while (0)
 
-/* We force the use of the SSE xor block because it can write around L2.
-   We may also be able to load into the L1 only depending on how the cpu
-   deals with a load to a line that is being prefetched.  */
-#define XOR_SELECT_TEMPLATE(FASTEST) \
-        AVX_SELECT(&xor_block_sse)
-
 #endif /* _ASM_X86_XOR_64_H */
diff --git a/arch/x86/include/uapi/asm/mce.h b/arch/x86/include/uapi/asm/mce.h
index 58c829871c31..a0eab85ce7b8 100644
--- a/arch/x86/include/uapi/asm/mce.h
+++ b/arch/x86/include/uapi/asm/mce.h
@@ -4,66 +4,6 @@
 #include <linux/types.h>
 #include <asm/ioctls.h>
 
-/*
- * Machine Check support for x86
- */
-
-/* MCG_CAP register defines */
-#define MCG_BANKCNT_MASK        0xff         /* Number of Banks */
-#define MCG_CTL_P               (1ULL<<8)    /* MCG_CTL register available */
-#define MCG_EXT_P               (1ULL<<9)    /* Extended registers available */
-#define MCG_CMCI_P              (1ULL<<10)   /* CMCI supported */
-#define MCG_EXT_CNT_MASK        0xff0000     /* Number of Extended registers */
-#define MCG_EXT_CNT_SHIFT       16
-#define MCG_EXT_CNT(c)          (((c) & MCG_EXT_CNT_MASK) >> MCG_EXT_CNT_SHIFT)
-#define MCG_SER_P               (1ULL<<24)   /* MCA recovery/new status bits */
-
-/* MCG_STATUS register defines */
-#define MCG_STATUS_RIPV  (1ULL<<0)   /* restart ip valid */
-#define MCG_STATUS_EIPV  (1ULL<<1)   /* ip points to correct instruction */
-#define MCG_STATUS_MCIP  (1ULL<<2)   /* machine check in progress */
-
-/* MCi_STATUS register defines */
-#define MCI_STATUS_VAL   (1ULL<<63)  /* valid error */
-#define MCI_STATUS_OVER  (1ULL<<62)  /* previous errors lost */
-#define MCI_STATUS_UC    (1ULL<<61)  /* uncorrected error */
-#define MCI_STATUS_EN    (1ULL<<60)  /* error enabled */
-#define MCI_STATUS_MISCV (1ULL<<59)  /* misc error reg. valid */
-#define MCI_STATUS_ADDRV (1ULL<<58)  /* addr reg. valid */
-#define MCI_STATUS_PCC   (1ULL<<57)  /* processor context corrupt */
-#define MCI_STATUS_S     (1ULL<<56)  /* Signaled machine check */
-#define MCI_STATUS_AR    (1ULL<<55)  /* Action required */
-#define MCACOD            0xffff     /* MCA Error Code */
-
-/* Architecturally defined codes from SDM Vol. 3B Chapter 15 */
-#define MCACOD_SCRUB    0x00C0  /* 0xC0-0xCF Memory Scrubbing */
-#define MCACOD_SCRUBMSK 0xfff0
-#define MCACOD_L3WB     0x017A  /* L3 Explicit Writeback */
-#define MCACOD_DATA     0x0134  /* Data Load */
-#define MCACOD_INSTR    0x0150  /* Instruction Fetch */
-
-/* MCi_MISC register defines */
-#define MCI_MISC_ADDR_LSB(m)    ((m) & 0x3f)
-#define MCI_MISC_ADDR_MODE(m)   (((m) >> 6) & 7)
-#define  MCI_MISC_ADDR_SEGOFF   0       /* segment offset */
-#define  MCI_MISC_ADDR_LINEAR   1       /* linear address */
-#define  MCI_MISC_ADDR_PHYS     2       /* physical address */
-#define  MCI_MISC_ADDR_MEM      3       /* memory address */
-#define  MCI_MISC_ADDR_GENERIC  7       /* generic */
-
-/* CTL2 register defines */
-#define MCI_CTL2_CMCI_EN                (1ULL << 30)
-#define MCI_CTL2_CMCI_THRESHOLD_MASK    0x7fffULL
-
-#define MCJ_CTX_MASK            3
-#define MCJ_CTX(flags)          ((flags) & MCJ_CTX_MASK)
-#define MCJ_CTX_RANDOM          0    /* inject context: random */
-#define MCJ_CTX_PROCESS         0x1  /* inject context: process */
-#define MCJ_CTX_IRQ             0x2  /* inject context: IRQ */
-#define MCJ_NMI_BROADCAST       0x4  /* do NMI broadcasting */
-#define MCJ_EXCEPTION           0x8  /* raise as exception */
-#define MCJ_IRQ_BRAODCAST       0x10 /* do IRQ broadcasting */
-
 /* Fields are zero when not available */
 struct mce {
         __u64 status;
@@ -87,35 +27,8 @@ struct mce {
         __u64 mcgcap;   /* MCGCAP MSR: machine check capabilities of CPU */
 };
 
-/*
- * This structure contains all data related to the MCE log.  Also
- * carries a signature to make it easier to find from external
- * debugging tools.  Each entry is only valid when its finished flag
- * is set.
- */
-
-#define MCE_LOG_LEN 32
-
-struct mce_log {
-        char signature[12]; /* "MACHINECHECK" */
-        unsigned len;       /* = MCE_LOG_LEN */
-        unsigned next;
-        unsigned flags;
-        unsigned recordlen;     /* length of struct mce */
-        struct mce entry[MCE_LOG_LEN];
-};
-
-#define MCE_OVERFLOW 0          /* bit 0 in flags means overflow */
-
-#define MCE_LOG_SIGNATURE       "MACHINECHECK"
-
 #define MCE_GET_RECORD_LEN   _IOR('M', 1, int)
 #define MCE_GET_LOG_LEN      _IOR('M', 2, int)
 #define MCE_GETCLEAR_FLAGS   _IOR('M', 3, int)
 
-/* Software defined banks */
-#define MCE_EXTENDED_BANK       128
-#define MCE_THERMAL_BANK        MCE_EXTENDED_BANK + 0
-#define K8_MCE_THRESHOLD_BASE      (MCE_EXTENDED_BANK + 1)
-
 #endif /* _UAPI_ASM_X86_MCE_H */
diff --git a/arch/x86/include/uapi/asm/msr-index.h b/arch/x86/include/uapi/asm/msr-index.h
index 433a59fb1a74..7a060f4b411f 100644
--- a/arch/x86/include/uapi/asm/msr-index.h
+++ b/arch/x86/include/uapi/asm/msr-index.h
@@ -44,6 +44,7 @@
 #define SNB_C1_AUTO_UNDEMOTE            (1UL << 27)
 #define SNB_C3_AUTO_UNDEMOTE            (1UL << 28)
 
+#define MSR_PLATFORM_INFO               0x000000ce
 #define MSR_MTRRcap                     0x000000fe
 #define MSR_IA32_BBL_CR_CTL             0x00000119
 #define MSR_IA32_BBL_CR_CTL3            0x0000011e
@@ -103,6 +104,8 @@
 #define DEBUGCTLMSR_BTS_OFF_USR         (1UL << 10)
 #define DEBUGCTLMSR_FREEZE_LBRS_ON_PMI  (1UL << 11)
 
+#define MSR_IA32_POWER_CTL              0x000001fc
+
 #define MSR_IA32_MC0_CTL                0x00000400
 #define MSR_IA32_MC0_STATUS             0x00000401
 #define MSR_IA32_MC0_ADDR               0x00000402
@@ -173,6 +176,7 @@
 #define MSR_AMD64_OSVW_ID_LENGTH        0xc0010140
 #define MSR_AMD64_OSVW_STATUS           0xc0010141
 #define MSR_AMD64_DC_CFG                0xc0011022
+#define MSR_AMD64_BU_CFG2               0xc001102a
 #define MSR_AMD64_IBSFETCHCTL           0xc0011030
 #define MSR_AMD64_IBSFETCHLINAD         0xc0011031
 #define MSR_AMD64_IBSFETCHPHYSAD        0xc0011032
@@ -194,6 +198,8 @@
 /* Fam 15h MSRs */
 #define MSR_F15H_PERF_CTL               0xc0010200
 #define MSR_F15H_PERF_CTR               0xc0010201
+#define MSR_F15H_NB_PERF_CTL            0xc0010240
+#define MSR_F15H_NB_PERF_CTR            0xc0010241
 
 /* Fam 10h MSRs */
 #define MSR_FAM10H_MMIO_CONF_BASE       0xc0010058
@@ -272,6 +278,7 @@
 #define MSR_IA32_PLATFORM_ID            0x00000017
 #define MSR_IA32_EBL_CR_POWERON         0x0000002a
 #define MSR_EBC_FREQUENCY_ID            0x0000002c
+#define MSR_SMI_COUNT                   0x00000034
 #define MSR_IA32_FEATURE_CONTROL        0x0000003a
 #define MSR_IA32_TSC_ADJUST             0x0000003b
 
diff --git a/arch/x86/include/uapi/asm/signal.h b/arch/x86/include/uapi/asm/signal.h
index aa7d6ae39e0e..8264f47cf53e 100644
--- a/arch/x86/include/uapi/asm/signal.h
+++ b/arch/x86/include/uapi/asm/signal.h
@@ -95,9 +95,9 @@ typedef unsigned long sigset_t;
 #ifndef __ASSEMBLY__
 
 
-#ifdef __i386__
 # ifndef __KERNEL__
 /* Here we must cater to libcs that poke about in kernel headers.  */
+#ifdef __i386__
 
 struct sigaction {
         union {
@@ -112,7 +112,6 @@ struct sigaction {
 #define sa_handler      _u._sa_handler
 #define sa_sigaction    _u._sa_sigaction
 
-# endif /* ! __KERNEL__ */
 #else /* __i386__ */
 
 struct sigaction {
@@ -122,11 +121,8 @@ struct sigaction {
         sigset_t sa_mask;               /* mask last for extensibility */
 };
 
-struct k_sigaction {
-        struct sigaction sa;
-};
-
 #endif /* !__i386__ */
+# endif /* ! __KERNEL__ */
 
 typedef struct sigaltstack {
         void __user *ss_sp;
diff --git a/arch/x86/include/uapi/asm/vmx.h b/arch/x86/include/uapi/asm/vmx.h
index 979d03bce135..2871fccfee68 100644
--- a/arch/x86/include/uapi/asm/vmx.h
+++ b/arch/x86/include/uapi/asm/vmx.h
@@ -62,10 +62,12 @@
 #define EXIT_REASON_MCE_DURING_VMENTRY  41
 #define EXIT_REASON_TPR_BELOW_THRESHOLD 43
 #define EXIT_REASON_APIC_ACCESS         44
+#define EXIT_REASON_EOI_INDUCED         45
 #define EXIT_REASON_EPT_VIOLATION       48
 #define EXIT_REASON_EPT_MISCONFIG       49
 #define EXIT_REASON_WBINVD              54
 #define EXIT_REASON_XSETBV              55
+#define EXIT_REASON_APIC_WRITE          56
 #define EXIT_REASON_INVPCID             58
 
 #define VMX_EXIT_REASONS \
@@ -103,7 +105,12 @@
         { EXIT_REASON_APIC_ACCESS,           "APIC_ACCESS" }, \
         { EXIT_REASON_EPT_VIOLATION,         "EPT_VIOLATION" }, \
         { EXIT_REASON_EPT_MISCONFIG,         "EPT_MISCONFIG" }, \
-        { EXIT_REASON_WBINVD,                "WBINVD" }
+        { EXIT_REASON_WBINVD,                "WBINVD" }, \
+        { EXIT_REASON_APIC_WRITE,            "APIC_WRITE" }, \
+        { EXIT_REASON_EOI_INDUCED,           "EOI_INDUCED" }, \
+        { EXIT_REASON_INVALID_STATE,         "INVALID_STATE" }, \
+        { EXIT_REASON_INVD,                  "INVD" }, \
+        { EXIT_REASON_INVPCID,               "INVPCID" }
 
 
 #endif /* _UAPIVMX_H */
diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile
index 34e923a53762..7bd3bd310106 100644
--- a/arch/x86/kernel/Makefile
+++ b/arch/x86/kernel/Makefile
@@ -65,8 +65,7 @@ obj-$(CONFIG_X86_TSC)		+= trace_clock.o
 obj-$(CONFIG_KEXEC)             += machine_kexec_$(BITS).o
 obj-$(CONFIG_KEXEC)             += relocate_kernel_$(BITS).o crash.o
 obj-$(CONFIG_CRASH_DUMP)        += crash_dump_$(BITS).o
-obj-$(CONFIG_KPROBES)           += kprobes.o
-obj-$(CONFIG_OPTPROBES)         += kprobes-opt.o
+obj-y                           += kprobes/
 obj-$(CONFIG_MODULES)           += module.o
 obj-$(CONFIG_DOUBLEFAULT)       += doublefault_32.o
 obj-$(CONFIG_KGDB)              += kgdb.o
@@ -88,6 +87,9 @@ obj-$(CONFIG_PARAVIRT_CLOCK)	+= pvclock.o
 
 obj-$(CONFIG_PCSPKR_PLATFORM)   += pcspeaker.o
 
+obj-$(CONFIG_MICROCODE_EARLY)           += microcode_core_early.o
+obj-$(CONFIG_MICROCODE_INTEL_EARLY)     += microcode_intel_early.o
+obj-$(CONFIG_MICROCODE_INTEL_LIB)       += microcode_intel_lib.o
 microcode-y                             := microcode_core.o
 microcode-$(CONFIG_MICROCODE_INTEL)     += microcode_intel.o
 microcode-$(CONFIG_MICROCODE_AMD)       += microcode_amd.o
diff --git a/arch/x86/kernel/acpi/boot.c b/arch/x86/kernel/acpi/boot.c
index bacf4b0d91f4..230c8ea878e5 100644
--- a/arch/x86/kernel/acpi/boot.c
+++ b/arch/x86/kernel/acpi/boot.c
@@ -51,7 +51,6 @@ EXPORT_SYMBOL(acpi_disabled);
 
 #ifdef  CONFIG_X86_64
 # include <asm/proto.h>
-# include <asm/numa_64.h>
 #endif                          /* X86 */
 
 #define BAD_MADT_ENTRY(entry, end) (                                        \
@@ -697,6 +696,10 @@ EXPORT_SYMBOL(acpi_map_lsapic);
 
 int acpi_unmap_lsapic(int cpu)
 {
+#ifdef CONFIG_ACPI_NUMA
+        set_apicid_to_node(per_cpu(x86_cpu_to_apicid, cpu), NUMA_NO_NODE);
+#endif
+
         per_cpu(x86_cpu_to_apicid, cpu) = -1;
         set_cpu_present(cpu, false);
         num_processors--;
diff --git a/arch/x86/kernel/acpi/sleep.c b/arch/x86/kernel/acpi/sleep.c
index d5e0d717005a..0532f5d6e4ef 100644
--- a/arch/x86/kernel/acpi/sleep.c
+++ b/arch/x86/kernel/acpi/sleep.c
@@ -69,7 +69,7 @@ int acpi_suspend_lowlevel(void)
 
 #ifndef CONFIG_64BIT
         header->pmode_entry = (u32)&wakeup_pmode_return;
-        header->pmode_cr3 = (u32)__pa(&initial_page_table);
+        header->pmode_cr3 = (u32)__pa_symbol(initial_page_table);
         saved_magic = 0x12345678;
 #else /* CONFIG_64BIT */
 #ifdef CONFIG_SMP
diff --git a/arch/x86/kernel/amd_gart_64.c b/arch/x86/kernel/amd_gart_64.c
index e66311200cbd..b574b295a2f9 100644
--- a/arch/x86/kernel/amd_gart_64.c
+++ b/arch/x86/kernel/amd_gart_64.c
@@ -768,10 +768,9 @@ int __init gart_iommu_init(void)
         aper_base       = info.aper_base;
         end_pfn         = (aper_base>>PAGE_SHIFT) + (aper_size>>PAGE_SHIFT);
 
-        if (end_pfn > max_low_pfn_mapped) {
-                start_pfn = (aper_base>>PAGE_SHIFT);
+        start_pfn = PFN_DOWN(aper_base);
+        if (!pfn_range_is_mapped(start_pfn, end_pfn))
                 init_memory_mapping(start_pfn<<PAGE_SHIFT, end_pfn<<PAGE_SHIFT);
-        }
 
         pr_info("PCI-DMA: using GART IOMMU.\n");
         iommu_size = check_iommu_size(info.aper_base, aper_size);
diff --git a/arch/x86/kernel/apb_timer.c b/arch/x86/kernel/apb_timer.c
index afdc3f756dea..c9876efecafb 100644
--- a/arch/x86/kernel/apb_timer.c
+++ b/arch/x86/kernel/apb_timer.c
@@ -240,7 +240,7 @@ static int apbt_cpuhp_notify(struct notifier_block *n,
                 dw_apb_clockevent_pause(adev->timer);
                 if (system_state == SYSTEM_RUNNING) {
                         pr_debug("skipping APBT CPU %lu offline\n", cpu);
-                } else if (adev) {
+                } else {
                         pr_debug("APBT clockevent for cpu %lu offline\n", cpu);
                         dw_apb_clockevent_stop(adev->timer);
                 }
@@ -311,7 +311,6 @@ void __init apbt_time_init(void)
 #ifdef CONFIG_SMP
         int i;
         struct sfi_timer_table_entry *p_mtmr;
-        unsigned int percpu_timer;
         struct apbt_dev *adev;
 #endif
 
@@ -346,13 +345,10 @@ void __init apbt_time_init(void)
                 return;
         }
         pr_debug("%s: %d CPUs online\n", __func__, num_online_cpus());
-        if (num_possible_cpus() <= sfi_mtimer_num) {
-                percpu_timer = 1;
+        if (num_possible_cpus() <= sfi_mtimer_num)
                 apbt_num_timers_used = num_possible_cpus();
-        } else {
-                percpu_timer = 0;
+        else
                 apbt_num_timers_used = 1;
-        }
         pr_debug("%s: %d APB timers used\n", __func__, apbt_num_timers_used);
 
         /* here we set up per CPU timer data structure */
diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c
index b994cc84aa7e..904611bf0e5a 100644
--- a/arch/x86/kernel/apic/apic.c
+++ b/arch/x86/kernel/apic/apic.c
@@ -131,7 +131,7 @@ static int __init parse_lapic(char *arg)
 {
         if (config_enabled(CONFIG_X86_32) && !arg)
                 force_enable_local_apic = 1;
-        else if (!strncmp(arg, "notscdeadline", 13))
+        else if (arg && !strncmp(arg, "notscdeadline", 13))
                 setup_clear_cpu_cap(X86_FEATURE_TSC_DEADLINE_TIMER);
         return 0;
 }
@@ -1477,8 +1477,7 @@ void __init bsp_end_local_APIC_setup(void)
          * Now that local APIC setup is completed for BP, configure the fault
          * handling for interrupt remapping.
          */
-        if (irq_remapping_enabled)
-                irq_remap_enable_fault_handling();
+        irq_remap_enable_fault_handling();
 
 }
 
@@ -2251,8 +2250,7 @@ static int lapic_suspend(void)
         local_irq_save(flags);
         disable_local_APIC();
 
-        if (irq_remapping_enabled)
-                irq_remapping_disable();
+        irq_remapping_disable();
 
         local_irq_restore(flags);
         return 0;
@@ -2268,16 +2266,15 @@ static void lapic_resume(void)
                 return;
 
         local_irq_save(flags);
-        if (irq_remapping_enabled) {
-                /*
-                 * IO-APIC and PIC have their own resume routines.
-                 * We just mask them here to make sure the interrupt
-                 * subsystem is completely quiet while we enable x2apic
-                 * and interrupt-remapping.
-                 */
-                mask_ioapic_entries();
-                legacy_pic->mask_all();
-        }
+
+        /*
+         * IO-APIC and PIC have their own resume routines.
+         * We just mask them here to make sure the interrupt
+         * subsystem is completely quiet while we enable x2apic
+         * and interrupt-remapping.
+         */
+        mask_ioapic_entries();
+        legacy_pic->mask_all();
 
         if (x2apic_mode)
                 enable_x2apic();
@@ -2320,8 +2317,7 @@ static void lapic_resume(void)
         apic_write(APIC_ESR, 0);
         apic_read(APIC_ESR);
 
-        if (irq_remapping_enabled)
-                irq_remapping_reenable(x2apic_mode);
+        irq_remapping_reenable(x2apic_mode);
 
         local_irq_restore(flags);
 }
diff --git a/arch/x86/kernel/apic/apic_numachip.c b/arch/x86/kernel/apic/apic_numachip.c
index 9c2aa89a11cb..9a9110918ca7 100644
--- a/arch/x86/kernel/apic/apic_numachip.c
+++ b/arch/x86/kernel/apic/apic_numachip.c
@@ -28,6 +28,7 @@
 #include <asm/apic.h>
 #include <asm/ipi.h>
 #include <asm/apic_flat_64.h>
+#include <asm/pgtable.h>
 
 static int numachip_system __read_mostly;
 
diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c
index b739d398bb29..9ed796ccc32c 100644
--- a/arch/x86/kernel/apic/io_apic.c
+++ b/arch/x86/kernel/apic/io_apic.c
@@ -68,22 +68,6 @@
 #define for_each_irq_pin(entry, head) \
         for (entry = head; entry; entry = entry->next)
 
-#ifdef CONFIG_IRQ_REMAP
-static void irq_remap_modify_chip_defaults(struct irq_chip *chip);
-static inline bool irq_remapped(struct irq_cfg *cfg)
-{
-        return cfg->irq_2_iommu.iommu != NULL;
-}
-#else
-static inline bool irq_remapped(struct irq_cfg *cfg)
-{
-        return false;
-}
-static inline void irq_remap_modify_chip_defaults(struct irq_chip *chip)
-{
-}
-#endif
-
 /*
  *      Is the SiS APIC rmw bug present ?
  *      -1 = don't know, 0 = no, 1 = yes
@@ -300,9 +284,9 @@ static struct irq_cfg *alloc_irq_and_cfg_at(unsigned int at, int node)
         return cfg;
 }
 
-static int alloc_irq_from(unsigned int from, int node)
+static int alloc_irqs_from(unsigned int from, unsigned int count, int node)
 {
-        return irq_alloc_desc_from(from, node);
+        return irq_alloc_descs_from(from, count, node);
 }
 
 static void free_irq_at(unsigned int at, struct irq_cfg *cfg)
@@ -326,7 +310,7 @@ static __attribute_const__ struct io_apic __iomem *io_apic_base(int idx)
                 + (mpc_ioapic_addr(idx) & ~PAGE_MASK);
 }
 
-static inline void io_apic_eoi(unsigned int apic, unsigned int vector)
+void io_apic_eoi(unsigned int apic, unsigned int vector)
 {
         struct io_apic __iomem *io_apic = io_apic_base(apic);
         writel(vector, &io_apic->eoi);
@@ -573,19 +557,10 @@ static void unmask_ioapic_irq(struct irq_data *data)
  * Otherwise, we simulate the EOI message manually by changing the trigger
  * mode to edge and then back to level, with RTE being masked during this.
  */
-static void __eoi_ioapic_pin(int apic, int pin, int vector, struct irq_cfg *cfg)
+void native_eoi_ioapic_pin(int apic, int pin, int vector)
 {
         if (mpc_ioapic_ver(apic) >= 0x20) {
-                /*
-                 * Intr-remapping uses pin number as the virtual vector
-                 * in the RTE. Actual vector is programmed in
-                 * intr-remapping table entry. Hence for the io-apic
-                 * EOI we use the pin number.
-                 */
-                if (cfg && irq_remapped(cfg))
-                        io_apic_eoi(apic, pin);
-                else
-                        io_apic_eoi(apic, vector);
+                io_apic_eoi(apic, vector);
         } else {
                 struct IO_APIC_route_entry entry, entry1;
 
@@ -606,14 +581,15 @@ static void __eoi_ioapic_pin(int apic, int pin, int vector, struct irq_cfg *cfg)
         }
 }
 
-static void eoi_ioapic_irq(unsigned int irq, struct irq_cfg *cfg)
+void eoi_ioapic_irq(unsigned int irq, struct irq_cfg *cfg)
 {
         struct irq_pin_list *entry;
         unsigned long flags;
 
         raw_spin_lock_irqsave(&ioapic_lock, flags);
         for_each_irq_pin(entry, cfg->irq_2_pin)
-                __eoi_ioapic_pin(entry->apic, entry->pin, cfg->vector, cfg);
+                x86_io_apic_ops.eoi_ioapic_pin(entry->apic, entry->pin,
+                                               cfg->vector);
         raw_spin_unlock_irqrestore(&ioapic_lock, flags);
 }
 
@@ -650,7 +626,7 @@ static void clear_IO_APIC_pin(unsigned int apic, unsigned int pin)
                 }
 
                 raw_spin_lock_irqsave(&ioapic_lock, flags);
-                __eoi_ioapic_pin(apic, pin, entry.vector, NULL);
+                x86_io_apic_ops.eoi_ioapic_pin(apic, pin, entry.vector);
                 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
         }
 
@@ -1304,25 +1280,18 @@ static void ioapic_register_intr(unsigned int irq, struct irq_cfg *cfg,
                 fasteoi = false;
         }
 
-        if (irq_remapped(cfg)) {
-                irq_set_status_flags(irq, IRQ_MOVE_PCNTXT);
-                irq_remap_modify_chip_defaults(chip);
+        if (setup_remapped_irq(irq, cfg, chip))
                 fasteoi = trigger != 0;
-        }
 
         hdl = fasteoi ? handle_fasteoi_irq : handle_edge_irq;
         irq_set_chip_and_handler_name(irq, chip, hdl,
                                       fasteoi ? "fasteoi" : "edge");
 }
 
-static int setup_ioapic_entry(int irq, struct IO_APIC_route_entry *entry,
-                               unsigned int destination, int vector,
-                               struct io_apic_irq_attr *attr)
+int native_setup_ioapic_entry(int irq, struct IO_APIC_route_entry *entry,
+                              unsigned int destination, int vector,
+                              struct io_apic_irq_attr *attr)
 {
-        if (irq_remapping_enabled)
-                return setup_ioapic_remapped_entry(irq, entry, destination,
-                                                   vector, attr);
-
         memset(entry, 0, sizeof(*entry));
 
         entry->delivery_mode = apic->irq_delivery_mode;
@@ -1370,8 +1339,8 @@ static void setup_ioapic_irq(unsigned int irq, struct irq_cfg *cfg,
                     attr->ioapic, mpc_ioapic_id(attr->ioapic), attr->ioapic_pin,
                     cfg->vector, irq, attr->trigger, attr->polarity, dest);
 
-        if (setup_ioapic_entry(irq, &entry, dest, cfg->vector, attr)) {
-                pr_warn("Failed to setup ioapic entry for ioapic %d, pin %d\n",
+        if (x86_io_apic_ops.setup_entry(irq, &entry, dest, cfg->vector, attr)) {
+                pr_warn("Failed to setup ioapic entry for ioapic  %d, pin %d\n",
                         mpc_ioapic_id(attr->ioapic), attr->ioapic_pin);
                 __clear_irq_vector(irq, cfg);
 
@@ -1479,9 +1448,6 @@ static void __init setup_timer_IRQ0_pin(unsigned int ioapic_idx,
         struct IO_APIC_route_entry entry;
         unsigned int dest;
 
-        if (irq_remapping_enabled)
-                return;
-
         memset(&entry, 0, sizeof(entry));
 
         /*
@@ -1513,9 +1479,63 @@ static void __init setup_timer_IRQ0_pin(unsigned int ioapic_idx,
         ioapic_write_entry(ioapic_idx, pin, entry);
 }
 
-__apicdebuginit(void) print_IO_APIC(int ioapic_idx)
+void native_io_apic_print_entries(unsigned int apic, unsigned int nr_entries)
 {
         int i;
+
+        pr_debug(" NR Dst Mask Trig IRR Pol Stat Dmod Deli Vect:\n");
+
+        for (i = 0; i <= nr_entries; i++) {
+                struct IO_APIC_route_entry entry;
+
+                entry = ioapic_read_entry(apic, i);
+
+                pr_debug(" %02x %02X  ", i, entry.dest);
+                pr_cont("%1d    %1d    %1d   %1d   %1d    "
+                        "%1d    %1d    %02X\n",
+                        entry.mask,
+                        entry.trigger,
+                        entry.irr,
+                        entry.polarity,
+                        entry.delivery_status,
+                        entry.dest_mode,
+                        entry.delivery_mode,
+                        entry.vector);
+        }
+}
+
+void intel_ir_io_apic_print_entries(unsigned int apic,
+                                    unsigned int nr_entries)
+{
+        int i;
+
+        pr_debug(" NR Indx Fmt Mask Trig IRR Pol Stat Indx2 Zero Vect:\n");
+
+        for (i = 0; i <= nr_entries; i++) {
+                struct IR_IO_APIC_route_entry *ir_entry;
+                struct IO_APIC_route_entry entry;
+
+                entry = ioapic_read_entry(apic, i);
+
+                ir_entry = (struct IR_IO_APIC_route_entry *)&entry;
+
+                pr_debug(" %02x %04X ", i, ir_entry->index);
+                pr_cont("%1d   %1d    %1d    %1d   %1d   "
+                        "%1d    %1d     %X    %02X\n",
+                        ir_entry->format,
+                        ir_entry->mask,
+                        ir_entry->trigger,
+                        ir_entry->irr,
+                        ir_entry->polarity,
+                        ir_entry->delivery_status,
+                        ir_entry->index2,
+                        ir_entry->zero,
+                        ir_entry->vector);
+        }
+}
+
+__apicdebuginit(void) print_IO_APIC(int ioapic_idx)
+{
         union IO_APIC_reg_00 reg_00;
         union IO_APIC_reg_01 reg_01;
         union IO_APIC_reg_02 reg_02;
@@ -1568,58 +1588,7 @@ __apicdebuginit(void) print_IO_APIC(int ioapic_idx)
 
         printk(KERN_DEBUG ".... IRQ redirection table:\n");
 
-        if (irq_remapping_enabled) {
-                printk(KERN_DEBUG " NR Indx Fmt Mask Trig IRR"
-                        " Pol Stat Indx2 Zero Vect:\n");
-        } else {
-                printk(KERN_DEBUG " NR Dst Mask Trig IRR Pol"
-                        " Stat Dmod Deli Vect:\n");
-        }
-
-        for (i = 0; i <= reg_01.bits.entries; i++) {
-                if (irq_remapping_enabled) {
-                        struct IO_APIC_route_entry entry;
-                        struct IR_IO_APIC_route_entry *ir_entry;
-
-                        entry = ioapic_read_entry(ioapic_idx, i);
-                        ir_entry = (struct IR_IO_APIC_route_entry *) &entry;
-                        printk(KERN_DEBUG " %02x %04X ",
-                                i,
-                                ir_entry->index
-                        );
-                        pr_cont("%1d   %1d    %1d    %1d   %1d   "
-                                "%1d    %1d     %X    %02X\n",
-                                ir_entry->format,
-                                ir_entry->mask,
-                                ir_entry->trigger,
-                                ir_entry->irr,
-                                ir_entry->polarity,
-                                ir_entry->delivery_status,
-                                ir_entry->index2,
-                                ir_entry->zero,
-                                ir_entry->vector
-                        );
-                } else {
-                        struct IO_APIC_route_entry entry;
-
-                        entry = ioapic_read_entry(ioapic_idx, i);
-                        printk(KERN_DEBUG " %02x %02X  ",
-                                i,
-                                entry.dest
-                        );
-                        pr_cont("%1d    %1d    %1d   %1d   %1d    "
-                                "%1d    %1d    %02X\n",
-                                entry.mask,
-                                entry.trigger,
-                                entry.irr,
-                                entry.polarity,
-                                entry.delivery_status,
-                                entry.dest_mode,
-                                entry.delivery_mode,
-                                entry.vector
-                        );
-                }
-        }
+        x86_io_apic_ops.print_entries(ioapic_idx, reg_01.bits.entries);
 }
 
 __apicdebuginit(void) print_IO_APICs(void)
@@ -1921,30 +1890,14 @@ void __init enable_IO_APIC(void)
         clear_IO_APIC();
 }
 
-/*
- * Not an __init, needed by the reboot code
- */
-void disable_IO_APIC(void)
+void native_disable_io_apic(void)
 {
         /*
-         * Clear the IO-APIC before rebooting:
-         */
-        clear_IO_APIC();
-
-        if (!legacy_pic->nr_legacy_irqs)
-                return;
-
-        /*
          * If the i8259 is routed through an IOAPIC
          * Put that IOAPIC in virtual wire mode
          * so legacy interrupts can be delivered.
-         *
-         * With interrupt-remapping, for now we will use virtual wire A mode,
-         * as virtual wire B is little complex (need to configure both
-         * IOAPIC RTE as well as interrupt-remapping table entry).
-         * As this gets called during crash dump, keep this simple for now.
          */
-        if (ioapic_i8259.pin != -1 && !irq_remapping_enabled) {
+        if (ioapic_i8259.pin != -1) {
                 struct IO_APIC_route_entry entry;
 
                 memset(&entry, 0, sizeof(entry));
@@ -1964,12 +1917,25 @@ void disable_IO_APIC(void)
                 ioapic_write_entry(ioapic_i8259.apic, ioapic_i8259.pin, entry);
         }
 
+        if (cpu_has_apic || apic_from_smp_config())
+                disconnect_bsp_APIC(ioapic_i8259.pin != -1);
+
+}
+
+/*
+ * Not an __init, needed by the reboot code
+ */
+void disable_IO_APIC(void)
+{
         /*
-         * Use virtual wire A mode when interrupt remapping is enabled.
+         * Clear the IO-APIC before rebooting:
          */
-        if (cpu_has_apic || apic_from_smp_config())
-                disconnect_bsp_APIC(!irq_remapping_enabled &&
-                                ioapic_i8259.pin != -1);
+        clear_IO_APIC();
+
+        if (!legacy_pic->nr_legacy_irqs)
+                return;
+
+        x86_io_apic_ops.disable();
 }
 
 #ifdef CONFIG_X86_32
@@ -2322,12 +2288,8 @@ static void __target_IO_APIC_irq(unsigned int irq, unsigned int dest, struct irq
 
                 apic = entry->apic;
                 pin = entry->pin;
-                /*
-                 * With interrupt-remapping, destination information comes
-                 * from interrupt-remapping table entry.
-                 */
-                if (!irq_remapped(cfg))
-                        io_apic_write(apic, 0x11 + pin*2, dest);
+
+                io_apic_write(apic, 0x11 + pin*2, dest);
                 reg = io_apic_read(apic, 0x10 + pin*2);
                 reg &= ~IO_APIC_REDIR_VECTOR_MASK;
                 reg |= vector;
@@ -2369,9 +2331,10 @@ int __ioapic_set_affinity(struct irq_data *data, const struct cpumask *mask,
         return 0;
 }
 
-static int
-ioapic_set_affinity(struct irq_data *data, const struct cpumask *mask,
-                    bool force)
+
+int native_ioapic_set_affinity(struct irq_data *data,
+                               const struct cpumask *mask,
+                               bool force)
 {
         unsigned int dest, irq = data->irq;
         unsigned long flags;
@@ -2548,33 +2511,6 @@ static void ack_apic_level(struct irq_data *data)
         ioapic_irqd_unmask(data, cfg, masked);
 }
 
-#ifdef CONFIG_IRQ_REMAP
-static void ir_ack_apic_edge(struct irq_data *data)
-{
-        ack_APIC_irq();
-}
-
-static void ir_ack_apic_level(struct irq_data *data)
-{
-        ack_APIC_irq();
-        eoi_ioapic_irq(data->irq, data->chip_data);
-}
-
-static void ir_print_prefix(struct irq_data *data, struct seq_file *p)
-{
-        seq_printf(p, " IR-%s", data->chip->name);
-}
-
-static void irq_remap_modify_chip_defaults(struct irq_chip *chip)
-{
-        chip->irq_print_chip = ir_print_prefix;
-        chip->irq_ack = ir_ack_apic_edge;
-        chip->irq_eoi = ir_ack_apic_level;
-
-        chip->irq_set_affinity = set_remapped_irq_affinity;
-}
-#endif /* CONFIG_IRQ_REMAP */
-
 static struct irq_chip ioapic_chip __read_mostly = {
         .name                   = "IO-APIC",
         .irq_startup            = startup_ioapic_irq,
@@ -2582,7 +2518,7 @@ static struct irq_chip ioapic_chip __read_mostly = {
         .irq_unmask             = unmask_ioapic_irq,
         .irq_ack                = ack_apic_edge,
         .irq_eoi                = ack_apic_level,
-        .irq_set_affinity       = ioapic_set_affinity,
+        .irq_set_affinity       = native_ioapic_set_affinity,
         .irq_retrigger          = ioapic_retrigger_irq,
 };
 
@@ -2781,8 +2717,7 @@ static inline void __init check_timer(void)
          * 8259A.
          */
         if (pin1 == -1) {
-                if (irq_remapping_enabled)
-                        panic("BIOS bug: timer not connected to IO-APIC");
+                panic_if_irq_remap("BIOS bug: timer not connected to IO-APIC");
                 pin1 = pin2;
                 apic1 = apic2;
                 no_pin1 = 1;
@@ -2814,8 +2749,7 @@ static inline void __init check_timer(void)
                                 clear_IO_APIC_pin(0, pin1);
                         goto out;
                 }
-                if (irq_remapping_enabled)
-                        panic("timer doesn't work through Interrupt-remapped IO-APIC");
+                panic_if_irq_remap("timer doesn't work through Interrupt-remapped IO-APIC");
                 local_irq_disable();
                 clear_IO_APIC_pin(apic1, pin1);
                 if (!no_pin1)
@@ -2982,37 +2916,58 @@ device_initcall(ioapic_init_ops);
 /*
  * Dynamic irq allocate and deallocation
  */
-unsigned int create_irq_nr(unsigned int from, int node)
+unsigned int __create_irqs(unsigned int from, unsigned int count, int node)
 {
-        struct irq_cfg *cfg;
+        struct irq_cfg **cfg;
         unsigned long flags;
-        unsigned int ret = 0;
-        int irq;
+        int irq, i;
 
         if (from < nr_irqs_gsi)
                 from = nr_irqs_gsi;
 
-        irq = alloc_irq_from(from, node);
-        if (irq < 0)
-                return 0;
-        cfg = alloc_irq_cfg(irq, node);
-        if (!cfg) {
-                free_irq_at(irq, NULL);
+        cfg = kzalloc_node(count * sizeof(cfg[0]), GFP_KERNEL, node);
+        if (!cfg)
                 return 0;
+
+        irq = alloc_irqs_from(from, count, node);
+        if (irq < 0)
+                goto out_cfgs;
+
+        for (i = 0; i < count; i++) {
+                cfg[i] = alloc_irq_cfg(irq + i, node);
+                if (!cfg[i])
+                        goto out_irqs;
         }
 
         raw_spin_lock_irqsave(&vector_lock, flags);
-        if (!__assign_irq_vector(irq, cfg, apic->target_cpus()))
-                ret = irq;
+        for (i = 0; i < count; i++)
+                if (__assign_irq_vector(irq + i, cfg[i], apic->target_cpus()))
+                        goto out_vecs;
         raw_spin_unlock_irqrestore(&vector_lock, flags);
 
-        if (ret) {
-                irq_set_chip_data(irq, cfg);
-                irq_clear_status_flags(irq, IRQ_NOREQUEST);
-        } else {
-                free_irq_at(irq, cfg);
+        for (i = 0; i < count; i++) {
+                irq_set_chip_data(irq + i, cfg[i]);
+                irq_clear_status_flags(irq + i, IRQ_NOREQUEST);
         }
-        return ret;
+
+        kfree(cfg);
+        return irq;
+
+out_vecs:
+        for (i--; i >= 0; i--)
+                __clear_irq_vector(irq + i, cfg[i]);
+        raw_spin_unlock_irqrestore(&vector_lock, flags);
+out_irqs:
+        for (i = 0; i < count; i++)
+                free_irq_at(irq + i, cfg[i]);
+out_cfgs:
+        kfree(cfg);
+        return 0;
+}
+
+unsigned int create_irq_nr(unsigned int from, int node)
+{
+        return __create_irqs(from, 1, node);
 }
 
 int create_irq(void)
@@ -3037,48 +2992,35 @@ void destroy_irq(unsigned int irq)
 
         irq_set_status_flags(irq, IRQ_NOREQUEST|IRQ_NOPROBE);
 
-        if (irq_remapped(cfg))
-                free_remapped_irq(irq);
+        free_remapped_irq(irq);
+
         raw_spin_lock_irqsave(&vector_lock, flags);
         __clear_irq_vector(irq, cfg);
         raw_spin_unlock_irqrestore(&vector_lock, flags);
         free_irq_at(irq, cfg);
 }
 
+void destroy_irqs(unsigned int irq, unsigned int count)
+{
+        unsigned int i;
+
+        for (i = 0; i < count; i++)
+                destroy_irq(irq + i);
+}
+
 /*
  * MSI message composition
  */
-#ifdef CONFIG_PCI_MSI
-static int msi_compose_msg(struct pci_dev *pdev, unsigned int irq,
-                           struct msi_msg *msg, u8 hpet_id)
+void native_compose_msi_msg(struct pci_dev *pdev,
+                            unsigned int irq, unsigned int dest,
+                            struct msi_msg *msg, u8 hpet_id)
 {
-        struct irq_cfg *cfg;
-        int err;
-        unsigned dest;
-
-        if (disable_apic)
-                return -ENXIO;
-
-        cfg = irq_cfg(irq);
-        err = assign_irq_vector(irq, cfg, apic->target_cpus());
-        if (err)
-                return err;
+        struct irq_cfg *cfg = irq_cfg(irq);
 
-        err = apic->cpu_mask_to_apicid_and(cfg->domain,
-                                           apic->target_cpus(), &dest);
-        if (err)
-                return err;
-
-        if (irq_remapped(cfg)) {
-                compose_remapped_msi_msg(pdev, irq, dest, msg, hpet_id);
-                return err;
-        }
+        msg->address_hi = MSI_ADDR_BASE_HI;
 
         if (x2apic_enabled())
-                msg->address_hi = MSI_ADDR_BASE_HI |
-                                  MSI_ADDR_EXT_DEST_ID(dest);
-        else
-                msg->address_hi = MSI_ADDR_BASE_HI;
+                msg->address_hi |= MSI_ADDR_EXT_DEST_ID(dest);
 
         msg->address_lo =
                 MSI_ADDR_BASE_LO |
@@ -3097,8 +3039,32 @@ static int msi_compose_msg(struct pci_dev *pdev, unsigned int irq,
                         MSI_DATA_DELIVERY_FIXED:
                         MSI_DATA_DELIVERY_LOWPRI) |
                 MSI_DATA_VECTOR(cfg->vector);
+}
 
-        return err;
+#ifdef CONFIG_PCI_MSI
+static int msi_compose_msg(struct pci_dev *pdev, unsigned int irq,
+                           struct msi_msg *msg, u8 hpet_id)
+{
+        struct irq_cfg *cfg;
+        int err;
+        unsigned dest;
+
+        if (disable_apic)
+                return -ENXIO;
+
+        cfg = irq_cfg(irq);
+        err = assign_irq_vector(irq, cfg, apic->target_cpus());
+        if (err)
+                return err;
+
+        err = apic->cpu_mask_to_apicid_and(cfg->domain,
+                                           apic->target_cpus(), &dest);
+        if (err)
+                return err;
+
+        x86_msi.compose_msi_msg(pdev, irq, dest, msg, hpet_id);
+
+        return 0;
 }
 
 static int
@@ -3136,23 +3102,28 @@ static struct irq_chip msi_chip = {
         .irq_retrigger          = ioapic_retrigger_irq,
 };
 
-static int setup_msi_irq(struct pci_dev *dev, struct msi_desc *msidesc, int irq)
+int setup_msi_irq(struct pci_dev *dev, struct msi_desc *msidesc,
+                  unsigned int irq_base, unsigned int irq_offset)
 {
         struct irq_chip *chip = &msi_chip;
         struct msi_msg msg;
+        unsigned int irq = irq_base + irq_offset;
         int ret;
 
         ret = msi_compose_msg(dev, irq, &msg, -1);
         if (ret < 0)
                 return ret;
 
-        irq_set_msi_desc(irq, msidesc);
-        write_msi_msg(irq, &msg);
+        irq_set_msi_desc_off(irq_base, irq_offset, msidesc);
 
-        if (irq_remapped(irq_get_chip_data(irq))) {
-                irq_set_status_flags(irq, IRQ_MOVE_PCNTXT);
-                irq_remap_modify_chip_defaults(chip);
-        }
+        /*
+         * MSI-X message is written per-IRQ, the offset is always 0.
+         * MSI message denotes a contiguous group of IRQs, written for 0th IRQ.
+         */
+        if (!irq_offset)
+                write_msi_msg(irq, &msg);
+
+        setup_remapped_irq(irq, irq_get_chip_data(irq), chip);
 
         irq_set_chip_and_handler_name(irq, chip, handle_edge_irq, "edge");
 
@@ -3163,46 +3134,26 @@ static int setup_msi_irq(struct pci_dev *dev, struct msi_desc *msidesc, int irq)
 
 int native_setup_msi_irqs(struct pci_dev *dev, int nvec, int type)
 {
-        int node, ret, sub_handle, index = 0;
         unsigned int irq, irq_want;
         struct msi_desc *msidesc;
+        int node, ret;
 
-        /* x86 doesn't support multiple MSI yet */
+        /* Multiple MSI vectors only supported with interrupt remapping */
         if (type == PCI_CAP_ID_MSI && nvec > 1)
                 return 1;
 
         node = dev_to_node(&dev->dev);
         irq_want = nr_irqs_gsi;
-        sub_handle = 0;
         list_for_each_entry(msidesc, &dev->msi_list, list) {
                 irq = create_irq_nr(irq_want, node);
                 if (irq == 0)
-                        return -1;
+                        return -ENOSPC;
+
                 irq_want = irq + 1;
-                if (!irq_remapping_enabled)
-                        goto no_ir;
 
-                if (!sub_handle) {
-                        /*
-                         * allocate the consecutive block of IRTE's
-                         * for 'nvec'
-                         */
-                        index = msi_alloc_remapped_irq(dev, irq, nvec);
-                        if (index < 0) {
-                                ret = index;
-                                goto error;
-                        }
-                } else {
-                        ret = msi_setup_remapped_irq(dev, irq, index,
-                                                     sub_handle);
-                        if (ret < 0)
-                                goto error;
-                }
-no_ir:
-                ret = setup_msi_irq(dev, msidesc, irq);
+                ret = setup_msi_irq(dev, msidesc, irq, 0);
                 if (ret < 0)
                         goto error;
-                sub_handle++;
         }
         return 0;
 
@@ -3298,26 +3249,19 @@ static struct irq_chip hpet_msi_type = {
         .irq_retrigger = ioapic_retrigger_irq,
 };
 
-int arch_setup_hpet_msi(unsigned int irq, unsigned int id)
+int default_setup_hpet_msi(unsigned int irq, unsigned int id)
 {
         struct irq_chip *chip = &hpet_msi_type;
         struct msi_msg msg;
         int ret;
 
-        if (irq_remapping_enabled) {
-                ret = setup_hpet_msi_remapped(irq, id);
-                if (ret)
-                        return ret;
-        }
-
         ret = msi_compose_msg(NULL, irq, &msg, id);
         if (ret < 0)
                 return ret;
 
         hpet_msi_write(irq_get_handler_data(irq), &msg);
         irq_set_status_flags(irq, IRQ_MOVE_PCNTXT);
-        if (irq_remapped(irq_get_chip_data(irq)))
-                irq_remap_modify_chip_defaults(chip);
+        setup_remapped_irq(irq, irq_get_chip_data(irq), chip);
 
         irq_set_chip_and_handler_name(irq, chip, handle_edge_irq, "edge");
         return 0;
@@ -3683,10 +3627,7 @@ void __init setup_ioapic_dest(void)
                 else
                         mask = apic->target_cpus();
 
-                if (irq_remapping_enabled)
-                        set_remapped_irq_affinity(idata, mask, false);
-                else
-                        ioapic_set_affinity(idata, mask, false);
+                x86_io_apic_ops.set_affinity(idata, mask, false);
         }
 
 }
diff --git a/arch/x86/kernel/apic/ipi.c b/arch/x86/kernel/apic/ipi.c
index cce91bf26676..7434d8556d09 100644
--- a/arch/x86/kernel/apic/ipi.c
+++ b/arch/x86/kernel/apic/ipi.c
@@ -106,7 +106,7 @@ void default_send_IPI_mask_logical(const struct cpumask *cpumask, int vector)
         unsigned long mask = cpumask_bits(cpumask)[0];
         unsigned long flags;
 
-        if (WARN_ONCE(!mask, "empty IPI mask"))
+        if (!mask)
                 return;
 
         local_irq_save(flags);
diff --git a/arch/x86/kernel/apic/x2apic_phys.c b/arch/x86/kernel/apic/x2apic_phys.c
index e03a1e180e81..562a76d433c8 100644
--- a/arch/x86/kernel/apic/x2apic_phys.c
+++ b/arch/x86/kernel/apic/x2apic_phys.c
@@ -20,18 +20,19 @@ static int set_x2apic_phys_mode(char *arg)
 }
 early_param("x2apic_phys", set_x2apic_phys_mode);
 
-static int x2apic_acpi_madt_oem_check(char *oem_id, char *oem_table_id)
+static bool x2apic_fadt_phys(void)
 {
-        if (x2apic_phys)
-                return x2apic_enabled();
-        else if ((acpi_gbl_FADT.header.revision >= FADT2_REVISION_ID) &&
-                (acpi_gbl_FADT.flags & ACPI_FADT_APIC_PHYSICAL) &&
-                x2apic_enabled()) {
+        if ((acpi_gbl_FADT.header.revision >= FADT2_REVISION_ID) &&
+                (acpi_gbl_FADT.flags & ACPI_FADT_APIC_PHYSICAL)) {
                 printk(KERN_DEBUG "System requires x2apic physical mode\n");
-                return 1;
+                return true;
         }
-        else
-                return 0;
+        return false;
+}
+
+static int x2apic_acpi_madt_oem_check(char *oem_id, char *oem_table_id)
+{
+        return x2apic_enabled() && (x2apic_phys || x2apic_fadt_phys());
 }
 
 static void
@@ -82,7 +83,7 @@ static void init_x2apic_ldr(void)
 
 static int x2apic_phys_probe(void)
 {
-        if (x2apic_mode && x2apic_phys)
+        if (x2apic_mode && (x2apic_phys || x2apic_fadt_phys()))
                 return 1;
 
         return apic == &apic_x2apic_phys;
diff --git a/arch/x86/kernel/apic/x2apic_uv_x.c b/arch/x86/kernel/apic/x2apic_uv_x.c
index 8cfade9510a4..794f6eb54cd3 100644
--- a/arch/x86/kernel/apic/x2apic_uv_x.c
+++ b/arch/x86/kernel/apic/x2apic_uv_x.c
@@ -5,7 +5,7 @@
  *
  * SGI UV APIC functions (note: not an Intel compatible APIC)
  *
- * Copyright (C) 2007-2010 Silicon Graphics, Inc. All rights reserved.
+ * Copyright (C) 2007-2013 Silicon Graphics, Inc. All rights reserved.
  */
 #include <linux/cpumask.h>
 #include <linux/hardirq.h>
@@ -91,10 +91,16 @@ static int __init early_get_pnodeid(void)
         m_n_config.v = uv_early_read_mmr(UVH_RH_GAM_CONFIG_MMR);
         uv_min_hub_revision_id = node_id.s.revision;
 
-        if (node_id.s.part_number == UV2_HUB_PART_NUMBER)
-                uv_min_hub_revision_id += UV2_HUB_REVISION_BASE - 1;
-        if (node_id.s.part_number == UV2_HUB_PART_NUMBER_X)
+        switch (node_id.s.part_number) {
+        case UV2_HUB_PART_NUMBER:
+        case UV2_HUB_PART_NUMBER_X:
                 uv_min_hub_revision_id += UV2_HUB_REVISION_BASE - 1;
+                break;
+        case UV3_HUB_PART_NUMBER:
+        case UV3_HUB_PART_NUMBER_X:
+                uv_min_hub_revision_id += UV3_HUB_REVISION_BASE - 1;
+                break;
+        }
 
         uv_hub_info->hub_revision = uv_min_hub_revision_id;
         pnode = (node_id.s.node_id >> 1) & ((1 << m_n_config.s.n_skt) - 1);
@@ -130,13 +136,16 @@ static void __init uv_set_apicid_hibit(void)
 
 static int __init uv_acpi_madt_oem_check(char *oem_id, char *oem_table_id)
 {
-        int pnodeid, is_uv1, is_uv2;
+        int pnodeid, is_uv1, is_uv2, is_uv3;
 
         is_uv1 = !strcmp(oem_id, "SGI");
         is_uv2 = !strcmp(oem_id, "SGI2");
-        if (is_uv1 || is_uv2) {
+        is_uv3 = !strncmp(oem_id, "SGI3", 4);   /* there are varieties of UV3 */
+        if (is_uv1 || is_uv2 || is_uv3) {
                 uv_hub_info->hub_revision =
-                        is_uv1 ? UV1_HUB_REVISION_BASE : UV2_HUB_REVISION_BASE;
+                        (is_uv1 ? UV1_HUB_REVISION_BASE :
+                        (is_uv2 ? UV2_HUB_REVISION_BASE :
+                                  UV3_HUB_REVISION_BASE));
                 pnodeid = early_get_pnodeid();
                 early_get_apic_pnode_shift();
                 x86_platform.is_untracked_pat_range =  uv_is_untracked_pat_range;
@@ -450,14 +459,17 @@ static __init void map_high(char *id, unsigned long base, int pshift,
 
         paddr = base << pshift;
         bytes = (1UL << bshift) * (max_pnode + 1);
-        printk(KERN_INFO "UV: Map %s_HI 0x%lx - 0x%lx\n", id, paddr,
-                                                paddr + bytes);
+        if (!paddr) {
+                pr_info("UV: Map %s_HI base address NULL\n", id);
+                return;
+        }
+        pr_info("UV: Map %s_HI 0x%lx - 0x%lx\n", id, paddr, paddr + bytes);
         if (map_type == map_uc)
                 init_extra_mapping_uc(paddr, bytes);
         else
                 init_extra_mapping_wb(paddr, bytes);
-
 }
+
 static __init void map_gru_high(int max_pnode)
 {
         union uvh_rh_gam_gru_overlay_config_mmr_u gru;
@@ -468,7 +480,8 @@ static __init void map_gru_high(int max_pnode)
                 map_high("GRU", gru.s.base, shift, shift, max_pnode, map_wb);
                 gru_start_paddr = ((u64)gru.s.base << shift);
                 gru_end_paddr = gru_start_paddr + (1UL << shift) * (max_pnode + 1);
-
+        } else {
+                pr_info("UV: GRU disabled\n");
         }
 }
 
@@ -480,23 +493,146 @@ static __init void map_mmr_high(int max_pnode)
         mmr.v = uv_read_local_mmr(UVH_RH_GAM_MMR_OVERLAY_CONFIG_MMR);
         if (mmr.s.enable)
                 map_high("MMR", mmr.s.base, shift, shift, max_pnode, map_uc);
+        else
+                pr_info("UV: MMR disabled\n");
+}
+
+/*
+ * This commonality works because both 0 & 1 versions of the MMIOH OVERLAY
+ * and REDIRECT MMR regs are exactly the same on UV3.
+ */
+struct mmioh_config {
+        unsigned long overlay;
+        unsigned long redirect;
+        char *id;
+};
+
+static __initdata struct mmioh_config mmiohs[] = {
+        {
+                UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR,
+                UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR,
+                "MMIOH0"
+        },
+        {
+                UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR,
+                UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR,
+                "MMIOH1"
+        },
+};
+
+static __init void map_mmioh_high_uv3(int index, int min_pnode, int max_pnode)
+{
+        union uv3h_rh_gam_mmioh_overlay_config0_mmr_u overlay;
+        unsigned long mmr;
+        unsigned long base;
+        int i, n, shift, m_io, max_io;
+        int nasid, lnasid, fi, li;
+        char *id;
+
+        id = mmiohs[index].id;
+        overlay.v = uv_read_local_mmr(mmiohs[index].overlay);
+        pr_info("UV: %s overlay 0x%lx base:0x%x m_io:%d\n",
+                id, overlay.v, overlay.s3.base, overlay.s3.m_io);
+        if (!overlay.s3.enable) {
+                pr_info("UV: %s disabled\n", id);
+                return;
+        }
+
+        shift = UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_BASE_SHFT;
+        base = (unsigned long)overlay.s3.base;
+        m_io = overlay.s3.m_io;
+        mmr = mmiohs[index].redirect;
+        n = UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR_DEPTH;
+        min_pnode *= 2;                         /* convert to NASID */
+        max_pnode *= 2;
+        max_io = lnasid = fi = li = -1;
+
+        for (i = 0; i < n; i++) {
+                union uv3h_rh_gam_mmioh_redirect_config0_mmr_u redirect;
+
+                redirect.v = uv_read_local_mmr(mmr + i * 8);
+                nasid = redirect.s3.nasid;
+                if (nasid < min_pnode || max_pnode < nasid)
+                        nasid = -1;             /* invalid NASID */
+
+                if (nasid == lnasid) {
+                        li = i;
+                        if (i != n-1)           /* last entry check */
+                                continue;
+                }
+
+                /* check if we have a cached (or last) redirect to print */
+                if (lnasid != -1 || (i == n-1 && nasid != -1))  {
+                        unsigned long addr1, addr2;
+                        int f, l;
+
+                        if (lnasid == -1) {
+                                f = l = i;
+                                lnasid = nasid;
+                        } else {
+                                f = fi;
+                                l = li;
+                        }
+                        addr1 = (base << shift) +
+                                f * (unsigned long)(1 << m_io);
+                        addr2 = (base << shift) +
+                                (l + 1) * (unsigned long)(1 << m_io);
+                        pr_info("UV: %s[%03d..%03d] NASID 0x%04x ADDR 0x%016lx - 0x%016lx\n",
+                                id, fi, li, lnasid, addr1, addr2);
+                        if (max_io < l)
+                                max_io = l;
+                }
+                fi = li = i;
+                lnasid = nasid;
+        }
+
+        pr_info("UV: %s base:0x%lx shift:%d M_IO:%d MAX_IO:%d\n",
+                id, base, shift, m_io, max_io);
+
+        if (max_io >= 0)
+                map_high(id, base, shift, m_io, max_io, map_uc);
 }
 
-static __init void map_mmioh_high(int max_pnode)
+static __init void map_mmioh_high(int min_pnode, int max_pnode)
 {
         union uvh_rh_gam_mmioh_overlay_config_mmr_u mmioh;
-        int shift;
+        unsigned long mmr, base;
+        int shift, enable, m_io, n_io;
 
-        mmioh.v = uv_read_local_mmr(UVH_RH_GAM_MMIOH_OVERLAY_CONFIG_MMR);
-        if (is_uv1_hub() && mmioh.s1.enable) {
-                shift = UV1H_RH_GAM_MMIOH_OVERLAY_CONFIG_MMR_BASE_SHFT;
-                map_high("MMIOH", mmioh.s1.base, shift, mmioh.s1.m_io,
-                        max_pnode, map_uc);
+        if (is_uv3_hub()) {
+                /* Map both MMIOH Regions */
+                map_mmioh_high_uv3(0, min_pnode, max_pnode);
+                map_mmioh_high_uv3(1, min_pnode, max_pnode);
+                return;
         }
-        if (is_uv2_hub() && mmioh.s2.enable) {
+
+        if (is_uv1_hub()) {
+                mmr = UV1H_RH_GAM_MMIOH_OVERLAY_CONFIG_MMR;
+                shift = UV1H_RH_GAM_MMIOH_OVERLAY_CONFIG_MMR_BASE_SHFT;
+                mmioh.v = uv_read_local_mmr(mmr);
+                enable = !!mmioh.s1.enable;
+                base = mmioh.s1.base;
+                m_io = mmioh.s1.m_io;
+                n_io = mmioh.s1.n_io;
+        } else if (is_uv2_hub()) {
+                mmr = UV2H_RH_GAM_MMIOH_OVERLAY_CONFIG_MMR;
                 shift = UV2H_RH_GAM_MMIOH_OVERLAY_CONFIG_MMR_BASE_SHFT;
-                map_high("MMIOH", mmioh.s2.base, shift, mmioh.s2.m_io,
-                        max_pnode, map_uc);
+                mmioh.v = uv_read_local_mmr(mmr);
+                enable = !!mmioh.s2.enable;
+                base = mmioh.s2.base;
+                m_io = mmioh.s2.m_io;
+                n_io = mmioh.s2.n_io;
+        } else
+                return;
+
+        if (enable) {
+                max_pnode &= (1 << n_io) - 1;
+                pr_info(
+                    "UV: base:0x%lx shift:%d N_IO:%d M_IO:%d max_pnode:0x%x\n",
+                        base, shift, m_io, n_io, max_pnode);
+                map_high("MMIOH", base, shift, m_io, max_pnode, map_uc);
+        } else {
+                pr_info("UV: MMIOH disabled\n");
         }
 }
 
@@ -724,42 +860,41 @@ void uv_nmi_init(void)
 void __init uv_system_init(void)
 {
         union uvh_rh_gam_config_mmr_u  m_n_config;
-        union uvh_rh_gam_mmioh_overlay_config_mmr_u mmioh;
         union uvh_node_id_u node_id;
         unsigned long gnode_upper, lowmem_redir_base, lowmem_redir_size;
-        int bytes, nid, cpu, lcpu, pnode, blade, i, j, m_val, n_val, n_io;
-        int gnode_extra, max_pnode = 0;
+        int bytes, nid, cpu, lcpu, pnode, blade, i, j, m_val, n_val;
+        int gnode_extra, min_pnode = 999999, max_pnode = -1;
         unsigned long mmr_base, present, paddr;
-        unsigned short pnode_mask, pnode_io_mask;
+        unsigned short pnode_mask;
+        char *hub = (is_uv1_hub() ? "UV1" :
+                    (is_uv2_hub() ? "UV2" :
+                                    "UV3"));
 
-        printk(KERN_INFO "UV: Found %s hub\n", is_uv1_hub() ? "UV1" : "UV2");
+        pr_info("UV: Found %s hub\n", hub);
         map_low_mmrs();
 
         m_n_config.v = uv_read_local_mmr(UVH_RH_GAM_CONFIG_MMR );
         m_val = m_n_config.s.m_skt;
         n_val = m_n_config.s.n_skt;
-        mmioh.v = uv_read_local_mmr(UVH_RH_GAM_MMIOH_OVERLAY_CONFIG_MMR);
-        n_io = is_uv1_hub() ? mmioh.s1.n_io : mmioh.s2.n_io;
+        pnode_mask = (1 << n_val) - 1;
         mmr_base =
             uv_read_local_mmr(UVH_RH_GAM_MMR_OVERLAY_CONFIG_MMR) &
             ~UV_MMR_ENABLE;
-        pnode_mask = (1 << n_val) - 1;
-        pnode_io_mask = (1 << n_io) - 1;
 
         node_id.v = uv_read_local_mmr(UVH_NODE_ID);
         gnode_extra = (node_id.s.node_id & ~((1 << n_val) - 1)) >> 1;
         gnode_upper = ((unsigned long)gnode_extra  << m_val);
-        printk(KERN_INFO "UV: N %d, M %d, N_IO: %d, gnode_upper 0x%lx, gnode_extra 0x%x, pnode_mask 0x%x, pnode_io_mask 0x%x\n",
-                        n_val, m_val, n_io, gnode_upper, gnode_extra, pnode_mask, pnode_io_mask);
+        pr_info("UV: N:%d M:%d pnode_mask:0x%x gnode_upper/extra:0x%lx/0x%x\n",
+                        n_val, m_val, pnode_mask, gnode_upper, gnode_extra);
 
-        printk(KERN_DEBUG "UV: global MMR base 0x%lx\n", mmr_base);
+        pr_info("UV: global MMR base 0x%lx\n", mmr_base);
 
         for(i = 0; i < UVH_NODE_PRESENT_TABLE_DEPTH; i++)
                 uv_possible_blades +=
                   hweight64(uv_read_local_mmr( UVH_NODE_PRESENT_TABLE + i * 8));
 
         /* uv_num_possible_blades() is really the hub count */
-        printk(KERN_INFO "UV: Found %d blades, %d hubs\n",
+        pr_info("UV: Found %d blades, %d hubs\n",
                         is_uv1_hub() ? uv_num_possible_blades() :
                         (uv_num_possible_blades() + 1) / 2,
                         uv_num_possible_blades());
@@ -794,6 +929,7 @@ void __init uv_system_init(void)
                         uv_blade_info[blade].nr_possible_cpus = 0;
                         uv_blade_info[blade].nr_online_cpus = 0;
                         spin_lock_init(&uv_blade_info[blade].nmi_lock);
+                        min_pnode = min(pnode, min_pnode);
                         max_pnode = max(pnode, max_pnode);
                         blade++;
                 }
@@ -856,7 +992,7 @@ void __init uv_system_init(void)
 
         map_gru_high(max_pnode);
         map_mmr_high(max_pnode);
-        map_mmioh_high(max_pnode & pnode_io_mask);
+        map_mmioh_high(min_pnode, max_pnode);
 
         uv_cpu_init();
         uv_scir_register_cpu_notifier();
diff --git a/arch/x86/kernel/apm_32.c b/arch/x86/kernel/apm_32.c
index d65464e43503..66b5faffe14a 100644
--- a/arch/x86/kernel/apm_32.c
+++ b/arch/x86/kernel/apm_32.c
@@ -232,6 +232,7 @@
 #include <linux/acpi.h>
 #include <linux/syscore_ops.h>
 #include <linux/i8253.h>
+#include <linux/cpuidle.h>
 
 #include <asm/uaccess.h>
 #include <asm/desc.h>
@@ -360,13 +361,35 @@ struct apm_user {
  * idle percentage above which bios idle calls are done
  */
 #ifdef CONFIG_APM_CPU_IDLE
-#warning deprecated CONFIG_APM_CPU_IDLE will be deleted in 2012
 #define DEFAULT_IDLE_THRESHOLD  95
 #else
 #define DEFAULT_IDLE_THRESHOLD  100
 #endif
 #define DEFAULT_IDLE_PERIOD     (100 / 3)
 
+static int apm_cpu_idle(struct cpuidle_device *dev,
+                        struct cpuidle_driver *drv, int index);
+
+static struct cpuidle_driver apm_idle_driver = {
+        .name = "apm_idle",
+        .owner = THIS_MODULE,
+        .en_core_tk_irqen = 1,
+        .states = {
+                { /* entry 0 is for polling */ },
+                { /* entry 1 is for APM idle */
+                        .name = "APM",
+                        .desc = "APM idle",
+                        .flags = CPUIDLE_FLAG_TIME_VALID,
+                        .exit_latency = 250,    /* WAG */
+                        .target_residency = 500,        /* WAG */
+                        .enter = &apm_cpu_idle
+                },
+        },
+        .state_count = 2,
+};
+
+static struct cpuidle_device apm_cpuidle_device;
+
 /*
  * Local variables
  */
@@ -377,7 +400,6 @@ static struct {
 static int clock_slowed;
 static int idle_threshold __read_mostly = DEFAULT_IDLE_THRESHOLD;
 static int idle_period __read_mostly = DEFAULT_IDLE_PERIOD;
-static int set_pm_idle;
 static int suspends_pending;
 static int standbys_pending;
 static int ignore_sys_suspend;
@@ -884,8 +906,6 @@ static void apm_do_busy(void)
 #define IDLE_CALC_LIMIT (HZ * 100)
 #define IDLE_LEAKY_MAX  16
 
-static void (*original_pm_idle)(void) __read_mostly;
-
 /**
  * apm_cpu_idle         -       cpu idling for APM capable Linux
  *
@@ -894,35 +914,36 @@ static void (*original_pm_idle)(void) __read_mostly;
  * Furthermore it calls the system default idle routine.
  */
 
-static void apm_cpu_idle(void)
+static int apm_cpu_idle(struct cpuidle_device *dev,
+        struct cpuidle_driver *drv, int index)
 {
         static int use_apm_idle; /* = 0 */
         static unsigned int last_jiffies; /* = 0 */
         static unsigned int last_stime; /* = 0 */
+        cputime_t stime;
 
         int apm_idle_done = 0;
         unsigned int jiffies_since_last_check = jiffies - last_jiffies;
         unsigned int bucket;
 
-        WARN_ONCE(1, "deprecated apm_cpu_idle will be deleted in 2012");
 recalc:
+        task_cputime(current, NULL, &stime);
         if (jiffies_since_last_check > IDLE_CALC_LIMIT) {
                 use_apm_idle = 0;
-                last_jiffies = jiffies;
-                last_stime = current->stime;
         } else if (jiffies_since_last_check > idle_period) {
                 unsigned int idle_percentage;
 
-                idle_percentage = current->stime - last_stime;
+                idle_percentage = stime - last_stime;
                 idle_percentage *= 100;
                 idle_percentage /= jiffies_since_last_check;
                 use_apm_idle = (idle_percentage > idle_threshold);
                 if (apm_info.forbid_idle)
                         use_apm_idle = 0;
-                last_jiffies = jiffies;
-                last_stime = current->stime;
         }
 
+        last_jiffies = jiffies;
+        last_stime = stime;
+
         bucket = IDLE_LEAKY_MAX;
 
         while (!need_resched()) {
@@ -950,10 +971,7 @@ recalc:
                                 break;
                         }
                 }
-                if (original_pm_idle)
-                        original_pm_idle();
-                else
-                        default_idle();
+                default_idle();
                 local_irq_disable();
                 jiffies_since_last_check = jiffies - last_jiffies;
                 if (jiffies_since_last_check > idle_period)
@@ -963,7 +981,7 @@ recalc:
         if (apm_idle_done)
                 apm_do_busy();
 
-        local_irq_enable();
+        return index;
 }
 
 /**
@@ -2381,9 +2399,9 @@ static int __init apm_init(void)
         if (HZ != 100)
                 idle_period = (idle_period * HZ) / 100;
         if (idle_threshold < 100) {
-                original_pm_idle = pm_idle;
-                pm_idle  = apm_cpu_idle;
-                set_pm_idle = 1;
+                if (!cpuidle_register_driver(&apm_idle_driver))
+                        if (cpuidle_register_device(&apm_cpuidle_device))
+                                cpuidle_unregister_driver(&apm_idle_driver);
         }
 
         return 0;
@@ -2393,15 +2411,9 @@ static void __exit apm_exit(void)
 {
         int error;
 
-        if (set_pm_idle) {
-                pm_idle = original_pm_idle;
-                /*
-                 * We are about to unload the current idle thread pm callback
-                 * (pm_idle), Wait for all processors to update cached/local
-                 * copies of pm_idle before proceeding.
-                 */
-                kick_all_cpus_sync();
-        }
+        cpuidle_unregister_device(&apm_cpuidle_device);
+        cpuidle_unregister_driver(&apm_idle_driver);
+
         if (((apm_info.bios.flags & APM_BIOS_DISENGAGED) == 0)
             && (apm_info.connection_version > 0x0100)) {
                 error = apm_engage_power_management(APM_DEVICE_ALL, 0);
diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c
index 15239fffd6fe..fa96eb0d02fb 100644
--- a/arch/x86/kernel/cpu/amd.c
+++ b/arch/x86/kernel/cpu/amd.c
@@ -12,7 +12,6 @@
 #include <asm/pci-direct.h>
 
 #ifdef CONFIG_X86_64
-# include <asm/numa_64.h>
 # include <asm/mmconfig.h>
 # include <asm/cacheflush.h>
 #endif
@@ -220,8 +219,7 @@ static void __cpuinit amd_k7_smp_check(struct cpuinfo_x86 *c)
          */
         WARN_ONCE(1, "WARNING: This combination of AMD"
                 " processors is not suitable for SMP.\n");
-        if (!test_taint(TAINT_UNSAFE_SMP))
-                add_taint(TAINT_UNSAFE_SMP);
+        add_taint(TAINT_UNSAFE_SMP, LOCKDEP_NOW_UNRELIABLE);
 
 valid_k7:
         ;
@@ -364,9 +362,9 @@ static void __cpuinit amd_detect_cmp(struct cpuinfo_x86 *c)
 #endif
 }
 
-int amd_get_nb_id(int cpu)
+u16 amd_get_nb_id(int cpu)
 {
-        int id = 0;
+        u16 id = 0;
 #ifdef CONFIG_SMP
         id = per_cpu(cpu_llc_id, cpu);
 #endif
@@ -518,10 +516,9 @@ static void __cpuinit early_init_amd(struct cpuinfo_x86 *c)
 static void __cpuinit init_amd(struct cpuinfo_x86 *c)
 {
         u32 dummy;
-
-#ifdef CONFIG_SMP
         unsigned long long value;
 
+#ifdef CONFIG_SMP
         /*
          * Disable TLB flush filter by setting HWCR.FFDIS on K8
          * bit 6 of msr C001_0015
@@ -559,12 +556,10 @@ static void __cpuinit init_amd(struct cpuinfo_x86 *c)
                  * (AMD Erratum #110, docId: 25759).
                  */
                 if (c->x86_model < 0x14 && cpu_has(c, X86_FEATURE_LAHF_LM)) {
-                        u64 val;
-
                         clear_cpu_cap(c, X86_FEATURE_LAHF_LM);
-                        if (!rdmsrl_amd_safe(0xc001100d, &val)) {
-                                val &= ~(1ULL << 32);
-                                wrmsrl_amd_safe(0xc001100d, val);
+                        if (!rdmsrl_amd_safe(0xc001100d, &value)) {
+                                value &= ~(1ULL << 32);
+                                wrmsrl_amd_safe(0xc001100d, value);
                         }
                 }
 
@@ -617,13 +612,12 @@ static void __cpuinit init_amd(struct cpuinfo_x86 *c)
         if ((c->x86 == 0x15) &&
             (c->x86_model >= 0x10) && (c->x86_model <= 0x1f) &&
             !cpu_has(c, X86_FEATURE_TOPOEXT)) {
-                u64 val;
 
-                if (!rdmsrl_safe(0xc0011005, &val)) {
-                        val |= 1ULL << 54;
-                        wrmsrl_safe(0xc0011005, val);
-                        rdmsrl(0xc0011005, val);
-                        if (val & (1ULL << 54)) {
+                if (!rdmsrl_safe(0xc0011005, &value)) {
+                        value |= 1ULL << 54;
+                        wrmsrl_safe(0xc0011005, value);
+                        rdmsrl(0xc0011005, value);
+                        if (value & (1ULL << 54)) {
                                 set_cpu_cap(c, X86_FEATURE_TOPOEXT);
                                 printk(KERN_INFO FW_INFO "CPU: Re-enabling "
                                   "disabled Topology Extensions Support\n");
@@ -637,11 +631,10 @@ static void __cpuinit init_amd(struct cpuinfo_x86 *c)
          */
         if ((c->x86 == 0x15) &&
             (c->x86_model >= 0x02) && (c->x86_model < 0x20)) {
-                u64 val;
 
-                if (!rdmsrl_safe(0xc0011021, &val) && !(val & 0x1E)) {
-                        val |= 0x1E;
-                        wrmsrl_safe(0xc0011021, val);
+                if (!rdmsrl_safe(0xc0011021, &value) && !(value & 0x1E)) {
+                        value |= 0x1E;
+                        wrmsrl_safe(0xc0011021, value);
                 }
         }
 
@@ -685,12 +678,10 @@ static void __cpuinit init_amd(struct cpuinfo_x86 *c)
                  * benefit in doing so.
                  */
                 if (!rdmsrl_safe(MSR_K8_TSEG_ADDR, &tseg)) {
+                        unsigned long pfn = tseg >> PAGE_SHIFT;
+
                         printk(KERN_DEBUG "tseg: %010llx\n", tseg);
-                        if ((tseg>>PMD_SHIFT) <
-                                (max_low_pfn_mapped>>(PMD_SHIFT-PAGE_SHIFT)) ||
-                                ((tseg>>PMD_SHIFT) <
-                                (max_pfn_mapped>>(PMD_SHIFT-PAGE_SHIFT)) &&
-                                (tseg>>PMD_SHIFT) >= (1ULL<<(32 - PMD_SHIFT))))
+                        if (pfn_range_is_mapped(pfn, pfn + 1))
                                 set_memory_4k((unsigned long)__va(tseg), 1);
                 }
         }
@@ -703,13 +694,11 @@ static void __cpuinit init_amd(struct cpuinfo_x86 *c)
         if (c->x86 > 0x11)
                 set_cpu_cap(c, X86_FEATURE_ARAT);
 
-        /*
-         * Disable GART TLB Walk Errors on Fam10h. We do this here
-         * because this is always needed when GART is enabled, even in a
-         * kernel which has no MCE support built in.
-         */
         if (c->x86 == 0x10) {
                 /*
+                 * Disable GART TLB Walk Errors on Fam10h. We do this here
+                 * because this is always needed when GART is enabled, even in a
+                 * kernel which has no MCE support built in.
                  * BIOS should disable GartTlbWlk Errors themself. If
                  * it doesn't do it here as suggested by the BKDG.
                  *
@@ -723,6 +712,21 @@ static void __cpuinit init_amd(struct cpuinfo_x86 *c)
                         mask |= (1 << 10);
                         wrmsrl_safe(MSR_AMD64_MCx_MASK(4), mask);
                 }
+
+                /*
+                 * On family 10h BIOS may not have properly enabled WC+ support,
+                 * causing it to be converted to CD memtype. This may result in
+                 * performance degradation for certain nested-paging guests.
+                 * Prevent this conversion by clearing bit 24 in
+                 * MSR_AMD64_BU_CFG2.
+                 *
+                 * NOTE: we want to use the _safe accessors so as not to #GP kvm
+                 * guests on older kvm hosts.
+                 */
+
+                rdmsrl_safe(MSR_AMD64_BU_CFG2, &value);
+                value &= ~(1ULL << 24);
+                wrmsrl_safe(MSR_AMD64_BU_CFG2, value);
         }
 
         rdmsr_safe(MSR_AMD64_PATCH_LEVEL, &c->microcode, &dummy);
diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index 92dfec986a48..af6455e3fcc9 100644
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -17,15 +17,6 @@
 #include <asm/paravirt.h>
 #include <asm/alternative.h>
 
-static int __init no_halt(char *s)
-{
-        WARN_ONCE(1, "\"no-hlt\" is deprecated, please use \"idle=poll\"\n");
-        boot_cpu_data.hlt_works_ok = 0;
-        return 1;
-}
-
-__setup("no-hlt", no_halt);
-
 static int __init no_387(char *s)
 {
         boot_cpu_data.hard_math = 0;
@@ -89,23 +80,6 @@ static void __init check_fpu(void)
                 pr_warn("Hmm, FPU with FDIV bug\n");
 }
 
-static void __init check_hlt(void)
-{
-        if (boot_cpu_data.x86 >= 5 || paravirt_enabled())
-                return;
-
-        pr_info("Checking 'hlt' instruction... ");
-        if (!boot_cpu_data.hlt_works_ok) {
-                pr_cont("disabled\n");
-                return;
-        }
-        halt();
-        halt();
-        halt();
-        halt();
-        pr_cont("OK\n");
-}
-
 /*
  * Check whether we are able to run this kernel safely on SMP.
  *
@@ -129,7 +103,6 @@ void __init check_bugs(void)
         print_cpu_info(&boot_cpu_data);
 #endif
         check_config();
-        check_hlt();
         init_utsname()->machine[1] =
                 '0' + (boot_cpu_data.x86 > 6 ? 6 : boot_cpu_data.x86);
         alternative_instructions();
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 9c3ab43a6954..d814772c5bed 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -37,6 +37,8 @@
 #include <asm/mce.h>
 #include <asm/msr.h>
 #include <asm/pat.h>
+#include <asm/microcode.h>
+#include <asm/microcode_intel.h>
 
 #ifdef CONFIG_X86_LOCAL_APIC
 #include <asm/uv/uv.h>
@@ -213,7 +215,7 @@ static inline int flag_is_changeable_p(u32 flag)
 }
 
 /* Probe for the CPUID instruction */
-static int __cpuinit have_cpuid_p(void)
+int __cpuinit have_cpuid_p(void)
 {
         return flag_is_changeable_p(X86_EFLAGS_ID);
 }
@@ -249,11 +251,6 @@ static inline int flag_is_changeable_p(u32 flag)
 {
         return 1;
 }
-/* Probe for the CPUID instruction */
-static inline int have_cpuid_p(void)
-{
-        return 1;
-}
 static inline void squash_the_stupid_serial_number(struct cpuinfo_x86 *c)
 {
 }
@@ -1223,6 +1220,12 @@ void __cpuinit cpu_init(void)
         int cpu;
         int i;
 
+        /*
+         * Load microcode on this cpu if a valid microcode is available.
+         * This is early microcode loading procedure.
+         */
+        load_ucode_ap();
+
         cpu = stack_smp_processor_id();
         t = &per_cpu(init_tss, cpu);
         oist = &per_cpu(orig_ist, cpu);
@@ -1314,6 +1317,8 @@ void __cpuinit cpu_init(void)
         struct tss_struct *t = &per_cpu(init_tss, cpu);
         struct thread_struct *thread = &curr->thread;
 
+        show_ucode_info_early();
+
         if (cpumask_test_and_set_cpu(cpu, cpu_initialized_mask)) {
                 printk(KERN_WARNING "CPU#%d already initialized!\n", cpu);
                 for (;;)
diff --git a/arch/x86/kernel/cpu/hypervisor.c b/arch/x86/kernel/cpu/hypervisor.c
index a8f8fa9769d6..1e7e84a02eba 100644
--- a/arch/x86/kernel/cpu/hypervisor.c
+++ b/arch/x86/kernel/cpu/hypervisor.c
@@ -79,3 +79,10 @@ void __init init_hypervisor_platform(void)
         if (x86_hyper->init_platform)
                 x86_hyper->init_platform();
 }
+
+bool __init hypervisor_x2apic_available(void)
+{
+        return x86_hyper                   &&
+               x86_hyper->x2apic_available &&
+               x86_hyper->x2apic_available();
+}
diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c
index fcaabd0432c5..1905ce98bee0 100644
--- a/arch/x86/kernel/cpu/intel.c
+++ b/arch/x86/kernel/cpu/intel.c
@@ -17,7 +17,6 @@
 
 #ifdef CONFIG_X86_64
 #include <linux/topology.h>
-#include <asm/numa_64.h>
 #endif
 
 #include "cpu.h"
@@ -168,7 +167,7 @@ int __cpuinit ppro_with_ram_bug(void)
 #ifdef CONFIG_X86_F00F_BUG
 static void __cpuinit trap_init_f00f_bug(void)
 {
-        __set_fixmap(FIX_F00F_IDT, __pa(&idt_table), PAGE_KERNEL_RO);
+        __set_fixmap(FIX_F00F_IDT, __pa_symbol(idt_table), PAGE_KERNEL_RO);
 
         /*
          * Update the IDT descriptor and reload the IDT so that
diff --git a/arch/x86/kernel/cpu/intel_cacheinfo.c b/arch/x86/kernel/cpu/intel_cacheinfo.c
index fe9edec6698a..7c6f7d548c0f 100644
--- a/arch/x86/kernel/cpu/intel_cacheinfo.c
+++ b/arch/x86/kernel/cpu/intel_cacheinfo.c
@@ -298,8 +298,7 @@ struct _cache_attr {
                          unsigned int);
 };
 
-#ifdef CONFIG_AMD_NB
-
+#if defined(CONFIG_AMD_NB) && defined(CONFIG_SYSFS)
 /*
  * L3 cache descriptors
  */
@@ -524,9 +523,9 @@ store_subcaches(struct _cpuid4_info *this_leaf, const char *buf, size_t count,
 static struct _cache_attr subcaches =
         __ATTR(subcaches, 0644, show_subcaches, store_subcaches);
 
-#else   /* CONFIG_AMD_NB */
+#else
 #define amd_init_l3_cache(x, y)
-#endif /* CONFIG_AMD_NB */
+#endif  /* CONFIG_AMD_NB && CONFIG_SYSFS */
 
 static int
 __cpuinit cpuid4_cache_lookup_regs(int index,
@@ -1227,7 +1226,7 @@ static struct notifier_block __cpuinitdata cacheinfo_cpu_notifier = {
         .notifier_call = cacheinfo_cpu_callback,
 };
 
-static int __cpuinit cache_sysfs_init(void)
+static int __init cache_sysfs_init(void)
 {
         int i;
 
diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c
index 80dbda84f1c3..7bc126346ace 100644
--- a/arch/x86/kernel/cpu/mcheck/mce.c
+++ b/arch/x86/kernel/cpu/mcheck/mce.c
@@ -512,11 +512,8 @@ int mce_available(struct cpuinfo_x86 *c)
 
 static void mce_schedule_work(void)
 {
-        if (!mce_ring_empty()) {
-                struct work_struct *work = &__get_cpu_var(mce_work);
-                if (!work_pending(work))
-                        schedule_work(work);
-        }
+        if (!mce_ring_empty())
+                schedule_work(&__get_cpu_var(mce_work));
 }
 
 DEFINE_PER_CPU(struct irq_work, mce_irq_work);
@@ -1085,7 +1082,7 @@ void do_machine_check(struct pt_regs *regs, long error_code)
                 /*
                  * Set taint even when machine check was not enabled.
                  */
-                add_taint(TAINT_MACHINE_CHECK);
+                add_taint(TAINT_MACHINE_CHECK, LOCKDEP_NOW_UNRELIABLE);
 
                 severity = mce_severity(&m, cfg->tolerant, NULL);
 
@@ -1351,12 +1348,7 @@ int mce_notify_irq(void)
                 /* wake processes polling /dev/mcelog */
                 wake_up_interruptible(&mce_chrdev_wait);
 
-                /*
-                 * There is no risk of missing notifications because
-                 * work_pending is always cleared before the function is
-                 * executed.
-                 */
-                if (mce_helper[0] && !work_pending(&mce_trigger_work))
+                if (mce_helper[0])
                         schedule_work(&mce_trigger_work);
 
                 if (__ratelimit(&ratelimit))
diff --git a/arch/x86/kernel/cpu/mcheck/p5.c b/arch/x86/kernel/cpu/mcheck/p5.c
index 2d5454cd2c4f..1c044b1ccc59 100644
--- a/arch/x86/kernel/cpu/mcheck/p5.c
+++ b/arch/x86/kernel/cpu/mcheck/p5.c
@@ -33,7 +33,7 @@ static void pentium_machine_check(struct pt_regs *regs, long error_code)
                         smp_processor_id());
         }
 
-        add_taint(TAINT_MACHINE_CHECK);
+        add_taint(TAINT_MACHINE_CHECK, LOCKDEP_NOW_UNRELIABLE);
 }
 
 /* Set up machine check reporting for processors with Intel style MCE: */
diff --git a/arch/x86/kernel/cpu/mcheck/winchip.c b/arch/x86/kernel/cpu/mcheck/winchip.c
index 2d7998fb628c..e9a701aecaa1 100644
--- a/arch/x86/kernel/cpu/mcheck/winchip.c
+++ b/arch/x86/kernel/cpu/mcheck/winchip.c
@@ -15,7 +15,7 @@
 static void winchip_machine_check(struct pt_regs *regs, long error_code)
 {
         printk(KERN_EMERG "CPU0: Machine Check Exception.\n");
-        add_taint(TAINT_MACHINE_CHECK);
+        add_taint(TAINT_MACHINE_CHECK, LOCKDEP_NOW_UNRELIABLE);
 }
 
 /* Set up machine check reporting on the Winchip C6 series */
diff --git a/arch/x86/kernel/cpu/mshyperv.c b/arch/x86/kernel/cpu/mshyperv.c
index 0a630dd4b620..a7d26d83fb70 100644
--- a/arch/x86/kernel/cpu/mshyperv.c
+++ b/arch/x86/kernel/cpu/mshyperv.c
@@ -14,10 +14,15 @@
 #include <linux/time.h>
 #include <linux/clocksource.h>
 #include <linux/module.h>
+#include <linux/hardirq.h>
+#include <linux/interrupt.h>
 #include <asm/processor.h>
 #include <asm/hypervisor.h>
 #include <asm/hyperv.h>
 #include <asm/mshyperv.h>
+#include <asm/desc.h>
+#include <asm/idle.h>
+#include <asm/irq_regs.h>
 
 struct ms_hyperv_info ms_hyperv;
 EXPORT_SYMBOL_GPL(ms_hyperv);
@@ -30,6 +35,13 @@ static bool __init ms_hyperv_platform(void)
         if (!boot_cpu_has(X86_FEATURE_HYPERVISOR))
                 return false;
 
+        /*
+         * Xen emulates Hyper-V to support enlightened Windows.
+         * Check to see first if we are on a Xen Hypervisor.
+         */
+        if (xen_cpuid_base())
+                return false;
+
         cpuid(HYPERV_CPUID_VENDOR_AND_MAX_FUNCTIONS,
               &eax, &hyp_signature[0], &hyp_signature[1], &hyp_signature[2]);
 
@@ -68,7 +80,14 @@ static void __init ms_hyperv_init_platform(void)
         printk(KERN_INFO "HyperV: features 0x%x, hints 0x%x\n",
                ms_hyperv.features, ms_hyperv.hints);
 
-        clocksource_register_hz(&hyperv_cs, NSEC_PER_SEC/100);
+        if (ms_hyperv.features & HV_X64_MSR_TIME_REF_COUNT_AVAILABLE)
+                clocksource_register_hz(&hyperv_cs, NSEC_PER_SEC/100);
+#if IS_ENABLED(CONFIG_HYPERV)
+        /*
+         * Setup the IDT for hypervisor callback.
+         */
+        alloc_intr_gate(HYPERVISOR_CALLBACK_VECTOR, hyperv_callback_vector);
+#endif
 }
 
 const __refconst struct hypervisor_x86 x86_hyper_ms_hyperv = {
@@ -77,3 +96,36 @@ const __refconst struct hypervisor_x86 x86_hyper_ms_hyperv = {
         .init_platform          = ms_hyperv_init_platform,
 };
 EXPORT_SYMBOL(x86_hyper_ms_hyperv);
+
+#if IS_ENABLED(CONFIG_HYPERV)
+static int vmbus_irq = -1;
+static irq_handler_t vmbus_isr;
+
+void hv_register_vmbus_handler(int irq, irq_handler_t handler)
+{
+        vmbus_irq = irq;
+        vmbus_isr = handler;
+}
+
+void hyperv_vector_handler(struct pt_regs *regs)
+{
+        struct pt_regs *old_regs = set_irq_regs(regs);
+        struct irq_desc *desc;
+
+        irq_enter();
+        exit_idle();
+
+        desc = irq_to_desc(vmbus_irq);
+
+        if (desc)
+                generic_handle_irq_desc(vmbus_irq, desc);
+
+        irq_exit();
+        set_irq_regs(old_regs);
+}
+#else
+void hv_register_vmbus_handler(int irq, irq_handler_t handler)
+{
+}
+#endif
+EXPORT_SYMBOL_GPL(hv_register_vmbus_handler);
diff --git a/arch/x86/kernel/cpu/mtrr/generic.c b/arch/x86/kernel/cpu/mtrr/generic.c
index e9fe907cd249..fa72a39e5d46 100644
--- a/arch/x86/kernel/cpu/mtrr/generic.c
+++ b/arch/x86/kernel/cpu/mtrr/generic.c
@@ -542,7 +542,7 @@ static void generic_get_mtrr(unsigned int reg, unsigned long *base,
 
                 if (tmp != mask_lo) {
                         printk(KERN_WARNING "mtrr: your BIOS has configured an incorrect mask, fixing it.\n");
-                        add_taint(TAINT_FIRMWARE_WORKAROUND);
+                        add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
                         mask_lo = tmp;
                 }
         }
diff --git a/arch/x86/kernel/cpu/perf_event.c b/arch/x86/kernel/cpu/perf_event.c
index 6774c17a5576..bf0f01aea994 100644
--- a/arch/x86/kernel/cpu/perf_event.c
+++ b/arch/x86/kernel/cpu/perf_event.c
@@ -829,7 +829,7 @@ static inline void x86_assign_hw_event(struct perf_event *event,
         } else {
                 hwc->config_base = x86_pmu_config_addr(hwc->idx);
                 hwc->event_base  = x86_pmu_event_addr(hwc->idx);
-                hwc->event_base_rdpmc = hwc->idx;
+                hwc->event_base_rdpmc = x86_pmu_rdpmc_index(hwc->idx);
         }
 }
 
@@ -1310,11 +1310,6 @@ static struct attribute_group x86_pmu_format_group = {
         .attrs = NULL,
 };
 
-struct perf_pmu_events_attr {
-        struct device_attribute attr;
-        u64 id;
-};
-
 /*
  * Remove all undefined events (x86_pmu.event_map(id) == 0)
  * out of events_attr attributes.
@@ -1348,11 +1343,9 @@ static ssize_t events_sysfs_show(struct device *dev, struct device_attribute *at
 #define EVENT_VAR(_id)  event_attr_##_id
 #define EVENT_PTR(_id) &event_attr_##_id.attr.attr
 
-#define EVENT_ATTR(_name, _id)                                  \
-static struct perf_pmu_events_attr EVENT_VAR(_id) = {           \
-        .attr = __ATTR(_name, 0444, events_sysfs_show, NULL),   \
-        .id   =  PERF_COUNT_HW_##_id,                           \
-};
+#define EVENT_ATTR(_name, _id)                                          \
+        PMU_EVENT_ATTR(_name, EVENT_VAR(_id), PERF_COUNT_HW_##_id,      \
+                        events_sysfs_show)
 
 EVENT_ATTR(cpu-cycles,                  CPU_CYCLES              );
 EVENT_ATTR(instructions,                INSTRUCTIONS            );
diff --git a/arch/x86/kernel/cpu/perf_event.h b/arch/x86/kernel/cpu/perf_event.h
index 115c1ea97746..7f5c75c2afdd 100644
--- a/arch/x86/kernel/cpu/perf_event.h
+++ b/arch/x86/kernel/cpu/perf_event.h
@@ -325,6 +325,8 @@ struct x86_pmu {
         int             (*schedule_events)(struct cpu_hw_events *cpuc, int n, int *assign);
         unsigned        eventsel;
         unsigned        perfctr;
+        int             (*addr_offset)(int index, bool eventsel);
+        int             (*rdpmc_index)(int index);
         u64             (*event_map)(int);
         int             max_events;
         int             num_counters;
@@ -446,28 +448,21 @@ extern u64 __read_mostly hw_cache_extra_regs
 
 u64 x86_perf_event_update(struct perf_event *event);
 
-static inline int x86_pmu_addr_offset(int index)
+static inline unsigned int x86_pmu_config_addr(int index)
 {
-        int offset;
-
-        /* offset = X86_FEATURE_PERFCTR_CORE ? index << 1 : index */
-        alternative_io(ASM_NOP2,
-                       "shll $1, %%eax",
-                       X86_FEATURE_PERFCTR_CORE,
-                       "=a" (offset),
-                       "a"  (index));
-
-        return offset;
+        return x86_pmu.eventsel + (x86_pmu.addr_offset ?
+                                   x86_pmu.addr_offset(index, true) : index);
 }
 
-static inline unsigned int x86_pmu_config_addr(int index)
+static inline unsigned int x86_pmu_event_addr(int index)
 {
-        return x86_pmu.eventsel + x86_pmu_addr_offset(index);
+        return x86_pmu.perfctr + (x86_pmu.addr_offset ?
+                                  x86_pmu.addr_offset(index, false) : index);
 }
 
-static inline unsigned int x86_pmu_event_addr(int index)
+static inline int x86_pmu_rdpmc_index(int index)
 {
-        return x86_pmu.perfctr + x86_pmu_addr_offset(index);
+        return x86_pmu.rdpmc_index ? x86_pmu.rdpmc_index(index) : index;
 }
 
 int x86_setup_perfctr(struct perf_event *event);
diff --git a/arch/x86/kernel/cpu/perf_event_amd.c b/arch/x86/kernel/cpu/perf_event_amd.c
index c93bc4e813a0..dfdab42aed27 100644
--- a/arch/x86/kernel/cpu/perf_event_amd.c
+++ b/arch/x86/kernel/cpu/perf_event_amd.c
@@ -132,21 +132,102 @@ static u64 amd_pmu_event_map(int hw_event)
         return amd_perfmon_event_map[hw_event];
 }
 
-static int amd_pmu_hw_config(struct perf_event *event)
+static struct event_constraint *amd_nb_event_constraint;
+
+/*
+ * Previously calculated offsets
+ */
+static unsigned int event_offsets[X86_PMC_IDX_MAX] __read_mostly;
+static unsigned int count_offsets[X86_PMC_IDX_MAX] __read_mostly;
+static unsigned int rdpmc_indexes[X86_PMC_IDX_MAX] __read_mostly;
+
+/*
+ * Legacy CPUs:
+ *   4 counters starting at 0xc0010000 each offset by 1
+ *
+ * CPUs with core performance counter extensions:
+ *   6 counters starting at 0xc0010200 each offset by 2
+ *
+ * CPUs with north bridge performance counter extensions:
+ *   4 additional counters starting at 0xc0010240 each offset by 2
+ *   (indexed right above either one of the above core counters)
+ */
+static inline int amd_pmu_addr_offset(int index, bool eventsel)
 {
-        int ret;
+        int offset, first, base;
 
-        /* pass precise event sampling to ibs: */
-        if (event->attr.precise_ip && get_ibs_caps())
-                return -ENOENT;
+        if (!index)
+                return index;
+
+        if (eventsel)
+                offset = event_offsets[index];
+        else
+                offset = count_offsets[index];
+
+        if (offset)
+                return offset;
+
+        if (amd_nb_event_constraint &&
+            test_bit(index, amd_nb_event_constraint->idxmsk)) {
+                /*
+                 * calculate the offset of NB counters with respect to
+                 * base eventsel or perfctr
+                 */
+
+                first = find_first_bit(amd_nb_event_constraint->idxmsk,
+                                       X86_PMC_IDX_MAX);
+
+                if (eventsel)
+                        base = MSR_F15H_NB_PERF_CTL - x86_pmu.eventsel;
+                else
+                        base = MSR_F15H_NB_PERF_CTR - x86_pmu.perfctr;
+
+                offset = base + ((index - first) << 1);
+        } else if (!cpu_has_perfctr_core)
+                offset = index;
+        else
+                offset = index << 1;
+
+        if (eventsel)
+                event_offsets[index] = offset;
+        else
+                count_offsets[index] = offset;
+
+        return offset;
+}
+
+static inline int amd_pmu_rdpmc_index(int index)
+{
+        int ret, first;
+
+        if (!index)
+                return index;
+
+        ret = rdpmc_indexes[index];
 
-        ret = x86_pmu_hw_config(event);
         if (ret)
                 return ret;
 
-        if (has_branch_stack(event))
-                return -EOPNOTSUPP;
+        if (amd_nb_event_constraint &&
+            test_bit(index, amd_nb_event_constraint->idxmsk)) {
+                /*
+                 * according to the mnual, ECX value of the NB counters is
+                 * the index of the NB counter (0, 1, 2 or 3) plus 6
+                 */
+
+                first = find_first_bit(amd_nb_event_constraint->idxmsk,
+                                       X86_PMC_IDX_MAX);
+                ret = index - first + 6;
+        } else
+                ret = index;
+
+        rdpmc_indexes[index] = ret;
+
+        return ret;
+}
 
+static int amd_core_hw_config(struct perf_event *event)
+{
         if (event->attr.exclude_host && event->attr.exclude_guest)
                 /*
                  * When HO == GO == 1 the hardware treats that as GO == HO == 0
@@ -156,14 +237,37 @@ static int amd_pmu_hw_config(struct perf_event *event)
                 event->hw.config &= ~(ARCH_PERFMON_EVENTSEL_USR |
                                       ARCH_PERFMON_EVENTSEL_OS);
         else if (event->attr.exclude_host)
-                event->hw.config |= AMD_PERFMON_EVENTSEL_GUESTONLY;
+                event->hw.config |= AMD64_EVENTSEL_GUESTONLY;
         else if (event->attr.exclude_guest)
-                event->hw.config |= AMD_PERFMON_EVENTSEL_HOSTONLY;
+                event->hw.config |= AMD64_EVENTSEL_HOSTONLY;
+
+        return 0;
+}
+
+/*
+ * NB counters do not support the following event select bits:
+ *   Host/Guest only
+ *   Counter mask
+ *   Invert counter mask
+ *   Edge detect
+ *   OS/User mode
+ */
+static int amd_nb_hw_config(struct perf_event *event)
+{
+        /* for NB, we only allow system wide counting mode */
+        if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)
+                return -EINVAL;
+
+        if (event->attr.exclude_user || event->attr.exclude_kernel ||
+            event->attr.exclude_host || event->attr.exclude_guest)
+                return -EINVAL;
 
-        if (event->attr.type != PERF_TYPE_RAW)
-                return 0;
+        event->hw.config &= ~(ARCH_PERFMON_EVENTSEL_USR |
+                              ARCH_PERFMON_EVENTSEL_OS);
 
-        event->hw.config |= event->attr.config & AMD64_RAW_EVENT_MASK;
+        if (event->hw.config & ~(AMD64_RAW_EVENT_MASK_NB |
+                                 ARCH_PERFMON_EVENTSEL_INT))
+                return -EINVAL;
 
         return 0;
 }
@@ -181,6 +285,11 @@ static inline int amd_is_nb_event(struct hw_perf_event *hwc)
         return (hwc->config & 0xe0) == 0xe0;
 }
 
+static inline int amd_is_perfctr_nb_event(struct hw_perf_event *hwc)
+{
+        return amd_nb_event_constraint && amd_is_nb_event(hwc);
+}
+
 static inline int amd_has_nb(struct cpu_hw_events *cpuc)
 {
         struct amd_nb *nb = cpuc->amd_nb;
@@ -188,20 +297,37 @@ static inline int amd_has_nb(struct cpu_hw_events *cpuc)
         return nb && nb->nb_id != -1;
 }
 
-static void amd_put_event_constraints(struct cpu_hw_events *cpuc,
-                                      struct perf_event *event)
+static int amd_pmu_hw_config(struct perf_event *event)
+{
+        int ret;
+
+        /* pass precise event sampling to ibs: */
+        if (event->attr.precise_ip && get_ibs_caps())
+                return -ENOENT;
+
+        if (has_branch_stack(event))
+                return -EOPNOTSUPP;
+
+        ret = x86_pmu_hw_config(event);
+        if (ret)
+                return ret;
+
+        if (event->attr.type == PERF_TYPE_RAW)
+                event->hw.config |= event->attr.config & AMD64_RAW_EVENT_MASK;
+
+        if (amd_is_perfctr_nb_event(&event->hw))
+                return amd_nb_hw_config(event);
+
+        return amd_core_hw_config(event);
+}
+
+static void __amd_put_nb_event_constraints(struct cpu_hw_events *cpuc,
+                                           struct perf_event *event)
 {
-        struct hw_perf_event *hwc = &event->hw;
         struct amd_nb *nb = cpuc->amd_nb;
         int i;
 
         /*
-         * only care about NB events
-         */
-        if (!(amd_has_nb(cpuc) && amd_is_nb_event(hwc)))
-                return;
-
-        /*
          * need to scan whole list because event may not have
          * been assigned during scheduling
          *
@@ -215,6 +341,19 @@ static void amd_put_event_constraints(struct cpu_hw_events *cpuc,
         }
 }
 
+static void amd_nb_interrupt_hw_config(struct hw_perf_event *hwc)
+{
+        int core_id = cpu_data(smp_processor_id()).cpu_core_id;
+
+        /* deliver interrupts only to this core */
+        if (hwc->config & ARCH_PERFMON_EVENTSEL_INT) {
+                hwc->config |= AMD64_EVENTSEL_INT_CORE_ENABLE;
+                hwc->config &= ~AMD64_EVENTSEL_INT_CORE_SEL_MASK;
+                hwc->config |= (u64)(core_id) <<
+                        AMD64_EVENTSEL_INT_CORE_SEL_SHIFT;
+        }
+}
+
  /*
   * AMD64 NorthBridge events need special treatment because
   * counter access needs to be synchronized across all cores
@@ -247,24 +386,24 @@ static void amd_put_event_constraints(struct cpu_hw_events *cpuc,
   *
   * Given that resources are allocated (cmpxchg), they must be
   * eventually freed for others to use. This is accomplished by
-  * calling amd_put_event_constraints().
+  * calling __amd_put_nb_event_constraints()
   *
   * Non NB events are not impacted by this restriction.
   */
 static struct event_constraint *
-amd_get_event_constraints(struct cpu_hw_events *cpuc, struct perf_event *event)
+__amd_get_nb_event_constraints(struct cpu_hw_events *cpuc, struct perf_event *event,
+                               struct event_constraint *c)
 {
         struct hw_perf_event *hwc = &event->hw;
         struct amd_nb *nb = cpuc->amd_nb;
-        struct perf_event *old = NULL;
-        int max = x86_pmu.num_counters;
-        int i, j, k = -1;
+        struct perf_event *old;
+        int idx, new = -1;
 
-        /*
-         * if not NB event or no NB, then no constraints
-         */
-        if (!(amd_has_nb(cpuc) && amd_is_nb_event(hwc)))
-                return &unconstrained;
+        if (!c)
+                c = &unconstrained;
+
+        if (cpuc->is_fake)
+                return c;
 
         /*
          * detect if already present, if so reuse
@@ -276,48 +415,36 @@ amd_get_event_constraints(struct cpu_hw_events *cpuc, struct perf_event *event)
          * because of successive calls to x86_schedule_events() from
          * hw_perf_group_sched_in() without hw_perf_enable()
          */
-        for (i = 0; i < max; i++) {
-                /*
-                 * keep track of first free slot
-                 */
-                if (k == -1 && !nb->owners[i])
-                        k = i;
+        for_each_set_bit(idx, c->idxmsk, x86_pmu.num_counters) {
+                if (new == -1 || hwc->idx == idx)
+                        /* assign free slot, prefer hwc->idx */
+                        old = cmpxchg(nb->owners + idx, NULL, event);
+                else if (nb->owners[idx] == event)
+                        /* event already present */
+                        old = event;
+                else
+                        continue;
+
+                if (old && old != event)
+                        continue;
+
+                /* reassign to this slot */
+                if (new != -1)
+                        cmpxchg(nb->owners + new, event, NULL);
+                new = idx;
 
                 /* already present, reuse */
-                if (nb->owners[i] == event)
-                        goto done;
-        }
-        /*
-         * not present, so grab a new slot
-         * starting either at:
-         */
-        if (hwc->idx != -1) {
-                /* previous assignment */
-                i = hwc->idx;
-        } else if (k != -1) {
-                /* start from free slot found */
-                i = k;
-        } else {
-                /*
-                 * event not found, no slot found in
-                 * first pass, try again from the
-                 * beginning
-                 */
-                i = 0;
-        }
-        j = i;
-        do {
-                old = cmpxchg(nb->owners+i, NULL, event);
-                if (!old)
+                if (old == event)
                         break;
-                if (++i == max)
-                        i = 0;
-        } while (i != j);
-done:
-        if (!old)
-                return &nb->event_constraints[i];
-
-        return &emptyconstraint;
+        }
+
+        if (new == -1)
+                return &emptyconstraint;
+
+        if (amd_is_perfctr_nb_event(hwc))
+                amd_nb_interrupt_hw_config(hwc);
+
+        return &nb->event_constraints[new];
 }
 
 static struct amd_nb *amd_alloc_nb(int cpu)
@@ -364,7 +491,7 @@ static void amd_pmu_cpu_starting(int cpu)
         struct amd_nb *nb;
         int i, nb_id;
 
-        cpuc->perf_ctr_virt_mask = AMD_PERFMON_EVENTSEL_HOSTONLY;
+        cpuc->perf_ctr_virt_mask = AMD64_EVENTSEL_HOSTONLY;
 
         if (boot_cpu_data.x86_max_cores < 2)
                 return;
@@ -407,6 +534,26 @@ static void amd_pmu_cpu_dead(int cpu)
         }
 }
 
+static struct event_constraint *
+amd_get_event_constraints(struct cpu_hw_events *cpuc, struct perf_event *event)
+{
+        /*
+         * if not NB event or no NB, then no constraints
+         */
+        if (!(amd_has_nb(cpuc) && amd_is_nb_event(&event->hw)))
+                return &unconstrained;
+
+        return __amd_get_nb_event_constraints(cpuc, event,
+                                              amd_nb_event_constraint);
+}
+
+static void amd_put_event_constraints(struct cpu_hw_events *cpuc,
+                                      struct perf_event *event)
+{
+        if (amd_has_nb(cpuc) && amd_is_nb_event(&event->hw))
+                __amd_put_nb_event_constraints(cpuc, event);
+}
+
 PMU_FORMAT_ATTR(event,  "config:0-7,32-35");
 PMU_FORMAT_ATTR(umask,  "config:8-15"   );
 PMU_FORMAT_ATTR(edge,   "config:18"     );
@@ -496,6 +643,9 @@ static struct event_constraint amd_f15_PMC30 = EVENT_CONSTRAINT_OVERLAP(0, 0x09,
 static struct event_constraint amd_f15_PMC50 = EVENT_CONSTRAINT(0, 0x3F, 0);
 static struct event_constraint amd_f15_PMC53 = EVENT_CONSTRAINT(0, 0x38, 0);
 
+static struct event_constraint amd_NBPMC96 = EVENT_CONSTRAINT(0, 0x3C0, 0);
+static struct event_constraint amd_NBPMC74 = EVENT_CONSTRAINT(0, 0xF0, 0);
+
 static struct event_constraint *
 amd_get_event_constraints_f15h(struct cpu_hw_events *cpuc, struct perf_event *event)
 {
@@ -561,8 +711,8 @@ amd_get_event_constraints_f15h(struct cpu_hw_events *cpuc, struct perf_event *ev
                         return &amd_f15_PMC20;
                 }
         case AMD_EVENT_NB:
-                /* not yet implemented */
-                return &emptyconstraint;
+                return __amd_get_nb_event_constraints(cpuc, event,
+                                                      amd_nb_event_constraint);
         default:
                 return &emptyconstraint;
         }
@@ -587,6 +737,8 @@ static __initconst const struct x86_pmu amd_pmu = {
         .schedule_events        = x86_schedule_events,
         .eventsel               = MSR_K7_EVNTSEL0,
         .perfctr                = MSR_K7_PERFCTR0,
+        .addr_offset            = amd_pmu_addr_offset,
+        .rdpmc_index            = amd_pmu_rdpmc_index,
         .event_map              = amd_pmu_event_map,
         .max_events             = ARRAY_SIZE(amd_perfmon_event_map),
         .num_counters           = AMD64_NUM_COUNTERS,
@@ -608,7 +760,7 @@ static __initconst const struct x86_pmu amd_pmu = {
 
 static int setup_event_constraints(void)
 {
-        if (boot_cpu_data.x86 >= 0x15)
+        if (boot_cpu_data.x86 == 0x15)
                 x86_pmu.get_event_constraints = amd_get_event_constraints_f15h;
         return 0;
 }
@@ -638,6 +790,23 @@ static int setup_perfctr_core(void)
         return 0;
 }
 
+static int setup_perfctr_nb(void)
+{
+        if (!cpu_has_perfctr_nb)
+                return -ENODEV;
+
+        x86_pmu.num_counters += AMD64_NUM_COUNTERS_NB;
+
+        if (cpu_has_perfctr_core)
+                amd_nb_event_constraint = &amd_NBPMC96;
+        else
+                amd_nb_event_constraint = &amd_NBPMC74;
+
+        printk(KERN_INFO "perf: AMD northbridge performance counters detected\n");
+
+        return 0;
+}
+
 __init int amd_pmu_init(void)
 {
         /* Performance-monitoring supported from K7 and later: */
@@ -648,6 +817,7 @@ __init int amd_pmu_init(void)
 
         setup_event_constraints();
         setup_perfctr_core();
+        setup_perfctr_nb();
 
         /* Events are common for all AMDs */
         memcpy(hw_cache_event_ids, amd_hw_cache_event_ids,
@@ -678,7 +848,7 @@ void amd_pmu_disable_virt(void)
          * SVM is disabled the Guest-only bits still gets set and the counter
          * will not count anything.
          */
-        cpuc->perf_ctr_virt_mask = AMD_PERFMON_EVENTSEL_HOSTONLY;
+        cpuc->perf_ctr_virt_mask = AMD64_EVENTSEL_HOSTONLY;
 
         /* Reload all events */
         x86_pmu_disable_all();
diff --git a/arch/x86/kernel/cpu/perf_event_amd_ibs.c b/arch/x86/kernel/cpu/perf_event_amd_ibs.c
index 6336bcbd0618..5f0581e713c2 100644
--- a/arch/x86/kernel/cpu/perf_event_amd_ibs.c
+++ b/arch/x86/kernel/cpu/perf_event_amd_ibs.c
@@ -528,7 +528,7 @@ static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs)
         if (!test_bit(IBS_STARTED, pcpu->state)) {
                 /*
                  * Catch spurious interrupts after stopping IBS: After
-                 * disabling IBS there could be still incomming NMIs
+                 * disabling IBS there could be still incoming NMIs
                  * with samples that even have the valid bit cleared.
                  * Mark all this NMIs as handled.
                  */
diff --git a/arch/x86/kernel/cpu/perf_event_intel.c b/arch/x86/kernel/cpu/perf_event_intel.c
index 93b9e1181f83..dab7580c47ae 100644
--- a/arch/x86/kernel/cpu/perf_event_intel.c
+++ b/arch/x86/kernel/cpu/perf_event_intel.c
@@ -101,12 +101,37 @@ static struct event_constraint intel_snb_event_constraints[] __read_mostly =
         FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
         FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
         FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */
+        INTEL_UEVENT_CONSTRAINT(0x04a3, 0xf), /* CYCLE_ACTIVITY.CYCLES_NO_DISPATCH */
+        INTEL_UEVENT_CONSTRAINT(0x05a3, 0xf), /* CYCLE_ACTIVITY.STALLS_L2_PENDING */
+        INTEL_UEVENT_CONSTRAINT(0x02a3, 0x4), /* CYCLE_ACTIVITY.CYCLES_L1D_PENDING */
+        INTEL_UEVENT_CONSTRAINT(0x06a3, 0x4), /* CYCLE_ACTIVITY.STALLS_L1D_PENDING */
         INTEL_EVENT_CONSTRAINT(0x48, 0x4), /* L1D_PEND_MISS.PENDING */
         INTEL_UEVENT_CONSTRAINT(0x01c0, 0x2), /* INST_RETIRED.PREC_DIST */
         INTEL_EVENT_CONSTRAINT(0xcd, 0x8), /* MEM_TRANS_RETIRED.LOAD_LATENCY */
         EVENT_CONSTRAINT_END
 };
 
+static struct event_constraint intel_ivb_event_constraints[] __read_mostly =
+{
+        FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
+        FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
+        FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */
+        INTEL_UEVENT_CONSTRAINT(0x0148, 0x4), /* L1D_PEND_MISS.PENDING */
+        INTEL_UEVENT_CONSTRAINT(0x0279, 0xf), /* IDQ.EMTPY */
+        INTEL_UEVENT_CONSTRAINT(0x019c, 0xf), /* IDQ_UOPS_NOT_DELIVERED.CORE */
+        INTEL_UEVENT_CONSTRAINT(0x04a3, 0xf), /* CYCLE_ACTIVITY.CYCLES_NO_EXECUTE */
+        INTEL_UEVENT_CONSTRAINT(0x05a3, 0xf), /* CYCLE_ACTIVITY.STALLS_L2_PENDING */
+        INTEL_UEVENT_CONSTRAINT(0x06a3, 0xf), /* CYCLE_ACTIVITY.STALLS_LDM_PENDING */
+        INTEL_UEVENT_CONSTRAINT(0x08a3, 0x4), /* CYCLE_ACTIVITY.CYCLES_L1D_PENDING */
+        INTEL_UEVENT_CONSTRAINT(0x0ca3, 0x4), /* CYCLE_ACTIVITY.STALLS_L1D_PENDING */
+        INTEL_UEVENT_CONSTRAINT(0x01c0, 0x2), /* INST_RETIRED.PREC_DIST */
+        INTEL_EVENT_CONSTRAINT(0xd0, 0xf), /* MEM_UOPS_RETIRED.* */
+        INTEL_EVENT_CONSTRAINT(0xd1, 0xf), /* MEM_LOAD_UOPS_RETIRED.* */
+        INTEL_EVENT_CONSTRAINT(0xd2, 0xf), /* MEM_LOAD_UOPS_LLC_HIT_RETIRED.* */
+        INTEL_EVENT_CONSTRAINT(0xd3, 0xf), /*  MEM_LOAD_UOPS_LLC_MISS_RETIRED.* */
+        EVENT_CONSTRAINT_END
+};
+
 static struct extra_reg intel_westmere_extra_regs[] __read_mostly =
 {
         INTEL_EVENT_EXTRA_REG(0xb7, MSR_OFFCORE_RSP_0, 0xffff, RSP_0),
@@ -2019,7 +2044,10 @@ __init int intel_pmu_init(void)
                 break;
 
         case 28: /* Atom */
-        case 54: /* Cedariew */
+        case 38: /* Lincroft */
+        case 39: /* Penwell */
+        case 53: /* Cloverview */
+        case 54: /* Cedarview */
                 memcpy(hw_cache_event_ids, atom_hw_cache_event_ids,
                        sizeof(hw_cache_event_ids));
 
@@ -2084,6 +2112,7 @@ __init int intel_pmu_init(void)
                 pr_cont("SandyBridge events, ");
                 break;
         case 58: /* IvyBridge */
+        case 62: /* IvyBridge EP */
                 memcpy(hw_cache_event_ids, snb_hw_cache_event_ids,
                        sizeof(hw_cache_event_ids));
                 memcpy(hw_cache_extra_regs, snb_hw_cache_extra_regs,
@@ -2091,7 +2120,7 @@ __init int intel_pmu_init(void)
 
                 intel_pmu_lbr_init_snb();
 
-                x86_pmu.event_constraints = intel_snb_event_constraints;
+                x86_pmu.event_constraints = intel_ivb_event_constraints;
                 x86_pmu.pebs_constraints = intel_ivb_pebs_event_constraints;
                 x86_pmu.pebs_aliases = intel_pebs_aliases_snb;
                 x86_pmu.extra_regs = intel_snb_extra_regs;
diff --git a/arch/x86/kernel/cpu/perf_event_intel_ds.c b/arch/x86/kernel/cpu/perf_event_intel_ds.c
index 826054a4f2ee..b05a575d56f4 100644
--- a/arch/x86/kernel/cpu/perf_event_intel_ds.c
+++ b/arch/x86/kernel/cpu/perf_event_intel_ds.c
@@ -729,3 +729,13 @@ void intel_ds_init(void)
                 }
         }
 }
+
+void perf_restore_debug_store(void)
+{
+        struct debug_store *ds = __this_cpu_read(cpu_hw_events.ds);
+
+        if (!x86_pmu.bts && !x86_pmu.pebs)
+                return;
+
+        wrmsrl(MSR_IA32_DS_AREA, (unsigned long)ds);
+}
diff --git a/arch/x86/kernel/cpu/perf_event_p6.c b/arch/x86/kernel/cpu/perf_event_p6.c
index f2af39f5dc3d..4820c232a0b9 100644
--- a/arch/x86/kernel/cpu/perf_event_p6.c
+++ b/arch/x86/kernel/cpu/perf_event_p6.c
@@ -19,7 +19,7 @@ static const u64 p6_perfmon_event_map[] =
 
 };
 
-static __initconst u64 p6_hw_cache_event_ids
+static u64 p6_hw_cache_event_ids
                                 [PERF_COUNT_HW_CACHE_MAX]
                                 [PERF_COUNT_HW_CACHE_OP_MAX]
                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
diff --git a/arch/x86/kernel/cpu/proc.c b/arch/x86/kernel/cpu/proc.c
index 3286a92e662a..e280253f6f94 100644
--- a/arch/x86/kernel/cpu/proc.c
+++ b/arch/x86/kernel/cpu/proc.c
@@ -28,7 +28,6 @@ static void show_cpuinfo_misc(struct seq_file *m, struct cpuinfo_x86 *c)
 {
         seq_printf(m,
                    "fdiv_bug\t: %s\n"
-                   "hlt_bug\t\t: %s\n"
                    "f00f_bug\t: %s\n"
                    "coma_bug\t: %s\n"
                    "fpu\t\t: %s\n"
@@ -36,7 +35,6 @@ static void show_cpuinfo_misc(struct seq_file *m, struct cpuinfo_x86 *c)
                    "cpuid level\t: %d\n"
                    "wp\t\t: %s\n",
                    c->fdiv_bug ? "yes" : "no",
-                   c->hlt_works_ok ? "no" : "yes",
                    c->f00f_bug ? "yes" : "no",
                    c->coma_bug ? "yes" : "no",
                    c->hard_math ? "yes" : "no",
diff --git a/arch/x86/kernel/cpu/vmware.c b/arch/x86/kernel/cpu/vmware.c
index d22d0c4edcfd..03a36321ec54 100644
--- a/arch/x86/kernel/cpu/vmware.c
+++ b/arch/x86/kernel/cpu/vmware.c
@@ -33,6 +33,9 @@
 
 #define VMWARE_PORT_CMD_GETVERSION      10
 #define VMWARE_PORT_CMD_GETHZ           45
+#define VMWARE_PORT_CMD_GETVCPU_INFO    68
+#define VMWARE_PORT_CMD_LEGACY_X2APIC   3
+#define VMWARE_PORT_CMD_VCPU_RESERVED   31
 
 #define VMWARE_PORT(cmd, eax, ebx, ecx, edx)                            \
         __asm__("inl (%%dx)" :                                          \
@@ -125,10 +128,20 @@ static void __cpuinit vmware_set_cpu_features(struct cpuinfo_x86 *c)
         set_cpu_cap(c, X86_FEATURE_TSC_RELIABLE);
 }
 
+/* Checks if hypervisor supports x2apic without VT-D interrupt remapping. */
+static bool __init vmware_legacy_x2apic_available(void)
+{
+        uint32_t eax, ebx, ecx, edx;
+        VMWARE_PORT(GETVCPU_INFO, eax, ebx, ecx, edx);
+        return (eax & (1 << VMWARE_PORT_CMD_VCPU_RESERVED)) == 0 &&
+               (eax & (1 << VMWARE_PORT_CMD_LEGACY_X2APIC)) != 0;
+}
+
 const __refconst struct hypervisor_x86 x86_hyper_vmware = {
         .name                   = "VMware",
         .detect                 = vmware_platform,
         .set_cpu_features       = vmware_set_cpu_features,
         .init_platform          = vmware_platform_setup,
+        .x2apic_available       = vmware_legacy_x2apic_available,
 };
 EXPORT_SYMBOL(x86_hyper_vmware);
diff --git a/arch/x86/kernel/cpuid.c b/arch/x86/kernel/cpuid.c
index 60c78917190c..1e4dbcfe6d31 100644
--- a/arch/x86/kernel/cpuid.c
+++ b/arch/x86/kernel/cpuid.c
@@ -85,7 +85,7 @@ static ssize_t cpuid_read(struct file *file, char __user *buf,
 {
         char __user *tmp = buf;
         struct cpuid_regs cmd;
-        int cpu = iminor(file->f_path.dentry->d_inode);
+        int cpu = iminor(file_inode(file));
         u64 pos = *ppos;
         ssize_t bytes = 0;
         int err = 0;
@@ -116,7 +116,7 @@ static int cpuid_open(struct inode *inode, struct file *file)
         unsigned int cpu;
         struct cpuinfo_x86 *c;
 
-        cpu = iminor(file->f_path.dentry->d_inode);
+        cpu = iminor(file_inode(file));
         if (cpu >= nr_cpu_ids || !cpu_online(cpu))
                 return -ENXIO;  /* No such CPU */
 
diff --git a/arch/x86/kernel/dumpstack.c b/arch/x86/kernel/dumpstack.c
index ae42418bc50f..c8797d55b245 100644
--- a/arch/x86/kernel/dumpstack.c
+++ b/arch/x86/kernel/dumpstack.c
@@ -232,7 +232,7 @@ void __kprobes oops_end(unsigned long flags, struct pt_regs *regs, int signr)
 
         bust_spinlocks(0);
         die_owner = -1;
-        add_taint(TAINT_DIE);
+        add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE);
         die_nest_count--;
         if (!die_nest_count)
                 /* Nest count reaches zero, release the lock. */
diff --git a/arch/x86/kernel/e820.c b/arch/x86/kernel/e820.c
index df06ade26bef..d32abeabbda5 100644
--- a/arch/x86/kernel/e820.c
+++ b/arch/x86/kernel/e820.c
@@ -835,7 +835,7 @@ static int __init parse_memopt(char *p)
 }
 early_param("mem", parse_memopt);
 
-static int __init parse_memmap_opt(char *p)
+static int __init parse_memmap_one(char *p)
 {
         char *oldp;
         u64 start_at, mem_size;
@@ -877,6 +877,20 @@ static int __init parse_memmap_opt(char *p)
 
         return *p == '\0' ? 0 : -EINVAL;
 }
+static int __init parse_memmap_opt(char *str)
+{
+        while (str) {
+                char *k = strchr(str, ',');
+
+                if (k)
+                        *k++ = 0;
+
+                parse_memmap_one(str);
+                str = k;
+        }
+
+        return 0;
+}
 early_param("memmap", parse_memmap_opt);
 
 void __init finish_e820_parsing(void)
diff --git a/arch/x86/kernel/entry_32.S b/arch/x86/kernel/entry_32.S
index 6ed91d9980e2..8f3e2dec1df3 100644
--- a/arch/x86/kernel/entry_32.S
+++ b/arch/x86/kernel/entry_32.S
@@ -699,51 +699,6 @@ END(syscall_badsys)
  */
         .popsection
 
-/*
- * System calls that need a pt_regs pointer.
- */
-#define PTREGSCALL0(name) \
-ENTRY(ptregs_##name) ;  \
-        leal 4(%esp),%eax; \
-        jmp sys_##name; \
-ENDPROC(ptregs_##name)
-
-#define PTREGSCALL1(name) \
-ENTRY(ptregs_##name) ; \
-        leal 4(%esp),%edx; \
-        movl (PT_EBX+4)(%esp),%eax; \
-        jmp sys_##name; \
-ENDPROC(ptregs_##name)
-
-#define PTREGSCALL2(name) \
-ENTRY(ptregs_##name) ; \
-        leal 4(%esp),%ecx; \
-        movl (PT_ECX+4)(%esp),%edx; \
-        movl (PT_EBX+4)(%esp),%eax; \
-        jmp sys_##name; \
-ENDPROC(ptregs_##name)
-
-#define PTREGSCALL3(name) \
-ENTRY(ptregs_##name) ; \
-        CFI_STARTPROC; \
-        leal 4(%esp),%eax; \
-        pushl_cfi %eax; \
-        movl PT_EDX(%eax),%ecx; \
-        movl PT_ECX(%eax),%edx; \
-        movl PT_EBX(%eax),%eax; \
-        call sys_##name; \
-        addl $4,%esp; \
-        CFI_ADJUST_CFA_OFFSET -4; \
-        ret; \
-        CFI_ENDPROC; \
-ENDPROC(ptregs_##name)
-
-PTREGSCALL1(iopl)
-PTREGSCALL0(sigreturn)
-PTREGSCALL0(rt_sigreturn)
-PTREGSCALL2(vm86)
-PTREGSCALL1(vm86old)
-
 .macro FIXUP_ESPFIX_STACK
 /*
  * Switch back for ESPFIX stack to the normal zerobased stack
@@ -1091,11 +1046,18 @@ ENTRY(xen_failsafe_callback)
         _ASM_EXTABLE(4b,9b)
 ENDPROC(xen_failsafe_callback)
 
-BUILD_INTERRUPT3(xen_hvm_callback_vector, XEN_HVM_EVTCHN_CALLBACK,
+BUILD_INTERRUPT3(xen_hvm_callback_vector, HYPERVISOR_CALLBACK_VECTOR,
                 xen_evtchn_do_upcall)
 
 #endif  /* CONFIG_XEN */
 
+#if IS_ENABLED(CONFIG_HYPERV)
+
+BUILD_INTERRUPT3(hyperv_callback_vector, HYPERVISOR_CALLBACK_VECTOR,
+        hyperv_vector_handler)
+
+#endif /* CONFIG_HYPERV */
+
 #ifdef CONFIG_FUNCTION_TRACER
 #ifdef CONFIG_DYNAMIC_FTRACE
 
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index cb3c591339aa..c1d01e6ca790 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -828,23 +828,6 @@ int_restore_rest:
         CFI_ENDPROC
 END(system_call)
 
-/*
- * Certain special system calls that need to save a complete full stack frame.
- */
-        .macro PTREGSCALL label,func,arg
-ENTRY(\label)
-        PARTIAL_FRAME 1 8               /* offset 8: return address */
-        subq $REST_SKIP, %rsp
-        CFI_ADJUST_CFA_OFFSET REST_SKIP
-        call save_rest
-        DEFAULT_FRAME 0 8               /* offset 8: return address */
-        leaq 8(%rsp), \arg      /* pt_regs pointer */
-        call \func
-        jmp ptregscall_common
-        CFI_ENDPROC
-END(\label)
-        .endm
-
         .macro FORK_LIKE func
 ENTRY(stub_\func)
         CFI_STARTPROC
@@ -861,10 +844,22 @@ ENTRY(stub_\func)
 END(stub_\func)
         .endm
 
+        .macro FIXED_FRAME label,func
+ENTRY(\label)
+        CFI_STARTPROC
+        PARTIAL_FRAME 0 8               /* offset 8: return address */
+        FIXUP_TOP_OF_STACK %r11, 8-ARGOFFSET
+        call \func
+        RESTORE_TOP_OF_STACK %r11, 8-ARGOFFSET
+        ret
+        CFI_ENDPROC
+END(\label)
+        .endm
+
         FORK_LIKE  clone
         FORK_LIKE  fork
         FORK_LIKE  vfork
-        PTREGSCALL stub_iopl, sys_iopl, %rsi
+        FIXED_FRAME stub_iopl, sys_iopl
 
 ENTRY(ptregscall_common)
         DEFAULT_FRAME 1 8       /* offset 8: return address */
@@ -886,7 +881,6 @@ ENTRY(stub_execve)
         SAVE_REST
         FIXUP_TOP_OF_STACK %r11
         call sys_execve
-        RESTORE_TOP_OF_STACK %r11
         movq %rax,RAX(%rsp)
         RESTORE_REST
         jmp int_ret_from_sys_call
@@ -902,7 +896,6 @@ ENTRY(stub_rt_sigreturn)
         addq $8, %rsp
         PARTIAL_FRAME 0
         SAVE_REST
-        movq %rsp,%rdi
         FIXUP_TOP_OF_STACK %r11
         call sys_rt_sigreturn
         movq %rax,RAX(%rsp) # fixme, this could be done at the higher layer
@@ -917,7 +910,6 @@ ENTRY(stub_x32_rt_sigreturn)
         addq $8, %rsp
         PARTIAL_FRAME 0
         SAVE_REST
-        movq %rsp,%rdi
         FIXUP_TOP_OF_STACK %r11
         call sys32_x32_rt_sigreturn
         movq %rax,RAX(%rsp) # fixme, this could be done at the higher layer
@@ -1454,11 +1446,16 @@ ENTRY(xen_failsafe_callback)
         CFI_ENDPROC
 END(xen_failsafe_callback)
 
-apicinterrupt XEN_HVM_EVTCHN_CALLBACK \
+apicinterrupt HYPERVISOR_CALLBACK_VECTOR \
         xen_hvm_callback_vector xen_evtchn_do_upcall
 
 #endif /* CONFIG_XEN */
 
+#if IS_ENABLED(CONFIG_HYPERV)
+apicinterrupt HYPERVISOR_CALLBACK_VECTOR \
+        hyperv_callback_vector hyperv_vector_handler
+#endif /* CONFIG_HYPERV */
+
 /*
  * Some functions should be protected against kprobes
  */
diff --git a/arch/x86/kernel/ftrace.c b/arch/x86/kernel/ftrace.c
index 1d414029f1d8..42a392a9fd02 100644
--- a/arch/x86/kernel/ftrace.c
+++ b/arch/x86/kernel/ftrace.c
@@ -89,7 +89,7 @@ do_ftrace_mod_code(unsigned long ip, const void *new_code)
          * kernel identity mapping to modify code.
          */
         if (within(ip, (unsigned long)_text, (unsigned long)_etext))
-                ip = (unsigned long)__va(__pa(ip));
+                ip = (unsigned long)__va(__pa_symbol(ip));
 
         return probe_kernel_write((void *)ip, new_code, MCOUNT_INSN_SIZE);
 }
@@ -279,7 +279,7 @@ static int ftrace_write(unsigned long ip, const char *val, int size)
          * kernel identity mapping to modify code.
          */
         if (within(ip, (unsigned long)_text, (unsigned long)_etext))
-                ip = (unsigned long)__va(__pa(ip));
+                ip = (unsigned long)__va(__pa_symbol(ip));
 
         return probe_kernel_write((void *)ip, val, size);
 }
diff --git a/arch/x86/kernel/head.c b/arch/x86/kernel/head.c
index 48d9d4ea1020..992f442ca155 100644
--- a/arch/x86/kernel/head.c
+++ b/arch/x86/kernel/head.c
@@ -5,8 +5,6 @@
 #include <asm/setup.h>
 #include <asm/bios_ebda.h>
 
-#define BIOS_LOWMEM_KILOBYTES 0x413
-
 /*
  * The BIOS places the EBDA/XBDA at the top of conventional
  * memory, and usually decreases the reported amount of
@@ -16,17 +14,30 @@
  * chipset: reserve a page before VGA to prevent PCI prefetch
  * into it (errata #56). Usually the page is reserved anyways,
  * unless you have no PS/2 mouse plugged in.
+ *
+ * This functions is deliberately very conservative.  Losing
+ * memory in the bottom megabyte is rarely a problem, as long
+ * as we have enough memory to install the trampoline.  Using
+ * memory that is in use by the BIOS or by some DMA device
+ * the BIOS didn't shut down *is* a big problem.
  */
+
+#define BIOS_LOWMEM_KILOBYTES   0x413
+#define LOWMEM_CAP              0x9f000U        /* Absolute maximum */
+#define INSANE_CUTOFF           0x20000U        /* Less than this = insane */
+
 void __init reserve_ebda_region(void)
 {
         unsigned int lowmem, ebda_addr;
 
-        /* To determine the position of the EBDA and the */
-        /* end of conventional memory, we need to look at */
-        /* the BIOS data area. In a paravirtual environment */
-        /* that area is absent. We'll just have to assume */
-        /* that the paravirt case can handle memory setup */
-        /* correctly, without our help. */
+        /*
+         * To determine the position of the EBDA and the
+         * end of conventional memory, we need to look at
+         * the BIOS data area. In a paravirtual environment
+         * that area is absent. We'll just have to assume
+         * that the paravirt case can handle memory setup
+         * correctly, without our help.
+         */
         if (paravirt_enabled())
                 return;
 
@@ -37,19 +48,23 @@ void __init reserve_ebda_region(void)
         /* start of EBDA area */
         ebda_addr = get_bios_ebda();
 
-        /* Fixup: bios puts an EBDA in the top 64K segment */
-        /* of conventional memory, but does not adjust lowmem. */
-        if ((lowmem - ebda_addr) <= 0x10000)
-                lowmem = ebda_addr;
+        /*
+         * Note: some old Dells seem to need 4k EBDA without
+         * reporting so, so just consider the memory above 0x9f000
+         * to be off limits (bugzilla 2990).
+         */
+
+        /* If the EBDA address is below 128K, assume it is bogus */
+        if (ebda_addr < INSANE_CUTOFF)
+                ebda_addr = LOWMEM_CAP;
 
-        /* Fixup: bios does not report an EBDA at all. */
-        /* Some old Dells seem to need 4k anyhow (bugzilla 2990) */
-        if ((ebda_addr == 0) && (lowmem >= 0x9f000))
-                lowmem = 0x9f000;
+        /* If lowmem is less than 128K, assume it is bogus */
+        if (lowmem < INSANE_CUTOFF)
+                lowmem = LOWMEM_CAP;
 
-        /* Paranoia: should never happen, but... */
-        if ((lowmem == 0) || (lowmem >= 0x100000))
-                lowmem = 0x9f000;
+        /* Use the lower of the lowmem and EBDA markers as the cutoff */
+        lowmem = min(lowmem, ebda_addr);
+        lowmem = min(lowmem, LOWMEM_CAP); /* Absolute cap */
 
         /* reserve all memory between lowmem and the 1MB mark */
         memblock_reserve(lowmem, 0x100000 - lowmem);
diff --git a/arch/x86/kernel/head32.c b/arch/x86/kernel/head32.c
index c18f59d10101..138463a24877 100644
--- a/arch/x86/kernel/head32.c
+++ b/arch/x86/kernel/head32.c
@@ -18,6 +18,7 @@
 #include <asm/io_apic.h>
 #include <asm/bios_ebda.h>
 #include <asm/tlbflush.h>
+#include <asm/bootparam_utils.h>
 
 static void __init i386_default_early_setup(void)
 {
@@ -30,19 +31,7 @@ static void __init i386_default_early_setup(void)
 
 void __init i386_start_kernel(void)
 {
-        memblock_reserve(__pa_symbol(&_text),
-                         __pa_symbol(&__bss_stop) - __pa_symbol(&_text));
-
-#ifdef CONFIG_BLK_DEV_INITRD
-        /* Reserve INITRD */
-        if (boot_params.hdr.type_of_loader && boot_params.hdr.ramdisk_image) {
-                /* Assume only end is not page aligned */
-                u64 ramdisk_image = boot_params.hdr.ramdisk_image;
-                u64 ramdisk_size  = boot_params.hdr.ramdisk_size;
-                u64 ramdisk_end   = PAGE_ALIGN(ramdisk_image + ramdisk_size);
-                memblock_reserve(ramdisk_image, ramdisk_end - ramdisk_image);
-        }
-#endif
+        sanitize_boot_params(&boot_params);
 
         /* Call the subarch specific early setup function */
         switch (boot_params.hdr.hardware_subarch) {
@@ -57,11 +46,5 @@ void __init i386_start_kernel(void)
                 break;
         }
 
-        /*
-         * At this point everything still needed from the boot loader
-         * or BIOS or kernel text should be early reserved or marked not
-         * RAM in e820. All other memory is free game.
-         */
-
         start_kernel();
 }
diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c
index 037df57a99ac..c5e403f6d869 100644
--- a/arch/x86/kernel/head64.c
+++ b/arch/x86/kernel/head64.c
@@ -25,12 +25,84 @@
 #include <asm/kdebug.h>
 #include <asm/e820.h>
 #include <asm/bios_ebda.h>
+#include <asm/bootparam_utils.h>
+#include <asm/microcode.h>
 
-static void __init zap_identity_mappings(void)
+/*
+ * Manage page tables very early on.
+ */
+extern pgd_t early_level4_pgt[PTRS_PER_PGD];
+extern pmd_t early_dynamic_pgts[EARLY_DYNAMIC_PAGE_TABLES][PTRS_PER_PMD];
+static unsigned int __initdata next_early_pgt = 2;
+
+/* Wipe all early page tables except for the kernel symbol map */
+static void __init reset_early_page_tables(void)
 {
-        pgd_t *pgd = pgd_offset_k(0UL);
-        pgd_clear(pgd);
-        __flush_tlb_all();
+        unsigned long i;
+
+        for (i = 0; i < PTRS_PER_PGD-1; i++)
+                early_level4_pgt[i].pgd = 0;
+
+        next_early_pgt = 0;
+
+        write_cr3(__pa(early_level4_pgt));
+}
+
+/* Create a new PMD entry */
+int __init early_make_pgtable(unsigned long address)
+{
+        unsigned long physaddr = address - __PAGE_OFFSET;
+        unsigned long i;
+        pgdval_t pgd, *pgd_p;
+        pudval_t pud, *pud_p;
+        pmdval_t pmd, *pmd_p;
+
+        /* Invalid address or early pgt is done ?  */
+        if (physaddr >= MAXMEM || read_cr3() != __pa(early_level4_pgt))
+                return -1;
+
+again:
+        pgd_p = &early_level4_pgt[pgd_index(address)].pgd;
+        pgd = *pgd_p;
+
+        /*
+         * The use of __START_KERNEL_map rather than __PAGE_OFFSET here is
+         * critical -- __PAGE_OFFSET would point us back into the dynamic
+         * range and we might end up looping forever...
+         */
+        if (pgd)
+                pud_p = (pudval_t *)((pgd & PTE_PFN_MASK) + __START_KERNEL_map - phys_base);
+        else {
+                if (next_early_pgt >= EARLY_DYNAMIC_PAGE_TABLES) {
+                        reset_early_page_tables();
+                        goto again;
+                }
+
+                pud_p = (pudval_t *)early_dynamic_pgts[next_early_pgt++];
+                for (i = 0; i < PTRS_PER_PUD; i++)
+                        pud_p[i] = 0;
+                *pgd_p = (pgdval_t)pud_p - __START_KERNEL_map + phys_base + _KERNPG_TABLE;
+        }
+        pud_p += pud_index(address);
+        pud = *pud_p;
+
+        if (pud)
+                pmd_p = (pmdval_t *)((pud & PTE_PFN_MASK) + __START_KERNEL_map - phys_base);
+        else {
+                if (next_early_pgt >= EARLY_DYNAMIC_PAGE_TABLES) {
+                        reset_early_page_tables();
+                        goto again;
+                }
+
+                pmd_p = (pmdval_t *)early_dynamic_pgts[next_early_pgt++];
+                for (i = 0; i < PTRS_PER_PMD; i++)
+                        pmd_p[i] = 0;
+                *pud_p = (pudval_t)pmd_p - __START_KERNEL_map + phys_base + _KERNPG_TABLE;
+        }
+        pmd = (physaddr & PMD_MASK) + (__PAGE_KERNEL_LARGE & ~_PAGE_GLOBAL);
+        pmd_p[pmd_index(address)] = pmd;
+
+        return 0;
 }
 
 /* Don't add a printk in there. printk relies on the PDA which is not initialized 
@@ -41,13 +113,25 @@ static void __init clear_bss(void)
                (unsigned long) __bss_stop - (unsigned long) __bss_start);
 }
 
+static unsigned long get_cmd_line_ptr(void)
+{
+        unsigned long cmd_line_ptr = boot_params.hdr.cmd_line_ptr;
+
+        cmd_line_ptr |= (u64)boot_params.ext_cmd_line_ptr << 32;
+
+        return cmd_line_ptr;
+}
+
 static void __init copy_bootdata(char *real_mode_data)
 {
         char * command_line;
+        unsigned long cmd_line_ptr;
 
         memcpy(&boot_params, real_mode_data, sizeof boot_params);
-        if (boot_params.hdr.cmd_line_ptr) {
-                command_line = __va(boot_params.hdr.cmd_line_ptr);
+        sanitize_boot_params(&boot_params);
+        cmd_line_ptr = get_cmd_line_ptr();
+        if (cmd_line_ptr) {
+                command_line = __va(cmd_line_ptr);
                 memcpy(boot_command_line, command_line, COMMAND_LINE_SIZE);
         }
 }
@@ -70,54 +154,40 @@ void __init x86_64_start_kernel(char * real_mode_data)
                                 (__START_KERNEL & PGDIR_MASK)));
         BUILD_BUG_ON(__fix_to_virt(__end_of_fixed_addresses) <= MODULES_END);
 
+        /* Kill off the identity-map trampoline */
+        reset_early_page_tables();
+
         /* clear bss before set_intr_gate with early_idt_handler */
         clear_bss();
 
-        /* Make NULL pointers segfault */
-        zap_identity_mappings();
-
-        max_pfn_mapped = KERNEL_IMAGE_SIZE >> PAGE_SHIFT;
-
-        for (i = 0; i < NUM_EXCEPTION_VECTORS; i++) {
-#ifdef CONFIG_EARLY_PRINTK
+        for (i = 0; i < NUM_EXCEPTION_VECTORS; i++)
                 set_intr_gate(i, &early_idt_handlers[i]);
-#else
-                set_intr_gate(i, early_idt_handler);
-#endif
-        }
         load_idt((const struct desc_ptr *)&idt_descr);
 
+        copy_bootdata(__va(real_mode_data));
+
+        /*
+         * Load microcode early on BSP.
+         */
+        load_ucode_bsp();
+
         if (console_loglevel == 10)
                 early_printk("Kernel alive\n");
 
+        clear_page(init_level4_pgt);
+        /* set init_level4_pgt kernel high mapping*/
+        init_level4_pgt[511] = early_level4_pgt[511];
+
         x86_64_start_reservations(real_mode_data);
 }
 
 void __init x86_64_start_reservations(char *real_mode_data)
 {
-        copy_bootdata(__va(real_mode_data));
-
-        memblock_reserve(__pa_symbol(&_text),
-                         __pa_symbol(&__bss_stop) - __pa_symbol(&_text));
-
-#ifdef CONFIG_BLK_DEV_INITRD
-        /* Reserve INITRD */
-        if (boot_params.hdr.type_of_loader && boot_params.hdr.ramdisk_image) {
-                /* Assume only end is not page aligned */
-                unsigned long ramdisk_image = boot_params.hdr.ramdisk_image;
-                unsigned long ramdisk_size  = boot_params.hdr.ramdisk_size;
-                unsigned long ramdisk_end   = PAGE_ALIGN(ramdisk_image + ramdisk_size);
-                memblock_reserve(ramdisk_image, ramdisk_end - ramdisk_image);
-        }
-#endif
+        /* version is always not zero if it is copied */
+        if (!boot_params.hdr.version)
+                copy_bootdata(__va(real_mode_data));
 
         reserve_ebda_region();
 
-        /*
-         * At this point everything still needed from the boot loader
-         * or BIOS or kernel text should be early reserved or marked not
-         * RAM in e820. All other memory is free game.
-         */
-
         start_kernel();
 }
diff --git a/arch/x86/kernel/head_32.S b/arch/x86/kernel/head_32.S
index c8932c79e78b..73afd11799ca 100644
--- a/arch/x86/kernel/head_32.S
+++ b/arch/x86/kernel/head_32.S
@@ -144,6 +144,11 @@ ENTRY(startup_32)
         movl %eax, pa(olpc_ofw_pgd)
 #endif
 
+#ifdef CONFIG_MICROCODE_EARLY
+        /* Early load ucode on BSP. */
+        call load_ucode_bsp
+#endif
+
 /*
  * Initialize page tables.  This creates a PDE and a set of page
  * tables, which are located immediately beyond __brk_base.  The variable
@@ -299,6 +304,12 @@ ENTRY(startup_32_smp)
         movl %eax,%ss
         leal -__PAGE_OFFSET(%ecx),%esp
 
+#ifdef CONFIG_MICROCODE_EARLY
+        /* Early load ucode on AP. */
+        call load_ucode_ap
+#endif
+
+
 default_entry:
 #define CR0_STATE       (X86_CR0_PE | X86_CR0_MP | X86_CR0_ET | \
                          X86_CR0_NE | X86_CR0_WP | X86_CR0_AM | \
@@ -307,36 +318,45 @@ default_entry:
         movl %eax,%cr0
 
 /*
- *      New page tables may be in 4Mbyte page mode and may
- *      be using the global pages. 
+ * We want to start out with EFLAGS unambiguously cleared. Some BIOSes leave
+ * bits like NT set. This would confuse the debugger if this code is traced. So
+ * initialize them properly now before switching to protected mode. That means
+ * DF in particular (even though we have cleared it earlier after copying the
+ * command line) because GCC expects it.
+ */
+        pushl $0
+        popfl
+
+/*
+ * New page tables may be in 4Mbyte page mode and may be using the global pages.
  *
- *      NOTE! If we are on a 486 we may have no cr4 at all!
- *      Specifically, cr4 exists if and only if CPUID exists
- *      and has flags other than the FPU flag set.
+ * NOTE! If we are on a 486 we may have no cr4 at all! Specifically, cr4 exists
+ * if and only if CPUID exists and has flags other than the FPU flag set.
  */
+        movl $-1,pa(X86_CPUID)          # preset CPUID level
         movl $X86_EFLAGS_ID,%ecx
         pushl %ecx
-        popfl
-        pushfl
-        popl %eax
-        pushl $0
-        popfl
+        popfl                           # set EFLAGS=ID
         pushfl
-        popl %edx
-        xorl %edx,%eax
-        testl %ecx,%eax
-        jz 6f                   # No ID flag = no CPUID = no CR4
+        popl %eax                       # get EFLAGS
+        testl $X86_EFLAGS_ID,%eax       # did EFLAGS.ID remained set?
+        jz enable_paging                # hw disallowed setting of ID bit
+                                        # which means no CPUID and no CR4
+
+        xorl %eax,%eax
+        cpuid
+        movl %eax,pa(X86_CPUID)         # save largest std CPUID function
 
         movl $1,%eax
         cpuid
-        andl $~1,%edx           # Ignore CPUID.FPU
-        jz 6f                   # No flags or only CPUID.FPU = no CR4
+        andl $~1,%edx                   # Ignore CPUID.FPU
+        jz enable_paging                # No flags or only CPUID.FPU = no CR4
 
         movl pa(mmu_cr4_features),%eax
         movl %eax,%cr4
 
         testb $X86_CR4_PAE, %al         # check if PAE is enabled
-        jz 6f
+        jz enable_paging
 
         /* Check if extended functions are implemented */
         movl $0x80000000, %eax
@@ -344,7 +364,7 @@ default_entry:
         /* Value must be in the range 0x80000001 to 0x8000ffff */
         subl $0x80000001, %eax
         cmpl $(0x8000ffff-0x80000001), %eax
-        ja 6f
+        ja enable_paging
 
         /* Clear bogus XD_DISABLE bits */
         call verify_cpu
@@ -353,7 +373,7 @@ default_entry:
         cpuid
         /* Execute Disable bit supported? */
         btl $(X86_FEATURE_NX & 31), %edx
-        jnc 6f
+        jnc enable_paging
 
         /* Setup EFER (Extended Feature Enable Register) */
         movl $MSR_EFER, %ecx
@@ -363,7 +383,7 @@ default_entry:
         /* Make changes effective */
         wrmsr
 
-6:
+enable_paging:
 
 /*
  * Enable paging
@@ -378,14 +398,6 @@ default_entry:
         addl $__PAGE_OFFSET, %esp
 
 /*
- * Initialize eflags.  Some BIOS's leave bits like NT set.  This would
- * confuse the debugger if this code is traced.
- * XXX - best to initialize before switching to protected mode.
- */
-        pushl $0
-        popfl
-
-/*
  * start system 32-bit setup. We need to re-do some of the things done
  * in 16-bit mode for the "real" operations.
  */
@@ -394,31 +406,11 @@ default_entry:
         jz 1f                           # Did we do this already?
         call *%eax
 1:
-        
-/* check if it is 486 or 386. */
+
 /*
- * XXX - this does a lot of unnecessary setup.  Alignment checks don't
- * apply at our cpl of 0 and the stack ought to be aligned already, and
- * we don't need to preserve eflags.
+ * Check if it is 486
  */
-        movl $-1,X86_CPUID      # -1 for no CPUID initially
-        movb $3,X86             # at least 386
-        pushfl                  # push EFLAGS
-        popl %eax               # get EFLAGS
-        movl %eax,%ecx          # save original EFLAGS
-        xorl $0x240000,%eax     # flip AC and ID bits in EFLAGS
-        pushl %eax              # copy to EFLAGS
-        popfl                   # set EFLAGS
-        pushfl                  # get new EFLAGS
-        popl %eax               # put it in eax
-        xorl %ecx,%eax          # change in flags
-        pushl %ecx              # restore original EFLAGS
-        popfl
-        testl $0x40000,%eax     # check if AC bit changed
-        je is386
-
-        movb $4,X86             # at least 486
-        testl $0x200000,%eax    # check if ID bit changed
+        cmpl $-1,X86_CPUID
         je is486
 
         /* get vendor info */
@@ -444,11 +436,10 @@ default_entry:
         movb %cl,X86_MASK
         movl %edx,X86_CAPABILITY
 
-is486:  movl $0x50022,%ecx      # set AM, WP, NE and MP
-        jmp 2f
-
-is386:  movl $2,%ecx            # set MP
-2:      movl %cr0,%eax
+is486:
+        movb $4,X86
+        movl $0x50022,%ecx      # set AM, WP, NE and MP
+        movl %cr0,%eax
         andl $0x80000011,%eax   # Save PG,PE,ET
         orl %ecx,%eax
         movl %eax,%cr0
@@ -473,7 +464,6 @@ is386:	movl $2,%ecx		# set MP
         xorl %eax,%eax                  # Clear LDT
         lldt %ax
 
-        cld                     # gcc2 wants the direction flag cleared at all times
         pushl $0                # fake return address for unwinder
         jmp *(initial_code)
 
diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S
index 980053c4b9cc..6859e9626442 100644
--- a/arch/x86/kernel/head_64.S
+++ b/arch/x86/kernel/head_64.S
@@ -47,14 +47,13 @@ L3_START_KERNEL = pud_index(__START_KERNEL_map)
         .code64
         .globl startup_64
 startup_64:
-
         /*
-         * At this point the CPU runs in 64bit mode CS.L = 1 CS.D = 1,
+         * At this point the CPU runs in 64bit mode CS.L = 1 CS.D = 0,
          * and someone has loaded an identity mapped page table
          * for us.  These identity mapped page tables map all of the
          * kernel pages and possibly all of memory.
          *
-         * %esi holds a physical pointer to real_mode_data.
+         * %rsi holds a physical pointer to real_mode_data.
          *
          * We come here either directly from a 64bit bootloader, or from
          * arch/x86_64/boot/compressed/head.S.
@@ -66,7 +65,8 @@ startup_64:
          * tables and then reload them.
          */
 
-        /* Compute the delta between the address I am compiled to run at and the
+        /*
+         * Compute the delta between the address I am compiled to run at and the
          * address I am actually running at.
          */
         leaq    _text(%rip), %rbp
@@ -78,45 +78,62 @@ startup_64:
         testl   %eax, %eax
         jnz     bad_address
 
-        /* Is the address too large? */
-        leaq    _text(%rip), %rdx
-        movq    $PGDIR_SIZE, %rax
-        cmpq    %rax, %rdx
-        jae     bad_address
-
-        /* Fixup the physical addresses in the page table
+        /*
+         * Is the address too large?
          */
-        addq    %rbp, init_level4_pgt + 0(%rip)
-        addq    %rbp, init_level4_pgt + (L4_PAGE_OFFSET*8)(%rip)
-        addq    %rbp, init_level4_pgt + (L4_START_KERNEL*8)(%rip)
+        leaq    _text(%rip), %rax
+        shrq    $MAX_PHYSMEM_BITS, %rax
+        jnz     bad_address
 
-        addq    %rbp, level3_ident_pgt + 0(%rip)
+        /*
+         * Fixup the physical addresses in the page table
+         */
+        addq    %rbp, early_level4_pgt + (L4_START_KERNEL*8)(%rip)
 
         addq    %rbp, level3_kernel_pgt + (510*8)(%rip)
         addq    %rbp, level3_kernel_pgt + (511*8)(%rip)
 
         addq    %rbp, level2_fixmap_pgt + (506*8)(%rip)
 
-        /* Add an Identity mapping if I am above 1G */
+        /*
+         * Set up the identity mapping for the switchover.  These
+         * entries should *NOT* have the global bit set!  This also
+         * creates a bunch of nonsense entries but that is fine --
+         * it avoids problems around wraparound.
+         */
         leaq    _text(%rip), %rdi
-        andq    $PMD_PAGE_MASK, %rdi
+        leaq    early_level4_pgt(%rip), %rbx
 
         movq    %rdi, %rax
-        shrq    $PUD_SHIFT, %rax
-        andq    $(PTRS_PER_PUD - 1), %rax
-        jz      ident_complete
+        shrq    $PGDIR_SHIFT, %rax
 
-        leaq    (level2_spare_pgt - __START_KERNEL_map + _KERNPG_TABLE)(%rbp), %rdx
-        leaq    level3_ident_pgt(%rip), %rbx
-        movq    %rdx, 0(%rbx, %rax, 8)
+        leaq    (4096 + _KERNPG_TABLE)(%rbx), %rdx
+        movq    %rdx, 0(%rbx,%rax,8)
+        movq    %rdx, 8(%rbx,%rax,8)
 
+        addq    $4096, %rdx
         movq    %rdi, %rax
-        shrq    $PMD_SHIFT, %rax
-        andq    $(PTRS_PER_PMD - 1), %rax
-        leaq    __PAGE_KERNEL_IDENT_LARGE_EXEC(%rdi), %rdx
-        leaq    level2_spare_pgt(%rip), %rbx
-        movq    %rdx, 0(%rbx, %rax, 8)
-ident_complete:
+        shrq    $PUD_SHIFT, %rax
+        andl    $(PTRS_PER_PUD-1), %eax
+        movq    %rdx, (4096+0)(%rbx,%rax,8)
+        movq    %rdx, (4096+8)(%rbx,%rax,8)
+
+        addq    $8192, %rbx
+        movq    %rdi, %rax
+        shrq    $PMD_SHIFT, %rdi
+        addq    $(__PAGE_KERNEL_LARGE_EXEC & ~_PAGE_GLOBAL), %rax
+        leaq    (_end - 1)(%rip), %rcx
+        shrq    $PMD_SHIFT, %rcx
+        subq    %rdi, %rcx
+        incl    %ecx
+
+1:
+        andq    $(PTRS_PER_PMD - 1), %rdi
+        movq    %rax, (%rbx,%rdi,8)
+        incq    %rdi
+        addq    $PMD_SIZE, %rax
+        decl    %ecx
+        jnz     1b
 
         /*
          * Fixup the kernel text+data virtual addresses. Note that
@@ -124,7 +141,6 @@ ident_complete:
          * cleanup_highmap() fixes this up along with the mappings
          * beyond _end.
          */
-
         leaq    level2_kernel_pgt(%rip), %rdi
         leaq    4096(%rdi), %r8
         /* See if it is a valid page table entry */
@@ -139,17 +155,14 @@ ident_complete:
         /* Fixup phys_base */
         addq    %rbp, phys_base(%rip)
 
-        /* Due to ENTRY(), sometimes the empty space gets filled with
-         * zeros. Better take a jmp than relying on empty space being
-         * filled with 0x90 (nop)
-         */
-        jmp secondary_startup_64
+        movq    $(early_level4_pgt - __START_KERNEL_map), %rax
+        jmp 1f
 ENTRY(secondary_startup_64)
         /*
-         * At this point the CPU runs in 64bit mode CS.L = 1 CS.D = 1,
+         * At this point the CPU runs in 64bit mode CS.L = 1 CS.D = 0,
          * and someone has loaded a mapped page table.
          *
-         * %esi holds a physical pointer to real_mode_data.
+         * %rsi holds a physical pointer to real_mode_data.
          *
          * We come here either from startup_64 (using physical addresses)
          * or from trampoline.S (using virtual addresses).
@@ -159,12 +172,14 @@ ENTRY(secondary_startup_64)
          * after the boot processor executes this code.
          */
 
+        movq    $(init_level4_pgt - __START_KERNEL_map), %rax
+1:
+
         /* Enable PAE mode and PGE */
-        movl    $(X86_CR4_PAE | X86_CR4_PGE), %eax
-        movq    %rax, %cr4
+        movl    $(X86_CR4_PAE | X86_CR4_PGE), %ecx
+        movq    %rcx, %cr4
 
         /* Setup early boot stage 4 level pagetables. */
-        movq    $(init_level4_pgt - __START_KERNEL_map), %rax
         addq    phys_base(%rip), %rax
         movq    %rax, %cr3
 
@@ -196,7 +211,7 @@ ENTRY(secondary_startup_64)
         movq    %rax, %cr0
 
         /* Setup a boot time stack */
-        movq stack_start(%rip),%rsp
+        movq stack_start(%rip), %rsp
 
         /* zero EFLAGS after setting rsp */
         pushq $0
@@ -236,15 +251,33 @@ ENTRY(secondary_startup_64)
         movl    initial_gs+4(%rip),%edx
         wrmsr   
 
-        /* esi is pointer to real mode structure with interesting info.
+        /* rsi is pointer to real mode structure with interesting info.
            pass it to C */
-        movl    %esi, %edi
+        movq    %rsi, %rdi
         
         /* Finally jump to run C code and to be on real kernel address
          * Since we are running on identity-mapped space we have to jump
          * to the full 64bit address, this is only possible as indirect
          * jump.  In addition we need to ensure %cs is set so we make this
          * a far return.
+         *
+         * Note: do not change to far jump indirect with 64bit offset.
+         *
+         * AMD does not support far jump indirect with 64bit offset.
+         * AMD64 Architecture Programmer's Manual, Volume 3: states only
+         *      JMP FAR mem16:16 FF /5 Far jump indirect,
+         *              with the target specified by a far pointer in memory.
+         *      JMP FAR mem16:32 FF /5 Far jump indirect,
+         *              with the target specified by a far pointer in memory.
+         *
+         * Intel64 does support 64bit offset.
+         * Software Developer Manual Vol 2: states:
+         *      FF /5 JMP m16:16 Jump far, absolute indirect,
+         *              address given in m16:16
+         *      FF /5 JMP m16:32 Jump far, absolute indirect,
+         *              address given in m16:32.
+         *      REX.W + FF /5 JMP m16:64 Jump far, absolute indirect,
+         *              address given in m16:64.
          */
         movq    initial_code(%rip),%rax
         pushq   $0              # fake return address to stop unwinder
@@ -270,13 +303,13 @@ ENDPROC(start_cpu0)
 
         /* SMP bootup changes these two */
         __REFDATA
-        .align  8
-        ENTRY(initial_code)
+        .balign 8
+        GLOBAL(initial_code)
         .quad   x86_64_start_kernel
-        ENTRY(initial_gs)
+        GLOBAL(initial_gs)
         .quad   INIT_PER_CPU_VAR(irq_stack_union)
 
-        ENTRY(stack_start)
+        GLOBAL(stack_start)
         .quad  init_thread_union+THREAD_SIZE-8
         .word  0
         __FINITDATA
@@ -284,7 +317,7 @@ ENDPROC(start_cpu0)
 bad_address:
         jmp bad_address
 
-        .section ".init.text","ax"
+        __INIT
         .globl early_idt_handlers
 early_idt_handlers:
         # 104(%rsp) %rflags
@@ -303,6 +336,7 @@ early_idt_handlers:
         i = i + 1
         .endr
 
+/* This is global to keep gas from relaxing the jumps */
 ENTRY(early_idt_handler)
         cld
 
@@ -321,14 +355,22 @@ ENTRY(early_idt_handler)
         pushq %r11              #  0(%rsp)
 
         cmpl $__KERNEL_CS,96(%rsp)
-        jne 10f
+        jne 11f
+
+        cmpl $14,72(%rsp)       # Page fault?
+        jnz 10f
+        GET_CR2_INTO(%rdi)      # can clobber any volatile register if pv
+        call early_make_pgtable
+        andl %eax,%eax
+        jz 20f                  # All good
 
+10:
         leaq 88(%rsp),%rdi      # Pointer to %rip
         call early_fixup_exception
         andl %eax,%eax
         jnz 20f                 # Found an exception entry
 
-10:
+11:
 #ifdef CONFIG_EARLY_PRINTK
         GET_CR2_INTO(%r9)       # can clobber any volatile register if pv
         movl 80(%rsp),%r8d      # error code
@@ -350,7 +392,7 @@ ENTRY(early_idt_handler)
 1:      hlt
         jmp 1b
 
-20:     # Exception table entry found
+20:     # Exception table entry found or page table generated
         popq %r11
         popq %r10
         popq %r9
@@ -363,6 +405,9 @@ ENTRY(early_idt_handler)
         addq $16,%rsp           # drop vector number and error code
         decl early_recursion_flag(%rip)
         INTERRUPT_RETURN
+ENDPROC(early_idt_handler)
+
+        __INITDATA
 
         .balign 4
 early_recursion_flag:
@@ -374,11 +419,10 @@ early_idt_msg:
 early_idt_ripmsg:
         .asciz "RIP %s\n"
 #endif /* CONFIG_EARLY_PRINTK */
-        .previous
 
 #define NEXT_PAGE(name) \
         .balign PAGE_SIZE; \
-ENTRY(name)
+GLOBAL(name)
 
 /* Automate the creation of 1 to 1 mapping pmd entries */
 #define PMDS(START, PERM, COUNT)                        \
@@ -388,24 +432,37 @@ ENTRY(name)
         i = i + 1 ;                                     \
         .endr
 
+        __INITDATA
+NEXT_PAGE(early_level4_pgt)
+        .fill   511,8,0
+        .quad   level3_kernel_pgt - __START_KERNEL_map + _PAGE_TABLE
+
+NEXT_PAGE(early_dynamic_pgts)
+        .fill   512*EARLY_DYNAMIC_PAGE_TABLES,8,0
+
         .data
-        /*
-         * This default setting generates an ident mapping at address 0x100000
-         * and a mapping for the kernel that precisely maps virtual address
-         * 0xffffffff80000000 to physical address 0x000000. (always using
-         * 2Mbyte large pages provided by PAE mode)
-         */
+
+#ifndef CONFIG_XEN
 NEXT_PAGE(init_level4_pgt)
-        .quad   level3_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE
-        .org    init_level4_pgt + L4_PAGE_OFFSET*8, 0
-        .quad   level3_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE
-        .org    init_level4_pgt + L4_START_KERNEL*8, 0
+        .fill   512,8,0
+#else
+NEXT_PAGE(init_level4_pgt)
+        .quad   level3_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE
+        .org    init_level4_pgt + L4_PAGE_OFFSET*8, 0
+        .quad   level3_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE
+        .org    init_level4_pgt + L4_START_KERNEL*8, 0
         /* (2^48-(2*1024*1024*1024))/(2^39) = 511 */
-        .quad   level3_kernel_pgt - __START_KERNEL_map + _PAGE_TABLE
+        .quad   level3_kernel_pgt - __START_KERNEL_map + _PAGE_TABLE
 
 NEXT_PAGE(level3_ident_pgt)
         .quad   level2_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE
-        .fill   511,8,0
+        .fill   511, 8, 0
+NEXT_PAGE(level2_ident_pgt)
+        /* Since I easily can, map the first 1G.
+         * Don't set NX because code runs from these pages.
+         */
+        PMDS(0, __PAGE_KERNEL_IDENT_LARGE_EXEC, PTRS_PER_PMD)
+#endif
 
 NEXT_PAGE(level3_kernel_pgt)
         .fill   L3_START_KERNEL,8,0
@@ -413,21 +470,6 @@ NEXT_PAGE(level3_kernel_pgt)
         .quad   level2_kernel_pgt - __START_KERNEL_map + _KERNPG_TABLE
         .quad   level2_fixmap_pgt - __START_KERNEL_map + _PAGE_TABLE
 
-NEXT_PAGE(level2_fixmap_pgt)
-        .fill   506,8,0
-        .quad   level1_fixmap_pgt - __START_KERNEL_map + _PAGE_TABLE
-        /* 8MB reserved for vsyscalls + a 2MB hole = 4 + 1 entries */
-        .fill   5,8,0
-
-NEXT_PAGE(level1_fixmap_pgt)
-        .fill   512,8,0
-
-NEXT_PAGE(level2_ident_pgt)
-        /* Since I easily can, map the first 1G.
-         * Don't set NX because code runs from these pages.
-         */
-        PMDS(0, __PAGE_KERNEL_IDENT_LARGE_EXEC, PTRS_PER_PMD)
-
 NEXT_PAGE(level2_kernel_pgt)
         /*
          * 512 MB kernel mapping. We spend a full page on this pagetable
@@ -442,11 +484,16 @@ NEXT_PAGE(level2_kernel_pgt)
         PMDS(0, __PAGE_KERNEL_LARGE_EXEC,
                 KERNEL_IMAGE_SIZE/PMD_SIZE)
 
-NEXT_PAGE(level2_spare_pgt)
-        .fill   512, 8, 0
+NEXT_PAGE(level2_fixmap_pgt)
+        .fill   506,8,0
+        .quad   level1_fixmap_pgt - __START_KERNEL_map + _PAGE_TABLE
+        /* 8MB reserved for vsyscalls + a 2MB hole = 4 + 1 entries */
+        .fill   5,8,0
+
+NEXT_PAGE(level1_fixmap_pgt)
+        .fill   512,8,0
 
 #undef PMDS
-#undef NEXT_PAGE
 
         .data
         .align 16
@@ -472,6 +519,5 @@ ENTRY(nmi_idt_table)
         .skip IDT_ENTRIES * 16
 
         __PAGE_ALIGNED_BSS
-        .align PAGE_SIZE
-ENTRY(empty_zero_page)
+NEXT_PAGE(empty_zero_page)
         .skip PAGE_SIZE
diff --git a/arch/x86/kernel/hpet.c b/arch/x86/kernel/hpet.c
index e28670f9a589..da85a8e830a1 100644
--- a/arch/x86/kernel/hpet.c
+++ b/arch/x86/kernel/hpet.c
@@ -478,7 +478,7 @@ static int hpet_msi_next_event(unsigned long delta,
 
 static int hpet_setup_msi_irq(unsigned int irq)
 {
-        if (arch_setup_hpet_msi(irq, hpet_blockid)) {
+        if (x86_msi.setup_hpet_msi(irq, hpet_blockid)) {
                 destroy_irq(irq);
                 return -EINVAL;
         }
diff --git a/arch/x86/kernel/i386_ksyms_32.c b/arch/x86/kernel/i386_ksyms_32.c
index 9c3bd4a2050e..0fa69127209a 100644
--- a/arch/x86/kernel/i386_ksyms_32.c
+++ b/arch/x86/kernel/i386_ksyms_32.c
@@ -26,6 +26,7 @@ EXPORT_SYMBOL(csum_partial_copy_generic);
 EXPORT_SYMBOL(__get_user_1);
 EXPORT_SYMBOL(__get_user_2);
 EXPORT_SYMBOL(__get_user_4);
+EXPORT_SYMBOL(__get_user_8);
 
 EXPORT_SYMBOL(__put_user_1);
 EXPORT_SYMBOL(__put_user_2);
diff --git a/arch/x86/kernel/ioport.c b/arch/x86/kernel/ioport.c
index 8c968974253d..4ddaf66ea35f 100644
--- a/arch/x86/kernel/ioport.c
+++ b/arch/x86/kernel/ioport.c
@@ -93,8 +93,9 @@ asmlinkage long sys_ioperm(unsigned long from, unsigned long num, int turn_on)
  * on system-call entry - see also fork() and the signal handling
  * code.
  */
-long sys_iopl(unsigned int level, struct pt_regs *regs)
+SYSCALL_DEFINE1(iopl, unsigned int, level)
 {
+        struct pt_regs *regs = current_pt_regs();
         unsigned int old = (regs->flags >> 12) & 3;
         struct thread_struct *t = &current->thread;
 
diff --git a/arch/x86/kernel/kprobes/Makefile b/arch/x86/kernel/kprobes/Makefile
new file mode 100644
index 000000000000..0d33169cc1a2
--- /dev/null
+++ b/arch/x86/kernel/kprobes/Makefile
@@ -0,0 +1,7 @@
+#
+# Makefile for kernel probes
+#
+
+obj-$(CONFIG_KPROBES)           += core.o
+obj-$(CONFIG_OPTPROBES)         += opt.o
+obj-$(CONFIG_KPROBES_ON_FTRACE) += ftrace.o
diff --git a/arch/x86/kernel/kprobes-common.h b/arch/x86/kernel/kprobes/common.h
index 3230b68ef29a..2e9d4b5af036 100644
--- a/arch/x86/kernel/kprobes-common.h
+++ b/arch/x86/kernel/kprobes/common.h
@@ -99,4 +99,15 @@ static inline unsigned long __recover_optprobed_insn(kprobe_opcode_t *buf, unsig
         return addr;
 }
 #endif
+
+#ifdef CONFIG_KPROBES_ON_FTRACE
+extern int skip_singlestep(struct kprobe *p, struct pt_regs *regs,
+                           struct kprobe_ctlblk *kcb);
+#else
+static inline int skip_singlestep(struct kprobe *p, struct pt_regs *regs,
+                                  struct kprobe_ctlblk *kcb)
+{
+        return 0;
+}
+#endif
 #endif
diff --git a/arch/x86/kernel/kprobes.c b/arch/x86/kernel/kprobes/core.c
index 57916c0d3cf6..7bfe318d3d8a 100644
--- a/arch/x86/kernel/kprobes.c
+++ b/arch/x86/kernel/kprobes/core.c
@@ -58,7 +58,7 @@
 #include <asm/insn.h>
 #include <asm/debugreg.h>
 
-#include "kprobes-common.h"
+#include "common.h"
 
 void jprobe_return_end(void);
 
@@ -78,7 +78,7 @@ DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
          * Groups, and some special opcodes can not boost.
          * This is non-const and volatile to keep gcc from statically
          * optimizing it out, as variable_test_bit makes gcc think only
-         * *(unsigned long*) is used. 
+         * *(unsigned long*) is used.
          */
 static volatile u32 twobyte_is_boostable[256 / 32] = {
         /*      0  1  2  3  4  5  6  7  8  9  a  b  c  d  e  f          */
@@ -117,7 +117,7 @@ static void __kprobes __synthesize_relative_insn(void *from, void *to, u8 op)
         struct __arch_relative_insn {
                 u8 op;
                 s32 raddr;
-        } __attribute__((packed)) *insn;
+        } __packed *insn;
 
         insn = (struct __arch_relative_insn *)from;
         insn->raddr = (s32)((long)(to) - ((long)(from) + 5));
@@ -375,6 +375,9 @@ static void __kprobes arch_copy_kprobe(struct kprobe *p)
         else
                 p->ainsn.boostable = -1;
 
+        /* Check whether the instruction modifies Interrupt Flag or not */
+        p->ainsn.if_modifier = is_IF_modifier(p->ainsn.insn);
+
         /* Also, displacement change doesn't affect the first byte */
         p->opcode = p->ainsn.insn[0];
 }
@@ -434,7 +437,7 @@ static void __kprobes set_current_kprobe(struct kprobe *p, struct pt_regs *regs,
         __this_cpu_write(current_kprobe, p);
         kcb->kprobe_saved_flags = kcb->kprobe_old_flags
                 = (regs->flags & (X86_EFLAGS_TF | X86_EFLAGS_IF));
-        if (is_IF_modifier(p->ainsn.insn))
+        if (p->ainsn.if_modifier)
                 kcb->kprobe_saved_flags &= ~X86_EFLAGS_IF;
 }
 
@@ -541,23 +544,6 @@ reenter_kprobe(struct kprobe *p, struct pt_regs *regs, struct kprobe_ctlblk *kcb
         return 1;
 }
 
-#ifdef KPROBES_CAN_USE_FTRACE
-static void __kprobes skip_singlestep(struct kprobe *p, struct pt_regs *regs,
-                                      struct kprobe_ctlblk *kcb)
-{
-        /*
-         * Emulate singlestep (and also recover regs->ip)
-         * as if there is a 5byte nop
-         */
-        regs->ip = (unsigned long)p->addr + MCOUNT_INSN_SIZE;
-        if (unlikely(p->post_handler)) {
-                kcb->kprobe_status = KPROBE_HIT_SSDONE;
-                p->post_handler(p, regs, 0);
-        }
-        __this_cpu_write(current_kprobe, NULL);
-}
-#endif
-
 /*
  * Interrupts are disabled on entry as trap3 is an interrupt gate and they
  * remain disabled throughout this function.
@@ -616,13 +602,8 @@ static int __kprobes kprobe_handler(struct pt_regs *regs)
         } else if (kprobe_running()) {
                 p = __this_cpu_read(current_kprobe);
                 if (p->break_handler && p->break_handler(p, regs)) {
-#ifdef KPROBES_CAN_USE_FTRACE
-                        if (kprobe_ftrace(p)) {
-                                skip_singlestep(p, regs, kcb);
-                                return 1;
-                        }
-#endif
-                        setup_singlestep(p, regs, kcb, 0);
+                        if (!skip_singlestep(p, regs, kcb))
+                                setup_singlestep(p, regs, kcb, 0);
                         return 1;
                 }
         } /* else: not a kprobe fault; let the kernel handle it */
@@ -674,7 +655,7 @@ static __used __kprobes void *trampoline_handler(struct pt_regs *regs)
 {
         struct kretprobe_instance *ri = NULL;
         struct hlist_head *head, empty_rp;
-        struct hlist_node *node, *tmp;
+        struct hlist_node *tmp;
         unsigned long flags, orig_ret_address = 0;
         unsigned long trampoline_address = (unsigned long)&kretprobe_trampoline;
         kprobe_opcode_t *correct_ret_addr = NULL;
@@ -704,7 +685,7 @@ static __used __kprobes void *trampoline_handler(struct pt_regs *regs)
          *       will be the real return address, and all the rest will
          *       point to kretprobe_trampoline.
          */
-        hlist_for_each_entry_safe(ri, node, tmp, head, hlist) {
+        hlist_for_each_entry_safe(ri, tmp, head, hlist) {
                 if (ri->task != current)
                         /* another task is sharing our hash bucket */
                         continue;
@@ -723,7 +704,7 @@ static __used __kprobes void *trampoline_handler(struct pt_regs *regs)
         kretprobe_assert(ri, orig_ret_address, trampoline_address);
 
         correct_ret_addr = ri->ret_addr;
-        hlist_for_each_entry_safe(ri, node, tmp, head, hlist) {
+        hlist_for_each_entry_safe(ri, tmp, head, hlist) {
                 if (ri->task != current)
                         /* another task is sharing our hash bucket */
                         continue;
@@ -750,7 +731,7 @@ static __used __kprobes void *trampoline_handler(struct pt_regs *regs)
 
         kretprobe_hash_unlock(current, &flags);
 
-        hlist_for_each_entry_safe(ri, node, tmp, &empty_rp, hlist) {
+        hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) {
                 hlist_del(&ri->hlist);
                 kfree(ri);
         }
@@ -1075,50 +1056,6 @@ int __kprobes longjmp_break_handler(struct kprobe *p, struct pt_regs *regs)
         return 0;
 }
 
-#ifdef KPROBES_CAN_USE_FTRACE
-/* Ftrace callback handler for kprobes */
-void __kprobes kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,
-                                     struct ftrace_ops *ops, struct pt_regs *regs)
-{
-        struct kprobe *p;
-        struct kprobe_ctlblk *kcb;
-        unsigned long flags;
-
-        /* Disable irq for emulating a breakpoint and avoiding preempt */
-        local_irq_save(flags);
-
-        p = get_kprobe((kprobe_opcode_t *)ip);
-        if (unlikely(!p) || kprobe_disabled(p))
-                goto end;
-
-        kcb = get_kprobe_ctlblk();
-        if (kprobe_running()) {
-                kprobes_inc_nmissed_count(p);
-        } else {
-                /* Kprobe handler expects regs->ip = ip + 1 as breakpoint hit */
-                regs->ip = ip + sizeof(kprobe_opcode_t);
-
-                __this_cpu_write(current_kprobe, p);
-                kcb->kprobe_status = KPROBE_HIT_ACTIVE;
-                if (!p->pre_handler || !p->pre_handler(p, regs))
-                        skip_singlestep(p, regs, kcb);
-                /*
-                 * If pre_handler returns !0, it sets regs->ip and
-                 * resets current kprobe.
-                 */
-        }
-end:
-        local_irq_restore(flags);
-}
-
-int __kprobes arch_prepare_kprobe_ftrace(struct kprobe *p)
-{
-        p->ainsn.insn = NULL;
-        p->ainsn.boostable = -1;
-        return 0;
-}
-#endif
-
 int __init arch_init_kprobes(void)
 {
         return arch_init_optprobes();
diff --git a/arch/x86/kernel/kprobes/ftrace.c b/arch/x86/kernel/kprobes/ftrace.c
new file mode 100644
index 000000000000..23ef5c556f06
--- /dev/null
+++ b/arch/x86/kernel/kprobes/ftrace.c
@@ -0,0 +1,93 @@
+/*
+ * Dynamic Ftrace based Kprobes Optimization
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * Copyright (C) Hitachi Ltd., 2012
+ */
+#include <linux/kprobes.h>
+#include <linux/ptrace.h>
+#include <linux/hardirq.h>
+#include <linux/preempt.h>
+#include <linux/ftrace.h>
+
+#include "common.h"
+
+static int __skip_singlestep(struct kprobe *p, struct pt_regs *regs,
+                             struct kprobe_ctlblk *kcb)
+{
+        /*
+         * Emulate singlestep (and also recover regs->ip)
+         * as if there is a 5byte nop
+         */
+        regs->ip = (unsigned long)p->addr + MCOUNT_INSN_SIZE;
+        if (unlikely(p->post_handler)) {
+                kcb->kprobe_status = KPROBE_HIT_SSDONE;
+                p->post_handler(p, regs, 0);
+        }
+        __this_cpu_write(current_kprobe, NULL);
+        return 1;
+}
+
+int __kprobes skip_singlestep(struct kprobe *p, struct pt_regs *regs,
+                              struct kprobe_ctlblk *kcb)
+{
+        if (kprobe_ftrace(p))
+                return __skip_singlestep(p, regs, kcb);
+        else
+                return 0;
+}
+
+/* Ftrace callback handler for kprobes */
+void __kprobes kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,
+                                     struct ftrace_ops *ops, struct pt_regs *regs)
+{
+        struct kprobe *p;
+        struct kprobe_ctlblk *kcb;
+        unsigned long flags;
+
+        /* Disable irq for emulating a breakpoint and avoiding preempt */
+        local_irq_save(flags);
+
+        p = get_kprobe((kprobe_opcode_t *)ip);
+        if (unlikely(!p) || kprobe_disabled(p))
+                goto end;
+
+        kcb = get_kprobe_ctlblk();
+        if (kprobe_running()) {
+                kprobes_inc_nmissed_count(p);
+        } else {
+                /* Kprobe handler expects regs->ip = ip + 1 as breakpoint hit */
+                regs->ip = ip + sizeof(kprobe_opcode_t);
+
+                __this_cpu_write(current_kprobe, p);
+                kcb->kprobe_status = KPROBE_HIT_ACTIVE;
+                if (!p->pre_handler || !p->pre_handler(p, regs))
+                        __skip_singlestep(p, regs, kcb);
+                /*
+                 * If pre_handler returns !0, it sets regs->ip and
+                 * resets current kprobe.
+                 */
+        }
+end:
+        local_irq_restore(flags);
+}
+
+int __kprobes arch_prepare_kprobe_ftrace(struct kprobe *p)
+{
+        p->ainsn.insn = NULL;
+        p->ainsn.boostable = -1;
+        return 0;
+}
diff --git a/arch/x86/kernel/kprobes-opt.c b/arch/x86/kernel/kprobes/opt.c
index c5e410eed403..76dc6f095724 100644
--- a/arch/x86/kernel/kprobes-opt.c
+++ b/arch/x86/kernel/kprobes/opt.c
@@ -37,7 +37,7 @@
 #include <asm/insn.h>
 #include <asm/debugreg.h>
 
-#include "kprobes-common.h"
+#include "common.h"
 
 unsigned long __recover_optprobed_insn(kprobe_opcode_t *buf, unsigned long addr)
 {
diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c
index 9c2bd8bd4b4c..b686a904d7c3 100644
--- a/arch/x86/kernel/kvm.c
+++ b/arch/x86/kernel/kvm.c
@@ -297,9 +297,9 @@ static void kvm_register_steal_time(void)
 
         memset(st, 0, sizeof(*st));
 
-        wrmsrl(MSR_KVM_STEAL_TIME, (__pa(st) | KVM_MSR_ENABLED));
-        printk(KERN_INFO "kvm-stealtime: cpu %d, msr %lx\n",
-                cpu, __pa(st));
+        wrmsrl(MSR_KVM_STEAL_TIME, (slow_virt_to_phys(st) | KVM_MSR_ENABLED));
+        pr_info("kvm-stealtime: cpu %d, msr %llx\n",
+                cpu, (unsigned long long) slow_virt_to_phys(st));
 }
 
 static DEFINE_PER_CPU(unsigned long, kvm_apic_eoi) = KVM_PV_EOI_DISABLED;
@@ -324,7 +324,7 @@ void __cpuinit kvm_guest_cpu_init(void)
                 return;
 
         if (kvm_para_has_feature(KVM_FEATURE_ASYNC_PF) && kvmapf) {
-                u64 pa = __pa(&__get_cpu_var(apf_reason));
+                u64 pa = slow_virt_to_phys(&__get_cpu_var(apf_reason));
 
 #ifdef CONFIG_PREEMPT
                 pa |= KVM_ASYNC_PF_SEND_ALWAYS;
@@ -340,7 +340,8 @@ void __cpuinit kvm_guest_cpu_init(void)
                 /* Size alignment is implied but just to make it explicit. */
                 BUILD_BUG_ON(__alignof__(kvm_apic_eoi) < 4);
                 __get_cpu_var(kvm_apic_eoi) = 0;
-                pa = __pa(&__get_cpu_var(kvm_apic_eoi)) | KVM_MSR_ENABLED;
+                pa = slow_virt_to_phys(&__get_cpu_var(kvm_apic_eoi))
+                        | KVM_MSR_ENABLED;
                 wrmsrl(MSR_KVM_PV_EOI_EN, pa);
         }
 
@@ -505,6 +506,7 @@ static bool __init kvm_detect(void)
 const struct hypervisor_x86 x86_hyper_kvm __refconst = {
         .name                   = "KVM",
         .detect                 = kvm_detect,
+        .x2apic_available       = kvm_para_available,
 };
 EXPORT_SYMBOL_GPL(x86_hyper_kvm);
 
diff --git a/arch/x86/kernel/kvmclock.c b/arch/x86/kernel/kvmclock.c
index 220a360010f8..0732f0089a3d 100644
--- a/arch/x86/kernel/kvmclock.c
+++ b/arch/x86/kernel/kvmclock.c
@@ -162,8 +162,8 @@ int kvm_register_clock(char *txt)
         int low, high, ret;
         struct pvclock_vcpu_time_info *src = &hv_clock[cpu].pvti;
 
-        low = (int)__pa(src) | 1;
-        high = ((u64)__pa(src) >> 32);
+        low = (int)slow_virt_to_phys(src) | 1;
+        high = ((u64)slow_virt_to_phys(src) >> 32);
         ret = native_write_msr_safe(msr_kvm_system_time, low, high);
         printk(KERN_INFO "kvm-clock: cpu %d, msr %x:%x, %s\n",
                cpu, high, low, txt);
@@ -218,6 +218,9 @@ static void kvm_shutdown(void)
 void __init kvmclock_init(void)
 {
         unsigned long mem;
+        int size;
+
+        size = PAGE_ALIGN(sizeof(struct pvclock_vsyscall_time_info)*NR_CPUS);
 
         if (!kvm_para_available())
                 return;
@@ -231,16 +234,14 @@ void __init kvmclock_init(void)
         printk(KERN_INFO "kvm-clock: Using msrs %x and %x",
                 msr_kvm_system_time, msr_kvm_wall_clock);
 
-        mem = memblock_alloc(sizeof(struct pvclock_vsyscall_time_info)*NR_CPUS,
-                             PAGE_SIZE);
+        mem = memblock_alloc(size, PAGE_SIZE);
         if (!mem)
                 return;
         hv_clock = __va(mem);
 
         if (kvm_register_clock("boot clock")) {
                 hv_clock = NULL;
-                memblock_free(mem,
-                        sizeof(struct pvclock_vsyscall_time_info)*NR_CPUS);
+                memblock_free(mem, size);
                 return;
         }
         pv_time_ops.sched_clock = kvm_clock_read;
@@ -275,7 +276,7 @@ int __init kvm_setup_vsyscall_timeinfo(void)
         struct pvclock_vcpu_time_info *vcpu_time;
         unsigned int size;
 
-        size = sizeof(struct pvclock_vsyscall_time_info)*NR_CPUS;
+        size = PAGE_ALIGN(sizeof(struct pvclock_vsyscall_time_info)*NR_CPUS);
 
         preempt_disable();
         cpu = smp_processor_id();
diff --git a/arch/x86/kernel/machine_kexec_64.c b/arch/x86/kernel/machine_kexec_64.c
index b3ea9db39db6..4eabc160696f 100644
--- a/arch/x86/kernel/machine_kexec_64.c
+++ b/arch/x86/kernel/machine_kexec_64.c
@@ -16,125 +16,12 @@
 #include <linux/io.h>
 #include <linux/suspend.h>
 
+#include <asm/init.h>
 #include <asm/pgtable.h>
 #include <asm/tlbflush.h>
 #include <asm/mmu_context.h>
 #include <asm/debugreg.h>
 
-static int init_one_level2_page(struct kimage *image, pgd_t *pgd,
-                                unsigned long addr)
-{
-        pud_t *pud;
-        pmd_t *pmd;
-        struct page *page;
-        int result = -ENOMEM;
-
-        addr &= PMD_MASK;
-        pgd += pgd_index(addr);
-        if (!pgd_present(*pgd)) {
-                page = kimage_alloc_control_pages(image, 0);
-                if (!page)
-                        goto out;
-                pud = (pud_t *)page_address(page);
-                clear_page(pud);
-                set_pgd(pgd, __pgd(__pa(pud) | _KERNPG_TABLE));
-        }
-        pud = pud_offset(pgd, addr);
-        if (!pud_present(*pud)) {
-                page = kimage_alloc_control_pages(image, 0);
-                if (!page)
-                        goto out;
-                pmd = (pmd_t *)page_address(page);
-                clear_page(pmd);
-                set_pud(pud, __pud(__pa(pmd) | _KERNPG_TABLE));
-        }
-        pmd = pmd_offset(pud, addr);
-        if (!pmd_present(*pmd))
-                set_pmd(pmd, __pmd(addr | __PAGE_KERNEL_LARGE_EXEC));
-        result = 0;
-out:
-        return result;
-}
-
-static void init_level2_page(pmd_t *level2p, unsigned long addr)
-{
-        unsigned long end_addr;
-
-        addr &= PAGE_MASK;
-        end_addr = addr + PUD_SIZE;
-        while (addr < end_addr) {
-                set_pmd(level2p++, __pmd(addr | __PAGE_KERNEL_LARGE_EXEC));
-                addr += PMD_SIZE;
-        }
-}
-
-static int init_level3_page(struct kimage *image, pud_t *level3p,
-                                unsigned long addr, unsigned long last_addr)
-{
-        unsigned long end_addr;
-        int result;
-
-        result = 0;
-        addr &= PAGE_MASK;
-        end_addr = addr + PGDIR_SIZE;
-        while ((addr < last_addr) && (addr < end_addr)) {
-                struct page *page;
-                pmd_t *level2p;
-
-                page = kimage_alloc_control_pages(image, 0);
-                if (!page) {
-                        result = -ENOMEM;
-                        goto out;
-                }
-                level2p = (pmd_t *)page_address(page);
-                init_level2_page(level2p, addr);
-                set_pud(level3p++, __pud(__pa(level2p) | _KERNPG_TABLE));
-                addr += PUD_SIZE;
-        }
-        /* clear the unused entries */
-        while (addr < end_addr) {
-                pud_clear(level3p++);
-                addr += PUD_SIZE;
-        }
-out:
-        return result;
-}
-
-
-static int init_level4_page(struct kimage *image, pgd_t *level4p,
-                                unsigned long addr, unsigned long last_addr)
-{
-        unsigned long end_addr;
-        int result;
-
-        result = 0;
-        addr &= PAGE_MASK;
-        end_addr = addr + (PTRS_PER_PGD * PGDIR_SIZE);
-        while ((addr < last_addr) && (addr < end_addr)) {
-                struct page *page;
-                pud_t *level3p;
-
-                page = kimage_alloc_control_pages(image, 0);
-                if (!page) {
-                        result = -ENOMEM;
-                        goto out;
-                }
-                level3p = (pud_t *)page_address(page);
-                result = init_level3_page(image, level3p, addr, last_addr);
-                if (result)
-                        goto out;
-                set_pgd(level4p++, __pgd(__pa(level3p) | _KERNPG_TABLE));
-                addr += PGDIR_SIZE;
-        }
-        /* clear the unused entries */
-        while (addr < end_addr) {
-                pgd_clear(level4p++);
-                addr += PGDIR_SIZE;
-        }
-out:
-        return result;
-}
-
 static void free_transition_pgtable(struct kimage *image)
 {
         free_page((unsigned long)image->arch.pud);
@@ -184,22 +71,62 @@ err:
         return result;
 }
 
+static void *alloc_pgt_page(void *data)
+{
+        struct kimage *image = (struct kimage *)data;
+        struct page *page;
+        void *p = NULL;
+
+        page = kimage_alloc_control_pages(image, 0);
+        if (page) {
+                p = page_address(page);
+                clear_page(p);
+        }
+
+        return p;
+}
 
 static int init_pgtable(struct kimage *image, unsigned long start_pgtable)
 {
+        struct x86_mapping_info info = {
+                .alloc_pgt_page = alloc_pgt_page,
+                .context        = image,
+                .pmd_flag       = __PAGE_KERNEL_LARGE_EXEC,
+        };
+        unsigned long mstart, mend;
         pgd_t *level4p;
         int result;
+        int i;
+
         level4p = (pgd_t *)__va(start_pgtable);
-        result = init_level4_page(image, level4p, 0, max_pfn << PAGE_SHIFT);
-        if (result)
-                return result;
+        clear_page(level4p);
+        for (i = 0; i < nr_pfn_mapped; i++) {
+                mstart = pfn_mapped[i].start << PAGE_SHIFT;
+                mend   = pfn_mapped[i].end << PAGE_SHIFT;
+
+                result = kernel_ident_mapping_init(&info,
+                                                 level4p, mstart, mend);
+                if (result)
+                        return result;
+        }
+
         /*
-         * image->start may be outside 0 ~ max_pfn, for example when
-         * jump back to original kernel from kexeced kernel
+         * segments's mem ranges could be outside 0 ~ max_pfn,
+         * for example when jump back to original kernel from kexeced kernel.
+         * or first kernel is booted with user mem map, and second kernel
+         * could be loaded out of that range.
          */
-        result = init_one_level2_page(image, level4p, image->start);
-        if (result)
-                return result;
+        for (i = 0; i < image->nr_segments; i++) {
+                mstart = image->segment[i].mem;
+                mend   = mstart + image->segment[i].memsz;
+
+                result = kernel_ident_mapping_init(&info,
+                                                 level4p, mstart, mend);
+
+                if (result)
+                        return result;
+        }
+
         return init_transition_pgtable(image, level4p);
 }
 
diff --git a/arch/x86/kernel/microcode_core.c b/arch/x86/kernel/microcode_core.c
index 3a04b224d0c0..22db92bbdf1a 100644
--- a/arch/x86/kernel/microcode_core.c
+++ b/arch/x86/kernel/microcode_core.c
@@ -364,10 +364,7 @@ static struct attribute_group mc_attr_group = {
 
 static void microcode_fini_cpu(int cpu)
 {
-        struct ucode_cpu_info *uci = ucode_cpu_info + cpu;
-
         microcode_ops->microcode_fini_cpu(cpu);
-        uci->valid = 0;
 }
 
 static enum ucode_state microcode_resume_cpu(int cpu)
@@ -383,6 +380,10 @@ static enum ucode_state microcode_resume_cpu(int cpu)
 static enum ucode_state microcode_init_cpu(int cpu, bool refresh_fw)
 {
         enum ucode_state ustate;
+        struct ucode_cpu_info *uci = ucode_cpu_info + cpu;
+
+        if (uci && uci->valid)
+                return UCODE_OK;
 
         if (collect_cpu_info(cpu))
                 return UCODE_ERROR;
diff --git a/arch/x86/kernel/microcode_core_early.c b/arch/x86/kernel/microcode_core_early.c
new file mode 100644
index 000000000000..577db8417d15
--- /dev/null
+++ b/arch/x86/kernel/microcode_core_early.c
@@ -0,0 +1,76 @@
+/*
+ *      X86 CPU microcode early update for Linux
+ *
+ *      Copyright (C) 2012 Fenghua Yu <fenghua.yu@intel.com>
+ *                         H Peter Anvin" <hpa@zytor.com>
+ *
+ *      This driver allows to early upgrade microcode on Intel processors
+ *      belonging to IA-32 family - PentiumPro, Pentium II,
+ *      Pentium III, Xeon, Pentium 4, etc.
+ *
+ *      Reference: Section 9.11 of Volume 3, IA-32 Intel Architecture
+ *      Software Developer's Manual.
+ *
+ *      This program is free software; you can redistribute it and/or
+ *      modify it under the terms of the GNU General Public License
+ *      as published by the Free Software Foundation; either version
+ *      2 of the License, or (at your option) any later version.
+ */
+#include <linux/module.h>
+#include <asm/microcode_intel.h>
+#include <asm/processor.h>
+
+#define QCHAR(a, b, c, d) ((a) + ((b) << 8) + ((c) << 16) + ((d) << 24))
+#define CPUID_INTEL1 QCHAR('G', 'e', 'n', 'u')
+#define CPUID_INTEL2 QCHAR('i', 'n', 'e', 'I')
+#define CPUID_INTEL3 QCHAR('n', 't', 'e', 'l')
+#define CPUID_AMD1 QCHAR('A', 'u', 't', 'h')
+#define CPUID_AMD2 QCHAR('e', 'n', 't', 'i')
+#define CPUID_AMD3 QCHAR('c', 'A', 'M', 'D')
+
+#define CPUID_IS(a, b, c, ebx, ecx, edx)        \
+                (!((ebx ^ (a))|(edx ^ (b))|(ecx ^ (c))))
+
+/*
+ * In early loading microcode phase on BSP, boot_cpu_data is not set up yet.
+ * x86_vendor() gets vendor id for BSP.
+ *
+ * In 32 bit AP case, accessing boot_cpu_data needs linear address. To simplify
+ * coding, we still use x86_vendor() to get vendor id for AP.
+ *
+ * x86_vendor() gets vendor information directly through cpuid.
+ */
+static int __cpuinit x86_vendor(void)
+{
+        u32 eax = 0x00000000;
+        u32 ebx, ecx = 0, edx;
+
+        if (!have_cpuid_p())
+                return X86_VENDOR_UNKNOWN;
+
+        native_cpuid(&eax, &ebx, &ecx, &edx);
+
+        if (CPUID_IS(CPUID_INTEL1, CPUID_INTEL2, CPUID_INTEL3, ebx, ecx, edx))
+                return X86_VENDOR_INTEL;
+
+        if (CPUID_IS(CPUID_AMD1, CPUID_AMD2, CPUID_AMD3, ebx, ecx, edx))
+                return X86_VENDOR_AMD;
+
+        return X86_VENDOR_UNKNOWN;
+}
+
+void __init load_ucode_bsp(void)
+{
+        int vendor = x86_vendor();
+
+        if (vendor == X86_VENDOR_INTEL)
+                load_ucode_intel_bsp();
+}
+
+void __cpuinit load_ucode_ap(void)
+{
+        int vendor = x86_vendor();
+
+        if (vendor == X86_VENDOR_INTEL)
+                load_ucode_intel_ap();
+}
diff --git a/arch/x86/kernel/microcode_intel.c b/arch/x86/kernel/microcode_intel.c
index 3544aed39338..5fb2cebf556b 100644
--- a/arch/x86/kernel/microcode_intel.c
+++ b/arch/x86/kernel/microcode_intel.c
@@ -79,7 +79,7 @@
 #include <linux/module.h>
 #include <linux/vmalloc.h>
 
-#include <asm/microcode.h>
+#include <asm/microcode_intel.h>
 #include <asm/processor.h>
 #include <asm/msr.h>
 
@@ -87,59 +87,6 @@ MODULE_DESCRIPTION("Microcode Update Driver");
 MODULE_AUTHOR("Tigran Aivazian <tigran@aivazian.fsnet.co.uk>");
 MODULE_LICENSE("GPL");
 
-struct microcode_header_intel {
-        unsigned int            hdrver;
-        unsigned int            rev;
-        unsigned int            date;
-        unsigned int            sig;
-        unsigned int            cksum;
-        unsigned int            ldrver;
-        unsigned int            pf;
-        unsigned int            datasize;
-        unsigned int            totalsize;
-        unsigned int            reserved[3];
-};
-
-struct microcode_intel {
-        struct microcode_header_intel hdr;
-        unsigned int            bits[0];
-};
-
-/* microcode format is extended from prescott processors */
-struct extended_signature {
-        unsigned int            sig;
-        unsigned int            pf;
-        unsigned int            cksum;
-};
-
-struct extended_sigtable {
-        unsigned int            count;
-        unsigned int            cksum;
-        unsigned int            reserved[3];
-        struct extended_signature sigs[0];
-};
-
-#define DEFAULT_UCODE_DATASIZE  (2000)
-#define MC_HEADER_SIZE          (sizeof(struct microcode_header_intel))
-#define DEFAULT_UCODE_TOTALSIZE (DEFAULT_UCODE_DATASIZE + MC_HEADER_SIZE)
-#define EXT_HEADER_SIZE         (sizeof(struct extended_sigtable))
-#define EXT_SIGNATURE_SIZE      (sizeof(struct extended_signature))
-#define DWSIZE                  (sizeof(u32))
-
-#define get_totalsize(mc) \
-        (((struct microcode_intel *)mc)->hdr.totalsize ? \
-         ((struct microcode_intel *)mc)->hdr.totalsize : \
-         DEFAULT_UCODE_TOTALSIZE)
-
-#define get_datasize(mc) \
-        (((struct microcode_intel *)mc)->hdr.datasize ? \
-         ((struct microcode_intel *)mc)->hdr.datasize : DEFAULT_UCODE_DATASIZE)
-
-#define sigmatch(s1, s2, p1, p2) \
-        (((s1) == (s2)) && (((p1) & (p2)) || (((p1) == 0) && ((p2) == 0))))
-
-#define exttable_size(et) ((et)->count * EXT_SIGNATURE_SIZE + EXT_HEADER_SIZE)
-
 static int collect_cpu_info(int cpu_num, struct cpu_signature *csig)
 {
         struct cpuinfo_x86 *c = &cpu_data(cpu_num);
@@ -162,128 +109,25 @@ static int collect_cpu_info(int cpu_num, struct cpu_signature *csig)
         return 0;
 }
 
-static inline int update_match_cpu(struct cpu_signature *csig, int sig, int pf)
-{
-        return (!sigmatch(sig, csig->sig, pf, csig->pf)) ? 0 : 1;
-}
-
-static inline int
-update_match_revision(struct microcode_header_intel *mc_header, int rev)
-{
-        return (mc_header->rev <= rev) ? 0 : 1;
-}
-
-static int microcode_sanity_check(void *mc)
-{
-        unsigned long total_size, data_size, ext_table_size;
-        struct microcode_header_intel *mc_header = mc;
-        struct extended_sigtable *ext_header = NULL;
-        int sum, orig_sum, ext_sigcount = 0, i;
-        struct extended_signature *ext_sig;
-
-        total_size = get_totalsize(mc_header);
-        data_size = get_datasize(mc_header);
-
-        if (data_size + MC_HEADER_SIZE > total_size) {
-                pr_err("error! Bad data size in microcode data file\n");
-                return -EINVAL;
-        }
-
-        if (mc_header->ldrver != 1 || mc_header->hdrver != 1) {
-                pr_err("error! Unknown microcode update format\n");
-                return -EINVAL;
-        }
-        ext_table_size = total_size - (MC_HEADER_SIZE + data_size);
-        if (ext_table_size) {
-                if ((ext_table_size < EXT_HEADER_SIZE)
-                 || ((ext_table_size - EXT_HEADER_SIZE) % EXT_SIGNATURE_SIZE)) {
-                        pr_err("error! Small exttable size in microcode data file\n");
-                        return -EINVAL;
-                }
-                ext_header = mc + MC_HEADER_SIZE + data_size;
-                if (ext_table_size != exttable_size(ext_header)) {
-                        pr_err("error! Bad exttable size in microcode data file\n");
-                        return -EFAULT;
-                }
-                ext_sigcount = ext_header->count;
-        }
-
-        /* check extended table checksum */
-        if (ext_table_size) {
-                int ext_table_sum = 0;
-                int *ext_tablep = (int *)ext_header;
-
-                i = ext_table_size / DWSIZE;
-                while (i--)
-                        ext_table_sum += ext_tablep[i];
-                if (ext_table_sum) {
-                        pr_warning("aborting, bad extended signature table checksum\n");
-                        return -EINVAL;
-                }
-        }
-
-        /* calculate the checksum */
-        orig_sum = 0;
-        i = (MC_HEADER_SIZE + data_size) / DWSIZE;
-        while (i--)
-                orig_sum += ((int *)mc)[i];
-        if (orig_sum) {
-                pr_err("aborting, bad checksum\n");
-                return -EINVAL;
-        }
-        if (!ext_table_size)
-                return 0;
-        /* check extended signature checksum */
-        for (i = 0; i < ext_sigcount; i++) {
-                ext_sig = (void *)ext_header + EXT_HEADER_SIZE +
-                          EXT_SIGNATURE_SIZE * i;
-                sum = orig_sum
-                        - (mc_header->sig + mc_header->pf + mc_header->cksum)
-                        + (ext_sig->sig + ext_sig->pf + ext_sig->cksum);
-                if (sum) {
-                        pr_err("aborting, bad checksum\n");
-                        return -EINVAL;
-                }
-        }
-        return 0;
-}
-
 /*
  * return 0 - no update found
  * return 1 - found update
  */
-static int
-get_matching_microcode(struct cpu_signature *cpu_sig, void *mc, int rev)
+static int get_matching_mc(struct microcode_intel *mc_intel, int cpu)
 {
-        struct microcode_header_intel *mc_header = mc;
-        struct extended_sigtable *ext_header;
-        unsigned long total_size = get_totalsize(mc_header);
-        int ext_sigcount, i;
-        struct extended_signature *ext_sig;
-
-        if (!update_match_revision(mc_header, rev))
-                return 0;
-
-        if (update_match_cpu(cpu_sig, mc_header->sig, mc_header->pf))
-                return 1;
+        struct cpu_signature cpu_sig;
+        unsigned int csig, cpf, crev;
 
-        /* Look for ext. headers: */
-        if (total_size <= get_datasize(mc_header) + MC_HEADER_SIZE)
-                return 0;
+        collect_cpu_info(cpu, &cpu_sig);
 
-        ext_header = mc + get_datasize(mc_header) + MC_HEADER_SIZE;
-        ext_sigcount = ext_header->count;
-        ext_sig = (void *)ext_header + EXT_HEADER_SIZE;
+        csig = cpu_sig.sig;
+        cpf = cpu_sig.pf;
+        crev = cpu_sig.rev;
 
-        for (i = 0; i < ext_sigcount; i++) {
-                if (update_match_cpu(cpu_sig, ext_sig->sig, ext_sig->pf))
-                        return 1;
-                ext_sig++;
-        }
-        return 0;
+        return get_matching_microcode(csig, cpf, mc_intel, crev);
 }
 
-static int apply_microcode(int cpu)
+int apply_microcode(int cpu)
 {
         struct microcode_intel *mc_intel;
         struct ucode_cpu_info *uci;
@@ -300,6 +144,14 @@ static int apply_microcode(int cpu)
         if (mc_intel == NULL)
                 return 0;
 
+        /*
+         * Microcode on this CPU could be updated earlier. Only apply the
+         * microcode patch in mc_intel when it is newer than the one on this
+         * CPU.
+         */
+        if (get_matching_mc(mc_intel, cpu) == 0)
+                return 0;
+
         /* write microcode via MSR 0x79 */
         wrmsr(MSR_IA32_UCODE_WRITE,
               (unsigned long) mc_intel->bits,
@@ -338,6 +190,7 @@ static enum ucode_state generic_load_microcode(int cpu, void *data, size_t size,
         unsigned int leftover = size;
         enum ucode_state state = UCODE_OK;
         unsigned int curr_mc_size = 0;
+        unsigned int csig, cpf;
 
         while (leftover) {
                 struct microcode_header_intel mc_header;
@@ -362,11 +215,13 @@ static enum ucode_state generic_load_microcode(int cpu, void *data, size_t size,
                 }
 
                 if (get_ucode_data(mc, ucode_ptr, mc_size) ||
-                    microcode_sanity_check(mc) < 0) {
+                    microcode_sanity_check(mc, 1) < 0) {
                         break;
                 }
 
-                if (get_matching_microcode(&uci->cpu_sig, mc, new_rev)) {
+                csig = uci->cpu_sig.sig;
+                cpf = uci->cpu_sig.pf;
+                if (get_matching_microcode(csig, cpf, mc, new_rev)) {
                         vfree(new_mc);
                         new_rev = mc_header.rev;
                         new_mc  = mc;
@@ -393,6 +248,13 @@ static enum ucode_state generic_load_microcode(int cpu, void *data, size_t size,
         vfree(uci->mc);
         uci->mc = (struct microcode_intel *)new_mc;
 
+        /*
+         * If early loading microcode is supported, save this mc into
+         * permanent memory. So it will be loaded early when a CPU is hot added
+         * or resumes.
+         */
+        save_mc_for_early(new_mc);
+
         pr_debug("CPU%d found a matching microcode update with version 0x%x (current=0x%x)\n",
                  cpu, new_rev, uci->cpu_sig.rev);
 out:
diff --git a/arch/x86/kernel/microcode_intel_early.c b/arch/x86/kernel/microcode_intel_early.c
new file mode 100644
index 000000000000..d893e8ed8ac9
--- /dev/null
+++ b/arch/x86/kernel/microcode_intel_early.c
@@ -0,0 +1,796 @@
+/*
+ *      Intel CPU microcode early update for Linux
+ *
+ *      Copyright (C) 2012 Fenghua Yu <fenghua.yu@intel.com>
+ *                         H Peter Anvin" <hpa@zytor.com>
+ *
+ *      This allows to early upgrade microcode on Intel processors
+ *      belonging to IA-32 family - PentiumPro, Pentium II,
+ *      Pentium III, Xeon, Pentium 4, etc.
+ *
+ *      Reference: Section 9.11 of Volume 3, IA-32 Intel Architecture
+ *      Software Developer's Manual.
+ *
+ *      This program is free software; you can redistribute it and/or
+ *      modify it under the terms of the GNU General Public License
+ *      as published by the Free Software Foundation; either version
+ *      2 of the License, or (at your option) any later version.
+ */
+#include <linux/module.h>
+#include <linux/mm.h>
+#include <linux/slab.h>
+#include <linux/earlycpio.h>
+#include <linux/initrd.h>
+#include <linux/cpu.h>
+#include <asm/msr.h>
+#include <asm/microcode_intel.h>
+#include <asm/processor.h>
+#include <asm/tlbflush.h>
+#include <asm/setup.h>
+
+unsigned long mc_saved_in_initrd[MAX_UCODE_COUNT];
+struct mc_saved_data {
+        unsigned int mc_saved_count;
+        struct microcode_intel **mc_saved;
+} mc_saved_data;
+
+static enum ucode_state __cpuinit
+generic_load_microcode_early(struct microcode_intel **mc_saved_p,
+                             unsigned int mc_saved_count,
+                             struct ucode_cpu_info *uci)
+{
+        struct microcode_intel *ucode_ptr, *new_mc = NULL;
+        int new_rev = uci->cpu_sig.rev;
+        enum ucode_state state = UCODE_OK;
+        unsigned int mc_size;
+        struct microcode_header_intel *mc_header;
+        unsigned int csig = uci->cpu_sig.sig;
+        unsigned int cpf = uci->cpu_sig.pf;
+        int i;
+
+        for (i = 0; i < mc_saved_count; i++) {
+                ucode_ptr = mc_saved_p[i];
+
+                mc_header = (struct microcode_header_intel *)ucode_ptr;
+                mc_size = get_totalsize(mc_header);
+                if (get_matching_microcode(csig, cpf, ucode_ptr, new_rev)) {
+                        new_rev = mc_header->rev;
+                        new_mc  = ucode_ptr;
+                }
+        }
+
+        if (!new_mc) {
+                state = UCODE_NFOUND;
+                goto out;
+        }
+
+        uci->mc = (struct microcode_intel *)new_mc;
+out:
+        return state;
+}
+
+static void __cpuinit
+microcode_pointer(struct microcode_intel **mc_saved,
+                  unsigned long *mc_saved_in_initrd,
+                  unsigned long initrd_start, int mc_saved_count)
+{
+        int i;
+
+        for (i = 0; i < mc_saved_count; i++)
+                mc_saved[i] = (struct microcode_intel *)
+                              (mc_saved_in_initrd[i] + initrd_start);
+}
+
+#ifdef CONFIG_X86_32
+static void __cpuinit
+microcode_phys(struct microcode_intel **mc_saved_tmp,
+               struct mc_saved_data *mc_saved_data)
+{
+        int i;
+        struct microcode_intel ***mc_saved;
+
+        mc_saved = (struct microcode_intel ***)
+                   __pa_nodebug(&mc_saved_data->mc_saved);
+        for (i = 0; i < mc_saved_data->mc_saved_count; i++) {
+                struct microcode_intel *p;
+
+                p = *(struct microcode_intel **)
+                        __pa_nodebug(mc_saved_data->mc_saved + i);
+                mc_saved_tmp[i] = (struct microcode_intel *)__pa_nodebug(p);
+        }
+}
+#endif
+
+static enum ucode_state __cpuinit
+load_microcode(struct mc_saved_data *mc_saved_data,
+               unsigned long *mc_saved_in_initrd,
+               unsigned long initrd_start,
+               struct ucode_cpu_info *uci)
+{
+        struct microcode_intel *mc_saved_tmp[MAX_UCODE_COUNT];
+        unsigned int count = mc_saved_data->mc_saved_count;
+
+        if (!mc_saved_data->mc_saved) {
+                microcode_pointer(mc_saved_tmp, mc_saved_in_initrd,
+                                  initrd_start, count);
+
+                return generic_load_microcode_early(mc_saved_tmp, count, uci);
+        } else {
+#ifdef CONFIG_X86_32
+                microcode_phys(mc_saved_tmp, mc_saved_data);
+                return generic_load_microcode_early(mc_saved_tmp, count, uci);
+#else
+                return generic_load_microcode_early(mc_saved_data->mc_saved,
+                                                    count, uci);
+#endif
+        }
+}
+
+static u8 get_x86_family(unsigned long sig)
+{
+        u8 x86;
+
+        x86 = (sig >> 8) & 0xf;
+
+        if (x86 == 0xf)
+                x86 += (sig >> 20) & 0xff;
+
+        return x86;
+}
+
+static u8 get_x86_model(unsigned long sig)
+{
+        u8 x86, x86_model;
+
+        x86 = get_x86_family(sig);
+        x86_model = (sig >> 4) & 0xf;
+
+        if (x86 == 0x6 || x86 == 0xf)
+                x86_model += ((sig >> 16) & 0xf) << 4;
+
+        return x86_model;
+}
+
+/*
+ * Given CPU signature and a microcode patch, this function finds if the
+ * microcode patch has matching family and model with the CPU.
+ */
+static enum ucode_state
+matching_model_microcode(struct microcode_header_intel *mc_header,
+                        unsigned long sig)
+{
+        u8 x86, x86_model;
+        u8 x86_ucode, x86_model_ucode;
+        struct extended_sigtable *ext_header;
+        unsigned long total_size = get_totalsize(mc_header);
+        unsigned long data_size = get_datasize(mc_header);
+        int ext_sigcount, i;
+        struct extended_signature *ext_sig;
+
+        x86 = get_x86_family(sig);
+        x86_model = get_x86_model(sig);
+
+        x86_ucode = get_x86_family(mc_header->sig);
+        x86_model_ucode = get_x86_model(mc_header->sig);
+
+        if (x86 == x86_ucode && x86_model == x86_model_ucode)
+                return UCODE_OK;
+
+        /* Look for ext. headers: */
+        if (total_size <= data_size + MC_HEADER_SIZE)
+                return UCODE_NFOUND;
+
+        ext_header = (struct extended_sigtable *)
+                     mc_header + data_size + MC_HEADER_SIZE;
+        ext_sigcount = ext_header->count;
+        ext_sig = (void *)ext_header + EXT_HEADER_SIZE;
+
+        for (i = 0; i < ext_sigcount; i++) {
+                x86_ucode = get_x86_family(ext_sig->sig);
+                x86_model_ucode = get_x86_model(ext_sig->sig);
+
+                if (x86 == x86_ucode && x86_model == x86_model_ucode)
+                        return UCODE_OK;
+
+                ext_sig++;
+        }
+
+        return UCODE_NFOUND;
+}
+
+static int
+save_microcode(struct mc_saved_data *mc_saved_data,
+               struct microcode_intel **mc_saved_src,
+               unsigned int mc_saved_count)
+{
+        int i, j;
+        struct microcode_intel **mc_saved_p;
+        int ret;
+
+        if (!mc_saved_count)
+                return -EINVAL;
+
+        /*
+         * Copy new microcode data.
+         */
+        mc_saved_p = kmalloc(mc_saved_count*sizeof(struct microcode_intel *),
+                             GFP_KERNEL);
+        if (!mc_saved_p)
+                return -ENOMEM;
+
+        for (i = 0; i < mc_saved_count; i++) {
+                struct microcode_intel *mc = mc_saved_src[i];
+                struct microcode_header_intel *mc_header = &mc->hdr;
+                unsigned long mc_size = get_totalsize(mc_header);
+                mc_saved_p[i] = kmalloc(mc_size, GFP_KERNEL);
+                if (!mc_saved_p[i]) {
+                        ret = -ENOMEM;
+                        goto err;
+                }
+                if (!mc_saved_src[i]) {
+                        ret = -EINVAL;
+                        goto err;
+                }
+                memcpy(mc_saved_p[i], mc, mc_size);
+        }
+
+        /*
+         * Point to newly saved microcode.
+         */
+        mc_saved_data->mc_saved = mc_saved_p;
+        mc_saved_data->mc_saved_count = mc_saved_count;
+
+        return 0;
+
+err:
+        for (j = 0; j <= i; j++)
+                kfree(mc_saved_p[j]);
+        kfree(mc_saved_p);
+
+        return ret;
+}
+
+/*
+ * A microcode patch in ucode_ptr is saved into mc_saved
+ * - if it has matching signature and newer revision compared to an existing
+ *   patch mc_saved.
+ * - or if it is a newly discovered microcode patch.
+ *
+ * The microcode patch should have matching model with CPU.
+ */
+static void _save_mc(struct microcode_intel **mc_saved, u8 *ucode_ptr,
+                     unsigned int *mc_saved_count_p)
+{
+        int i;
+        int found = 0;
+        unsigned int mc_saved_count = *mc_saved_count_p;
+        struct microcode_header_intel *mc_header;
+
+        mc_header = (struct microcode_header_intel *)ucode_ptr;
+        for (i = 0; i < mc_saved_count; i++) {
+                unsigned int sig, pf;
+                unsigned int new_rev;
+                struct microcode_header_intel *mc_saved_header =
+                             (struct microcode_header_intel *)mc_saved[i];
+                sig = mc_saved_header->sig;
+                pf = mc_saved_header->pf;
+                new_rev = mc_header->rev;
+
+                if (get_matching_sig(sig, pf, ucode_ptr, new_rev)) {
+                        found = 1;
+                        if (update_match_revision(mc_header, new_rev)) {
+                                /*
+                                 * Found an older ucode saved before.
+                                 * Replace the older one with this newer
+                                 * one.
+                                 */
+                                mc_saved[i] =
+                                        (struct microcode_intel *)ucode_ptr;
+                                break;
+                        }
+                }
+        }
+        if (i >= mc_saved_count && !found)
+                /*
+                 * This ucode is first time discovered in ucode file.
+                 * Save it to memory.
+                 */
+                mc_saved[mc_saved_count++] =
+                                 (struct microcode_intel *)ucode_ptr;
+
+        *mc_saved_count_p = mc_saved_count;
+}
+
+/*
+ * Get microcode matching with BSP's model. Only CPUs with the same model as
+ * BSP can stay in the platform.
+ */
+static enum ucode_state __init
+get_matching_model_microcode(int cpu, unsigned long start,
+                             void *data, size_t size,
+                             struct mc_saved_data *mc_saved_data,
+                             unsigned long *mc_saved_in_initrd,
+                             struct ucode_cpu_info *uci)
+{
+        u8 *ucode_ptr = data;
+        unsigned int leftover = size;
+        enum ucode_state state = UCODE_OK;
+        unsigned int mc_size;
+        struct microcode_header_intel *mc_header;
+        struct microcode_intel *mc_saved_tmp[MAX_UCODE_COUNT];
+        unsigned int mc_saved_count = mc_saved_data->mc_saved_count;
+        int i;
+
+        while (leftover) {
+                mc_header = (struct microcode_header_intel *)ucode_ptr;
+
+                mc_size = get_totalsize(mc_header);
+                if (!mc_size || mc_size > leftover ||
+                        microcode_sanity_check(ucode_ptr, 0) < 0)
+                        break;
+
+                leftover -= mc_size;
+
+                /*
+                 * Since APs with same family and model as the BSP may boot in
+                 * the platform, we need to find and save microcode patches
+                 * with the same family and model as the BSP.
+                 */
+                if (matching_model_microcode(mc_header, uci->cpu_sig.sig) !=
+                         UCODE_OK) {
+                        ucode_ptr += mc_size;
+                        continue;
+                }
+
+                _save_mc(mc_saved_tmp, ucode_ptr, &mc_saved_count);
+
+                ucode_ptr += mc_size;
+        }
+
+        if (leftover) {
+                state = UCODE_ERROR;
+                goto out;
+        }
+
+        if (mc_saved_count == 0) {
+                state = UCODE_NFOUND;
+                goto out;
+        }
+
+        for (i = 0; i < mc_saved_count; i++)
+                mc_saved_in_initrd[i] = (unsigned long)mc_saved_tmp[i] - start;
+
+        mc_saved_data->mc_saved_count = mc_saved_count;
+out:
+        return state;
+}
+
+#define native_rdmsr(msr, val1, val2)           \
+do {                                            \
+        u64 __val = native_read_msr((msr));     \
+        (void)((val1) = (u32)__val);            \
+        (void)((val2) = (u32)(__val >> 32));    \
+} while (0)
+
+#define native_wrmsr(msr, low, high)            \
+        native_write_msr(msr, low, high);
+
+static int __cpuinit collect_cpu_info_early(struct ucode_cpu_info *uci)
+{
+        unsigned int val[2];
+        u8 x86, x86_model;
+        struct cpu_signature csig;
+        unsigned int eax, ebx, ecx, edx;
+
+        csig.sig = 0;
+        csig.pf = 0;
+        csig.rev = 0;
+
+        memset(uci, 0, sizeof(*uci));
+
+        eax = 0x00000001;
+        ecx = 0;
+        native_cpuid(&eax, &ebx, &ecx, &edx);
+        csig.sig = eax;
+
+        x86 = get_x86_family(csig.sig);
+        x86_model = get_x86_model(csig.sig);
+
+        if ((x86_model >= 5) || (x86 > 6)) {
+                /* get processor flags from MSR 0x17 */
+                native_rdmsr(MSR_IA32_PLATFORM_ID, val[0], val[1]);
+                csig.pf = 1 << ((val[1] >> 18) & 7);
+        }
+        native_wrmsr(MSR_IA32_UCODE_REV, 0, 0);
+
+        /* As documented in the SDM: Do a CPUID 1 here */
+        sync_core();
+
+        /* get the current revision from MSR 0x8B */
+        native_rdmsr(MSR_IA32_UCODE_REV, val[0], val[1]);
+
+        csig.rev = val[1];
+
+        uci->cpu_sig = csig;
+        uci->valid = 1;
+
+        return 0;
+}
+
+#ifdef DEBUG
+static void __ref show_saved_mc(void)
+{
+        int i, j;
+        unsigned int sig, pf, rev, total_size, data_size, date;
+        struct ucode_cpu_info uci;
+
+        if (mc_saved_data.mc_saved_count == 0) {
+                pr_debug("no micorcode data saved.\n");
+                return;
+        }
+        pr_debug("Total microcode saved: %d\n", mc_saved_data.mc_saved_count);
+
+        collect_cpu_info_early(&uci);
+
+        sig = uci.cpu_sig.sig;
+        pf = uci.cpu_sig.pf;
+        rev = uci.cpu_sig.rev;
+        pr_debug("CPU%d: sig=0x%x, pf=0x%x, rev=0x%x\n",
+                 smp_processor_id(), sig, pf, rev);
+
+        for (i = 0; i < mc_saved_data.mc_saved_count; i++) {
+                struct microcode_header_intel *mc_saved_header;
+                struct extended_sigtable *ext_header;
+                int ext_sigcount;
+                struct extended_signature *ext_sig;
+
+                mc_saved_header = (struct microcode_header_intel *)
+                                  mc_saved_data.mc_saved[i];
+                sig = mc_saved_header->sig;
+                pf = mc_saved_header->pf;
+                rev = mc_saved_header->rev;
+                total_size = get_totalsize(mc_saved_header);
+                data_size = get_datasize(mc_saved_header);
+                date = mc_saved_header->date;
+
+                pr_debug("mc_saved[%d]: sig=0x%x, pf=0x%x, rev=0x%x, toal size=0x%x, date = %04x-%02x-%02x\n",
+                         i, sig, pf, rev, total_size,
+                         date & 0xffff,
+                         date >> 24,
+                         (date >> 16) & 0xff);
+
+                /* Look for ext. headers: */
+                if (total_size <= data_size + MC_HEADER_SIZE)
+                        continue;
+
+                ext_header = (struct extended_sigtable *)
+                             mc_saved_header + data_size + MC_HEADER_SIZE;
+                ext_sigcount = ext_header->count;
+                ext_sig = (void *)ext_header + EXT_HEADER_SIZE;
+
+                for (j = 0; j < ext_sigcount; j++) {
+                        sig = ext_sig->sig;
+                        pf = ext_sig->pf;
+
+                        pr_debug("\tExtended[%d]: sig=0x%x, pf=0x%x\n",
+                                 j, sig, pf);
+
+                        ext_sig++;
+                }
+
+        }
+}
+#else
+static inline void show_saved_mc(void)
+{
+}
+#endif
+
+#if defined(CONFIG_MICROCODE_INTEL_EARLY) && defined(CONFIG_HOTPLUG_CPU)
+/*
+ * Save this mc into mc_saved_data. So it will be loaded early when a CPU is
+ * hot added or resumes.
+ *
+ * Please make sure this mc should be a valid microcode patch before calling
+ * this function.
+ */
+int save_mc_for_early(u8 *mc)
+{
+        struct microcode_intel *mc_saved_tmp[MAX_UCODE_COUNT];
+        unsigned int mc_saved_count_init;
+        unsigned int mc_saved_count;
+        struct microcode_intel **mc_saved;
+        int ret = 0;
+        int i;
+
+        /*
+         * Hold hotplug lock so mc_saved_data is not accessed by a CPU in
+         * hotplug.
+         */
+        cpu_hotplug_driver_lock();
+
+        mc_saved_count_init = mc_saved_data.mc_saved_count;
+        mc_saved_count = mc_saved_data.mc_saved_count;
+        mc_saved = mc_saved_data.mc_saved;
+
+        if (mc_saved && mc_saved_count)
+                memcpy(mc_saved_tmp, mc_saved,
+                       mc_saved_count * sizeof(struct mirocode_intel *));
+        /*
+         * Save the microcode patch mc in mc_save_tmp structure if it's a newer
+         * version.
+         */
+
+        _save_mc(mc_saved_tmp, mc, &mc_saved_count);
+
+        /*
+         * Save the mc_save_tmp in global mc_saved_data.
+         */
+        ret = save_microcode(&mc_saved_data, mc_saved_tmp, mc_saved_count);
+        if (ret) {
+                pr_err("Can not save microcode patch.\n");
+                goto out;
+        }
+
+        show_saved_mc();
+
+        /*
+         * Free old saved microcod data.
+         */
+        if (mc_saved) {
+                for (i = 0; i < mc_saved_count_init; i++)
+                        kfree(mc_saved[i]);
+                kfree(mc_saved);
+        }
+
+out:
+        cpu_hotplug_driver_unlock();
+
+        return ret;
+}
+EXPORT_SYMBOL_GPL(save_mc_for_early);
+#endif
+
+static __initdata char ucode_name[] = "kernel/x86/microcode/GenuineIntel.bin";
+static __init enum ucode_state
+scan_microcode(unsigned long start, unsigned long end,
+                struct mc_saved_data *mc_saved_data,
+                unsigned long *mc_saved_in_initrd,
+                struct ucode_cpu_info *uci)
+{
+        unsigned int size = end - start + 1;
+        struct cpio_data cd;
+        long offset = 0;
+#ifdef CONFIG_X86_32
+        char *p = (char *)__pa_nodebug(ucode_name);
+#else
+        char *p = ucode_name;
+#endif
+
+        cd.data = NULL;
+        cd.size = 0;
+
+        cd = find_cpio_data(p, (void *)start, size, &offset);
+        if (!cd.data)
+                return UCODE_ERROR;
+
+
+        return get_matching_model_microcode(0, start, cd.data, cd.size,
+                                            mc_saved_data, mc_saved_in_initrd,
+                                            uci);
+}
+
+/*
+ * Print ucode update info.
+ */
+static void __cpuinit
+print_ucode_info(struct ucode_cpu_info *uci, unsigned int date)
+{
+        int cpu = smp_processor_id();
+
+        pr_info("CPU%d microcode updated early to revision 0x%x, date = %04x-%02x-%02x\n",
+                cpu,
+                uci->cpu_sig.rev,
+                date & 0xffff,
+                date >> 24,
+                (date >> 16) & 0xff);
+}
+
+#ifdef CONFIG_X86_32
+
+static int delay_ucode_info;
+static int current_mc_date;
+
+/*
+ * Print early updated ucode info after printk works. This is delayed info dump.
+ */
+void __cpuinit show_ucode_info_early(void)
+{
+        struct ucode_cpu_info uci;
+
+        if (delay_ucode_info) {
+                collect_cpu_info_early(&uci);
+                print_ucode_info(&uci, current_mc_date);
+                delay_ucode_info = 0;
+        }
+}
+
+/*
+ * At this point, we can not call printk() yet. Keep microcode patch number in
+ * mc_saved_data.mc_saved and delay printing microcode info in
+ * show_ucode_info_early() until printk() works.
+ */
+static void __cpuinit print_ucode(struct ucode_cpu_info *uci)
+{
+        struct microcode_intel *mc_intel;
+        int *delay_ucode_info_p;
+        int *current_mc_date_p;
+
+        mc_intel = uci->mc;
+        if (mc_intel == NULL)
+                return;
+
+        delay_ucode_info_p = (int *)__pa_nodebug(&delay_ucode_info);
+        current_mc_date_p = (int *)__pa_nodebug(&current_mc_date);
+
+        *delay_ucode_info_p = 1;
+        *current_mc_date_p = mc_intel->hdr.date;
+}
+#else
+
+/*
+ * Flush global tlb. We only do this in x86_64 where paging has been enabled
+ * already and PGE should be enabled as well.
+ */
+static inline void __cpuinit flush_tlb_early(void)
+{
+        __native_flush_tlb_global_irq_disabled();
+}
+
+static inline void __cpuinit print_ucode(struct ucode_cpu_info *uci)
+{
+        struct microcode_intel *mc_intel;
+
+        mc_intel = uci->mc;
+        if (mc_intel == NULL)
+                return;
+
+        print_ucode_info(uci, mc_intel->hdr.date);
+}
+#endif
+
+static int __cpuinit apply_microcode_early(struct mc_saved_data *mc_saved_data,
+                                           struct ucode_cpu_info *uci)
+{
+        struct microcode_intel *mc_intel;
+        unsigned int val[2];
+
+        mc_intel = uci->mc;
+        if (mc_intel == NULL)
+                return 0;
+
+        /* write microcode via MSR 0x79 */
+        native_wrmsr(MSR_IA32_UCODE_WRITE,
+              (unsigned long) mc_intel->bits,
+              (unsigned long) mc_intel->bits >> 16 >> 16);
+        native_wrmsr(MSR_IA32_UCODE_REV, 0, 0);
+
+        /* As documented in the SDM: Do a CPUID 1 here */
+        sync_core();
+
+        /* get the current revision from MSR 0x8B */
+        native_rdmsr(MSR_IA32_UCODE_REV, val[0], val[1]);
+        if (val[1] != mc_intel->hdr.rev)
+                return -1;
+
+#ifdef CONFIG_X86_64
+        /* Flush global tlb. This is precaution. */
+        flush_tlb_early();
+#endif
+        uci->cpu_sig.rev = val[1];
+
+        print_ucode(uci);
+
+        return 0;
+}
+
+/*
+ * This function converts microcode patch offsets previously stored in
+ * mc_saved_in_initrd to pointers and stores the pointers in mc_saved_data.
+ */
+int __init save_microcode_in_initrd(void)
+{
+        unsigned int count = mc_saved_data.mc_saved_count;
+        struct microcode_intel *mc_saved[MAX_UCODE_COUNT];
+        int ret = 0;
+
+        if (count == 0)
+                return ret;
+
+        microcode_pointer(mc_saved, mc_saved_in_initrd, initrd_start, count);
+        ret = save_microcode(&mc_saved_data, mc_saved, count);
+        if (ret)
+                pr_err("Can not save microcod patches from initrd");
+
+        show_saved_mc();
+
+        return ret;
+}
+
+static void __init
+_load_ucode_intel_bsp(struct mc_saved_data *mc_saved_data,
+                      unsigned long *mc_saved_in_initrd,
+                      unsigned long initrd_start_early,
+                      unsigned long initrd_end_early,
+                      struct ucode_cpu_info *uci)
+{
+        collect_cpu_info_early(uci);
+        scan_microcode(initrd_start_early, initrd_end_early, mc_saved_data,
+                       mc_saved_in_initrd, uci);
+        load_microcode(mc_saved_data, mc_saved_in_initrd,
+                       initrd_start_early, uci);
+        apply_microcode_early(mc_saved_data, uci);
+}
+
+void __init
+load_ucode_intel_bsp(void)
+{
+        u64 ramdisk_image, ramdisk_size;
+        unsigned long initrd_start_early, initrd_end_early;
+        struct ucode_cpu_info uci;
+#ifdef CONFIG_X86_32
+        struct boot_params *boot_params_p;
+
+        boot_params_p = (struct boot_params *)__pa_nodebug(&boot_params);
+        ramdisk_image = boot_params_p->hdr.ramdisk_image;
+        ramdisk_size  = boot_params_p->hdr.ramdisk_size;
+        initrd_start_early = ramdisk_image;
+        initrd_end_early = initrd_start_early + ramdisk_size;
+
+        _load_ucode_intel_bsp(
+                (struct mc_saved_data *)__pa_nodebug(&mc_saved_data),
+                (unsigned long *)__pa_nodebug(&mc_saved_in_initrd),
+                initrd_start_early, initrd_end_early, &uci);
+#else
+        ramdisk_image = boot_params.hdr.ramdisk_image;
+        ramdisk_size  = boot_params.hdr.ramdisk_size;
+        initrd_start_early = ramdisk_image + PAGE_OFFSET;
+        initrd_end_early = initrd_start_early + ramdisk_size;
+
+        _load_ucode_intel_bsp(&mc_saved_data, mc_saved_in_initrd,
+                              initrd_start_early, initrd_end_early, &uci);
+#endif
+}
+
+void __cpuinit load_ucode_intel_ap(void)
+{
+        struct mc_saved_data *mc_saved_data_p;
+        struct ucode_cpu_info uci;
+        unsigned long *mc_saved_in_initrd_p;
+        unsigned long initrd_start_addr;
+#ifdef CONFIG_X86_32
+        unsigned long *initrd_start_p;
+
+        mc_saved_in_initrd_p =
+                (unsigned long *)__pa_nodebug(mc_saved_in_initrd);
+        mc_saved_data_p = (struct mc_saved_data *)__pa_nodebug(&mc_saved_data);
+        initrd_start_p = (unsigned long *)__pa_nodebug(&initrd_start);
+        initrd_start_addr = (unsigned long)__pa_nodebug(*initrd_start_p);
+#else
+        mc_saved_data_p = &mc_saved_data;
+        mc_saved_in_initrd_p = mc_saved_in_initrd;
+        initrd_start_addr = initrd_start;
+#endif
+
+        /*
+         * If there is no valid ucode previously saved in memory, no need to
+         * update ucode on this AP.
+         */
+        if (mc_saved_data_p->mc_saved_count == 0)
+                return;
+
+        collect_cpu_info_early(&uci);
+        load_microcode(mc_saved_data_p, mc_saved_in_initrd_p,
+                       initrd_start_addr, &uci);
+        apply_microcode_early(mc_saved_data_p, &uci);
+}
diff --git a/arch/x86/kernel/microcode_intel_lib.c b/arch/x86/kernel/microcode_intel_lib.c
new file mode 100644
index 000000000000..ce69320d0179
--- /dev/null
+++ b/arch/x86/kernel/microcode_intel_lib.c
@@ -0,0 +1,174 @@
+/*
+ *      Intel CPU Microcode Update Driver for Linux
+ *
+ *      Copyright (C) 2012 Fenghua Yu <fenghua.yu@intel.com>
+ *                         H Peter Anvin" <hpa@zytor.com>
+ *
+ *      This driver allows to upgrade microcode on Intel processors
+ *      belonging to IA-32 family - PentiumPro, Pentium II,
+ *      Pentium III, Xeon, Pentium 4, etc.
+ *
+ *      Reference: Section 8.11 of Volume 3a, IA-32 Intel? Architecture
+ *      Software Developer's Manual
+ *      Order Number 253668 or free download from:
+ *
+ *      http://developer.intel.com/Assets/PDF/manual/253668.pdf
+ *
+ *      For more information, go to http://www.urbanmyth.org/microcode
+ *
+ *      This program is free software; you can redistribute it and/or
+ *      modify it under the terms of the GNU General Public License
+ *      as published by the Free Software Foundation; either version
+ *      2 of the License, or (at your option) any later version.
+ *
+ */
+#include <linux/firmware.h>
+#include <linux/uaccess.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+
+#include <asm/microcode_intel.h>
+#include <asm/processor.h>
+#include <asm/msr.h>
+
+static inline int
+update_match_cpu(unsigned int csig, unsigned int cpf,
+                 unsigned int sig, unsigned int pf)
+{
+        return (!sigmatch(sig, csig, pf, cpf)) ? 0 : 1;
+}
+
+int
+update_match_revision(struct microcode_header_intel *mc_header, int rev)
+{
+        return (mc_header->rev <= rev) ? 0 : 1;
+}
+
+int microcode_sanity_check(void *mc, int print_err)
+{
+        unsigned long total_size, data_size, ext_table_size;
+        struct microcode_header_intel *mc_header = mc;
+        struct extended_sigtable *ext_header = NULL;
+        int sum, orig_sum, ext_sigcount = 0, i;
+        struct extended_signature *ext_sig;
+
+        total_size = get_totalsize(mc_header);
+        data_size = get_datasize(mc_header);
+
+        if (data_size + MC_HEADER_SIZE > total_size) {
+                if (print_err)
+                        pr_err("error! Bad data size in microcode data file\n");
+                return -EINVAL;
+        }
+
+        if (mc_header->ldrver != 1 || mc_header->hdrver != 1) {
+                if (print_err)
+                        pr_err("error! Unknown microcode update format\n");
+                return -EINVAL;
+        }
+        ext_table_size = total_size - (MC_HEADER_SIZE + data_size);
+        if (ext_table_size) {
+                if ((ext_table_size < EXT_HEADER_SIZE)
+                 || ((ext_table_size - EXT_HEADER_SIZE) % EXT_SIGNATURE_SIZE)) {
+                        if (print_err)
+                                pr_err("error! Small exttable size in microcode data file\n");
+                        return -EINVAL;
+                }
+                ext_header = mc + MC_HEADER_SIZE + data_size;
+                if (ext_table_size != exttable_size(ext_header)) {
+                        if (print_err)
+                                pr_err("error! Bad exttable size in microcode data file\n");
+                        return -EFAULT;
+                }
+                ext_sigcount = ext_header->count;
+        }
+
+        /* check extended table checksum */
+        if (ext_table_size) {
+                int ext_table_sum = 0;
+                int *ext_tablep = (int *)ext_header;
+
+                i = ext_table_size / DWSIZE;
+                while (i--)
+                        ext_table_sum += ext_tablep[i];
+                if (ext_table_sum) {
+                        if (print_err)
+                                pr_warn("aborting, bad extended signature table checksum\n");
+                        return -EINVAL;
+                }
+        }
+
+        /* calculate the checksum */
+        orig_sum = 0;
+        i = (MC_HEADER_SIZE + data_size) / DWSIZE;
+        while (i--)
+                orig_sum += ((int *)mc)[i];
+        if (orig_sum) {
+                if (print_err)
+                        pr_err("aborting, bad checksum\n");
+                return -EINVAL;
+        }
+        if (!ext_table_size)
+                return 0;
+        /* check extended signature checksum */
+        for (i = 0; i < ext_sigcount; i++) {
+                ext_sig = (void *)ext_header + EXT_HEADER_SIZE +
+                          EXT_SIGNATURE_SIZE * i;
+                sum = orig_sum
+                        - (mc_header->sig + mc_header->pf + mc_header->cksum)
+                        + (ext_sig->sig + ext_sig->pf + ext_sig->cksum);
+                if (sum) {
+                        if (print_err)
+                                pr_err("aborting, bad checksum\n");
+                        return -EINVAL;
+                }
+        }
+        return 0;
+}
+EXPORT_SYMBOL_GPL(microcode_sanity_check);
+
+/*
+ * return 0 - no update found
+ * return 1 - found update
+ */
+int get_matching_sig(unsigned int csig, int cpf, void *mc, int rev)
+{
+        struct microcode_header_intel *mc_header = mc;
+        struct extended_sigtable *ext_header;
+        unsigned long total_size = get_totalsize(mc_header);
+        int ext_sigcount, i;
+        struct extended_signature *ext_sig;
+
+        if (update_match_cpu(csig, cpf, mc_header->sig, mc_header->pf))
+                return 1;
+
+        /* Look for ext. headers: */
+        if (total_size <= get_datasize(mc_header) + MC_HEADER_SIZE)
+                return 0;
+
+        ext_header = mc + get_datasize(mc_header) + MC_HEADER_SIZE;
+        ext_sigcount = ext_header->count;
+        ext_sig = (void *)ext_header + EXT_HEADER_SIZE;
+
+        for (i = 0; i < ext_sigcount; i++) {
+                if (update_match_cpu(csig, cpf, ext_sig->sig, ext_sig->pf))
+                        return 1;
+                ext_sig++;
+        }
+        return 0;
+}
+
+/*
+ * return 0 - no update found
+ * return 1 - found update
+ */
+int get_matching_microcode(unsigned int csig, int cpf, void *mc, int rev)
+{
+        struct microcode_header_intel *mc_header = mc;
+
+        if (!update_match_revision(mc_header, rev))
+                return 0;
+
+        return get_matching_sig(csig, cpf, mc, rev);
+}
+EXPORT_SYMBOL_GPL(get_matching_microcode);
diff --git a/arch/x86/kernel/msr.c b/arch/x86/kernel/msr.c
index 4929502c1372..ce130493b802 100644
--- a/arch/x86/kernel/msr.c
+++ b/arch/x86/kernel/msr.c
@@ -71,7 +71,7 @@ static ssize_t msr_read(struct file *file, char __user *buf,
         u32 __user *tmp = (u32 __user *) buf;
         u32 data[2];
         u32 reg = *ppos;
-        int cpu = iminor(file->f_path.dentry->d_inode);
+        int cpu = iminor(file_inode(file));
         int err = 0;
         ssize_t bytes = 0;
 
@@ -99,7 +99,7 @@ static ssize_t msr_write(struct file *file, const char __user *buf,
         const u32 __user *tmp = (const u32 __user *)buf;
         u32 data[2];
         u32 reg = *ppos;
-        int cpu = iminor(file->f_path.dentry->d_inode);
+        int cpu = iminor(file_inode(file));
         int err = 0;
         ssize_t bytes = 0;
 
@@ -125,7 +125,7 @@ static long msr_ioctl(struct file *file, unsigned int ioc, unsigned long arg)
 {
         u32 __user *uregs = (u32 __user *)arg;
         u32 regs[8];
-        int cpu = iminor(file->f_path.dentry->d_inode);
+        int cpu = iminor(file_inode(file));
         int err;
 
         switch (ioc) {
@@ -171,13 +171,12 @@ static long msr_ioctl(struct file *file, unsigned int ioc, unsigned long arg)
 
 static int msr_open(struct inode *inode, struct file *file)
 {
-        unsigned int cpu;
+        unsigned int cpu = iminor(file_inode(file));
         struct cpuinfo_x86 *c;
 
         if (!capable(CAP_SYS_RAWIO))
                 return -EPERM;
 
-        cpu = iminor(file->f_path.dentry->d_inode);
         if (cpu >= nr_cpu_ids || !cpu_online(cpu))
                 return -ENXIO;  /* No such CPU */
 
diff --git a/arch/x86/kernel/nmi.c b/arch/x86/kernel/nmi.c
index f84f5c57de35..60308053fdb2 100644
--- a/arch/x86/kernel/nmi.c
+++ b/arch/x86/kernel/nmi.c
@@ -509,3 +509,4 @@ void local_touch_nmi(void)
 {
         __this_cpu_write(last_nmi_rip, 0);
 }
+EXPORT_SYMBOL_GPL(local_touch_nmi);
diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c
index 2ed787f15bf0..14ae10031ff0 100644
--- a/arch/x86/kernel/process.c
+++ b/arch/x86/kernel/process.c
@@ -268,13 +268,7 @@ void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p,
 unsigned long boot_option_idle_override = IDLE_NO_OVERRIDE;
 EXPORT_SYMBOL(boot_option_idle_override);
 
-/*
- * Powermanagement idle function, if any..
- */
-void (*pm_idle)(void);
-#ifdef CONFIG_APM_MODULE
-EXPORT_SYMBOL(pm_idle);
-#endif
+static void (*x86_idle)(void);
 
 #ifndef CONFIG_SMP
 static inline void play_dead(void)
@@ -351,7 +345,7 @@ void cpu_idle(void)
                         rcu_idle_enter();
 
                         if (cpuidle_idle_call())
-                                pm_idle();
+                                x86_idle();
 
                         rcu_idle_exit();
                         start_critical_timings();
@@ -375,7 +369,6 @@ void cpu_idle(void)
  */
 void default_idle(void)
 {
-        trace_power_start_rcuidle(POWER_CSTATE, 1, smp_processor_id());
         trace_cpu_idle_rcuidle(1, smp_processor_id());
         current_thread_info()->status &= ~TS_POLLING;
         /*
@@ -389,21 +382,22 @@ void default_idle(void)
         else
                 local_irq_enable();
         current_thread_info()->status |= TS_POLLING;
-        trace_power_end_rcuidle(smp_processor_id());
         trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, smp_processor_id());
 }
 #ifdef CONFIG_APM_MODULE
 EXPORT_SYMBOL(default_idle);
 #endif
 
-bool set_pm_idle_to_default(void)
+#ifdef CONFIG_XEN
+bool xen_set_default_idle(void)
 {
-        bool ret = !!pm_idle;
+        bool ret = !!x86_idle;
 
-        pm_idle = default_idle;
+        x86_idle = default_idle;
 
         return ret;
 }
+#endif
 void stop_this_cpu(void *dummy)
 {
         local_irq_disable();
@@ -413,31 +407,8 @@ void stop_this_cpu(void *dummy)
         set_cpu_online(smp_processor_id(), false);
         disable_local_APIC();
 
-        for (;;) {
-                if (hlt_works(smp_processor_id()))
-                        halt();
-        }
-}
-
-/* Default MONITOR/MWAIT with no hints, used for default C1 state */
-static void mwait_idle(void)
-{
-        if (!need_resched()) {
-                trace_power_start_rcuidle(POWER_CSTATE, 1, smp_processor_id());
-                trace_cpu_idle_rcuidle(1, smp_processor_id());
-                if (this_cpu_has(X86_FEATURE_CLFLUSH_MONITOR))
-                        clflush((void *)&current_thread_info()->flags);
-
-                __monitor((void *)&current_thread_info()->flags, 0, 0);
-                smp_mb();
-                if (!need_resched())
-                        __sti_mwait(0, 0);
-                else
-                        local_irq_enable();
-                trace_power_end_rcuidle(smp_processor_id());
-                trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, smp_processor_id());
-        } else
-                local_irq_enable();
+        for (;;)
+                halt();
 }
 
 /*
@@ -447,62 +418,13 @@ static void mwait_idle(void)
  */
 static void poll_idle(void)
 {
-        trace_power_start_rcuidle(POWER_CSTATE, 0, smp_processor_id());
         trace_cpu_idle_rcuidle(0, smp_processor_id());
         local_irq_enable();
         while (!need_resched())
                 cpu_relax();
-        trace_power_end_rcuidle(smp_processor_id());
         trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, smp_processor_id());
 }
 
-/*
- * mwait selection logic:
- *
- * It depends on the CPU. For AMD CPUs that support MWAIT this is
- * wrong. Family 0x10 and 0x11 CPUs will enter C1 on HLT. Powersavings
- * then depend on a clock divisor and current Pstate of the core. If
- * all cores of a processor are in halt state (C1) the processor can
- * enter the C1E (C1 enhanced) state. If mwait is used this will never
- * happen.
- *
- * idle=mwait overrides this decision and forces the usage of mwait.
- */
-
-#define MWAIT_INFO                      0x05
-#define MWAIT_ECX_EXTENDED_INFO         0x01
-#define MWAIT_EDX_C1                    0xf0
-
-int mwait_usable(const struct cpuinfo_x86 *c)
-{
-        u32 eax, ebx, ecx, edx;
-
-        /* Use mwait if idle=mwait boot option is given */
-        if (boot_option_idle_override == IDLE_FORCE_MWAIT)
-                return 1;
-
-        /*
-         * Any idle= boot option other than idle=mwait means that we must not
-         * use mwait. Eg: idle=halt or idle=poll or idle=nomwait
-         */
-        if (boot_option_idle_override != IDLE_NO_OVERRIDE)
-                return 0;
-
-        if (c->cpuid_level < MWAIT_INFO)
-                return 0;
-
-        cpuid(MWAIT_INFO, &eax, &ebx, &ecx, &edx);
-        /* Check, whether EDX has extended info about MWAIT */
-        if (!(ecx & MWAIT_ECX_EXTENDED_INFO))
-                return 1;
-
-        /*
-         * edx enumeratios MONITOR/MWAIT extensions. Check, whether
-         * C1  supports MWAIT
-         */
-        return (edx & MWAIT_EDX_C1);
-}
-
 bool amd_e400_c1e_detected;
 EXPORT_SYMBOL(amd_e400_c1e_detected);
 
@@ -567,31 +489,24 @@ static void amd_e400_idle(void)
 void __cpuinit select_idle_routine(const struct cpuinfo_x86 *c)
 {
 #ifdef CONFIG_SMP
-        if (pm_idle == poll_idle && smp_num_siblings > 1) {
+        if (x86_idle == poll_idle && smp_num_siblings > 1)
                 pr_warn_once("WARNING: polling idle and HT enabled, performance may degrade\n");
-        }
 #endif
-        if (pm_idle)
+        if (x86_idle)
                 return;
 
-        if (cpu_has(c, X86_FEATURE_MWAIT) && mwait_usable(c)) {
-                /*
-                 * One CPU supports mwait => All CPUs supports mwait
-                 */
-                pr_info("using mwait in idle threads\n");
-                pm_idle = mwait_idle;
-        } else if (cpu_has_amd_erratum(amd_erratum_400)) {
+        if (cpu_has_amd_erratum(amd_erratum_400)) {
                 /* E400: APIC timer interrupt does not wake up CPU from C1e */
                 pr_info("using AMD E400 aware idle routine\n");
-                pm_idle = amd_e400_idle;
+                x86_idle = amd_e400_idle;
         } else
-                pm_idle = default_idle;
+                x86_idle = default_idle;
 }
 
 void __init init_amd_e400_c1e_mask(void)
 {
         /* If we're using amd_e400_idle, we need to allocate amd_e400_c1e_mask. */
-        if (pm_idle == amd_e400_idle)
+        if (x86_idle == amd_e400_idle)
                 zalloc_cpumask_var(&amd_e400_c1e_mask, GFP_KERNEL);
 }
 
@@ -602,11 +517,8 @@ static int __init idle_setup(char *str)
 
         if (!strcmp(str, "poll")) {
                 pr_info("using polling idle threads\n");
-                pm_idle = poll_idle;
+                x86_idle = poll_idle;
                 boot_option_idle_override = IDLE_POLL;
-        } else if (!strcmp(str, "mwait")) {
-                boot_option_idle_override = IDLE_FORCE_MWAIT;
-                WARN_ONCE(1, "\"idle=mwait\" will be removed in 2012\n");
         } else if (!strcmp(str, "halt")) {
                 /*
                  * When the boot option of idle=halt is added, halt is
@@ -615,7 +527,7 @@ static int __init idle_setup(char *str)
                  * To continue to load the CPU idle driver, don't touch
                  * the boot_option_idle_override.
                  */
-                pm_idle = default_idle;
+                x86_idle = default_idle;
                 boot_option_idle_override = IDLE_HALT;
         } else if (!strcmp(str, "nomwait")) {
                 /*
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index 6e68a6194965..0f49677da51e 100644
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -117,7 +117,7 @@ void release_thread(struct task_struct *dead_task)
 {
         if (dead_task->mm) {
                 if (dead_task->mm->context.size) {
-                        pr_warn("WARNING: dead process %8s still has LDT? <%p/%d>\n",
+                        pr_warn("WARNING: dead process %s still has LDT? <%p/%d>\n",
                                 dead_task->comm,
                                 dead_task->mm->context.ldt,
                                 dead_task->mm->context.size);
diff --git a/arch/x86/kernel/ptrace.c b/arch/x86/kernel/ptrace.c
index b629bbe0d9bd..29a8120e6fe8 100644
--- a/arch/x86/kernel/ptrace.c
+++ b/arch/x86/kernel/ptrace.c
@@ -22,7 +22,7 @@
 #include <linux/perf_event.h>
 #include <linux/hw_breakpoint.h>
 #include <linux/rcupdate.h>
-#include <linux/module.h>
+#include <linux/export.h>
 #include <linux/context_tracking.h>
 
 #include <asm/uaccess.h>
diff --git a/arch/x86/kernel/pvclock.c b/arch/x86/kernel/pvclock.c
index 85c39590c1a4..2cb9470ea85b 100644
--- a/arch/x86/kernel/pvclock.c
+++ b/arch/x86/kernel/pvclock.c
@@ -185,7 +185,7 @@ int __init pvclock_init_vsyscall(struct pvclock_vsyscall_time_info *i,
 
         for (idx = 0; idx <= (PVCLOCK_FIXMAP_END-PVCLOCK_FIXMAP_BEGIN); idx++) {
                 __set_fixmap(PVCLOCK_FIXMAP_BEGIN + idx,
-                             __pa_symbol(i) + (idx*PAGE_SIZE),
+                             __pa(i) + (idx*PAGE_SIZE),
                              PAGE_KERNEL_VVAR);
         }
 
diff --git a/arch/x86/kernel/rtc.c b/arch/x86/kernel/rtc.c
index 801602b5d745..2e8f3d3b5641 100644
--- a/arch/x86/kernel/rtc.c
+++ b/arch/x86/kernel/rtc.c
@@ -149,7 +149,6 @@ unsigned long mach_get_cmos_time(void)
         if (century) {
                 century = bcd2bin(century);
                 year += century * 100;
-                printk(KERN_INFO "Extended CMOS year: %d\n", century * 100);
         } else
                 year += CMOS_YEARS_OFFS;
 
diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c
index 1abb7969173a..90d8cc930f5e 100644
--- a/arch/x86/kernel/setup.c
+++ b/arch/x86/kernel/setup.c
@@ -108,17 +108,16 @@
 #include <asm/topology.h>
 #include <asm/apicdef.h>
 #include <asm/amd_nb.h>
-#ifdef CONFIG_X86_64
-#include <asm/numa_64.h>
-#endif
 #include <asm/mce.h>
 #include <asm/alternative.h>
 #include <asm/prom.h>
 
 /*
- * end_pfn only includes RAM, while max_pfn_mapped includes all e820 entries.
- * The direct mapping extends to max_pfn_mapped, so that we can directly access
- * apertures, ACPI and other tables without having to play with fixmaps.
+ * max_low_pfn_mapped: highest direct mapped pfn under 4GB
+ * max_pfn_mapped:     highest direct mapped pfn over 4GB
+ *
+ * The direct mapping only covers E820_RAM regions, so the ranges and gaps are
+ * represented by pfn_mapped
  */
 unsigned long max_low_pfn_mapped;
 unsigned long max_pfn_mapped;
@@ -172,9 +171,15 @@ static struct resource bss_resource = {
 
 #ifdef CONFIG_X86_32
 /* cpu data as detected by the assembly code in head.S */
-struct cpuinfo_x86 new_cpu_data __cpuinitdata = {0, 0, 0, 0, -1, 1, 0, 0, -1};
+struct cpuinfo_x86 new_cpu_data __cpuinitdata = {
+        .wp_works_ok = -1,
+        .fdiv_bug = -1,
+};
 /* common cpu data for all cpus */
-struct cpuinfo_x86 boot_cpu_data __read_mostly = {0, 0, 0, 0, -1, 1, 0, 0, -1};
+struct cpuinfo_x86 boot_cpu_data __read_mostly = {
+        .wp_works_ok = -1,
+        .fdiv_bug = -1,
+};
 EXPORT_SYMBOL(boot_cpu_data);
 
 unsigned int def_to_bigsmp;
@@ -276,18 +281,7 @@ void * __init extend_brk(size_t size, size_t align)
         return ret;
 }
 
-#ifdef CONFIG_X86_64
-static void __init init_gbpages(void)
-{
-        if (direct_gbpages && cpu_has_gbpages)
-                printk(KERN_INFO "Using GB pages for direct mapping\n");
-        else
-                direct_gbpages = 0;
-}
-#else
-static inline void init_gbpages(void)
-{
-}
+#ifdef CONFIG_X86_32
 static void __init cleanup_highmap(void)
 {
 }
@@ -296,8 +290,8 @@ static void __init cleanup_highmap(void)
 static void __init reserve_brk(void)
 {
         if (_brk_end > _brk_start)
-                memblock_reserve(__pa(_brk_start),
-                                 __pa(_brk_end) - __pa(_brk_start));
+                memblock_reserve(__pa_symbol(_brk_start),
+                                 _brk_end - _brk_start);
 
         /* Mark brk area as locked down and no longer taking any
            new allocations */
@@ -306,27 +300,43 @@ static void __init reserve_brk(void)
 
 #ifdef CONFIG_BLK_DEV_INITRD
 
+static u64 __init get_ramdisk_image(void)
+{
+        u64 ramdisk_image = boot_params.hdr.ramdisk_image;
+
+        ramdisk_image |= (u64)boot_params.ext_ramdisk_image << 32;
+
+        return ramdisk_image;
+}
+static u64 __init get_ramdisk_size(void)
+{
+        u64 ramdisk_size = boot_params.hdr.ramdisk_size;
+
+        ramdisk_size |= (u64)boot_params.ext_ramdisk_size << 32;
+
+        return ramdisk_size;
+}
+
 #define MAX_MAP_CHUNK   (NR_FIX_BTMAPS << PAGE_SHIFT)
 static void __init relocate_initrd(void)
 {
         /* Assume only end is not page aligned */
-        u64 ramdisk_image = boot_params.hdr.ramdisk_image;
-        u64 ramdisk_size  = boot_params.hdr.ramdisk_size;
+        u64 ramdisk_image = get_ramdisk_image();
+        u64 ramdisk_size  = get_ramdisk_size();
         u64 area_size     = PAGE_ALIGN(ramdisk_size);
-        u64 end_of_lowmem = max_low_pfn_mapped << PAGE_SHIFT;
         u64 ramdisk_here;
         unsigned long slop, clen, mapaddr;
         char *p, *q;
 
-        /* We need to move the initrd down into lowmem */
-        ramdisk_here = memblock_find_in_range(0, end_of_lowmem, area_size,
-                                         PAGE_SIZE);
+        /* We need to move the initrd down into directly mapped mem */
+        ramdisk_here = memblock_find_in_range(0, PFN_PHYS(max_pfn_mapped),
+                                                 area_size, PAGE_SIZE);
 
         if (!ramdisk_here)
                 panic("Cannot find place for new RAMDISK of size %lld\n",
                          ramdisk_size);
 
-        /* Note: this includes all the lowmem currently occupied by
+        /* Note: this includes all the mem currently occupied by
            the initrd, we rely on that fact to keep the data intact. */
         memblock_reserve(ramdisk_here, area_size);
         initrd_start = ramdisk_here + PAGE_OFFSET;
@@ -336,17 +346,7 @@ static void __init relocate_initrd(void)
 
         q = (char *)initrd_start;
 
-        /* Copy any lowmem portion of the initrd */
-        if (ramdisk_image < end_of_lowmem) {
-                clen = end_of_lowmem - ramdisk_image;
-                p = (char *)__va(ramdisk_image);
-                memcpy(q, p, clen);
-                q += clen;
-                ramdisk_image += clen;
-                ramdisk_size  -= clen;
-        }
-
-        /* Copy the highmem portion of the initrd */
+        /* Copy the initrd */
         while (ramdisk_size) {
                 slop = ramdisk_image & ~PAGE_MASK;
                 clen = ramdisk_size;
@@ -360,22 +360,35 @@ static void __init relocate_initrd(void)
                 ramdisk_image += clen;
                 ramdisk_size  -= clen;
         }
-        /* high pages is not converted by early_res_to_bootmem */
-        ramdisk_image = boot_params.hdr.ramdisk_image;
-        ramdisk_size  = boot_params.hdr.ramdisk_size;
+
+        ramdisk_image = get_ramdisk_image();
+        ramdisk_size  = get_ramdisk_size();
         printk(KERN_INFO "Move RAMDISK from [mem %#010llx-%#010llx] to"
                 " [mem %#010llx-%#010llx]\n",
                 ramdisk_image, ramdisk_image + ramdisk_size - 1,
                 ramdisk_here, ramdisk_here + ramdisk_size - 1);
 }
 
+static void __init early_reserve_initrd(void)
+{
+        /* Assume only end is not page aligned */
+        u64 ramdisk_image = get_ramdisk_image();
+        u64 ramdisk_size  = get_ramdisk_size();
+        u64 ramdisk_end   = PAGE_ALIGN(ramdisk_image + ramdisk_size);
+
+        if (!boot_params.hdr.type_of_loader ||
+            !ramdisk_image || !ramdisk_size)
+                return;         /* No initrd provided by bootloader */
+
+        memblock_reserve(ramdisk_image, ramdisk_end - ramdisk_image);
+}
 static void __init reserve_initrd(void)
 {
         /* Assume only end is not page aligned */
-        u64 ramdisk_image = boot_params.hdr.ramdisk_image;
-        u64 ramdisk_size  = boot_params.hdr.ramdisk_size;
+        u64 ramdisk_image = get_ramdisk_image();
+        u64 ramdisk_size  = get_ramdisk_size();
         u64 ramdisk_end   = PAGE_ALIGN(ramdisk_image + ramdisk_size);
-        u64 end_of_lowmem = max_low_pfn_mapped << PAGE_SHIFT;
+        u64 mapped_size;
 
         if (!boot_params.hdr.type_of_loader ||
             !ramdisk_image || !ramdisk_size)
@@ -383,22 +396,18 @@ static void __init reserve_initrd(void)
 
         initrd_start = 0;
 
-        if (ramdisk_size >= (end_of_lowmem>>1)) {
+        mapped_size = memblock_mem_size(max_pfn_mapped);
+        if (ramdisk_size >= (mapped_size>>1))
                 panic("initrd too large to handle, "
                        "disabling initrd (%lld needed, %lld available)\n",
-                       ramdisk_size, end_of_lowmem>>1);
-        }
+                       ramdisk_size, mapped_size>>1);
 
         printk(KERN_INFO "RAMDISK: [mem %#010llx-%#010llx]\n", ramdisk_image,
                         ramdisk_end - 1);
 
-
-        if (ramdisk_end <= end_of_lowmem) {
-                /* All in lowmem, easy case */
-                /*
-                 * don't need to reserve again, already reserved early
-                 * in i386_start_kernel
-                 */
+        if (pfn_range_is_mapped(PFN_DOWN(ramdisk_image),
+                                PFN_DOWN(ramdisk_end))) {
+                /* All are mapped, easy case */
                 initrd_start = ramdisk_image + PAGE_OFFSET;
                 initrd_end = initrd_start + ramdisk_size;
                 return;
@@ -409,6 +418,9 @@ static void __init reserve_initrd(void)
         memblock_free(ramdisk_image, ramdisk_end - ramdisk_image);
 }
 #else
+static void __init early_reserve_initrd(void)
+{
+}
 static void __init reserve_initrd(void)
 {
 }
@@ -419,8 +431,6 @@ static void __init parse_setup_data(void)
         struct setup_data *data;
         u64 pa_data;
 
-        if (boot_params.hdr.version < 0x0209)
-                return;
         pa_data = boot_params.hdr.setup_data;
         while (pa_data) {
                 u32 data_len, map_len;
@@ -456,8 +466,6 @@ static void __init e820_reserve_setup_data(void)
         u64 pa_data;
         int found = 0;
 
-        if (boot_params.hdr.version < 0x0209)
-                return;
         pa_data = boot_params.hdr.setup_data;
         while (pa_data) {
                 data = early_memremap(pa_data, sizeof(*data));
@@ -481,8 +489,6 @@ static void __init memblock_x86_reserve_range_setup_data(void)
         struct setup_data *data;
         u64 pa_data;
 
-        if (boot_params.hdr.version < 0x0209)
-                return;
         pa_data = boot_params.hdr.setup_data;
         while (pa_data) {
                 data = early_memremap(pa_data, sizeof(*data));
@@ -501,17 +507,51 @@ static void __init memblock_x86_reserve_range_setup_data(void)
 /*
  * Keep the crash kernel below this limit.  On 32 bits earlier kernels
  * would limit the kernel to the low 512 MiB due to mapping restrictions.
- * On 64 bits, kexec-tools currently limits us to 896 MiB; increase this
- * limit once kexec-tools are fixed.
  */
 #ifdef CONFIG_X86_32
 # define CRASH_KERNEL_ADDR_MAX  (512 << 20)
 #else
-# define CRASH_KERNEL_ADDR_MAX  (896 << 20)
+# define CRASH_KERNEL_ADDR_MAX  MAXMEM
 #endif
 
+static void __init reserve_crashkernel_low(void)
+{
+#ifdef CONFIG_X86_64
+        const unsigned long long alignment = 16<<20;    /* 16M */
+        unsigned long long low_base = 0, low_size = 0;
+        unsigned long total_low_mem;
+        unsigned long long base;
+        int ret;
+
+        total_low_mem = memblock_mem_size(1UL<<(32-PAGE_SHIFT));
+        ret = parse_crashkernel_low(boot_command_line, total_low_mem,
+                                                &low_size, &base);
+        if (ret != 0 || low_size <= 0)
+                return;
+
+        low_base = memblock_find_in_range(low_size, (1ULL<<32),
+                                        low_size, alignment);
+
+        if (!low_base) {
+                pr_info("crashkernel low reservation failed - No suitable area found.\n");
+
+                return;
+        }
+
+        memblock_reserve(low_base, low_size);
+        pr_info("Reserving %ldMB of low memory at %ldMB for crashkernel (System low RAM: %ldMB)\n",
+                        (unsigned long)(low_size >> 20),
+                        (unsigned long)(low_base >> 20),
+                        (unsigned long)(total_low_mem >> 20));
+        crashk_low_res.start = low_base;
+        crashk_low_res.end   = low_base + low_size - 1;
+        insert_resource(&iomem_resource, &crashk_low_res);
+#endif
+}
+
 static void __init reserve_crashkernel(void)
 {
+        const unsigned long long alignment = 16<<20;    /* 16M */
         unsigned long long total_mem;
         unsigned long long crash_size, crash_base;
         int ret;
@@ -525,8 +565,6 @@ static void __init reserve_crashkernel(void)
 
         /* 0 means: find the address automatically */
         if (crash_base <= 0) {
-                const unsigned long long alignment = 16<<20;    /* 16M */
-
                 /*
                  *  kexec want bzImage is below CRASH_KERNEL_ADDR_MAX
                  */
@@ -537,6 +575,7 @@ static void __init reserve_crashkernel(void)
                         pr_info("crashkernel reservation failed - No suitable area found.\n");
                         return;
                 }
+
         } else {
                 unsigned long long start;
 
@@ -558,6 +597,9 @@ static void __init reserve_crashkernel(void)
         crashk_res.start = crash_base;
         crashk_res.end   = crash_base + crash_size - 1;
         insert_resource(&iomem_resource, &crashk_res);
+
+        if (crash_base >= (1ULL<<32))
+                reserve_crashkernel_low();
 }
 #else
 static void __init reserve_crashkernel(void)
@@ -608,8 +650,6 @@ static __init void reserve_ibft_region(void)
                 memblock_reserve(addr, size);
 }
 
-static unsigned reserve_low = CONFIG_X86_RESERVE_LOW << 10;
-
 static bool __init snb_gfx_workaround_needed(void)
 {
 #ifdef CONFIG_PCI
@@ -698,8 +738,7 @@ static void __init trim_bios_range(void)
          * since some BIOSes are known to corrupt low memory.  See the
          * Kconfig help text for X86_RESERVE_LOW.
          */
-        e820_update_range(0, ALIGN(reserve_low, PAGE_SIZE),
-                          E820_RAM, E820_RESERVED);
+        e820_update_range(0, PAGE_SIZE, E820_RAM, E820_RESERVED);
 
         /*
          * special case: Some BIOSen report the PC BIOS
@@ -711,6 +750,29 @@ static void __init trim_bios_range(void)
         sanitize_e820_map(e820.map, ARRAY_SIZE(e820.map), &e820.nr_map);
 }
 
+/* called before trim_bios_range() to spare extra sanitize */
+static void __init e820_add_kernel_range(void)
+{
+        u64 start = __pa_symbol(_text);
+        u64 size = __pa_symbol(_end) - start;
+
+        /*
+         * Complain if .text .data and .bss are not marked as E820_RAM and
+         * attempt to fix it by adding the range. We may have a confused BIOS,
+         * or the user may have used memmap=exactmap or memmap=xxM$yyM to
+         * exclude kernel range. If we really are running on top non-RAM,
+         * we will crash later anyways.
+         */
+        if (e820_all_mapped(start, start + size, E820_RAM))
+                return;
+
+        pr_warn(".text .data .bss are not marked as E820_RAM!\n");
+        e820_remove_range(start, size, E820_RAM, 0);
+        e820_add_region(start, size, E820_RAM);
+}
+
+static unsigned reserve_low = CONFIG_X86_RESERVE_LOW << 10;
+
 static int __init parse_reservelow(char *p)
 {
         unsigned long long size;
@@ -733,6 +795,11 @@ static int __init parse_reservelow(char *p)
 
 early_param("reservelow", parse_reservelow);
 
+static void __init trim_low_memory_range(void)
+{
+        memblock_reserve(0, ALIGN(reserve_low, PAGE_SIZE));
+}
+        
 /*
  * Determine if we were loaded by an EFI loader.  If so, then we have also been
  * passed the efi memmap, systab, etc., so we should use these data structures
@@ -748,6 +815,17 @@ early_param("reservelow", parse_reservelow);
 
 void __init setup_arch(char **cmdline_p)
 {
+        memblock_reserve(__pa_symbol(_text),
+                         (unsigned long)__bss_stop - (unsigned long)_text);
+
+        early_reserve_initrd();
+
+        /*
+         * At this point everything still needed from the boot loader
+         * or BIOS or kernel text should be early reserved or marked not
+         * RAM in e820. All other memory is free game.
+         */
+
 #ifdef CONFIG_X86_32
         memcpy(&boot_cpu_data, &new_cpu_data, sizeof(new_cpu_data));
         visws_early_detect();
@@ -835,12 +913,12 @@ void __init setup_arch(char **cmdline_p)
         init_mm.end_data = (unsigned long) _edata;
         init_mm.brk = _brk_end;
 
-        code_resource.start = virt_to_phys(_text);
-        code_resource.end = virt_to_phys(_etext)-1;
-        data_resource.start = virt_to_phys(_etext);
-        data_resource.end = virt_to_phys(_edata)-1;
-        bss_resource.start = virt_to_phys(&__bss_start);
-        bss_resource.end = virt_to_phys(&__bss_stop)-1;
+        code_resource.start = __pa_symbol(_text);
+        code_resource.end = __pa_symbol(_etext)-1;
+        data_resource.start = __pa_symbol(_etext);
+        data_resource.end = __pa_symbol(_edata)-1;
+        bss_resource.start = __pa_symbol(__bss_start);
+        bss_resource.end = __pa_symbol(__bss_stop)-1;
 
 #ifdef CONFIG_CMDLINE_BOOL
 #ifdef CONFIG_CMDLINE_OVERRIDE
@@ -906,6 +984,7 @@ void __init setup_arch(char **cmdline_p)
         insert_resource(&iomem_resource, &data_resource);
         insert_resource(&iomem_resource, &bss_resource);
 
+        e820_add_kernel_range();
         trim_bios_range();
 #ifdef CONFIG_X86_32
         if (ppro_with_ram_bug()) {
@@ -955,6 +1034,8 @@ void __init setup_arch(char **cmdline_p)
 
         reserve_ibft_region();
 
+        early_alloc_pgt_buf();
+
         /*
          * Need to conclude brk, before memblock_x86_fill()
          *  it could use memblock_find_in_range, could overlap with
@@ -964,7 +1045,7 @@ void __init setup_arch(char **cmdline_p)
 
         cleanup_highmap();
 
-        memblock.current_limit = get_max_mapped();
+        memblock.current_limit = ISA_END_ADDRESS;
         memblock_x86_fill();
 
         /*
@@ -981,41 +1062,22 @@ void __init setup_arch(char **cmdline_p)
         setup_bios_corruption_check();
 #endif
 
+#ifdef CONFIG_X86_32
         printk(KERN_DEBUG "initial memory mapped: [mem 0x00000000-%#010lx]\n",
                         (max_pfn_mapped<<PAGE_SHIFT) - 1);
+#endif
 
-        setup_real_mode();
+        reserve_real_mode();
 
         trim_platform_memory_ranges();
+        trim_low_memory_range();
 
-        init_gbpages();
-
-        /* max_pfn_mapped is updated here */
-        max_low_pfn_mapped = init_memory_mapping(0, max_low_pfn<<PAGE_SHIFT);
-        max_pfn_mapped = max_low_pfn_mapped;
-
-#ifdef CONFIG_X86_64
-        if (max_pfn > max_low_pfn) {
-                int i;
-                unsigned long start, end;
-                unsigned long start_pfn, end_pfn;
-
-                for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn,
-                                                         NULL) {
+        init_mem_mapping();
 
-                        end = PFN_PHYS(end_pfn);
-                        if (end <= (1UL<<32))
-                                continue;
+        early_trap_pf_init();
 
-                        start = PFN_PHYS(start_pfn);
-                        max_pfn_mapped = init_memory_mapping(
-                                                max((1UL<<32), start), end);
-                }
+        setup_real_mode();
 
-                /* can we preseve max_low_pfn ?*/
-                max_low_pfn = max_pfn;
-        }
-#endif
         memblock.current_limit = get_max_mapped();
         dma_contiguous_reserve(0);
 
diff --git a/arch/x86/kernel/signal.c b/arch/x86/kernel/signal.c
index d6bf1f34a6e9..69562992e457 100644
--- a/arch/x86/kernel/signal.c
+++ b/arch/x86/kernel/signal.c
@@ -278,7 +278,7 @@ static const struct {
 };
 
 static int
-__setup_frame(int sig, struct k_sigaction *ka, sigset_t *set,
+__setup_frame(int sig, struct ksignal *ksig, sigset_t *set,
               struct pt_regs *regs)
 {
         struct sigframe __user *frame;
@@ -286,7 +286,7 @@ __setup_frame(int sig, struct k_sigaction *ka, sigset_t *set,
         int err = 0;
         void __user *fpstate = NULL;
 
-        frame = get_sigframe(ka, regs, sizeof(*frame), &fpstate);
+        frame = get_sigframe(&ksig->ka, regs, sizeof(*frame), &fpstate);
 
         if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame)))
                 return -EFAULT;
@@ -307,8 +307,8 @@ __setup_frame(int sig, struct k_sigaction *ka, sigset_t *set,
                 restorer = VDSO32_SYMBOL(current->mm->context.vdso, sigreturn);
         else
                 restorer = &frame->retcode;
-        if (ka->sa.sa_flags & SA_RESTORER)
-                restorer = ka->sa.sa_restorer;
+        if (ksig->ka.sa.sa_flags & SA_RESTORER)
+                restorer = ksig->ka.sa.sa_restorer;
 
         /* Set up to return from userspace.  */
         err |= __put_user(restorer, &frame->pretcode);
@@ -327,7 +327,7 @@ __setup_frame(int sig, struct k_sigaction *ka, sigset_t *set,
 
         /* Set up registers for signal handler */
         regs->sp = (unsigned long)frame;
-        regs->ip = (unsigned long)ka->sa.sa_handler;
+        regs->ip = (unsigned long)ksig->ka.sa.sa_handler;
         regs->ax = (unsigned long)sig;
         regs->dx = 0;
         regs->cx = 0;
@@ -340,7 +340,7 @@ __setup_frame(int sig, struct k_sigaction *ka, sigset_t *set,
         return 0;
 }
 
-static int __setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
+static int __setup_rt_frame(int sig, struct ksignal *ksig,
                             sigset_t *set, struct pt_regs *regs)
 {
         struct rt_sigframe __user *frame;
@@ -348,7 +348,7 @@ static int __setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
         int err = 0;
         void __user *fpstate = NULL;
 
-        frame = get_sigframe(ka, regs, sizeof(*frame), &fpstate);
+        frame = get_sigframe(&ksig->ka, regs, sizeof(*frame), &fpstate);
 
         if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame)))
                 return -EFAULT;
@@ -368,8 +368,8 @@ static int __setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
 
                 /* Set up to return from userspace.  */
                 restorer = VDSO32_SYMBOL(current->mm->context.vdso, rt_sigreturn);
-                if (ka->sa.sa_flags & SA_RESTORER)
-                        restorer = ka->sa.sa_restorer;
+                if (ksig->ka.sa.sa_flags & SA_RESTORER)
+                        restorer = ksig->ka.sa.sa_restorer;
                 put_user_ex(restorer, &frame->pretcode);
 
                 /*
@@ -382,7 +382,7 @@ static int __setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
                 put_user_ex(*((u64 *)&rt_retcode), (u64 *)frame->retcode);
         } put_user_catch(err);
         
-        err |= copy_siginfo_to_user(&frame->info, info);
+        err |= copy_siginfo_to_user(&frame->info, &ksig->info);
         err |= setup_sigcontext(&frame->uc.uc_mcontext, fpstate,
                                 regs, set->sig[0]);
         err |= __copy_to_user(&frame->uc.uc_sigmask, set, sizeof(*set));
@@ -392,7 +392,7 @@ static int __setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
 
         /* Set up registers for signal handler */
         regs->sp = (unsigned long)frame;
-        regs->ip = (unsigned long)ka->sa.sa_handler;
+        regs->ip = (unsigned long)ksig->ka.sa.sa_handler;
         regs->ax = (unsigned long)sig;
         regs->dx = (unsigned long)&frame->info;
         regs->cx = (unsigned long)&frame->uc;
@@ -405,20 +405,20 @@ static int __setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
         return 0;
 }
 #else /* !CONFIG_X86_32 */
-static int __setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
+static int __setup_rt_frame(int sig, struct ksignal *ksig,
                             sigset_t *set, struct pt_regs *regs)
 {
         struct rt_sigframe __user *frame;
         void __user *fp = NULL;
         int err = 0;
 
-        frame = get_sigframe(ka, regs, sizeof(struct rt_sigframe), &fp);
+        frame = get_sigframe(&ksig->ka, regs, sizeof(struct rt_sigframe), &fp);
 
         if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame)))
                 return -EFAULT;
 
-        if (ka->sa.sa_flags & SA_SIGINFO) {
-                if (copy_siginfo_to_user(&frame->info, info))
+        if (ksig->ka.sa.sa_flags & SA_SIGINFO) {
+                if (copy_siginfo_to_user(&frame->info, &ksig->info))
                         return -EFAULT;
         }
 
@@ -434,8 +434,8 @@ static int __setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
                 /* Set up to return from userspace.  If provided, use a stub
                    already in userspace.  */
                 /* x86-64 should always use SA_RESTORER. */
-                if (ka->sa.sa_flags & SA_RESTORER) {
-                        put_user_ex(ka->sa.sa_restorer, &frame->pretcode);
+                if (ksig->ka.sa.sa_flags & SA_RESTORER) {
+                        put_user_ex(ksig->ka.sa.sa_restorer, &frame->pretcode);
                 } else {
                         /* could use a vstub here */
                         err |= -EFAULT;
@@ -457,7 +457,7 @@ static int __setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
            next argument after the signal number on the stack. */
         regs->si = (unsigned long)&frame->info;
         regs->dx = (unsigned long)&frame->uc;
-        regs->ip = (unsigned long) ka->sa.sa_handler;
+        regs->ip = (unsigned long) ksig->ka.sa.sa_handler;
 
         regs->sp = (unsigned long)frame;
 
@@ -469,8 +469,8 @@ static int __setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
 }
 #endif /* CONFIG_X86_32 */
 
-static int x32_setup_rt_frame(int sig, struct k_sigaction *ka,
-                              siginfo_t *info, compat_sigset_t *set,
+static int x32_setup_rt_frame(struct ksignal *ksig,
+                              compat_sigset_t *set,
                               struct pt_regs *regs)
 {
 #ifdef CONFIG_X86_X32_ABI
@@ -479,13 +479,13 @@ static int x32_setup_rt_frame(int sig, struct k_sigaction *ka,
         int err = 0;
         void __user *fpstate = NULL;
 
-        frame = get_sigframe(ka, regs, sizeof(*frame), &fpstate);
+        frame = get_sigframe(&ksig->ka, regs, sizeof(*frame), &fpstate);
 
         if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame)))
                 return -EFAULT;
 
-        if (ka->sa.sa_flags & SA_SIGINFO) {
-                if (copy_siginfo_to_user32(&frame->info, info))
+        if (ksig->ka.sa.sa_flags & SA_SIGINFO) {
+                if (copy_siginfo_to_user32(&frame->info, &ksig->info))
                         return -EFAULT;
         }
 
@@ -499,8 +499,8 @@ static int x32_setup_rt_frame(int sig, struct k_sigaction *ka,
                 err |= __compat_save_altstack(&frame->uc.uc_stack, regs->sp);
                 put_user_ex(0, &frame->uc.uc__pad0);
 
-                if (ka->sa.sa_flags & SA_RESTORER) {
-                        restorer = ka->sa.sa_restorer;
+                if (ksig->ka.sa.sa_flags & SA_RESTORER) {
+                        restorer = ksig->ka.sa.sa_restorer;
                 } else {
                         /* could use a vstub here */
                         restorer = NULL;
@@ -518,10 +518,10 @@ static int x32_setup_rt_frame(int sig, struct k_sigaction *ka,
 
         /* Set up registers for signal handler */
         regs->sp = (unsigned long) frame;
-        regs->ip = (unsigned long) ka->sa.sa_handler;
+        regs->ip = (unsigned long) ksig->ka.sa.sa_handler;
 
         /* We use the x32 calling convention here... */
-        regs->di = sig;
+        regs->di = ksig->sig;
         regs->si = (unsigned long) &frame->info;
         regs->dx = (unsigned long) &frame->uc;
 
@@ -535,70 +535,13 @@ static int x32_setup_rt_frame(int sig, struct k_sigaction *ka,
         return 0;
 }
 
-#ifdef CONFIG_X86_32
-/*
- * Atomically swap in the new signal mask, and wait for a signal.
- */
-asmlinkage int
-sys_sigsuspend(int history0, int history1, old_sigset_t mask)
-{
-        sigset_t blocked;
-        siginitset(&blocked, mask);
-        return sigsuspend(&blocked);
-}
-
-asmlinkage int
-sys_sigaction(int sig, const struct old_sigaction __user *act,
-              struct old_sigaction __user *oact)
-{
-        struct k_sigaction new_ka, old_ka;
-        int ret = 0;
-
-        if (act) {
-                old_sigset_t mask;
-
-                if (!access_ok(VERIFY_READ, act, sizeof(*act)))
-                        return -EFAULT;
-
-                get_user_try {
-                        get_user_ex(new_ka.sa.sa_handler, &act->sa_handler);
-                        get_user_ex(new_ka.sa.sa_flags, &act->sa_flags);
-                        get_user_ex(mask, &act->sa_mask);
-                        get_user_ex(new_ka.sa.sa_restorer, &act->sa_restorer);
-                } get_user_catch(ret);
-
-                if (ret)
-                        return -EFAULT;
-                siginitset(&new_ka.sa.sa_mask, mask);
-        }
-
-        ret = do_sigaction(sig, act ? &new_ka : NULL, oact ? &old_ka : NULL);
-
-        if (!ret && oact) {
-                if (!access_ok(VERIFY_WRITE, oact, sizeof(*oact)))
-                        return -EFAULT;
-
-                put_user_try {
-                        put_user_ex(old_ka.sa.sa_handler, &oact->sa_handler);
-                        put_user_ex(old_ka.sa.sa_flags, &oact->sa_flags);
-                        put_user_ex(old_ka.sa.sa_mask.sig[0], &oact->sa_mask);
-                        put_user_ex(old_ka.sa.sa_restorer, &oact->sa_restorer);
-                } put_user_catch(ret);
-
-                if (ret)
-                        return -EFAULT;
-        }
-
-        return ret;
-}
-#endif /* CONFIG_X86_32 */
-
 /*
  * Do a signal return; undo the signal stack.
  */
 #ifdef CONFIG_X86_32
-unsigned long sys_sigreturn(struct pt_regs *regs)
+unsigned long sys_sigreturn(void)
 {
+        struct pt_regs *regs = current_pt_regs();
         struct sigframe __user *frame;
         unsigned long ax;
         sigset_t set;
@@ -625,8 +568,9 @@ badframe:
 }
 #endif /* CONFIG_X86_32 */
 
-long sys_rt_sigreturn(struct pt_regs *regs)
+long sys_rt_sigreturn(void)
 {
+        struct pt_regs *regs = current_pt_regs();
         struct rt_sigframe __user *frame;
         unsigned long ax;
         sigset_t set;
@@ -667,30 +611,29 @@ static int signr_convert(int sig)
 }
 
 static int
-setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
-                struct pt_regs *regs)
+setup_rt_frame(struct ksignal *ksig, struct pt_regs *regs)
 {
-        int usig = signr_convert(sig);
+        int usig = signr_convert(ksig->sig);
         sigset_t *set = sigmask_to_save();
         compat_sigset_t *cset = (compat_sigset_t *) set;
 
         /* Set up the stack frame */
         if (is_ia32_frame()) {
-                if (ka->sa.sa_flags & SA_SIGINFO)
-                        return ia32_setup_rt_frame(usig, ka, info, cset, regs);
+                if (ksig->ka.sa.sa_flags & SA_SIGINFO)
+                        return ia32_setup_rt_frame(usig, ksig, cset, regs);
                 else
-                        return ia32_setup_frame(usig, ka, cset, regs);
+                        return ia32_setup_frame(usig, ksig, cset, regs);
         } else if (is_x32_frame()) {
-                return x32_setup_rt_frame(usig, ka, info, cset, regs);
+                return x32_setup_rt_frame(ksig, cset, regs);
         } else {
-                return __setup_rt_frame(sig, ka, info, set, regs);
+                return __setup_rt_frame(ksig->sig, ksig, set, regs);
         }
 }
 
 static void
-handle_signal(unsigned long sig, siginfo_t *info, struct k_sigaction *ka,
-                struct pt_regs *regs)
+handle_signal(struct ksignal *ksig, struct pt_regs *regs)
 {
+        bool failed;
         /* Are we from a system call? */
         if (syscall_get_nr(current, regs) >= 0) {
                 /* If so, check system call restarting.. */
@@ -701,7 +644,7 @@ handle_signal(unsigned long sig, siginfo_t *info, struct k_sigaction *ka,
                         break;
 
                 case -ERESTARTSYS:
-                        if (!(ka->sa.sa_flags & SA_RESTART)) {
+                        if (!(ksig->ka.sa.sa_flags & SA_RESTART)) {
                                 regs->ax = -EINTR;
                                 break;
                         }
@@ -721,26 +664,21 @@ handle_signal(unsigned long sig, siginfo_t *info, struct k_sigaction *ka,
             likely(test_and_clear_thread_flag(TIF_FORCED_TF)))
                 regs->flags &= ~X86_EFLAGS_TF;
 
-        if (setup_rt_frame(sig, ka, info, regs) < 0) {
-                force_sigsegv(sig, current);
-                return;
+        failed = (setup_rt_frame(ksig, regs) < 0);
+        if (!failed) {
+                /*
+                 * Clear the direction flag as per the ABI for function entry.
+                 */
+                regs->flags &= ~X86_EFLAGS_DF;
+                /*
+                 * Clear TF when entering the signal handler, but
+                 * notify any tracer that was single-stepping it.
+                 * The tracer may want to single-step inside the
+                 * handler too.
+                 */
+                regs->flags &= ~X86_EFLAGS_TF;
         }
-
-        /*
-         * Clear the direction flag as per the ABI for function entry.
-         */
-        regs->flags &= ~X86_EFLAGS_DF;
-
-        /*
-         * Clear TF when entering the signal handler, but
-         * notify any tracer that was single-stepping it.
-         * The tracer may want to single-step inside the
-         * handler too.
-         */
-        regs->flags &= ~X86_EFLAGS_TF;
-
-        signal_delivered(sig, info, ka, regs,
-                         test_thread_flag(TIF_SINGLESTEP));
+        signal_setup_done(failed, ksig, test_thread_flag(TIF_SINGLESTEP));
 }
 
 #ifdef CONFIG_X86_32
@@ -757,14 +695,11 @@ handle_signal(unsigned long sig, siginfo_t *info, struct k_sigaction *ka,
  */
 static void do_signal(struct pt_regs *regs)
 {
-        struct k_sigaction ka;
-        siginfo_t info;
-        int signr;
+        struct ksignal ksig;
 
-        signr = get_signal_to_deliver(&info, &ka, regs, NULL);
-        if (signr > 0) {
+        if (get_signal(&ksig)) {
                 /* Whee! Actually deliver the signal.  */
-                handle_signal(signr, &info, &ka, regs);
+                handle_signal(&ksig, regs);
                 return;
         }
 
@@ -843,8 +778,9 @@ void signal_fault(struct pt_regs *regs, void __user *frame, char *where)
 }
 
 #ifdef CONFIG_X86_X32_ABI
-asmlinkage long sys32_x32_rt_sigreturn(struct pt_regs *regs)
+asmlinkage long sys32_x32_rt_sigreturn(void)
 {
+        struct pt_regs *regs = current_pt_regs();
         struct rt_sigframe_x32 __user *frame;
         sigset_t set;
         unsigned long ax;
diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index ed0fe385289d..9f190a2a00e9 100644
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -1365,11 +1365,10 @@ static inline void mwait_play_dead(void)
         unsigned int eax, ebx, ecx, edx;
         unsigned int highest_cstate = 0;
         unsigned int highest_subcstate = 0;
-        int i;
         void *mwait_ptr;
-        struct cpuinfo_x86 *c = __this_cpu_ptr(&cpu_info);
+        int i;
 
-        if (!(this_cpu_has(X86_FEATURE_MWAIT) && mwait_usable(c)))
+        if (!this_cpu_has(X86_FEATURE_MWAIT))
                 return;
         if (!this_cpu_has(X86_FEATURE_CLFLSH))
                 return;
diff --git a/arch/x86/kernel/sys_x86_64.c b/arch/x86/kernel/sys_x86_64.c
index 97ef74b88e0f..dbded5aedb81 100644
--- a/arch/x86/kernel/sys_x86_64.c
+++ b/arch/x86/kernel/sys_x86_64.c
@@ -157,7 +157,7 @@ arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0,
         if (flags & MAP_FIXED)
                 return addr;
 
-        /* for MAP_32BIT mappings we force the legact mmap base */
+        /* for MAP_32BIT mappings we force the legacy mmap base */
         if (!test_thread_flag(TIF_ADDR32) && (flags & MAP_32BIT))
                 goto bottomup;
 
diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c
index ecffca11f4e9..68bda7a84159 100644
--- a/arch/x86/kernel/traps.c
+++ b/arch/x86/kernel/traps.c
@@ -688,10 +688,19 @@ void __init early_trap_init(void)
         set_intr_gate_ist(X86_TRAP_DB, &debug, DEBUG_STACK);
         /* int3 can be called from all */
         set_system_intr_gate_ist(X86_TRAP_BP, &int3, DEBUG_STACK);
+#ifdef CONFIG_X86_32
         set_intr_gate(X86_TRAP_PF, &page_fault);
+#endif
         load_idt(&idt_descr);
 }
 
+void __init early_trap_pf_init(void)
+{
+#ifdef CONFIG_X86_64
+        set_intr_gate(X86_TRAP_PF, &page_fault);
+#endif
+}
+
 void __init trap_init(void)
 {
         int i;
diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c
index 06ccb5073a3f..4b9ea101fe3b 100644
--- a/arch/x86/kernel/tsc.c
+++ b/arch/x86/kernel/tsc.c
@@ -623,7 +623,8 @@ static void set_cyc2ns_scale(unsigned long cpu_khz, int cpu)
         ns_now = __cycles_2_ns(tsc_now);
 
         if (cpu_khz) {
-                *scale = (NSEC_PER_MSEC << CYC2NS_SCALE_FACTOR)/cpu_khz;
+                *scale = ((NSEC_PER_MSEC << CYC2NS_SCALE_FACTOR) +
+                                cpu_khz / 2) / cpu_khz;
                 *offset = ns_now - mult_frac(tsc_now, *scale,
                                              (1UL << CYC2NS_SCALE_FACTOR));
         }
diff --git a/arch/x86/kernel/uprobes.c b/arch/x86/kernel/uprobes.c
index c71025b67462..0ba4cfb4f412 100644
--- a/arch/x86/kernel/uprobes.c
+++ b/arch/x86/kernel/uprobes.c
@@ -680,8 +680,10 @@ static bool __skip_sstep(struct arch_uprobe *auprobe, struct pt_regs *regs)
                 if (auprobe->insn[i] == 0x66)
                         continue;
 
-                if (auprobe->insn[i] == 0x90)
+                if (auprobe->insn[i] == 0x90) {
+                        regs->ip += i + 1;
                         return true;
+                }
 
                 break;
         }
diff --git a/arch/x86/kernel/vm86_32.c b/arch/x86/kernel/vm86_32.c
index 1dfe69cc78a8..1cf5766dde16 100644
--- a/arch/x86/kernel/vm86_32.c
+++ b/arch/x86/kernel/vm86_32.c
@@ -202,7 +202,7 @@ out:
 static int do_vm86_irq_handling(int subfunction, int irqnumber);
 static void do_sys_vm86(struct kernel_vm86_struct *info, struct task_struct *tsk);
 
-int sys_vm86old(struct vm86_struct __user *v86, struct pt_regs *regs)
+int sys_vm86old(struct vm86_struct __user *v86)
 {
         struct kernel_vm86_struct info; /* declare this _on top_,
                                          * this avoids wasting of stack space.
@@ -222,7 +222,7 @@ int sys_vm86old(struct vm86_struct __user *v86, struct pt_regs *regs)
         if (tmp)
                 goto out;
         memset(&info.vm86plus, 0, (int)&info.regs32 - (int)&info.vm86plus);
-        info.regs32 = regs;
+        info.regs32 = current_pt_regs();
         tsk->thread.vm86_info = v86;
         do_sys_vm86(&info, tsk);
         ret = 0;        /* we never return here */
@@ -231,7 +231,7 @@ out:
 }
 
 
-int sys_vm86(unsigned long cmd, unsigned long arg, struct pt_regs *regs)
+int sys_vm86(unsigned long cmd, unsigned long arg)
 {
         struct kernel_vm86_struct info; /* declare this _on top_,
                                          * this avoids wasting of stack space.
@@ -272,7 +272,7 @@ int sys_vm86(unsigned long cmd, unsigned long arg, struct pt_regs *regs)
         ret = -EFAULT;
         if (tmp)
                 goto out;
-        info.regs32 = regs;
+        info.regs32 = current_pt_regs();
         info.vm86plus.is_vm86pus = 1;
         tsk->thread.vm86_info = (struct vm86_struct __user *)v86;
         do_sys_vm86(&info, tsk);
diff --git a/arch/x86/kernel/x8664_ksyms_64.c b/arch/x86/kernel/x8664_ksyms_64.c
index 1330dd102950..b014d9414d08 100644
--- a/arch/x86/kernel/x8664_ksyms_64.c
+++ b/arch/x86/kernel/x8664_ksyms_64.c
@@ -59,6 +59,9 @@ EXPORT_SYMBOL(memcpy);
 EXPORT_SYMBOL(__memcpy);
 EXPORT_SYMBOL(memmove);
 
+#ifndef CONFIG_DEBUG_VIRTUAL
+EXPORT_SYMBOL(phys_base);
+#endif
 EXPORT_SYMBOL(empty_zero_page);
 #ifndef CONFIG_PARAVIRT
 EXPORT_SYMBOL(native_load_gs_index);
diff --git a/arch/x86/kernel/x86_init.c b/arch/x86/kernel/x86_init.c
index 7a3d075a814a..45a14dbbddaf 100644
--- a/arch/x86/kernel/x86_init.c
+++ b/arch/x86/kernel/x86_init.c
@@ -19,6 +19,7 @@
 #include <asm/time.h>
 #include <asm/irq.h>
 #include <asm/io_apic.h>
+#include <asm/hpet.h>
 #include <asm/pat.h>
 #include <asm/tsc.h>
 #include <asm/iommu.h>
@@ -62,10 +63,6 @@ struct x86_init_ops x86_init __initdata = {
                 .banner                 = default_banner,
         },
 
-        .mapping = {
-                .pagetable_reserve              = native_pagetable_reserve,
-        },
-
         .paging = {
                 .pagetable_init         = native_pagetable_init,
         },
@@ -111,15 +108,22 @@ struct x86_platform_ops x86_platform = {
 
 EXPORT_SYMBOL_GPL(x86_platform);
 struct x86_msi_ops x86_msi = {
-        .setup_msi_irqs = native_setup_msi_irqs,
-        .teardown_msi_irq = native_teardown_msi_irq,
-        .teardown_msi_irqs = default_teardown_msi_irqs,
-        .restore_msi_irqs = default_restore_msi_irqs,
+        .setup_msi_irqs         = native_setup_msi_irqs,
+        .compose_msi_msg        = native_compose_msi_msg,
+        .teardown_msi_irq       = native_teardown_msi_irq,
+        .teardown_msi_irqs      = default_teardown_msi_irqs,
+        .restore_msi_irqs       = default_restore_msi_irqs,
+        .setup_hpet_msi         = default_setup_hpet_msi,
 };
 
 struct x86_io_apic_ops x86_io_apic_ops = {
-        .init   = native_io_apic_init_mappings,
-        .read   = native_io_apic_read,
-        .write  = native_io_apic_write,
-        .modify = native_io_apic_modify,
+        .init                   = native_io_apic_init_mappings,
+        .read                   = native_io_apic_read,
+        .write                  = native_io_apic_write,
+        .modify                 = native_io_apic_modify,
+        .disable                = native_disable_io_apic,
+        .print_entries          = native_io_apic_print_entries,
+        .set_affinity           = native_ioapic_set_affinity,
+        .setup_entry            = native_setup_ioapic_entry,
+        .eoi_ioapic_pin         = native_eoi_ioapic_pin,
 };
diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c
index a27e76371108..a335cc6cde72 100644
--- a/arch/x86/kvm/emulate.c
+++ b/arch/x86/kvm/emulate.c
@@ -24,6 +24,7 @@
 #include "kvm_cache_regs.h"
 #include <linux/module.h>
 #include <asm/kvm_emulate.h>
+#include <linux/stringify.h>
 
 #include "x86.h"
 #include "tss.h"
@@ -43,7 +44,7 @@
 #define OpCL               9ull  /* CL register (for shifts) */
 #define OpImmByte         10ull  /* 8-bit sign extended immediate */
 #define OpOne             11ull  /* Implied 1 */
-#define OpImm             12ull  /* Sign extended immediate */
+#define OpImm             12ull  /* Sign extended up to 32-bit immediate */
 #define OpMem16           13ull  /* Memory operand (16-bit). */
 #define OpMem32           14ull  /* Memory operand (32-bit). */
 #define OpImmU            15ull  /* Immediate operand, zero extended */
@@ -58,6 +59,7 @@
 #define OpFS              24ull  /* FS */
 #define OpGS              25ull  /* GS */
 #define OpMem8            26ull  /* 8-bit zero extended memory operand */
+#define OpImm64           27ull  /* Sign extended 16/32/64-bit immediate */
 
 #define OpBits             5  /* Width of operand field */
 #define OpMask             ((1ull << OpBits) - 1)
@@ -101,6 +103,7 @@
 #define SrcMemFAddr (OpMemFAddr << SrcShift)
 #define SrcAcc      (OpAcc << SrcShift)
 #define SrcImmU16   (OpImmU16 << SrcShift)
+#define SrcImm64    (OpImm64 << SrcShift)
 #define SrcDX       (OpDX << SrcShift)
 #define SrcMem8     (OpMem8 << SrcShift)
 #define SrcMask     (OpMask << SrcShift)
@@ -113,6 +116,7 @@
 #define GroupDual   (2<<15)     /* Alternate decoding of mod == 3 */
 #define Prefix      (3<<15)     /* Instruction varies with 66/f2/f3 prefix */
 #define RMExt       (4<<15)     /* Opcode extension in ModRM r/m if mod == 3 */
+#define Escape      (5<<15)     /* Escape to coprocessor instruction */
 #define Sse         (1<<18)     /* SSE Vector instruction */
 /* Generic ModRM decode. */
 #define ModRM       (1<<19)
@@ -146,6 +150,8 @@
 #define Aligned     ((u64)1 << 41)  /* Explicitly aligned (e.g. MOVDQA) */
 #define Unaligned   ((u64)1 << 42)  /* Explicitly unaligned (e.g. MOVDQU) */
 #define Avx         ((u64)1 << 43)  /* Advanced Vector Extensions */
+#define Fastop      ((u64)1 << 44)  /* Use opcode::u.fastop */
+#define NoWrite     ((u64)1 << 45)  /* No writeback */
 
 #define X2(x...) x, x
 #define X3(x...) X2(x), x
@@ -156,6 +162,27 @@
 #define X8(x...) X4(x), X4(x)
 #define X16(x...) X8(x), X8(x)
 
+#define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
+#define FASTOP_SIZE 8
+
+/*
+ * fastop functions have a special calling convention:
+ *
+ * dst:    [rdx]:rax  (in/out)
+ * src:    rbx        (in/out)
+ * src2:   rcx        (in)
+ * flags:  rflags     (in/out)
+ *
+ * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
+ * different operand sizes can be reached by calculation, rather than a jump
+ * table (which would be bigger than the code).
+ *
+ * fastop functions are declared as taking a never-defined fastop parameter,
+ * so they can't be called from C directly.
+ */
+
+struct fastop;
+
 struct opcode {
         u64 flags : 56;
         u64 intercept : 8;
@@ -164,6 +191,8 @@ struct opcode {
                 const struct opcode *group;
                 const struct group_dual *gdual;
                 const struct gprefix *gprefix;
+                const struct escape *esc;
+                void (*fastop)(struct fastop *fake);
         } u;
         int (*check_perm)(struct x86_emulate_ctxt *ctxt);
 };
@@ -180,6 +209,11 @@ struct gprefix {
         struct opcode pfx_f3;
 };
 
+struct escape {
+        struct opcode op[8];
+        struct opcode high[64];
+};
+
 /* EFLAGS bit definitions. */
 #define EFLG_ID (1<<21)
 #define EFLG_VIP (1<<20)
@@ -407,6 +441,97 @@ static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
                 }                                                       \
         } while (0)
 
+static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));
+
+#define FOP_ALIGN ".align " __stringify(FASTOP_SIZE) " \n\t"
+#define FOP_RET   "ret \n\t"
+
+#define FOP_START(op) \
+        extern void em_##op(struct fastop *fake); \
+        asm(".pushsection .text, \"ax\" \n\t" \
+            ".global em_" #op " \n\t" \
+            FOP_ALIGN \
+            "em_" #op ": \n\t"
+
+#define FOP_END \
+            ".popsection")
+
+#define FOPNOP() FOP_ALIGN FOP_RET
+
+#define FOP1E(op,  dst) \
+        FOP_ALIGN #op " %" #dst " \n\t" FOP_RET
+
+#define FASTOP1(op) \
+        FOP_START(op) \
+        FOP1E(op##b, al) \
+        FOP1E(op##w, ax) \
+        FOP1E(op##l, eax) \
+        ON64(FOP1E(op##q, rax)) \
+        FOP_END
+
+#define FOP2E(op,  dst, src)       \
+        FOP_ALIGN #op " %" #src ", %" #dst " \n\t" FOP_RET
+
+#define FASTOP2(op) \
+        FOP_START(op) \
+        FOP2E(op##b, al, bl) \
+        FOP2E(op##w, ax, bx) \
+        FOP2E(op##l, eax, ebx) \
+        ON64(FOP2E(op##q, rax, rbx)) \
+        FOP_END
+
+/* 2 operand, word only */
+#define FASTOP2W(op) \
+        FOP_START(op) \
+        FOPNOP() \
+        FOP2E(op##w, ax, bx) \
+        FOP2E(op##l, eax, ebx) \
+        ON64(FOP2E(op##q, rax, rbx)) \
+        FOP_END
+
+/* 2 operand, src is CL */
+#define FASTOP2CL(op) \
+        FOP_START(op) \
+        FOP2E(op##b, al, cl) \
+        FOP2E(op##w, ax, cl) \
+        FOP2E(op##l, eax, cl) \
+        ON64(FOP2E(op##q, rax, cl)) \
+        FOP_END
+
+#define FOP3E(op,  dst, src, src2) \
+        FOP_ALIGN #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
+
+/* 3-operand, word-only, src2=cl */
+#define FASTOP3WCL(op) \
+        FOP_START(op) \
+        FOPNOP() \
+        FOP3E(op##w, ax, bx, cl) \
+        FOP3E(op##l, eax, ebx, cl) \
+        ON64(FOP3E(op##q, rax, rbx, cl)) \
+        FOP_END
+
+/* Special case for SETcc - 1 instruction per cc */
+#define FOP_SETCC(op) ".align 4; " #op " %al; ret \n\t"
+
+FOP_START(setcc)
+FOP_SETCC(seto)
+FOP_SETCC(setno)
+FOP_SETCC(setc)
+FOP_SETCC(setnc)
+FOP_SETCC(setz)
+FOP_SETCC(setnz)
+FOP_SETCC(setbe)
+FOP_SETCC(setnbe)
+FOP_SETCC(sets)
+FOP_SETCC(setns)
+FOP_SETCC(setp)
+FOP_SETCC(setnp)
+FOP_SETCC(setl)
+FOP_SETCC(setnl)
+FOP_SETCC(setle)
+FOP_SETCC(setnle)
+FOP_END;
+
 #define __emulate_1op_rax_rdx(ctxt, _op, _suffix, _ex)                  \
         do {                                                            \
                 unsigned long _tmp;                                     \
@@ -663,7 +788,7 @@ static int __linearize(struct x86_emulate_ctxt *ctxt,
         ulong la;
         u32 lim;
         u16 sel;
-        unsigned cpl, rpl;
+        unsigned cpl;
 
         la = seg_base(ctxt, addr.seg) + addr.ea;
         switch (ctxt->mode) {
@@ -697,11 +822,6 @@ static int __linearize(struct x86_emulate_ctxt *ctxt,
                                 goto bad;
                 }
                 cpl = ctxt->ops->cpl(ctxt);
-                if (ctxt->mode == X86EMUL_MODE_REAL)
-                        rpl = 0;
-                else
-                        rpl = sel & 3;
-                cpl = max(cpl, rpl);
                 if (!(desc.type & 8)) {
                         /* data segment */
                         if (cpl > desc.dpl)
@@ -852,39 +972,50 @@ static int read_descriptor(struct x86_emulate_ctxt *ctxt,
         return rc;
 }
 
-static int test_cc(unsigned int condition, unsigned int flags)
-{
-        int rc = 0;
-
-        switch ((condition & 15) >> 1) {
-        case 0: /* o */
-                rc |= (flags & EFLG_OF);
-                break;
-        case 1: /* b/c/nae */
-                rc |= (flags & EFLG_CF);
-                break;
-        case 2: /* z/e */
-                rc |= (flags & EFLG_ZF);
-                break;
-        case 3: /* be/na */
-                rc |= (flags & (EFLG_CF|EFLG_ZF));
-                break;
-        case 4: /* s */
-                rc |= (flags & EFLG_SF);
-                break;
-        case 5: /* p/pe */
-                rc |= (flags & EFLG_PF);
-                break;
-        case 7: /* le/ng */
-                rc |= (flags & EFLG_ZF);
-                /* fall through */
-        case 6: /* l/nge */
-                rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
-                break;
-        }
-
-        /* Odd condition identifiers (lsb == 1) have inverted sense. */
-        return (!!rc ^ (condition & 1));
+FASTOP2(add);
+FASTOP2(or);
+FASTOP2(adc);
+FASTOP2(sbb);
+FASTOP2(and);
+FASTOP2(sub);
+FASTOP2(xor);
+FASTOP2(cmp);
+FASTOP2(test);
+
+FASTOP3WCL(shld);
+FASTOP3WCL(shrd);
+
+FASTOP2W(imul);
+
+FASTOP1(not);
+FASTOP1(neg);
+FASTOP1(inc);
+FASTOP1(dec);
+
+FASTOP2CL(rol);
+FASTOP2CL(ror);
+FASTOP2CL(rcl);
+FASTOP2CL(rcr);
+FASTOP2CL(shl);
+FASTOP2CL(shr);
+FASTOP2CL(sar);
+
+FASTOP2W(bsf);
+FASTOP2W(bsr);
+FASTOP2W(bt);
+FASTOP2W(bts);
+FASTOP2W(btr);
+FASTOP2W(btc);
+
+static u8 test_cc(unsigned int condition, unsigned long flags)
+{
+        u8 rc;
+        void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
+
+        flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
+        asm("push %[flags]; popf; call *%[fastop]"
+            : "=a"(rc) : [fastop]"r"(fop), [flags]"r"(flags));
+        return rc;
 }
 
 static void fetch_register_operand(struct operand *op)
@@ -994,6 +1125,53 @@ static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
         ctxt->ops->put_fpu(ctxt);
 }
 
+static int em_fninit(struct x86_emulate_ctxt *ctxt)
+{
+        if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
+                return emulate_nm(ctxt);
+
+        ctxt->ops->get_fpu(ctxt);
+        asm volatile("fninit");
+        ctxt->ops->put_fpu(ctxt);
+        return X86EMUL_CONTINUE;
+}
+
+static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
+{
+        u16 fcw;
+
+        if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
+                return emulate_nm(ctxt);
+
+        ctxt->ops->get_fpu(ctxt);
+        asm volatile("fnstcw %0": "+m"(fcw));
+        ctxt->ops->put_fpu(ctxt);
+
+        /* force 2 byte destination */
+        ctxt->dst.bytes = 2;
+        ctxt->dst.val = fcw;
+
+        return X86EMUL_CONTINUE;
+}
+
+static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
+{
+        u16 fsw;
+
+        if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
+                return emulate_nm(ctxt);
+
+        ctxt->ops->get_fpu(ctxt);
+        asm volatile("fnstsw %0": "+m"(fsw));
+        ctxt->ops->put_fpu(ctxt);
+
+        /* force 2 byte destination */
+        ctxt->dst.bytes = 2;
+        ctxt->dst.val = fsw;
+
+        return X86EMUL_CONTINUE;
+}
+
 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
                                     struct operand *op)
 {
@@ -1534,6 +1712,9 @@ static int writeback(struct x86_emulate_ctxt *ctxt)
 {
         int rc;
 
+        if (ctxt->d & NoWrite)
+                return X86EMUL_CONTINUE;
+
         switch (ctxt->dst.type) {
         case OP_REG:
                 write_register_operand(&ctxt->dst);
@@ -1918,47 +2099,6 @@ static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
         return X86EMUL_CONTINUE;
 }
 
-static int em_grp2(struct x86_emulate_ctxt *ctxt)
-{
-        switch (ctxt->modrm_reg) {
-        case 0: /* rol */
-                emulate_2op_SrcB(ctxt, "rol");
-                break;
-        case 1: /* ror */
-                emulate_2op_SrcB(ctxt, "ror");
-                break;
-        case 2: /* rcl */
-                emulate_2op_SrcB(ctxt, "rcl");
-                break;
-        case 3: /* rcr */
-                emulate_2op_SrcB(ctxt, "rcr");
-                break;
-        case 4: /* sal/shl */
-        case 6: /* sal/shl */
-                emulate_2op_SrcB(ctxt, "sal");
-                break;
-        case 5: /* shr */
-                emulate_2op_SrcB(ctxt, "shr");
-                break;
-        case 7: /* sar */
-                emulate_2op_SrcB(ctxt, "sar");
-                break;
-        }
-        return X86EMUL_CONTINUE;
-}
-
-static int em_not(struct x86_emulate_ctxt *ctxt)
-{
-        ctxt->dst.val = ~ctxt->dst.val;
-        return X86EMUL_CONTINUE;
-}
-
-static int em_neg(struct x86_emulate_ctxt *ctxt)
-{
-        emulate_1op(ctxt, "neg");
-        return X86EMUL_CONTINUE;
-}
-
 static int em_mul_ex(struct x86_emulate_ctxt *ctxt)
 {
         u8 ex = 0;
@@ -2000,12 +2140,6 @@ static int em_grp45(struct x86_emulate_ctxt *ctxt)
         int rc = X86EMUL_CONTINUE;
 
         switch (ctxt->modrm_reg) {
-        case 0: /* inc */
-                emulate_1op(ctxt, "inc");
-                break;
-        case 1: /* dec */
-                emulate_1op(ctxt, "dec");
-                break;
         case 2: /* call near abs */ {
                 long int old_eip;
                 old_eip = ctxt->_eip;
@@ -2075,7 +2209,7 @@ static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
         /* Save real source value, then compare EAX against destination. */
         ctxt->src.orig_val = ctxt->src.val;
         ctxt->src.val = reg_read(ctxt, VCPU_REGS_RAX);
-        emulate_2op_SrcV(ctxt, "cmp");
+        fastop(ctxt, em_cmp);
 
         if (ctxt->eflags & EFLG_ZF) {
                 /* Success: write back to memory. */
@@ -2843,7 +2977,7 @@ static int em_das(struct x86_emulate_ctxt *ctxt)
         ctxt->src.type = OP_IMM;
         ctxt->src.val = 0;
         ctxt->src.bytes = 1;
-        emulate_2op_SrcV(ctxt, "or");
+        fastop(ctxt, em_or);
         ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
         if (cf)
                 ctxt->eflags |= X86_EFLAGS_CF;
@@ -2852,6 +2986,24 @@ static int em_das(struct x86_emulate_ctxt *ctxt)
         return X86EMUL_CONTINUE;
 }
 
+static int em_aad(struct x86_emulate_ctxt *ctxt)
+{
+        u8 al = ctxt->dst.val & 0xff;
+        u8 ah = (ctxt->dst.val >> 8) & 0xff;
+
+        al = (al + (ah * ctxt->src.val)) & 0xff;
+
+        ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
+
+        /* Set PF, ZF, SF */
+        ctxt->src.type = OP_IMM;
+        ctxt->src.val = 0;
+        ctxt->src.bytes = 1;
+        fastop(ctxt, em_or);
+
+        return X86EMUL_CONTINUE;
+}
+
 static int em_call(struct x86_emulate_ctxt *ctxt)
 {
         long rel = ctxt->src.val;
@@ -2900,64 +3052,6 @@ static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
         return X86EMUL_CONTINUE;
 }
 
-static int em_add(struct x86_emulate_ctxt *ctxt)
-{
-        emulate_2op_SrcV(ctxt, "add");
-        return X86EMUL_CONTINUE;
-}
-
-static int em_or(struct x86_emulate_ctxt *ctxt)
-{
-        emulate_2op_SrcV(ctxt, "or");
-        return X86EMUL_CONTINUE;
-}
-
-static int em_adc(struct x86_emulate_ctxt *ctxt)
-{
-        emulate_2op_SrcV(ctxt, "adc");
-        return X86EMUL_CONTINUE;
-}
-
-static int em_sbb(struct x86_emulate_ctxt *ctxt)
-{
-        emulate_2op_SrcV(ctxt, "sbb");
-        return X86EMUL_CONTINUE;
-}
-
-static int em_and(struct x86_emulate_ctxt *ctxt)
-{
-        emulate_2op_SrcV(ctxt, "and");
-        return X86EMUL_CONTINUE;
-}
-
-static int em_sub(struct x86_emulate_ctxt *ctxt)
-{
-        emulate_2op_SrcV(ctxt, "sub");
-        return X86EMUL_CONTINUE;
-}
-
-static int em_xor(struct x86_emulate_ctxt *ctxt)
-{
-        emulate_2op_SrcV(ctxt, "xor");
-        return X86EMUL_CONTINUE;
-}
-
-static int em_cmp(struct x86_emulate_ctxt *ctxt)
-{
-        emulate_2op_SrcV(ctxt, "cmp");
-        /* Disable writeback. */
-        ctxt->dst.type = OP_NONE;
-        return X86EMUL_CONTINUE;
-}
-
-static int em_test(struct x86_emulate_ctxt *ctxt)
-{
-        emulate_2op_SrcV(ctxt, "test");
-        /* Disable writeback. */
-        ctxt->dst.type = OP_NONE;
-        return X86EMUL_CONTINUE;
-}
-
 static int em_xchg(struct x86_emulate_ctxt *ctxt)
 {
         /* Write back the register source. */
@@ -2970,16 +3064,10 @@ static int em_xchg(struct x86_emulate_ctxt *ctxt)
         return X86EMUL_CONTINUE;
 }
 
-static int em_imul(struct x86_emulate_ctxt *ctxt)
-{
-        emulate_2op_SrcV_nobyte(ctxt, "imul");
-        return X86EMUL_CONTINUE;
-}
-
 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
 {
         ctxt->dst.val = ctxt->src2.val;
-        return em_imul(ctxt);
+        return fastop(ctxt, em_imul);
 }
 
 static int em_cwd(struct x86_emulate_ctxt *ctxt)
@@ -3300,47 +3388,6 @@ static int em_sti(struct x86_emulate_ctxt *ctxt)
         return X86EMUL_CONTINUE;
 }
 
-static int em_bt(struct x86_emulate_ctxt *ctxt)
-{
-        /* Disable writeback. */
-        ctxt->dst.type = OP_NONE;
-        /* only subword offset */
-        ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
-
-        emulate_2op_SrcV_nobyte(ctxt, "bt");
-        return X86EMUL_CONTINUE;
-}
-
-static int em_bts(struct x86_emulate_ctxt *ctxt)
-{
-        emulate_2op_SrcV_nobyte(ctxt, "bts");
-        return X86EMUL_CONTINUE;
-}
-
-static int em_btr(struct x86_emulate_ctxt *ctxt)
-{
-        emulate_2op_SrcV_nobyte(ctxt, "btr");
-        return X86EMUL_CONTINUE;
-}
-
-static int em_btc(struct x86_emulate_ctxt *ctxt)
-{
-        emulate_2op_SrcV_nobyte(ctxt, "btc");
-        return X86EMUL_CONTINUE;
-}
-
-static int em_bsf(struct x86_emulate_ctxt *ctxt)
-{
-        emulate_2op_SrcV_nobyte(ctxt, "bsf");
-        return X86EMUL_CONTINUE;
-}
-
-static int em_bsr(struct x86_emulate_ctxt *ctxt)
-{
-        emulate_2op_SrcV_nobyte(ctxt, "bsr");
-        return X86EMUL_CONTINUE;
-}
-
 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
 {
         u32 eax, ebx, ecx, edx;
@@ -3572,7 +3619,9 @@ static int check_perm_out(struct x86_emulate_ctxt *ctxt)
 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
+#define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
+#define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
 #define II(_f, _e, _i) \
         { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
 #define IIP(_f, _e, _i, _p) \
@@ -3583,12 +3632,13 @@ static int check_perm_out(struct x86_emulate_ctxt *ctxt)
 #define D2bv(_f)      D((_f) | ByteOp), D(_f)
 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
 #define I2bv(_f, _e)  I((_f) | ByteOp, _e), I(_f, _e)
+#define F2bv(_f, _e)  F((_f) | ByteOp, _e), F(_f, _e)
 #define I2bvIP(_f, _e, _i, _p) \
         IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
 
-#define I6ALU(_f, _e) I2bv((_f) | DstMem | SrcReg | ModRM, _e),         \
-                I2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e),     \
-                I2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
+#define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e),         \
+                F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e),     \
+                F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
 
 static const struct opcode group7_rm1[] = {
         DI(SrcNone | Priv, monitor),
@@ -3614,25 +3664,36 @@ static const struct opcode group7_rm7[] = {
 };
 
 static const struct opcode group1[] = {
-        I(Lock, em_add),
-        I(Lock | PageTable, em_or),
-        I(Lock, em_adc),
-        I(Lock, em_sbb),
-        I(Lock | PageTable, em_and),
-        I(Lock, em_sub),
-        I(Lock, em_xor),
-        I(0, em_cmp),
+        F(Lock, em_add),
+        F(Lock | PageTable, em_or),
+        F(Lock, em_adc),
+        F(Lock, em_sbb),
+        F(Lock | PageTable, em_and),
+        F(Lock, em_sub),
+        F(Lock, em_xor),
+        F(NoWrite, em_cmp),
 };
 
 static const struct opcode group1A[] = {
         I(DstMem | SrcNone | Mov | Stack, em_pop), N, N, N, N, N, N, N,
 };
 
+static const struct opcode group2[] = {
+        F(DstMem | ModRM, em_rol),
+        F(DstMem | ModRM, em_ror),
+        F(DstMem | ModRM, em_rcl),
+        F(DstMem | ModRM, em_rcr),
+        F(DstMem | ModRM, em_shl),
+        F(DstMem | ModRM, em_shr),
+        F(DstMem | ModRM, em_shl),
+        F(DstMem | ModRM, em_sar),
+};
+
 static const struct opcode group3[] = {
-        I(DstMem | SrcImm, em_test),
-        I(DstMem | SrcImm, em_test),
-        I(DstMem | SrcNone | Lock, em_not),
-        I(DstMem | SrcNone | Lock, em_neg),
+        F(DstMem | SrcImm | NoWrite, em_test),
+        F(DstMem | SrcImm | NoWrite, em_test),
+        F(DstMem | SrcNone | Lock, em_not),
+        F(DstMem | SrcNone | Lock, em_neg),
         I(SrcMem, em_mul_ex),
         I(SrcMem, em_imul_ex),
         I(SrcMem, em_div_ex),
@@ -3640,14 +3701,14 @@ static const struct opcode group3[] = {
 };
 
 static const struct opcode group4[] = {
-        I(ByteOp | DstMem | SrcNone | Lock, em_grp45),
-        I(ByteOp | DstMem | SrcNone | Lock, em_grp45),
+        F(ByteOp | DstMem | SrcNone | Lock, em_inc),
+        F(ByteOp | DstMem | SrcNone | Lock, em_dec),
         N, N, N, N, N, N,
 };
 
 static const struct opcode group5[] = {
-        I(DstMem | SrcNone | Lock,              em_grp45),
-        I(DstMem | SrcNone | Lock,              em_grp45),
+        F(DstMem | SrcNone | Lock,              em_inc),
+        F(DstMem | SrcNone | Lock,              em_dec),
         I(SrcMem | Stack,                       em_grp45),
         I(SrcMemFAddr | ImplicitOps | Stack,    em_call_far),
         I(SrcMem | Stack,                       em_grp45),
@@ -3682,10 +3743,10 @@ static const struct group_dual group7 = { {
 
 static const struct opcode group8[] = {
         N, N, N, N,
-        I(DstMem | SrcImmByte,                          em_bt),
-        I(DstMem | SrcImmByte | Lock | PageTable,       em_bts),
-        I(DstMem | SrcImmByte | Lock,                   em_btr),
-        I(DstMem | SrcImmByte | Lock | PageTable,       em_btc),
+        F(DstMem | SrcImmByte | NoWrite,                em_bt),
+        F(DstMem | SrcImmByte | Lock | PageTable,       em_bts),
+        F(DstMem | SrcImmByte | Lock,                   em_btr),
+        F(DstMem | SrcImmByte | Lock | PageTable,       em_btc),
 };
 
 static const struct group_dual group9 = { {
@@ -3707,33 +3768,96 @@ static const struct gprefix pfx_vmovntpx = {
         I(0, em_mov), N, N, N,
 };
 
+static const struct escape escape_d9 = { {
+        N, N, N, N, N, N, N, I(DstMem, em_fnstcw),
+}, {
+        /* 0xC0 - 0xC7 */
+        N, N, N, N, N, N, N, N,
+        /* 0xC8 - 0xCF */
+        N, N, N, N, N, N, N, N,
+        /* 0xD0 - 0xC7 */
+        N, N, N, N, N, N, N, N,
+        /* 0xD8 - 0xDF */
+        N, N, N, N, N, N, N, N,
+        /* 0xE0 - 0xE7 */
+        N, N, N, N, N, N, N, N,
+        /* 0xE8 - 0xEF */
+        N, N, N, N, N, N, N, N,
+        /* 0xF0 - 0xF7 */
+        N, N, N, N, N, N, N, N,
+        /* 0xF8 - 0xFF */
+        N, N, N, N, N, N, N, N,
+} };
+
+static const struct escape escape_db = { {
+        N, N, N, N, N, N, N, N,
+}, {
+        /* 0xC0 - 0xC7 */
+        N, N, N, N, N, N, N, N,
+        /* 0xC8 - 0xCF */
+        N, N, N, N, N, N, N, N,
+        /* 0xD0 - 0xC7 */
+        N, N, N, N, N, N, N, N,
+        /* 0xD8 - 0xDF */
+        N, N, N, N, N, N, N, N,
+        /* 0xE0 - 0xE7 */
+        N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
+        /* 0xE8 - 0xEF */
+        N, N, N, N, N, N, N, N,
+        /* 0xF0 - 0xF7 */
+        N, N, N, N, N, N, N, N,
+        /* 0xF8 - 0xFF */
+        N, N, N, N, N, N, N, N,
+} };
+
+static const struct escape escape_dd = { {
+        N, N, N, N, N, N, N, I(DstMem, em_fnstsw),
+}, {
+        /* 0xC0 - 0xC7 */
+        N, N, N, N, N, N, N, N,
+        /* 0xC8 - 0xCF */
+        N, N, N, N, N, N, N, N,
+        /* 0xD0 - 0xC7 */
+        N, N, N, N, N, N, N, N,
+        /* 0xD8 - 0xDF */
+        N, N, N, N, N, N, N, N,
+        /* 0xE0 - 0xE7 */
+        N, N, N, N, N, N, N, N,
+        /* 0xE8 - 0xEF */
+        N, N, N, N, N, N, N, N,
+        /* 0xF0 - 0xF7 */
+        N, N, N, N, N, N, N, N,
+        /* 0xF8 - 0xFF */
+        N, N, N, N, N, N, N, N,
+} };
+
 static const struct opcode opcode_table[256] = {
         /* 0x00 - 0x07 */
-        I6ALU(Lock, em_add),
+        F6ALU(Lock, em_add),
         I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
         I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
         /* 0x08 - 0x0F */
-        I6ALU(Lock | PageTable, em_or),
+        F6ALU(Lock | PageTable, em_or),
         I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
         N,
         /* 0x10 - 0x17 */
-        I6ALU(Lock, em_adc),
+        F6ALU(Lock, em_adc),
         I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
         I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
         /* 0x18 - 0x1F */
-        I6ALU(Lock, em_sbb),
+        F6ALU(Lock, em_sbb),
         I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
         I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
         /* 0x20 - 0x27 */
-        I6ALU(Lock | PageTable, em_and), N, N,
+        F6ALU(Lock | PageTable, em_and), N, N,
         /* 0x28 - 0x2F */
-        I6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
+        F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
         /* 0x30 - 0x37 */
-        I6ALU(Lock, em_xor), N, N,
+        F6ALU(Lock, em_xor), N, N,
         /* 0x38 - 0x3F */
-        I6ALU(0, em_cmp), N, N,
+        F6ALU(NoWrite, em_cmp), N, N,
         /* 0x40 - 0x4F */
-        X16(D(DstReg)),
+        X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
         /* 0x50 - 0x57 */
         X8(I(SrcReg | Stack, em_push)),
         /* 0x58 - 0x5F */
@@ -3757,7 +3881,7 @@ static const struct opcode opcode_table[256] = {
         G(DstMem | SrcImm, group1),
         G(ByteOp | DstMem | SrcImm | No64, group1),
         G(DstMem | SrcImmByte, group1),
-        I2bv(DstMem | SrcReg | ModRM, em_test),
+        F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
         I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
         /* 0x88 - 0x8F */
         I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
@@ -3777,18 +3901,18 @@ static const struct opcode opcode_table[256] = {
         I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
         I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
         I2bv(SrcSI | DstDI | Mov | String, em_mov),
-        I2bv(SrcSI | DstDI | String, em_cmp),
+        F2bv(SrcSI | DstDI | String | NoWrite, em_cmp),
         /* 0xA8 - 0xAF */
-        I2bv(DstAcc | SrcImm, em_test),
+        F2bv(DstAcc | SrcImm | NoWrite, em_test),
         I2bv(SrcAcc | DstDI | Mov | String, em_mov),
         I2bv(SrcSI | DstAcc | Mov | String, em_mov),
-        I2bv(SrcAcc | DstDI | String, em_cmp),
+        F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp),
         /* 0xB0 - 0xB7 */
         X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
         /* 0xB8 - 0xBF */
-        X8(I(DstReg | SrcImm | Mov, em_mov)),
+        X8(I(DstReg | SrcImm64 | Mov, em_mov)),
         /* 0xC0 - 0xC7 */
-        D2bv(DstMem | SrcImmByte | ModRM),
+        G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
         I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
         I(ImplicitOps | Stack, em_ret),
         I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
@@ -3800,10 +3924,11 @@ static const struct opcode opcode_table[256] = {
         D(ImplicitOps), DI(SrcImmByte, intn),
         D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
         /* 0xD0 - 0xD7 */
-        D2bv(DstMem | SrcOne | ModRM), D2bv(DstMem | ModRM),
-        N, N, N, N,
+        G(Src2One | ByteOp, group2), G(Src2One, group2),
+        G(Src2CL | ByteOp, group2), G(Src2CL, group2),
+        N, I(DstAcc | SrcImmByte | No64, em_aad), N, N,
         /* 0xD8 - 0xDF */
-        N, N, N, N, N, N, N, N,
+        N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
         /* 0xE0 - 0xE7 */
         X3(I(SrcImmByte, em_loop)),
         I(SrcImmByte, em_jcxz),
@@ -3870,28 +3995,29 @@ static const struct opcode twobyte_table[256] = {
         X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
         /* 0xA0 - 0xA7 */
         I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
-        II(ImplicitOps, em_cpuid, cpuid), I(DstMem | SrcReg | ModRM | BitOp, em_bt),
-        D(DstMem | SrcReg | Src2ImmByte | ModRM),
-        D(DstMem | SrcReg | Src2CL | ModRM), N, N,
+        II(ImplicitOps, em_cpuid, cpuid),
+        F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
+        F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
+        F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
         /* 0xA8 - 0xAF */
         I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
         DI(ImplicitOps, rsm),
-        I(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
-        D(DstMem | SrcReg | Src2ImmByte | ModRM),
-        D(DstMem | SrcReg | Src2CL | ModRM),
-        D(ModRM), I(DstReg | SrcMem | ModRM, em_imul),
+        F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
+        F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
+        F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
+        D(ModRM), F(DstReg | SrcMem | ModRM, em_imul),
         /* 0xB0 - 0xB7 */
         I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_cmpxchg),
         I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
-        I(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
+        F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
         I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
         I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
         D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
         /* 0xB8 - 0xBF */
         N, N,
         G(BitOp, group8),
-        I(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
-        I(DstReg | SrcMem | ModRM, em_bsf), I(DstReg | SrcMem | ModRM, em_bsr),
+        F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
+        F(DstReg | SrcMem | ModRM, em_bsf), F(DstReg | SrcMem | ModRM, em_bsr),
         D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
         /* 0xC0 - 0xC7 */
         D2bv(DstMem | SrcReg | ModRM | Lock),
@@ -3950,6 +4076,9 @@ static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
         case 4:
                 op->val = insn_fetch(s32, ctxt);
                 break;
+        case 8:
+                op->val = insn_fetch(s64, ctxt);
+                break;
         }
         if (!sign_extension) {
                 switch (op->bytes) {
@@ -4028,6 +4157,9 @@ static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
         case OpImm:
                 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
                 break;
+        case OpImm64:
+                rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
+                break;
         case OpMem8:
                 ctxt->memop.bytes = 1;
                 goto mem_common;
@@ -4222,6 +4354,12 @@ done_prefixes:
                         case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
                         }
                         break;
+                case Escape:
+                        if (ctxt->modrm > 0xbf)
+                                opcode = opcode.u.esc->high[ctxt->modrm - 0xc0];
+                        else
+                                opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
+                        break;
                 default:
                         return EMULATION_FAILED;
                 }
@@ -4354,6 +4492,16 @@ static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
                 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
 }
 
+static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
+{
+        ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
+        fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
+        asm("push %[flags]; popf; call *%[fastop]; pushf; pop %[flags]\n"
+            : "+a"(ctxt->dst.val), "+b"(ctxt->src.val), [flags]"+D"(flags)
+        : "c"(ctxt->src2.val), [fastop]"S"(fop));
+        ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
+        return X86EMUL_CONTINUE;
+}
 
 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
 {
@@ -4483,6 +4631,13 @@ special_insn:
         }
 
         if (ctxt->execute) {
+                if (ctxt->d & Fastop) {
+                        void (*fop)(struct fastop *) = (void *)ctxt->execute;
+                        rc = fastop(ctxt, fop);
+                        if (rc != X86EMUL_CONTINUE)
+                                goto done;
+                        goto writeback;
+                }
                 rc = ctxt->execute(ctxt);
                 if (rc != X86EMUL_CONTINUE)
                         goto done;
@@ -4493,12 +4648,6 @@ special_insn:
                 goto twobyte_insn;
 
         switch (ctxt->b) {
-        case 0x40 ... 0x47: /* inc r16/r32 */
-                emulate_1op(ctxt, "inc");
-                break;
-        case 0x48 ... 0x4f: /* dec r16/r32 */
-                emulate_1op(ctxt, "dec");
-                break;
         case 0x63:              /* movsxd */
                 if (ctxt->mode != X86EMUL_MODE_PROT64)
                         goto cannot_emulate;
@@ -4523,9 +4672,6 @@ special_insn:
                 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
                 }
                 break;
-        case 0xc0 ... 0xc1:
-                rc = em_grp2(ctxt);
-                break;
         case 0xcc:              /* int3 */
                 rc = emulate_int(ctxt, 3);
                 break;
@@ -4536,13 +4682,6 @@ special_insn:
                 if (ctxt->eflags & EFLG_OF)
                         rc = emulate_int(ctxt, 4);
                 break;
-        case 0xd0 ... 0xd1:     /* Grp2 */
-                rc = em_grp2(ctxt);
-                break;
-        case 0xd2 ... 0xd3:     /* Grp2 */
-                ctxt->src.val = reg_read(ctxt, VCPU_REGS_RCX);
-                rc = em_grp2(ctxt);
-                break;
         case 0xe9: /* jmp rel */
         case 0xeb: /* jmp rel short */
                 jmp_rel(ctxt, ctxt->src.val);
@@ -4661,14 +4800,6 @@ twobyte_insn:
         case 0x90 ... 0x9f:     /* setcc r/m8 */
                 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
                 break;
-        case 0xa4: /* shld imm8, r, r/m */
-        case 0xa5: /* shld cl, r, r/m */
-                emulate_2op_cl(ctxt, "shld");
-                break;
-        case 0xac: /* shrd imm8, r, r/m */
-        case 0xad: /* shrd cl, r, r/m */
-                emulate_2op_cl(ctxt, "shrd");
-                break;
         case 0xae:              /* clflush */
                 break;
         case 0xb6 ... 0xb7:     /* movzx */
@@ -4682,7 +4813,7 @@ twobyte_insn:
                                                         (s16) ctxt->src.val;
                 break;
         case 0xc0 ... 0xc1:     /* xadd */
-                emulate_2op_SrcV(ctxt, "add");
+                fastop(ctxt, em_add);
                 /* Write back the register source. */
                 ctxt->src.val = ctxt->dst.orig_val;
                 write_register_operand(&ctxt->src);
diff --git a/arch/x86/kvm/i8254.c b/arch/x86/kvm/i8254.c
index 11300d2fa714..c1d30b2fc9bb 100644
--- a/arch/x86/kvm/i8254.c
+++ b/arch/x86/kvm/i8254.c
@@ -122,7 +122,6 @@ static s64 __kpit_elapsed(struct kvm *kvm)
          */
         remaining = hrtimer_get_remaining(&ps->timer);
         elapsed = ps->period - ktime_to_ns(remaining);
-        elapsed = mod_64(elapsed, ps->period);
 
         return elapsed;
 }
diff --git a/arch/x86/kvm/i8259.c b/arch/x86/kvm/i8259.c
index 848206df0967..cc31f7c06d3d 100644
--- a/arch/x86/kvm/i8259.c
+++ b/arch/x86/kvm/i8259.c
@@ -241,6 +241,8 @@ int kvm_pic_read_irq(struct kvm *kvm)
         int irq, irq2, intno;
         struct kvm_pic *s = pic_irqchip(kvm);
 
+        s->output = 0;
+
         pic_lock(s);
         irq = pic_get_irq(&s->pics[0]);
         if (irq >= 0) {
diff --git a/arch/x86/kvm/irq.c b/arch/x86/kvm/irq.c
index 7e06ba1618bd..484bc874688b 100644
--- a/arch/x86/kvm/irq.c
+++ b/arch/x86/kvm/irq.c
@@ -38,49 +38,81 @@ int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
 EXPORT_SYMBOL(kvm_cpu_has_pending_timer);
 
 /*
+ * check if there is pending interrupt from
+ * non-APIC source without intack.
+ */
+static int kvm_cpu_has_extint(struct kvm_vcpu *v)
+{
+        if (kvm_apic_accept_pic_intr(v))
+                return pic_irqchip(v->kvm)->output;     /* PIC */
+        else
+                return 0;
+}
+
+/*
+ * check if there is injectable interrupt:
+ * when virtual interrupt delivery enabled,
+ * interrupt from apic will handled by hardware,
+ * we don't need to check it here.
+ */
+int kvm_cpu_has_injectable_intr(struct kvm_vcpu *v)
+{
+        if (!irqchip_in_kernel(v->kvm))
+                return v->arch.interrupt.pending;
+
+        if (kvm_cpu_has_extint(v))
+                return 1;
+
+        if (kvm_apic_vid_enabled(v->kvm))
+                return 0;
+
+        return kvm_apic_has_interrupt(v) != -1; /* LAPIC */
+}
+
+/*
  * check if there is pending interrupt without
  * intack.
  */
 int kvm_cpu_has_interrupt(struct kvm_vcpu *v)
 {
-        struct kvm_pic *s;
-
         if (!irqchip_in_kernel(v->kvm))
                 return v->arch.interrupt.pending;
 
-        if (kvm_apic_has_interrupt(v) == -1) {  /* LAPIC */
-                if (kvm_apic_accept_pic_intr(v)) {
-                        s = pic_irqchip(v->kvm);        /* PIC */
-                        return s->output;
-                } else
-                        return 0;
-        }
-        return 1;
+        if (kvm_cpu_has_extint(v))
+                return 1;
+
+        return kvm_apic_has_interrupt(v) != -1; /* LAPIC */
 }
 EXPORT_SYMBOL_GPL(kvm_cpu_has_interrupt);
 
 /*
+ * Read pending interrupt(from non-APIC source)
+ * vector and intack.
+ */
+static int kvm_cpu_get_extint(struct kvm_vcpu *v)
+{
+        if (kvm_cpu_has_extint(v))
+                return kvm_pic_read_irq(v->kvm); /* PIC */
+        return -1;
+}
+
+/*
  * Read pending interrupt vector and intack.
  */
 int kvm_cpu_get_interrupt(struct kvm_vcpu *v)
 {
-        struct kvm_pic *s;
         int vector;
 
         if (!irqchip_in_kernel(v->kvm))
                 return v->arch.interrupt.nr;
 
-        vector = kvm_get_apic_interrupt(v);     /* APIC */
-        if (vector == -1) {
-                if (kvm_apic_accept_pic_intr(v)) {
-                        s = pic_irqchip(v->kvm);
-                        s->output = 0;          /* PIC */
-                        vector = kvm_pic_read_irq(v->kvm);
-                }
-        }
-        return vector;
+        vector = kvm_cpu_get_extint(v);
+
+        if (kvm_apic_vid_enabled(v->kvm) || vector != -1)
+                return vector;                  /* PIC */
+
+        return kvm_get_apic_interrupt(v);       /* APIC */
 }
-EXPORT_SYMBOL_GPL(kvm_cpu_get_interrupt);
 
 void kvm_inject_pending_timer_irqs(struct kvm_vcpu *vcpu)
 {
diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c
index 9392f527f107..02b51dd4e4ad 100644
--- a/arch/x86/kvm/lapic.c
+++ b/arch/x86/kvm/lapic.c
@@ -140,31 +140,56 @@ static inline int apic_enabled(struct kvm_lapic *apic)
         (LVT_MASK | APIC_MODE_MASK | APIC_INPUT_POLARITY | \
          APIC_LVT_REMOTE_IRR | APIC_LVT_LEVEL_TRIGGER)
 
-static inline int apic_x2apic_mode(struct kvm_lapic *apic)
-{
-        return apic->vcpu->arch.apic_base & X2APIC_ENABLE;
-}
-
 static inline int kvm_apic_id(struct kvm_lapic *apic)
 {
         return (kvm_apic_get_reg(apic, APIC_ID) >> 24) & 0xff;
 }
 
-static inline u16 apic_cluster_id(struct kvm_apic_map *map, u32 ldr)
+void kvm_calculate_eoi_exitmap(struct kvm_vcpu *vcpu,
+                                struct kvm_lapic_irq *irq,
+                                u64 *eoi_exit_bitmap)
 {
-        u16 cid;
-        ldr >>= 32 - map->ldr_bits;
-        cid = (ldr >> map->cid_shift) & map->cid_mask;
+        struct kvm_lapic **dst;
+        struct kvm_apic_map *map;
+        unsigned long bitmap = 1;
+        int i;
 
-        BUG_ON(cid >= ARRAY_SIZE(map->logical_map));
+        rcu_read_lock();
+        map = rcu_dereference(vcpu->kvm->arch.apic_map);
 
-        return cid;
-}
+        if (unlikely(!map)) {
+                __set_bit(irq->vector, (unsigned long *)eoi_exit_bitmap);
+                goto out;
+        }
 
-static inline u16 apic_logical_id(struct kvm_apic_map *map, u32 ldr)
-{
-        ldr >>= (32 - map->ldr_bits);
-        return ldr & map->lid_mask;
+        if (irq->dest_mode == 0) { /* physical mode */
+                if (irq->delivery_mode == APIC_DM_LOWEST ||
+                                irq->dest_id == 0xff) {
+                        __set_bit(irq->vector,
+                                  (unsigned long *)eoi_exit_bitmap);
+                        goto out;
+                }
+                dst = &map->phys_map[irq->dest_id & 0xff];
+        } else {
+                u32 mda = irq->dest_id << (32 - map->ldr_bits);
+
+                dst = map->logical_map[apic_cluster_id(map, mda)];
+
+                bitmap = apic_logical_id(map, mda);
+        }
+
+        for_each_set_bit(i, &bitmap, 16) {
+                if (!dst[i])
+                        continue;
+                if (dst[i]->vcpu == vcpu) {
+                        __set_bit(irq->vector,
+                                  (unsigned long *)eoi_exit_bitmap);
+                        break;
+                }
+        }
+
+out:
+        rcu_read_unlock();
 }
 
 static void recalculate_apic_map(struct kvm *kvm)
@@ -230,6 +255,8 @@ out:
 
         if (old)
                 kfree_rcu(old, rcu);
+
+        kvm_ioapic_make_eoibitmap_request(kvm);
 }
 
 static inline void kvm_apic_set_id(struct kvm_lapic *apic, u8 id)
@@ -345,6 +372,10 @@ static inline int apic_find_highest_irr(struct kvm_lapic *apic)
 {
         int result;
 
+        /*
+         * Note that irr_pending is just a hint. It will be always
+         * true with virtual interrupt delivery enabled.
+         */
         if (!apic->irr_pending)
                 return -1;
 
@@ -461,6 +492,8 @@ static void pv_eoi_clr_pending(struct kvm_vcpu *vcpu)
 static inline int apic_find_highest_isr(struct kvm_lapic *apic)
 {
         int result;
+
+        /* Note that isr_count is always 1 with vid enabled */
         if (!apic->isr_count)
                 return -1;
         if (likely(apic->highest_isr_cache != -1))
@@ -740,6 +773,19 @@ int kvm_apic_compare_prio(struct kvm_vcpu *vcpu1, struct kvm_vcpu *vcpu2)
         return vcpu1->arch.apic_arb_prio - vcpu2->arch.apic_arb_prio;
 }
 
+static void kvm_ioapic_send_eoi(struct kvm_lapic *apic, int vector)
+{
+        if (!(kvm_apic_get_reg(apic, APIC_SPIV) & APIC_SPIV_DIRECTED_EOI) &&
+            kvm_ioapic_handles_vector(apic->vcpu->kvm, vector)) {
+                int trigger_mode;
+                if (apic_test_vector(vector, apic->regs + APIC_TMR))
+                        trigger_mode = IOAPIC_LEVEL_TRIG;
+                else
+                        trigger_mode = IOAPIC_EDGE_TRIG;
+                kvm_ioapic_update_eoi(apic->vcpu->kvm, vector, trigger_mode);
+        }
+}
+
 static int apic_set_eoi(struct kvm_lapic *apic)
 {
         int vector = apic_find_highest_isr(apic);
@@ -756,19 +802,26 @@ static int apic_set_eoi(struct kvm_lapic *apic)
         apic_clear_isr(vector, apic);
         apic_update_ppr(apic);
 
-        if (!(kvm_apic_get_reg(apic, APIC_SPIV) & APIC_SPIV_DIRECTED_EOI) &&
-            kvm_ioapic_handles_vector(apic->vcpu->kvm, vector)) {
-                int trigger_mode;
-                if (apic_test_vector(vector, apic->regs + APIC_TMR))
-                        trigger_mode = IOAPIC_LEVEL_TRIG;
-                else
-                        trigger_mode = IOAPIC_EDGE_TRIG;
-                kvm_ioapic_update_eoi(apic->vcpu->kvm, vector, trigger_mode);
-        }
+        kvm_ioapic_send_eoi(apic, vector);
         kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
         return vector;
 }
 
+/*
+ * this interface assumes a trap-like exit, which has already finished
+ * desired side effect including vISR and vPPR update.
+ */
+void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, int vector)
+{
+        struct kvm_lapic *apic = vcpu->arch.apic;
+
+        trace_kvm_eoi(apic, vector);
+
+        kvm_ioapic_send_eoi(apic, vector);
+        kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
+}
+EXPORT_SYMBOL_GPL(kvm_apic_set_eoi_accelerated);
+
 static void apic_send_ipi(struct kvm_lapic *apic)
 {
         u32 icr_low = kvm_apic_get_reg(apic, APIC_ICR);
@@ -1212,6 +1265,21 @@ void kvm_lapic_set_eoi(struct kvm_vcpu *vcpu)
 }
 EXPORT_SYMBOL_GPL(kvm_lapic_set_eoi);
 
+/* emulate APIC access in a trap manner */
+void kvm_apic_write_nodecode(struct kvm_vcpu *vcpu, u32 offset)
+{
+        u32 val = 0;
+
+        /* hw has done the conditional check and inst decode */
+        offset &= 0xff0;
+
+        apic_reg_read(vcpu->arch.apic, offset, 4, &val);
+
+        /* TODO: optimize to just emulate side effect w/o one more write */
+        apic_reg_write(vcpu->arch.apic, offset, val);
+}
+EXPORT_SYMBOL_GPL(kvm_apic_write_nodecode);
+
 void kvm_free_lapic(struct kvm_vcpu *vcpu)
 {
         struct kvm_lapic *apic = vcpu->arch.apic;
@@ -1288,6 +1356,7 @@ u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu)
 
 void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value)
 {
+        u64 old_value = vcpu->arch.apic_base;
         struct kvm_lapic *apic = vcpu->arch.apic;
 
         if (!apic) {
@@ -1309,11 +1378,16 @@ void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value)
                 value &= ~MSR_IA32_APICBASE_BSP;
 
         vcpu->arch.apic_base = value;
-        if (apic_x2apic_mode(apic)) {
-                u32 id = kvm_apic_id(apic);
-                u32 ldr = ((id >> 4) << 16) | (1 << (id & 0xf));
-                kvm_apic_set_ldr(apic, ldr);
+        if ((old_value ^ value) & X2APIC_ENABLE) {
+                if (value & X2APIC_ENABLE) {
+                        u32 id = kvm_apic_id(apic);
+                        u32 ldr = ((id >> 4) << 16) | (1 << (id & 0xf));
+                        kvm_apic_set_ldr(apic, ldr);
+                        kvm_x86_ops->set_virtual_x2apic_mode(vcpu, true);
+                } else
+                        kvm_x86_ops->set_virtual_x2apic_mode(vcpu, false);
         }
+
         apic->base_address = apic->vcpu->arch.apic_base &
                              MSR_IA32_APICBASE_BASE;
 
@@ -1359,8 +1433,8 @@ void kvm_lapic_reset(struct kvm_vcpu *vcpu)
                 apic_set_reg(apic, APIC_ISR + 0x10 * i, 0);
                 apic_set_reg(apic, APIC_TMR + 0x10 * i, 0);
         }
-        apic->irr_pending = false;
-        apic->isr_count = 0;
+        apic->irr_pending = kvm_apic_vid_enabled(vcpu->kvm);
+        apic->isr_count = kvm_apic_vid_enabled(vcpu->kvm);
         apic->highest_isr_cache = -1;
         update_divide_count(apic);
         atomic_set(&apic->lapic_timer.pending, 0);
@@ -1575,8 +1649,10 @@ void kvm_apic_post_state_restore(struct kvm_vcpu *vcpu,
         update_divide_count(apic);
         start_apic_timer(apic);
         apic->irr_pending = true;
-        apic->isr_count = count_vectors(apic->regs + APIC_ISR);
+        apic->isr_count = kvm_apic_vid_enabled(vcpu->kvm) ?
+                                1 : count_vectors(apic->regs + APIC_ISR);
         apic->highest_isr_cache = -1;
+        kvm_x86_ops->hwapic_isr_update(vcpu->kvm, apic_find_highest_isr(apic));
         kvm_make_request(KVM_REQ_EVENT, vcpu);
 }
 
diff --git a/arch/x86/kvm/lapic.h b/arch/x86/kvm/lapic.h
index e5ebf9f3571f..1676d34ddb4e 100644
--- a/arch/x86/kvm/lapic.h
+++ b/arch/x86/kvm/lapic.h
@@ -64,6 +64,9 @@ int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu);
 u64 kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu);
 void kvm_set_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu, u64 data);
 
+void kvm_apic_write_nodecode(struct kvm_vcpu *vcpu, u32 offset);
+void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, int vector);
+
 void kvm_lapic_set_vapic_addr(struct kvm_vcpu *vcpu, gpa_t vapic_addr);
 void kvm_lapic_sync_from_vapic(struct kvm_vcpu *vcpu);
 void kvm_lapic_sync_to_vapic(struct kvm_vcpu *vcpu);
@@ -124,4 +127,35 @@ static inline int kvm_lapic_enabled(struct kvm_vcpu *vcpu)
         return kvm_apic_present(vcpu) && kvm_apic_sw_enabled(vcpu->arch.apic);
 }
 
+static inline int apic_x2apic_mode(struct kvm_lapic *apic)
+{
+        return apic->vcpu->arch.apic_base & X2APIC_ENABLE;
+}
+
+static inline bool kvm_apic_vid_enabled(struct kvm *kvm)
+{
+        return kvm_x86_ops->vm_has_apicv(kvm);
+}
+
+static inline u16 apic_cluster_id(struct kvm_apic_map *map, u32 ldr)
+{
+        u16 cid;
+        ldr >>= 32 - map->ldr_bits;
+        cid = (ldr >> map->cid_shift) & map->cid_mask;
+
+        BUG_ON(cid >= ARRAY_SIZE(map->logical_map));
+
+        return cid;
+}
+
+static inline u16 apic_logical_id(struct kvm_apic_map *map, u32 ldr)
+{
+        ldr >>= (32 - map->ldr_bits);
+        return ldr & map->lid_mask;
+}
+
+void kvm_calculate_eoi_exitmap(struct kvm_vcpu *vcpu,
+                                struct kvm_lapic_irq *irq,
+                                u64 *eoi_bitmap);
+
 #endif
diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c
index 01d7c2ad05f5..956ca358108a 100644
--- a/arch/x86/kvm/mmu.c
+++ b/arch/x86/kvm/mmu.c
@@ -448,7 +448,8 @@ static bool __check_direct_spte_mmio_pf(u64 spte)
 
 static bool spte_is_locklessly_modifiable(u64 spte)
 {
-        return !(~spte & (SPTE_HOST_WRITEABLE | SPTE_MMU_WRITEABLE));
+        return (spte & (SPTE_HOST_WRITEABLE | SPTE_MMU_WRITEABLE)) ==
+                (SPTE_HOST_WRITEABLE | SPTE_MMU_WRITEABLE);
 }
 
 static bool spte_has_volatile_bits(u64 spte)
@@ -831,8 +832,7 @@ static int mapping_level(struct kvm_vcpu *vcpu, gfn_t large_gfn)
         if (host_level == PT_PAGE_TABLE_LEVEL)
                 return host_level;
 
-        max_level = kvm_x86_ops->get_lpage_level() < host_level ?
-                kvm_x86_ops->get_lpage_level() : host_level;
+        max_level = min(kvm_x86_ops->get_lpage_level(), host_level);
 
         for (level = PT_DIRECTORY_LEVEL; level <= max_level; ++level)
                 if (has_wrprotected_page(vcpu->kvm, large_gfn, level))
@@ -1142,7 +1142,7 @@ spte_write_protect(struct kvm *kvm, u64 *sptep, bool *flush, bool pt_protect)
 }
 
 static bool __rmap_write_protect(struct kvm *kvm, unsigned long *rmapp,
-                                 int level, bool pt_protect)
+                                 bool pt_protect)
 {
         u64 *sptep;
         struct rmap_iterator iter;
@@ -1180,7 +1180,7 @@ void kvm_mmu_write_protect_pt_masked(struct kvm *kvm,
         while (mask) {
                 rmapp = __gfn_to_rmap(slot->base_gfn + gfn_offset + __ffs(mask),
                                       PT_PAGE_TABLE_LEVEL, slot);
-                __rmap_write_protect(kvm, rmapp, PT_PAGE_TABLE_LEVEL, false);
+                __rmap_write_protect(kvm, rmapp, false);
 
                 /* clear the first set bit */
                 mask &= mask - 1;
@@ -1199,7 +1199,7 @@ static bool rmap_write_protect(struct kvm *kvm, u64 gfn)
         for (i = PT_PAGE_TABLE_LEVEL;
              i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
                 rmapp = __gfn_to_rmap(gfn, i, slot);
-                write_protected |= __rmap_write_protect(kvm, rmapp, i, true);
+                write_protected |= __rmap_write_protect(kvm, rmapp, true);
         }
 
         return write_protected;
@@ -1460,28 +1460,14 @@ static inline void kvm_mod_used_mmu_pages(struct kvm *kvm, int nr)
         percpu_counter_add(&kvm_total_used_mmu_pages, nr);
 }
 
-/*
- * Remove the sp from shadow page cache, after call it,
- * we can not find this sp from the cache, and the shadow
- * page table is still valid.
- * It should be under the protection of mmu lock.
- */
-static void kvm_mmu_isolate_page(struct kvm_mmu_page *sp)
+static void kvm_mmu_free_page(struct kvm_mmu_page *sp)
 {
         ASSERT(is_empty_shadow_page(sp->spt));
         hlist_del(&sp->hash_link);
-        if (!sp->role.direct)
-                free_page((unsigned long)sp->gfns);
-}
-
-/*
- * Free the shadow page table and the sp, we can do it
- * out of the protection of mmu lock.
- */
-static void kvm_mmu_free_page(struct kvm_mmu_page *sp)
-{
         list_del(&sp->link);
         free_page((unsigned long)sp->spt);
+        if (!sp->role.direct)
+                free_page((unsigned long)sp->gfns);
         kmem_cache_free(mmu_page_header_cache, sp);
 }
 
@@ -1522,7 +1508,6 @@ static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu,
                 sp->gfns = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache);
         set_page_private(virt_to_page(sp->spt), (unsigned long)sp);
         list_add(&sp->link, &vcpu->kvm->arch.active_mmu_pages);
-        bitmap_zero(sp->slot_bitmap, KVM_MEM_SLOTS_NUM);
         sp->parent_ptes = 0;
         mmu_page_add_parent_pte(vcpu, sp, parent_pte);
         kvm_mod_used_mmu_pages(vcpu->kvm, +1);
@@ -1659,13 +1644,13 @@ static int kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp,
 static void kvm_mmu_commit_zap_page(struct kvm *kvm,
                                     struct list_head *invalid_list);
 
-#define for_each_gfn_sp(kvm, sp, gfn, pos)                              \
-  hlist_for_each_entry(sp, pos,                                         \
+#define for_each_gfn_sp(kvm, sp, gfn)                                   \
+  hlist_for_each_entry(sp,                                              \
    &(kvm)->arch.mmu_page_hash[kvm_page_table_hashfn(gfn)], hash_link)   \
         if ((sp)->gfn != (gfn)) {} else
 
-#define for_each_gfn_indirect_valid_sp(kvm, sp, gfn, pos)               \
-  hlist_for_each_entry(sp, pos,                                         \
+#define for_each_gfn_indirect_valid_sp(kvm, sp, gfn)                    \
+  hlist_for_each_entry(sp,                                              \
    &(kvm)->arch.mmu_page_hash[kvm_page_table_hashfn(gfn)], hash_link)   \
                 if ((sp)->gfn != (gfn) || (sp)->role.direct ||          \
                         (sp)->role.invalid) {} else
@@ -1721,11 +1706,10 @@ static int kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
 static void kvm_sync_pages(struct kvm_vcpu *vcpu,  gfn_t gfn)
 {
         struct kvm_mmu_page *s;
-        struct hlist_node *node;
         LIST_HEAD(invalid_list);
         bool flush = false;
 
-        for_each_gfn_indirect_valid_sp(vcpu->kvm, s, gfn, node) {
+        for_each_gfn_indirect_valid_sp(vcpu->kvm, s, gfn) {
                 if (!s->unsync)
                         continue;
 
@@ -1863,7 +1847,6 @@ static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu,
         union kvm_mmu_page_role role;
         unsigned quadrant;
         struct kvm_mmu_page *sp;
-        struct hlist_node *node;
         bool need_sync = false;
 
         role = vcpu->arch.mmu.base_role;
@@ -1878,7 +1861,7 @@ static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu,
                 quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1;
                 role.quadrant = quadrant;
         }
-        for_each_gfn_sp(vcpu->kvm, sp, gfn, node) {
+        for_each_gfn_sp(vcpu->kvm, sp, gfn) {
                 if (!need_sync && sp->unsync)
                         need_sync = true;
 
@@ -1973,9 +1956,9 @@ static void link_shadow_page(u64 *sptep, struct kvm_mmu_page *sp)
 {
         u64 spte;
 
-        spte = __pa(sp->spt)
-                | PT_PRESENT_MASK | PT_ACCESSED_MASK
-                | PT_WRITABLE_MASK | PT_USER_MASK;
+        spte = __pa(sp->spt) | PT_PRESENT_MASK | PT_WRITABLE_MASK |
+               shadow_user_mask | shadow_x_mask | shadow_accessed_mask;
+
         mmu_spte_set(sptep, spte);
 }
 
@@ -2126,7 +2109,6 @@ static void kvm_mmu_commit_zap_page(struct kvm *kvm,
         do {
                 sp = list_first_entry(invalid_list, struct kvm_mmu_page, link);
                 WARN_ON(!sp->role.invalid || sp->root_count);
-                kvm_mmu_isolate_page(sp);
                 kvm_mmu_free_page(sp);
         } while (!list_empty(invalid_list));
 }
@@ -2144,6 +2126,8 @@ void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int goal_nr_mmu_pages)
          * change the value
          */
 
+        spin_lock(&kvm->mmu_lock);
+
         if (kvm->arch.n_used_mmu_pages > goal_nr_mmu_pages) {
                 while (kvm->arch.n_used_mmu_pages > goal_nr_mmu_pages &&
                         !list_empty(&kvm->arch.active_mmu_pages)) {
@@ -2158,19 +2142,20 @@ void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int goal_nr_mmu_pages)
         }
 
         kvm->arch.n_max_mmu_pages = goal_nr_mmu_pages;
+
+        spin_unlock(&kvm->mmu_lock);
 }
 
 int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn)
 {
         struct kvm_mmu_page *sp;
-        struct hlist_node *node;
         LIST_HEAD(invalid_list);
         int r;
 
         pgprintk("%s: looking for gfn %llx\n", __func__, gfn);
         r = 0;
         spin_lock(&kvm->mmu_lock);
-        for_each_gfn_indirect_valid_sp(kvm, sp, gfn, node) {
+        for_each_gfn_indirect_valid_sp(kvm, sp, gfn) {
                 pgprintk("%s: gfn %llx role %x\n", __func__, gfn,
                          sp->role.word);
                 r = 1;
@@ -2183,14 +2168,6 @@ int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn)
 }
 EXPORT_SYMBOL_GPL(kvm_mmu_unprotect_page);
 
-static void page_header_update_slot(struct kvm *kvm, void *pte, gfn_t gfn)
-{
-        int slot = memslot_id(kvm, gfn);
-        struct kvm_mmu_page *sp = page_header(__pa(pte));
-
-        __set_bit(slot, sp->slot_bitmap);
-}
-
 /*
  * The function is based on mtrr_type_lookup() in
  * arch/x86/kernel/cpu/mtrr/generic.c
@@ -2308,9 +2285,8 @@ static void __kvm_unsync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
 static void kvm_unsync_pages(struct kvm_vcpu *vcpu,  gfn_t gfn)
 {
         struct kvm_mmu_page *s;
-        struct hlist_node *node;
 
-        for_each_gfn_indirect_valid_sp(vcpu->kvm, s, gfn, node) {
+        for_each_gfn_indirect_valid_sp(vcpu->kvm, s, gfn) {
                 if (s->unsync)
                         continue;
                 WARN_ON(s->role.level != PT_PAGE_TABLE_LEVEL);
@@ -2322,19 +2298,17 @@ static int mmu_need_write_protect(struct kvm_vcpu *vcpu, gfn_t gfn,
                                   bool can_unsync)
 {
         struct kvm_mmu_page *s;
-        struct hlist_node *node;
         bool need_unsync = false;
 
-        for_each_gfn_indirect_valid_sp(vcpu->kvm, s, gfn, node) {
+        for_each_gfn_indirect_valid_sp(vcpu->kvm, s, gfn) {
                 if (!can_unsync)
                         return 1;
 
                 if (s->role.level != PT_PAGE_TABLE_LEVEL)
                         return 1;
 
-                if (!need_unsync && !s->unsync) {
+                if (!s->unsync)
                         need_unsync = true;
-                }
         }
         if (need_unsync)
                 kvm_unsync_pages(vcpu, gfn);
@@ -2342,8 +2316,7 @@ static int mmu_need_write_protect(struct kvm_vcpu *vcpu, gfn_t gfn,
 }
 
 static int set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
-                    unsigned pte_access, int user_fault,
-                    int write_fault, int level,
+                    unsigned pte_access, int level,
                     gfn_t gfn, pfn_t pfn, bool speculative,
                     bool can_unsync, bool host_writable)
 {
@@ -2378,20 +2351,13 @@ static int set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
 
         spte |= (u64)pfn << PAGE_SHIFT;
 
-        if ((pte_access & ACC_WRITE_MASK)
-            || (!vcpu->arch.mmu.direct_map && write_fault
-                && !is_write_protection(vcpu) && !user_fault)) {
+        if (pte_access & ACC_WRITE_MASK) {
 
                 /*
-                 * There are two cases:
-                 * - the one is other vcpu creates new sp in the window
-                 *   between mapping_level() and acquiring mmu-lock.
-                 * - the another case is the new sp is created by itself
-                 *   (page-fault path) when guest uses the target gfn as
-                 *   its page table.
-                 * Both of these cases can be fixed by allowing guest to
-                 * retry the access, it will refault, then we can establish
-                 * the mapping by using small page.
+                 * Other vcpu creates new sp in the window between
+                 * mapping_level() and acquiring mmu-lock. We can
+                 * allow guest to retry the access, the mapping can
+                 * be fixed if guest refault.
                  */
                 if (level > PT_PAGE_TABLE_LEVEL &&
                     has_wrprotected_page(vcpu->kvm, gfn, level))
@@ -2399,19 +2365,6 @@ static int set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
 
                 spte |= PT_WRITABLE_MASK | SPTE_MMU_WRITEABLE;
 
-                if (!vcpu->arch.mmu.direct_map
-                    && !(pte_access & ACC_WRITE_MASK)) {
-                        spte &= ~PT_USER_MASK;
-                        /*
-                         * If we converted a user page to a kernel page,
-                         * so that the kernel can write to it when cr0.wp=0,
-                         * then we should prevent the kernel from executing it
-                         * if SMEP is enabled.
-                         */
-                        if (kvm_read_cr4_bits(vcpu, X86_CR4_SMEP))
-                                spte |= PT64_NX_MASK;
-                }
-
                 /*
                  * Optimization: for pte sync, if spte was writable the hash
                  * lookup is unnecessary (and expensive). Write protection
@@ -2441,19 +2394,15 @@ done:
 }
 
 static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
-                         unsigned pt_access, unsigned pte_access,
-                         int user_fault, int write_fault,
-                         int *emulate, int level, gfn_t gfn,
-                         pfn_t pfn, bool speculative,
+                         unsigned pte_access, int write_fault, int *emulate,
+                         int level, gfn_t gfn, pfn_t pfn, bool speculative,
                          bool host_writable)
 {
         int was_rmapped = 0;
         int rmap_count;
 
-        pgprintk("%s: spte %llx access %x write_fault %d"
-                 " user_fault %d gfn %llx\n",
-                 __func__, *sptep, pt_access,
-                 write_fault, user_fault, gfn);
+        pgprintk("%s: spte %llx write_fault %d gfn %llx\n", __func__,
+                 *sptep, write_fault, gfn);
 
         if (is_rmap_spte(*sptep)) {
                 /*
@@ -2477,9 +2426,8 @@ static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
                         was_rmapped = 1;
         }
 
-        if (set_spte(vcpu, sptep, pte_access, user_fault, write_fault,
-                      level, gfn, pfn, speculative, true,
-                      host_writable)) {
+        if (set_spte(vcpu, sptep, pte_access, level, gfn, pfn, speculative,
+              true, host_writable)) {
                 if (write_fault)
                         *emulate = 1;
                 kvm_mmu_flush_tlb(vcpu);
@@ -2497,7 +2445,6 @@ static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
                 ++vcpu->kvm->stat.lpages;
 
         if (is_shadow_present_pte(*sptep)) {
-                page_header_update_slot(vcpu->kvm, sptep, gfn);
                 if (!was_rmapped) {
                         rmap_count = rmap_add(vcpu, sptep, gfn);
                         if (rmap_count > RMAP_RECYCLE_THRESHOLD)
@@ -2571,10 +2518,9 @@ static int direct_pte_prefetch_many(struct kvm_vcpu *vcpu,
                 return -1;
 
         for (i = 0; i < ret; i++, gfn++, start++)
-                mmu_set_spte(vcpu, start, ACC_ALL,
-                             access, 0, 0, NULL,
-                             sp->role.level, gfn,
-                             page_to_pfn(pages[i]), true, true);
+                mmu_set_spte(vcpu, start, access, 0, NULL,
+                             sp->role.level, gfn, page_to_pfn(pages[i]),
+                             true, true);
 
         return 0;
 }
@@ -2633,11 +2579,9 @@ static int __direct_map(struct kvm_vcpu *vcpu, gpa_t v, int write,
 
         for_each_shadow_entry(vcpu, (u64)gfn << PAGE_SHIFT, iterator) {
                 if (iterator.level == level) {
-                        unsigned pte_access = ACC_ALL;
-
-                        mmu_set_spte(vcpu, iterator.sptep, ACC_ALL, pte_access,
-                                     0, write, &emulate,
-                                     level, gfn, pfn, prefault, map_writable);
+                        mmu_set_spte(vcpu, iterator.sptep, ACC_ALL,
+                                     write, &emulate, level, gfn, pfn,
+                                     prefault, map_writable);
                         direct_pte_prefetch(vcpu, iterator.sptep);
                         ++vcpu->stat.pf_fixed;
                         break;
@@ -2652,11 +2596,7 @@ static int __direct_map(struct kvm_vcpu *vcpu, gpa_t v, int write,
                                               iterator.level - 1,
                                               1, ACC_ALL, iterator.sptep);
 
-                        mmu_spte_set(iterator.sptep,
-                                     __pa(sp->spt)
-                                     | PT_PRESENT_MASK | PT_WRITABLE_MASK
-                                     | shadow_user_mask | shadow_x_mask
-                                     | shadow_accessed_mask);
+                        link_shadow_page(iterator.sptep, sp);
                 }
         }
         return emulate;
@@ -3719,6 +3659,7 @@ int kvm_init_shadow_mmu(struct kvm_vcpu *vcpu, struct kvm_mmu *context)
         else
                 r = paging32_init_context(vcpu, context);
 
+        vcpu->arch.mmu.base_role.nxe = is_nx(vcpu);
         vcpu->arch.mmu.base_role.cr4_pae = !!is_pae(vcpu);
         vcpu->arch.mmu.base_role.cr0_wp  = is_write_protection(vcpu);
         vcpu->arch.mmu.base_role.smep_andnot_wp
@@ -3885,7 +3826,7 @@ static u64 mmu_pte_write_fetch_gpte(struct kvm_vcpu *vcpu, gpa_t *gpa,
                 /* Handle a 32-bit guest writing two halves of a 64-bit gpte */
                 *gpa &= ~(gpa_t)7;
                 *bytes = 8;
-                r = kvm_read_guest(vcpu->kvm, *gpa, &gentry, min(*bytes, 8));
+                r = kvm_read_guest(vcpu->kvm, *gpa, &gentry, 8);
                 if (r)
                         gentry = 0;
                 new = (const u8 *)&gentry;
@@ -3987,7 +3928,6 @@ void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
         gfn_t gfn = gpa >> PAGE_SHIFT;
         union kvm_mmu_page_role mask = { .word = 0 };
         struct kvm_mmu_page *sp;
-        struct hlist_node *node;
         LIST_HEAD(invalid_list);
         u64 entry, gentry, *spte;
         int npte;
@@ -4018,7 +3958,7 @@ void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
         kvm_mmu_audit(vcpu, AUDIT_PRE_PTE_WRITE);
 
         mask.cr0_wp = mask.cr4_pae = mask.nxe = 1;
-        for_each_gfn_indirect_valid_sp(vcpu->kvm, sp, gfn, node) {
+        for_each_gfn_indirect_valid_sp(vcpu->kvm, sp, gfn) {
                 if (detect_write_misaligned(sp, gpa, bytes) ||
                       detect_write_flooding(sp)) {
                         zap_page |= !!kvm_mmu_prepare_zap_page(vcpu->kvm, sp,
@@ -4039,7 +3979,7 @@ void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
                               !((sp->role.word ^ vcpu->arch.mmu.base_role.word)
                               & mask.word) && rmap_can_add(vcpu))
                                 mmu_pte_write_new_pte(vcpu, sp, spte, &gentry);
-                        if (!remote_flush && need_remote_flush(entry, *spte))
+                        if (need_remote_flush(entry, *spte))
                                 remote_flush = true;
                         ++spte;
                 }
@@ -4198,26 +4138,36 @@ int kvm_mmu_setup(struct kvm_vcpu *vcpu)
 
 void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot)
 {
-        struct kvm_mmu_page *sp;
-        bool flush = false;
+        struct kvm_memory_slot *memslot;
+        gfn_t last_gfn;
+        int i;
 
-        list_for_each_entry(sp, &kvm->arch.active_mmu_pages, link) {
-                int i;
-                u64 *pt;
+        memslot = id_to_memslot(kvm->memslots, slot);
+        last_gfn = memslot->base_gfn + memslot->npages - 1;
 
-                if (!test_bit(slot, sp->slot_bitmap))
-                        continue;
+        spin_lock(&kvm->mmu_lock);
 
-                pt = sp->spt;
-                for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
-                        if (!is_shadow_present_pte(pt[i]) ||
-                              !is_last_spte(pt[i], sp->role.level))
-                                continue;
+        for (i = PT_PAGE_TABLE_LEVEL;
+             i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
+                unsigned long *rmapp;
+                unsigned long last_index, index;
 
-                        spte_write_protect(kvm, &pt[i], &flush, false);
+                rmapp = memslot->arch.rmap[i - PT_PAGE_TABLE_LEVEL];
+                last_index = gfn_to_index(last_gfn, memslot->base_gfn, i);
+
+                for (index = 0; index <= last_index; ++index, ++rmapp) {
+                        if (*rmapp)
+                                __rmap_write_protect(kvm, rmapp, false);
+
+                        if (need_resched() || spin_needbreak(&kvm->mmu_lock)) {
+                                kvm_flush_remote_tlbs(kvm);
+                                cond_resched_lock(&kvm->mmu_lock);
+                        }
                 }
         }
+
         kvm_flush_remote_tlbs(kvm);
+        spin_unlock(&kvm->mmu_lock);
 }
 
 void kvm_mmu_zap_all(struct kvm *kvm)
diff --git a/arch/x86/kvm/mmutrace.h b/arch/x86/kvm/mmutrace.h
index cd6e98333ba3..b8f6172f4174 100644
--- a/arch/x86/kvm/mmutrace.h
+++ b/arch/x86/kvm/mmutrace.h
@@ -195,12 +195,6 @@ DEFINE_EVENT(kvm_mmu_page_class, kvm_mmu_prepare_zap_page,
         TP_ARGS(sp)
 );
 
-DEFINE_EVENT(kvm_mmu_page_class, kvm_mmu_delay_free_pages,
-        TP_PROTO(struct kvm_mmu_page *sp),
-
-        TP_ARGS(sp)
-);
-
 TRACE_EVENT(
         mark_mmio_spte,
         TP_PROTO(u64 *sptep, gfn_t gfn, unsigned access),
diff --git a/arch/x86/kvm/paging_tmpl.h b/arch/x86/kvm/paging_tmpl.h
index 891eb6d93b8b..105dd5bd550e 100644
--- a/arch/x86/kvm/paging_tmpl.h
+++ b/arch/x86/kvm/paging_tmpl.h
@@ -151,7 +151,7 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
         pt_element_t pte;
         pt_element_t __user *uninitialized_var(ptep_user);
         gfn_t table_gfn;
-        unsigned index, pt_access, pte_access, accessed_dirty, shift;
+        unsigned index, pt_access, pte_access, accessed_dirty;
         gpa_t pte_gpa;
         int offset;
         const int write_fault = access & PFERR_WRITE_MASK;
@@ -249,16 +249,12 @@ retry_walk:
 
         if (!write_fault)
                 protect_clean_gpte(&pte_access, pte);
-
-        /*
-         * On a write fault, fold the dirty bit into accessed_dirty by shifting it one
-         * place right.
-         *
-         * On a read fault, do nothing.
-         */
-        shift = write_fault >> ilog2(PFERR_WRITE_MASK);
-        shift *= PT_DIRTY_SHIFT - PT_ACCESSED_SHIFT;
-        accessed_dirty &= pte >> shift;
+        else
+                /*
+                 * On a write fault, fold the dirty bit into accessed_dirty by
+                 * shifting it one place right.
+                 */
+                accessed_dirty &= pte >> (PT_DIRTY_SHIFT - PT_ACCESSED_SHIFT);
 
         if (unlikely(!accessed_dirty)) {
                 ret = FNAME(update_accessed_dirty_bits)(vcpu, mmu, walker, write_fault);
@@ -330,8 +326,8 @@ FNAME(prefetch_gpte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
          * we call mmu_set_spte() with host_writable = true because
          * pte_prefetch_gfn_to_pfn always gets a writable pfn.
          */
-        mmu_set_spte(vcpu, spte, sp->role.access, pte_access, 0, 0,
-                     NULL, PT_PAGE_TABLE_LEVEL, gfn, pfn, true, true);
+        mmu_set_spte(vcpu, spte, pte_access, 0, NULL, PT_PAGE_TABLE_LEVEL,
+                     gfn, pfn, true, true);
 
         return true;
 }
@@ -405,7 +401,7 @@ static void FNAME(pte_prefetch)(struct kvm_vcpu *vcpu, struct guest_walker *gw,
  */
 static int FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr,
                          struct guest_walker *gw,
-                         int user_fault, int write_fault, int hlevel,
+                         int write_fault, int hlevel,
                          pfn_t pfn, bool map_writable, bool prefault)
 {
         struct kvm_mmu_page *sp = NULL;
@@ -413,9 +409,6 @@ static int FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr,
         unsigned direct_access, access = gw->pt_access;
         int top_level, emulate = 0;
 
-        if (!is_present_gpte(gw->ptes[gw->level - 1]))
-                return 0;
-
         direct_access = gw->pte_access;
 
         top_level = vcpu->arch.mmu.root_level;
@@ -477,9 +470,8 @@ static int FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr,
         }
 
         clear_sp_write_flooding_count(it.sptep);
-        mmu_set_spte(vcpu, it.sptep, access, gw->pte_access,
-                     user_fault, write_fault, &emulate, it.level,
-                     gw->gfn, pfn, prefault, map_writable);
+        mmu_set_spte(vcpu, it.sptep, gw->pte_access, write_fault, &emulate,
+                     it.level, gw->gfn, pfn, prefault, map_writable);
         FNAME(pte_prefetch)(vcpu, gw, it.sptep);
 
         return emulate;
@@ -491,6 +483,46 @@ out_gpte_changed:
         return 0;
 }
 
+ /*
+ * To see whether the mapped gfn can write its page table in the current
+ * mapping.
+ *
+ * It is the helper function of FNAME(page_fault). When guest uses large page
+ * size to map the writable gfn which is used as current page table, we should
+ * force kvm to use small page size to map it because new shadow page will be
+ * created when kvm establishes shadow page table that stop kvm using large
+ * page size. Do it early can avoid unnecessary #PF and emulation.
+ *
+ * @write_fault_to_shadow_pgtable will return true if the fault gfn is
+ * currently used as its page table.
+ *
+ * Note: the PDPT page table is not checked for PAE-32 bit guest. It is ok
+ * since the PDPT is always shadowed, that means, we can not use large page
+ * size to map the gfn which is used as PDPT.
+ */
+static bool
+FNAME(is_self_change_mapping)(struct kvm_vcpu *vcpu,
+                              struct guest_walker *walker, int user_fault,
+                              bool *write_fault_to_shadow_pgtable)
+{
+        int level;
+        gfn_t mask = ~(KVM_PAGES_PER_HPAGE(walker->level) - 1);
+        bool self_changed = false;
+
+        if (!(walker->pte_access & ACC_WRITE_MASK ||
+              (!is_write_protection(vcpu) && !user_fault)))
+                return false;
+
+        for (level = walker->level; level <= walker->max_level; level++) {
+                gfn_t gfn = walker->gfn ^ walker->table_gfn[level - 1];
+
+                self_changed |= !(gfn & mask);
+                *write_fault_to_shadow_pgtable |= !gfn;
+        }
+
+        return self_changed;
+}
+
 /*
  * Page fault handler.  There are several causes for a page fault:
  *   - there is no shadow pte for the guest pte
@@ -516,7 +548,7 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr, u32 error_code,
         int level = PT_PAGE_TABLE_LEVEL;
         int force_pt_level;
         unsigned long mmu_seq;
-        bool map_writable;
+        bool map_writable, is_self_change_mapping;
 
         pgprintk("%s: addr %lx err %x\n", __func__, addr, error_code);
 
@@ -544,8 +576,14 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr, u32 error_code,
                 return 0;
         }
 
+        vcpu->arch.write_fault_to_shadow_pgtable = false;
+
+        is_self_change_mapping = FNAME(is_self_change_mapping)(vcpu,
+              &walker, user_fault, &vcpu->arch.write_fault_to_shadow_pgtable);
+
         if (walker.level >= PT_DIRECTORY_LEVEL)
-                force_pt_level = mapping_level_dirty_bitmap(vcpu, walker.gfn);
+                force_pt_level = mapping_level_dirty_bitmap(vcpu, walker.gfn)
+                   || is_self_change_mapping;
         else
                 force_pt_level = 1;
         if (!force_pt_level) {
@@ -564,6 +602,26 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr, u32 error_code,
                                 walker.gfn, pfn, walker.pte_access, &r))
                 return r;
 
+        /*
+         * Do not change pte_access if the pfn is a mmio page, otherwise
+         * we will cache the incorrect access into mmio spte.
+         */
+        if (write_fault && !(walker.pte_access & ACC_WRITE_MASK) &&
+             !is_write_protection(vcpu) && !user_fault &&
+              !is_noslot_pfn(pfn)) {
+                walker.pte_access |= ACC_WRITE_MASK;
+                walker.pte_access &= ~ACC_USER_MASK;
+
+                /*
+                 * If we converted a user page to a kernel page,
+                 * so that the kernel can write to it when cr0.wp=0,
+                 * then we should prevent the kernel from executing it
+                 * if SMEP is enabled.
+                 */
+                if (kvm_read_cr4_bits(vcpu, X86_CR4_SMEP))
+                        walker.pte_access &= ~ACC_EXEC_MASK;
+        }
+
         spin_lock(&vcpu->kvm->mmu_lock);
         if (mmu_notifier_retry(vcpu->kvm, mmu_seq))
                 goto out_unlock;
@@ -572,7 +630,7 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr, u32 error_code,
         kvm_mmu_free_some_pages(vcpu);
         if (!force_pt_level)
                 transparent_hugepage_adjust(vcpu, &walker.gfn, &pfn, &level);
-        r = FNAME(fetch)(vcpu, addr, &walker, user_fault, write_fault,
+        r = FNAME(fetch)(vcpu, addr, &walker, write_fault,
                          level, pfn, map_writable, prefault);
         ++vcpu->stat.pf_fixed;
         kvm_mmu_audit(vcpu, AUDIT_POST_PAGE_FAULT);
@@ -747,7 +805,7 @@ static int FNAME(sync_page)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
 
                 host_writable = sp->spt[i] & SPTE_HOST_WRITEABLE;
 
-                set_spte(vcpu, &sp->spt[i], pte_access, 0, 0,
+                set_spte(vcpu, &sp->spt[i], pte_access,
                          PT_PAGE_TABLE_LEVEL, gfn,
                          spte_to_pfn(sp->spt[i]), true, false,
                          host_writable);
diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c
index d29d3cd1c156..e1b1ce21bc00 100644
--- a/arch/x86/kvm/svm.c
+++ b/arch/x86/kvm/svm.c
@@ -3571,6 +3571,26 @@ static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
                 set_cr_intercept(svm, INTERCEPT_CR8_WRITE);
 }
 
+static void svm_set_virtual_x2apic_mode(struct kvm_vcpu *vcpu, bool set)
+{
+        return;
+}
+
+static int svm_vm_has_apicv(struct kvm *kvm)
+{
+        return 0;
+}
+
+static void svm_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap)
+{
+        return;
+}
+
+static void svm_hwapic_isr_update(struct kvm *kvm, int isr)
+{
+        return;
+}
+
 static int svm_nmi_allowed(struct kvm_vcpu *vcpu)
 {
         struct vcpu_svm *svm = to_svm(vcpu);
@@ -4290,6 +4310,10 @@ static struct kvm_x86_ops svm_x86_ops = {
         .enable_nmi_window = enable_nmi_window,
         .enable_irq_window = enable_irq_window,
         .update_cr8_intercept = update_cr8_intercept,
+        .set_virtual_x2apic_mode = svm_set_virtual_x2apic_mode,
+        .vm_has_apicv = svm_vm_has_apicv,
+        .load_eoi_exitmap = svm_load_eoi_exitmap,
+        .hwapic_isr_update = svm_hwapic_isr_update,
 
         .set_tss_addr = svm_set_tss_addr,
         .get_tdp_level = get_npt_level,
diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index 9120ae1901e4..6667042714cc 100644
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -84,6 +84,8 @@ module_param(vmm_exclusive, bool, S_IRUGO);
 static bool __read_mostly fasteoi = 1;
 module_param(fasteoi, bool, S_IRUGO);
 
+static bool __read_mostly enable_apicv_reg_vid;
+
 /*
  * If nested=1, nested virtualization is supported, i.e., guests may use
  * VMX and be a hypervisor for its own guests. If nested=0, guests may not
@@ -92,12 +94,8 @@ module_param(fasteoi, bool, S_IRUGO);
 static bool __read_mostly nested = 0;
 module_param(nested, bool, S_IRUGO);
 
-#define KVM_GUEST_CR0_MASK_UNRESTRICTED_GUEST                           \
-        (X86_CR0_WP | X86_CR0_NE | X86_CR0_NW | X86_CR0_CD)
-#define KVM_GUEST_CR0_MASK                                              \
-        (KVM_GUEST_CR0_MASK_UNRESTRICTED_GUEST | X86_CR0_PG | X86_CR0_PE)
-#define KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST                         \
-        (X86_CR0_WP | X86_CR0_NE)
+#define KVM_GUEST_CR0_MASK (X86_CR0_NW | X86_CR0_CD)
+#define KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST (X86_CR0_WP | X86_CR0_NE)
 #define KVM_VM_CR0_ALWAYS_ON                                            \
         (KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST | X86_CR0_PG | X86_CR0_PE)
 #define KVM_CR4_GUEST_OWNED_BITS                                      \
@@ -624,6 +622,8 @@ static void vmx_set_segment(struct kvm_vcpu *vcpu,
                             struct kvm_segment *var, int seg);
 static void vmx_get_segment(struct kvm_vcpu *vcpu,
                             struct kvm_segment *var, int seg);
+static bool guest_state_valid(struct kvm_vcpu *vcpu);
+static u32 vmx_segment_access_rights(struct kvm_segment *var);
 
 static DEFINE_PER_CPU(struct vmcs *, vmxarea);
 static DEFINE_PER_CPU(struct vmcs *, current_vmcs);
@@ -638,6 +638,8 @@ static unsigned long *vmx_io_bitmap_a;
 static unsigned long *vmx_io_bitmap_b;
 static unsigned long *vmx_msr_bitmap_legacy;
 static unsigned long *vmx_msr_bitmap_longmode;
+static unsigned long *vmx_msr_bitmap_legacy_x2apic;
+static unsigned long *vmx_msr_bitmap_longmode_x2apic;
 
 static bool cpu_has_load_ia32_efer;
 static bool cpu_has_load_perf_global_ctrl;
@@ -762,6 +764,24 @@ static inline bool cpu_has_vmx_virtualize_apic_accesses(void)
                 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
 }
 
+static inline bool cpu_has_vmx_virtualize_x2apic_mode(void)
+{
+        return vmcs_config.cpu_based_2nd_exec_ctrl &
+                SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE;
+}
+
+static inline bool cpu_has_vmx_apic_register_virt(void)
+{
+        return vmcs_config.cpu_based_2nd_exec_ctrl &
+                SECONDARY_EXEC_APIC_REGISTER_VIRT;
+}
+
+static inline bool cpu_has_vmx_virtual_intr_delivery(void)
+{
+        return vmcs_config.cpu_based_2nd_exec_ctrl &
+                SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY;
+}
+
 static inline bool cpu_has_vmx_flexpriority(void)
 {
         return cpu_has_vmx_tpr_shadow() &&
@@ -1694,7 +1714,6 @@ static unsigned long vmx_get_rflags(struct kvm_vcpu *vcpu)
 static void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
 {
         __set_bit(VCPU_EXREG_RFLAGS, (ulong *)&vcpu->arch.regs_avail);
-        __clear_bit(VCPU_EXREG_CPL, (ulong *)&vcpu->arch.regs_avail);
         to_vmx(vcpu)->rflags = rflags;
         if (to_vmx(vcpu)->rmode.vm86_active) {
                 to_vmx(vcpu)->rmode.save_rflags = rflags;
@@ -1820,6 +1839,25 @@ static void move_msr_up(struct vcpu_vmx *vmx, int from, int to)
         vmx->guest_msrs[from] = tmp;
 }
 
+static void vmx_set_msr_bitmap(struct kvm_vcpu *vcpu)
+{
+        unsigned long *msr_bitmap;
+
+        if (irqchip_in_kernel(vcpu->kvm) && apic_x2apic_mode(vcpu->arch.apic)) {
+                if (is_long_mode(vcpu))
+                        msr_bitmap = vmx_msr_bitmap_longmode_x2apic;
+                else
+                        msr_bitmap = vmx_msr_bitmap_legacy_x2apic;
+        } else {
+                if (is_long_mode(vcpu))
+                        msr_bitmap = vmx_msr_bitmap_longmode;
+                else
+                        msr_bitmap = vmx_msr_bitmap_legacy;
+        }
+
+        vmcs_write64(MSR_BITMAP, __pa(msr_bitmap));
+}
+
 /*
  * Set up the vmcs to automatically save and restore system
  * msrs.  Don't touch the 64-bit msrs if the guest is in legacy
@@ -1828,7 +1866,6 @@ static void move_msr_up(struct vcpu_vmx *vmx, int from, int to)
 static void setup_msrs(struct vcpu_vmx *vmx)
 {
         int save_nmsrs, index;
-        unsigned long *msr_bitmap;
 
         save_nmsrs = 0;
 #ifdef CONFIG_X86_64
@@ -1860,14 +1897,8 @@ static void setup_msrs(struct vcpu_vmx *vmx)
 
         vmx->save_nmsrs = save_nmsrs;
 
-        if (cpu_has_vmx_msr_bitmap()) {
-                if (is_long_mode(&vmx->vcpu))
-                        msr_bitmap = vmx_msr_bitmap_longmode;
-                else
-                        msr_bitmap = vmx_msr_bitmap_legacy;
-
-                vmcs_write64(MSR_BITMAP, __pa(msr_bitmap));
-        }
+        if (cpu_has_vmx_msr_bitmap())
+                vmx_set_msr_bitmap(&vmx->vcpu);
 }
 
 /*
@@ -2533,13 +2564,16 @@ static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf)
         if (_cpu_based_exec_control & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) {
                 min2 = 0;
                 opt2 = SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
+                        SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
                         SECONDARY_EXEC_WBINVD_EXITING |
                         SECONDARY_EXEC_ENABLE_VPID |
                         SECONDARY_EXEC_ENABLE_EPT |
                         SECONDARY_EXEC_UNRESTRICTED_GUEST |
                         SECONDARY_EXEC_PAUSE_LOOP_EXITING |
                         SECONDARY_EXEC_RDTSCP |
-                        SECONDARY_EXEC_ENABLE_INVPCID;
+                        SECONDARY_EXEC_ENABLE_INVPCID |
+                        SECONDARY_EXEC_APIC_REGISTER_VIRT |
+                        SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY;
                 if (adjust_vmx_controls(min2, opt2,
                                         MSR_IA32_VMX_PROCBASED_CTLS2,
                                         &_cpu_based_2nd_exec_control) < 0)
@@ -2550,6 +2584,13 @@ static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf)
                                 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES))
                 _cpu_based_exec_control &= ~CPU_BASED_TPR_SHADOW;
 #endif
+
+        if (!(_cpu_based_exec_control & CPU_BASED_TPR_SHADOW))
+                _cpu_based_2nd_exec_control &= ~(
+                                SECONDARY_EXEC_APIC_REGISTER_VIRT |
+                                SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
+                                SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
+
         if (_cpu_based_2nd_exec_control & SECONDARY_EXEC_ENABLE_EPT) {
                 /* CR3 accesses and invlpg don't need to cause VM Exits when EPT
                    enabled */
@@ -2747,6 +2788,15 @@ static __init int hardware_setup(void)
         if (!cpu_has_vmx_ple())
                 ple_gap = 0;
 
+        if (!cpu_has_vmx_apic_register_virt() ||
+                                !cpu_has_vmx_virtual_intr_delivery())
+                enable_apicv_reg_vid = 0;
+
+        if (enable_apicv_reg_vid)
+                kvm_x86_ops->update_cr8_intercept = NULL;
+        else
+                kvm_x86_ops->hwapic_irr_update = NULL;
+
         if (nested)
                 nested_vmx_setup_ctls_msrs();
 
@@ -2758,18 +2808,28 @@ static __exit void hardware_unsetup(void)
         free_kvm_area();
 }
 
-static void fix_pmode_dataseg(struct kvm_vcpu *vcpu, int seg, struct kvm_segment *save)
+static bool emulation_required(struct kvm_vcpu *vcpu)
 {
-        const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
-        struct kvm_segment tmp = *save;
+        return emulate_invalid_guest_state && !guest_state_valid(vcpu);
+}
 
-        if (!(vmcs_readl(sf->base) == tmp.base && tmp.s)) {
-                tmp.base = vmcs_readl(sf->base);
-                tmp.selector = vmcs_read16(sf->selector);
-                tmp.dpl = tmp.selector & SELECTOR_RPL_MASK;
-                tmp.s = 1;
+static void fix_pmode_seg(struct kvm_vcpu *vcpu, int seg,
+                struct kvm_segment *save)
+{
+        if (!emulate_invalid_guest_state) {
+                /*
+                 * CS and SS RPL should be equal during guest entry according
+                 * to VMX spec, but in reality it is not always so. Since vcpu
+                 * is in the middle of the transition from real mode to
+                 * protected mode it is safe to assume that RPL 0 is a good
+                 * default value.
+                 */
+                if (seg == VCPU_SREG_CS || seg == VCPU_SREG_SS)
+                        save->selector &= ~SELECTOR_RPL_MASK;
+                save->dpl = save->selector & SELECTOR_RPL_MASK;
+                save->s = 1;
         }
-        vmx_set_segment(vcpu, &tmp, seg);
+        vmx_set_segment(vcpu, save, seg);
 }
 
 static void enter_pmode(struct kvm_vcpu *vcpu)
@@ -2777,7 +2837,17 @@ static void enter_pmode(struct kvm_vcpu *vcpu)
         unsigned long flags;
         struct vcpu_vmx *vmx = to_vmx(vcpu);
 
-        vmx->emulation_required = 1;
+        /*
+         * Update real mode segment cache. It may be not up-to-date if sement
+         * register was written while vcpu was in a guest mode.
+         */
+        vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_ES], VCPU_SREG_ES);
+        vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_DS], VCPU_SREG_DS);
+        vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_FS], VCPU_SREG_FS);
+        vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_GS], VCPU_SREG_GS);
+        vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_SS], VCPU_SREG_SS);
+        vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_CS], VCPU_SREG_CS);
+
         vmx->rmode.vm86_active = 0;
 
         vmx_segment_cache_clear(vmx);
@@ -2794,22 +2864,16 @@ static void enter_pmode(struct kvm_vcpu *vcpu)
 
         update_exception_bitmap(vcpu);
 
-        if (emulate_invalid_guest_state)
-                return;
-
-        fix_pmode_dataseg(vcpu, VCPU_SREG_ES, &vmx->rmode.segs[VCPU_SREG_ES]);
-        fix_pmode_dataseg(vcpu, VCPU_SREG_DS, &vmx->rmode.segs[VCPU_SREG_DS]);
-        fix_pmode_dataseg(vcpu, VCPU_SREG_FS, &vmx->rmode.segs[VCPU_SREG_FS]);
-        fix_pmode_dataseg(vcpu, VCPU_SREG_GS, &vmx->rmode.segs[VCPU_SREG_GS]);
-
-        vmx_segment_cache_clear(vmx);
+        fix_pmode_seg(vcpu, VCPU_SREG_CS, &vmx->rmode.segs[VCPU_SREG_CS]);
+        fix_pmode_seg(vcpu, VCPU_SREG_SS, &vmx->rmode.segs[VCPU_SREG_SS]);
+        fix_pmode_seg(vcpu, VCPU_SREG_ES, &vmx->rmode.segs[VCPU_SREG_ES]);
+        fix_pmode_seg(vcpu, VCPU_SREG_DS, &vmx->rmode.segs[VCPU_SREG_DS]);
+        fix_pmode_seg(vcpu, VCPU_SREG_FS, &vmx->rmode.segs[VCPU_SREG_FS]);
+        fix_pmode_seg(vcpu, VCPU_SREG_GS, &vmx->rmode.segs[VCPU_SREG_GS]);
 
-        vmcs_write16(GUEST_SS_SELECTOR, 0);
-        vmcs_write32(GUEST_SS_AR_BYTES, 0x93);
-
-        vmcs_write16(GUEST_CS_SELECTOR,
-                     vmcs_read16(GUEST_CS_SELECTOR) & ~SELECTOR_RPL_MASK);
-        vmcs_write32(GUEST_CS_AR_BYTES, 0x9b);
+        /* CPL is always 0 when CPU enters protected mode */
+        __set_bit(VCPU_EXREG_CPL, (ulong *)&vcpu->arch.regs_avail);
+        vmx->cpl = 0;
 }
 
 static gva_t rmode_tss_base(struct kvm *kvm)
@@ -2831,36 +2895,51 @@ static gva_t rmode_tss_base(struct kvm *kvm)
 static void fix_rmode_seg(int seg, struct kvm_segment *save)
 {
         const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
-
-        vmcs_write16(sf->selector, save->base >> 4);
-        vmcs_write32(sf->base, save->base & 0xffff0);
-        vmcs_write32(sf->limit, 0xffff);
-        vmcs_write32(sf->ar_bytes, 0xf3);
-        if (save->base & 0xf)
-                printk_once(KERN_WARNING "kvm: segment base is not paragraph"
-                            " aligned when entering protected mode (seg=%d)",
-                            seg);
+        struct kvm_segment var = *save;
+
+        var.dpl = 0x3;
+        if (seg == VCPU_SREG_CS)
+                var.type = 0x3;
+
+        if (!emulate_invalid_guest_state) {
+                var.selector = var.base >> 4;
+                var.base = var.base & 0xffff0;
+                var.limit = 0xffff;
+                var.g = 0;
+                var.db = 0;
+                var.present = 1;
+                var.s = 1;
+                var.l = 0;
+                var.unusable = 0;
+                var.type = 0x3;
+                var.avl = 0;
+                if (save->base & 0xf)
+                        printk_once(KERN_WARNING "kvm: segment base is not "
+                                        "paragraph aligned when entering "
+                                        "protected mode (seg=%d)", seg);
+        }
+
+        vmcs_write16(sf->selector, var.selector);
+        vmcs_write32(sf->base, var.base);
+        vmcs_write32(sf->limit, var.limit);
+        vmcs_write32(sf->ar_bytes, vmx_segment_access_rights(&var));
 }
 
 static void enter_rmode(struct kvm_vcpu *vcpu)
 {
         unsigned long flags;
         struct vcpu_vmx *vmx = to_vmx(vcpu);
-        struct kvm_segment var;
-
-        if (enable_unrestricted_guest)
-                return;
 
         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_TR], VCPU_SREG_TR);
         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_ES], VCPU_SREG_ES);
         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_DS], VCPU_SREG_DS);
         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_FS], VCPU_SREG_FS);
         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_GS], VCPU_SREG_GS);
+        vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_SS], VCPU_SREG_SS);
+        vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_CS], VCPU_SREG_CS);
 
-        vmx->emulation_required = 1;
         vmx->rmode.vm86_active = 1;
 
-
         /*
          * Very old userspace does not call KVM_SET_TSS_ADDR before entering
          * vcpu. Call it here with phys address pointing 16M below 4G.
@@ -2888,28 +2967,13 @@ static void enter_rmode(struct kvm_vcpu *vcpu)
         vmcs_writel(GUEST_CR4, vmcs_readl(GUEST_CR4) | X86_CR4_VME);
         update_exception_bitmap(vcpu);
 
-        if (emulate_invalid_guest_state)
-                goto continue_rmode;
-
-        vmx_get_segment(vcpu, &var, VCPU_SREG_SS);
-        vmx_set_segment(vcpu, &var, VCPU_SREG_SS);
-
-        vmx_get_segment(vcpu, &var, VCPU_SREG_CS);
-        vmx_set_segment(vcpu, &var, VCPU_SREG_CS);
-
-        vmx_get_segment(vcpu, &var, VCPU_SREG_ES);
-        vmx_set_segment(vcpu, &var, VCPU_SREG_ES);
-
-        vmx_get_segment(vcpu, &var, VCPU_SREG_DS);
-        vmx_set_segment(vcpu, &var, VCPU_SREG_DS);
+        fix_rmode_seg(VCPU_SREG_SS, &vmx->rmode.segs[VCPU_SREG_SS]);
+        fix_rmode_seg(VCPU_SREG_CS, &vmx->rmode.segs[VCPU_SREG_CS]);
+        fix_rmode_seg(VCPU_SREG_ES, &vmx->rmode.segs[VCPU_SREG_ES]);
+        fix_rmode_seg(VCPU_SREG_DS, &vmx->rmode.segs[VCPU_SREG_DS]);
+        fix_rmode_seg(VCPU_SREG_GS, &vmx->rmode.segs[VCPU_SREG_GS]);
+        fix_rmode_seg(VCPU_SREG_FS, &vmx->rmode.segs[VCPU_SREG_FS]);
 
-        vmx_get_segment(vcpu, &var, VCPU_SREG_GS);
-        vmx_set_segment(vcpu, &var, VCPU_SREG_GS);
-
-        vmx_get_segment(vcpu, &var, VCPU_SREG_FS);
-        vmx_set_segment(vcpu, &var, VCPU_SREG_FS);
-
-continue_rmode:
         kvm_mmu_reset_context(vcpu);
 }
 
@@ -3068,17 +3132,18 @@ static void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
         struct vcpu_vmx *vmx = to_vmx(vcpu);
         unsigned long hw_cr0;
 
+        hw_cr0 = (cr0 & ~KVM_GUEST_CR0_MASK);
         if (enable_unrestricted_guest)
-                hw_cr0 = (cr0 & ~KVM_GUEST_CR0_MASK_UNRESTRICTED_GUEST)
-                        | KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST;
-        else
-                hw_cr0 = (cr0 & ~KVM_GUEST_CR0_MASK) | KVM_VM_CR0_ALWAYS_ON;
+                hw_cr0 |= KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST;
+        else {
+                hw_cr0 |= KVM_VM_CR0_ALWAYS_ON;
 
-        if (vmx->rmode.vm86_active && (cr0 & X86_CR0_PE))
-                enter_pmode(vcpu);
+                if (vmx->rmode.vm86_active && (cr0 & X86_CR0_PE))
+                        enter_pmode(vcpu);
 
-        if (!vmx->rmode.vm86_active && !(cr0 & X86_CR0_PE))
-                enter_rmode(vcpu);
+                if (!vmx->rmode.vm86_active && !(cr0 & X86_CR0_PE))
+                        enter_rmode(vcpu);
+        }
 
 #ifdef CONFIG_X86_64
         if (vcpu->arch.efer & EFER_LME) {
@@ -3098,7 +3163,9 @@ static void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
         vmcs_writel(CR0_READ_SHADOW, cr0);
         vmcs_writel(GUEST_CR0, hw_cr0);
         vcpu->arch.cr0 = cr0;
-        __clear_bit(VCPU_EXREG_CPL, (ulong *)&vcpu->arch.regs_avail);
+
+        /* depends on vcpu->arch.cr0 to be set to a new value */
+        vmx->emulation_required = emulation_required(vcpu);
 }
 
 static u64 construct_eptp(unsigned long root_hpa)
@@ -3155,6 +3222,14 @@ static int vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
                 if (!is_paging(vcpu)) {
                         hw_cr4 &= ~X86_CR4_PAE;
                         hw_cr4 |= X86_CR4_PSE;
+                        /*
+                         * SMEP is disabled if CPU is in non-paging mode in
+                         * hardware. However KVM always uses paging mode to
+                         * emulate guest non-paging mode with TDP.
+                         * To emulate this behavior, SMEP needs to be manually
+                         * disabled when guest switches to non-paging mode.
+                         */
+                        hw_cr4 &= ~X86_CR4_SMEP;
                 } else if (!(cr4 & X86_CR4_PAE)) {
                         hw_cr4 &= ~X86_CR4_PAE;
                 }
@@ -3171,10 +3246,7 @@ static void vmx_get_segment(struct kvm_vcpu *vcpu,
         struct vcpu_vmx *vmx = to_vmx(vcpu);
         u32 ar;
 
-        if (vmx->rmode.vm86_active
-            && (seg == VCPU_SREG_TR || seg == VCPU_SREG_ES
-                || seg == VCPU_SREG_DS || seg == VCPU_SREG_FS
-                || seg == VCPU_SREG_GS)) {
+        if (vmx->rmode.vm86_active && seg != VCPU_SREG_LDTR) {
                 *var = vmx->rmode.segs[seg];
                 if (seg == VCPU_SREG_TR
                     || var->selector == vmx_read_guest_seg_selector(vmx, seg))
@@ -3187,8 +3259,6 @@ static void vmx_get_segment(struct kvm_vcpu *vcpu,
         var->limit = vmx_read_guest_seg_limit(vmx, seg);
         var->selector = vmx_read_guest_seg_selector(vmx, seg);
         ar = vmx_read_guest_seg_ar(vmx, seg);
-        if ((ar & AR_UNUSABLE_MASK) && !emulate_invalid_guest_state)
-                ar = 0;
         var->type = ar & 15;
         var->s = (ar >> 4) & 1;
         var->dpl = (ar >> 5) & 3;
@@ -3211,8 +3281,10 @@ static u64 vmx_get_segment_base(struct kvm_vcpu *vcpu, int seg)
         return vmx_read_guest_seg_base(to_vmx(vcpu), seg);
 }
 
-static int __vmx_get_cpl(struct kvm_vcpu *vcpu)
+static int vmx_get_cpl(struct kvm_vcpu *vcpu)
 {
+        struct vcpu_vmx *vmx = to_vmx(vcpu);
+
         if (!is_protmode(vcpu))
                 return 0;
 
@@ -3220,24 +3292,9 @@ static int __vmx_get_cpl(struct kvm_vcpu *vcpu)
             && (kvm_get_rflags(vcpu) & X86_EFLAGS_VM)) /* if virtual 8086 */
                 return 3;
 
-        return vmx_read_guest_seg_selector(to_vmx(vcpu), VCPU_SREG_CS) & 3;
-}
-
-static int vmx_get_cpl(struct kvm_vcpu *vcpu)
-{
-        struct vcpu_vmx *vmx = to_vmx(vcpu);
-
-        /*
-         * If we enter real mode with cs.sel & 3 != 0, the normal CPL calculations
-         * fail; use the cache instead.
-         */
-        if (unlikely(vmx->emulation_required && emulate_invalid_guest_state)) {
-                return vmx->cpl;
-        }
-
         if (!test_bit(VCPU_EXREG_CPL, (ulong *)&vcpu->arch.regs_avail)) {
                 __set_bit(VCPU_EXREG_CPL, (ulong *)&vcpu->arch.regs_avail);
-                vmx->cpl = __vmx_get_cpl(vcpu);
+                vmx->cpl = vmx_read_guest_seg_selector(vmx, VCPU_SREG_CS) & 3;
         }
 
         return vmx->cpl;
@@ -3269,28 +3326,23 @@ static void vmx_set_segment(struct kvm_vcpu *vcpu,
 {
         struct vcpu_vmx *vmx = to_vmx(vcpu);
         const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
-        u32 ar;
 
         vmx_segment_cache_clear(vmx);
+        if (seg == VCPU_SREG_CS)
+                __clear_bit(VCPU_EXREG_CPL, (ulong *)&vcpu->arch.regs_avail);
 
-        if (vmx->rmode.vm86_active && seg == VCPU_SREG_TR) {
-                vmcs_write16(sf->selector, var->selector);
-                vmx->rmode.segs[VCPU_SREG_TR] = *var;
-                return;
+        if (vmx->rmode.vm86_active && seg != VCPU_SREG_LDTR) {
+                vmx->rmode.segs[seg] = *var;
+                if (seg == VCPU_SREG_TR)
+                        vmcs_write16(sf->selector, var->selector);
+                else if (var->s)
+                        fix_rmode_seg(seg, &vmx->rmode.segs[seg]);
+                goto out;
         }
+
         vmcs_writel(sf->base, var->base);
         vmcs_write32(sf->limit, var->limit);
         vmcs_write16(sf->selector, var->selector);
-        if (vmx->rmode.vm86_active && var->s) {
-                vmx->rmode.segs[seg] = *var;
-                /*
-                 * Hack real-mode segments into vm86 compatibility.
-                 */
-                if (var->base == 0xffff0000 && var->selector == 0xf000)
-                        vmcs_writel(sf->base, 0xf0000);
-                ar = 0xf3;
-        } else
-                ar = vmx_segment_access_rights(var);
 
         /*
          *   Fix the "Accessed" bit in AR field of segment registers for older
@@ -3304,42 +3356,12 @@ static void vmx_set_segment(struct kvm_vcpu *vcpu,
          * kvm hack.
          */
         if (enable_unrestricted_guest && (seg != VCPU_SREG_LDTR))
-                ar |= 0x1; /* Accessed */
+                var->type |= 0x1; /* Accessed */
 
-        vmcs_write32(sf->ar_bytes, ar);
-        __clear_bit(VCPU_EXREG_CPL, (ulong *)&vcpu->arch.regs_avail);
+        vmcs_write32(sf->ar_bytes, vmx_segment_access_rights(var));
 
-        /*
-         * Fix segments for real mode guest in hosts that don't have
-         * "unrestricted_mode" or it was disabled.
-         * This is done to allow migration of the guests from hosts with
-         * unrestricted guest like Westmere to older host that don't have
-         * unrestricted guest like Nehelem.
-         */
-        if (vmx->rmode.vm86_active) {
-                switch (seg) {
-                case VCPU_SREG_CS:
-                        vmcs_write32(GUEST_CS_AR_BYTES, 0xf3);
-                        vmcs_write32(GUEST_CS_LIMIT, 0xffff);
-                        if (vmcs_readl(GUEST_CS_BASE) == 0xffff0000)
-                                vmcs_writel(GUEST_CS_BASE, 0xf0000);
-                        vmcs_write16(GUEST_CS_SELECTOR,
-                                     vmcs_readl(GUEST_CS_BASE) >> 4);
-                        break;
-                case VCPU_SREG_ES:
-                case VCPU_SREG_DS:
-                case VCPU_SREG_GS:
-                case VCPU_SREG_FS:
-                        fix_rmode_seg(seg, &vmx->rmode.segs[seg]);
-                        break;
-                case VCPU_SREG_SS:
-                        vmcs_write16(GUEST_SS_SELECTOR,
-                                     vmcs_readl(GUEST_SS_BASE) >> 4);
-                        vmcs_write32(GUEST_SS_LIMIT, 0xffff);
-                        vmcs_write32(GUEST_SS_AR_BYTES, 0xf3);
-                        break;
-                }
-        }
+out:
+        vmx->emulation_required |= emulation_required(vcpu);
 }
 
 static void vmx_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
@@ -3380,13 +3402,16 @@ static bool rmode_segment_valid(struct kvm_vcpu *vcpu, int seg)
         u32 ar;
 
         vmx_get_segment(vcpu, &var, seg);
+        var.dpl = 0x3;
+        if (seg == VCPU_SREG_CS)
+                var.type = 0x3;
         ar = vmx_segment_access_rights(&var);
 
         if (var.base != (var.selector << 4))
                 return false;
-        if (var.limit < 0xffff)
+        if (var.limit != 0xffff)
                 return false;
-        if (((ar | (3 << AR_DPL_SHIFT)) & ~(AR_G_MASK | AR_DB_MASK)) != 0xf3)
+        if (ar != 0xf3)
                 return false;
 
         return true;
@@ -3521,6 +3546,9 @@ static bool cs_ss_rpl_check(struct kvm_vcpu *vcpu)
  */
 static bool guest_state_valid(struct kvm_vcpu *vcpu)
 {
+        if (enable_unrestricted_guest)
+                return true;
+
         /* real mode guest state checks */
         if (!is_protmode(vcpu)) {
                 if (!rmode_segment_valid(vcpu, VCPU_SREG_CS))
@@ -3644,12 +3672,9 @@ static void seg_setup(int seg)
         vmcs_write16(sf->selector, 0);
         vmcs_writel(sf->base, 0);
         vmcs_write32(sf->limit, 0xffff);
-        if (enable_unrestricted_guest) {
-                ar = 0x93;
-                if (seg == VCPU_SREG_CS)
-                        ar |= 0x08; /* code segment */
-        } else
-                ar = 0xf3;
+        ar = 0x93;
+        if (seg == VCPU_SREG_CS)
+                ar |= 0x08; /* code segment */
 
         vmcs_write32(sf->ar_bytes, ar);
 }
@@ -3667,7 +3692,7 @@ static int alloc_apic_access_page(struct kvm *kvm)
         kvm_userspace_mem.flags = 0;
         kvm_userspace_mem.guest_phys_addr = 0xfee00000ULL;
         kvm_userspace_mem.memory_size = PAGE_SIZE;
-        r = __kvm_set_memory_region(kvm, &kvm_userspace_mem, 0);
+        r = __kvm_set_memory_region(kvm, &kvm_userspace_mem, false);
         if (r)
                 goto out;
 
@@ -3697,7 +3722,7 @@ static int alloc_identity_pagetable(struct kvm *kvm)
         kvm_userspace_mem.guest_phys_addr =
                 kvm->arch.ept_identity_map_addr;
         kvm_userspace_mem.memory_size = PAGE_SIZE;
-        r = __kvm_set_memory_region(kvm, &kvm_userspace_mem, 0);
+        r = __kvm_set_memory_region(kvm, &kvm_userspace_mem, false);
         if (r)
                 goto out;
 
@@ -3739,7 +3764,10 @@ static void free_vpid(struct vcpu_vmx *vmx)
         spin_unlock(&vmx_vpid_lock);
 }
 
-static void __vmx_disable_intercept_for_msr(unsigned long *msr_bitmap, u32 msr)
+#define MSR_TYPE_R      1
+#define MSR_TYPE_W      2
+static void __vmx_disable_intercept_for_msr(unsigned long *msr_bitmap,
+                                                u32 msr, int type)
 {
         int f = sizeof(unsigned long);
 
@@ -3752,20 +3780,93 @@ static void __vmx_disable_intercept_for_msr(unsigned long *msr_bitmap, u32 msr)
          * We can control MSRs 0x00000000-0x00001fff and 0xc0000000-0xc0001fff.
          */
         if (msr <= 0x1fff) {
-                __clear_bit(msr, msr_bitmap + 0x000 / f); /* read-low */
-                __clear_bit(msr, msr_bitmap + 0x800 / f); /* write-low */
+                if (type & MSR_TYPE_R)
+                        /* read-low */
+                        __clear_bit(msr, msr_bitmap + 0x000 / f);
+
+                if (type & MSR_TYPE_W)
+                        /* write-low */
+                        __clear_bit(msr, msr_bitmap + 0x800 / f);
+
         } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
                 msr &= 0x1fff;
-                __clear_bit(msr, msr_bitmap + 0x400 / f); /* read-high */
-                __clear_bit(msr, msr_bitmap + 0xc00 / f); /* write-high */
+                if (type & MSR_TYPE_R)
+                        /* read-high */
+                        __clear_bit(msr, msr_bitmap + 0x400 / f);
+
+                if (type & MSR_TYPE_W)
+                        /* write-high */
+                        __clear_bit(msr, msr_bitmap + 0xc00 / f);
+
+        }
+}
+
+static void __vmx_enable_intercept_for_msr(unsigned long *msr_bitmap,
+                                                u32 msr, int type)
+{
+        int f = sizeof(unsigned long);
+
+        if (!cpu_has_vmx_msr_bitmap())
+                return;
+
+        /*
+         * See Intel PRM Vol. 3, 20.6.9 (MSR-Bitmap Address). Early manuals
+         * have the write-low and read-high bitmap offsets the wrong way round.
+         * We can control MSRs 0x00000000-0x00001fff and 0xc0000000-0xc0001fff.
+         */
+        if (msr <= 0x1fff) {
+                if (type & MSR_TYPE_R)
+                        /* read-low */
+                        __set_bit(msr, msr_bitmap + 0x000 / f);
+
+                if (type & MSR_TYPE_W)
+                        /* write-low */
+                        __set_bit(msr, msr_bitmap + 0x800 / f);
+
+        } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
+                msr &= 0x1fff;
+                if (type & MSR_TYPE_R)
+                        /* read-high */
+                        __set_bit(msr, msr_bitmap + 0x400 / f);
+
+                if (type & MSR_TYPE_W)
+                        /* write-high */
+                        __set_bit(msr, msr_bitmap + 0xc00 / f);
+
         }
 }
 
 static void vmx_disable_intercept_for_msr(u32 msr, bool longmode_only)
 {
         if (!longmode_only)
-                __vmx_disable_intercept_for_msr(vmx_msr_bitmap_legacy, msr);
-        __vmx_disable_intercept_for_msr(vmx_msr_bitmap_longmode, msr);
+                __vmx_disable_intercept_for_msr(vmx_msr_bitmap_legacy,
+                                                msr, MSR_TYPE_R | MSR_TYPE_W);
+        __vmx_disable_intercept_for_msr(vmx_msr_bitmap_longmode,
+                                                msr, MSR_TYPE_R | MSR_TYPE_W);
+}
+
+static void vmx_enable_intercept_msr_read_x2apic(u32 msr)
+{
+        __vmx_enable_intercept_for_msr(vmx_msr_bitmap_legacy_x2apic,
+                        msr, MSR_TYPE_R);
+        __vmx_enable_intercept_for_msr(vmx_msr_bitmap_longmode_x2apic,
+                        msr, MSR_TYPE_R);
+}
+
+static void vmx_disable_intercept_msr_read_x2apic(u32 msr)
+{
+        __vmx_disable_intercept_for_msr(vmx_msr_bitmap_legacy_x2apic,
+                        msr, MSR_TYPE_R);
+        __vmx_disable_intercept_for_msr(vmx_msr_bitmap_longmode_x2apic,
+                        msr, MSR_TYPE_R);
+}
+
+static void vmx_disable_intercept_msr_write_x2apic(u32 msr)
+{
+        __vmx_disable_intercept_for_msr(vmx_msr_bitmap_legacy_x2apic,
+                        msr, MSR_TYPE_W);
+        __vmx_disable_intercept_for_msr(vmx_msr_bitmap_longmode_x2apic,
+                        msr, MSR_TYPE_W);
 }
 
 /*
@@ -3844,6 +3945,11 @@ static u32 vmx_exec_control(struct vcpu_vmx *vmx)
         return exec_control;
 }
 
+static int vmx_vm_has_apicv(struct kvm *kvm)
+{
+        return enable_apicv_reg_vid && irqchip_in_kernel(kvm);
+}
+
 static u32 vmx_secondary_exec_control(struct vcpu_vmx *vmx)
 {
         u32 exec_control = vmcs_config.cpu_based_2nd_exec_ctrl;
@@ -3861,6 +3967,10 @@ static u32 vmx_secondary_exec_control(struct vcpu_vmx *vmx)
                 exec_control &= ~SECONDARY_EXEC_UNRESTRICTED_GUEST;
         if (!ple_gap)
                 exec_control &= ~SECONDARY_EXEC_PAUSE_LOOP_EXITING;
+        if (!vmx_vm_has_apicv(vmx->vcpu.kvm))
+                exec_control &= ~(SECONDARY_EXEC_APIC_REGISTER_VIRT |
+                                  SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
+        exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE;
         return exec_control;
 }
 
@@ -3905,6 +4015,15 @@ static int vmx_vcpu_setup(struct vcpu_vmx *vmx)
                                 vmx_secondary_exec_control(vmx));
         }
 
+        if (enable_apicv_reg_vid) {
+                vmcs_write64(EOI_EXIT_BITMAP0, 0);
+                vmcs_write64(EOI_EXIT_BITMAP1, 0);
+                vmcs_write64(EOI_EXIT_BITMAP2, 0);
+                vmcs_write64(EOI_EXIT_BITMAP3, 0);
+
+                vmcs_write16(GUEST_INTR_STATUS, 0);
+        }
+
         if (ple_gap) {
                 vmcs_write32(PLE_GAP, ple_gap);
                 vmcs_write32(PLE_WINDOW, ple_window);
@@ -3990,14 +4109,9 @@ static int vmx_vcpu_reset(struct kvm_vcpu *vcpu)
         vmx_segment_cache_clear(vmx);
 
         seg_setup(VCPU_SREG_CS);
-        /*
-         * GUEST_CS_BASE should really be 0xffff0000, but VT vm86 mode
-         * insists on having GUEST_CS_BASE == GUEST_CS_SELECTOR << 4.  Sigh.
-         */
-        if (kvm_vcpu_is_bsp(&vmx->vcpu)) {
+        if (kvm_vcpu_is_bsp(&vmx->vcpu))
                 vmcs_write16(GUEST_CS_SELECTOR, 0xf000);
-                vmcs_writel(GUEST_CS_BASE, 0x000f0000);
-        } else {
+        else {
                 vmcs_write16(GUEST_CS_SELECTOR, vmx->vcpu.arch.sipi_vector << 8);
                 vmcs_writel(GUEST_CS_BASE, vmx->vcpu.arch.sipi_vector << 12);
         }
@@ -4073,9 +4187,6 @@ static int vmx_vcpu_reset(struct kvm_vcpu *vcpu)
 
         ret = 0;
 
-        /* HACK: Don't enable emulation on guest boot/reset */
-        vmx->emulation_required = 0;
-
         return ret;
 }
 
@@ -4251,7 +4362,7 @@ static int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr)
                 .flags = 0,
         };
 
-        ret = kvm_set_memory_region(kvm, &tss_mem, 0);
+        ret = kvm_set_memory_region(kvm, &tss_mem, false);
         if (ret)
                 return ret;
         kvm->arch.tss_addr = addr;
@@ -4261,28 +4372,9 @@ static int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr)
         return 0;
 }
 
-static int handle_rmode_exception(struct kvm_vcpu *vcpu,
-                                  int vec, u32 err_code)
+static bool rmode_exception(struct kvm_vcpu *vcpu, int vec)
 {
-        /*
-         * Instruction with address size override prefix opcode 0x67
-         * Cause the #SS fault with 0 error code in VM86 mode.
-         */
-        if (((vec == GP_VECTOR) || (vec == SS_VECTOR)) && err_code == 0)
-                if (emulate_instruction(vcpu, 0) == EMULATE_DONE)
-                        return 1;
-        /*
-         * Forward all other exceptions that are valid in real mode.
-         * FIXME: Breaks guest debugging in real mode, needs to be fixed with
-         *        the required debugging infrastructure rework.
-         */
         switch (vec) {
-        case DB_VECTOR:
-                if (vcpu->guest_debug &
-                    (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))
-                        return 0;
-                kvm_queue_exception(vcpu, vec);
-                return 1;
         case BP_VECTOR:
                 /*
                  * Update instruction length as we may reinject the exception
@@ -4291,7 +4383,12 @@ static int handle_rmode_exception(struct kvm_vcpu *vcpu,
                 to_vmx(vcpu)->vcpu.arch.event_exit_inst_len =
                         vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
                 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
-                        return 0;
+                        return false;
+                /* fall through */
+        case DB_VECTOR:
+                if (vcpu->guest_debug &
+                        (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))
+                        return false;
                 /* fall through */
         case DE_VECTOR:
         case OF_VECTOR:
@@ -4301,10 +4398,37 @@ static int handle_rmode_exception(struct kvm_vcpu *vcpu,
         case SS_VECTOR:
         case GP_VECTOR:
         case MF_VECTOR:
-                kvm_queue_exception(vcpu, vec);
-                return 1;
+                return true;
+        break;
         }
-        return 0;
+        return false;
+}
+
+static int handle_rmode_exception(struct kvm_vcpu *vcpu,
+                                  int vec, u32 err_code)
+{
+        /*
+         * Instruction with address size override prefix opcode 0x67
+         * Cause the #SS fault with 0 error code in VM86 mode.
+         */
+        if (((vec == GP_VECTOR) || (vec == SS_VECTOR)) && err_code == 0) {
+                if (emulate_instruction(vcpu, 0) == EMULATE_DONE) {
+                        if (vcpu->arch.halt_request) {
+                                vcpu->arch.halt_request = 0;
+                                return kvm_emulate_halt(vcpu);
+                        }
+                        return 1;
+                }
+                return 0;
+        }
+
+        /*
+         * Forward all other exceptions that are valid in real mode.
+         * FIXME: Breaks guest debugging in real mode, needs to be fixed with
+         *        the required debugging infrastructure rework.
+         */
+        kvm_queue_exception(vcpu, vec);
+        return 1;
 }
 
 /*
@@ -4392,17 +4516,11 @@ static int handle_exception(struct kvm_vcpu *vcpu)
                 return kvm_mmu_page_fault(vcpu, cr2, error_code, NULL, 0);
         }
 
-        if (vmx->rmode.vm86_active &&
-            handle_rmode_exception(vcpu, intr_info & INTR_INFO_VECTOR_MASK,
-                                                                error_code)) {
-                if (vcpu->arch.halt_request) {
-                        vcpu->arch.halt_request = 0;
-                        return kvm_emulate_halt(vcpu);
-                }
-                return 1;
-        }
-
         ex_no = intr_info & INTR_INFO_VECTOR_MASK;
+
+        if (vmx->rmode.vm86_active && rmode_exception(vcpu, ex_no))
+                return handle_rmode_exception(vcpu, ex_no, error_code);
+
         switch (ex_no) {
         case DB_VECTOR:
                 dr6 = vmcs_readl(EXIT_QUALIFICATION);
@@ -4820,6 +4938,26 @@ static int handle_apic_access(struct kvm_vcpu *vcpu)
         return emulate_instruction(vcpu, 0) == EMULATE_DONE;
 }
 
+static int handle_apic_eoi_induced(struct kvm_vcpu *vcpu)
+{
+        unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
+        int vector = exit_qualification & 0xff;
+
+        /* EOI-induced VM exit is trap-like and thus no need to adjust IP */
+        kvm_apic_set_eoi_accelerated(vcpu, vector);
+        return 1;
+}
+
+static int handle_apic_write(struct kvm_vcpu *vcpu)
+{
+        unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
+        u32 offset = exit_qualification & 0xfff;
+
+        /* APIC-write VM exit is trap-like and thus no need to adjust IP */
+        kvm_apic_write_nodecode(vcpu, offset);
+        return 1;
+}
+
 static int handle_task_switch(struct kvm_vcpu *vcpu)
 {
         struct vcpu_vmx *vmx = to_vmx(vcpu);
@@ -5065,7 +5203,7 @@ static int handle_invalid_guest_state(struct kvm_vcpu *vcpu)
                         schedule();
         }
 
-        vmx->emulation_required = !guest_state_valid(vcpu);
+        vmx->emulation_required = emulation_required(vcpu);
 out:
         return ret;
 }
@@ -5754,6 +5892,8 @@ static int (*const kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = {
         [EXIT_REASON_VMON]                    = handle_vmon,
         [EXIT_REASON_TPR_BELOW_THRESHOLD]     = handle_tpr_below_threshold,
         [EXIT_REASON_APIC_ACCESS]             = handle_apic_access,
+        [EXIT_REASON_APIC_WRITE]              = handle_apic_write,
+        [EXIT_REASON_EOI_INDUCED]             = handle_apic_eoi_induced,
         [EXIT_REASON_WBINVD]                  = handle_wbinvd,
         [EXIT_REASON_XSETBV]                  = handle_xsetbv,
         [EXIT_REASON_TASK_SWITCH]             = handle_task_switch,
@@ -5780,7 +5920,7 @@ static bool nested_vmx_exit_handled_msr(struct kvm_vcpu *vcpu,
         u32 msr_index = vcpu->arch.regs[VCPU_REGS_RCX];
         gpa_t bitmap;
 
-        if (!nested_cpu_has(get_vmcs12(vcpu), CPU_BASED_USE_MSR_BITMAPS))
+        if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS))
                 return 1;
 
         /*
@@ -6008,7 +6148,7 @@ static int vmx_handle_exit(struct kvm_vcpu *vcpu)
         u32 vectoring_info = vmx->idt_vectoring_info;
 
         /* If guest state is invalid, start emulating */
-        if (vmx->emulation_required && emulate_invalid_guest_state)
+        if (vmx->emulation_required)
                 return handle_invalid_guest_state(vcpu);
 
         /*
@@ -6103,6 +6243,85 @@ static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
         vmcs_write32(TPR_THRESHOLD, irr);
 }
 
+static void vmx_set_virtual_x2apic_mode(struct kvm_vcpu *vcpu, bool set)
+{
+        u32 sec_exec_control;
+
+        /*
+         * There is not point to enable virtualize x2apic without enable
+         * apicv
+         */
+        if (!cpu_has_vmx_virtualize_x2apic_mode() ||
+                                !vmx_vm_has_apicv(vcpu->kvm))
+                return;
+
+        if (!vm_need_tpr_shadow(vcpu->kvm))
+                return;
+
+        sec_exec_control = vmcs_read32(SECONDARY_VM_EXEC_CONTROL);
+
+        if (set) {
+                sec_exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
+                sec_exec_control |= SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE;
+        } else {
+                sec_exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE;
+                sec_exec_control |= SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
+        }
+        vmcs_write32(SECONDARY_VM_EXEC_CONTROL, sec_exec_control);
+
+        vmx_set_msr_bitmap(vcpu);
+}
+
+static void vmx_hwapic_isr_update(struct kvm *kvm, int isr)
+{
+        u16 status;
+        u8 old;
+
+        if (!vmx_vm_has_apicv(kvm))
+                return;
+
+        if (isr == -1)
+                isr = 0;
+
+        status = vmcs_read16(GUEST_INTR_STATUS);
+        old = status >> 8;
+        if (isr != old) {
+                status &= 0xff;
+                status |= isr << 8;
+                vmcs_write16(GUEST_INTR_STATUS, status);
+        }
+}
+
+static void vmx_set_rvi(int vector)
+{
+        u16 status;
+        u8 old;
+
+        status = vmcs_read16(GUEST_INTR_STATUS);
+        old = (u8)status & 0xff;
+        if ((u8)vector != old) {
+                status &= ~0xff;
+                status |= (u8)vector;
+                vmcs_write16(GUEST_INTR_STATUS, status);
+        }
+}
+
+static void vmx_hwapic_irr_update(struct kvm_vcpu *vcpu, int max_irr)
+{
+        if (max_irr == -1)
+                return;
+
+        vmx_set_rvi(max_irr);
+}
+
+static void vmx_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap)
+{
+        vmcs_write64(EOI_EXIT_BITMAP0, eoi_exit_bitmap[0]);
+        vmcs_write64(EOI_EXIT_BITMAP1, eoi_exit_bitmap[1]);
+        vmcs_write64(EOI_EXIT_BITMAP2, eoi_exit_bitmap[2]);
+        vmcs_write64(EOI_EXIT_BITMAP3, eoi_exit_bitmap[3]);
+}
+
 static void vmx_complete_atomic_exit(struct vcpu_vmx *vmx)
 {
         u32 exit_intr_info;
@@ -6291,7 +6510,7 @@ static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu)
 
         /* Don't enter VMX if guest state is invalid, let the exit handler
            start emulation until we arrive back to a valid state */
-        if (vmx->emulation_required && emulate_invalid_guest_state)
+        if (vmx->emulation_required)
                 return;
 
         if (test_bit(VCPU_REGS_RSP, (unsigned long *)&vcpu->arch.regs_dirty))
@@ -7366,6 +7585,11 @@ static struct kvm_x86_ops vmx_x86_ops = {
         .enable_nmi_window = enable_nmi_window,
         .enable_irq_window = enable_irq_window,
         .update_cr8_intercept = update_cr8_intercept,
+        .set_virtual_x2apic_mode = vmx_set_virtual_x2apic_mode,
+        .vm_has_apicv = vmx_vm_has_apicv,
+        .load_eoi_exitmap = vmx_load_eoi_exitmap,
+        .hwapic_irr_update = vmx_hwapic_irr_update,
+        .hwapic_isr_update = vmx_hwapic_isr_update,
 
         .set_tss_addr = vmx_set_tss_addr,
         .get_tdp_level = get_ept_level,
@@ -7398,7 +7622,7 @@ static struct kvm_x86_ops vmx_x86_ops = {
 
 static int __init vmx_init(void)
 {
-        int r, i;
+        int r, i, msr;
 
         rdmsrl_safe(MSR_EFER, &host_efer);
 
@@ -7419,11 +7643,19 @@ static int __init vmx_init(void)
         if (!vmx_msr_bitmap_legacy)
                 goto out1;
 
+        vmx_msr_bitmap_legacy_x2apic =
+                                (unsigned long *)__get_free_page(GFP_KERNEL);
+        if (!vmx_msr_bitmap_legacy_x2apic)
+                goto out2;
 
         vmx_msr_bitmap_longmode = (unsigned long *)__get_free_page(GFP_KERNEL);
         if (!vmx_msr_bitmap_longmode)
-                goto out2;
+                goto out3;
 
+        vmx_msr_bitmap_longmode_x2apic =
+                                (unsigned long *)__get_free_page(GFP_KERNEL);
+        if (!vmx_msr_bitmap_longmode_x2apic)
+                goto out4;
 
         /*
          * Allow direct access to the PC debug port (it is often used for I/O
@@ -7455,6 +7687,28 @@ static int __init vmx_init(void)
         vmx_disable_intercept_for_msr(MSR_IA32_SYSENTER_CS, false);
         vmx_disable_intercept_for_msr(MSR_IA32_SYSENTER_ESP, false);
         vmx_disable_intercept_for_msr(MSR_IA32_SYSENTER_EIP, false);
+        memcpy(vmx_msr_bitmap_legacy_x2apic,
+                        vmx_msr_bitmap_legacy, PAGE_SIZE);
+        memcpy(vmx_msr_bitmap_longmode_x2apic,
+                        vmx_msr_bitmap_longmode, PAGE_SIZE);
+
+        if (enable_apicv_reg_vid) {
+                for (msr = 0x800; msr <= 0x8ff; msr++)
+                        vmx_disable_intercept_msr_read_x2apic(msr);
+
+                /* According SDM, in x2apic mode, the whole id reg is used.
+                 * But in KVM, it only use the highest eight bits. Need to
+                 * intercept it */
+                vmx_enable_intercept_msr_read_x2apic(0x802);
+                /* TMCCT */
+                vmx_enable_intercept_msr_read_x2apic(0x839);
+                /* TPR */
+                vmx_disable_intercept_msr_write_x2apic(0x808);
+                /* EOI */
+                vmx_disable_intercept_msr_write_x2apic(0x80b);
+                /* SELF-IPI */
+                vmx_disable_intercept_msr_write_x2apic(0x83f);
+        }
 
         if (enable_ept) {
                 kvm_mmu_set_mask_ptes(0ull,
@@ -7468,8 +7722,10 @@ static int __init vmx_init(void)
 
         return 0;
 
-out3:
+out4:
         free_page((unsigned long)vmx_msr_bitmap_longmode);
+out3:
+        free_page((unsigned long)vmx_msr_bitmap_legacy_x2apic);
 out2:
         free_page((unsigned long)vmx_msr_bitmap_legacy);
 out1:
@@ -7481,6 +7737,8 @@ out:
 
 static void __exit vmx_exit(void)
 {
+        free_page((unsigned long)vmx_msr_bitmap_legacy_x2apic);
+        free_page((unsigned long)vmx_msr_bitmap_longmode_x2apic);
         free_page((unsigned long)vmx_msr_bitmap_legacy);
         free_page((unsigned long)vmx_msr_bitmap_longmode);
         free_page((unsigned long)vmx_io_bitmap_b);
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index c243b81e3c74..f19ac0aca60d 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -872,8 +872,6 @@ static int set_efer(struct kvm_vcpu *vcpu, u64 efer)
 
         kvm_x86_ops->set_efer(vcpu, efer);
 
-        vcpu->arch.mmu.base_role.nxe = (efer & EFER_NX) && !tdp_enabled;
-
         /* Update reserved bits */
         if ((efer ^ old_efer) & EFER_NX)
                 kvm_mmu_reset_context(vcpu);
@@ -1408,25 +1406,15 @@ static int kvm_guest_time_update(struct kvm_vcpu *v)
         unsigned long flags, this_tsc_khz;
         struct kvm_vcpu_arch *vcpu = &v->arch;
         struct kvm_arch *ka = &v->kvm->arch;
-        void *shared_kaddr;
         s64 kernel_ns, max_kernel_ns;
         u64 tsc_timestamp, host_tsc;
-        struct pvclock_vcpu_time_info *guest_hv_clock;
+        struct pvclock_vcpu_time_info guest_hv_clock;
         u8 pvclock_flags;
         bool use_master_clock;
 
         kernel_ns = 0;
         host_tsc = 0;
 
-        /* Keep irq disabled to prevent changes to the clock */
-        local_irq_save(flags);
-        this_tsc_khz = __get_cpu_var(cpu_tsc_khz);
-        if (unlikely(this_tsc_khz == 0)) {
-                local_irq_restore(flags);
-                kvm_make_request(KVM_REQ_CLOCK_UPDATE, v);
-                return 1;
-        }
-
         /*
          * If the host uses TSC clock, then passthrough TSC as stable
          * to the guest.
@@ -1438,6 +1426,15 @@ static int kvm_guest_time_update(struct kvm_vcpu *v)
                 kernel_ns = ka->master_kernel_ns;
         }
         spin_unlock(&ka->pvclock_gtod_sync_lock);
+
+        /* Keep irq disabled to prevent changes to the clock */
+        local_irq_save(flags);
+        this_tsc_khz = __get_cpu_var(cpu_tsc_khz);
+        if (unlikely(this_tsc_khz == 0)) {
+                local_irq_restore(flags);
+                kvm_make_request(KVM_REQ_CLOCK_UPDATE, v);
+                return 1;
+        }
         if (!use_master_clock) {
                 host_tsc = native_read_tsc();
                 kernel_ns = get_kernel_ns();
@@ -1465,7 +1462,7 @@ static int kvm_guest_time_update(struct kvm_vcpu *v)
 
         local_irq_restore(flags);
 
-        if (!vcpu->time_page)
+        if (!vcpu->pv_time_enabled)
                 return 0;
 
         /*
@@ -1527,12 +1524,12 @@ static int kvm_guest_time_update(struct kvm_vcpu *v)
          */
         vcpu->hv_clock.version += 2;
 
-        shared_kaddr = kmap_atomic(vcpu->time_page);
-
-        guest_hv_clock = shared_kaddr + vcpu->time_offset;
+        if (unlikely(kvm_read_guest_cached(v->kvm, &vcpu->pv_time,
+                &guest_hv_clock, sizeof(guest_hv_clock))))
+                return 0;
 
         /* retain PVCLOCK_GUEST_STOPPED if set in guest copy */
-        pvclock_flags = (guest_hv_clock->flags & PVCLOCK_GUEST_STOPPED);
+        pvclock_flags = (guest_hv_clock.flags & PVCLOCK_GUEST_STOPPED);
 
         if (vcpu->pvclock_set_guest_stopped_request) {
                 pvclock_flags |= PVCLOCK_GUEST_STOPPED;
@@ -1545,12 +1542,9 @@ static int kvm_guest_time_update(struct kvm_vcpu *v)
 
         vcpu->hv_clock.flags = pvclock_flags;
 
-        memcpy(shared_kaddr + vcpu->time_offset, &vcpu->hv_clock,
-               sizeof(vcpu->hv_clock));
-
-        kunmap_atomic(shared_kaddr);
-
-        mark_page_dirty(v->kvm, vcpu->time >> PAGE_SHIFT);
+        kvm_write_guest_cached(v->kvm, &vcpu->pv_time,
+                                &vcpu->hv_clock,
+                                sizeof(vcpu->hv_clock));
         return 0;
 }
 
@@ -1839,10 +1833,7 @@ static int kvm_pv_enable_async_pf(struct kvm_vcpu *vcpu, u64 data)
 
 static void kvmclock_reset(struct kvm_vcpu *vcpu)
 {
-        if (vcpu->arch.time_page) {
-                kvm_release_page_dirty(vcpu->arch.time_page);
-                vcpu->arch.time_page = NULL;
-        }
+        vcpu->arch.pv_time_enabled = false;
 }
 
 static void accumulate_steal_time(struct kvm_vcpu *vcpu)
@@ -1881,6 +1872,14 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
         u64 data = msr_info->data;
 
         switch (msr) {
+        case MSR_AMD64_NB_CFG:
+        case MSR_IA32_UCODE_REV:
+        case MSR_IA32_UCODE_WRITE:
+        case MSR_VM_HSAVE_PA:
+        case MSR_AMD64_PATCH_LOADER:
+        case MSR_AMD64_BU_CFG2:
+                break;
+
         case MSR_EFER:
                 return set_efer(vcpu, data);
         case MSR_K7_HWCR:
@@ -1900,8 +1899,6 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
                         return 1;
                 }
                 break;
-        case MSR_AMD64_NB_CFG:
-                break;
         case MSR_IA32_DEBUGCTLMSR:
                 if (!data) {
                         /* We support the non-activated case already */
@@ -1914,11 +1911,6 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
                 vcpu_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTLMSR 0x%llx, nop\n",
                             __func__, data);
                 break;
-        case MSR_IA32_UCODE_REV:
-        case MSR_IA32_UCODE_WRITE:
-        case MSR_VM_HSAVE_PA:
-        case MSR_AMD64_PATCH_LOADER:
-                break;
         case 0x200 ... 0x2ff:
                 return set_msr_mtrr(vcpu, msr, data);
         case MSR_IA32_APICBASE:
@@ -1948,6 +1940,7 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
                 break;
         case MSR_KVM_SYSTEM_TIME_NEW:
         case MSR_KVM_SYSTEM_TIME: {
+                u64 gpa_offset;
                 kvmclock_reset(vcpu);
 
                 vcpu->arch.time = data;
@@ -1957,14 +1950,17 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
                 if (!(data & 1))
                         break;
 
-                /* ...but clean it before doing the actual write */
-                vcpu->arch.time_offset = data & ~(PAGE_MASK | 1);
+                gpa_offset = data & ~(PAGE_MASK | 1);
 
-                vcpu->arch.time_page =
-                                gfn_to_page(vcpu->kvm, data >> PAGE_SHIFT);
+                /* Check that the address is 32-byte aligned. */
+                if (gpa_offset & (sizeof(struct pvclock_vcpu_time_info) - 1))
+                        break;
 
-                if (is_error_page(vcpu->arch.time_page))
-                        vcpu->arch.time_page = NULL;
+                if (kvm_gfn_to_hva_cache_init(vcpu->kvm,
+                     &vcpu->arch.pv_time, data & ~1ULL))
+                        vcpu->arch.pv_time_enabled = false;
+                else
+                        vcpu->arch.pv_time_enabled = true;
 
                 break;
         }
@@ -2253,6 +2249,7 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
         case MSR_K8_INT_PENDING_MSG:
         case MSR_AMD64_NB_CFG:
         case MSR_FAM10H_MMIO_CONF_BASE:
+        case MSR_AMD64_BU_CFG2:
                 data = 0;
                 break;
         case MSR_P6_PERFCTR0:
@@ -2520,7 +2517,7 @@ int kvm_dev_ioctl_check_extension(long ext)
                 r = KVM_MAX_VCPUS;
                 break;
         case KVM_CAP_NR_MEMSLOTS:
-                r = KVM_MEMORY_SLOTS;
+                r = KVM_USER_MEM_SLOTS;
                 break;
         case KVM_CAP_PV_MMU:    /* obsolete */
                 r = 0;
@@ -2967,7 +2964,7 @@ static int kvm_vcpu_ioctl_x86_set_xcrs(struct kvm_vcpu *vcpu,
  */
 static int kvm_set_guest_paused(struct kvm_vcpu *vcpu)
 {
-        if (!vcpu->arch.time_page)
+        if (!vcpu->arch.pv_time_enabled)
                 return -EINVAL;
         vcpu->arch.pvclock_set_guest_stopped_request = true;
         kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
@@ -3272,12 +3269,10 @@ static int kvm_vm_ioctl_set_nr_mmu_pages(struct kvm *kvm,
                 return -EINVAL;
 
         mutex_lock(&kvm->slots_lock);
-        spin_lock(&kvm->mmu_lock);
 
         kvm_mmu_change_mmu_pages(kvm, kvm_nr_mmu_pages);
         kvm->arch.n_requested_mmu_pages = kvm_nr_mmu_pages;
 
-        spin_unlock(&kvm->mmu_lock);
         mutex_unlock(&kvm->slots_lock);
         return 0;
 }
@@ -3437,7 +3432,7 @@ int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log)
         mutex_lock(&kvm->slots_lock);
 
         r = -EINVAL;
-        if (log->slot >= KVM_MEMORY_SLOTS)
+        if (log->slot >= KVM_USER_MEM_SLOTS)
                 goto out;
 
         memslot = id_to_memslot(kvm->memslots, log->slot);
@@ -4493,8 +4488,10 @@ static bool emulator_get_segment(struct x86_emulate_ctxt *ctxt, u16 *selector,
         kvm_get_segment(emul_to_vcpu(ctxt), &var, seg);
         *selector = var.selector;
 
-        if (var.unusable)
+        if (var.unusable) {
+                memset(desc, 0, sizeof(*desc));
                 return false;
+        }
 
         if (var.g)
                 var.limit >>= 12;
@@ -4755,26 +4752,26 @@ static int handle_emulation_failure(struct kvm_vcpu *vcpu)
         return r;
 }
 
-static bool reexecute_instruction(struct kvm_vcpu *vcpu, gva_t gva)
+static bool reexecute_instruction(struct kvm_vcpu *vcpu, gva_t cr2,
+                                  bool write_fault_to_shadow_pgtable)
 {
-        gpa_t gpa;
+        gpa_t gpa = cr2;
         pfn_t pfn;
 
-        if (tdp_enabled)
-                return false;
-
-        /*
-         * if emulation was due to access to shadowed page table
-         * and it failed try to unshadow page and re-enter the
-         * guest to let CPU execute the instruction.
-         */
-        if (kvm_mmu_unprotect_page_virt(vcpu, gva))
-                return true;
-
-        gpa = kvm_mmu_gva_to_gpa_system(vcpu, gva, NULL);
+        if (!vcpu->arch.mmu.direct_map) {
+                /*
+                 * Write permission should be allowed since only
+                 * write access need to be emulated.
+                 */
+                gpa = kvm_mmu_gva_to_gpa_write(vcpu, cr2, NULL);
 
-        if (gpa == UNMAPPED_GVA)
-                return true; /* let cpu generate fault */
+                /*
+                 * If the mapping is invalid in guest, let cpu retry
+                 * it to generate fault.
+                 */
+                if (gpa == UNMAPPED_GVA)
+                        return true;
+        }
 
         /*
          * Do not retry the unhandleable instruction if it faults on the
@@ -4783,12 +4780,43 @@ static bool reexecute_instruction(struct kvm_vcpu *vcpu, gva_t gva)
          * instruction -> ...
          */
         pfn = gfn_to_pfn(vcpu->kvm, gpa_to_gfn(gpa));
-        if (!is_error_noslot_pfn(pfn)) {
-                kvm_release_pfn_clean(pfn);
+
+        /*
+         * If the instruction failed on the error pfn, it can not be fixed,
+         * report the error to userspace.
+         */
+        if (is_error_noslot_pfn(pfn))
+                return false;
+
+        kvm_release_pfn_clean(pfn);
+
+        /* The instructions are well-emulated on direct mmu. */
+        if (vcpu->arch.mmu.direct_map) {
+                unsigned int indirect_shadow_pages;
+
+                spin_lock(&vcpu->kvm->mmu_lock);
+                indirect_shadow_pages = vcpu->kvm->arch.indirect_shadow_pages;
+                spin_unlock(&vcpu->kvm->mmu_lock);
+
+                if (indirect_shadow_pages)
+                        kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));
+
                 return true;
         }
 
-        return false;
+        /*
+         * if emulation was due to access to shadowed page table
+         * and it failed try to unshadow page and re-enter the
+         * guest to let CPU execute the instruction.
+         */
+        kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));
+
+        /*
+         * If the access faults on its page table, it can not
+         * be fixed by unprotecting shadow page and it should
+         * be reported to userspace.
+         */
+        return !write_fault_to_shadow_pgtable;
 }
 
 static bool retry_instruction(struct x86_emulate_ctxt *ctxt,
@@ -4830,7 +4858,7 @@ static bool retry_instruction(struct x86_emulate_ctxt *ctxt,
         if (!vcpu->arch.mmu.direct_map)
                 gpa = kvm_mmu_gva_to_gpa_write(vcpu, cr2, NULL);
 
-        kvm_mmu_unprotect_page(vcpu->kvm, gpa >> PAGE_SHIFT);
+        kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));
 
         return true;
 }
@@ -4847,7 +4875,13 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu,
         int r;
         struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
         bool writeback = true;
+        bool write_fault_to_spt = vcpu->arch.write_fault_to_shadow_pgtable;
 
+        /*
+         * Clear write_fault_to_shadow_pgtable here to ensure it is
+         * never reused.
+         */
+        vcpu->arch.write_fault_to_shadow_pgtable = false;
         kvm_clear_exception_queue(vcpu);
 
         if (!(emulation_type & EMULTYPE_NO_DECODE)) {
@@ -4866,7 +4900,8 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu,
                 if (r != EMULATION_OK)  {
                         if (emulation_type & EMULTYPE_TRAP_UD)
                                 return EMULATE_FAIL;
-                        if (reexecute_instruction(vcpu, cr2))
+                        if (reexecute_instruction(vcpu, cr2,
+                                                  write_fault_to_spt))
                                 return EMULATE_DONE;
                         if (emulation_type & EMULTYPE_SKIP)
                                 return EMULATE_FAIL;
@@ -4896,7 +4931,7 @@ restart:
                 return EMULATE_DONE;
 
         if (r == EMULATION_FAILED) {
-                if (reexecute_instruction(vcpu, cr2))
+                if (reexecute_instruction(vcpu, cr2, write_fault_to_spt))
                         return EMULATE_DONE;
 
                 return handle_emulation_failure(vcpu);
@@ -5539,7 +5574,7 @@ static void inject_pending_event(struct kvm_vcpu *vcpu)
                         vcpu->arch.nmi_injected = true;
                         kvm_x86_ops->set_nmi(vcpu);
                 }
-        } else if (kvm_cpu_has_interrupt(vcpu)) {
+        } else if (kvm_cpu_has_injectable_intr(vcpu)) {
                 if (kvm_x86_ops->interrupt_allowed(vcpu)) {
                         kvm_queue_interrupt(vcpu, kvm_cpu_get_interrupt(vcpu),
                                             false);
@@ -5607,6 +5642,16 @@ static void kvm_gen_update_masterclock(struct kvm *kvm)
 #endif
 }
 
+static void update_eoi_exitmap(struct kvm_vcpu *vcpu)
+{
+        u64 eoi_exit_bitmap[4];
+
+        memset(eoi_exit_bitmap, 0, 32);
+
+        kvm_ioapic_calculate_eoi_exitmap(vcpu, eoi_exit_bitmap);
+        kvm_x86_ops->load_eoi_exitmap(vcpu, eoi_exit_bitmap);
+}
+
 static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
 {
         int r;
@@ -5660,6 +5705,8 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
                         kvm_handle_pmu_event(vcpu);
                 if (kvm_check_request(KVM_REQ_PMI, vcpu))
                         kvm_deliver_pmi(vcpu);
+                if (kvm_check_request(KVM_REQ_EOIBITMAP, vcpu))
+                        update_eoi_exitmap(vcpu);
         }
 
         if (kvm_check_request(KVM_REQ_EVENT, vcpu) || req_int_win) {
@@ -5668,10 +5715,17 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
                 /* enable NMI/IRQ window open exits if needed */
                 if (vcpu->arch.nmi_pending)
                         kvm_x86_ops->enable_nmi_window(vcpu);
-                else if (kvm_cpu_has_interrupt(vcpu) || req_int_win)
+                else if (kvm_cpu_has_injectable_intr(vcpu) || req_int_win)
                         kvm_x86_ops->enable_irq_window(vcpu);
 
                 if (kvm_lapic_enabled(vcpu)) {
+                        /*
+                         * Update architecture specific hints for APIC
+                         * virtual interrupt delivery.
+                         */
+                        if (kvm_x86_ops->hwapic_irr_update)
+                                kvm_x86_ops->hwapic_irr_update(vcpu,
+                                        kvm_lapic_find_highest_irr(vcpu));
                         update_cr8_intercept(vcpu);
                         kvm_lapic_sync_to_vapic(vcpu);
                 }
@@ -6661,6 +6715,7 @@ int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
                 goto fail_free_wbinvd_dirty_mask;
 
         vcpu->arch.ia32_tsc_adjust_msr = 0x0;
+        vcpu->arch.pv_time_enabled = false;
         kvm_async_pf_hash_reset(vcpu);
         kvm_pmu_init(vcpu);
 
@@ -6851,48 +6906,43 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm,
                                 struct kvm_memory_slot *memslot,
                                 struct kvm_memory_slot old,
                                 struct kvm_userspace_memory_region *mem,
-                                int user_alloc)
+                                bool user_alloc)
 {
         int npages = memslot->npages;
-        int map_flags = MAP_PRIVATE | MAP_ANONYMOUS;
-
-        /* Prevent internal slot pages from being moved by fork()/COW. */
-        if (memslot->id >= KVM_MEMORY_SLOTS)
-                map_flags = MAP_SHARED | MAP_ANONYMOUS;
 
-        /*To keep backward compatibility with older userspace,
-         *x86 needs to handle !user_alloc case.
+        /*
+         * Only private memory slots need to be mapped here since
+         * KVM_SET_MEMORY_REGION ioctl is no longer supported.
          */
-        if (!user_alloc) {
-                if (npages && !old.npages) {
-                        unsigned long userspace_addr;
+        if ((memslot->id >= KVM_USER_MEM_SLOTS) && npages && !old.npages) {
+                unsigned long userspace_addr;
 
-                        userspace_addr = vm_mmap(NULL, 0,
-                                                 npages * PAGE_SIZE,
-                                                 PROT_READ | PROT_WRITE,
-                                                 map_flags,
-                                                 0);
+                /*
+                 * MAP_SHARED to prevent internal slot pages from being moved
+                 * by fork()/COW.
+                 */
+                userspace_addr = vm_mmap(NULL, 0, npages * PAGE_SIZE,
+                                         PROT_READ | PROT_WRITE,
+                                         MAP_SHARED | MAP_ANONYMOUS, 0);
 
-                        if (IS_ERR((void *)userspace_addr))
-                                return PTR_ERR((void *)userspace_addr);
+                if (IS_ERR((void *)userspace_addr))
+                        return PTR_ERR((void *)userspace_addr);
 
-                        memslot->userspace_addr = userspace_addr;
-                }
+                memslot->userspace_addr = userspace_addr;
         }
 
-
         return 0;
 }
 
 void kvm_arch_commit_memory_region(struct kvm *kvm,
                                 struct kvm_userspace_memory_region *mem,
                                 struct kvm_memory_slot old,
-                                int user_alloc)
+                                bool user_alloc)
 {
 
         int nr_mmu_pages = 0, npages = mem->memory_size >> PAGE_SHIFT;
 
-        if (!user_alloc && !old.user_alloc && old.npages && !npages) {
+        if ((mem->slot >= KVM_USER_MEM_SLOTS) && old.npages && !npages) {
                 int ret;
 
                 ret = vm_munmap(old.userspace_addr,
@@ -6906,11 +6956,15 @@ void kvm_arch_commit_memory_region(struct kvm *kvm,
         if (!kvm->arch.n_requested_mmu_pages)
                 nr_mmu_pages = kvm_mmu_calculate_mmu_pages(kvm);
 
-        spin_lock(&kvm->mmu_lock);
         if (nr_mmu_pages)
                 kvm_mmu_change_mmu_pages(kvm, nr_mmu_pages);
-        kvm_mmu_slot_remove_write_access(kvm, mem->slot);
-        spin_unlock(&kvm->mmu_lock);
+        /*
+         * Write protect all pages for dirty logging.
+         * Existing largepage mappings are destroyed here and new ones will
+         * not be created until the end of the logging.
+         */
+        if (npages && (mem->flags & KVM_MEM_LOG_DIRTY_PAGES))
+                kvm_mmu_slot_remove_write_access(kvm, mem->slot);
         /*
          * If memory slot is created, or moved, we need to clear all
          * mmio sptes.
diff --git a/arch/x86/lguest/Kconfig b/arch/x86/lguest/Kconfig
index 7872a3330fb5..29043d2048a0 100644
--- a/arch/x86/lguest/Kconfig
+++ b/arch/x86/lguest/Kconfig
@@ -2,6 +2,7 @@ config LGUEST_GUEST
         bool "Lguest guest support"
         select PARAVIRT
         depends on X86_32
+        select TTY
         select VIRTUALIZATION
         select VIRTIO
         select VIRTIO_CONSOLE
diff --git a/arch/x86/lguest/boot.c b/arch/x86/lguest/boot.c
index df4176cdbb32..1cbd89ca5569 100644
--- a/arch/x86/lguest/boot.c
+++ b/arch/x86/lguest/boot.c
@@ -552,7 +552,8 @@ static void lguest_write_cr3(unsigned long cr3)
         current_cr3 = cr3;
 
         /* These two page tables are simple, linear, and used during boot */
-        if (cr3 != __pa(swapper_pg_dir) && cr3 != __pa(initial_page_table))
+        if (cr3 != __pa_symbol(swapper_pg_dir) &&
+            cr3 != __pa_symbol(initial_page_table))
                 cr3_changed = true;
 }
 
diff --git a/arch/x86/lib/getuser.S b/arch/x86/lib/getuser.S
index 156b9c804670..a4512359656a 100644
--- a/arch/x86/lib/getuser.S
+++ b/arch/x86/lib/getuser.S
@@ -15,11 +15,10 @@
  * __get_user_X
  *
  * Inputs:      %[r|e]ax contains the address.
- *              The register is modified, but all changes are undone
- *              before returning because the C code doesn't know about it.
  *
  * Outputs:     %[r|e]ax is error code (0 or -EFAULT)
  *              %[r|e]dx contains zero-extended value
+ *              %ecx contains the high half for 32-bit __get_user_8
  *
  *
  * These functions should not modify any other registers,
@@ -42,7 +41,7 @@ ENTRY(__get_user_1)
         cmp TI_addr_limit(%_ASM_DX),%_ASM_AX
         jae bad_get_user
         ASM_STAC
-1:      movzb (%_ASM_AX),%edx
+1:      movzbl (%_ASM_AX),%edx
         xor %eax,%eax
         ASM_CLAC
         ret
@@ -72,29 +71,42 @@ ENTRY(__get_user_4)
         cmp TI_addr_limit(%_ASM_DX),%_ASM_AX
         jae bad_get_user
         ASM_STAC
-3:      mov -3(%_ASM_AX),%edx
+3:      movl -3(%_ASM_AX),%edx
         xor %eax,%eax
         ASM_CLAC
         ret
         CFI_ENDPROC
 ENDPROC(__get_user_4)
 
-#ifdef CONFIG_X86_64
 ENTRY(__get_user_8)
         CFI_STARTPROC
+#ifdef CONFIG_X86_64
         add $7,%_ASM_AX
         jc bad_get_user
         GET_THREAD_INFO(%_ASM_DX)
         cmp TI_addr_limit(%_ASM_DX),%_ASM_AX
-        jae     bad_get_user
+        jae bad_get_user
         ASM_STAC
-4:      movq -7(%_ASM_AX),%_ASM_DX
+4:      movq -7(%_ASM_AX),%rdx
         xor %eax,%eax
         ASM_CLAC
         ret
+#else
+        add $7,%_ASM_AX
+        jc bad_get_user_8
+        GET_THREAD_INFO(%_ASM_DX)
+        cmp TI_addr_limit(%_ASM_DX),%_ASM_AX
+        jae bad_get_user_8
+        ASM_STAC
+4:      movl -7(%_ASM_AX),%edx
+5:      movl -3(%_ASM_AX),%ecx
+        xor %eax,%eax
+        ASM_CLAC
+        ret
+#endif
         CFI_ENDPROC
 ENDPROC(__get_user_8)
-#endif
+
 
 bad_get_user:
         CFI_STARTPROC
@@ -105,9 +117,24 @@ bad_get_user:
         CFI_ENDPROC
 END(bad_get_user)
 
+#ifdef CONFIG_X86_32
+bad_get_user_8:
+        CFI_STARTPROC
+        xor %edx,%edx
+        xor %ecx,%ecx
+        mov $(-EFAULT),%_ASM_AX
+        ASM_CLAC
+        ret
+        CFI_ENDPROC
+END(bad_get_user_8)
+#endif
+
         _ASM_EXTABLE(1b,bad_get_user)
         _ASM_EXTABLE(2b,bad_get_user)
         _ASM_EXTABLE(3b,bad_get_user)
 #ifdef CONFIG_X86_64
         _ASM_EXTABLE(4b,bad_get_user)
+#else
+        _ASM_EXTABLE(4b,bad_get_user_8)
+        _ASM_EXTABLE(5b,bad_get_user_8)
 #endif
diff --git a/arch/x86/lib/usercopy_64.c b/arch/x86/lib/usercopy_64.c
index 05928aae911e..906fea315791 100644
--- a/arch/x86/lib/usercopy_64.c
+++ b/arch/x86/lib/usercopy_64.c
@@ -74,10 +74,10 @@ copy_user_handle_tail(char *to, char *from, unsigned len, unsigned zerorest)
         char c;
         unsigned zero_len;
 
-        for (; len; --len) {
+        for (; len; --len, to++) {
                 if (__get_user_nocheck(c, from++, sizeof(char)))
                         break;
-                if (__put_user_nocheck(c, to++, sizeof(char)))
+                if (__put_user_nocheck(c, to, sizeof(char)))
                         break;
         }
 
diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c
index 027088f2f7dd..2b97525246d4 100644
--- a/arch/x86/mm/fault.c
+++ b/arch/x86/mm/fault.c
@@ -748,13 +748,15 @@ __bad_area_nosemaphore(struct pt_regs *regs, unsigned long error_code,
                                 return;
                 }
 #endif
+                /* Kernel addresses are always protection faults: */
+                if (address >= TASK_SIZE)
+                        error_code |= PF_PROT;
 
-                if (unlikely(show_unhandled_signals))
+                if (likely(show_unhandled_signals))
                         show_signal_msg(regs, error_code, address, tsk);
 
-                /* Kernel addresses are always protection faults: */
                 tsk->thread.cr2         = address;
-                tsk->thread.error_code  = error_code | (address >= TASK_SIZE);
+                tsk->thread.error_code  = error_code;
                 tsk->thread.trap_nr     = X86_TRAP_PF;
 
                 force_sig_info_fault(SIGSEGV, si_code, address, tsk, 0);
@@ -937,14 +939,8 @@ spurious_fault(unsigned long error_code, unsigned long address)
         if (pmd_large(*pmd))
                 return spurious_fault_check(error_code, (pte_t *) pmd);
 
-        /*
-         * Note: don't use pte_present() here, since it returns true
-         * if the _PAGE_PROTNONE bit is set.  However, this aliases the
-         * _PAGE_GLOBAL bit, which for kernel pages give false positives
-         * when CONFIG_DEBUG_PAGEALLOC is used.
-         */
         pte = pte_offset_kernel(pmd, address);
-        if (!(pte_flags(*pte) & _PAGE_PRESENT))
+        if (!pte_present(*pte))
                 return 0;
 
         ret = spurious_fault_check(error_code, pte);
diff --git a/arch/x86/mm/init.c b/arch/x86/mm/init.c
index d7aea41563b3..59b7fc453277 100644
--- a/arch/x86/mm/init.c
+++ b/arch/x86/mm/init.c
@@ -16,87 +16,134 @@
 #include <asm/tlb.h>
 #include <asm/proto.h>
 #include <asm/dma.h>            /* for MAX_DMA_PFN */
+#include <asm/microcode.h>
 
-unsigned long __initdata pgt_buf_start;
-unsigned long __meminitdata pgt_buf_end;
-unsigned long __meminitdata pgt_buf_top;
+#include "mm_internal.h"
 
-int after_bootmem;
+static unsigned long __initdata pgt_buf_start;
+static unsigned long __initdata pgt_buf_end;
+static unsigned long __initdata pgt_buf_top;
 
-int direct_gbpages
-#ifdef CONFIG_DIRECT_GBPAGES
-                                = 1
-#endif
-;
+static unsigned long min_pfn_mapped;
 
-struct map_range {
-        unsigned long start;
-        unsigned long end;
-        unsigned page_size_mask;
-};
+static bool __initdata can_use_brk_pgt = true;
 
 /*
- * First calculate space needed for kernel direct mapping page tables to cover
- * mr[0].start to mr[nr_range - 1].end, while accounting for possible 2M and 1GB
- * pages. Then find enough contiguous space for those page tables.
+ * Pages returned are already directly mapped.
+ *
+ * Changing that is likely to break Xen, see commit:
+ *
+ *    279b706 x86,xen: introduce x86_init.mapping.pagetable_reserve
+ *
+ * for detailed information.
  */
-static void __init find_early_table_space(struct map_range *mr, int nr_range)
+__ref void *alloc_low_pages(unsigned int num)
 {
+        unsigned long pfn;
         int i;
-        unsigned long puds = 0, pmds = 0, ptes = 0, tables;
-        unsigned long start = 0, good_end;
-        phys_addr_t base;
 
-        for (i = 0; i < nr_range; i++) {
-                unsigned long range, extra;
+        if (after_bootmem) {
+                unsigned int order;
 
-                range = mr[i].end - mr[i].start;
-                puds += (range + PUD_SIZE - 1) >> PUD_SHIFT;
+                order = get_order((unsigned long)num << PAGE_SHIFT);
+                return (void *)__get_free_pages(GFP_ATOMIC | __GFP_NOTRACK |
+                                                __GFP_ZERO, order);
+        }
 
-                if (mr[i].page_size_mask & (1 << PG_LEVEL_1G)) {
-                        extra = range - ((range >> PUD_SHIFT) << PUD_SHIFT);
-                        pmds += (extra + PMD_SIZE - 1) >> PMD_SHIFT;
-                } else {
-                        pmds += (range + PMD_SIZE - 1) >> PMD_SHIFT;
-                }
+        if ((pgt_buf_end + num) > pgt_buf_top || !can_use_brk_pgt) {
+                unsigned long ret;
+                if (min_pfn_mapped >= max_pfn_mapped)
+                        panic("alloc_low_page: ran out of memory");
+                ret = memblock_find_in_range(min_pfn_mapped << PAGE_SHIFT,
+                                        max_pfn_mapped << PAGE_SHIFT,
+                                        PAGE_SIZE * num , PAGE_SIZE);
+                if (!ret)
+                        panic("alloc_low_page: can not alloc memory");
+                memblock_reserve(ret, PAGE_SIZE * num);
+                pfn = ret >> PAGE_SHIFT;
+        } else {
+                pfn = pgt_buf_end;
+                pgt_buf_end += num;
+                printk(KERN_DEBUG "BRK [%#010lx, %#010lx] PGTABLE\n",
+                        pfn << PAGE_SHIFT, (pgt_buf_end << PAGE_SHIFT) - 1);
+        }
 
-                if (mr[i].page_size_mask & (1 << PG_LEVEL_2M)) {
-                        extra = range - ((range >> PMD_SHIFT) << PMD_SHIFT);
-#ifdef CONFIG_X86_32
-                        extra += PMD_SIZE;
-#endif
-                        ptes += (extra + PAGE_SIZE - 1) >> PAGE_SHIFT;
-                } else {
-                        ptes += (range + PAGE_SIZE - 1) >> PAGE_SHIFT;
-                }
+        for (i = 0; i < num; i++) {
+                void *adr;
+
+                adr = __va((pfn + i) << PAGE_SHIFT);
+                clear_page(adr);
         }
 
-        tables = roundup(puds * sizeof(pud_t), PAGE_SIZE);
-        tables += roundup(pmds * sizeof(pmd_t), PAGE_SIZE);
-        tables += roundup(ptes * sizeof(pte_t), PAGE_SIZE);
+        return __va(pfn << PAGE_SHIFT);
+}
 
-#ifdef CONFIG_X86_32
-        /* for fixmap */
-        tables += roundup(__end_of_fixed_addresses * sizeof(pte_t), PAGE_SIZE);
-#endif
-        good_end = max_pfn_mapped << PAGE_SHIFT;
+/* need 4 4k for initial PMD_SIZE, 4k for 0-ISA_END_ADDRESS */
+#define INIT_PGT_BUF_SIZE       (5 * PAGE_SIZE)
+RESERVE_BRK(early_pgt_alloc, INIT_PGT_BUF_SIZE);
+void  __init early_alloc_pgt_buf(void)
+{
+        unsigned long tables = INIT_PGT_BUF_SIZE;
+        phys_addr_t base;
 
-        base = memblock_find_in_range(start, good_end, tables, PAGE_SIZE);
-        if (!base)
-                panic("Cannot find space for the kernel page tables");
+        base = __pa(extend_brk(tables, PAGE_SIZE));
 
         pgt_buf_start = base >> PAGE_SHIFT;
         pgt_buf_end = pgt_buf_start;
         pgt_buf_top = pgt_buf_start + (tables >> PAGE_SHIFT);
+}
 
-        printk(KERN_DEBUG "kernel direct mapping tables up to %#lx @ [mem %#010lx-%#010lx]\n",
-                mr[nr_range - 1].end - 1, pgt_buf_start << PAGE_SHIFT,
-                (pgt_buf_top << PAGE_SHIFT) - 1);
+int after_bootmem;
+
+int direct_gbpages
+#ifdef CONFIG_DIRECT_GBPAGES
+                                = 1
+#endif
+;
+
+static void __init init_gbpages(void)
+{
+#ifdef CONFIG_X86_64
+        if (direct_gbpages && cpu_has_gbpages)
+                printk(KERN_INFO "Using GB pages for direct mapping\n");
+        else
+                direct_gbpages = 0;
+#endif
 }
 
-void __init native_pagetable_reserve(u64 start, u64 end)
+struct map_range {
+        unsigned long start;
+        unsigned long end;
+        unsigned page_size_mask;
+};
+
+static int page_size_mask;
+
+static void __init probe_page_size_mask(void)
 {
-        memblock_reserve(start, end - start);
+        init_gbpages();
+
+#if !defined(CONFIG_DEBUG_PAGEALLOC) && !defined(CONFIG_KMEMCHECK)
+        /*
+         * For CONFIG_DEBUG_PAGEALLOC, identity mapping will use small pages.
+         * This will simplify cpa(), which otherwise needs to support splitting
+         * large pages into small in interrupt context, etc.
+         */
+        if (direct_gbpages)
+                page_size_mask |= 1 << PG_LEVEL_1G;
+        if (cpu_has_pse)
+                page_size_mask |= 1 << PG_LEVEL_2M;
+#endif
+
+        /* Enable PSE if available */
+        if (cpu_has_pse)
+                set_in_cr4(X86_CR4_PSE);
+
+        /* Enable PGE if available */
+        if (cpu_has_pge) {
+                set_in_cr4(X86_CR4_PGE);
+                __supported_pte_mask |= _PAGE_GLOBAL;
+        }
 }
 
 #ifdef CONFIG_X86_32
@@ -122,58 +169,51 @@ static int __meminit save_mr(struct map_range *mr, int nr_range,
 }
 
 /*
- * Setup the direct mapping of the physical memory at PAGE_OFFSET.
- * This runs before bootmem is initialized and gets pages directly from
- * the physical memory. To access them they are temporarily mapped.
+ * adjust the page_size_mask for small range to go with
+ *      big page size instead small one if nearby are ram too.
  */
-unsigned long __init_refok init_memory_mapping(unsigned long start,
-                                               unsigned long end)
+static void __init_refok adjust_range_page_size_mask(struct map_range *mr,
+                                                         int nr_range)
 {
-        unsigned long page_size_mask = 0;
-        unsigned long start_pfn, end_pfn;
-        unsigned long ret = 0;
-        unsigned long pos;
-
-        struct map_range mr[NR_RANGE_MR];
-        int nr_range, i;
-        int use_pse, use_gbpages;
+        int i;
 
-        printk(KERN_INFO "init_memory_mapping: [mem %#010lx-%#010lx]\n",
-               start, end - 1);
+        for (i = 0; i < nr_range; i++) {
+                if ((page_size_mask & (1<<PG_LEVEL_2M)) &&
+                    !(mr[i].page_size_mask & (1<<PG_LEVEL_2M))) {
+                        unsigned long start = round_down(mr[i].start, PMD_SIZE);
+                        unsigned long end = round_up(mr[i].end, PMD_SIZE);
 
-#if defined(CONFIG_DEBUG_PAGEALLOC) || defined(CONFIG_KMEMCHECK)
-        /*
-         * For CONFIG_DEBUG_PAGEALLOC, identity mapping will use small pages.
-         * This will simplify cpa(), which otherwise needs to support splitting
-         * large pages into small in interrupt context, etc.
-         */
-        use_pse = use_gbpages = 0;
-#else
-        use_pse = cpu_has_pse;
-        use_gbpages = direct_gbpages;
+#ifdef CONFIG_X86_32
+                        if ((end >> PAGE_SHIFT) > max_low_pfn)
+                                continue;
 #endif
 
-        /* Enable PSE if available */
-        if (cpu_has_pse)
-                set_in_cr4(X86_CR4_PSE);
+                        if (memblock_is_region_memory(start, end - start))
+                                mr[i].page_size_mask |= 1<<PG_LEVEL_2M;
+                }
+                if ((page_size_mask & (1<<PG_LEVEL_1G)) &&
+                    !(mr[i].page_size_mask & (1<<PG_LEVEL_1G))) {
+                        unsigned long start = round_down(mr[i].start, PUD_SIZE);
+                        unsigned long end = round_up(mr[i].end, PUD_SIZE);
 
-        /* Enable PGE if available */
-        if (cpu_has_pge) {
-                set_in_cr4(X86_CR4_PGE);
-                __supported_pte_mask |= _PAGE_GLOBAL;
+                        if (memblock_is_region_memory(start, end - start))
+                                mr[i].page_size_mask |= 1<<PG_LEVEL_1G;
+                }
         }
+}
 
-        if (use_gbpages)
-                page_size_mask |= 1 << PG_LEVEL_1G;
-        if (use_pse)
-                page_size_mask |= 1 << PG_LEVEL_2M;
+static int __meminit split_mem_range(struct map_range *mr, int nr_range,
+                                     unsigned long start,
+                                     unsigned long end)
+{
+        unsigned long start_pfn, end_pfn, limit_pfn;
+        unsigned long pfn;
+        int i;
 
-        memset(mr, 0, sizeof(mr));
-        nr_range = 0;
+        limit_pfn = PFN_DOWN(end);
 
         /* head if not big page alignment ? */
-        start_pfn = start >> PAGE_SHIFT;
-        pos = start_pfn << PAGE_SHIFT;
+        pfn = start_pfn = PFN_DOWN(start);
 #ifdef CONFIG_X86_32
         /*
          * Don't use a large page for the first 2/4MB of memory
@@ -181,66 +221,60 @@ unsigned long __init_refok init_memory_mapping(unsigned long start,
          * and overlapping MTRRs into large pages can cause
          * slowdowns.
          */
-        if (pos == 0)
-                end_pfn = 1<<(PMD_SHIFT - PAGE_SHIFT);
+        if (pfn == 0)
+                end_pfn = PFN_DOWN(PMD_SIZE);
         else
-                end_pfn = ((pos + (PMD_SIZE - 1))>>PMD_SHIFT)
-                                 << (PMD_SHIFT - PAGE_SHIFT);
+                end_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE));
 #else /* CONFIG_X86_64 */
-        end_pfn = ((pos + (PMD_SIZE - 1)) >> PMD_SHIFT)
-                        << (PMD_SHIFT - PAGE_SHIFT);
+        end_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE));
 #endif
-        if (end_pfn > (end >> PAGE_SHIFT))
-                end_pfn = end >> PAGE_SHIFT;
+        if (end_pfn > limit_pfn)
+                end_pfn = limit_pfn;
         if (start_pfn < end_pfn) {
                 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0);
-                pos = end_pfn << PAGE_SHIFT;
+                pfn = end_pfn;
         }
 
         /* big page (2M) range */
-        start_pfn = ((pos + (PMD_SIZE - 1))>>PMD_SHIFT)
-                         << (PMD_SHIFT - PAGE_SHIFT);
+        start_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE));
 #ifdef CONFIG_X86_32
-        end_pfn = (end>>PMD_SHIFT) << (PMD_SHIFT - PAGE_SHIFT);
+        end_pfn = round_down(limit_pfn, PFN_DOWN(PMD_SIZE));
 #else /* CONFIG_X86_64 */
-        end_pfn = ((pos + (PUD_SIZE - 1))>>PUD_SHIFT)
-                         << (PUD_SHIFT - PAGE_SHIFT);
-        if (end_pfn > ((end>>PMD_SHIFT)<<(PMD_SHIFT - PAGE_SHIFT)))
-                end_pfn = ((end>>PMD_SHIFT)<<(PMD_SHIFT - PAGE_SHIFT));
+        end_pfn = round_up(pfn, PFN_DOWN(PUD_SIZE));
+        if (end_pfn > round_down(limit_pfn, PFN_DOWN(PMD_SIZE)))
+                end_pfn = round_down(limit_pfn, PFN_DOWN(PMD_SIZE));
 #endif
 
         if (start_pfn < end_pfn) {
                 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn,
                                 page_size_mask & (1<<PG_LEVEL_2M));
-                pos = end_pfn << PAGE_SHIFT;
+                pfn = end_pfn;
         }
 
 #ifdef CONFIG_X86_64
         /* big page (1G) range */
-        start_pfn = ((pos + (PUD_SIZE - 1))>>PUD_SHIFT)
-                         << (PUD_SHIFT - PAGE_SHIFT);
-        end_pfn = (end >> PUD_SHIFT) << (PUD_SHIFT - PAGE_SHIFT);
+        start_pfn = round_up(pfn, PFN_DOWN(PUD_SIZE));
+        end_pfn = round_down(limit_pfn, PFN_DOWN(PUD_SIZE));
         if (start_pfn < end_pfn) {
                 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn,
                                 page_size_mask &
                                  ((1<<PG_LEVEL_2M)|(1<<PG_LEVEL_1G)));
-                pos = end_pfn << PAGE_SHIFT;
+                pfn = end_pfn;
         }
 
         /* tail is not big page (1G) alignment */
-        start_pfn = ((pos + (PMD_SIZE - 1))>>PMD_SHIFT)
-                         << (PMD_SHIFT - PAGE_SHIFT);
-        end_pfn = (end >> PMD_SHIFT) << (PMD_SHIFT - PAGE_SHIFT);
+        start_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE));
+        end_pfn = round_down(limit_pfn, PFN_DOWN(PMD_SIZE));
         if (start_pfn < end_pfn) {
                 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn,
                                 page_size_mask & (1<<PG_LEVEL_2M));
-                pos = end_pfn << PAGE_SHIFT;
+                pfn = end_pfn;
         }
 #endif
 
         /* tail is not big page (2M) alignment */
-        start_pfn = pos>>PAGE_SHIFT;
-        end_pfn = end>>PAGE_SHIFT;
+        start_pfn = pfn;
+        end_pfn = limit_pfn;
         nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0);
 
         /* try to merge same page size and continuous */
@@ -257,59 +291,168 @@ unsigned long __init_refok init_memory_mapping(unsigned long start,
                 nr_range--;
         }
 
+        if (!after_bootmem)
+                adjust_range_page_size_mask(mr, nr_range);
+
         for (i = 0; i < nr_range; i++)
                 printk(KERN_DEBUG " [mem %#010lx-%#010lx] page %s\n",
                                 mr[i].start, mr[i].end - 1,
                         (mr[i].page_size_mask & (1<<PG_LEVEL_1G))?"1G":(
                          (mr[i].page_size_mask & (1<<PG_LEVEL_2M))?"2M":"4k"));
 
-        /*
-         * Find space for the kernel direct mapping tables.
-         *
-         * Later we should allocate these tables in the local node of the
-         * memory mapped. Unfortunately this is done currently before the
-         * nodes are discovered.
-         */
-        if (!after_bootmem)
-                find_early_table_space(mr, nr_range);
+        return nr_range;
+}
+
+struct range pfn_mapped[E820_X_MAX];
+int nr_pfn_mapped;
+
+static void add_pfn_range_mapped(unsigned long start_pfn, unsigned long end_pfn)
+{
+        nr_pfn_mapped = add_range_with_merge(pfn_mapped, E820_X_MAX,
+                                             nr_pfn_mapped, start_pfn, end_pfn);
+        nr_pfn_mapped = clean_sort_range(pfn_mapped, E820_X_MAX);
+
+        max_pfn_mapped = max(max_pfn_mapped, end_pfn);
+
+        if (start_pfn < (1UL<<(32-PAGE_SHIFT)))
+                max_low_pfn_mapped = max(max_low_pfn_mapped,
+                                         min(end_pfn, 1UL<<(32-PAGE_SHIFT)));
+}
+
+bool pfn_range_is_mapped(unsigned long start_pfn, unsigned long end_pfn)
+{
+        int i;
+
+        for (i = 0; i < nr_pfn_mapped; i++)
+                if ((start_pfn >= pfn_mapped[i].start) &&
+                    (end_pfn <= pfn_mapped[i].end))
+                        return true;
+
+        return false;
+}
+
+/*
+ * Setup the direct mapping of the physical memory at PAGE_OFFSET.
+ * This runs before bootmem is initialized and gets pages directly from
+ * the physical memory. To access them they are temporarily mapped.
+ */
+unsigned long __init_refok init_memory_mapping(unsigned long start,
+                                               unsigned long end)
+{
+        struct map_range mr[NR_RANGE_MR];
+        unsigned long ret = 0;
+        int nr_range, i;
+
+        pr_info("init_memory_mapping: [mem %#010lx-%#010lx]\n",
+               start, end - 1);
+
+        memset(mr, 0, sizeof(mr));
+        nr_range = split_mem_range(mr, 0, start, end);
 
         for (i = 0; i < nr_range; i++)
                 ret = kernel_physical_mapping_init(mr[i].start, mr[i].end,
                                                    mr[i].page_size_mask);
 
-#ifdef CONFIG_X86_32
-        early_ioremap_page_table_range_init();
+        add_pfn_range_mapped(start >> PAGE_SHIFT, ret >> PAGE_SHIFT);
 
-        load_cr3(swapper_pg_dir);
-#endif
+        return ret >> PAGE_SHIFT;
+}
 
-        __flush_tlb_all();
+/*
+ * would have hole in the middle or ends, and only ram parts will be mapped.
+ */
+static unsigned long __init init_range_memory_mapping(
+                                           unsigned long r_start,
+                                           unsigned long r_end)
+{
+        unsigned long start_pfn, end_pfn;
+        unsigned long mapped_ram_size = 0;
+        int i;
 
-        /*
-         * Reserve the kernel pagetable pages we used (pgt_buf_start -
-         * pgt_buf_end) and free the other ones (pgt_buf_end - pgt_buf_top)
-         * so that they can be reused for other purposes.
-         *
-         * On native it just means calling memblock_reserve, on Xen it also
-         * means marking RW the pagetable pages that we allocated before
-         * but that haven't been used.
-         *
-         * In fact on xen we mark RO the whole range pgt_buf_start -
-         * pgt_buf_top, because we have to make sure that when
-         * init_memory_mapping reaches the pagetable pages area, it maps
-         * RO all the pagetable pages, including the ones that are beyond
-         * pgt_buf_end at that time.
-         */
-        if (!after_bootmem && pgt_buf_end > pgt_buf_start)
-                x86_init.mapping.pagetable_reserve(PFN_PHYS(pgt_buf_start),
-                                PFN_PHYS(pgt_buf_end));
+        for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, NULL) {
+                u64 start = clamp_val(PFN_PHYS(start_pfn), r_start, r_end);
+                u64 end = clamp_val(PFN_PHYS(end_pfn), r_start, r_end);
+                if (start >= end)
+                        continue;
 
-        if (!after_bootmem)
-                early_memtest(start, end);
+                /*
+                 * if it is overlapping with brk pgt, we need to
+                 * alloc pgt buf from memblock instead.
+                 */
+                can_use_brk_pgt = max(start, (u64)pgt_buf_end<<PAGE_SHIFT) >=
+                                    min(end, (u64)pgt_buf_top<<PAGE_SHIFT);
+                init_memory_mapping(start, end);
+                mapped_ram_size += end - start;
+                can_use_brk_pgt = true;
+        }
 
-        return ret >> PAGE_SHIFT;
+        return mapped_ram_size;
 }
 
+/* (PUD_SHIFT-PMD_SHIFT)/2 */
+#define STEP_SIZE_SHIFT 5
+void __init init_mem_mapping(void)
+{
+        unsigned long end, real_end, start, last_start;
+        unsigned long step_size;
+        unsigned long addr;
+        unsigned long mapped_ram_size = 0;
+        unsigned long new_mapped_ram_size;
+
+        probe_page_size_mask();
+
+#ifdef CONFIG_X86_64
+        end = max_pfn << PAGE_SHIFT;
+#else
+        end = max_low_pfn << PAGE_SHIFT;
+#endif
+
+        /* the ISA range is always mapped regardless of memory holes */
+        init_memory_mapping(0, ISA_END_ADDRESS);
+
+        /* xen has big range in reserved near end of ram, skip it at first.*/
+        addr = memblock_find_in_range(ISA_END_ADDRESS, end, PMD_SIZE, PMD_SIZE);
+        real_end = addr + PMD_SIZE;
+
+        /* step_size need to be small so pgt_buf from BRK could cover it */
+        step_size = PMD_SIZE;
+        max_pfn_mapped = 0; /* will get exact value next */
+        min_pfn_mapped = real_end >> PAGE_SHIFT;
+        last_start = start = real_end;
+        while (last_start > ISA_END_ADDRESS) {
+                if (last_start > step_size) {
+                        start = round_down(last_start - 1, step_size);
+                        if (start < ISA_END_ADDRESS)
+                                start = ISA_END_ADDRESS;
+                } else
+                        start = ISA_END_ADDRESS;
+                new_mapped_ram_size = init_range_memory_mapping(start,
+                                                        last_start);
+                last_start = start;
+                min_pfn_mapped = last_start >> PAGE_SHIFT;
+                /* only increase step_size after big range get mapped */
+                if (new_mapped_ram_size > mapped_ram_size)
+                        step_size <<= STEP_SIZE_SHIFT;
+                mapped_ram_size += new_mapped_ram_size;
+        }
+
+        if (real_end < end)
+                init_range_memory_mapping(real_end, end);
+
+#ifdef CONFIG_X86_64
+        if (max_pfn > max_low_pfn) {
+                /* can we preseve max_low_pfn ?*/
+                max_low_pfn = max_pfn;
+        }
+#else
+        early_ioremap_page_table_range_init();
+#endif
+
+        load_cr3(swapper_pg_dir);
+        __flush_tlb_all();
+
+        early_memtest(0, max_pfn_mapped << PAGE_SHIFT);
+}
 
 /*
  * devmem_is_allowed() checks to see if /dev/mem access to a certain address
@@ -391,6 +534,15 @@ void free_initmem(void)
 #ifdef CONFIG_BLK_DEV_INITRD
 void __init free_initrd_mem(unsigned long start, unsigned long end)
 {
+#ifdef CONFIG_MICROCODE_EARLY
+        /*
+         * Remember, initrd memory may contain microcode or other useful things.
+         * Before we lose initrd mem, we need to find a place to hold them
+         * now that normal virtual memory is enabled.
+         */
+        save_microcode_in_initrd();
+#endif
+
         /*
          * end could be not aligned, and We can not align that,
          * decompresser could be confused by aligned initrd_end
diff --git a/arch/x86/mm/init_32.c b/arch/x86/mm/init_32.c
index 745d66b843c8..2d19001151d5 100644
--- a/arch/x86/mm/init_32.c
+++ b/arch/x86/mm/init_32.c
@@ -53,25 +53,14 @@
 #include <asm/page_types.h>
 #include <asm/init.h>
 
+#include "mm_internal.h"
+
 unsigned long highstart_pfn, highend_pfn;
 
 static noinline int do_test_wp_bit(void);
 
 bool __read_mostly __vmalloc_start_set = false;
 
-static __init void *alloc_low_page(void)
-{
-        unsigned long pfn = pgt_buf_end++;
-        void *adr;
-
-        if (pfn >= pgt_buf_top)
-                panic("alloc_low_page: ran out of memory");
-
-        adr = __va(pfn * PAGE_SIZE);
-        clear_page(adr);
-        return adr;
-}
-
 /*
  * Creates a middle page table and puts a pointer to it in the
  * given global directory entry. This only returns the gd entry
@@ -84,10 +73,7 @@ static pmd_t * __init one_md_table_init(pgd_t *pgd)
 
 #ifdef CONFIG_X86_PAE
         if (!(pgd_val(*pgd) & _PAGE_PRESENT)) {
-                if (after_bootmem)
-                        pmd_table = (pmd_t *)alloc_bootmem_pages(PAGE_SIZE);
-                else
-                        pmd_table = (pmd_t *)alloc_low_page();
+                pmd_table = (pmd_t *)alloc_low_page();
                 paravirt_alloc_pmd(&init_mm, __pa(pmd_table) >> PAGE_SHIFT);
                 set_pgd(pgd, __pgd(__pa(pmd_table) | _PAGE_PRESENT));
                 pud = pud_offset(pgd, 0);
@@ -109,17 +95,7 @@ static pmd_t * __init one_md_table_init(pgd_t *pgd)
 static pte_t * __init one_page_table_init(pmd_t *pmd)
 {
         if (!(pmd_val(*pmd) & _PAGE_PRESENT)) {
-                pte_t *page_table = NULL;
-
-                if (after_bootmem) {
-#if defined(CONFIG_DEBUG_PAGEALLOC) || defined(CONFIG_KMEMCHECK)
-                        page_table = (pte_t *) alloc_bootmem_pages(PAGE_SIZE);
-#endif
-                        if (!page_table)
-                                page_table =
-                                (pte_t *)alloc_bootmem_pages(PAGE_SIZE);
-                } else
-                        page_table = (pte_t *)alloc_low_page();
+                pte_t *page_table = (pte_t *)alloc_low_page();
 
                 paravirt_alloc_pte(&init_mm, __pa(page_table) >> PAGE_SHIFT);
                 set_pmd(pmd, __pmd(__pa(page_table) | _PAGE_TABLE));
@@ -146,8 +122,39 @@ pte_t * __init populate_extra_pte(unsigned long vaddr)
         return one_page_table_init(pmd) + pte_idx;
 }
 
+static unsigned long __init
+page_table_range_init_count(unsigned long start, unsigned long end)
+{
+        unsigned long count = 0;
+#ifdef CONFIG_HIGHMEM
+        int pmd_idx_kmap_begin = fix_to_virt(FIX_KMAP_END) >> PMD_SHIFT;
+        int pmd_idx_kmap_end = fix_to_virt(FIX_KMAP_BEGIN) >> PMD_SHIFT;
+        int pgd_idx, pmd_idx;
+        unsigned long vaddr;
+
+        if (pmd_idx_kmap_begin == pmd_idx_kmap_end)
+                return 0;
+
+        vaddr = start;
+        pgd_idx = pgd_index(vaddr);
+
+        for ( ; (pgd_idx < PTRS_PER_PGD) && (vaddr != end); pgd_idx++) {
+                for (; (pmd_idx < PTRS_PER_PMD) && (vaddr != end);
+                                                        pmd_idx++) {
+                        if ((vaddr >> PMD_SHIFT) >= pmd_idx_kmap_begin &&
+                            (vaddr >> PMD_SHIFT) <= pmd_idx_kmap_end)
+                                count++;
+                        vaddr += PMD_SIZE;
+                }
+                pmd_idx = 0;
+        }
+#endif
+        return count;
+}
+
 static pte_t *__init page_table_kmap_check(pte_t *pte, pmd_t *pmd,
-                                           unsigned long vaddr, pte_t *lastpte)
+                                           unsigned long vaddr, pte_t *lastpte,
+                                           void **adr)
 {
 #ifdef CONFIG_HIGHMEM
         /*
@@ -161,16 +168,15 @@ static pte_t *__init page_table_kmap_check(pte_t *pte, pmd_t *pmd,
 
         if (pmd_idx_kmap_begin != pmd_idx_kmap_end
             && (vaddr >> PMD_SHIFT) >= pmd_idx_kmap_begin
-            && (vaddr >> PMD_SHIFT) <= pmd_idx_kmap_end
-            && ((__pa(pte) >> PAGE_SHIFT) < pgt_buf_start
-                || (__pa(pte) >> PAGE_SHIFT) >= pgt_buf_end)) {
+            && (vaddr >> PMD_SHIFT) <= pmd_idx_kmap_end) {
                 pte_t *newpte;
                 int i;
 
                 BUG_ON(after_bootmem);
-                newpte = alloc_low_page();
+                newpte = *adr;
                 for (i = 0; i < PTRS_PER_PTE; i++)
                         set_pte(newpte + i, pte[i]);
+                *adr = (void *)(((unsigned long)(*adr)) + PAGE_SIZE);
 
                 paravirt_alloc_pte(&init_mm, __pa(newpte) >> PAGE_SHIFT);
                 set_pmd(pmd, __pmd(__pa(newpte)|_PAGE_TABLE));
@@ -204,6 +210,11 @@ page_table_range_init(unsigned long start, unsigned long end, pgd_t *pgd_base)
         pgd_t *pgd;
         pmd_t *pmd;
         pte_t *pte = NULL;
+        unsigned long count = page_table_range_init_count(start, end);
+        void *adr = NULL;
+
+        if (count)
+                adr = alloc_low_pages(count);
 
         vaddr = start;
         pgd_idx = pgd_index(vaddr);
@@ -216,7 +227,7 @@ page_table_range_init(unsigned long start, unsigned long end, pgd_t *pgd_base)
                 for (; (pmd_idx < PTRS_PER_PMD) && (vaddr != end);
                                                         pmd++, pmd_idx++) {
                         pte = page_table_kmap_check(one_page_table_init(pmd),
-                                                    pmd, vaddr, pte);
+                                                    pmd, vaddr, pte, &adr);
 
                         vaddr += PMD_SIZE;
                 }
@@ -310,6 +321,7 @@ repeat:
                                         __pgprot(PTE_IDENT_ATTR |
                                                  _PAGE_PSE);
 
+                                pfn &= PMD_MASK >> PAGE_SHIFT;
                                 addr2 = (pfn + PTRS_PER_PTE-1) * PAGE_SIZE +
                                         PAGE_OFFSET + PAGE_SIZE-1;
 
@@ -455,9 +467,14 @@ void __init native_pagetable_init(void)
 
         /*
          * Remove any mappings which extend past the end of physical
-         * memory from the boot time page table:
+         * memory from the boot time page table.
+         * In virtual address space, we should have at least two pages
+         * from VMALLOC_END to pkmap or fixmap according to VMALLOC_END
+         * definition. And max_low_pfn is set to VMALLOC_END physical
+         * address. If initial memory mapping is doing right job, we
+         * should have pte used near max_low_pfn or one pmd is not present.
          */
-        for (pfn = max_low_pfn + 1; pfn < 1<<(32-PAGE_SHIFT); pfn++) {
+        for (pfn = max_low_pfn; pfn < 1<<(32-PAGE_SHIFT); pfn++) {
                 va = PAGE_OFFSET + (pfn<<PAGE_SHIFT);
                 pgd = base + pgd_index(va);
                 if (!pgd_present(*pgd))
@@ -468,10 +485,19 @@ void __init native_pagetable_init(void)
                 if (!pmd_present(*pmd))
                         break;
 
+                /* should not be large page here */
+                if (pmd_large(*pmd)) {
+                        pr_warn("try to clear pte for ram above max_low_pfn: pfn: %lx pmd: %p pmd phys: %lx, but pmd is big page and is not using pte !\n",
+                                pfn, pmd, __pa(pmd));
+                        BUG_ON(1);
+                }
+
                 pte = pte_offset_kernel(pmd, va);
                 if (!pte_present(*pte))
                         break;
 
+                printk(KERN_DEBUG "clearing pte for ram above max_low_pfn: pfn: %lx pmd: %p pmd phys: %lx pte: %p pte phys: %lx\n",
+                                pfn, pmd, __pa(pmd), pte, __pa(pte));
                 pte_clear(NULL, va, pte);
         }
         paravirt_alloc_pmd(&init_mm, __pa(base) >> PAGE_SHIFT);
@@ -550,7 +576,7 @@ early_param("highmem", parse_highmem);
  * artificially via the highmem=x boot parameter then create
  * it:
  */
-void __init lowmem_pfn_init(void)
+static void __init lowmem_pfn_init(void)
 {
         /* max_low_pfn is 0, we already have early_res support */
         max_low_pfn = max_pfn;
@@ -586,7 +612,7 @@ void __init lowmem_pfn_init(void)
  * We have more RAM than fits into lowmem - we try to put it into
  * highmem, also taking the highmem=x boot parameter into account:
  */
-void __init highmem_pfn_init(void)
+static void __init highmem_pfn_init(void)
 {
         max_low_pfn = MAXMEM_PFN;
 
@@ -669,8 +695,6 @@ void __init setup_bootmem_allocator(void)
         printk(KERN_INFO "  mapped low ram: 0 - %08lx\n",
                  max_pfn_mapped<<PAGE_SHIFT);
         printk(KERN_INFO "  low ram: 0 - %08lx\n", max_low_pfn<<PAGE_SHIFT);
-
-        after_bootmem = 1;
 }
 
 /*
@@ -753,6 +777,8 @@ void __init mem_init(void)
                 if (page_is_ram(tmp) && PageReserved(pfn_to_page(tmp)))
                         reservedpages++;
 
+        after_bootmem = 1;
+
         codesize =  (unsigned long) &_etext - (unsigned long) &_text;
         datasize =  (unsigned long) &_edata - (unsigned long) &_etext;
         initsize =  (unsigned long) &__init_end - (unsigned long) &__init_begin;
@@ -836,6 +862,18 @@ int arch_add_memory(int nid, u64 start, u64 size)
 
         return __add_pages(nid, zone, start_pfn, nr_pages);
 }
+
+#ifdef CONFIG_MEMORY_HOTREMOVE
+int arch_remove_memory(u64 start, u64 size)
+{
+        unsigned long start_pfn = start >> PAGE_SHIFT;
+        unsigned long nr_pages = size >> PAGE_SHIFT;
+        struct zone *zone;
+
+        zone = page_zone(pfn_to_page(start_pfn));
+        return __remove_pages(zone, start_pfn, nr_pages);
+}
+#endif
 #endif
 
 /*
diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
index 2ead3c8a4c84..474e28f10815 100644
--- a/arch/x86/mm/init_64.c
+++ b/arch/x86/mm/init_64.c
@@ -54,6 +54,82 @@
 #include <asm/uv/uv.h>
 #include <asm/setup.h>
 
+#include "mm_internal.h"
+
+static void ident_pmd_init(unsigned long pmd_flag, pmd_t *pmd_page,
+                           unsigned long addr, unsigned long end)
+{
+        addr &= PMD_MASK;
+        for (; addr < end; addr += PMD_SIZE) {
+                pmd_t *pmd = pmd_page + pmd_index(addr);
+
+                if (!pmd_present(*pmd))
+                        set_pmd(pmd, __pmd(addr | pmd_flag));
+        }
+}
+static int ident_pud_init(struct x86_mapping_info *info, pud_t *pud_page,
+                          unsigned long addr, unsigned long end)
+{
+        unsigned long next;
+
+        for (; addr < end; addr = next) {
+                pud_t *pud = pud_page + pud_index(addr);
+                pmd_t *pmd;
+
+                next = (addr & PUD_MASK) + PUD_SIZE;
+                if (next > end)
+                        next = end;
+
+                if (pud_present(*pud)) {
+                        pmd = pmd_offset(pud, 0);
+                        ident_pmd_init(info->pmd_flag, pmd, addr, next);
+                        continue;
+                }
+                pmd = (pmd_t *)info->alloc_pgt_page(info->context);
+                if (!pmd)
+                        return -ENOMEM;
+                ident_pmd_init(info->pmd_flag, pmd, addr, next);
+                set_pud(pud, __pud(__pa(pmd) | _KERNPG_TABLE));
+        }
+
+        return 0;
+}
+
+int kernel_ident_mapping_init(struct x86_mapping_info *info, pgd_t *pgd_page,
+                              unsigned long addr, unsigned long end)
+{
+        unsigned long next;
+        int result;
+        int off = info->kernel_mapping ? pgd_index(__PAGE_OFFSET) : 0;
+
+        for (; addr < end; addr = next) {
+                pgd_t *pgd = pgd_page + pgd_index(addr) + off;
+                pud_t *pud;
+
+                next = (addr & PGDIR_MASK) + PGDIR_SIZE;
+                if (next > end)
+                        next = end;
+
+                if (pgd_present(*pgd)) {
+                        pud = pud_offset(pgd, 0);
+                        result = ident_pud_init(info, pud, addr, next);
+                        if (result)
+                                return result;
+                        continue;
+                }
+
+                pud = (pud_t *)info->alloc_pgt_page(info->context);
+                if (!pud)
+                        return -ENOMEM;
+                result = ident_pud_init(info, pud, addr, next);
+                if (result)
+                        return result;
+                set_pgd(pgd, __pgd(__pa(pud) | _KERNPG_TABLE));
+        }
+
+        return 0;
+}
+
 static int __init parse_direct_gbpages_off(char *arg)
 {
         direct_gbpages = 0;
@@ -302,10 +378,18 @@ void __init init_extra_mapping_uc(unsigned long phys, unsigned long size)
 void __init cleanup_highmap(void)
 {
         unsigned long vaddr = __START_KERNEL_map;
-        unsigned long vaddr_end = __START_KERNEL_map + (max_pfn_mapped << PAGE_SHIFT);
+        unsigned long vaddr_end = __START_KERNEL_map + KERNEL_IMAGE_SIZE;
         unsigned long end = roundup((unsigned long)_brk_end, PMD_SIZE) - 1;
         pmd_t *pmd = level2_kernel_pgt;
 
+        /*
+         * Native path, max_pfn_mapped is not set yet.
+         * Xen has valid max_pfn_mapped set in
+         *      arch/x86/xen/mmu.c:xen_setup_kernel_pagetable().
+         */
+        if (max_pfn_mapped)
+                vaddr_end = __START_KERNEL_map + (max_pfn_mapped << PAGE_SHIFT);
+
         for (; vaddr + PMD_SIZE - 1 < vaddr_end; pmd++, vaddr += PMD_SIZE) {
                 if (pmd_none(*pmd))
                         continue;
@@ -314,69 +398,24 @@ void __init cleanup_highmap(void)
         }
 }
 
-static __ref void *alloc_low_page(unsigned long *phys)
-{
-        unsigned long pfn = pgt_buf_end++;
-        void *adr;
-
-        if (after_bootmem) {
-                adr = (void *)get_zeroed_page(GFP_ATOMIC | __GFP_NOTRACK);
-                *phys = __pa(adr);
-
-                return adr;
-        }
-
-        if (pfn >= pgt_buf_top)
-                panic("alloc_low_page: ran out of memory");
-
-        adr = early_memremap(pfn * PAGE_SIZE, PAGE_SIZE);
-        clear_page(adr);
-        *phys  = pfn * PAGE_SIZE;
-        return adr;
-}
-
-static __ref void *map_low_page(void *virt)
-{
-        void *adr;
-        unsigned long phys, left;
-
-        if (after_bootmem)
-                return virt;
-
-        phys = __pa(virt);
-        left = phys & (PAGE_SIZE - 1);
-        adr = early_memremap(phys & PAGE_MASK, PAGE_SIZE);
-        adr = (void *)(((unsigned long)adr) | left);
-
-        return adr;
-}
-
-static __ref void unmap_low_page(void *adr)
-{
-        if (after_bootmem)
-                return;
-
-        early_iounmap((void *)((unsigned long)adr & PAGE_MASK), PAGE_SIZE);
-}
-
 static unsigned long __meminit
 phys_pte_init(pte_t *pte_page, unsigned long addr, unsigned long end,
               pgprot_t prot)
 {
-        unsigned pages = 0;
+        unsigned long pages = 0, next;
         unsigned long last_map_addr = end;
         int i;
 
         pte_t *pte = pte_page + pte_index(addr);
 
-        for(i = pte_index(addr); i < PTRS_PER_PTE; i++, addr += PAGE_SIZE, pte++) {
-
+        for (i = pte_index(addr); i < PTRS_PER_PTE; i++, addr = next, pte++) {
+                next = (addr & PAGE_MASK) + PAGE_SIZE;
                 if (addr >= end) {
-                        if (!after_bootmem) {
-                                for(; i < PTRS_PER_PTE; i++, pte++)
-                                        set_pte(pte, __pte(0));
-                        }
-                        break;
+                        if (!after_bootmem &&
+                            !e820_any_mapped(addr & PAGE_MASK, next, E820_RAM) &&
+                            !e820_any_mapped(addr & PAGE_MASK, next, E820_RESERVED_KERN))
+                                set_pte(pte, __pte(0));
+                        continue;
                 }
 
                 /*
@@ -414,28 +453,25 @@ phys_pmd_init(pmd_t *pmd_page, unsigned long address, unsigned long end,
         int i = pmd_index(address);
 
         for (; i < PTRS_PER_PMD; i++, address = next) {
-                unsigned long pte_phys;
                 pmd_t *pmd = pmd_page + pmd_index(address);
                 pte_t *pte;
                 pgprot_t new_prot = prot;
 
+                next = (address & PMD_MASK) + PMD_SIZE;
                 if (address >= end) {
-                        if (!after_bootmem) {
-                                for (; i < PTRS_PER_PMD; i++, pmd++)
-                                        set_pmd(pmd, __pmd(0));
-                        }
-                        break;
+                        if (!after_bootmem &&
+                            !e820_any_mapped(address & PMD_MASK, next, E820_RAM) &&
+                            !e820_any_mapped(address & PMD_MASK, next, E820_RESERVED_KERN))
+                                set_pmd(pmd, __pmd(0));
+                        continue;
                 }
 
-                next = (address & PMD_MASK) + PMD_SIZE;
-
                 if (pmd_val(*pmd)) {
                         if (!pmd_large(*pmd)) {
                                 spin_lock(&init_mm.page_table_lock);
-                                pte = map_low_page((pte_t *)pmd_page_vaddr(*pmd));
+                                pte = (pte_t *)pmd_page_vaddr(*pmd);
                                 last_map_addr = phys_pte_init(pte, address,
                                                                 end, prot);
-                                unmap_low_page(pte);
                                 spin_unlock(&init_mm.page_table_lock);
                                 continue;
                         }
@@ -464,19 +500,18 @@ phys_pmd_init(pmd_t *pmd_page, unsigned long address, unsigned long end,
                         pages++;
                         spin_lock(&init_mm.page_table_lock);
                         set_pte((pte_t *)pmd,
-                                pfn_pte(address >> PAGE_SHIFT,
+                                pfn_pte((address & PMD_MASK) >> PAGE_SHIFT,
                                         __pgprot(pgprot_val(prot) | _PAGE_PSE)));
                         spin_unlock(&init_mm.page_table_lock);
                         last_map_addr = next;
                         continue;
                 }
 
-                pte = alloc_low_page(&pte_phys);
+                pte = alloc_low_page();
                 last_map_addr = phys_pte_init(pte, address, end, new_prot);
-                unmap_low_page(pte);
 
                 spin_lock(&init_mm.page_table_lock);
-                pmd_populate_kernel(&init_mm, pmd, __va(pte_phys));
+                pmd_populate_kernel(&init_mm, pmd, pte);
                 spin_unlock(&init_mm.page_table_lock);
         }
         update_page_count(PG_LEVEL_2M, pages);
@@ -492,27 +527,24 @@ phys_pud_init(pud_t *pud_page, unsigned long addr, unsigned long end,
         int i = pud_index(addr);
 
         for (; i < PTRS_PER_PUD; i++, addr = next) {
-                unsigned long pmd_phys;
                 pud_t *pud = pud_page + pud_index(addr);
                 pmd_t *pmd;
                 pgprot_t prot = PAGE_KERNEL;
 
-                if (addr >= end)
-                        break;
-
                 next = (addr & PUD_MASK) + PUD_SIZE;
-
-                if (!after_bootmem && !e820_any_mapped(addr, next, 0)) {
-                        set_pud(pud, __pud(0));
+                if (addr >= end) {
+                        if (!after_bootmem &&
+                            !e820_any_mapped(addr & PUD_MASK, next, E820_RAM) &&
+                            !e820_any_mapped(addr & PUD_MASK, next, E820_RESERVED_KERN))
+                                set_pud(pud, __pud(0));
                         continue;
                 }
 
                 if (pud_val(*pud)) {
                         if (!pud_large(*pud)) {
-                                pmd = map_low_page(pmd_offset(pud, 0));
+                                pmd = pmd_offset(pud, 0);
                                 last_map_addr = phys_pmd_init(pmd, addr, end,
                                                          page_size_mask, prot);
-                                unmap_low_page(pmd);
                                 __flush_tlb_all();
                                 continue;
                         }
@@ -541,19 +573,19 @@ phys_pud_init(pud_t *pud_page, unsigned long addr, unsigned long end,
                         pages++;
                         spin_lock(&init_mm.page_table_lock);
                         set_pte((pte_t *)pud,
-                                pfn_pte(addr >> PAGE_SHIFT, PAGE_KERNEL_LARGE));
+                                pfn_pte((addr & PUD_MASK) >> PAGE_SHIFT,
+                                        PAGE_KERNEL_LARGE));
                         spin_unlock(&init_mm.page_table_lock);
                         last_map_addr = next;
                         continue;
                 }
 
-                pmd = alloc_low_page(&pmd_phys);
+                pmd = alloc_low_page();
                 last_map_addr = phys_pmd_init(pmd, addr, end, page_size_mask,
                                               prot);
-                unmap_low_page(pmd);
 
                 spin_lock(&init_mm.page_table_lock);
-                pud_populate(&init_mm, pud, __va(pmd_phys));
+                pud_populate(&init_mm, pud, pmd);
                 spin_unlock(&init_mm.page_table_lock);
         }
         __flush_tlb_all();
@@ -578,34 +610,29 @@ kernel_physical_mapping_init(unsigned long start,
 
         for (; start < end; start = next) {
                 pgd_t *pgd = pgd_offset_k(start);
-                unsigned long pud_phys;
                 pud_t *pud;
 
-                next = (start + PGDIR_SIZE) & PGDIR_MASK;
-                if (next > end)
-                        next = end;
+                next = (start & PGDIR_MASK) + PGDIR_SIZE;
 
                 if (pgd_val(*pgd)) {
-                        pud = map_low_page((pud_t *)pgd_page_vaddr(*pgd));
+                        pud = (pud_t *)pgd_page_vaddr(*pgd);
                         last_map_addr = phys_pud_init(pud, __pa(start),
                                                  __pa(end), page_size_mask);
-                        unmap_low_page(pud);
                         continue;
                 }
 
-                pud = alloc_low_page(&pud_phys);
-                last_map_addr = phys_pud_init(pud, __pa(start), __pa(next),
+                pud = alloc_low_page();
+                last_map_addr = phys_pud_init(pud, __pa(start), __pa(end),
                                                  page_size_mask);
-                unmap_low_page(pud);
 
                 spin_lock(&init_mm.page_table_lock);
-                pgd_populate(&init_mm, pgd, __va(pud_phys));
+                pgd_populate(&init_mm, pgd, pud);
                 spin_unlock(&init_mm.page_table_lock);
                 pgd_changed = true;
         }
 
         if (pgd_changed)
-                sync_global_pgds(addr, end);
+                sync_global_pgds(addr, end - 1);
 
         __flush_tlb_all();
 
@@ -664,13 +691,11 @@ int arch_add_memory(int nid, u64 start, u64 size)
 {
         struct pglist_data *pgdat = NODE_DATA(nid);
         struct zone *zone = pgdat->node_zones + ZONE_NORMAL;
-        unsigned long last_mapped_pfn, start_pfn = start >> PAGE_SHIFT;
+        unsigned long start_pfn = start >> PAGE_SHIFT;
         unsigned long nr_pages = size >> PAGE_SHIFT;
         int ret;
 
-        last_mapped_pfn = init_memory_mapping(start, start + size);
-        if (last_mapped_pfn > max_pfn_mapped)
-                max_pfn_mapped = last_mapped_pfn;
+        init_memory_mapping(start, start + size);
 
         ret = __add_pages(nid, zone, start_pfn, nr_pages);
         WARN_ON_ONCE(ret);
@@ -682,10 +707,357 @@ int arch_add_memory(int nid, u64 start, u64 size)
 }
 EXPORT_SYMBOL_GPL(arch_add_memory);
 
+#define PAGE_INUSE 0xFD
+
+static void __meminit free_pagetable(struct page *page, int order)
+{
+        struct zone *zone;
+        bool bootmem = false;
+        unsigned long magic;
+        unsigned int nr_pages = 1 << order;
+
+        /* bootmem page has reserved flag */
+        if (PageReserved(page)) {
+                __ClearPageReserved(page);
+                bootmem = true;
+
+                magic = (unsigned long)page->lru.next;
+                if (magic == SECTION_INFO || magic == MIX_SECTION_INFO) {
+                        while (nr_pages--)
+                                put_page_bootmem(page++);
+                } else
+                        __free_pages_bootmem(page, order);
+        } else
+                free_pages((unsigned long)page_address(page), order);
+
+        /*
+         * SECTION_INFO pages and MIX_SECTION_INFO pages
+         * are all allocated by bootmem.
+         */
+        if (bootmem) {
+                zone = page_zone(page);
+                zone_span_writelock(zone);
+                zone->present_pages += nr_pages;
+                zone_span_writeunlock(zone);
+                totalram_pages += nr_pages;
+        }
+}
+
+static void __meminit free_pte_table(pte_t *pte_start, pmd_t *pmd)
+{
+        pte_t *pte;
+        int i;
+
+        for (i = 0; i < PTRS_PER_PTE; i++) {
+                pte = pte_start + i;
+                if (pte_val(*pte))
+                        return;
+        }
+
+        /* free a pte talbe */
+        free_pagetable(pmd_page(*pmd), 0);
+        spin_lock(&init_mm.page_table_lock);
+        pmd_clear(pmd);
+        spin_unlock(&init_mm.page_table_lock);
+}
+
+static void __meminit free_pmd_table(pmd_t *pmd_start, pud_t *pud)
+{
+        pmd_t *pmd;
+        int i;
+
+        for (i = 0; i < PTRS_PER_PMD; i++) {
+                pmd = pmd_start + i;
+                if (pmd_val(*pmd))
+                        return;
+        }
+
+        /* free a pmd talbe */
+        free_pagetable(pud_page(*pud), 0);
+        spin_lock(&init_mm.page_table_lock);
+        pud_clear(pud);
+        spin_unlock(&init_mm.page_table_lock);
+}
+
+/* Return true if pgd is changed, otherwise return false. */
+static bool __meminit free_pud_table(pud_t *pud_start, pgd_t *pgd)
+{
+        pud_t *pud;
+        int i;
+
+        for (i = 0; i < PTRS_PER_PUD; i++) {
+                pud = pud_start + i;
+                if (pud_val(*pud))
+                        return false;
+        }
+
+        /* free a pud table */
+        free_pagetable(pgd_page(*pgd), 0);
+        spin_lock(&init_mm.page_table_lock);
+        pgd_clear(pgd);
+        spin_unlock(&init_mm.page_table_lock);
+
+        return true;
+}
+
+static void __meminit
+remove_pte_table(pte_t *pte_start, unsigned long addr, unsigned long end,
+                 bool direct)
+{
+        unsigned long next, pages = 0;
+        pte_t *pte;
+        void *page_addr;
+        phys_addr_t phys_addr;
+
+        pte = pte_start + pte_index(addr);
+        for (; addr < end; addr = next, pte++) {
+                next = (addr + PAGE_SIZE) & PAGE_MASK;
+                if (next > end)
+                        next = end;
+
+                if (!pte_present(*pte))
+                        continue;
+
+                /*
+                 * We mapped [0,1G) memory as identity mapping when
+                 * initializing, in arch/x86/kernel/head_64.S. These
+                 * pagetables cannot be removed.
+                 */
+                phys_addr = pte_val(*pte) + (addr & PAGE_MASK);
+                if (phys_addr < (phys_addr_t)0x40000000)
+                        return;
+
+                if (IS_ALIGNED(addr, PAGE_SIZE) &&
+                    IS_ALIGNED(next, PAGE_SIZE)) {
+                        /*
+                         * Do not free direct mapping pages since they were
+                         * freed when offlining, or simplely not in use.
+                         */
+                        if (!direct)
+                                free_pagetable(pte_page(*pte), 0);
+
+                        spin_lock(&init_mm.page_table_lock);
+                        pte_clear(&init_mm, addr, pte);
+                        spin_unlock(&init_mm.page_table_lock);
+
+                        /* For non-direct mapping, pages means nothing. */
+                        pages++;
+                } else {
+                        /*
+                         * If we are here, we are freeing vmemmap pages since
+                         * direct mapped memory ranges to be freed are aligned.
+                         *
+                         * If we are not removing the whole page, it means
+                         * other page structs in this page are being used and
+                         * we canot remove them. So fill the unused page_structs
+                         * with 0xFD, and remove the page when it is wholly
+                         * filled with 0xFD.
+                         */
+                        memset((void *)addr, PAGE_INUSE, next - addr);
+
+                        page_addr = page_address(pte_page(*pte));
+                        if (!memchr_inv(page_addr, PAGE_INUSE, PAGE_SIZE)) {
+                                free_pagetable(pte_page(*pte), 0);
+
+                                spin_lock(&init_mm.page_table_lock);
+                                pte_clear(&init_mm, addr, pte);
+                                spin_unlock(&init_mm.page_table_lock);
+                        }
+                }
+        }
+
+        /* Call free_pte_table() in remove_pmd_table(). */
+        flush_tlb_all();
+        if (direct)
+                update_page_count(PG_LEVEL_4K, -pages);
+}
+
+static void __meminit
+remove_pmd_table(pmd_t *pmd_start, unsigned long addr, unsigned long end,
+                 bool direct)
+{
+        unsigned long next, pages = 0;
+        pte_t *pte_base;
+        pmd_t *pmd;
+        void *page_addr;
+
+        pmd = pmd_start + pmd_index(addr);
+        for (; addr < end; addr = next, pmd++) {
+                next = pmd_addr_end(addr, end);
+
+                if (!pmd_present(*pmd))
+                        continue;
+
+                if (pmd_large(*pmd)) {
+                        if (IS_ALIGNED(addr, PMD_SIZE) &&
+                            IS_ALIGNED(next, PMD_SIZE)) {
+                                if (!direct)
+                                        free_pagetable(pmd_page(*pmd),
+                                                       get_order(PMD_SIZE));
+
+                                spin_lock(&init_mm.page_table_lock);
+                                pmd_clear(pmd);
+                                spin_unlock(&init_mm.page_table_lock);
+                                pages++;
+                        } else {
+                                /* If here, we are freeing vmemmap pages. */
+                                memset((void *)addr, PAGE_INUSE, next - addr);
+
+                                page_addr = page_address(pmd_page(*pmd));
+                                if (!memchr_inv(page_addr, PAGE_INUSE,
+                                                PMD_SIZE)) {
+                                        free_pagetable(pmd_page(*pmd),
+                                                       get_order(PMD_SIZE));
+
+                                        spin_lock(&init_mm.page_table_lock);
+                                        pmd_clear(pmd);
+                                        spin_unlock(&init_mm.page_table_lock);
+                                }
+                        }
+
+                        continue;
+                }
+
+                pte_base = (pte_t *)pmd_page_vaddr(*pmd);
+                remove_pte_table(pte_base, addr, next, direct);
+                free_pte_table(pte_base, pmd);
+        }
+
+        /* Call free_pmd_table() in remove_pud_table(). */
+        if (direct)
+                update_page_count(PG_LEVEL_2M, -pages);
+}
+
+static void __meminit
+remove_pud_table(pud_t *pud_start, unsigned long addr, unsigned long end,
+                 bool direct)
+{
+        unsigned long next, pages = 0;
+        pmd_t *pmd_base;
+        pud_t *pud;
+        void *page_addr;
+
+        pud = pud_start + pud_index(addr);
+        for (; addr < end; addr = next, pud++) {
+                next = pud_addr_end(addr, end);
+
+                if (!pud_present(*pud))
+                        continue;
+
+                if (pud_large(*pud)) {
+                        if (IS_ALIGNED(addr, PUD_SIZE) &&
+                            IS_ALIGNED(next, PUD_SIZE)) {
+                                if (!direct)
+                                        free_pagetable(pud_page(*pud),
+                                                       get_order(PUD_SIZE));
+
+                                spin_lock(&init_mm.page_table_lock);
+                                pud_clear(pud);
+                                spin_unlock(&init_mm.page_table_lock);
+                                pages++;
+                        } else {
+                                /* If here, we are freeing vmemmap pages. */
+                                memset((void *)addr, PAGE_INUSE, next - addr);
+
+                                page_addr = page_address(pud_page(*pud));
+                                if (!memchr_inv(page_addr, PAGE_INUSE,
+                                                PUD_SIZE)) {
+                                        free_pagetable(pud_page(*pud),
+                                                       get_order(PUD_SIZE));
+
+                                        spin_lock(&init_mm.page_table_lock);
+                                        pud_clear(pud);
+                                        spin_unlock(&init_mm.page_table_lock);
+                                }
+                        }
+
+                        continue;
+                }
+
+                pmd_base = (pmd_t *)pud_page_vaddr(*pud);
+                remove_pmd_table(pmd_base, addr, next, direct);
+                free_pmd_table(pmd_base, pud);
+        }
+
+        if (direct)
+                update_page_count(PG_LEVEL_1G, -pages);
+}
+
+/* start and end are both virtual address. */
+static void __meminit
+remove_pagetable(unsigned long start, unsigned long end, bool direct)
+{
+        unsigned long next;
+        pgd_t *pgd;
+        pud_t *pud;
+        bool pgd_changed = false;
+
+        for (; start < end; start = next) {
+                next = pgd_addr_end(start, end);
+
+                pgd = pgd_offset_k(start);
+                if (!pgd_present(*pgd))
+                        continue;
+
+                pud = (pud_t *)pgd_page_vaddr(*pgd);
+                remove_pud_table(pud, start, next, direct);
+                if (free_pud_table(pud, pgd))
+                        pgd_changed = true;
+        }
+
+        if (pgd_changed)
+                sync_global_pgds(start, end - 1);
+
+        flush_tlb_all();
+}
+
+void __ref vmemmap_free(struct page *memmap, unsigned long nr_pages)
+{
+        unsigned long start = (unsigned long)memmap;
+        unsigned long end = (unsigned long)(memmap + nr_pages);
+
+        remove_pagetable(start, end, false);
+}
+
+static void __meminit
+kernel_physical_mapping_remove(unsigned long start, unsigned long end)
+{
+        start = (unsigned long)__va(start);
+        end = (unsigned long)__va(end);
+
+        remove_pagetable(start, end, true);
+}
+
+#ifdef CONFIG_MEMORY_HOTREMOVE
+int __ref arch_remove_memory(u64 start, u64 size)
+{
+        unsigned long start_pfn = start >> PAGE_SHIFT;
+        unsigned long nr_pages = size >> PAGE_SHIFT;
+        struct zone *zone;
+        int ret;
+
+        zone = page_zone(pfn_to_page(start_pfn));
+        kernel_physical_mapping_remove(start, start + size);
+        ret = __remove_pages(zone, start_pfn, nr_pages);
+        WARN_ON_ONCE(ret);
+
+        return ret;
+}
+#endif
 #endif /* CONFIG_MEMORY_HOTPLUG */
 
 static struct kcore_list kcore_vsyscall;
 
+static void __init register_page_bootmem_info(void)
+{
+#ifdef CONFIG_NUMA
+        int i;
+
+        for_each_online_node(i)
+                register_page_bootmem_info_node(NODE_DATA(i));
+#endif
+}
+
 void __init mem_init(void)
 {
         long codesize, reservedpages, datasize, initsize;
@@ -698,11 +1070,8 @@ void __init mem_init(void)
         reservedpages = 0;
 
         /* this will put all low memory onto the freelists */
-#ifdef CONFIG_NUMA
-        totalram_pages = numa_free_all_bootmem();
-#else
+        register_page_bootmem_info();
         totalram_pages = free_all_bootmem();
-#endif
 
         absent_pages = absent_pages_in_range(0, max_pfn);
         reservedpages = max_pfn - totalram_pages - absent_pages;
@@ -772,12 +1141,11 @@ void set_kernel_text_ro(void)
 void mark_rodata_ro(void)
 {
         unsigned long start = PFN_ALIGN(_text);
-        unsigned long rodata_start =
-                ((unsigned long)__start_rodata + PAGE_SIZE - 1) & PAGE_MASK;
+        unsigned long rodata_start = PFN_ALIGN(__start_rodata);
         unsigned long end = (unsigned long) &__end_rodata_hpage_align;
-        unsigned long text_end = PAGE_ALIGN((unsigned long) &__stop___ex_table);
-        unsigned long rodata_end = PAGE_ALIGN((unsigned long) &__end_rodata);
-        unsigned long data_start = (unsigned long) &_sdata;
+        unsigned long text_end = PFN_ALIGN(&__stop___ex_table);
+        unsigned long rodata_end = PFN_ALIGN(&__end_rodata);
+        unsigned long all_end = PFN_ALIGN(&_end);
 
         printk(KERN_INFO "Write protecting the kernel read-only data: %luk\n",
                (end - start) >> 10);
@@ -786,10 +1154,10 @@ void mark_rodata_ro(void)
         kernel_set_to_readonly = 1;
 
         /*
-         * The rodata section (but not the kernel text!) should also be
-         * not-executable.
+         * The rodata/data/bss/brk section (but not the kernel text!)
+         * should also be not-executable.
          */
-        set_memory_nx(rodata_start, (end - rodata_start) >> PAGE_SHIFT);
+        set_memory_nx(rodata_start, (all_end - rodata_start) >> PAGE_SHIFT);
 
         rodata_test();
 
@@ -802,12 +1170,12 @@ void mark_rodata_ro(void)
 #endif
 
         free_init_pages("unused kernel memory",
-                        (unsigned long) page_address(virt_to_page(text_end)),
-                        (unsigned long)
-                                 page_address(virt_to_page(rodata_start)));
+                        (unsigned long) __va(__pa_symbol(text_end)),
+                        (unsigned long) __va(__pa_symbol(rodata_start)));
+
         free_init_pages("unused kernel memory",
-                        (unsigned long) page_address(virt_to_page(rodata_end)),
-                        (unsigned long) page_address(virt_to_page(data_start)));
+                        (unsigned long) __va(__pa_symbol(rodata_end)),
+                        (unsigned long) __va(__pa_symbol(_sdata)));
 }
 
 #endif
@@ -831,6 +1199,9 @@ int kern_addr_valid(unsigned long addr)
         if (pud_none(*pud))
                 return 0;
 
+        if (pud_large(*pud))
+                return pfn_valid(pud_pfn(*pud));
+
         pmd = pmd_offset(pud, addr);
         if (pmd_none(*pmd))
                 return 0;
@@ -981,10 +1352,70 @@ vmemmap_populate(struct page *start_page, unsigned long size, int node)
                 }
 
         }
-        sync_global_pgds((unsigned long)start_page, end);
+        sync_global_pgds((unsigned long)start_page, end - 1);
         return 0;
 }
 
+#if defined(CONFIG_MEMORY_HOTPLUG_SPARSE) && defined(CONFIG_HAVE_BOOTMEM_INFO_NODE)
+void register_page_bootmem_memmap(unsigned long section_nr,
+                                  struct page *start_page, unsigned long size)
+{
+        unsigned long addr = (unsigned long)start_page;
+        unsigned long end = (unsigned long)(start_page + size);
+        unsigned long next;
+        pgd_t *pgd;
+        pud_t *pud;
+        pmd_t *pmd;
+        unsigned int nr_pages;
+        struct page *page;
+
+        for (; addr < end; addr = next) {
+                pte_t *pte = NULL;
+
+                pgd = pgd_offset_k(addr);
+                if (pgd_none(*pgd)) {
+                        next = (addr + PAGE_SIZE) & PAGE_MASK;
+                        continue;
+                }
+                get_page_bootmem(section_nr, pgd_page(*pgd), MIX_SECTION_INFO);
+
+                pud = pud_offset(pgd, addr);
+                if (pud_none(*pud)) {
+                        next = (addr + PAGE_SIZE) & PAGE_MASK;
+                        continue;
+                }
+                get_page_bootmem(section_nr, pud_page(*pud), MIX_SECTION_INFO);
+
+                if (!cpu_has_pse) {
+                        next = (addr + PAGE_SIZE) & PAGE_MASK;
+                        pmd = pmd_offset(pud, addr);
+                        if (pmd_none(*pmd))
+                                continue;
+                        get_page_bootmem(section_nr, pmd_page(*pmd),
+                                         MIX_SECTION_INFO);
+
+                        pte = pte_offset_kernel(pmd, addr);
+                        if (pte_none(*pte))
+                                continue;
+                        get_page_bootmem(section_nr, pte_page(*pte),
+                                         SECTION_INFO);
+                } else {
+                        next = pmd_addr_end(addr, end);
+
+                        pmd = pmd_offset(pud, addr);
+                        if (pmd_none(*pmd))
+                                continue;
+
+                        nr_pages = 1 << (get_order(PMD_SIZE));
+                        page = pmd_page(*pmd);
+                        while (nr_pages--)
+                                get_page_bootmem(section_nr, page++,
+                                                 SECTION_INFO);
+                }
+        }
+}
+#endif
+
 void __meminit vmemmap_populate_print_last(void)
 {
         if (p_start) {
diff --git a/arch/x86/mm/memtest.c b/arch/x86/mm/memtest.c
index c80b9fb95734..8dabbed409ee 100644
--- a/arch/x86/mm/memtest.c
+++ b/arch/x86/mm/memtest.c
@@ -9,6 +9,7 @@
 #include <linux/memblock.h>
 
 static u64 patterns[] __initdata = {
+        /* The first entry has to be 0 to leave memtest with zeroed memory */
         0,
         0xffffffffffffffffULL,
         0x5555555555555555ULL,
@@ -110,15 +111,8 @@ void __init early_memtest(unsigned long start, unsigned long end)
                 return;
 
         printk(KERN_INFO "early_memtest: # of tests: %d\n", memtest_pattern);
-        for (i = 0; i < memtest_pattern; i++) {
+        for (i = memtest_pattern-1; i < UINT_MAX; --i) {
                 idx = i % ARRAY_SIZE(patterns);
                 do_one_pass(patterns[idx], start, end);
         }
-
-        if (idx > 0) {
-                printk(KERN_INFO "early_memtest: wipe out "
-                       "test pattern from memory\n");
-                /* additional test with pattern 0 will do this */
-                do_one_pass(0, start, end);
-        }
 }
diff --git a/arch/x86/mm/mm_internal.h b/arch/x86/mm/mm_internal.h
new file mode 100644
index 000000000000..6b563a118891
--- /dev/null
+++ b/arch/x86/mm/mm_internal.h
@@ -0,0 +1,19 @@
+#ifndef __X86_MM_INTERNAL_H
+#define __X86_MM_INTERNAL_H
+
+void *alloc_low_pages(unsigned int num);
+static inline void *alloc_low_page(void)
+{
+        return alloc_low_pages(1);
+}
+
+void early_ioremap_page_table_range_init(void);
+
+unsigned long kernel_physical_mapping_init(unsigned long start,
+                                             unsigned long end,
+                                             unsigned long page_size_mask);
+void zone_sizes_init(void);
+
+extern int after_bootmem;
+
+#endif  /* __X86_MM_INTERNAL_H */
diff --git a/arch/x86/mm/numa.c b/arch/x86/mm/numa.c
index 2d125be1bae9..72fe01e9e414 100644
--- a/arch/x86/mm/numa.c
+++ b/arch/x86/mm/numa.c
@@ -56,7 +56,7 @@ early_param("numa", numa_setup);
 /*
  * apicid, cpu, node mappings
  */
-s16 __apicid_to_node[MAX_LOCAL_APIC] __cpuinitdata = {
+s16 __apicid_to_node[MAX_LOCAL_APIC] = {
         [0 ... MAX_LOCAL_APIC-1] = NUMA_NO_NODE
 };
 
@@ -78,7 +78,7 @@ EXPORT_SYMBOL(node_to_cpumask_map);
 DEFINE_EARLY_PER_CPU(int, x86_cpu_to_node_map, NUMA_NO_NODE);
 EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_node_map);
 
-void __cpuinit numa_set_node(int cpu, int node)
+void numa_set_node(int cpu, int node)
 {
         int *cpu_to_node_map = early_per_cpu_ptr(x86_cpu_to_node_map);
 
@@ -97,11 +97,10 @@ void __cpuinit numa_set_node(int cpu, int node)
 #endif
         per_cpu(x86_cpu_to_node_map, cpu) = node;
 
-        if (node != NUMA_NO_NODE)
-                set_cpu_numa_node(cpu, node);
+        set_cpu_numa_node(cpu, node);
 }
 
-void __cpuinit numa_clear_node(int cpu)
+void numa_clear_node(int cpu)
 {
         numa_set_node(cpu, NUMA_NO_NODE);
 }
@@ -193,7 +192,6 @@ int __init numa_add_memblk(int nid, u64 start, u64 end)
 static void __init setup_node_data(int nid, u64 start, u64 end)
 {
         const size_t nd_size = roundup(sizeof(pg_data_t), PAGE_SIZE);
-        bool remapped = false;
         u64 nd_pa;
         void *nd;
         int tnid;
@@ -205,37 +203,28 @@ static void __init setup_node_data(int nid, u64 start, u64 end)
         if (end && (end - start) < NODE_MIN_SIZE)
                 return;
 
-        /* initialize remap allocator before aligning to ZONE_ALIGN */
-        init_alloc_remap(nid, start, end);
-
         start = roundup(start, ZONE_ALIGN);
 
         printk(KERN_INFO "Initmem setup node %d [mem %#010Lx-%#010Lx]\n",
                nid, start, end - 1);
 
         /*
-         * Allocate node data.  Try remap allocator first, node-local
-         * memory and then any node.  Never allocate in DMA zone.
+         * Allocate node data.  Try node-local memory and then any node.
+         * Never allocate in DMA zone.
          */
-        nd = alloc_remap(nid, nd_size);
-        if (nd) {
-                nd_pa = __pa(nd);
-                remapped = true;
-        } else {
-                nd_pa = memblock_alloc_nid(nd_size, SMP_CACHE_BYTES, nid);
-                if (!nd_pa) {
-                        pr_err("Cannot find %zu bytes in node %d\n",
-                               nd_size, nid);
-                        return;
-                }
-                nd = __va(nd_pa);
+        nd_pa = memblock_alloc_nid(nd_size, SMP_CACHE_BYTES, nid);
+        if (!nd_pa) {
+                pr_err("Cannot find %zu bytes in node %d\n",
+                       nd_size, nid);
+                return;
         }
+        nd = __va(nd_pa);
 
         /* report and initialize */
-        printk(KERN_INFO "  NODE_DATA [mem %#010Lx-%#010Lx]%s\n",
-               nd_pa, nd_pa + nd_size - 1, remapped ? " (remapped)" : "");
+        printk(KERN_INFO "  NODE_DATA [mem %#010Lx-%#010Lx]\n",
+               nd_pa, nd_pa + nd_size - 1);
         tnid = early_pfn_to_nid(nd_pa >> PAGE_SHIFT);
-        if (!remapped && tnid != nid)
+        if (tnid != nid)
                 printk(KERN_INFO "    NODE_DATA(%d) on node %d\n", nid, tnid);
 
         node_data[nid] = nd;
diff --git a/arch/x86/mm/numa_32.c b/arch/x86/mm/numa_32.c
index 534255a36b6b..73a6d7395bd3 100644
--- a/arch/x86/mm/numa_32.c
+++ b/arch/x86/mm/numa_32.c
@@ -73,167 +73,6 @@ unsigned long node_memmap_size_bytes(int nid, unsigned long start_pfn,
 
 extern unsigned long highend_pfn, highstart_pfn;
 
-#define LARGE_PAGE_BYTES (PTRS_PER_PTE * PAGE_SIZE)
-
-static void *node_remap_start_vaddr[MAX_NUMNODES];
-void set_pmd_pfn(unsigned long vaddr, unsigned long pfn, pgprot_t flags);
-
-/*
- * Remap memory allocator
- */
-static unsigned long node_remap_start_pfn[MAX_NUMNODES];
-static void *node_remap_end_vaddr[MAX_NUMNODES];
-static void *node_remap_alloc_vaddr[MAX_NUMNODES];
-
-/**
- * alloc_remap - Allocate remapped memory
- * @nid: NUMA node to allocate memory from
- * @size: The size of allocation
- *
- * Allocate @size bytes from the remap area of NUMA node @nid.  The
- * size of the remap area is predetermined by init_alloc_remap() and
- * only the callers considered there should call this function.  For
- * more info, please read the comment on top of init_alloc_remap().
- *
- * The caller must be ready to handle allocation failure from this
- * function and fall back to regular memory allocator in such cases.
- *
- * CONTEXT:
- * Single CPU early boot context.
- *
- * RETURNS:
- * Pointer to the allocated memory on success, %NULL on failure.
- */
-void *alloc_remap(int nid, unsigned long size)
-{
-        void *allocation = node_remap_alloc_vaddr[nid];
-
-        size = ALIGN(size, L1_CACHE_BYTES);
-
-        if (!allocation || (allocation + size) > node_remap_end_vaddr[nid])
-                return NULL;
-
-        node_remap_alloc_vaddr[nid] += size;
-        memset(allocation, 0, size);
-
-        return allocation;
-}
-
-#ifdef CONFIG_HIBERNATION
-/**
- * resume_map_numa_kva - add KVA mapping to the temporary page tables created
- *                       during resume from hibernation
- * @pgd_base - temporary resume page directory
- */
-void resume_map_numa_kva(pgd_t *pgd_base)
-{
-        int node;
-
-        for_each_online_node(node) {
-                unsigned long start_va, start_pfn, nr_pages, pfn;
-
-                start_va = (unsigned long)node_remap_start_vaddr[node];
-                start_pfn = node_remap_start_pfn[node];
-                nr_pages = (node_remap_end_vaddr[node] -
-                            node_remap_start_vaddr[node]) >> PAGE_SHIFT;
-
-                printk(KERN_DEBUG "%s: node %d\n", __func__, node);
-
-                for (pfn = 0; pfn < nr_pages; pfn += PTRS_PER_PTE) {
-                        unsigned long vaddr = start_va + (pfn << PAGE_SHIFT);
-                        pgd_t *pgd = pgd_base + pgd_index(vaddr);
-                        pud_t *pud = pud_offset(pgd, vaddr);
-                        pmd_t *pmd = pmd_offset(pud, vaddr);
-
-                        set_pmd(pmd, pfn_pmd(start_pfn + pfn,
-                                                PAGE_KERNEL_LARGE_EXEC));
-
-                        printk(KERN_DEBUG "%s: %08lx -> pfn %08lx\n",
-                                __func__, vaddr, start_pfn + pfn);
-                }
-        }
-}
-#endif
-
-/**
- * init_alloc_remap - Initialize remap allocator for a NUMA node
- * @nid: NUMA node to initizlie remap allocator for
- *
- * NUMA nodes may end up without any lowmem.  As allocating pgdat and
- * memmap on a different node with lowmem is inefficient, a special
- * remap allocator is implemented which can be used by alloc_remap().
- *
- * For each node, the amount of memory which will be necessary for
- * pgdat and memmap is calculated and two memory areas of the size are
- * allocated - one in the node and the other in lowmem; then, the area
- * in the node is remapped to the lowmem area.
- *
- * As pgdat and memmap must be allocated in lowmem anyway, this
- * doesn't waste lowmem address space; however, the actual lowmem
- * which gets remapped over is wasted.  The amount shouldn't be
- * problematic on machines this feature will be used.
- *
- * Initialization failure isn't fatal.  alloc_remap() is used
- * opportunistically and the callers will fall back to other memory
- * allocation mechanisms on failure.
- */
-void __init init_alloc_remap(int nid, u64 start, u64 end)
-{
-        unsigned long start_pfn = start >> PAGE_SHIFT;
-        unsigned long end_pfn = end >> PAGE_SHIFT;
-        unsigned long size, pfn;
-        u64 node_pa, remap_pa;
-        void *remap_va;
-
-        /*
-         * The acpi/srat node info can show hot-add memroy zones where
-         * memory could be added but not currently present.
-         */
-        printk(KERN_DEBUG "node %d pfn: [%lx - %lx]\n",
-               nid, start_pfn, end_pfn);
-
-        /* calculate the necessary space aligned to large page size */
-        size = node_memmap_size_bytes(nid, start_pfn, end_pfn);
-        size += ALIGN(sizeof(pg_data_t), PAGE_SIZE);
-        size = ALIGN(size, LARGE_PAGE_BYTES);
-
-        /* allocate node memory and the lowmem remap area */
-        node_pa = memblock_find_in_range(start, end, size, LARGE_PAGE_BYTES);
-        if (!node_pa) {
-                pr_warning("remap_alloc: failed to allocate %lu bytes for node %d\n",
-                           size, nid);
-                return;
-        }
-        memblock_reserve(node_pa, size);
-
-        remap_pa = memblock_find_in_range(min_low_pfn << PAGE_SHIFT,
-                                          max_low_pfn << PAGE_SHIFT,
-                                          size, LARGE_PAGE_BYTES);
-        if (!remap_pa) {
-                pr_warning("remap_alloc: failed to allocate %lu bytes remap area for node %d\n",
-                           size, nid);
-                memblock_free(node_pa, size);
-                return;
-        }
-        memblock_reserve(remap_pa, size);
-        remap_va = phys_to_virt(remap_pa);
-
-        /* perform actual remap */
-        for (pfn = 0; pfn < size >> PAGE_SHIFT; pfn += PTRS_PER_PTE)
-                set_pmd_pfn((unsigned long)remap_va + (pfn << PAGE_SHIFT),
-                            (node_pa >> PAGE_SHIFT) + pfn,
-                            PAGE_KERNEL_LARGE);
-
-        /* initialize remap allocator parameters */
-        node_remap_start_pfn[nid] = node_pa >> PAGE_SHIFT;
-        node_remap_start_vaddr[nid] = remap_va;
-        node_remap_end_vaddr[nid] = remap_va + size;
-        node_remap_alloc_vaddr[nid] = remap_va;
-
-        printk(KERN_DEBUG "remap_alloc: node %d [%08llx-%08llx) -> [%p-%p)\n",
-               nid, node_pa, node_pa + size, remap_va, remap_va + size);
-}
-
 void __init initmem_init(void)
 {
         x86_numa_init();
diff --git a/arch/x86/mm/numa_64.c b/arch/x86/mm/numa_64.c
index 92e27119ee1a..9405ffc91502 100644
--- a/arch/x86/mm/numa_64.c
+++ b/arch/x86/mm/numa_64.c
@@ -10,16 +10,3 @@ void __init initmem_init(void)
 {
         x86_numa_init();
 }
-
-unsigned long __init numa_free_all_bootmem(void)
-{
-        unsigned long pages = 0;
-        int i;
-
-        for_each_online_node(i)
-                pages += free_all_bootmem_node(NODE_DATA(i));
-
-        pages += free_low_memory_core_early(MAX_NUMNODES);
-
-        return pages;
-}
diff --git a/arch/x86/mm/numa_internal.h b/arch/x86/mm/numa_internal.h
index 7178c3afe05e..ad86ec91e640 100644
--- a/arch/x86/mm/numa_internal.h
+++ b/arch/x86/mm/numa_internal.h
@@ -21,12 +21,6 @@ void __init numa_reset_distance(void);
 
 void __init x86_numa_init(void);
 
-#ifdef CONFIG_X86_64
-static inline void init_alloc_remap(int nid, u64 start, u64 end)        { }
-#else
-void __init init_alloc_remap(int nid, u64 start, u64 end);
-#endif
-
 #ifdef CONFIG_NUMA_EMU
 void __init numa_emulation(struct numa_meminfo *numa_meminfo,
                            int numa_dist_cnt);
diff --git a/arch/x86/mm/pageattr.c b/arch/x86/mm/pageattr.c
index a718e0d23503..091934e1d0d9 100644
--- a/arch/x86/mm/pageattr.c
+++ b/arch/x86/mm/pageattr.c
@@ -94,12 +94,12 @@ static inline void split_page_count(int level) { }
 
 static inline unsigned long highmap_start_pfn(void)
 {
-        return __pa(_text) >> PAGE_SHIFT;
+        return __pa_symbol(_text) >> PAGE_SHIFT;
 }
 
 static inline unsigned long highmap_end_pfn(void)
 {
-        return __pa(roundup(_brk_end, PMD_SIZE)) >> PAGE_SHIFT;
+        return __pa_symbol(roundup(_brk_end, PMD_SIZE)) >> PAGE_SHIFT;
 }
 
 #endif
@@ -276,8 +276,8 @@ static inline pgprot_t static_protections(pgprot_t prot, unsigned long address,
          * The .rodata section needs to be read-only. Using the pfn
          * catches all aliases.
          */
-        if (within(pfn, __pa((unsigned long)__start_rodata) >> PAGE_SHIFT,
-                   __pa((unsigned long)__end_rodata) >> PAGE_SHIFT))
+        if (within(pfn, __pa_symbol(__start_rodata) >> PAGE_SHIFT,
+                   __pa_symbol(__end_rodata) >> PAGE_SHIFT))
                 pgprot_val(forbidden) |= _PAGE_RW;
 
 #if defined(CONFIG_X86_64) && defined(CONFIG_DEBUG_RODATA)
@@ -364,6 +364,37 @@ pte_t *lookup_address(unsigned long address, unsigned int *level)
 EXPORT_SYMBOL_GPL(lookup_address);
 
 /*
+ * This is necessary because __pa() does not work on some
+ * kinds of memory, like vmalloc() or the alloc_remap()
+ * areas on 32-bit NUMA systems.  The percpu areas can
+ * end up in this kind of memory, for instance.
+ *
+ * This could be optimized, but it is only intended to be
+ * used at inititalization time, and keeping it
+ * unoptimized should increase the testing coverage for
+ * the more obscure platforms.
+ */
+phys_addr_t slow_virt_to_phys(void *__virt_addr)
+{
+        unsigned long virt_addr = (unsigned long)__virt_addr;
+        phys_addr_t phys_addr;
+        unsigned long offset;
+        enum pg_level level;
+        unsigned long psize;
+        unsigned long pmask;
+        pte_t *pte;
+
+        pte = lookup_address(virt_addr, &level);
+        BUG_ON(!pte);
+        psize = page_level_size(level);
+        pmask = page_level_mask(level);
+        offset = virt_addr & ~pmask;
+        phys_addr = pte_pfn(*pte) << PAGE_SHIFT;
+        return (phys_addr | offset);
+}
+EXPORT_SYMBOL_GPL(slow_virt_to_phys);
+
+/*
  * Set the new pmd in all the pgds we know about:
  */
 static void __set_pmd_pte(pte_t *kpte, unsigned long address, pte_t pte)
@@ -396,7 +427,7 @@ try_preserve_large_page(pte_t *kpte, unsigned long address,
         pte_t new_pte, old_pte, *tmp;
         pgprot_t old_prot, new_prot, req_prot;
         int i, do_split = 1;
-        unsigned int level;
+        enum pg_level level;
 
         if (cpa->force_split)
                 return 1;
@@ -412,15 +443,12 @@ try_preserve_large_page(pte_t *kpte, unsigned long address,
 
         switch (level) {
         case PG_LEVEL_2M:
-                psize = PMD_PAGE_SIZE;
-                pmask = PMD_PAGE_MASK;
-                break;
 #ifdef CONFIG_X86_64
         case PG_LEVEL_1G:
-                psize = PUD_PAGE_SIZE;
-                pmask = PUD_PAGE_MASK;
-                break;
 #endif
+                psize = page_level_size(level);
+                pmask = page_level_mask(level);
+                break;
         default:
                 do_split = -EINVAL;
                 goto out_unlock;
@@ -445,6 +473,19 @@ try_preserve_large_page(pte_t *kpte, unsigned long address,
         pgprot_val(req_prot) |= pgprot_val(cpa->mask_set);
 
         /*
+         * Set the PSE and GLOBAL flags only if the PRESENT flag is
+         * set otherwise pmd_present/pmd_huge will return true even on
+         * a non present pmd. The canon_pgprot will clear _PAGE_GLOBAL
+         * for the ancient hardware that doesn't support it.
+         */
+        if (pgprot_val(new_prot) & _PAGE_PRESENT)
+                pgprot_val(new_prot) |= _PAGE_PSE | _PAGE_GLOBAL;
+        else
+                pgprot_val(new_prot) &= ~(_PAGE_PSE | _PAGE_GLOBAL);
+
+        new_prot = canon_pgprot(new_prot);
+
+        /*
          * old_pte points to the large page base address. So we need
          * to add the offset of the virtual address:
          */
@@ -489,7 +530,7 @@ try_preserve_large_page(pte_t *kpte, unsigned long address,
                  * The address is aligned and the number of pages
                  * covers the full page.
                  */
-                new_pte = pfn_pte(pte_pfn(old_pte), canon_pgprot(new_prot));
+                new_pte = pfn_pte(pte_pfn(old_pte), new_prot);
                 __set_pmd_pte(kpte, address, new_pte);
                 cpa->flags |= CPA_FLUSHTLB;
                 do_split = 0;
@@ -501,21 +542,13 @@ out_unlock:
         return do_split;
 }
 
-static int split_large_page(pte_t *kpte, unsigned long address)
+int __split_large_page(pte_t *kpte, unsigned long address, pte_t *pbase)
 {
         unsigned long pfn, pfninc = 1;
         unsigned int i, level;
-        pte_t *pbase, *tmp;
+        pte_t *tmp;
         pgprot_t ref_prot;
-        struct page *base;
-
-        if (!debug_pagealloc)
-                spin_unlock(&cpa_lock);
-        base = alloc_pages(GFP_KERNEL | __GFP_NOTRACK, 0);
-        if (!debug_pagealloc)
-                spin_lock(&cpa_lock);
-        if (!base)
-                return -ENOMEM;
+        struct page *base = virt_to_page(pbase);
 
         spin_lock(&pgd_lock);
         /*
@@ -523,10 +556,11 @@ static int split_large_page(pte_t *kpte, unsigned long address)
          * up for us already:
          */
         tmp = lookup_address(address, &level);
-        if (tmp != kpte)
-                goto out_unlock;
+        if (tmp != kpte) {
+                spin_unlock(&pgd_lock);
+                return 1;
+        }
 
-        pbase = (pte_t *)page_address(base);
         paravirt_alloc_pte(&init_mm, page_to_pfn(base));
         ref_prot = pte_pgprot(pte_clrhuge(*kpte));
         /*
@@ -540,27 +574,40 @@ static int split_large_page(pte_t *kpte, unsigned long address)
 #ifdef CONFIG_X86_64
         if (level == PG_LEVEL_1G) {
                 pfninc = PMD_PAGE_SIZE >> PAGE_SHIFT;
-                pgprot_val(ref_prot) |= _PAGE_PSE;
+                /*
+                 * Set the PSE flags only if the PRESENT flag is set
+                 * otherwise pmd_present/pmd_huge will return true
+                 * even on a non present pmd.
+                 */
+                if (pgprot_val(ref_prot) & _PAGE_PRESENT)
+                        pgprot_val(ref_prot) |= _PAGE_PSE;
+                else
+                        pgprot_val(ref_prot) &= ~_PAGE_PSE;
         }
 #endif
 
         /*
+         * Set the GLOBAL flags only if the PRESENT flag is set
+         * otherwise pmd/pte_present will return true even on a non
+         * present pmd/pte. The canon_pgprot will clear _PAGE_GLOBAL
+         * for the ancient hardware that doesn't support it.
+         */
+        if (pgprot_val(ref_prot) & _PAGE_PRESENT)
+                pgprot_val(ref_prot) |= _PAGE_GLOBAL;
+        else
+                pgprot_val(ref_prot) &= ~_PAGE_GLOBAL;
+
+        /*
          * Get the target pfn from the original entry:
          */
         pfn = pte_pfn(*kpte);
         for (i = 0; i < PTRS_PER_PTE; i++, pfn += pfninc)
-                set_pte(&pbase[i], pfn_pte(pfn, ref_prot));
+                set_pte(&pbase[i], pfn_pte(pfn, canon_pgprot(ref_prot)));
 
-        if (address >= (unsigned long)__va(0) &&
-                address < (unsigned long)__va(max_low_pfn_mapped << PAGE_SHIFT))
+        if (pfn_range_is_mapped(PFN_DOWN(__pa(address)),
+                                PFN_DOWN(__pa(address)) + 1))
                 split_page_count(level);
 
-#ifdef CONFIG_X86_64
-        if (address >= (unsigned long)__va(1UL<<32) &&
-                address < (unsigned long)__va(max_pfn_mapped << PAGE_SHIFT))
-                split_page_count(level);
-#endif
-
         /*
          * Install the new, split up pagetable.
          *
@@ -579,17 +626,27 @@ static int split_large_page(pte_t *kpte, unsigned long address)
          * going on.
          */
         __flush_tlb_all();
+        spin_unlock(&pgd_lock);
 
-        base = NULL;
+        return 0;
+}
 
-out_unlock:
-        /*
-         * If we dropped out via the lookup_address check under
-         * pgd_lock then stick the page back into the pool:
-         */
-        if (base)
+static int split_large_page(pte_t *kpte, unsigned long address)
+{
+        pte_t *pbase;
+        struct page *base;
+
+        if (!debug_pagealloc)
+                spin_unlock(&cpa_lock);
+        base = alloc_pages(GFP_KERNEL | __GFP_NOTRACK, 0);
+        if (!debug_pagealloc)
+                spin_lock(&cpa_lock);
+        if (!base)
+                return -ENOMEM;
+
+        pbase = (pte_t *)page_address(base);
+        if (__split_large_page(kpte, address, pbase))
                 __free_page(base);
-        spin_unlock(&pgd_lock);
 
         return 0;
 }
@@ -660,6 +717,18 @@ repeat:
                 new_prot = static_protections(new_prot, address, pfn);
 
                 /*
+                 * Set the GLOBAL flags only if the PRESENT flag is
+                 * set otherwise pte_present will return true even on
+                 * a non present pte. The canon_pgprot will clear
+                 * _PAGE_GLOBAL for the ancient hardware that doesn't
+                 * support it.
+                 */
+                if (pgprot_val(new_prot) & _PAGE_PRESENT)
+                        pgprot_val(new_prot) |= _PAGE_GLOBAL;
+                else
+                        pgprot_val(new_prot) &= ~_PAGE_GLOBAL;
+
+                /*
                  * We need to keep the pfn from the existing PTE,
                  * after all we're only going to change it's attributes
                  * not the memory it points to
@@ -729,13 +798,9 @@ static int cpa_process_alias(struct cpa_data *cpa)
         unsigned long vaddr;
         int ret;
 
-        if (cpa->pfn >= max_pfn_mapped)
+        if (!pfn_range_is_mapped(cpa->pfn, cpa->pfn + 1))
                 return 0;
 
-#ifdef CONFIG_X86_64
-        if (cpa->pfn >= max_low_pfn_mapped && cpa->pfn < (1UL<<(32-PAGE_SHIFT)))
-                return 0;
-#endif
         /*
          * No need to redo, when the primary call touched the direct
          * mapping already:
diff --git a/arch/x86/mm/pat.c b/arch/x86/mm/pat.c
index 0eb572eda406..657438858e83 100644
--- a/arch/x86/mm/pat.c
+++ b/arch/x86/mm/pat.c
@@ -560,10 +560,17 @@ int kernel_map_sync_memtype(u64 base, unsigned long size, unsigned long flags)
 {
         unsigned long id_sz;
 
-        if (base >= __pa(high_memory))
+        if (base > __pa(high_memory-1))
                 return 0;
 
-        id_sz = (__pa(high_memory) < base + size) ?
+        /*
+         * some areas in the middle of the kernel identity range
+         * are not mapped, like the PCI space.
+         */
+        if (!page_is_ram(base >> PAGE_SHIFT))
+                return 0;
+
+        id_sz = (__pa(high_memory-1) <= base + size) ?
                                 __pa(high_memory) - base :
                                 size;
 
diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c
index e27fbf887f3b..193350b51f90 100644
--- a/arch/x86/mm/pgtable.c
+++ b/arch/x86/mm/pgtable.c
@@ -334,7 +334,12 @@ int pmdp_set_access_flags(struct vm_area_struct *vma,
         if (changed && dirty) {
                 *pmdp = entry;
                 pmd_update_defer(vma->vm_mm, address, pmdp);
-                flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
+                /*
+                 * We had a write-protection fault here and changed the pmd
+                 * to to more permissive. No need to flush the TLB for that,
+                 * #PF is architecturally guaranteed to do that and in the
+                 * worst-case we'll generate a spurious fault.
+                 */
         }
 
         return changed;
diff --git a/arch/x86/mm/physaddr.c b/arch/x86/mm/physaddr.c
index d2e2735327b4..e666cbbb9261 100644
--- a/arch/x86/mm/physaddr.c
+++ b/arch/x86/mm/physaddr.c
@@ -1,3 +1,4 @@
+#include <linux/bootmem.h>
 #include <linux/mmdebug.h>
 #include <linux/module.h>
 #include <linux/mm.h>
@@ -8,33 +9,54 @@
 
 #ifdef CONFIG_X86_64
 
+#ifdef CONFIG_DEBUG_VIRTUAL
 unsigned long __phys_addr(unsigned long x)
 {
-        if (x >= __START_KERNEL_map) {
-                x -= __START_KERNEL_map;
-                VIRTUAL_BUG_ON(x >= KERNEL_IMAGE_SIZE);
-                x += phys_base;
+        unsigned long y = x - __START_KERNEL_map;
+
+        /* use the carry flag to determine if x was < __START_KERNEL_map */
+        if (unlikely(x > y)) {
+                x = y + phys_base;
+
+                VIRTUAL_BUG_ON(y >= KERNEL_IMAGE_SIZE);
         } else {
-                VIRTUAL_BUG_ON(x < PAGE_OFFSET);
-                x -= PAGE_OFFSET;
-                VIRTUAL_BUG_ON(!phys_addr_valid(x));
+                x = y + (__START_KERNEL_map - PAGE_OFFSET);
+
+                /* carry flag will be set if starting x was >= PAGE_OFFSET */
+                VIRTUAL_BUG_ON((x > y) || !phys_addr_valid(x));
         }
+
         return x;
 }
 EXPORT_SYMBOL(__phys_addr);
 
+unsigned long __phys_addr_symbol(unsigned long x)
+{
+        unsigned long y = x - __START_KERNEL_map;
+
+        /* only check upper bounds since lower bounds will trigger carry */
+        VIRTUAL_BUG_ON(y >= KERNEL_IMAGE_SIZE);
+
+        return y + phys_base;
+}
+EXPORT_SYMBOL(__phys_addr_symbol);
+#endif
+
 bool __virt_addr_valid(unsigned long x)
 {
-        if (x >= __START_KERNEL_map) {
-                x -= __START_KERNEL_map;
-                if (x >= KERNEL_IMAGE_SIZE)
+        unsigned long y = x - __START_KERNEL_map;
+
+        /* use the carry flag to determine if x was < __START_KERNEL_map */
+        if (unlikely(x > y)) {
+                x = y + phys_base;
+
+                if (y >= KERNEL_IMAGE_SIZE)
                         return false;
-                x += phys_base;
         } else {
-                if (x < PAGE_OFFSET)
-                        return false;
-                x -= PAGE_OFFSET;
-                if (!phys_addr_valid(x))
+                x = y + (__START_KERNEL_map - PAGE_OFFSET);
+
+                /* carry flag will be set if starting x was >= PAGE_OFFSET */
+                if ((x > y) || !phys_addr_valid(x))
                         return false;
         }
 
@@ -47,10 +69,16 @@ EXPORT_SYMBOL(__virt_addr_valid);
 #ifdef CONFIG_DEBUG_VIRTUAL
 unsigned long __phys_addr(unsigned long x)
 {
+        unsigned long phys_addr = x - PAGE_OFFSET;
         /* VMALLOC_* aren't constants  */
         VIRTUAL_BUG_ON(x < PAGE_OFFSET);
         VIRTUAL_BUG_ON(__vmalloc_start_set && is_vmalloc_addr((void *) x));
-        return x - PAGE_OFFSET;
+        /* max_low_pfn is set early, but not _that_ early */
+        if (max_low_pfn) {
+                VIRTUAL_BUG_ON((phys_addr >> PAGE_SHIFT) > max_low_pfn);
+                BUG_ON(slow_virt_to_phys((void *)x) != phys_addr);
+        }
+        return phys_addr;
 }
 EXPORT_SYMBOL(__phys_addr);
 #endif
diff --git a/arch/x86/mm/srat.c b/arch/x86/mm/srat.c
index 4ddf497ca65b..cdd0da9dd530 100644
--- a/arch/x86/mm/srat.c
+++ b/arch/x86/mm/srat.c
@@ -149,39 +149,40 @@ acpi_numa_memory_affinity_init(struct acpi_srat_mem_affinity *ma)
         int node, pxm;
 
         if (srat_disabled())
-                return -1;
-        if (ma->header.length != sizeof(struct acpi_srat_mem_affinity)) {
-                bad_srat();
-                return -1;
-        }
+                goto out_err;
+        if (ma->header.length != sizeof(struct acpi_srat_mem_affinity))
+                goto out_err_bad_srat;
         if ((ma->flags & ACPI_SRAT_MEM_ENABLED) == 0)
-                return -1;
-
+                goto out_err;
         if ((ma->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE) && !save_add_info())
-                return -1;
+                goto out_err;
+
         start = ma->base_address;
         end = start + ma->length;
         pxm = ma->proximity_domain;
         if (acpi_srat_revision <= 1)
                 pxm &= 0xff;
+
         node = setup_node(pxm);
         if (node < 0) {
                 printk(KERN_ERR "SRAT: Too many proximity domains.\n");
-                bad_srat();
-                return -1;
+                goto out_err_bad_srat;
         }
 
-        if (numa_add_memblk(node, start, end) < 0) {
-                bad_srat();
-                return -1;
-        }
+        if (numa_add_memblk(node, start, end) < 0)
+                goto out_err_bad_srat;
 
         node_set(node, numa_nodes_parsed);
 
         printk(KERN_INFO "SRAT: Node %u PXM %u [mem %#010Lx-%#010Lx]\n",
                node, pxm,
                (unsigned long long) start, (unsigned long long) end - 1);
+
         return 0;
+out_err_bad_srat:
+        bad_srat();
+out_err:
+        return -1;
 }
 
 void __init acpi_numa_arch_fixup(void) {}
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index 13a6b29e2e5d..282375f13c7e 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -335,7 +335,7 @@ static const struct file_operations fops_tlbflush = {
         .llseek = default_llseek,
 };
 
-static int __cpuinit create_tlb_flushall_shift(void)
+static int __init create_tlb_flushall_shift(void)
 {
         debugfs_create_file("tlb_flushall_shift", S_IRUSR | S_IWUSR,
                             arch_debugfs_dir, NULL, &fops_tlbflush);
diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index d11a47099d33..3cbe45381bbb 100644
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -1,6 +1,6 @@
 /* bpf_jit_comp.c : BPF JIT compiler
  *
- * Copyright (C) 2011 Eric Dumazet (eric.dumazet@gmail.com)
+ * Copyright (C) 2011-2013 Eric Dumazet (eric.dumazet@gmail.com)
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
@@ -124,6 +124,26 @@ static inline void bpf_flush_icache(void *start, void *end)
 #define CHOOSE_LOAD_FUNC(K, func) \
         ((int)K < 0 ? ((int)K >= SKF_LL_OFF ? func##_negative_offset : func) : func##_positive_offset)
 
+/* Helper to find the offset of pkt_type in sk_buff
+ * We want to make sure its still a 3bit field starting at a byte boundary.
+ */
+#define PKT_TYPE_MAX 7
+static int pkt_type_offset(void)
+{
+        struct sk_buff skb_probe = {
+                .pkt_type = ~0,
+        };
+        char *ct = (char *)&skb_probe;
+        unsigned int off;
+
+        for (off = 0; off < sizeof(struct sk_buff); off++) {
+                if (ct[off] == PKT_TYPE_MAX)
+                        return off;
+        }
+        pr_err_once("Please fix pkt_type_offset(), as pkt_type couldn't be found\n");
+        return -1;
+}
+
 void bpf_jit_compile(struct sk_filter *fp)
 {
         u8 temp[64];
@@ -216,6 +236,7 @@ void bpf_jit_compile(struct sk_filter *fp)
                 case BPF_S_ANC_VLAN_TAG:
                 case BPF_S_ANC_VLAN_TAG_PRESENT:
                 case BPF_S_ANC_QUEUE:
+                case BPF_S_ANC_PKTTYPE:
                 case BPF_S_LD_W_ABS:
                 case BPF_S_LD_H_ABS:
                 case BPF_S_LD_B_ABS:
@@ -536,6 +557,23 @@ void bpf_jit_compile(struct sk_filter *fp)
                                         EMIT3(0x83, 0xe0, 0x01); /* and    $0x1,%eax */
                                 }
                                 break;
+                        case BPF_S_ANC_PKTTYPE:
+                        {
+                                int off = pkt_type_offset();
+
+                                if (off < 0)
+                                        goto out;
+                                if (is_imm8(off)) {
+                                        /* movzbl off8(%rdi),%eax */
+                                        EMIT4(0x0f, 0xb6, 0x47, off);
+                                } else {
+                                        /* movbl off32(%rdi),%eax */
+                                        EMIT3(0x0f, 0xb6, 0x87);
+                                        EMIT(off, 4);
+                                }
+                                EMIT3(0x83, 0xe0, PKT_TYPE_MAX); /* and    $0x7,%eax */
+                                break;
+                        }
                         case BPF_S_LD_W_ABS:
                                 func = CHOOSE_LOAD_FUNC(K, sk_load_word);
 common_load:                    seen |= SEEN_DATAREF;
diff --git a/arch/x86/pci/acpi.c b/arch/x86/pci/acpi.c
index 53ea60458e01..3e724256dbee 100644
--- a/arch/x86/pci/acpi.c
+++ b/arch/x86/pci/acpi.c
@@ -521,6 +521,7 @@ struct pci_bus *pci_acpi_scan_root(struct acpi_pci_root *root)
         sd = &info->sd;
         sd->domain = domain;
         sd->node = node;
+        sd->acpi = device->handle;
         /*
          * Maybe the desired pci bus has been already scanned. In such case
          * it is unnecessary to scan the pci bus with the given domain,busnum.
@@ -592,6 +593,14 @@ struct pci_bus *pci_acpi_scan_root(struct acpi_pci_root *root)
         return bus;
 }
 
+int pcibios_root_bridge_prepare(struct pci_host_bridge *bridge)
+{
+        struct pci_sysdata *sd = bridge->bus->sysdata;
+
+        ACPI_HANDLE_SET(&bridge->dev, sd->acpi);
+        return 0;
+}
+
 int __init pci_acpi_init(void)
 {
         struct pci_dev *dev = NULL;
diff --git a/arch/x86/pci/common.c b/arch/x86/pci/common.c
index ccd0ab3ab899..901177d75ff5 100644
--- a/arch/x86/pci/common.c
+++ b/arch/x86/pci/common.c
@@ -34,7 +34,6 @@ int noioapicreroute = 1;
 #endif
 int pcibios_last_bus = -1;
 unsigned long pirq_table_addr;
-struct pci_bus *pci_root_bus;
 const struct pci_raw_ops *__read_mostly raw_pci_ops;
 const struct pci_raw_ops *__read_mostly raw_pci_ext_ops;
 
diff --git a/arch/x86/pci/i386.c b/arch/x86/pci/i386.c
index dd8ca6f7223b..94919e307f8e 100644
--- a/arch/x86/pci/i386.c
+++ b/arch/x86/pci/i386.c
@@ -51,6 +51,7 @@ struct pcibios_fwaddrmap {
 
 static LIST_HEAD(pcibios_fwaddrmappings);
 static DEFINE_SPINLOCK(pcibios_fwaddrmap_lock);
+static bool pcibios_fw_addr_done;
 
 /* Must be called with 'pcibios_fwaddrmap_lock' lock held. */
 static struct pcibios_fwaddrmap *pcibios_fwaddrmap_lookup(struct pci_dev *dev)
@@ -72,6 +73,9 @@ pcibios_save_fw_addr(struct pci_dev *dev, int idx, resource_size_t fw_addr)
         unsigned long flags;
         struct pcibios_fwaddrmap *map;
 
+        if (pcibios_fw_addr_done)
+                return;
+
         spin_lock_irqsave(&pcibios_fwaddrmap_lock, flags);
         map = pcibios_fwaddrmap_lookup(dev);
         if (!map) {
@@ -97,6 +101,9 @@ resource_size_t pcibios_retrieve_fw_addr(struct pci_dev *dev, int idx)
         struct pcibios_fwaddrmap *map;
         resource_size_t fw_addr = 0;
 
+        if (pcibios_fw_addr_done)
+                return 0;
+
         spin_lock_irqsave(&pcibios_fwaddrmap_lock, flags);
         map = pcibios_fwaddrmap_lookup(dev);
         if (map)
@@ -106,7 +113,7 @@ resource_size_t pcibios_retrieve_fw_addr(struct pci_dev *dev, int idx)
         return fw_addr;
 }
 
-static void pcibios_fw_addr_list_del(void)
+static void __init pcibios_fw_addr_list_del(void)
 {
         unsigned long flags;
         struct pcibios_fwaddrmap *entry, *next;
@@ -118,6 +125,7 @@ static void pcibios_fw_addr_list_del(void)
                 kfree(entry);
         }
         spin_unlock_irqrestore(&pcibios_fwaddrmap_lock, flags);
+        pcibios_fw_addr_done = true;
 }
 
 static int
@@ -193,46 +201,46 @@ EXPORT_SYMBOL(pcibios_align_resource);
  *          as well.
  */
 
-static void __init pcibios_allocate_bus_resources(struct list_head *bus_list)
+static void pcibios_allocate_bridge_resources(struct pci_dev *dev)
 {
-        struct pci_bus *bus;
-        struct pci_dev *dev;
         int idx;
         struct resource *r;
 
-        /* Depth-First Search on bus tree */
-        list_for_each_entry(bus, bus_list, node) {
-                if ((dev = bus->self)) {
-                        for (idx = PCI_BRIDGE_RESOURCES;
-                            idx < PCI_NUM_RESOURCES; idx++) {
-                                r = &dev->resource[idx];
-                                if (!r->flags)
-                                        continue;
-                                if (!r->start ||
-                                    pci_claim_resource(dev, idx) < 0) {
-                                        /*
-                                         * Something is wrong with the region.
-                                         * Invalidate the resource to prevent
-                                         * child resource allocations in this
-                                         * range.
-                                         */
-                                        r->start = r->end = 0;
-                                        r->flags = 0;
-                                }
-                        }
+        for (idx = PCI_BRIDGE_RESOURCES; idx < PCI_NUM_RESOURCES; idx++) {
+                r = &dev->resource[idx];
+                if (!r->flags)
+                        continue;
+                if (!r->start || pci_claim_resource(dev, idx) < 0) {
+                        /*
+                         * Something is wrong with the region.
+                         * Invalidate the resource to prevent
+                         * child resource allocations in this
+                         * range.
+                         */
+                        r->start = r->end = 0;
+                        r->flags = 0;
                 }
-                pcibios_allocate_bus_resources(&bus->children);
         }
 }
 
+static void pcibios_allocate_bus_resources(struct pci_bus *bus)
+{
+        struct pci_bus *child;
+
+        /* Depth-First Search on bus tree */
+        if (bus->self)
+                pcibios_allocate_bridge_resources(bus->self);
+        list_for_each_entry(child, &bus->children, node)
+                pcibios_allocate_bus_resources(child);
+}
+
 struct pci_check_idx_range {
         int start;
         int end;
 };
 
-static void __init pcibios_allocate_resources(int pass)
+static void pcibios_allocate_dev_resources(struct pci_dev *dev, int pass)
 {
-        struct pci_dev *dev = NULL;
         int idx, disabled, i;
         u16 command;
         struct resource *r;
@@ -244,14 +252,13 @@ static void __init pcibios_allocate_resources(int pass)
 #endif
         };
 
-        for_each_pci_dev(dev) {
-                pci_read_config_word(dev, PCI_COMMAND, &command);
-                for (i = 0; i < ARRAY_SIZE(idx_range); i++)
+        pci_read_config_word(dev, PCI_COMMAND, &command);
+        for (i = 0; i < ARRAY_SIZE(idx_range); i++)
                 for (idx = idx_range[i].start; idx <= idx_range[i].end; idx++) {
                         r = &dev->resource[idx];
-                        if (r->parent)          /* Already allocated */
+                        if (r->parent)  /* Already allocated */
                                 continue;
-                        if (!r->start)          /* Address not assigned at all */
+                        if (!r->start)  /* Address not assigned at all */
                                 continue;
                         if (r->flags & IORESOURCE_IO)
                                 disabled = !(command & PCI_COMMAND_IO);
@@ -270,44 +277,74 @@ static void __init pcibios_allocate_resources(int pass)
                                 }
                         }
                 }
-                if (!pass) {
-                        r = &dev->resource[PCI_ROM_RESOURCE];
-                        if (r->flags & IORESOURCE_ROM_ENABLE) {
-                                /* Turn the ROM off, leave the resource region,
-                                 * but keep it unregistered. */
-                                u32 reg;
-                                dev_dbg(&dev->dev, "disabling ROM %pR\n", r);
-                                r->flags &= ~IORESOURCE_ROM_ENABLE;
-                                pci_read_config_dword(dev,
-                                                dev->rom_base_reg, &reg);
-                                pci_write_config_dword(dev, dev->rom_base_reg,
+        if (!pass) {
+                r = &dev->resource[PCI_ROM_RESOURCE];
+                if (r->flags & IORESOURCE_ROM_ENABLE) {
+                        /* Turn the ROM off, leave the resource region,
+                         * but keep it unregistered. */
+                        u32 reg;
+                        dev_dbg(&dev->dev, "disabling ROM %pR\n", r);
+                        r->flags &= ~IORESOURCE_ROM_ENABLE;
+                        pci_read_config_dword(dev, dev->rom_base_reg, &reg);
+                        pci_write_config_dword(dev, dev->rom_base_reg,
                                                 reg & ~PCI_ROM_ADDRESS_ENABLE);
-                        }
                 }
         }
 }
 
-static int __init pcibios_assign_resources(void)
+static void pcibios_allocate_resources(struct pci_bus *bus, int pass)
+{
+        struct pci_dev *dev;
+        struct pci_bus *child;
+
+        list_for_each_entry(dev, &bus->devices, bus_list) {
+                pcibios_allocate_dev_resources(dev, pass);
+
+                child = dev->subordinate;
+                if (child)
+                        pcibios_allocate_resources(child, pass);
+        }
+}
+
+static void pcibios_allocate_dev_rom_resource(struct pci_dev *dev)
 {
-        struct pci_dev *dev = NULL;
         struct resource *r;
 
-        if (!(pci_probe & PCI_ASSIGN_ROMS)) {
-                /*
-                 * Try to use BIOS settings for ROMs, otherwise let
-                 * pci_assign_unassigned_resources() allocate the new
-                 * addresses.
-                 */
-                for_each_pci_dev(dev) {
-                        r = &dev->resource[PCI_ROM_RESOURCE];
-                        if (!r->flags || !r->start)
-                                continue;
-                        if (pci_claim_resource(dev, PCI_ROM_RESOURCE) < 0) {
-                                r->end -= r->start;
-                                r->start = 0;
-                        }
-                }
+        /*
+         * Try to use BIOS settings for ROMs, otherwise let
+         * pci_assign_unassigned_resources() allocate the new
+         * addresses.
+         */
+        r = &dev->resource[PCI_ROM_RESOURCE];
+        if (!r->flags || !r->start)
+                return;
+
+        if (pci_claim_resource(dev, PCI_ROM_RESOURCE) < 0) {
+                r->end -= r->start;
+                r->start = 0;
         }
+}
+static void pcibios_allocate_rom_resources(struct pci_bus *bus)
+{
+        struct pci_dev *dev;
+        struct pci_bus *child;
+
+        list_for_each_entry(dev, &bus->devices, bus_list) {
+                pcibios_allocate_dev_rom_resource(dev);
+
+                child = dev->subordinate;
+                if (child)
+                        pcibios_allocate_rom_resources(child);
+        }
+}
+
+static int __init pcibios_assign_resources(void)
+{
+        struct pci_bus *bus;
+
+        if (!(pci_probe & PCI_ASSIGN_ROMS))
+                list_for_each_entry(bus, &pci_root_buses, node)
+                        pcibios_allocate_rom_resources(bus);
 
         pci_assign_unassigned_resources();
         pcibios_fw_addr_list_del();
@@ -315,12 +352,32 @@ static int __init pcibios_assign_resources(void)
         return 0;
 }
 
+void pcibios_resource_survey_bus(struct pci_bus *bus)
+{
+        dev_printk(KERN_DEBUG, &bus->dev, "Allocating resources\n");
+
+        pcibios_allocate_bus_resources(bus);
+
+        pcibios_allocate_resources(bus, 0);
+        pcibios_allocate_resources(bus, 1);
+
+        if (!(pci_probe & PCI_ASSIGN_ROMS))
+                pcibios_allocate_rom_resources(bus);
+}
+
 void __init pcibios_resource_survey(void)
 {
+        struct pci_bus *bus;
+
         DBG("PCI: Allocating resources\n");
-        pcibios_allocate_bus_resources(&pci_root_buses);
-        pcibios_allocate_resources(0);
-        pcibios_allocate_resources(1);
+
+        list_for_each_entry(bus, &pci_root_buses, node)
+                pcibios_allocate_bus_resources(bus);
+
+        list_for_each_entry(bus, &pci_root_buses, node)
+                pcibios_allocate_resources(bus, 0);
+        list_for_each_entry(bus, &pci_root_buses, node)
+                pcibios_allocate_resources(bus, 1);
 
         e820_reserve_resources_late();
         /*
diff --git a/arch/x86/pci/legacy.c b/arch/x86/pci/legacy.c
index 4a2ab9cb3659..4db96fb1c232 100644
--- a/arch/x86/pci/legacy.c
+++ b/arch/x86/pci/legacy.c
@@ -30,7 +30,7 @@ int __init pci_legacy_init(void)
         }
 
         printk("PCI: Probing PCI hardware\n");
-        pci_root_bus = pcibios_scan_root(0);
+        pcibios_scan_root(0);
         return 0;
 }
 
diff --git a/arch/x86/pci/mmconfig-shared.c b/arch/x86/pci/mmconfig-shared.c
index fb29968a7cd5..082e88129712 100644
--- a/arch/x86/pci/mmconfig-shared.c
+++ b/arch/x86/pci/mmconfig-shared.c
@@ -548,8 +548,7 @@ static int __init acpi_mcfg_check_entry(struct acpi_table_mcfg *mcfg,
         if (cfg->address < 0xFFFFFFFF)
                 return 0;
 
-        if (!strcmp(mcfg->header.oem_id, "SGI") ||
-                        !strcmp(mcfg->header.oem_id, "SGI2"))
+        if (!strncmp(mcfg->header.oem_id, "SGI", 3))
                 return 0;
 
         if (mcfg->header.revision >= 1) {
diff --git a/arch/x86/pci/numaq_32.c b/arch/x86/pci/numaq_32.c
index b96b14c250b6..72c229f9ebcf 100644
--- a/arch/x86/pci/numaq_32.c
+++ b/arch/x86/pci/numaq_32.c
@@ -152,7 +152,7 @@ int __init pci_numaq_init(void)
 
         raw_pci_ops = &pci_direct_conf1_mq;
 
-        pci_root_bus = pcibios_scan_root(0);
+        pcibios_scan_root(0);
         if (num_online_nodes() > 1)
                 for_each_online_node(quad) {
                         if (quad == 0)
diff --git a/arch/x86/pci/xen.c b/arch/x86/pci/xen.c
index 56ab74989cf1..94e76620460f 100644
--- a/arch/x86/pci/xen.c
+++ b/arch/x86/pci/xen.c
@@ -162,6 +162,9 @@ static int xen_setup_msi_irqs(struct pci_dev *dev, int nvec, int type)
         struct msi_desc *msidesc;
         int *v;
 
+        if (type == PCI_CAP_ID_MSI && nvec > 1)
+                return 1;
+
         v = kzalloc(sizeof(int) * max(1, nvec), GFP_KERNEL);
         if (!v)
                 return -ENOMEM;
@@ -220,6 +223,9 @@ static int xen_hvm_setup_msi_irqs(struct pci_dev *dev, int nvec, int type)
         struct msi_desc *msidesc;
         struct msi_msg msg;
 
+        if (type == PCI_CAP_ID_MSI && nvec > 1)
+                return 1;
+
         list_for_each_entry(msidesc, &dev->msi_list, list) {
                 __read_msi_msg(msidesc, &msg);
                 pirq = MSI_ADDR_EXT_DEST_ID(msg.address_hi) |
@@ -263,6 +269,9 @@ static int xen_initdom_setup_msi_irqs(struct pci_dev *dev, int nvec, int type)
         int ret = 0;
         struct msi_desc *msidesc;
 
+        if (type == PCI_CAP_ID_MSI && nvec > 1)
+                return 1;
+
         list_for_each_entry(msidesc, &dev->msi_list, list) {
                 struct physdev_map_pirq map_irq;
                 domid_t domid;
diff --git a/arch/x86/platform/Makefile b/arch/x86/platform/Makefile
index 8d874396cb29..01e0231a113e 100644
--- a/arch/x86/platform/Makefile
+++ b/arch/x86/platform/Makefile
@@ -2,10 +2,12 @@
 obj-y   += ce4100/
 obj-y   += efi/
 obj-y   += geode/
+obj-y   += goldfish/
 obj-y   += iris/
 obj-y   += mrst/
 obj-y   += olpc/
 obj-y   += scx200/
 obj-y   += sfi/
+obj-y   += ts5500/
 obj-y   += visws/
 obj-y   += uv/
diff --git a/arch/x86/platform/efi/efi-bgrt.c b/arch/x86/platform/efi/efi-bgrt.c
index d9c1b95af17c..7145ec63c520 100644
--- a/arch/x86/platform/efi/efi-bgrt.c
+++ b/arch/x86/platform/efi/efi-bgrt.c
@@ -11,20 +11,21 @@
  * published by the Free Software Foundation.
  */
 #include <linux/kernel.h>
+#include <linux/init.h>
 #include <linux/acpi.h>
 #include <linux/efi.h>
 #include <linux/efi-bgrt.h>
 
 struct acpi_table_bgrt *bgrt_tab;
-void *bgrt_image;
-size_t bgrt_image_size;
+void *__initdata bgrt_image;
+size_t __initdata bgrt_image_size;
 
 struct bmp_header {
         u16 id;
         u32 size;
 } __packed;
 
-void efi_bgrt_init(void)
+void __init efi_bgrt_init(void)
 {
         acpi_status status;
         void __iomem *image;
diff --git a/arch/x86/platform/efi/efi.c b/arch/x86/platform/efi/efi.c
index 430cd784a0de..fff986da6239 100644
--- a/arch/x86/platform/efi/efi.c
+++ b/arch/x86/platform/efi/efi.c
@@ -80,9 +80,10 @@ int efi_enabled(int facility)
 }
 EXPORT_SYMBOL(efi_enabled);
 
+static bool __initdata disable_runtime = false;
 static int __init setup_noefi(char *arg)
 {
-        clear_bit(EFI_BOOT, &x86_efi_facility);
+        disable_runtime = true;
         return 0;
 }
 early_param("noefi", setup_noefi);
@@ -412,8 +413,8 @@ void __init efi_reserve_boot_services(void)
                  * - Not within any part of the kernel
                  * - Not the bios reserved area
                 */
-                if ((start+size >= virt_to_phys(_text)
-                                && start <= virt_to_phys(_end)) ||
+                if ((start+size >= __pa_symbol(_text)
+                                && start <= __pa_symbol(_end)) ||
                         !e820_all_mapped(start, start+size, E820_RAM) ||
                         memblock_is_region_reserved(start, size)) {
                         /* Could not reserve, skip it */
@@ -730,7 +731,7 @@ void __init efi_init(void)
         if (!efi_is_native())
                 pr_info("No EFI runtime due to 32/64-bit mismatch with kernel\n");
         else {
-                if (efi_runtime_init())
+                if (disable_runtime || efi_runtime_init())
                         return;
                 set_bit(EFI_RUNTIME_SERVICES, &x86_efi_facility);
         }
@@ -839,7 +840,7 @@ void __init efi_enter_virtual_mode(void)
         efi_memory_desc_t *md, *prev_md = NULL;
         efi_status_t status;
         unsigned long size;
-        u64 end, systab, end_pfn;
+        u64 end, systab, start_pfn, end_pfn;
         void *p, *va, *new_memmap = NULL;
         int count = 0;
 
@@ -892,10 +893,9 @@ void __init efi_enter_virtual_mode(void)
                 size = md->num_pages << EFI_PAGE_SHIFT;
                 end = md->phys_addr + size;
 
+                start_pfn = PFN_DOWN(md->phys_addr);
                 end_pfn = PFN_UP(end);
-                if (end_pfn <= max_low_pfn_mapped
-                    || (end_pfn > (1UL << (32 - PAGE_SHIFT))
-                        && end_pfn <= max_pfn_mapped)) {
+                if (pfn_range_is_mapped(start_pfn, end_pfn)) {
                         va = __va(md->phys_addr);
 
                         if (!(md->attribute & EFI_MEMORY_WB))
diff --git a/arch/x86/platform/goldfish/Makefile b/arch/x86/platform/goldfish/Makefile
new file mode 100644
index 000000000000..f030b532fdf3
--- /dev/null
+++ b/arch/x86/platform/goldfish/Makefile
@@ -0,0 +1 @@
+obj-$(CONFIG_GOLDFISH)  += goldfish.o
diff --git a/arch/x86/platform/goldfish/goldfish.c b/arch/x86/platform/goldfish/goldfish.c
new file mode 100644
index 000000000000..1693107a518e
--- /dev/null
+++ b/arch/x86/platform/goldfish/goldfish.c
@@ -0,0 +1,51 @@
+/*
+ * Copyright (C) 2007 Google, Inc.
+ * Copyright (C) 2011 Intel, Inc.
+ * Copyright (C) 2013 Intel, Inc.
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#include <linux/kernel.h>
+#include <linux/irq.h>
+#include <linux/platform_device.h>
+
+/*
+ * Where in virtual device memory the IO devices (timers, system controllers
+ * and so on)
+ */
+
+#define GOLDFISH_PDEV_BUS_BASE  (0xff001000)
+#define GOLDFISH_PDEV_BUS_END   (0xff7fffff)
+#define GOLDFISH_PDEV_BUS_IRQ   (4)
+
+#define GOLDFISH_TTY_BASE       (0x2000)
+
+static struct resource goldfish_pdev_bus_resources[] = {
+        {
+                .start  = GOLDFISH_PDEV_BUS_BASE,
+                .end    = GOLDFISH_PDEV_BUS_END,
+                .flags  = IORESOURCE_MEM,
+        },
+        {
+                .start  = GOLDFISH_PDEV_BUS_IRQ,
+                .end    = GOLDFISH_PDEV_BUS_IRQ,
+                .flags  = IORESOURCE_IRQ,
+        }
+};
+
+static int __init goldfish_init(void)
+{
+        platform_device_register_simple("goldfish_pdev_bus", -1,
+                                                goldfish_pdev_bus_resources, 2);
+        return 0;
+}
+device_initcall(goldfish_init);
diff --git a/arch/x86/platform/olpc/olpc-xo15-sci.c b/arch/x86/platform/olpc/olpc-xo15-sci.c
index 2fdca25905ae..fef7d0ba7e3a 100644
--- a/arch/x86/platform/olpc/olpc-xo15-sci.c
+++ b/arch/x86/platform/olpc/olpc-xo15-sci.c
@@ -195,7 +195,7 @@ err_sysfs:
         return r;
 }
 
-static int xo15_sci_remove(struct acpi_device *device, int type)
+static int xo15_sci_remove(struct acpi_device *device)
 {
         acpi_disable_gpe(NULL, xo15_sci_gpe);
         acpi_remove_gpe_handler(NULL, xo15_sci_gpe, xo15_sci_gpe_handler);
diff --git a/arch/x86/platform/sfi/sfi.c b/arch/x86/platform/sfi/sfi.c
index 7785b72ecc3a..bcd1a703e3e6 100644
--- a/arch/x86/platform/sfi/sfi.c
+++ b/arch/x86/platform/sfi/sfi.c
@@ -35,7 +35,7 @@
 static unsigned long sfi_lapic_addr __initdata = APIC_DEFAULT_PHYS_BASE;
 
 /* All CPUs enumerated by SFI must be present and enabled */
-static void __cpuinit mp_sfi_register_lapic(u8 id)
+static void __init mp_sfi_register_lapic(u8 id)
 {
         if (MAX_LOCAL_APIC - id <= 0) {
                 pr_warning("Processor #%d invalid (max %d)\n",
diff --git a/arch/x86/platform/ts5500/Makefile b/arch/x86/platform/ts5500/Makefile
new file mode 100644
index 000000000000..c54e348c96a7
--- /dev/null
+++ b/arch/x86/platform/ts5500/Makefile
@@ -0,0 +1 @@
+obj-$(CONFIG_TS5500)    += ts5500.o
diff --git a/arch/x86/platform/ts5500/ts5500.c b/arch/x86/platform/ts5500/ts5500.c
new file mode 100644
index 000000000000..39febb214e8c
--- /dev/null
+++ b/arch/x86/platform/ts5500/ts5500.c
@@ -0,0 +1,339 @@
+/*
+ * Technologic Systems TS-5500 Single Board Computer support
+ *
+ * Copyright (C) 2013 Savoir-faire Linux Inc.
+ *      Vivien Didelot <vivien.didelot@savoirfairelinux.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License as published by the Free Software
+ * Foundation; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ *
+ * This driver registers the Technologic Systems TS-5500 Single Board Computer
+ * (SBC) and its devices, and exposes information to userspace such as jumpers'
+ * state or available options. For further information about sysfs entries, see
+ * Documentation/ABI/testing/sysfs-platform-ts5500.
+ *
+ * This code actually supports the TS-5500 platform, but it may be extended to
+ * support similar Technologic Systems x86-based platforms, such as the TS-5600.
+ */
+
+#include <linux/delay.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/leds.h>
+#include <linux/module.h>
+#include <linux/platform_data/gpio-ts5500.h>
+#include <linux/platform_data/max197.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+
+/* Product code register */
+#define TS5500_PRODUCT_CODE_ADDR        0x74
+#define TS5500_PRODUCT_CODE             0x60    /* TS-5500 product code */
+
+/* SRAM/RS-485/ADC options, and RS-485 RTS/Automatic RS-485 flags register */
+#define TS5500_SRAM_RS485_ADC_ADDR      0x75
+#define TS5500_SRAM                     BIT(0)  /* SRAM option */
+#define TS5500_RS485                    BIT(1)  /* RS-485 option */
+#define TS5500_ADC                      BIT(2)  /* A/D converter option */
+#define TS5500_RS485_RTS                BIT(6)  /* RTS for RS-485 */
+#define TS5500_RS485_AUTO               BIT(7)  /* Automatic RS-485 */
+
+/* External Reset/Industrial Temperature Range options register */
+#define TS5500_ERESET_ITR_ADDR          0x76
+#define TS5500_ERESET                   BIT(0)  /* External Reset option */
+#define TS5500_ITR                      BIT(1)  /* Indust. Temp. Range option */
+
+/* LED/Jumpers register */
+#define TS5500_LED_JP_ADDR              0x77
+#define TS5500_LED                      BIT(0)  /* LED flag */
+#define TS5500_JP1                      BIT(1)  /* Automatic CMOS */
+#define TS5500_JP2                      BIT(2)  /* Enable Serial Console */
+#define TS5500_JP3                      BIT(3)  /* Write Enable Drive A */
+#define TS5500_JP4                      BIT(4)  /* Fast Console (115K baud) */
+#define TS5500_JP5                      BIT(5)  /* User Jumper */
+#define TS5500_JP6                      BIT(6)  /* Console on COM1 (req. JP2) */
+#define TS5500_JP7                      BIT(7)  /* Undocumented (Unused) */
+
+/* A/D Converter registers */
+#define TS5500_ADC_CONV_BUSY_ADDR       0x195   /* Conversion state register */
+#define TS5500_ADC_CONV_BUSY            BIT(0)
+#define TS5500_ADC_CONV_INIT_LSB_ADDR   0x196   /* Start conv. / LSB register */
+#define TS5500_ADC_CONV_MSB_ADDR        0x197   /* MSB register */
+#define TS5500_ADC_CONV_DELAY           12      /* usec */
+
+/**
+ * struct ts5500_sbc - TS-5500 board description
+ * @id:         Board product ID.
+ * @sram:       Flag for SRAM option.
+ * @rs485:      Flag for RS-485 option.
+ * @adc:        Flag for Analog/Digital converter option.
+ * @ereset:     Flag for External Reset option.
+ * @itr:        Flag for Industrial Temperature Range option.
+ * @jumpers:    Bitfield for jumpers' state.
+ */
+struct ts5500_sbc {
+        int     id;
+        bool    sram;
+        bool    rs485;
+        bool    adc;
+        bool    ereset;
+        bool    itr;
+        u8      jumpers;
+};
+
+/* Board signatures in BIOS shadow RAM */
+static const struct {
+        const char * const string;
+        const ssize_t offset;
+} ts5500_signatures[] __initdata = {
+        { "TS-5x00 AMD Elan", 0xb14 },
+};
+
+static int __init ts5500_check_signature(void)
+{
+        void __iomem *bios;
+        int i, ret = -ENODEV;
+
+        bios = ioremap(0xf0000, 0x10000);
+        if (!bios)
+                return -ENOMEM;
+
+        for (i = 0; i < ARRAY_SIZE(ts5500_signatures); i++) {
+                if (check_signature(bios + ts5500_signatures[i].offset,
+                                    ts5500_signatures[i].string,
+                                    strlen(ts5500_signatures[i].string))) {
+                        ret = 0;
+                        break;
+                }
+        }
+
+        iounmap(bios);
+        return ret;
+}
+
+static int __init ts5500_detect_config(struct ts5500_sbc *sbc)
+{
+        u8 tmp;
+        int ret = 0;
+
+        if (!request_region(TS5500_PRODUCT_CODE_ADDR, 4, "ts5500"))
+                return -EBUSY;
+
+        tmp = inb(TS5500_PRODUCT_CODE_ADDR);
+        if (tmp != TS5500_PRODUCT_CODE) {
+                pr_err("This platform is not a TS-5500 (found ID 0x%x)\n", tmp);
+                ret = -ENODEV;
+                goto cleanup;
+        }
+        sbc->id = tmp;
+
+        tmp = inb(TS5500_SRAM_RS485_ADC_ADDR);
+        sbc->sram = tmp & TS5500_SRAM;
+        sbc->rs485 = tmp & TS5500_RS485;
+        sbc->adc = tmp & TS5500_ADC;
+
+        tmp = inb(TS5500_ERESET_ITR_ADDR);
+        sbc->ereset = tmp & TS5500_ERESET;
+        sbc->itr = tmp & TS5500_ITR;
+
+        tmp = inb(TS5500_LED_JP_ADDR);
+        sbc->jumpers = tmp & ~TS5500_LED;
+
+cleanup:
+        release_region(TS5500_PRODUCT_CODE_ADDR, 4);
+        return ret;
+}
+
+static ssize_t ts5500_show_id(struct device *dev,
+                              struct device_attribute *attr, char *buf)
+{
+        struct ts5500_sbc *sbc = dev_get_drvdata(dev);
+
+        return sprintf(buf, "0x%.2x\n", sbc->id);
+}
+
+static ssize_t ts5500_show_jumpers(struct device *dev,
+                                   struct device_attribute *attr,
+                                   char *buf)
+{
+        struct ts5500_sbc *sbc = dev_get_drvdata(dev);
+
+        return sprintf(buf, "0x%.2x\n", sbc->jumpers >> 1);
+}
+
+#define TS5500_SHOW(field)                                      \
+        static ssize_t ts5500_show_##field(struct device *dev,  \
+                        struct device_attribute *attr,          \
+                        char *buf)                              \
+        {                                                       \
+                struct ts5500_sbc *sbc = dev_get_drvdata(dev);  \
+                return sprintf(buf, "%d\n", sbc->field);        \
+        }
+
+TS5500_SHOW(sram)
+TS5500_SHOW(rs485)
+TS5500_SHOW(adc)
+TS5500_SHOW(ereset)
+TS5500_SHOW(itr)
+
+static DEVICE_ATTR(id, S_IRUGO, ts5500_show_id, NULL);
+static DEVICE_ATTR(jumpers, S_IRUGO, ts5500_show_jumpers, NULL);
+static DEVICE_ATTR(sram, S_IRUGO, ts5500_show_sram, NULL);
+static DEVICE_ATTR(rs485, S_IRUGO, ts5500_show_rs485, NULL);
+static DEVICE_ATTR(adc, S_IRUGO, ts5500_show_adc, NULL);
+static DEVICE_ATTR(ereset, S_IRUGO, ts5500_show_ereset, NULL);
+static DEVICE_ATTR(itr, S_IRUGO, ts5500_show_itr, NULL);
+
+static struct attribute *ts5500_attributes[] = {
+        &dev_attr_id.attr,
+        &dev_attr_jumpers.attr,
+        &dev_attr_sram.attr,
+        &dev_attr_rs485.attr,
+        &dev_attr_adc.attr,
+        &dev_attr_ereset.attr,
+        &dev_attr_itr.attr,
+        NULL
+};
+
+static const struct attribute_group ts5500_attr_group = {
+        .attrs = ts5500_attributes,
+};
+
+static struct resource ts5500_dio1_resource[] = {
+        DEFINE_RES_IRQ_NAMED(7, "DIO1 interrupt"),
+};
+
+static struct platform_device ts5500_dio1_pdev = {
+        .name = "ts5500-dio1",
+        .id = -1,
+        .resource = ts5500_dio1_resource,
+        .num_resources = 1,
+};
+
+static struct resource ts5500_dio2_resource[] = {
+        DEFINE_RES_IRQ_NAMED(6, "DIO2 interrupt"),
+};
+
+static struct platform_device ts5500_dio2_pdev = {
+        .name = "ts5500-dio2",
+        .id = -1,
+        .resource = ts5500_dio2_resource,
+        .num_resources = 1,
+};
+
+static void ts5500_led_set(struct led_classdev *led_cdev,
+                           enum led_brightness brightness)
+{
+        outb(!!brightness, TS5500_LED_JP_ADDR);
+}
+
+static enum led_brightness ts5500_led_get(struct led_classdev *led_cdev)
+{
+        return (inb(TS5500_LED_JP_ADDR) & TS5500_LED) ? LED_FULL : LED_OFF;
+}
+
+static struct led_classdev ts5500_led_cdev = {
+        .name = "ts5500:green:",
+        .brightness_set = ts5500_led_set,
+        .brightness_get = ts5500_led_get,
+};
+
+static int ts5500_adc_convert(u8 ctrl)
+{
+        u8 lsb, msb;
+
+        /* Start conversion (ensure the 3 MSB are set to 0) */
+        outb(ctrl & 0x1f, TS5500_ADC_CONV_INIT_LSB_ADDR);
+
+        /*
+         * The platform has CPLD logic driving the A/D converter.
+         * The conversion must complete within 11 microseconds,
+         * otherwise we have to re-initiate a conversion.
+         */
+        udelay(TS5500_ADC_CONV_DELAY);
+        if (inb(TS5500_ADC_CONV_BUSY_ADDR) & TS5500_ADC_CONV_BUSY)
+                return -EBUSY;
+
+        /* Read the raw data */
+        lsb = inb(TS5500_ADC_CONV_INIT_LSB_ADDR);
+        msb = inb(TS5500_ADC_CONV_MSB_ADDR);
+
+        return (msb << 8) | lsb;
+}
+
+static struct max197_platform_data ts5500_adc_pdata = {
+        .convert = ts5500_adc_convert,
+};
+
+static struct platform_device ts5500_adc_pdev = {
+        .name = "max197",
+        .id = -1,
+        .dev = {
+                .platform_data = &ts5500_adc_pdata,
+        },
+};
+
+static int __init ts5500_init(void)
+{
+        struct platform_device *pdev;
+        struct ts5500_sbc *sbc;
+        int err;
+
+        /*
+         * There is no DMI available or PCI bridge subvendor info,
+         * only the BIOS provides a 16-bit identification call.
+         * It is safer to find a signature in the BIOS shadow RAM.
+         */
+        err = ts5500_check_signature();
+        if (err)
+                return err;
+
+        pdev = platform_device_register_simple("ts5500", -1, NULL, 0);
+        if (IS_ERR(pdev))
+                return PTR_ERR(pdev);
+
+        sbc = devm_kzalloc(&pdev->dev, sizeof(struct ts5500_sbc), GFP_KERNEL);
+        if (!sbc) {
+                err = -ENOMEM;
+                goto error;
+        }
+
+        err = ts5500_detect_config(sbc);
+        if (err)
+                goto error;
+
+        platform_set_drvdata(pdev, sbc);
+
+        err = sysfs_create_group(&pdev->dev.kobj, &ts5500_attr_group);
+        if (err)
+                goto error;
+
+        ts5500_dio1_pdev.dev.parent = &pdev->dev;
+        if (platform_device_register(&ts5500_dio1_pdev))
+                dev_warn(&pdev->dev, "DIO1 block registration failed\n");
+        ts5500_dio2_pdev.dev.parent = &pdev->dev;
+        if (platform_device_register(&ts5500_dio2_pdev))
+                dev_warn(&pdev->dev, "DIO2 block registration failed\n");
+
+        if (led_classdev_register(&pdev->dev, &ts5500_led_cdev))
+                dev_warn(&pdev->dev, "LED registration failed\n");
+
+        if (sbc->adc) {
+                ts5500_adc_pdev.dev.parent = &pdev->dev;
+                if (platform_device_register(&ts5500_adc_pdev))
+                        dev_warn(&pdev->dev, "ADC registration failed\n");
+        }
+
+        return 0;
+error:
+        platform_device_unregister(pdev);
+        return err;
+}
+device_initcall(ts5500_init);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Savoir-faire Linux Inc. <kernel@savoirfairelinux.com>");
+MODULE_DESCRIPTION("Technologic Systems TS-5500 platform driver");
diff --git a/arch/x86/platform/uv/tlb_uv.c b/arch/x86/platform/uv/tlb_uv.c
index dbbdca5f508c..0f92173a12b6 100644
--- a/arch/x86/platform/uv/tlb_uv.c
+++ b/arch/x86/platform/uv/tlb_uv.c
@@ -1467,7 +1467,7 @@ static ssize_t ptc_proc_write(struct file *file, const char __user *user,
         }
 
         if (input_arg == 0) {
-                elements = sizeof(stat_description)/sizeof(*stat_description);
+                elements = ARRAY_SIZE(stat_description);
                 printk(KERN_DEBUG "# cpu:      cpu number\n");
                 printk(KERN_DEBUG "Sender statistics:\n");
                 for (i = 0; i < elements; i++)
@@ -1508,7 +1508,7 @@ static int parse_tunables_write(struct bau_control *bcp, char *instr,
         char *q;
         int cnt = 0;
         int val;
-        int e = sizeof(tunables) / sizeof(*tunables);
+        int e = ARRAY_SIZE(tunables);
 
         p = instr + strspn(instr, WHITESPACE);
         q = p;
diff --git a/arch/x86/platform/uv/uv_time.c b/arch/x86/platform/uv/uv_time.c
index 5032e0d19b86..98718f604eb6 100644
--- a/arch/x86/platform/uv/uv_time.c
+++ b/arch/x86/platform/uv/uv_time.c
@@ -15,7 +15,7 @@
  *  along with this program; if not, write to the Free Software
  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
  *
- *  Copyright (c) 2009 Silicon Graphics, Inc.  All Rights Reserved.
+ *  Copyright (c) 2009-2013 Silicon Graphics, Inc.  All Rights Reserved.
  *  Copyright (c) Dimitri Sivanich
  */
 #include <linux/clockchips.h>
@@ -102,9 +102,10 @@ static int uv_intr_pending(int pnode)
         if (is_uv1_hub())
                 return uv_read_global_mmr64(pnode, UVH_EVENT_OCCURRED0) &
                         UV1H_EVENT_OCCURRED0_RTC1_MASK;
-        else
-                return uv_read_global_mmr64(pnode, UV2H_EVENT_OCCURRED2) &
-                        UV2H_EVENT_OCCURRED2_RTC_1_MASK;
+        else if (is_uvx_hub())
+                return uv_read_global_mmr64(pnode, UVXH_EVENT_OCCURRED2) &
+                        UVXH_EVENT_OCCURRED2_RTC_1_MASK;
+        return 0;
 }
 
 /* Setup interrupt and return non-zero if early expiration occurred. */
@@ -122,8 +123,8 @@ static int uv_setup_intr(int cpu, u64 expires)
                 uv_write_global_mmr64(pnode, UVH_EVENT_OCCURRED0_ALIAS,
                                 UV1H_EVENT_OCCURRED0_RTC1_MASK);
         else
-                uv_write_global_mmr64(pnode, UV2H_EVENT_OCCURRED2_ALIAS,
-                                UV2H_EVENT_OCCURRED2_RTC_1_MASK);
+                uv_write_global_mmr64(pnode, UVXH_EVENT_OCCURRED2_ALIAS,
+                                UVXH_EVENT_OCCURRED2_RTC_1_MASK);
 
         val = (X86_PLATFORM_IPI_VECTOR << UVH_RTC1_INT_CONFIG_VECTOR_SHFT) |
                 ((u64)apicid << UVH_RTC1_INT_CONFIG_APIC_ID_SHFT);
diff --git a/arch/x86/power/cpu.c b/arch/x86/power/cpu.c
index 120cee1c3f8d..3c68768d7a75 100644
--- a/arch/x86/power/cpu.c
+++ b/arch/x86/power/cpu.c
@@ -11,6 +11,7 @@
 #include <linux/suspend.h>
 #include <linux/export.h>
 #include <linux/smp.h>
+#include <linux/perf_event.h>
 
 #include <asm/pgtable.h>
 #include <asm/proto.h>
@@ -228,6 +229,7 @@ static void __restore_processor_state(struct saved_context *ctxt)
         do_fpu_end();
         x86_platform.restore_sched_clock_state();
         mtrr_bp_restore();
+        perf_restore_debug_store();
 }
 
 /* Needed by apm.c */
diff --git a/arch/x86/power/hibernate_32.c b/arch/x86/power/hibernate_32.c
index 74202c1910cd..7d28c885d238 100644
--- a/arch/x86/power/hibernate_32.c
+++ b/arch/x86/power/hibernate_32.c
@@ -129,8 +129,6 @@ static int resume_physical_mapping_init(pgd_t *pgd_base)
                 }
         }
 
-        resume_map_numa_kva(pgd_base);
-
         return 0;
 }
 
diff --git a/arch/x86/power/hibernate_64.c b/arch/x86/power/hibernate_64.c
index 460f314d13e5..a0fde91c16cf 100644
--- a/arch/x86/power/hibernate_64.c
+++ b/arch/x86/power/hibernate_64.c
@@ -11,6 +11,8 @@
 #include <linux/gfp.h>
 #include <linux/smp.h>
 #include <linux/suspend.h>
+
+#include <asm/init.h>
 #include <asm/proto.h>
 #include <asm/page.h>
 #include <asm/pgtable.h>
@@ -39,41 +41,21 @@ pgd_t *temp_level4_pgt;
 
 void *relocated_restore_code;
 
-static int res_phys_pud_init(pud_t *pud, unsigned long address, unsigned long end)
+static void *alloc_pgt_page(void *context)
 {
-        long i, j;
-
-        i = pud_index(address);
-        pud = pud + i;
-        for (; i < PTRS_PER_PUD; pud++, i++) {
-                unsigned long paddr;
-                pmd_t *pmd;
-
-                paddr = address + i*PUD_SIZE;
-                if (paddr >= end)
-                        break;
-
-                pmd = (pmd_t *)get_safe_page(GFP_ATOMIC);
-                if (!pmd)
-                        return -ENOMEM;
-                set_pud(pud, __pud(__pa(pmd) | _KERNPG_TABLE));
-                for (j = 0; j < PTRS_PER_PMD; pmd++, j++, paddr += PMD_SIZE) {
-                        unsigned long pe;
-
-                        if (paddr >= end)
-                                break;
-                        pe = __PAGE_KERNEL_LARGE_EXEC | paddr;
-                        pe &= __supported_pte_mask;
-                        set_pmd(pmd, __pmd(pe));
-                }
-        }
-        return 0;
+        return (void *)get_safe_page(GFP_ATOMIC);
 }
 
 static int set_up_temporary_mappings(void)
 {
-        unsigned long start, end, next;
-        int error;
+        struct x86_mapping_info info = {
+                .alloc_pgt_page = alloc_pgt_page,
+                .pmd_flag       = __PAGE_KERNEL_LARGE_EXEC,
+                .kernel_mapping = true,
+        };
+        unsigned long mstart, mend;
+        int result;
+        int i;
 
         temp_level4_pgt = (pgd_t *)get_safe_page(GFP_ATOMIC);
         if (!temp_level4_pgt)
@@ -84,21 +66,17 @@ static int set_up_temporary_mappings(void)
                 init_level4_pgt[pgd_index(__START_KERNEL_map)]);
 
         /* Set up the direct mapping from scratch */
-        start = (unsigned long)pfn_to_kaddr(0);
-        end = (unsigned long)pfn_to_kaddr(max_pfn);
-
-        for (; start < end; start = next) {
-                pud_t *pud = (pud_t *)get_safe_page(GFP_ATOMIC);
-                if (!pud)
-                        return -ENOMEM;
-                next = start + PGDIR_SIZE;
-                if (next > end)
-                        next = end;
-                if ((error = res_phys_pud_init(pud, __pa(start), __pa(next))))
-                        return error;
-                set_pgd(temp_level4_pgt + pgd_index(start),
-                        mk_kernel_pgd(__pa(pud)));
+        for (i = 0; i < nr_pfn_mapped; i++) {
+                mstart = pfn_mapped[i].start << PAGE_SHIFT;
+                mend   = pfn_mapped[i].end << PAGE_SHIFT;
+
+                result = kernel_ident_mapping_init(&info, temp_level4_pgt,
+                                                   mstart, mend);
+
+                if (result)
+                        return result;
         }
+
         return 0;
 }
 
diff --git a/arch/x86/realmode/init.c b/arch/x86/realmode/init.c
index cbca565af5bd..a44f457e70a1 100644
--- a/arch/x86/realmode/init.c
+++ b/arch/x86/realmode/init.c
@@ -8,9 +8,26 @@
 struct real_mode_header *real_mode_header;
 u32 *trampoline_cr4_features;
 
-void __init setup_real_mode(void)
+void __init reserve_real_mode(void)
 {
         phys_addr_t mem;
+        unsigned char *base;
+        size_t size = PAGE_ALIGN(real_mode_blob_end - real_mode_blob);
+
+        /* Has to be under 1M so we can execute real-mode AP code. */
+        mem = memblock_find_in_range(0, 1<<20, size, PAGE_SIZE);
+        if (!mem)
+                panic("Cannot allocate trampoline\n");
+
+        base = __va(mem);
+        memblock_reserve(mem, size);
+        real_mode_header = (struct real_mode_header *) base;
+        printk(KERN_DEBUG "Base memory trampoline at [%p] %llx size %zu\n",
+               base, (unsigned long long)mem, size);
+}
+
+void __init setup_real_mode(void)
+{
         u16 real_mode_seg;
         u32 *rel;
         u32 count;
@@ -25,16 +42,7 @@ void __init setup_real_mode(void)
         u64 efer;
 #endif
 
-        /* Has to be in very low memory so we can execute real-mode AP code. */
-        mem = memblock_find_in_range(0, 1<<20, size, PAGE_SIZE);
-        if (!mem)
-                panic("Cannot allocate trampoline\n");
-
-        base = __va(mem);
-        memblock_reserve(mem, size);
-        real_mode_header = (struct real_mode_header *) base;
-        printk(KERN_DEBUG "Base memory trampoline at [%p] %llx size %zu\n",
-               base, (unsigned long long)mem, size);
+        base = (unsigned char *)real_mode_header;
 
         memcpy(base, real_mode_blob, size);
 
@@ -62,9 +70,9 @@ void __init setup_real_mode(void)
                 __va(real_mode_header->trampoline_header);
 
 #ifdef CONFIG_X86_32
-        trampoline_header->start = __pa(startup_32_smp);
+        trampoline_header->start = __pa_symbol(startup_32_smp);
         trampoline_header->gdt_limit = __BOOT_DS + 7;
-        trampoline_header->gdt_base = __pa(boot_gdt);
+        trampoline_header->gdt_base = __pa_symbol(boot_gdt);
 #else
         /*
          * Some AMD processors will #GP(0) if EFER.LMA is set in WRMSR
@@ -78,16 +86,18 @@ void __init setup_real_mode(void)
         *trampoline_cr4_features = read_cr4();
 
         trampoline_pgd = (u64 *) __va(real_mode_header->trampoline_pgd);
-        trampoline_pgd[0] = __pa(level3_ident_pgt) + _KERNPG_TABLE;
-        trampoline_pgd[511] = __pa(level3_kernel_pgt) + _KERNPG_TABLE;
+        trampoline_pgd[0] = init_level4_pgt[pgd_index(__PAGE_OFFSET)].pgd;
+        trampoline_pgd[511] = init_level4_pgt[511].pgd;
 #endif
 }
 
 /*
- * set_real_mode_permissions() gets called very early, to guarantee the
- * availability of low memory.  This is before the proper kernel page
+ * reserve_real_mode() gets called very early, to guarantee the
+ * availability of low memory. This is before the proper kernel page
  * tables are set up, so we cannot set page permissions in that
- * function.  Thus, we use an arch_initcall instead.
+ * function. Also trampoline code will be executed by APs so we
+ * need to mark it executable at do_pre_smp_initcalls() at least,
+ * thus run it as a early_initcall().
  */
 static int __init set_real_mode_permissions(void)
 {
@@ -111,5 +121,4 @@ static int __init set_real_mode_permissions(void)
 
         return 0;
 }
-
-arch_initcall(set_real_mode_permissions);
+early_initcall(set_real_mode_permissions);
diff --git a/arch/x86/syscalls/syscall_32.tbl b/arch/x86/syscalls/syscall_32.tbl
index 28e3fa9056ea..e6d55f0064df 100644
--- a/arch/x86/syscalls/syscall_32.tbl
+++ b/arch/x86/syscalls/syscall_32.tbl
@@ -25,7 +25,7 @@
 16      i386    lchown                  sys_lchown16
 17      i386    break
 18      i386    oldstat                 sys_stat
-19      i386    lseek                   sys_lseek                       sys32_lseek
+19      i386    lseek                   sys_lseek                       compat_sys_lseek
 20      i386    getpid                  sys_getpid
 21      i386    mount                   sys_mount                       compat_sys_mount
 22      i386    umount                  sys_oldumount
@@ -73,12 +73,12 @@
 64      i386    getppid                 sys_getppid
 65      i386    getpgrp                 sys_getpgrp
 66      i386    setsid                  sys_setsid
-67      i386    sigaction               sys_sigaction                   sys32_sigaction
+67      i386    sigaction               sys_sigaction                   compat_sys_sigaction
 68      i386    sgetmask                sys_sgetmask
 69      i386    ssetmask                sys_ssetmask
 70      i386    setreuid                sys_setreuid16
 71      i386    setregid                sys_setregid16
-72      i386    sigsuspend              sys_sigsuspend                  sys32_sigsuspend
+72      i386    sigsuspend              sys_sigsuspend                  sys_sigsuspend
 73      i386    sigpending              sys_sigpending                  compat_sys_sigpending
 74      i386    sethostname             sys_sethostname
 75      i386    setrlimit               sys_setrlimit                   compat_sys_setrlimit
@@ -98,8 +98,8 @@
 89      i386    readdir                 sys_old_readdir                 compat_sys_old_readdir
 90      i386    mmap                    sys_old_mmap                    sys32_mmap
 91      i386    munmap                  sys_munmap
-92      i386    truncate                sys_truncate
-93      i386    ftruncate               sys_ftruncate
+92      i386    truncate                sys_truncate                    compat_sys_truncate
+93      i386    ftruncate               sys_ftruncate                   compat_sys_ftruncate
 94      i386    fchmod                  sys_fchmod
 95      i386    fchown                  sys_fchown16
 96      i386    getpriority             sys_getpriority
@@ -116,16 +116,16 @@
 107     i386    lstat                   sys_newlstat                    compat_sys_newlstat
 108     i386    fstat                   sys_newfstat                    compat_sys_newfstat
 109     i386    olduname                sys_uname
-110     i386    iopl                    ptregs_iopl                     stub32_iopl
+110     i386    iopl                    sys_iopl
 111     i386    vhangup                 sys_vhangup
 112     i386    idle
-113     i386    vm86old                 ptregs_vm86old                  sys32_vm86_warning
+113     i386    vm86old                 sys_vm86old                     sys32_vm86_warning
 114     i386    wait4                   sys_wait4                       compat_sys_wait4
 115     i386    swapoff                 sys_swapoff
 116     i386    sysinfo                 sys_sysinfo                     compat_sys_sysinfo
 117     i386    ipc                     sys_ipc                         sys32_ipc
 118     i386    fsync                   sys_fsync
-119     i386    sigreturn               ptregs_sigreturn                stub32_sigreturn
+119     i386    sigreturn               sys_sigreturn                   stub32_sigreturn
 120     i386    clone                   sys_clone                       stub32_clone
 121     i386    setdomainname           sys_setdomainname
 122     i386    uname                   sys_newuname
@@ -167,24 +167,24 @@
 158     i386    sched_yield             sys_sched_yield
 159     i386    sched_get_priority_max  sys_sched_get_priority_max
 160     i386    sched_get_priority_min  sys_sched_get_priority_min
-161     i386    sched_rr_get_interval   sys_sched_rr_get_interval       sys32_sched_rr_get_interval
+161     i386    sched_rr_get_interval   sys_sched_rr_get_interval       compat_sys_sched_rr_get_interval
 162     i386    nanosleep               sys_nanosleep                   compat_sys_nanosleep
 163     i386    mremap                  sys_mremap
 164     i386    setresuid               sys_setresuid16
 165     i386    getresuid               sys_getresuid16
-166     i386    vm86                    ptregs_vm86                     sys32_vm86_warning
+166     i386    vm86                    sys_vm86                        sys32_vm86_warning
 167     i386    query_module
 168     i386    poll                    sys_poll
 169     i386    nfsservctl
 170     i386    setresgid               sys_setresgid16
 171     i386    getresgid               sys_getresgid16
 172     i386    prctl                   sys_prctl
-173     i386    rt_sigreturn            ptregs_rt_sigreturn             stub32_rt_sigreturn
-174     i386    rt_sigaction            sys_rt_sigaction                sys32_rt_sigaction
+173     i386    rt_sigreturn            sys_rt_sigreturn                stub32_rt_sigreturn
+174     i386    rt_sigaction            sys_rt_sigaction                compat_sys_rt_sigaction
 175     i386    rt_sigprocmask          sys_rt_sigprocmask
-176     i386    rt_sigpending           sys_rt_sigpending               sys32_rt_sigpending
+176     i386    rt_sigpending           sys_rt_sigpending               compat_sys_rt_sigpending
 177     i386    rt_sigtimedwait         sys_rt_sigtimedwait             compat_sys_rt_sigtimedwait
-178     i386    rt_sigqueueinfo         sys_rt_sigqueueinfo             sys32_rt_sigqueueinfo
+178     i386    rt_sigqueueinfo         sys_rt_sigqueueinfo             compat_sys_rt_sigqueueinfo
 179     i386    rt_sigsuspend           sys_rt_sigsuspend
 180     i386    pread64                 sys_pread64                     sys32_pread
 181     i386    pwrite64                sys_pwrite64                    sys32_pwrite
diff --git a/arch/x86/syscalls/syscall_64.tbl b/arch/x86/syscalls/syscall_64.tbl
index dc97328bd90a..38ae65dfd14f 100644
--- a/arch/x86/syscalls/syscall_64.tbl
+++ b/arch/x86/syscalls/syscall_64.tbl
@@ -325,7 +325,7 @@
 # x32-specific system call numbers start at 512 to avoid cache impact
 # for native 64-bit operation.
 #
-512     x32     rt_sigaction            sys32_rt_sigaction
+512     x32     rt_sigaction            compat_sys_rt_sigaction
 513     x32     rt_sigreturn            stub_x32_rt_sigreturn
 514     x32     ioctl                   compat_sys_ioctl
 515     x32     readv                   compat_sys_readv
@@ -335,9 +335,9 @@
 519     x32     recvmsg                 compat_sys_recvmsg
 520     x32     execve                  stub_x32_execve
 521     x32     ptrace                  compat_sys_ptrace
-522     x32     rt_sigpending           sys32_rt_sigpending
+522     x32     rt_sigpending           compat_sys_rt_sigpending
 523     x32     rt_sigtimedwait         compat_sys_rt_sigtimedwait
-524     x32     rt_sigqueueinfo         sys32_rt_sigqueueinfo
+524     x32     rt_sigqueueinfo         compat_sys_rt_sigqueueinfo
 525     x32     sigaltstack             compat_sys_sigaltstack
 526     x32     timer_create            compat_sys_timer_create
 527     x32     mq_notify               compat_sys_mq_notify
diff --git a/arch/x86/tools/insn_sanity.c b/arch/x86/tools/insn_sanity.c
index cc2f8c131286..872eb60e7806 100644
--- a/arch/x86/tools/insn_sanity.c
+++ b/arch/x86/tools/insn_sanity.c
@@ -55,7 +55,7 @@ static FILE		*input_file;	/* Input file name */
 static void usage(const char *err)
 {
         if (err)
-                fprintf(stderr, "Error: %s\n\n", err);
+                fprintf(stderr, "%s: Error: %s\n\n", prog, err);
         fprintf(stderr, "Usage: %s [-y|-n|-v] [-s seed[,no]] [-m max] [-i input]\n", prog);
         fprintf(stderr, "\t-y   64bit mode\n");
         fprintf(stderr, "\t-n   32bit mode\n");
@@ -269,7 +269,13 @@ int main(int argc, char **argv)
                 insns++;
         }
 
-        fprintf(stdout, "%s: decoded and checked %d %s instructions with %d errors (seed:0x%x)\n", (errors) ? "Failure" : "Success", insns, (input_file) ? "given" : "random", errors, seed);
+        fprintf(stdout, "%s: %s: decoded and checked %d %s instructions with %d errors (seed:0x%x)\n",
+                prog,
+                (errors) ? "Failure" : "Success",
+                insns,
+                (input_file) ? "given" : "random",
+                errors,
+                seed);
 
         return errors ? 1 : 0;
 }
diff --git a/arch/x86/um/Kconfig b/arch/x86/um/Kconfig
index 53c90fd412d1..14ef8d1dbc33 100644
--- a/arch/x86/um/Kconfig
+++ b/arch/x86/um/Kconfig
@@ -13,7 +13,6 @@ endmenu
 config UML_X86
         def_bool y
         select GENERIC_FIND_FIRST_BIT
-        select GENERIC_SIGALTSTACK
 
 config 64BIT
         bool "64-bit kernel" if SUBARCH = "x86"
@@ -25,6 +24,8 @@ config X86_32
         select ARCH_WANT_IPC_PARSE_VERSION
         select MODULES_USE_ELF_REL
         select CLONE_BACKWARDS
+        select OLD_SIGSUSPEND3
+        select OLD_SIGACTION
 
 config X86_64
         def_bool 64BIT
@@ -37,9 +38,8 @@ config RWSEM_GENERIC_SPINLOCK
         def_bool !RWSEM_XCHGADD_ALGORITHM
 
 config 3_LEVEL_PGTABLES
-        bool "Three-level pagetables (EXPERIMENTAL)" if !64BIT
+        bool "Three-level pagetables" if !64BIT
         default 64BIT
-        depends on EXPERIMENTAL
         help
         Three-level pagetables will let UML have more than 4G of physical
         memory.  All the memory that can't be mapped directly will be treated
diff --git a/arch/x86/um/Makefile b/arch/x86/um/Makefile
index 5d065b2222d3..eafa324eb7a5 100644
--- a/arch/x86/um/Makefile
+++ b/arch/x86/um/Makefile
@@ -10,7 +10,7 @@ endif
 
 obj-y = bug.o bugs_$(BITS).o delay.o fault.o ksyms.o ldt.o \
         ptrace_$(BITS).o ptrace_user.o setjmp_$(BITS).o signal.o \
-        stub_$(BITS).o stub_segv.o syscalls_$(BITS).o \
+        stub_$(BITS).o stub_segv.o \
         sys_call_table_$(BITS).o sysrq_$(BITS).o tls_$(BITS).o \
         mem_$(BITS).o subarch.o os-$(OS)/
 
@@ -25,7 +25,7 @@ subarch-$(CONFIG_HIGHMEM) += ../mm/highmem_32.o
 
 else
 
-obj-y += vdso/
+obj-y += syscalls_64.o vdso/
 
 subarch-y = ../lib/csum-partial_64.o ../lib/memcpy_64.o ../lib/thunk_64.o \
                 ../lib/rwsem.o
diff --git a/arch/x86/um/fault.c b/arch/x86/um/fault.c
index 8784ab30d91b..84ac7f7b0257 100644
--- a/arch/x86/um/fault.c
+++ b/arch/x86/um/fault.c
@@ -20,7 +20,7 @@ int arch_fixup(unsigned long address, struct uml_pt_regs *regs)
         const struct exception_table_entry *fixup;
 
         fixup = search_exception_tables(address);
-        if (fixup != 0) {
+        if (fixup) {
                 UPT_IP(regs) = fixup->fixup;
                 return 1;
         }
diff --git a/arch/x86/um/shared/sysdep/syscalls_32.h b/arch/x86/um/shared/sysdep/syscalls_32.h
index 8436079be914..68fd2cf526fd 100644
--- a/arch/x86/um/shared/sysdep/syscalls_32.h
+++ b/arch/x86/um/shared/sysdep/syscalls_32.h
@@ -8,11 +8,6 @@
 
 typedef long syscall_handler_t(struct pt_regs);
 
-/* Not declared on x86, incompatible declarations on x86_64, so these have
- * to go here rather than in sys_call_table.c
- */
-extern syscall_handler_t sys_rt_sigaction;
-
 extern syscall_handler_t *sys_call_table[];
 
 #define EXECUTE_SYSCALL(syscall, regs) \
diff --git a/arch/x86/um/signal.c b/arch/x86/um/signal.c
index 71cef48ea5cd..ae7319db18ee 100644
--- a/arch/x86/um/signal.c
+++ b/arch/x86/um/signal.c
@@ -464,7 +464,7 @@ int setup_signal_stack_si(unsigned long stack_top, int sig,
         return 0;
 }
 
-long sys_sigreturn(struct pt_regs *regs)
+long sys_sigreturn(void)
 {
         unsigned long sp = PT_REGS_SP(&current->thread.regs);
         struct sigframe __user *frame = (struct sigframe __user *)(sp - 8);
@@ -577,7 +577,7 @@ int setup_signal_stack_si(unsigned long stack_top, int sig,
 }
 #endif
 
-long sys_rt_sigreturn(struct pt_regs *regs)
+long sys_rt_sigreturn(void)
 {
         unsigned long sp = PT_REGS_SP(&current->thread.regs);
         struct rt_sigframe __user *frame =
@@ -601,14 +601,3 @@ long sys_rt_sigreturn(struct pt_regs *regs)
         force_sig(SIGSEGV, current);
         return 0;
 }
-
-#ifdef CONFIG_X86_32
-long ptregs_sigreturn(void)
-{
-        return sys_sigreturn(NULL);
-}
-long ptregs_rt_sigreturn(void)
-{
-        return sys_rt_sigreturn(NULL);
-}
-#endif
diff --git a/arch/x86/um/sys_call_table_32.c b/arch/x86/um/sys_call_table_32.c
index a0c3b0d1a122..531d4269e2e3 100644
--- a/arch/x86/um/sys_call_table_32.c
+++ b/arch/x86/um/sys_call_table_32.c
@@ -24,10 +24,6 @@
 
 #define old_mmap sys_old_mmap
 
-#define ptregs_iopl sys_iopl
-#define ptregs_vm86old sys_vm86old
-#define ptregs_vm86 sys_vm86
-
 #define __SYSCALL_I386(nr, sym, compat) extern asmlinkage void sym(void) ;
 #include <asm/syscalls_32.h>
 
diff --git a/arch/x86/um/syscalls_32.c b/arch/x86/um/syscalls_32.c
deleted file mode 100644
index e8bcea99acdb..000000000000
--- a/arch/x86/um/syscalls_32.c
+++ /dev/null
@@ -1,38 +0,0 @@
-/* 
- * Copyright (C) 2000 - 2003 Jeff Dike (jdike@addtoit.com)
- * Licensed under the GPL
- */
-
-#include <linux/syscalls.h>
-#include <sysdep/syscalls.h>
-
-long sys_sigaction(int sig, const struct old_sigaction __user *act,
-                         struct old_sigaction __user *oact)
-{
-        struct k_sigaction new_ka, old_ka;
-        int ret;
-
-        if (act) {
-                old_sigset_t mask;
-                if (!access_ok(VERIFY_READ, act, sizeof(*act)) ||
-                    __get_user(new_ka.sa.sa_handler, &act->sa_handler) ||
-                    __get_user(new_ka.sa.sa_restorer, &act->sa_restorer) ||
-                    __get_user(new_ka.sa.sa_flags, &act->sa_flags) ||
-                    __get_user(mask, &act->sa_mask))
-                        return -EFAULT;
-                siginitset(&new_ka.sa.sa_mask, mask);
-        }
-
-        ret = do_sigaction(sig, act ? &new_ka : NULL, oact ? &old_ka : NULL);
-
-        if (!ret && oact) {
-                if (!access_ok(VERIFY_WRITE, oact, sizeof(*oact)) ||
-                    __put_user(old_ka.sa.sa_handler, &oact->sa_handler) ||
-                    __put_user(old_ka.sa.sa_restorer, &oact->sa_restorer) ||
-                    __put_user(old_ka.sa.sa_flags, &oact->sa_flags) ||
-                    __put_user(old_ka.sa.sa_mask.sig[0], &oact->sa_mask))
-                        return -EFAULT;
-        }
-
-        return ret;
-}
diff --git a/arch/x86/vdso/vclock_gettime.c b/arch/x86/vdso/vclock_gettime.c
index 205ad328aa52..c74436e687bf 100644
--- a/arch/x86/vdso/vclock_gettime.c
+++ b/arch/x86/vdso/vclock_gettime.c
@@ -60,7 +60,7 @@ notrace static cycle_t vread_tsc(void)
 
 static notrace cycle_t vread_hpet(void)
 {
-        return readl((const void __iomem *)fix_to_virt(VSYSCALL_HPET) + 0xf0);
+        return readl((const void __iomem *)fix_to_virt(VSYSCALL_HPET) + HPET_COUNTER);
 }
 
 #ifdef CONFIG_PARAVIRT_CLOCK
diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
index 138e5667409a..c8e1c7b95c3b 100644
--- a/arch/x86/xen/enlighten.c
+++ b/arch/x86/xen/enlighten.c
@@ -67,6 +67,7 @@
 #include <asm/hypervisor.h>
 #include <asm/mwait.h>
 #include <asm/pci_x86.h>
+#include <asm/pat.h>
 
 #ifdef CONFIG_ACPI
 #include <linux/acpi.h>
@@ -1417,7 +1418,14 @@ asmlinkage void __init xen_start_kernel(void)
          */
         acpi_numa = -1;
 #endif
-
+#ifdef CONFIG_X86_PAT
+        /*
+         * For right now disable the PAT. We should remove this once
+         * git commit 8eaffa67b43e99ae581622c5133e20b0f48bcef1
+         * (xen/pat: Disable PAT support for now) is reverted.
+         */
+        pat_enabled = 0;
+#endif
         /* Don't do the full vcpu_info placement stuff until we have a
            possible map and a non-dummy shared_info. */
         per_cpu(xen_vcpu, 0) = &HYPERVISOR_shared_info->vcpu_info[0];
@@ -1517,72 +1525,51 @@ asmlinkage void __init xen_start_kernel(void)
 #endif
 }
 
-#ifdef CONFIG_XEN_PVHVM
-#define HVM_SHARED_INFO_ADDR 0xFE700000UL
-static struct shared_info *xen_hvm_shared_info;
-static unsigned long xen_hvm_sip_phys;
-static int xen_major, xen_minor;
-
-static void xen_hvm_connect_shared_info(unsigned long pfn)
+void __ref xen_hvm_init_shared_info(void)
 {
+        int cpu;
         struct xen_add_to_physmap xatp;
+        static struct shared_info *shared_info_page = 0;
 
+        if (!shared_info_page)
+                shared_info_page = (struct shared_info *)
+                        extend_brk(PAGE_SIZE, PAGE_SIZE);
         xatp.domid = DOMID_SELF;
         xatp.idx = 0;
         xatp.space = XENMAPSPACE_shared_info;
-        xatp.gpfn = pfn;
+        xatp.gpfn = __pa(shared_info_page) >> PAGE_SHIFT;
         if (HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp))
                 BUG();
 
-}
-static void __init xen_hvm_set_shared_info(struct shared_info *sip)
-{
-        int cpu;
-
-        HYPERVISOR_shared_info = sip;
+        HYPERVISOR_shared_info = (struct shared_info *)shared_info_page;
 
         /* xen_vcpu is a pointer to the vcpu_info struct in the shared_info
          * page, we use it in the event channel upcall and in some pvclock
          * related functions. We don't need the vcpu_info placement
          * optimizations because we don't use any pv_mmu or pv_irq op on
-         * HVM. */
-        for_each_online_cpu(cpu)
+         * HVM.
+         * When xen_hvm_init_shared_info is run at boot time only vcpu 0 is
+         * online but xen_hvm_init_shared_info is run at resume time too and
+         * in that case multiple vcpus might be online. */
+        for_each_online_cpu(cpu) {
                 per_cpu(xen_vcpu, cpu) = &HYPERVISOR_shared_info->vcpu_info[cpu];
-}
-
-/* Reconnect the shared_info pfn to a (new) mfn */
-void xen_hvm_resume_shared_info(void)
-{
-        xen_hvm_connect_shared_info(xen_hvm_sip_phys >> PAGE_SHIFT);
-}
-
-/* Xen tools prior to Xen 4 do not provide a E820_Reserved area for guest usage.
- * On these old tools the shared info page will be placed in E820_Ram.
- * Xen 4 provides a E820_Reserved area at 0xFC000000, and this code expects
- * that nothing is mapped up to HVM_SHARED_INFO_ADDR.
- * Xen 4.3+ provides an explicit 1MB area at HVM_SHARED_INFO_ADDR which is used
- * here for the shared info page. */
-static void __init xen_hvm_init_shared_info(void)
-{
-        if (xen_major < 4) {
-                xen_hvm_shared_info = extend_brk(PAGE_SIZE, PAGE_SIZE);
-                xen_hvm_sip_phys = __pa(xen_hvm_shared_info);
-        } else {
-                xen_hvm_sip_phys = HVM_SHARED_INFO_ADDR;
-                set_fixmap(FIX_PARAVIRT_BOOTMAP, xen_hvm_sip_phys);
-                xen_hvm_shared_info =
-                (struct shared_info *)fix_to_virt(FIX_PARAVIRT_BOOTMAP);
         }
-        xen_hvm_connect_shared_info(xen_hvm_sip_phys >> PAGE_SHIFT);
-        xen_hvm_set_shared_info(xen_hvm_shared_info);
 }
 
+#ifdef CONFIG_XEN_PVHVM
 static void __init init_hvm_pv_info(void)
 {
-        uint32_t ecx, edx, pages, msr, base;
+        int major, minor;
+        uint32_t eax, ebx, ecx, edx, pages, msr, base;
         u64 pfn;
 
         base = xen_cpuid_base();
+        cpuid(base + 1, &eax, &ebx, &ecx, &edx);
+
+        major = eax >> 16;
+        minor = eax & 0xffff;
+        printk(KERN_INFO "Xen version %d.%d.\n", major, minor);
+
         cpuid(base + 2, &pages, &msr, &ecx, &edx);
 
         pfn = __pa(hypercall_page);
@@ -1633,22 +1620,12 @@ static void __init xen_hvm_guest_init(void)
 
 static bool __init xen_hvm_platform(void)
 {
-        uint32_t eax, ebx, ecx, edx, base;
-
         if (xen_pv_domain())
                 return false;
 
-        base = xen_cpuid_base();
-        if (!base)
+        if (!xen_cpuid_base())
                 return false;
 
-        cpuid(base + 1, &eax, &ebx, &ecx, &edx);
-
-        xen_major = eax >> 16;
-        xen_minor = eax & 0xffff;
-
-        printk(KERN_INFO "Xen version %d.%d.\n", xen_major, xen_minor);
-
         return true;
 }
 
@@ -1668,6 +1645,7 @@ const struct hypervisor_x86 x86_hyper_xen_hvm __refconst = {
         .name                   = "Xen HVM",
         .detect                 = xen_hvm_platform,
         .init_platform          = xen_hvm_guest_init,
+        .x2apic_available       = xen_x2apic_para_available,
 };
 EXPORT_SYMBOL(x86_hyper_xen_hvm);
 #endif
diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
index 01de35c77221..6afbb2ca9a0a 100644
--- a/arch/x86/xen/mmu.c
+++ b/arch/x86/xen/mmu.c
@@ -1178,20 +1178,6 @@ static void xen_exit_mmap(struct mm_struct *mm)
 
 static void xen_post_allocator_init(void);
 
-static __init void xen_mapping_pagetable_reserve(u64 start, u64 end)
-{
-        /* reserve the range used */
-        native_pagetable_reserve(start, end);
-
-        /* set as RW the rest */
-        printk(KERN_DEBUG "xen: setting RW the range %llx - %llx\n", end,
-                        PFN_PHYS(pgt_buf_top));
-        while (end < PFN_PHYS(pgt_buf_top)) {
-                make_lowmem_page_readwrite(__va(end));
-                end += PAGE_SIZE;
-        }
-}
-
 #ifdef CONFIG_X86_64
 static void __init xen_cleanhighmap(unsigned long vaddr,
                                     unsigned long vaddr_end)
@@ -1422,7 +1408,6 @@ static void __xen_write_cr3(bool kernel, unsigned long cr3)
                 xen_mc_callback(set_current_cr3, (void *)cr3);
         }
 }
-
 static void xen_write_cr3(unsigned long cr3)
 {
         BUG_ON(preemptible());
@@ -1448,6 +1433,43 @@ static void xen_write_cr3(unsigned long cr3)
         xen_mc_issue(PARAVIRT_LAZY_CPU);  /* interrupts restored */
 }
 
+#ifdef CONFIG_X86_64
+/*
+ * At the start of the day - when Xen launches a guest, it has already
+ * built pagetables for the guest. We diligently look over them
+ * in xen_setup_kernel_pagetable and graft as appropiate them in the
+ * init_level4_pgt and its friends. Then when we are happy we load
+ * the new init_level4_pgt - and continue on.
+ *
+ * The generic code starts (start_kernel) and 'init_mem_mapping' sets
+ * up the rest of the pagetables. When it has completed it loads the cr3.
+ * N.B. that baremetal would start at 'start_kernel' (and the early
+ * #PF handler would create bootstrap pagetables) - so we are running
+ * with the same assumptions as what to do when write_cr3 is executed
+ * at this point.
+ *
+ * Since there are no user-page tables at all, we have two variants
+ * of xen_write_cr3 - the early bootup (this one), and the late one
+ * (xen_write_cr3). The reason we have to do that is that in 64-bit
+ * the Linux kernel and user-space are both in ring 3 while the
+ * hypervisor is in ring 0.
+ */
+static void __init xen_write_cr3_init(unsigned long cr3)
+{
+        BUG_ON(preemptible());
+
+        xen_mc_batch();  /* disables interrupts */
+
+        /* Update while interrupts are disabled, so its atomic with
+           respect to ipis */
+        this_cpu_write(xen_cr3, cr3);
+
+        __xen_write_cr3(true, cr3);
+
+        xen_mc_issue(PARAVIRT_LAZY_CPU);  /* interrupts restored */
+}
+#endif
+
 static int xen_pgd_alloc(struct mm_struct *mm)
 {
         pgd_t *pgd = mm->pgd;
@@ -1503,19 +1525,6 @@ static pte_t __init mask_rw_pte(pte_t *ptep, pte_t pte)
 #else /* CONFIG_X86_64 */
 static pte_t __init mask_rw_pte(pte_t *ptep, pte_t pte)
 {
-        unsigned long pfn = pte_pfn(pte);
-
-        /*
-         * If the new pfn is within the range of the newly allocated
-         * kernel pagetable, and it isn't being mapped into an
-         * early_ioremap fixmap slot as a freshly allocated page, make sure
-         * it is RO.
-         */
-        if (((!is_early_ioremap_ptep(ptep) &&
-                        pfn >= pgt_buf_start && pfn < pgt_buf_top)) ||
-                        (is_early_ioremap_ptep(ptep) && pfn != (pgt_buf_end - 1)))
-                pte = pte_wrprotect(pte);
-
         return pte;
 }
 #endif /* CONFIG_X86_64 */
@@ -2111,6 +2120,7 @@ static void __init xen_post_allocator_init(void)
 #endif
 
 #ifdef CONFIG_X86_64
+        pv_mmu_ops.write_cr3 = &xen_write_cr3;
         SetPagePinned(virt_to_page(level3_user_vsyscall));
 #endif
         xen_mark_init_mm_pinned();
@@ -2129,11 +2139,7 @@ static const struct pv_mmu_ops xen_mmu_ops __initconst = {
         .write_cr2 = xen_write_cr2,
 
         .read_cr3 = xen_read_cr3,
-#ifdef CONFIG_X86_32
         .write_cr3 = xen_write_cr3_init,
-#else
-        .write_cr3 = xen_write_cr3,
-#endif
 
         .flush_tlb_user = xen_flush_tlb,
         .flush_tlb_kernel = xen_flush_tlb,
@@ -2197,7 +2203,6 @@ static const struct pv_mmu_ops xen_mmu_ops __initconst = {
 
 void __init xen_init_mmu_ops(void)
 {
-        x86_init.mapping.pagetable_reserve = xen_mapping_pagetable_reserve;
         x86_init.paging.pagetable_init = xen_pagetable_init;
         pv_mmu_ops = xen_mmu_ops;
 
diff --git a/arch/x86/xen/setup.c b/arch/x86/xen/setup.c
index 8971a26d21ab..94eac5c85cdc 100644
--- a/arch/x86/xen/setup.c
+++ b/arch/x86/xen/setup.c
@@ -556,12 +556,9 @@ void __init xen_arch_setup(void)
                COMMAND_LINE_SIZE : MAX_GUEST_CMDLINE);
 
         /* Set up idle, making sure it calls safe_halt() pvop */
-#ifdef CONFIG_X86_32
-        boot_cpu_data.hlt_works_ok = 1;
-#endif
         disable_cpuidle();
         disable_cpufreq();
-        WARN_ON(set_pm_idle_to_default());
+        WARN_ON(xen_set_default_idle());
         fiddle_vdso();
 #ifdef CONFIG_NUMA
         numa_off = 1;
diff --git a/arch/x86/xen/smp.c b/arch/x86/xen/smp.c
index 34bc4cee8887..09ea61d2e02f 100644
--- a/arch/x86/xen/smp.c
+++ b/arch/x86/xen/smp.c
@@ -300,8 +300,6 @@ cpu_initialize_context(unsigned int cpu, struct task_struct *idle)
         gdt = get_cpu_gdt_table(cpu);
 
         ctxt->flags = VGCF_IN_KERNEL;
-        ctxt->user_regs.ds = __USER_DS;
-        ctxt->user_regs.es = __USER_DS;
         ctxt->user_regs.ss = __KERNEL_DS;
 #ifdef CONFIG_X86_32
         ctxt->user_regs.fs = __KERNEL_PERCPU;
@@ -310,35 +308,41 @@ cpu_initialize_context(unsigned int cpu, struct task_struct *idle)
         ctxt->gs_base_kernel = per_cpu_offset(cpu);
 #endif
         ctxt->user_regs.eip = (unsigned long)cpu_bringup_and_idle;
-        ctxt->user_regs.eflags = 0x1000; /* IOPL_RING1 */
 
         memset(&ctxt->fpu_ctxt, 0, sizeof(ctxt->fpu_ctxt));
 
-        xen_copy_trap_info(ctxt->trap_ctxt);
+        {
+                ctxt->user_regs.eflags = 0x1000; /* IOPL_RING1 */
+                ctxt->user_regs.ds = __USER_DS;
+                ctxt->user_regs.es = __USER_DS;
 
-        ctxt->ldt_ents = 0;
+                xen_copy_trap_info(ctxt->trap_ctxt);
 
-        BUG_ON((unsigned long)gdt & ~PAGE_MASK);
+                ctxt->ldt_ents = 0;
 
-        gdt_mfn = arbitrary_virt_to_mfn(gdt);
-        make_lowmem_page_readonly(gdt);
-        make_lowmem_page_readonly(mfn_to_virt(gdt_mfn));
+                BUG_ON((unsigned long)gdt & ~PAGE_MASK);
 
-        ctxt->gdt_frames[0] = gdt_mfn;
-        ctxt->gdt_ents      = GDT_ENTRIES;
+                gdt_mfn = arbitrary_virt_to_mfn(gdt);
+                make_lowmem_page_readonly(gdt);
+                make_lowmem_page_readonly(mfn_to_virt(gdt_mfn));
 
-        ctxt->user_regs.cs = __KERNEL_CS;
-        ctxt->user_regs.esp = idle->thread.sp0 - sizeof(struct pt_regs);
+                ctxt->gdt_frames[0] = gdt_mfn;
+                ctxt->gdt_ents      = GDT_ENTRIES;
 
-        ctxt->kernel_ss = __KERNEL_DS;
-        ctxt->kernel_sp = idle->thread.sp0;
+                ctxt->kernel_ss = __KERNEL_DS;
+                ctxt->kernel_sp = idle->thread.sp0;
 
 #ifdef CONFIG_X86_32
-        ctxt->event_callback_cs     = __KERNEL_CS;
-        ctxt->failsafe_callback_cs  = __KERNEL_CS;
+                ctxt->event_callback_cs     = __KERNEL_CS;
+                ctxt->failsafe_callback_cs  = __KERNEL_CS;
 #endif
-        ctxt->event_callback_eip    = (unsigned long)xen_hypervisor_callback;
-        ctxt->failsafe_callback_eip = (unsigned long)xen_failsafe_callback;
+                ctxt->event_callback_eip    =
+                                        (unsigned long)xen_hypervisor_callback;
+                ctxt->failsafe_callback_eip =
+                                        (unsigned long)xen_failsafe_callback;
+        }
+        ctxt->user_regs.cs = __KERNEL_CS;
+        ctxt->user_regs.esp = idle->thread.sp0 - sizeof(struct pt_regs);
 
         per_cpu(xen_cr3, cpu) = __pa(swapper_pg_dir);
         ctxt->ctrlreg[3] = xen_pfn_to_cr3(virt_to_mfn(swapper_pg_dir));
diff --git a/arch/x86/xen/spinlock.c b/arch/x86/xen/spinlock.c
index 83e866d714ce..f7a080ef0354 100644
--- a/arch/x86/xen/spinlock.c
+++ b/arch/x86/xen/spinlock.c
@@ -328,7 +328,6 @@ static noinline void xen_spin_unlock_slow(struct xen_spinlock *xl)
                 if (per_cpu(lock_spinners, cpu) == xl) {
                         ADD_STATS(released_slow_kicked, 1);
                         xen_send_IPI_one(cpu, XEN_SPIN_UNLOCK_VECTOR);
-                        break;
                 }
         }
 }
diff --git a/arch/x86/xen/suspend.c b/arch/x86/xen/suspend.c
index ae8a00c39de4..45329c8c226e 100644
--- a/arch/x86/xen/suspend.c
+++ b/arch/x86/xen/suspend.c
@@ -30,7 +30,7 @@ void xen_arch_hvm_post_suspend(int suspend_cancelled)
 {
 #ifdef CONFIG_XEN_PVHVM
         int cpu;
-        xen_hvm_resume_shared_info();
+        xen_hvm_init_shared_info();
         xen_callback_vector();
         xen_unplug_emulated_devices();
         if (xen_feature(XENFEAT_hvm_safe_pvclock)) {
diff --git a/arch/x86/xen/xen-asm_32.S b/arch/x86/xen/xen-asm_32.S
index f9643fc50de5..33ca6e42a4ca 100644
--- a/arch/x86/xen/xen-asm_32.S
+++ b/arch/x86/xen/xen-asm_32.S
@@ -89,11 +89,11 @@ ENTRY(xen_iret)
          */
 #ifdef CONFIG_SMP
         GET_THREAD_INFO(%eax)
-        movl TI_cpu(%eax), %eax
-        movl __per_cpu_offset(,%eax,4), %eax
-        mov xen_vcpu(%eax), %eax
+        movl %ss:TI_cpu(%eax), %eax
+        movl %ss:__per_cpu_offset(,%eax,4), %eax
+        mov %ss:xen_vcpu(%eax), %eax
 #else
-        movl xen_vcpu, %eax
+        movl %ss:xen_vcpu, %eax
 #endif
 
         /* check IF state we're restoring */
@@ -106,11 +106,11 @@ ENTRY(xen_iret)
          * resuming the code, so we don't have to be worried about
          * being preempted to another CPU.
          */
-        setz XEN_vcpu_info_mask(%eax)
+        setz %ss:XEN_vcpu_info_mask(%eax)
 xen_iret_start_crit:
 
         /* check for unmasked and pending */
-        cmpw $0x0001, XEN_vcpu_info_pending(%eax)
+        cmpw $0x0001, %ss:XEN_vcpu_info_pending(%eax)
 
         /*
          * If there's something pending, mask events again so we can
@@ -118,7 +118,7 @@ xen_iret_start_crit:
          * touch XEN_vcpu_info_mask.
          */
         jne 1f
-        movb $1, XEN_vcpu_info_mask(%eax)
+        movb $1, %ss:XEN_vcpu_info_mask(%eax)
 
 1:      popl %eax
 
diff --git a/arch/x86/xen/xen-ops.h b/arch/x86/xen/xen-ops.h
index d2e73d19d366..a95b41744ad0 100644
--- a/arch/x86/xen/xen-ops.h
+++ b/arch/x86/xen/xen-ops.h
@@ -40,7 +40,7 @@ void xen_enable_syscall(void);
 void xen_vcpu_restore(void);
 
 void xen_callback_vector(void);
-void xen_hvm_resume_shared_info(void);
+void xen_hvm_init_shared_info(void);
 void xen_unplug_emulated_devices(void);
 
 void __init xen_build_dynamic_phys_to_machine(void);