284 files changed, 10826 insertions, 8836 deletions

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 13ffa5df37d7..93698794aa3a 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -24,7 +24,7 @@ config X86
         select HAVE_UNSTABLE_SCHED_CLOCK
         select HAVE_IDE
         select HAVE_OPROFILE
-        select HAVE_PERF_COUNTERS if (!M386 && !M486)
+        select HAVE_PERF_EVENTS if (!M386 && !M486)
         select HAVE_IOREMAP_PROT
         select HAVE_KPROBES
         select ARCH_WANT_OPTIONAL_GPIOLIB
@@ -38,7 +38,7 @@ config X86
         select HAVE_FUNCTION_GRAPH_FP_TEST
         select HAVE_FUNCTION_TRACE_MCOUNT_TEST
         select HAVE_FTRACE_NMI_ENTER if DYNAMIC_FTRACE
-        select HAVE_FTRACE_SYSCALLS
+        select HAVE_SYSCALL_TRACEPOINTS
         select HAVE_KVM
         select HAVE_ARCH_KGDB
         select HAVE_ARCH_TRACEHOOK
@@ -150,7 +150,10 @@ config ARCH_HAS_CACHE_LINE_SIZE
 config HAVE_SETUP_PER_CPU_AREA
         def_bool y
 
-config HAVE_DYNAMIC_PER_CPU_AREA
+config NEED_PER_CPU_EMBED_FIRST_CHUNK
+        def_bool y
+
+config NEED_PER_CPU_PAGE_FIRST_CHUNK
         def_bool y
 
 config HAVE_CPUMASK_OF_CPU_MAP
@@ -179,6 +182,10 @@ config ARCH_SUPPORTS_OPTIMIZED_INLINING
 config ARCH_SUPPORTS_DEBUG_PAGEALLOC
         def_bool y
 
+config HAVE_INTEL_TXT
+        def_bool y
+        depends on EXPERIMENTAL && DMAR && ACPI
+
 # Use the generic interrupt handling code in kernel/irq/:
 config GENERIC_HARDIRQS
         bool
@@ -318,6 +325,7 @@ config X86_EXTENDED_PLATFORM
                 SGI 320/540 (Visual Workstation)
                 Summit/EXA (IBM x440)
                 Unisys ES7000 IA32 series
+                Moorestown MID devices
 
           If you have one of these systems, or if you want to build a
           generic distribution kernel, say Y here - otherwise say N.
@@ -377,6 +385,18 @@ config X86_ELAN
 
           If unsure, choose "PC-compatible" instead.
 
+config X86_MRST
+       bool "Moorestown MID platform"
+        depends on X86_32
+        depends on X86_EXTENDED_PLATFORM
+        ---help---
+          Moorestown is Intel's Low Power Intel Architecture (LPIA) based Moblin
+          Internet Device(MID) platform. Moorestown consists of two chips:
+          Lincroft (CPU core, graphics, and memory controller) and Langwell IOH.
+          Unlike standard x86 PCs, Moorestown does not have many legacy devices
+          nor standard legacy replacement devices/features. e.g. Moorestown does
+          not contain i8259, i8254, HPET, legacy BIOS, most of the io ports.
+
 config X86_RDC321X
         bool "RDC R-321x SoC"
         depends on X86_32
@@ -586,7 +606,6 @@ config GART_IOMMU
         bool "GART IOMMU support" if EMBEDDED
         default y
         select SWIOTLB
-        select AGP
         depends on X86_64 && PCI
         ---help---
           Support for full DMA access of devices with 32bit memory access only
@@ -777,41 +796,17 @@ config X86_REROUTE_FOR_BROKEN_BOOT_IRQS
           increased on these systems.
 
 config X86_MCE
-        bool "Machine Check Exception"
+        bool "Machine Check / overheating reporting"
         ---help---
-          Machine Check Exception support allows the processor to notify the
-          kernel if it detects a problem (e.g. overheating, component failure).
+          Machine Check support allows the processor to notify the
+          kernel if it detects a problem (e.g. overheating, data corruption).
           The action the kernel takes depends on the severity of the problem,
-          ranging from a warning message on the console, to halting the machine.
-          Your processor must be a Pentium or newer to support this - check the
-          flags in /proc/cpuinfo for mce.  Note that some older Pentium systems
-          have a design flaw which leads to false MCE events - hence MCE is
-          disabled on all P5 processors, unless explicitly enabled with "mce"
-          as a boot argument.  Similarly, if MCE is built in and creates a
-          problem on some new non-standard machine, you can boot with "nomce"
-          to disable it.  MCE support simply ignores non-MCE processors like
-          the 386 and 486, so nearly everyone can say Y here.
-
-config X86_OLD_MCE
-        depends on X86_32 && X86_MCE
-        bool "Use legacy machine check code (will go away)"
-        default n
-        select X86_ANCIENT_MCE
-        ---help---
-          Use the old i386 machine check code. This is merely intended for
-          testing in a transition period. Try this if you run into any machine
-          check related software problems, but report the problem to
-          linux-kernel.  When in doubt say no.
-
-config X86_NEW_MCE
-        depends on X86_MCE
-        bool
-        default y if (!X86_OLD_MCE && X86_32) || X86_64
+          ranging from warning messages to halting the machine.
 
 config X86_MCE_INTEL
         def_bool y
         prompt "Intel MCE features"
-        depends on X86_NEW_MCE && X86_LOCAL_APIC
+        depends on X86_MCE && X86_LOCAL_APIC
         ---help---
            Additional support for intel specific MCE features such as
            the thermal monitor.
@@ -819,14 +814,14 @@ config X86_MCE_INTEL
 config X86_MCE_AMD
         def_bool y
         prompt "AMD MCE features"
-        depends on X86_NEW_MCE && X86_LOCAL_APIC
+        depends on X86_MCE && X86_LOCAL_APIC
         ---help---
            Additional support for AMD specific MCE features such as
            the DRAM Error Threshold.
 
 config X86_ANCIENT_MCE
         def_bool n
-        depends on X86_32
+        depends on X86_32 && X86_MCE
         prompt "Support for old Pentium 5 / WinChip machine checks"
         ---help---
           Include support for machine check handling on old Pentium 5 or WinChip
@@ -839,36 +834,16 @@ config X86_MCE_THRESHOLD
         default y
 
 config X86_MCE_INJECT
-        depends on X86_NEW_MCE
+        depends on X86_MCE
         tristate "Machine check injector support"
         ---help---
           Provide support for injecting machine checks for testing purposes.
           If you don't know what a machine check is and you don't do kernel
           QA it is safe to say n.
 
-config X86_MCE_NONFATAL
-        tristate "Check for non-fatal errors on AMD Athlon/Duron / Intel Pentium 4"
-        depends on X86_OLD_MCE
-        ---help---
-          Enabling this feature starts a timer that triggers every 5 seconds which
-          will look at the machine check registers to see if anything happened.
-          Non-fatal problems automatically get corrected (but still logged).
-          Disable this if you don't want to see these messages.
-          Seeing the messages this option prints out may be indicative of dying
-          or out-of-spec (ie, overclocked) hardware.
-          This option only does something on certain CPUs.
-          (AMD Athlon/Duron and Intel Pentium 4)
-
-config X86_MCE_P4THERMAL
-        bool "check for P4 thermal throttling interrupt."
-        depends on X86_OLD_MCE && X86_MCE && (X86_UP_APIC || SMP)
-        ---help---
-          Enabling this feature will cause a message to be printed when the P4
-          enters thermal throttling.
-
 config X86_THERMAL_VECTOR
         def_bool y
-        depends on X86_MCE_P4THERMAL || X86_MCE_INTEL
+        depends on X86_MCE_INTEL
 
 config VM86
         bool "Enable VM86 support" if EMBEDDED
@@ -1229,6 +1204,10 @@ config ARCH_DISCONTIGMEM_DEFAULT
         def_bool y
         depends on NUMA && X86_32
 
+config ARCH_PROC_KCORE_TEXT
+        def_bool y
+        depends on X86_64 && PROC_KCORE
+
 config ARCH_SPARSEMEM_DEFAULT
         def_bool y
         depends on X86_64
@@ -1414,6 +1393,10 @@ config X86_PAT
 
           If unsure, say Y.
 
+config ARCH_USES_PG_UNCACHED
+        def_bool y
+        depends on X86_PAT
+
 config EFI
         bool "EFI runtime service support"
         depends on ACPI
@@ -1683,6 +1666,8 @@ source "kernel/power/Kconfig"
 
 source "drivers/acpi/Kconfig"
 
+source "drivers/sfi/Kconfig"
+
 config X86_APM_BOOT
         bool
         default y
@@ -1878,7 +1863,7 @@ config PCI_DIRECT
 
 config PCI_MMCONFIG
         def_bool y
-        depends on X86_32 && PCI && ACPI && (PCI_GOMMCONFIG || PCI_GOANY)
+        depends on X86_32 && PCI && (ACPI || SFI) && (PCI_GOMMCONFIG || PCI_GOANY)
 
 config PCI_OLPC
         def_bool y
@@ -1916,7 +1901,7 @@ config DMAR_DEFAULT_ON
 config DMAR_BROKEN_GFX_WA
         def_bool n
         prompt "Workaround broken graphics drivers (going away soon)"
-        depends on DMAR
+        depends on DMAR && BROKEN
         ---help---
           Current Graphics drivers tend to use physical address
           for DMA and avoid using DMA APIs. Setting this config
diff --git a/arch/x86/Kconfig.cpu b/arch/x86/Kconfig.cpu
index 8130334329c0..527519b8a9f9 100644
--- a/arch/x86/Kconfig.cpu
+++ b/arch/x86/Kconfig.cpu
@@ -262,6 +262,15 @@ config MCORE2
           family in /proc/cpuinfo. Newer ones have 6 and older ones 15
           (not a typo)
 
+config MATOM
+        bool "Intel Atom"
+        ---help---
+
+          Select this for the Intel Atom platform. Intel Atom CPUs have an
+          in-order pipelining architecture and thus can benefit from
+          accordingly optimized code. Use a recent GCC with specific Atom
+          support in order to fully benefit from selecting this option.
+
 config GENERIC_CPU
         bool "Generic-x86-64"
         depends on X86_64
@@ -295,7 +304,7 @@ config X86_CPU
 config X86_L1_CACHE_BYTES
         int
         default "128" if MPSC
-        default "64" if GENERIC_CPU || MK8 || MCORE2 || X86_32
+        default "64" if GENERIC_CPU || MK8 || MCORE2 || MATOM || X86_32
 
 config X86_INTERNODE_CACHE_BYTES
         int
@@ -310,7 +319,7 @@ config X86_L1_CACHE_SHIFT
         default "7" if MPENTIUM4 || MPSC
         default "4" if X86_ELAN || M486 || M386 || MGEODEGX1
         default "5" if MWINCHIP3D || MWINCHIPC6 || MCRUSOE || MEFFICEON || MCYRIXIII || MK6 || MPENTIUMIII || MPENTIUMII || M686 || M586MMX || M586TSC || M586 || MVIAC3_2 || MGEODE_LX
-        default "6" if MK7 || MK8 || MPENTIUMM || MCORE2 || MVIAC7 || X86_GENERIC || GENERIC_CPU
+        default "6" if MK7 || MK8 || MPENTIUMM || MCORE2 || MATOM || MVIAC7 || X86_GENERIC || GENERIC_CPU
 
 config X86_XADD
         def_bool y
@@ -359,7 +368,7 @@ config X86_INTEL_USERCOPY
 
 config X86_USE_PPRO_CHECKSUM
         def_bool y
-        depends on MWINCHIP3D || MWINCHIPC6 || MCYRIXIII || MK7 || MK6 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MK8 || MVIAC3_2 || MEFFICEON || MGEODE_LX || MCORE2
+        depends on MWINCHIP3D || MWINCHIPC6 || MCYRIXIII || MK7 || MK6 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MK8 || MVIAC3_2 || MEFFICEON || MGEODE_LX || MCORE2 || MATOM
 
 config X86_USE_3DNOW
         def_bool y
@@ -387,7 +396,7 @@ config X86_P6_NOP
 
 config X86_TSC
         def_bool y
-        depends on ((MWINCHIP3D || MCRUSOE || MEFFICEON || MCYRIXIII || MK7 || MK6 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || M586MMX || M586TSC || MK8 || MVIAC3_2 || MVIAC7 || MGEODEGX1 || MGEODE_LX || MCORE2) && !X86_NUMAQ) || X86_64
+        depends on ((MWINCHIP3D || MCRUSOE || MEFFICEON || MCYRIXIII || MK7 || MK6 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || M586MMX || M586TSC || MK8 || MVIAC3_2 || MVIAC7 || MGEODEGX1 || MGEODE_LX || MCORE2 || MATOM) && !X86_NUMAQ) || X86_64
 
 config X86_CMPXCHG64
         def_bool y
@@ -397,7 +406,7 @@ config X86_CMPXCHG64
 # generates cmov.
 config X86_CMOV
         def_bool y
-        depends on (MK8 || MK7 || MCORE2 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MCRUSOE || MEFFICEON || X86_64)
+        depends on (MK8 || MK7 || MCORE2 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MCRUSOE || MEFFICEON || X86_64 || MATOM)
 
 config X86_MINIMUM_CPU_FAMILY
         int
diff --git a/arch/x86/Makefile b/arch/x86/Makefile
index 1b68659c41b4..a012ee8ef803 100644
--- a/arch/x86/Makefile
+++ b/arch/x86/Makefile
@@ -32,8 +32,8 @@ ifeq ($(CONFIG_X86_32),y)
 
         # Disable unit-at-a-time mode on pre-gcc-4.0 compilers, it makes gcc use
         # a lot more stack due to the lack of sharing of stacklots:
-        KBUILD_CFLAGS += $(shell if [ $(call cc-version) -lt 0400 ] ; then \
-                echo $(call cc-option,-fno-unit-at-a-time); fi ;)
+        KBUILD_CFLAGS += $(call cc-ifversion, -lt, 0400, \
+                                $(call cc-option,-fno-unit-at-a-time))
 
         # CPU-specific tuning. Anything which can be shared with UML should go here.
         include $(srctree)/arch/x86/Makefile_32.cpu
@@ -55,6 +55,8 @@ else
 
         cflags-$(CONFIG_MCORE2) += \
                 $(call cc-option,-march=core2,$(call cc-option,-mtune=generic))
+        cflags-$(CONFIG_MATOM) += $(call cc-option,-march=atom) \
+                $(call cc-option,-mtune=atom,$(call cc-option,-mtune=generic))
         cflags-$(CONFIG_GENERIC_CPU) += $(call cc-option,-mtune=generic)
         KBUILD_CFLAGS += $(cflags-y)
 
@@ -72,7 +74,7 @@ endif
 
 ifdef CONFIG_CC_STACKPROTECTOR
         cc_has_sp := $(srctree)/scripts/gcc-x86_$(BITS)-has-stack-protector.sh
-        ifeq ($(shell $(CONFIG_SHELL) $(cc_has_sp) $(CC)),y)
+        ifeq ($(shell $(CONFIG_SHELL) $(cc_has_sp) $(CC) $(biarch)),y)
                 stackp-y := -fstack-protector
                 stackp-$(CONFIG_CC_STACKPROTECTOR_ALL) += -fstack-protector-all
                 KBUILD_CFLAGS += $(stackp-y)
@@ -177,8 +179,8 @@ archclean:
 define archhelp
   echo  '* bzImage      - Compressed kernel image (arch/x86/boot/bzImage)'
   echo  '  install      - Install kernel using'
-  echo  '                  (your) ~/bin/installkernel or'
-  echo  '                  (distribution) /sbin/installkernel or'
+  echo  '                  (your) ~/bin/$(INSTALLKERNEL) or'
+  echo  '                  (distribution) /sbin/$(INSTALLKERNEL) or'
   echo  '                  install to $$(INSTALL_PATH) and run lilo'
   echo  '  fdimage      - Create 1.4MB boot floppy image (arch/x86/boot/fdimage)'
   echo  '  fdimage144   - Create 1.4MB boot floppy image (arch/x86/boot/fdimage)'
diff --git a/arch/x86/Makefile_32.cpu b/arch/x86/Makefile_32.cpu
index 80177ec052f0..30e9a264f69d 100644
--- a/arch/x86/Makefile_32.cpu
+++ b/arch/x86/Makefile_32.cpu
@@ -33,6 +33,8 @@ cflags-$(CONFIG_MCYRIXIII)	+= $(call cc-option,-march=c3,-march=i486) $(align)-f
 cflags-$(CONFIG_MVIAC3_2)       += $(call cc-option,-march=c3-2,-march=i686)
 cflags-$(CONFIG_MVIAC7)         += -march=i686
 cflags-$(CONFIG_MCORE2)         += -march=i686 $(call tune,core2)
+cflags-$(CONFIG_MATOM)          += $(call cc-option,-march=atom,$(call cc-option,-march=core2,-march=i686)) \
+        $(call cc-option,-mtune=atom,$(call cc-option,-mtune=generic))
 
 # AMD Elan support
 cflags-$(CONFIG_X86_ELAN)       += -march=i486
diff --git a/arch/x86/boot/install.sh b/arch/x86/boot/install.sh
index 8d60ee15dfd9..d13ec1c38640 100644
--- a/arch/x86/boot/install.sh
+++ b/arch/x86/boot/install.sh
@@ -33,8 +33,8 @@ verify "$3"
 
 # User may have a custom install script
 
-if [ -x ~/bin/${CROSS_COMPILE}installkernel ]; then exec ~/bin/${CROSS_COMPILE}installkernel "$@"; fi
-if [ -x /sbin/${CROSS_COMPILE}installkernel ]; then exec /sbin/${CROSS_COMPILE}installkernel "$@"; fi
+if [ -x ~/bin/${INSTALLKERNEL} ]; then exec ~/bin/${INSTALLKERNEL} "$@"; fi
+if [ -x /sbin/${INSTALLKERNEL} ]; then exec /sbin/${INSTALLKERNEL} "$@"; fi
 
 # Default install - same as make zlilo
 
diff --git a/arch/x86/boot/video-vesa.c b/arch/x86/boot/video-vesa.c
index 275dd177f198..11e8c6eb80a1 100644
--- a/arch/x86/boot/video-vesa.c
+++ b/arch/x86/boot/video-vesa.c
@@ -31,7 +31,6 @@ static inline void vesa_store_mode_params_graphics(void) {}
 
 static int vesa_probe(void)
 {
-#if defined(CONFIG_VIDEO_VESA) || defined(CONFIG_FIRMWARE_EDID)
         struct biosregs ireg, oreg;
         u16 mode;
         addr_t mode_ptr;
@@ -49,8 +48,7 @@ static int vesa_probe(void)
             vginfo.signature != VESA_MAGIC ||
             vginfo.version < 0x0102)
                 return 0;       /* Not present */
-#endif /* CONFIG_VIDEO_VESA || CONFIG_FIRMWARE_EDID */
-#ifdef CONFIG_VIDEO_VESA
+
         set_fs(vginfo.video_mode_ptr.seg);
         mode_ptr = vginfo.video_mode_ptr.off;
 
@@ -102,9 +100,6 @@ static int vesa_probe(void)
         }
 
         return nmodes;
-#else
-        return 0;
-#endif /* CONFIG_VIDEO_VESA */
 }
 
 static int vesa_set_mode(struct mode_info *mode)
diff --git a/arch/x86/boot/video-vga.c b/arch/x86/boot/video-vga.c
index 8f8d827e254d..819caa1f2008 100644
--- a/arch/x86/boot/video-vga.c
+++ b/arch/x86/boot/video-vga.c
@@ -47,14 +47,6 @@ static u8 vga_set_basic_mode(void)
 
         initregs(&ireg);
 
-#ifdef CONFIG_VIDEO_400_HACK
-        if (adapter >= ADAPTER_VGA) {
-                ireg.ax = 0x1202;
-                ireg.bx = 0x0030;
-                intcall(0x10, &ireg, NULL);
-        }
-#endif
-
         ax = 0x0f00;
         intcall(0x10, &ireg, &oreg);
         mode = oreg.al;
@@ -62,11 +54,9 @@ static u8 vga_set_basic_mode(void)
         set_fs(0);
         rows = rdfs8(0x484);    /* rows minus one */
 
-#ifndef CONFIG_VIDEO_400_HACK
         if ((oreg.ax == 0x5003 || oreg.ax == 0x5007) &&
             (rows == 0 || rows == 24))
                 return mode;
-#endif
 
         if (mode != 3 && mode != 7)
                 mode = 3;
diff --git a/arch/x86/boot/video.c b/arch/x86/boot/video.c
index bad728b76fc2..d42da3802499 100644
--- a/arch/x86/boot/video.c
+++ b/arch/x86/boot/video.c
@@ -221,7 +221,6 @@ static unsigned int mode_menu(void)
         }
 }
 
-#ifdef CONFIG_VIDEO_RETAIN
 /* Save screen content to the heap */
 static struct saved_screen {
         int x, y;
@@ -299,10 +298,6 @@ static void restore_screen(void)
         ireg.dl = saved.curx;
         intcall(0x10, &ireg, NULL);
 }
-#else
-#define save_screen()           ((void)0)
-#define restore_screen()        ((void)0)
-#endif
 
 void set_video(void)
 {
diff --git a/arch/x86/boot/video.h b/arch/x86/boot/video.h
index 5bb174a997fc..ff339c5db311 100644
--- a/arch/x86/boot/video.h
+++ b/arch/x86/boot/video.h
@@ -17,19 +17,8 @@
 
 #include <linux/types.h>
 
-/* Enable autodetection of SVGA adapters and modes. */
-#undef CONFIG_VIDEO_SVGA
-
-/* Enable autodetection of VESA modes */
-#define CONFIG_VIDEO_VESA
-
-/* Retain screen contents when switching modes */
-#define CONFIG_VIDEO_RETAIN
-
-/* Force 400 scan lines for standard modes (hack to fix bad BIOS behaviour */
-#undef CONFIG_VIDEO_400_HACK
-
-/* This code uses an extended set of video mode numbers. These include:
+/*
+ * This code uses an extended set of video mode numbers. These include:
  * Aliases for standard modes
  *      NORMAL_VGA (-1)
  *      EXTENDED_VGA (-2)
@@ -67,13 +56,8 @@
 /* The "recalculate timings" flag */
 #define VIDEO_RECALC 0x8000
 
-/* Define DO_STORE according to CONFIG_VIDEO_RETAIN */
-#ifdef CONFIG_VIDEO_RETAIN
 void store_screen(void);
 #define DO_STORE() store_screen()
-#else
-#define DO_STORE() ((void)0)
-#endif /* CONFIG_VIDEO_RETAIN */
 
 /*
  * Mode table structures
diff --git a/arch/x86/configs/i386_defconfig b/arch/x86/configs/i386_defconfig
index edb992ebef92..d28fad19654a 100644
--- a/arch/x86/configs/i386_defconfig
+++ b/arch/x86/configs/i386_defconfig
@@ -2355,7 +2355,7 @@ CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST=y
 CONFIG_HAVE_DYNAMIC_FTRACE=y
 CONFIG_HAVE_FTRACE_MCOUNT_RECORD=y
 CONFIG_HAVE_HW_BRANCH_TRACER=y
-CONFIG_HAVE_FTRACE_SYSCALLS=y
+CONFIG_HAVE_SYSCALL_TRACEPOINTS=y
 CONFIG_RING_BUFFER=y
 CONFIG_TRACING=y
 CONFIG_TRACING_SUPPORT=y
diff --git a/arch/x86/configs/x86_64_defconfig b/arch/x86/configs/x86_64_defconfig
index cee1dd2e69b2..6c86acd847a4 100644
--- a/arch/x86/configs/x86_64_defconfig
+++ b/arch/x86/configs/x86_64_defconfig
@@ -2329,7 +2329,7 @@ CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST=y
 CONFIG_HAVE_DYNAMIC_FTRACE=y
 CONFIG_HAVE_FTRACE_MCOUNT_RECORD=y
 CONFIG_HAVE_HW_BRANCH_TRACER=y
-CONFIG_HAVE_FTRACE_SYSCALLS=y
+CONFIG_HAVE_SYSCALL_TRACEPOINTS=y
 CONFIG_RING_BUFFER=y
 CONFIG_TRACING=y
 CONFIG_TRACING_SUPPORT=y
diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c
index c580c5ec1cad..585edebe12cf 100644
--- a/arch/x86/crypto/aesni-intel_glue.c
+++ b/arch/x86/crypto/aesni-intel_glue.c
@@ -59,13 +59,6 @@ asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
                               const u8 *in, unsigned int len, u8 *iv);
 
-static inline int kernel_fpu_using(void)
-{
-        if (in_interrupt() && !(read_cr0() & X86_CR0_TS))
-                return 1;
-        return 0;
-}
-
 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx)
 {
         unsigned long addr = (unsigned long)raw_ctx;
@@ -89,7 +82,7 @@ static int aes_set_key_common(struct crypto_tfm *tfm, void *raw_ctx,
                 return -EINVAL;
         }
 
-        if (kernel_fpu_using())
+        if (irq_fpu_usable())
                 err = crypto_aes_expand_key(ctx, in_key, key_len);
         else {
                 kernel_fpu_begin();
@@ -110,7 +103,7 @@ static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 {
         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
 
-        if (kernel_fpu_using())
+        if (irq_fpu_usable())
                 crypto_aes_encrypt_x86(ctx, dst, src);
         else {
                 kernel_fpu_begin();
@@ -123,7 +116,7 @@ static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 {
         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
 
-        if (kernel_fpu_using())
+        if (irq_fpu_usable())
                 crypto_aes_decrypt_x86(ctx, dst, src);
         else {
                 kernel_fpu_begin();
@@ -349,7 +342,7 @@ static int ablk_encrypt(struct ablkcipher_request *req)
         struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
         struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 
-        if (kernel_fpu_using()) {
+        if (irq_fpu_usable()) {
                 struct ablkcipher_request *cryptd_req =
                         ablkcipher_request_ctx(req);
                 memcpy(cryptd_req, req, sizeof(*req));
@@ -370,7 +363,7 @@ static int ablk_decrypt(struct ablkcipher_request *req)
         struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
         struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 
-        if (kernel_fpu_using()) {
+        if (irq_fpu_usable()) {
                 struct ablkcipher_request *cryptd_req =
                         ablkcipher_request_ctx(req);
                 memcpy(cryptd_req, req, sizeof(*req));
@@ -636,7 +629,7 @@ static int __init aesni_init(void)
         int err;
 
         if (!cpu_has_aes) {
-                printk(KERN_ERR "Intel AES-NI instructions are not detected.\n");
+                printk(KERN_INFO "Intel AES-NI instructions are not detected.\n");
                 return -ENODEV;
         }
         if ((err = crypto_register_alg(&aesni_alg)))
diff --git a/arch/x86/ia32/ia32entry.S b/arch/x86/ia32/ia32entry.S
index e590261ba059..74619c4f9fda 100644
--- a/arch/x86/ia32/ia32entry.S
+++ b/arch/x86/ia32/ia32entry.S
@@ -537,7 +537,7 @@ ia32_sys_call_table:
         .quad sys_mkdir
         .quad sys_rmdir         /* 40 */
         .quad sys_dup
-        .quad sys32_pipe
+        .quad sys_pipe
         .quad compat_sys_times
         .quad quiet_ni_syscall                  /* old prof syscall holder */
         .quad sys_brk           /* 45 */
@@ -831,5 +831,5 @@ ia32_sys_call_table:
         .quad compat_sys_preadv
         .quad compat_sys_pwritev
         .quad compat_sys_rt_tgsigqueueinfo      /* 335 */
-        .quad sys_perf_counter_open
+        .quad sys_perf_event_open
 ia32_syscall_end:
diff --git a/arch/x86/ia32/sys_ia32.c b/arch/x86/ia32/sys_ia32.c
index 085a8c35f149..9f5527198825 100644
--- a/arch/x86/ia32/sys_ia32.c
+++ b/arch/x86/ia32/sys_ia32.c
@@ -189,20 +189,6 @@ asmlinkage long sys32_mprotect(unsigned long start, size_t len,
         return sys_mprotect(start, len, prot);
 }
 
-asmlinkage long sys32_pipe(int __user *fd)
-{
-        int retval;
-        int fds[2];
-
-        retval = do_pipe_flags(fds, 0);
-        if (retval)
-                goto out;
-        if (copy_to_user(fd, fds, sizeof(fds)))
-                retval = -EFAULT;
-out:
-        return retval;
-}
-
 asmlinkage long sys32_rt_sigaction(int sig, struct sigaction32 __user *act,
                                    struct sigaction32 __user *oact,
                                    unsigned int sigsetsize)
diff --git a/arch/x86/include/asm/acpi.h b/arch/x86/include/asm/acpi.h
index 20d1465a2ab0..4518dc500903 100644
--- a/arch/x86/include/asm/acpi.h
+++ b/arch/x86/include/asm/acpi.h
@@ -144,7 +144,6 @@ static inline unsigned int acpi_processor_cstate_check(unsigned int max_cstate)
 
 #else /* !CONFIG_ACPI */
 
-#define acpi_disabled 1
 #define acpi_lapic 0
 #define acpi_ioapic 0
 static inline void acpi_noirq_set(void) { }
diff --git a/arch/x86/include/asm/agp.h b/arch/x86/include/asm/agp.h
index 9825cd64c9b6..eec2a70d4376 100644
--- a/arch/x86/include/asm/agp.h
+++ b/arch/x86/include/asm/agp.h
@@ -22,10 +22,6 @@
  */
 #define flush_agp_cache() wbinvd()
 
-/* Convert a physical address to an address suitable for the GART. */
-#define phys_to_gart(x) (x)
-#define gart_to_phys(x) (x)
-
 /* GATT allocation. Returns/accepts GATT kernel virtual address. */
 #define alloc_gatt_pages(order)         \
         ((char *)__get_free_pages(GFP_KERNEL, (order)))
diff --git a/arch/x86/include/asm/alternative.h b/arch/x86/include/asm/alternative.h
index 1a37bcdc8606..c240efc74e00 100644
--- a/arch/x86/include/asm/alternative.h
+++ b/arch/x86/include/asm/alternative.h
@@ -73,8 +73,6 @@ static inline void alternatives_smp_module_del(struct module *mod) {}
 static inline void alternatives_smp_switch(int smp) {}
 #endif  /* CONFIG_SMP */
 
-const unsigned char *const *find_nop_table(void);
-
 /* alternative assembly primitive: */
 #define ALTERNATIVE(oldinstr, newinstr, feature)                        \
                                                                         \
@@ -144,8 +142,6 @@ static inline void apply_paravirt(struct paravirt_patch_site *start,
 #define __parainstructions_end  NULL
 #endif
 
-extern void add_nops(void *insns, unsigned int len);
-
 /*
  * Clear and restore the kernel write-protection flag on the local CPU.
  * Allows the kernel to edit read-only pages.
@@ -161,10 +157,7 @@ extern void add_nops(void *insns, unsigned int len);
  * Intel's errata.
  * On the local CPU you need to be protected again NMI or MCE handlers seeing an
  * inconsistent instruction while you patch.
- * The _early version expects the memory to already be RW.
  */
-
 extern void *text_poke(void *addr, const void *opcode, size_t len);
-extern void *text_poke_early(void *addr, const void *opcode, size_t len);
 
 #endif /* _ASM_X86_ALTERNATIVE_H */
diff --git a/arch/x86/include/asm/amd_iommu.h b/arch/x86/include/asm/amd_iommu.h
index bdf96f119f06..ac95995b7bad 100644
--- a/arch/x86/include/asm/amd_iommu.h
+++ b/arch/x86/include/asm/amd_iommu.h
@@ -25,6 +25,7 @@
 #ifdef CONFIG_AMD_IOMMU
 extern int amd_iommu_init(void);
 extern int amd_iommu_init_dma_ops(void);
+extern int amd_iommu_init_passthrough(void);
 extern void amd_iommu_detect(void);
 extern irqreturn_t amd_iommu_int_handler(int irq, void *data);
 extern void amd_iommu_flush_all_domains(void);
diff --git a/arch/x86/include/asm/amd_iommu_types.h b/arch/x86/include/asm/amd_iommu_types.h
index 0c878caaa0a2..2a2cc7a78a81 100644
--- a/arch/x86/include/asm/amd_iommu_types.h
+++ b/arch/x86/include/asm/amd_iommu_types.h
@@ -143,22 +143,29 @@
 #define EVT_BUFFER_SIZE         8192 /* 512 entries */
 #define EVT_LEN_MASK            (0x9ULL << 56)
 
+#define PAGE_MODE_NONE    0x00
 #define PAGE_MODE_1_LEVEL 0x01
 #define PAGE_MODE_2_LEVEL 0x02
 #define PAGE_MODE_3_LEVEL 0x03
-
-#define IOMMU_PDE_NL_0   0x000ULL
-#define IOMMU_PDE_NL_1   0x200ULL
-#define IOMMU_PDE_NL_2   0x400ULL
-#define IOMMU_PDE_NL_3   0x600ULL
-
-#define IOMMU_PTE_L2_INDEX(address) (((address) >> 30) & 0x1ffULL)
-#define IOMMU_PTE_L1_INDEX(address) (((address) >> 21) & 0x1ffULL)
-#define IOMMU_PTE_L0_INDEX(address) (((address) >> 12) & 0x1ffULL)
-
-#define IOMMU_MAP_SIZE_L1 (1ULL << 21)
-#define IOMMU_MAP_SIZE_L2 (1ULL << 30)
-#define IOMMU_MAP_SIZE_L3 (1ULL << 39)
+#define PAGE_MODE_4_LEVEL 0x04
+#define PAGE_MODE_5_LEVEL 0x05
+#define PAGE_MODE_6_LEVEL 0x06
+
+#define PM_LEVEL_SHIFT(x)       (12 + ((x) * 9))
+#define PM_LEVEL_SIZE(x)        (((x) < 6) ? \
+                                  ((1ULL << PM_LEVEL_SHIFT((x))) - 1): \
+                                   (0xffffffffffffffffULL))
+#define PM_LEVEL_INDEX(x, a)    (((a) >> PM_LEVEL_SHIFT((x))) & 0x1ffULL)
+#define PM_LEVEL_ENC(x)         (((x) << 9) & 0xe00ULL)
+#define PM_LEVEL_PDE(x, a)      ((a) | PM_LEVEL_ENC((x)) | \
+                                 IOMMU_PTE_P | IOMMU_PTE_IR | IOMMU_PTE_IW)
+#define PM_PTE_LEVEL(pte)       (((pte) >> 9) & 0x7ULL)
+
+#define PM_MAP_4k               0
+#define PM_ADDR_MASK            0x000ffffffffff000ULL
+#define PM_MAP_MASK(lvl)        (PM_ADDR_MASK & \
+                                (~((1ULL << (12 + ((lvl) * 9))) - 1)))
+#define PM_ALIGNED(lvl, addr)   ((PM_MAP_MASK(lvl) & (addr)) == (addr))
 
 #define IOMMU_PTE_P  (1ULL << 0)
 #define IOMMU_PTE_TV (1ULL << 1)
@@ -167,11 +174,6 @@
 #define IOMMU_PTE_IR (1ULL << 61)
 #define IOMMU_PTE_IW (1ULL << 62)
 
-#define IOMMU_L1_PDE(address) \
-        ((address) | IOMMU_PDE_NL_1 | IOMMU_PTE_P | IOMMU_PTE_IR | IOMMU_PTE_IW)
-#define IOMMU_L2_PDE(address) \
-        ((address) | IOMMU_PDE_NL_2 | IOMMU_PTE_P | IOMMU_PTE_IR | IOMMU_PTE_IW)
-
 #define IOMMU_PAGE_MASK (((1ULL << 52) - 1) & ~0xfffULL)
 #define IOMMU_PTE_PRESENT(pte) ((pte) & IOMMU_PTE_P)
 #define IOMMU_PTE_PAGE(pte) (phys_to_virt((pte) & IOMMU_PAGE_MASK))
@@ -194,11 +196,14 @@
 #define PD_DMA_OPS_MASK         (1UL << 0) /* domain used for dma_ops */
 #define PD_DEFAULT_MASK         (1UL << 1) /* domain is a default dma_ops
                                               domain for an IOMMU */
+#define PD_PASSTHROUGH_MASK     (1UL << 2) /* domain has no page
+                                              translation */
+
 extern bool amd_iommu_dump;
 #define DUMP_printk(format, arg...)                                     \
         do {                                                            \
                 if (amd_iommu_dump)                                             \
-                        printk(KERN_INFO "AMD IOMMU: " format, ## arg); \
+                        printk(KERN_INFO "AMD-Vi: " format, ## arg);    \
         } while(0);
 
 /*
@@ -226,6 +231,7 @@ struct protection_domain {
         int mode;               /* paging mode (0-6 levels) */
         u64 *pt_root;           /* page table root pointer */
         unsigned long flags;    /* flags to find out type of domain */
+        bool updated;           /* complete domain flush required */
         unsigned dev_cnt;       /* devices assigned to this domain */
         void *priv;             /* private data */
 };
@@ -337,6 +343,9 @@ struct amd_iommu {
         /* if one, we need to send a completion wait command */
         bool need_sync;
 
+        /* becomes true if a command buffer reset is running */
+        bool reset_in_progress;
+
         /* default dma_ops domain for that IOMMU */
         struct dma_ops_domain *default_dom;
 };
@@ -457,4 +466,7 @@ static inline void amd_iommu_stats_init(void) { }
 
 #endif /* CONFIG_AMD_IOMMU_STATS */
 
+/* some function prototypes */
+extern void amd_iommu_reset_cmd_buffer(struct amd_iommu *iommu);
+
 #endif /* _ASM_X86_AMD_IOMMU_TYPES_H */
diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h
index bb7d47925847..474d80d3e6cc 100644
--- a/arch/x86/include/asm/apic.h
+++ b/arch/x86/include/asm/apic.h
@@ -66,13 +66,23 @@ static inline void default_inquire_remote_apic(int apicid)
 }
 
 /*
+ * With 82489DX we can't rely on apic feature bit
+ * retrieved via cpuid but still have to deal with
+ * such an apic chip so we assume that SMP configuration
+ * is found from MP table (64bit case uses ACPI mostly
+ * which set smp presence flag as well so we are safe
+ * to use this helper too).
+ */
+static inline bool apic_from_smp_config(void)
+{
+        return smp_found_config && !disable_apic;
+}
+
+/*
  * Basic functions accessing APICs.
  */
 #ifdef CONFIG_PARAVIRT
 #include <asm/paravirt.h>
-#else
-#define setup_boot_clock setup_boot_APIC_clock
-#define setup_secondary_clock setup_secondary_APIC_clock
 #endif
 
 #ifdef CONFIG_X86_64
@@ -183,6 +193,10 @@ static inline int x2apic_enabled(void)
 }
 
 #define x2apic_supported()      (cpu_has_x2apic)
+static inline void x2apic_force_phys(void)
+{
+        x2apic_phys = 1;
+}
 #else
 static inline void check_x2apic(void)
 {
@@ -194,6 +208,9 @@ static inline int x2apic_enabled(void)
 {
         return 0;
 }
+static inline void x2apic_force_phys(void)
+{
+}
 
 #define x2apic_preenabled 0
 #define x2apic_supported()      0
@@ -245,6 +262,8 @@ static inline void lapic_shutdown(void) { }
 static inline void init_apic_mappings(void) { }
 static inline void disable_local_APIC(void) { }
 static inline void apic_disable(void) { }
+# define setup_boot_APIC_clock x86_init_noop
+# define setup_secondary_APIC_clock x86_init_noop
 #endif /* !CONFIG_X86_LOCAL_APIC */
 
 #ifdef CONFIG_X86_64
@@ -293,7 +312,7 @@ struct apic {
         int (*cpu_present_to_apicid)(int mps_cpu);
         physid_mask_t (*apicid_to_cpu_present)(int phys_apicid);
         void (*setup_portio_remap)(void);
-        int (*check_phys_apicid_present)(int boot_cpu_physical_apicid);
+        int (*check_phys_apicid_present)(int phys_apicid);
         void (*enable_apic_mode)(void);
         int (*phys_pkg_id)(int cpuid_apic, int index_msb);
 
@@ -427,7 +446,7 @@ extern struct apic apic_x2apic_uv_x;
 DECLARE_PER_CPU(int, x2apic_extra_bits);
 
 extern int default_cpu_present_to_apicid(int mps_cpu);
-extern int default_check_phys_apicid_present(int boot_cpu_physical_apicid);
+extern int default_check_phys_apicid_present(int phys_apicid);
 #endif
 
 static inline void default_wait_for_init_deassert(atomic_t *deassert)
@@ -543,9 +562,9 @@ static inline int __default_cpu_present_to_apicid(int mps_cpu)
 }
 
 static inline int
-__default_check_phys_apicid_present(int boot_cpu_physical_apicid)
+__default_check_phys_apicid_present(int phys_apicid)
 {
-        return physid_isset(boot_cpu_physical_apicid, phys_cpu_present_map);
+        return physid_isset(phys_apicid, phys_cpu_present_map);
 }
 
 #ifdef CONFIG_X86_32
@@ -555,13 +574,13 @@ static inline int default_cpu_present_to_apicid(int mps_cpu)
 }
 
 static inline int
-default_check_phys_apicid_present(int boot_cpu_physical_apicid)
+default_check_phys_apicid_present(int phys_apicid)
 {
-        return __default_check_phys_apicid_present(boot_cpu_physical_apicid);
+        return __default_check_phys_apicid_present(phys_apicid);
 }
 #else
 extern int default_cpu_present_to_apicid(int mps_cpu);
-extern int default_check_phys_apicid_present(int boot_cpu_physical_apicid);
+extern int default_check_phys_apicid_present(int phys_apicid);
 #endif
 
 static inline physid_mask_t default_apicid_to_cpu_present(int phys_apicid)
diff --git a/arch/x86/include/asm/apicdef.h b/arch/x86/include/asm/apicdef.h
index 7ddb36ab933b..3b62da926de9 100644
--- a/arch/x86/include/asm/apicdef.h
+++ b/arch/x86/include/asm/apicdef.h
@@ -8,12 +8,14 @@
  * Ingo Molnar <mingo@redhat.com>, 1999, 2000
  */
 
-#define APIC_DEFAULT_PHYS_BASE  0xfee00000
+#define IO_APIC_DEFAULT_PHYS_BASE       0xfec00000
+#define APIC_DEFAULT_PHYS_BASE          0xfee00000
 
 #define APIC_ID         0x20
 
 #define APIC_LVR        0x30
 #define         APIC_LVR_MASK           0xFF00FF
+#define         APIC_LVR_DIRECTED_EOI   (1 << 24)
 #define         GET_APIC_VERSION(x)     ((x) & 0xFFu)
 #define         GET_APIC_MAXLVT(x)      (((x) >> 16) & 0xFFu)
 #ifdef CONFIG_X86_32
@@ -40,6 +42,7 @@
 #define         APIC_DFR_CLUSTER                0x0FFFFFFFul
 #define         APIC_DFR_FLAT                   0xFFFFFFFFul
 #define APIC_SPIV       0xF0
+#define         APIC_SPIV_DIRECTED_EOI          (1 << 12)
 #define         APIC_SPIV_FOCUS_DISABLED        (1 << 9)
 #define         APIC_SPIV_APIC_ENABLED          (1 << 8)
 #define APIC_ISR        0x100
diff --git a/arch/x86/include/asm/asm.h b/arch/x86/include/asm/asm.h
index 56be78f582f0..b3ed1e1460ff 100644
--- a/arch/x86/include/asm/asm.h
+++ b/arch/x86/include/asm/asm.h
@@ -3,7 +3,7 @@
 
 #ifdef __ASSEMBLY__
 # define __ASM_FORM(x)  x
-# define __ASM_EX_SEC   .section __ex_table
+# define __ASM_EX_SEC   .section __ex_table, "a"
 #else
 # define __ASM_FORM(x)  " " #x " "
 # define __ASM_EX_SEC   " .section __ex_table,\"a\"\n"
@@ -38,10 +38,18 @@
 #define _ASM_DI         __ASM_REG(di)
 
 /* Exception table entry */
+#ifdef __ASSEMBLY__
+# define _ASM_EXTABLE(from,to)      \
+        __ASM_EX_SEC ;              \
+        _ASM_ALIGN ;                \
+        _ASM_PTR from , to ;        \
+        .previous
+#else
 # define _ASM_EXTABLE(from,to) \
         __ASM_EX_SEC    \
         _ASM_ALIGN "\n" \
         _ASM_PTR #from "," #to "\n" \
         " .previous\n"
+#endif
 
 #endif /* _ASM_X86_ASM_H */
diff --git a/arch/x86/include/asm/bootparam.h b/arch/x86/include/asm/bootparam.h
index 1724e8de317c..6be33d83c716 100644
--- a/arch/x86/include/asm/bootparam.h
+++ b/arch/x86/include/asm/bootparam.h
@@ -85,7 +85,8 @@ struct efi_info {
 struct boot_params {
         struct screen_info screen_info;                 /* 0x000 */
         struct apm_bios_info apm_bios_info;             /* 0x040 */
-        __u8  _pad2[12];                                /* 0x054 */
+        __u8  _pad2[4];                                 /* 0x054 */
+        __u64  tboot_addr;                              /* 0x058 */
         struct ist_info ist_info;                       /* 0x060 */
         __u8  _pad3[16];                                /* 0x070 */
         __u8  hd0_info[16];     /* obsolete! */         /* 0x080 */
@@ -109,4 +110,14 @@ struct boot_params {
         __u8  _pad9[276];                               /* 0xeec */
 } __attribute__((packed));
 
+enum {
+        X86_SUBARCH_PC = 0,
+        X86_SUBARCH_LGUEST,
+        X86_SUBARCH_XEN,
+        X86_SUBARCH_MRST,
+        X86_NR_SUBARCHS,
+};
+
+
+
 #endif /* _ASM_X86_BOOTPARAM_H */
diff --git a/arch/x86/include/asm/cache.h b/arch/x86/include/asm/cache.h
index 5d367caa0e36..549860d3be8f 100644
--- a/arch/x86/include/asm/cache.h
+++ b/arch/x86/include/asm/cache.h
@@ -1,6 +1,8 @@
 #ifndef _ASM_X86_CACHE_H
 #define _ASM_X86_CACHE_H
 
+#include <linux/linkage.h>
+
 /* L1 cache line size */
 #define L1_CACHE_SHIFT  (CONFIG_X86_L1_CACHE_SHIFT)
 #define L1_CACHE_BYTES  (1 << L1_CACHE_SHIFT)
@@ -13,7 +15,7 @@
 #ifdef CONFIG_SMP
 #define __cacheline_aligned_in_smp                                      \
         __attribute__((__aligned__(1 << (INTERNODE_CACHE_SHIFT))))      \
-        __attribute__((__section__(".data.page_aligned")))
+        __page_aligned_data
 #endif
 #endif
 
diff --git a/arch/x86/include/asm/cacheflush.h b/arch/x86/include/asm/cacheflush.h
index e55dfc1ad453..b54f6afe7ec4 100644
--- a/arch/x86/include/asm/cacheflush.h
+++ b/arch/x86/include/asm/cacheflush.h
@@ -43,8 +43,58 @@ static inline void copy_from_user_page(struct vm_area_struct *vma,
         memcpy(dst, src, len);
 }
 
-#define PG_non_WB                               PG_arch_1
-PAGEFLAG(NonWB, non_WB)
+#define PG_WC                           PG_arch_1
+PAGEFLAG(WC, WC)
+
+#ifdef CONFIG_X86_PAT
+/*
+ * X86 PAT uses page flags WC and Uncached together to keep track of
+ * memory type of pages that have backing page struct. X86 PAT supports 3
+ * different memory types, _PAGE_CACHE_WB, _PAGE_CACHE_WC and
+ * _PAGE_CACHE_UC_MINUS and fourth state where page's memory type has not
+ * been changed from its default (value of -1 used to denote this).
+ * Note we do not support _PAGE_CACHE_UC here.
+ *
+ * Caller must hold memtype_lock for atomicity.
+ */
+static inline unsigned long get_page_memtype(struct page *pg)
+{
+        if (!PageUncached(pg) && !PageWC(pg))
+                return -1;
+        else if (!PageUncached(pg) && PageWC(pg))
+                return _PAGE_CACHE_WC;
+        else if (PageUncached(pg) && !PageWC(pg))
+                return _PAGE_CACHE_UC_MINUS;
+        else
+                return _PAGE_CACHE_WB;
+}
+
+static inline void set_page_memtype(struct page *pg, unsigned long memtype)
+{
+        switch (memtype) {
+        case _PAGE_CACHE_WC:
+                ClearPageUncached(pg);
+                SetPageWC(pg);
+                break;
+        case _PAGE_CACHE_UC_MINUS:
+                SetPageUncached(pg);
+                ClearPageWC(pg);
+                break;
+        case _PAGE_CACHE_WB:
+                SetPageUncached(pg);
+                SetPageWC(pg);
+                break;
+        default:
+        case -1:
+                ClearPageUncached(pg);
+                ClearPageWC(pg);
+                break;
+        }
+}
+#else
+static inline unsigned long get_page_memtype(struct page *pg) { return -1; }
+static inline void set_page_memtype(struct page *pg, unsigned long memtype) { }
+#endif
 
 /*
  * The set_memory_* API can be used to change various attributes of a virtual
diff --git a/arch/x86/include/asm/cpufeature.h b/arch/x86/include/asm/cpufeature.h
index 4a28d22d4793..9cfc88b97742 100644
--- a/arch/x86/include/asm/cpufeature.h
+++ b/arch/x86/include/asm/cpufeature.h
@@ -95,6 +95,8 @@
 #define X86_FEATURE_NONSTOP_TSC (3*32+24) /* TSC does not stop in C states */
 #define X86_FEATURE_CLFLUSH_MONITOR (3*32+25) /* "" clflush reqd with monitor */
 #define X86_FEATURE_EXTD_APICID (3*32+26) /* has extended APICID (8 bits) */
+#define X86_FEATURE_AMD_DCM     (3*32+27) /* multi-node processor */
+#define X86_FEATURE_APERFMPERF  (3*32+28) /* APERFMPERF */
 
 /* Intel-defined CPU features, CPUID level 0x00000001 (ecx), word 4 */
 #define X86_FEATURE_XMM3        (4*32+ 0) /* "pni" SSE-3 */
diff --git a/arch/x86/include/asm/current.h b/arch/x86/include/asm/current.h
index c68c361697e1..4d447b732d82 100644
--- a/arch/x86/include/asm/current.h
+++ b/arch/x86/include/asm/current.h
@@ -11,7 +11,7 @@ DECLARE_PER_CPU(struct task_struct *, current_task);
 
 static __always_inline struct task_struct *get_current(void)
 {
-        return percpu_read(current_task);
+        return percpu_read_stable(current_task);
 }
 
 #define current get_current()
diff --git a/arch/x86/include/asm/desc.h b/arch/x86/include/asm/desc.h
index c993e9e0fed4..e8de2f6f5ca5 100644
--- a/arch/x86/include/asm/desc.h
+++ b/arch/x86/include/asm/desc.h
@@ -291,11 +291,24 @@ static inline unsigned long get_desc_base(const struct desc_struct *desc)
         return desc->base0 | ((desc->base1) << 16) | ((desc->base2) << 24);
 }
 
+static inline void set_desc_base(struct desc_struct *desc, unsigned long base)
+{
+        desc->base0 = base & 0xffff;
+        desc->base1 = (base >> 16) & 0xff;
+        desc->base2 = (base >> 24) & 0xff;
+}
+
 static inline unsigned long get_desc_limit(const struct desc_struct *desc)
 {
         return desc->limit0 | (desc->limit << 16);
 }
 
+static inline void set_desc_limit(struct desc_struct *desc, unsigned long limit)
+{
+        desc->limit0 = limit & 0xffff;
+        desc->limit = (limit >> 16) & 0xf;
+}
+
 static inline void _set_gate(int gate, unsigned type, void *addr,
                              unsigned dpl, unsigned ist, unsigned seg)
 {
diff --git a/arch/x86/include/asm/desc_defs.h b/arch/x86/include/asm/desc_defs.h
index a6adefa28b94..9d6684849fd9 100644
--- a/arch/x86/include/asm/desc_defs.h
+++ b/arch/x86/include/asm/desc_defs.h
@@ -34,6 +34,12 @@ struct desc_struct {
         };
 } __attribute__((packed));
 
+#define GDT_ENTRY_INIT(flags, base, limit) { { { \
+                .a = ((limit) & 0xffff) | (((base) & 0xffff) << 16), \
+                .b = (((base) & 0xff0000) >> 16) | (((flags) & 0xf0ff) << 8) | \
+                        ((limit) & 0xf0000) | ((base) & 0xff000000), \
+        } } }
+
 enum {
         GATE_INTERRUPT = 0xE,
         GATE_TRAP = 0xF,
diff --git a/arch/x86/include/asm/device.h b/arch/x86/include/asm/device.h
index 4994a20acbcb..cee34e9ca45b 100644
--- a/arch/x86/include/asm/device.h
+++ b/arch/x86/include/asm/device.h
@@ -13,4 +13,7 @@ struct dma_map_ops *dma_ops;
 #endif
 };
 
+struct pdev_archdata {
+};
+
 #endif /* _ASM_X86_DEVICE_H */
diff --git a/arch/x86/include/asm/dma-mapping.h b/arch/x86/include/asm/dma-mapping.h
index 1c3f9435f1c9..0ee770d23d0e 100644
--- a/arch/x86/include/asm/dma-mapping.h
+++ b/arch/x86/include/asm/dma-mapping.h
@@ -55,6 +55,24 @@ extern int dma_set_mask(struct device *dev, u64 mask);
 extern void *dma_generic_alloc_coherent(struct device *dev, size_t size,
                                         dma_addr_t *dma_addr, gfp_t flag);
 
+static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size)
+{
+        if (!dev->dma_mask)
+                return 0;
+
+        return addr + size <= *dev->dma_mask;
+}
+
+static inline dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr)
+{
+        return paddr;
+}
+
+static inline phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr)
+{
+        return daddr;
+}
+
 static inline void
 dma_cache_sync(struct device *dev, void *vaddr, size_t size,
         enum dma_data_direction dir)
diff --git a/arch/x86/include/asm/do_timer.h b/arch/x86/include/asm/do_timer.h
deleted file mode 100644
index 23ecda0b28a0..000000000000
--- a/arch/x86/include/asm/do_timer.h
+++ /dev/null
@@ -1,16 +0,0 @@
-/* defines for inline arch setup functions */
-#include <linux/clockchips.h>
-
-#include <asm/i8259.h>
-#include <asm/i8253.h>
-
-/**
- * do_timer_interrupt_hook - hook into timer tick
- *
- * Call the pit clock event handler. see asm/i8253.h
- **/
-
-static inline void do_timer_interrupt_hook(void)
-{
-        global_clock_event->event_handler(global_clock_event);
-}
diff --git a/arch/x86/include/asm/dwarf2.h b/arch/x86/include/asm/dwarf2.h
index 3afc5e87cfdd..ae6253ab9029 100644
--- a/arch/x86/include/asm/dwarf2.h
+++ b/arch/x86/include/asm/dwarf2.h
@@ -87,9 +87,25 @@
         CFI_RESTORE \reg
         .endm
 #else /*!CONFIG_X86_64*/
+        .macro pushl_cfi reg
+        pushl \reg
+        CFI_ADJUST_CFA_OFFSET 4
+        .endm
 
-        /* 32bit defenitions are missed yet */
+        .macro popl_cfi reg
+        popl \reg
+        CFI_ADJUST_CFA_OFFSET -4
+        .endm
 
+        .macro movl_cfi reg offset=0
+        movl %\reg, \offset(%esp)
+        CFI_REL_OFFSET \reg, \offset
+        .endm
+
+        .macro movl_cfi_restore offset reg
+        movl \offset(%esp), %\reg
+        CFI_RESTORE \reg
+        .endm
 #endif /*!CONFIG_X86_64*/
 #endif /*__ASSEMBLY__*/
 
diff --git a/arch/x86/include/asm/e820.h b/arch/x86/include/asm/e820.h
index 7ecba4d85089..40b4e614fe71 100644
--- a/arch/x86/include/asm/e820.h
+++ b/arch/x86/include/asm/e820.h
@@ -126,8 +126,6 @@ extern void e820_reserve_resources(void);
 extern void e820_reserve_resources_late(void);
 extern void setup_memory_map(void);
 extern char *default_machine_specific_memory_setup(void);
-extern char *machine_specific_memory_setup(void);
-extern char *memory_setup(void);
 #endif /* __KERNEL__ */
 #endif /* __ASSEMBLY__ */
 
diff --git a/arch/x86/include/asm/elf.h b/arch/x86/include/asm/elf.h
index 83c1bc8d2e8a..456a304b8172 100644
--- a/arch/x86/include/asm/elf.h
+++ b/arch/x86/include/asm/elf.h
@@ -299,6 +299,8 @@ do {									\
 
 #ifdef CONFIG_X86_32
 
+#define STACK_RND_MASK (0x7ff)
+
 #define VDSO_HIGH_BASE          (__fix_to_virt(FIX_VDSO))
 
 #define ARCH_DLINFO             ARCH_DLINFO_IA32(vdso_enabled)
diff --git a/arch/x86/include/asm/entry_arch.h b/arch/x86/include/asm/entry_arch.h
index ff8cbfa07851..f5693c81a1db 100644
--- a/arch/x86/include/asm/entry_arch.h
+++ b/arch/x86/include/asm/entry_arch.h
@@ -49,7 +49,7 @@ BUILD_INTERRUPT(apic_timer_interrupt,LOCAL_TIMER_VECTOR)
 BUILD_INTERRUPT(error_interrupt,ERROR_APIC_VECTOR)
 BUILD_INTERRUPT(spurious_interrupt,SPURIOUS_APIC_VECTOR)
 
-#ifdef CONFIG_PERF_COUNTERS
+#ifdef CONFIG_PERF_EVENTS
 BUILD_INTERRUPT(perf_pending_interrupt, LOCAL_PENDING_VECTOR)
 #endif
 
@@ -61,7 +61,7 @@ BUILD_INTERRUPT(thermal_interrupt,THERMAL_APIC_VECTOR)
 BUILD_INTERRUPT(threshold_interrupt,THRESHOLD_APIC_VECTOR)
 #endif
 
-#ifdef CONFIG_X86_NEW_MCE
+#ifdef CONFIG_X86_MCE
 BUILD_INTERRUPT(mce_self_interrupt,MCE_SELF_VECTOR)
 #endif
 
diff --git a/arch/x86/include/asm/fixmap.h b/arch/x86/include/asm/fixmap.h
index 7b2d71df39a6..14f9890eb495 100644
--- a/arch/x86/include/asm/fixmap.h
+++ b/arch/x86/include/asm/fixmap.h
@@ -132,6 +132,9 @@ enum fixed_addresses {
 #ifdef CONFIG_X86_32
         FIX_WP_TEST,
 #endif
+#ifdef CONFIG_INTEL_TXT
+        FIX_TBOOT_BASE,
+#endif
         __end_of_fixed_addresses
 };
 
diff --git a/arch/x86/include/asm/ftrace.h b/arch/x86/include/asm/ftrace.h
index bd2c6511c887..db24c2278be0 100644
--- a/arch/x86/include/asm/ftrace.h
+++ b/arch/x86/include/asm/ftrace.h
@@ -28,13 +28,6 @@
 
 #endif
 
-/* FIXME: I don't want to stay hardcoded */
-#ifdef CONFIG_X86_64
-# define FTRACE_SYSCALL_MAX     296
-#else
-# define FTRACE_SYSCALL_MAX     333
-#endif
-
 #ifdef CONFIG_FUNCTION_TRACER
 #define MCOUNT_ADDR             ((long)(mcount))
 #define MCOUNT_INSN_SIZE        5 /* sizeof mcount call */
diff --git a/arch/x86/include/asm/hypervisor.h b/arch/x86/include/asm/hypervisor.h
index 369f5c5d09a1..b78c0941e422 100644
--- a/arch/x86/include/asm/hypervisor.h
+++ b/arch/x86/include/asm/hypervisor.h
@@ -20,7 +20,7 @@
 #ifndef ASM_X86__HYPERVISOR_H
 #define ASM_X86__HYPERVISOR_H
 
-extern unsigned long get_hypervisor_tsc_freq(void);
 extern void init_hypervisor(struct cpuinfo_x86 *c);
+extern void init_hypervisor_platform(void);
 
 #endif
diff --git a/arch/x86/include/asm/i387.h b/arch/x86/include/asm/i387.h
index 175adf58dd4f..0b20bbb758f2 100644
--- a/arch/x86/include/asm/i387.h
+++ b/arch/x86/include/asm/i387.h
@@ -26,6 +26,7 @@ extern void fpu_init(void);
 extern void mxcsr_feature_mask_init(void);
 extern int init_fpu(struct task_struct *child);
 extern asmlinkage void math_state_restore(void);
+extern void __math_state_restore(void);
 extern void init_thread_xstate(void);
 extern int dump_fpu(struct pt_regs *, struct user_i387_struct *);
 
@@ -301,6 +302,14 @@ static inline void kernel_fpu_end(void)
         preempt_enable();
 }
 
+static inline bool irq_fpu_usable(void)
+{
+        struct pt_regs *regs;
+
+        return !in_interrupt() || !(regs = get_irq_regs()) || \
+                user_mode(regs) || (read_cr0() & X86_CR0_TS);
+}
+
 /*
  * Some instructions like VIA's padlock instructions generate a spurious
  * DNA fault but don't modify SSE registers. And these instructions
diff --git a/arch/x86/include/asm/io_apic.h b/arch/x86/include/asm/io_apic.h
index 330ee807f89e..7c7c16cde1f8 100644
--- a/arch/x86/include/asm/io_apic.h
+++ b/arch/x86/include/asm/io_apic.h
@@ -143,6 +143,8 @@ extern int noioapicreroute;
 /* 1 if the timer IRQ uses the '8259A Virtual Wire' mode */
 extern int timer_through_8259;
 
+extern void io_apic_disable_legacy(void);
+
 /*
  * If we use the IO-APIC for IRQ routing, disable automatic
  * assignment of PCI IRQ's.
@@ -150,11 +152,10 @@ extern int timer_through_8259;
 #define io_apic_assign_pci_irqs \
         (mp_irq_entries && !skip_ioapic_setup && io_apic_irqs)
 
-#ifdef CONFIG_ACPI
+extern u8 io_apic_unique_id(u8 id);
 extern int io_apic_get_unique_id(int ioapic, int apic_id);
 extern int io_apic_get_version(int ioapic);
 extern int io_apic_get_redir_entries(int ioapic);
-#endif /* CONFIG_ACPI */
 
 struct io_apic_irq_attr;
 extern int io_apic_set_pci_routing(struct device *dev, int irq,
@@ -177,13 +178,26 @@ extern int setup_ioapic_entry(int apic, int irq,
                               int polarity, int vector, int pin);
 extern void ioapic_write_entry(int apic, int pin,
                                struct IO_APIC_route_entry e);
+extern void setup_ioapic_ids_from_mpc(void);
+
+struct mp_ioapic_gsi{
+        int gsi_base;
+        int gsi_end;
+};
+extern struct mp_ioapic_gsi  mp_gsi_routing[];
+int mp_find_ioapic(int gsi);
+int mp_find_ioapic_pin(int ioapic, int gsi);
+void __init mp_register_ioapic(int id, u32 address, u32 gsi_base);
+
 #else  /* !CONFIG_X86_IO_APIC */
+
 #define io_apic_assign_pci_irqs 0
+#define setup_ioapic_ids_from_mpc x86_init_noop
 static const int timer_through_8259 = 0;
 static inline void ioapic_init_mappings(void)   { }
 static inline void ioapic_insert_resources(void) { }
-
 static inline void probe_nr_irqs_gsi(void)      { }
+
 #endif
 
 #endif /* _ASM_X86_IO_APIC_H */
diff --git a/arch/x86/include/asm/ioctls.h b/arch/x86/include/asm/ioctls.h
index 0d5b23b7b06e..ec34c760665e 100644
--- a/arch/x86/include/asm/ioctls.h
+++ b/arch/x86/include/asm/ioctls.h
@@ -1,94 +1 @@
-#ifndef _ASM_X86_IOCTLS_H
-#define _ASM_X86_IOCTLS_H
-
-#include <asm/ioctl.h>
-
-/* 0x54 is just a magic number to make these relatively unique ('T') */
-
-#define TCGETS          0x5401
-#define TCSETS          0x5402 /* Clashes with SNDCTL_TMR_START sound ioctl */
-#define TCSETSW         0x5403
-#define TCSETSF         0x5404
-#define TCGETA          0x5405
-#define TCSETA          0x5406
-#define TCSETAW         0x5407
-#define TCSETAF         0x5408
-#define TCSBRK          0x5409
-#define TCXONC          0x540A
-#define TCFLSH          0x540B
-#define TIOCEXCL        0x540C
-#define TIOCNXCL        0x540D
-#define TIOCSCTTY       0x540E
-#define TIOCGPGRP       0x540F
-#define TIOCSPGRP       0x5410
-#define TIOCOUTQ        0x5411
-#define TIOCSTI         0x5412
-#define TIOCGWINSZ      0x5413
-#define TIOCSWINSZ      0x5414
-#define TIOCMGET        0x5415
-#define TIOCMBIS        0x5416
-#define TIOCMBIC        0x5417
-#define TIOCMSET        0x5418
-#define TIOCGSOFTCAR    0x5419
-#define TIOCSSOFTCAR    0x541A
-#define FIONREAD        0x541B
-#define TIOCINQ         FIONREAD
-#define TIOCLINUX       0x541C
-#define TIOCCONS        0x541D
-#define TIOCGSERIAL     0x541E
-#define TIOCSSERIAL     0x541F
-#define TIOCPKT         0x5420
-#define FIONBIO         0x5421
-#define TIOCNOTTY       0x5422
-#define TIOCSETD        0x5423
-#define TIOCGETD        0x5424
-#define TCSBRKP         0x5425  /* Needed for POSIX tcsendbreak() */
-/* #define TIOCTTYGSTRUCT 0x5426 - Former debugging-only ioctl */
-#define TIOCSBRK        0x5427  /* BSD compatibility */
-#define TIOCCBRK        0x5428  /* BSD compatibility */
-#define TIOCGSID        0x5429  /* Return the session ID of FD */
-#define TCGETS2         _IOR('T', 0x2A, struct termios2)
-#define TCSETS2         _IOW('T', 0x2B, struct termios2)
-#define TCSETSW2        _IOW('T', 0x2C, struct termios2)
-#define TCSETSF2        _IOW('T', 0x2D, struct termios2)
-#define TIOCGRS485      0x542E
-#define TIOCSRS485      0x542F
-#define TIOCGPTN        _IOR('T', 0x30, unsigned int)
-                                /* Get Pty Number (of pty-mux device) */
-#define TIOCSPTLCK      _IOW('T', 0x31, int)  /* Lock/unlock Pty */
-#define TCGETX          0x5432 /* SYS5 TCGETX compatibility */
-#define TCSETX          0x5433
-#define TCSETXF         0x5434
-#define TCSETXW         0x5435
-
-#define FIONCLEX        0x5450
-#define FIOCLEX         0x5451
-#define FIOASYNC        0x5452
-#define TIOCSERCONFIG   0x5453
-#define TIOCSERGWILD    0x5454
-#define TIOCSERSWILD    0x5455
-#define TIOCGLCKTRMIOS  0x5456
-#define TIOCSLCKTRMIOS  0x5457
-#define TIOCSERGSTRUCT  0x5458 /* For debugging only */
-#define TIOCSERGETLSR   0x5459 /* Get line status register */
-#define TIOCSERGETMULTI 0x545A /* Get multiport config  */
-#define TIOCSERSETMULTI 0x545B /* Set multiport config */
-
-#define TIOCMIWAIT      0x545C  /* wait for a change on serial input line(s) */
-#define TIOCGICOUNT     0x545D  /* read serial port inline interrupt counts */
-#define TIOCGHAYESESP   0x545E  /* Get Hayes ESP configuration */
-#define TIOCSHAYESESP   0x545F  /* Set Hayes ESP configuration */
-#define FIOQSIZE        0x5460
-
-/* Used for packet mode */
-#define TIOCPKT_DATA             0
-#define TIOCPKT_FLUSHREAD        1
-#define TIOCPKT_FLUSHWRITE       2
-#define TIOCPKT_STOP             4
-#define TIOCPKT_START            8
-#define TIOCPKT_NOSTOP          16
-#define TIOCPKT_DOSTOP          32
-
-#define TIOCSER_TEMT    0x01    /* Transmitter physically empty */
-
-#endif /* _ASM_X86_IOCTLS_H */
+#include <asm-generic/ioctls.h>
diff --git a/arch/x86/include/asm/iomap.h b/arch/x86/include/asm/iomap.h
index 0e9fe1d9d971..f35eb45d6576 100644
--- a/arch/x86/include/asm/iomap.h
+++ b/arch/x86/include/asm/iomap.h
@@ -26,13 +26,16 @@
 #include <asm/pgtable.h>
 #include <asm/tlbflush.h>
 
-int
-is_io_mapping_possible(resource_size_t base, unsigned long size);
-
 void *
 iomap_atomic_prot_pfn(unsigned long pfn, enum km_type type, pgprot_t prot);
 
 void
 iounmap_atomic(void *kvaddr, enum km_type type);
 
+int
+iomap_create_wc(resource_size_t base, unsigned long size, pgprot_t *prot);
+
+void
+iomap_free(resource_size_t base, unsigned long size);
+
 #endif /* _ASM_X86_IOMAP_H */
diff --git a/arch/x86/include/asm/ipcbuf.h b/arch/x86/include/asm/ipcbuf.h
index ee678fd51594..84c7e51cb6d0 100644
--- a/arch/x86/include/asm/ipcbuf.h
+++ b/arch/x86/include/asm/ipcbuf.h
@@ -1,28 +1 @@
-#ifndef _ASM_X86_IPCBUF_H
-#define _ASM_X86_IPCBUF_H
-
-/*
- * The ipc64_perm structure for x86 architecture.
- * Note extra padding because this structure is passed back and forth
- * between kernel and user space.
- *
- * Pad space is left for:
- * - 32-bit mode_t and seq
- * - 2 miscellaneous 32-bit values
- */
-
-struct ipc64_perm {
-        __kernel_key_t          key;
-        __kernel_uid32_t        uid;
-        __kernel_gid32_t        gid;
-        __kernel_uid32_t        cuid;
-        __kernel_gid32_t        cgid;
-        __kernel_mode_t         mode;
-        unsigned short          __pad1;
-        unsigned short          seq;
-        unsigned short          __pad2;
-        unsigned long           __unused1;
-        unsigned long           __unused2;
-};
-
-#endif /* _ASM_X86_IPCBUF_H */
+#include <asm-generic/ipcbuf.h>
diff --git a/arch/x86/include/asm/irq.h b/arch/x86/include/asm/irq.h
index f38481bcd455..ddda6cbed6f4 100644
--- a/arch/x86/include/asm/irq.h
+++ b/arch/x86/include/asm/irq.h
@@ -37,7 +37,6 @@ extern void fixup_irqs(void);
 #endif
 
 extern void (*generic_interrupt_extension)(void);
-extern void init_IRQ(void);
 extern void native_init_IRQ(void);
 extern bool handle_irq(unsigned irq, struct pt_regs *regs);
 
@@ -47,4 +46,6 @@ extern unsigned int do_IRQ(struct pt_regs *regs);
 extern DECLARE_BITMAP(used_vectors, NR_VECTORS);
 extern int vector_used_by_percpu_irq(unsigned int vector);
 
+extern void init_ISA_irqs(void);
+
 #endif /* _ASM_X86_IRQ_H */
diff --git a/arch/x86/include/asm/irqflags.h b/arch/x86/include/asm/irqflags.h
index c6ccbe7e81ad..9e2b952f810a 100644
--- a/arch/x86/include/asm/irqflags.h
+++ b/arch/x86/include/asm/irqflags.h
@@ -13,14 +13,13 @@ static inline unsigned long native_save_fl(void)
         unsigned long flags;
 
         /*
-         * Note: this needs to be "=r" not "=rm", because we have the
-         * stack offset from what gcc expects at the time the "pop" is
-         * executed, and so a memory reference with respect to the stack
-         * would end up using the wrong address.
+         * "=rm" is safe here, because "pop" adjusts the stack before
+         * it evaluates its effective address -- this is part of the
+         * documented behavior of the "pop" instruction.
          */
         asm volatile("# __raw_save_flags\n\t"
                      "pushf ; pop %0"
-                     : "=r" (flags)
+                     : "=rm" (flags)
                      : /* no input */
                      : "memory");
 
diff --git a/arch/x86/include/asm/kvm.h b/arch/x86/include/asm/kvm.h
index 125be8b19568..4a5fe914dc59 100644
--- a/arch/x86/include/asm/kvm.h
+++ b/arch/x86/include/asm/kvm.h
@@ -17,6 +17,8 @@
 #define __KVM_HAVE_USER_NMI
 #define __KVM_HAVE_GUEST_DEBUG
 #define __KVM_HAVE_MSIX
+#define __KVM_HAVE_MCE
+#define __KVM_HAVE_PIT_STATE2
 
 /* Architectural interrupt line count. */
 #define KVM_NR_INTERRUPTS 256
@@ -236,6 +238,14 @@ struct kvm_pit_state {
         struct kvm_pit_channel_state channels[3];
 };
 
+#define KVM_PIT_FLAGS_HPET_LEGACY  0x00000001
+
+struct kvm_pit_state2 {
+        struct kvm_pit_channel_state channels[3];
+        __u32 flags;
+        __u32 reserved[9];
+};
+
 struct kvm_reinject_control {
         __u8 pit_reinject;
         __u8 reserved[31];
diff --git a/arch/x86/include/asm/kvm_x86_emulate.h b/arch/x86/include/asm/kvm_emulate.h
index b7ed2c423116..b7ed2c423116 100644
--- a/arch/x86/include/asm/kvm_x86_emulate.h
+++ b/arch/x86/include/asm/kvm_emulate.h
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index eabdc1cfab5c..3be000435fad 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -14,6 +14,7 @@
 #include <linux/types.h>
 #include <linux/mm.h>
 #include <linux/mmu_notifier.h>
+#include <linux/tracepoint.h>
 
 #include <linux/kvm.h>
 #include <linux/kvm_para.h>
@@ -37,12 +38,14 @@
 #define CR3_L_MODE_RESERVED_BITS (CR3_NONPAE_RESERVED_BITS |    \
                                   0xFFFFFF0000000000ULL)
 
-#define KVM_GUEST_CR0_MASK                                 \
-        (X86_CR0_PG | X86_CR0_PE | X86_CR0_WP | X86_CR0_NE \
-         | X86_CR0_NW | X86_CR0_CD)
+#define KVM_GUEST_CR0_MASK_UNRESTRICTED_GUEST                           \
+        (X86_CR0_WP | X86_CR0_NE | X86_CR0_NW | X86_CR0_CD)
+#define KVM_GUEST_CR0_MASK                                              \
+        (KVM_GUEST_CR0_MASK_UNRESTRICTED_GUEST | X86_CR0_PG | X86_CR0_PE)
+#define KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST                         \
+        (X86_CR0_WP | X86_CR0_NE | X86_CR0_TS | X86_CR0_MP)
 #define KVM_VM_CR0_ALWAYS_ON                                            \
-        (X86_CR0_PG | X86_CR0_PE | X86_CR0_WP | X86_CR0_NE | X86_CR0_TS \
-         | X86_CR0_MP)
+        (KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST | X86_CR0_PG | X86_CR0_PE)
 #define KVM_GUEST_CR4_MASK                                              \
         (X86_CR4_VME | X86_CR4_PSE | X86_CR4_PAE | X86_CR4_PGE | X86_CR4_VMXE)
 #define KVM_PMODE_VM_CR4_ALWAYS_ON (X86_CR4_PAE | X86_CR4_VMXE)
@@ -51,12 +54,12 @@
 #define INVALID_PAGE (~(hpa_t)0)
 #define UNMAPPED_GVA (~(gpa_t)0)
 
-/* shadow tables are PAE even on non-PAE hosts */
-#define KVM_HPAGE_SHIFT 21
-#define KVM_HPAGE_SIZE (1UL << KVM_HPAGE_SHIFT)
-#define KVM_HPAGE_MASK (~(KVM_HPAGE_SIZE - 1))
-
-#define KVM_PAGES_PER_HPAGE (KVM_HPAGE_SIZE / PAGE_SIZE)
+/* KVM Hugepage definitions for x86 */
+#define KVM_NR_PAGE_SIZES       3
+#define KVM_HPAGE_SHIFT(x)      (PAGE_SHIFT + (((x) - 1) * 9))
+#define KVM_HPAGE_SIZE(x)       (1UL << KVM_HPAGE_SHIFT(x))
+#define KVM_HPAGE_MASK(x)       (~(KVM_HPAGE_SIZE(x) - 1))
+#define KVM_PAGES_PER_HPAGE(x)  (KVM_HPAGE_SIZE(x) / PAGE_SIZE)
 
 #define DE_VECTOR 0
 #define DB_VECTOR 1
@@ -120,6 +123,10 @@ enum kvm_reg {
         NR_VCPU_REGS
 };
 
+enum kvm_reg_ex {
+        VCPU_EXREG_PDPTR = NR_VCPU_REGS,
+};
+
 enum {
         VCPU_SREG_ES,
         VCPU_SREG_CS,
@@ -131,7 +138,7 @@ enum {
         VCPU_SREG_LDTR,
 };
 
-#include <asm/kvm_x86_emulate.h>
+#include <asm/kvm_emulate.h>
 
 #define KVM_NR_MEM_OBJS 40
 
@@ -308,7 +315,6 @@ struct kvm_vcpu_arch {
         struct {
                 gfn_t gfn;      /* presumed gfn during guest pte update */
                 pfn_t pfn;      /* pfn corresponding to that gfn */
-                int largepage;
                 unsigned long mmu_seq;
         } update_pte;
 
@@ -334,16 +340,6 @@ struct kvm_vcpu_arch {
                 u8 nr;
         } interrupt;
 
-        struct {
-                int vm86_active;
-                u8 save_iopl;
-                struct kvm_save_segment {
-                        u16 selector;
-                        unsigned long base;
-                        u32 limit;
-                        u32 ar;
-                } tr, es, ds, fs, gs;
-        } rmode;
         int halt_request; /* real mode on Intel only */
 
         int cpuid_nent;
@@ -366,13 +362,15 @@ struct kvm_vcpu_arch {
         u32 pat;
 
         int switch_db_regs;
-        unsigned long host_db[KVM_NR_DB_REGS];
-        unsigned long host_dr6;
-        unsigned long host_dr7;
         unsigned long db[KVM_NR_DB_REGS];
         unsigned long dr6;
         unsigned long dr7;
         unsigned long eff_db[KVM_NR_DB_REGS];
+
+        u64 mcg_cap;
+        u64 mcg_status;
+        u64 mcg_ctl;
+        u64 *mce_banks;
 };
 
 struct kvm_mem_alias {
@@ -409,6 +407,7 @@ struct kvm_arch{
 
         struct page *ept_identity_pagetable;
         bool ept_identity_pagetable_done;
+        gpa_t ept_identity_map_addr;
 
         unsigned long irq_sources_bitmap;
         unsigned long irq_states[KVM_IOAPIC_NUM_PINS];
@@ -526,6 +525,9 @@ struct kvm_x86_ops {
         int (*set_tss_addr)(struct kvm *kvm, unsigned int addr);
         int (*get_tdp_level)(void);
         u64 (*get_mt_mask)(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio);
+        bool (*gb_page_enable)(void);
+
+        const struct trace_print_flags *exit_reasons_str;
 };
 
 extern struct kvm_x86_ops *kvm_x86_ops;
@@ -618,6 +620,7 @@ void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr);
 void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code);
 void kvm_inject_page_fault(struct kvm_vcpu *vcpu, unsigned long cr2,
                            u32 error_code);
+bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl);
 
 int kvm_pic_set_irq(void *opaque, int irq, int level);
 
@@ -752,8 +755,6 @@ static inline void kvm_inject_gp(struct kvm_vcpu *vcpu, u32 error_code)
         kvm_queue_exception_e(vcpu, GP_VECTOR, error_code);
 }
 
-#define MSR_IA32_TIME_STAMP_COUNTER             0x010
-
 #define TSS_IOPB_BASE_OFFSET 0x66
 #define TSS_BASE_SIZE 0x68
 #define TSS_IOPB_SIZE (65536 / 8)
@@ -796,5 +797,8 @@ asmlinkage void kvm_handle_fault_on_reboot(void);
 int kvm_unmap_hva(struct kvm *kvm, unsigned long hva);
 int kvm_age_hva(struct kvm *kvm, unsigned long hva);
 int cpuid_maxphyaddr(struct kvm_vcpu *vcpu);
+int kvm_cpu_has_interrupt(struct kvm_vcpu *vcpu);
+int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu);
+int kvm_cpu_get_interrupt(struct kvm_vcpu *v);
 
 #endif /* _ASM_X86_KVM_HOST_H */
diff --git a/arch/x86/include/asm/kvm_para.h b/arch/x86/include/asm/kvm_para.h
index b8a3305ae093..c584076a47f4 100644
--- a/arch/x86/include/asm/kvm_para.h
+++ b/arch/x86/include/asm/kvm_para.h
@@ -1,6 +1,8 @@
 #ifndef _ASM_X86_KVM_PARA_H
 #define _ASM_X86_KVM_PARA_H
 
+#include <linux/types.h>
+
 /* This CPUID returns the signature 'KVMKVMKVM' in ebx, ecx, and edx.  It
  * should be used to determine that a VM is running under KVM.
  */
diff --git a/arch/x86/include/asm/lguest.h b/arch/x86/include/asm/lguest.h
index 5136dad57cbb..0d97deba1e35 100644
--- a/arch/x86/include/asm/lguest.h
+++ b/arch/x86/include/asm/lguest.h
@@ -90,8 +90,9 @@ static inline void lguest_set_ts(void)
 }
 
 /* Full 4G segment descriptors, suitable for CS and DS. */
-#define FULL_EXEC_SEGMENT ((struct desc_struct){ { {0x0000ffff, 0x00cf9b00} } })
-#define FULL_SEGMENT ((struct desc_struct){ { {0x0000ffff, 0x00cf9300} } })
+#define FULL_EXEC_SEGMENT \
+        ((struct desc_struct)GDT_ENTRY_INIT(0xc09b, 0, 0xfffff))
+#define FULL_SEGMENT ((struct desc_struct)GDT_ENTRY_INIT(0xc093, 0, 0xfffff))
 
 #endif /* __ASSEMBLY__ */
 
diff --git a/arch/x86/include/asm/mce.h b/arch/x86/include/asm/mce.h
index 5cdd8d100ec9..b608a64c5814 100644
--- a/arch/x86/include/asm/mce.h
+++ b/arch/x86/include/asm/mce.h
@@ -9,7 +9,7 @@
  */
 
 #define MCG_BANKCNT_MASK        0xff         /* Number of Banks */
-#define MCG_CTL_P               (1ULL<<8)    /* MCG_CAP register available */
+#define MCG_CTL_P               (1ULL<<8)    /* MCG_CTL register available */
 #define MCG_EXT_P               (1ULL<<9)    /* Extended registers available */
 #define MCG_CMCI_P              (1ULL<<10)   /* CMCI supported */
 #define MCG_EXT_CNT_MASK        0xff0000     /* Number of Extended registers */
@@ -38,6 +38,14 @@
 #define MCM_ADDR_MEM     3      /* memory address */
 #define MCM_ADDR_GENERIC 7      /* generic */
 
+#define MCJ_CTX_MASK            3
+#define MCJ_CTX(flags)          ((flags) & MCJ_CTX_MASK)
+#define MCJ_CTX_RANDOM          0    /* inject context: random */
+#define MCJ_CTX_PROCESS         1    /* inject context: process */
+#define MCJ_CTX_IRQ             2    /* inject context: IRQ */
+#define MCJ_NMI_BROADCAST       4    /* do NMI broadcasting */
+#define MCJ_EXCEPTION           8    /* raise as exception */
+
 /* Fields are zero when not available */
 struct mce {
         __u64 status;
@@ -48,8 +56,8 @@ struct mce {
         __u64 tsc;      /* cpu time stamp counter */
         __u64 time;     /* wall time_t when error was detected */
         __u8  cpuvendor;        /* cpu vendor as encoded in system.h */
-        __u8  pad1;
-        __u16 pad2;
+        __u8  inject_flags;     /* software inject flags */
+        __u16  pad;
         __u32 cpuid;    /* CPUID 1 EAX */
         __u8  cs;               /* code segment */
         __u8  bank;     /* machine check bank */
@@ -115,13 +123,6 @@ void mcheck_init(struct cpuinfo_x86 *c);
 static inline void mcheck_init(struct cpuinfo_x86 *c) {}
 #endif
 
-#ifdef CONFIG_X86_OLD_MCE
-extern int nr_mce_banks;
-void amd_mcheck_init(struct cpuinfo_x86 *c);
-void intel_p4_mcheck_init(struct cpuinfo_x86 *c);
-void intel_p6_mcheck_init(struct cpuinfo_x86 *c);
-#endif
-
 #ifdef CONFIG_X86_ANCIENT_MCE
 void intel_p5_mcheck_init(struct cpuinfo_x86 *c);
 void winchip_mcheck_init(struct cpuinfo_x86 *c);
@@ -137,10 +138,11 @@ void mce_log(struct mce *m);
 DECLARE_PER_CPU(struct sys_device, mce_dev);
 
 /*
- * To support more than 128 would need to escape the predefined
- * Linux defined extended banks first.
+ * Maximum banks number.
+ * This is the limit of the current register layout on
+ * Intel CPUs.
  */
-#define MAX_NR_BANKS (MCE_EXTENDED_BANK - 1)
+#define MAX_NR_BANKS 32
 
 #ifdef CONFIG_X86_MCE_INTEL
 extern int mce_cmci_disabled;
@@ -208,11 +210,7 @@ extern void (*threshold_cpu_callback)(unsigned long action, unsigned int cpu);
 
 void intel_init_thermal(struct cpuinfo_x86 *c);
 
-#ifdef CONFIG_X86_NEW_MCE
 void mce_log_therm_throt_event(__u64 status);
-#else
-static inline void mce_log_therm_throt_event(__u64 status) {}
-#endif
 
 #endif /* __KERNEL__ */
 #endif /* _ASM_X86_MCE_H */
diff --git a/arch/x86/include/asm/mman.h b/arch/x86/include/asm/mman.h
index 751af2550ed9..593e51d4643f 100644
--- a/arch/x86/include/asm/mman.h
+++ b/arch/x86/include/asm/mman.h
@@ -1,20 +1,8 @@
 #ifndef _ASM_X86_MMAN_H
 #define _ASM_X86_MMAN_H
 
-#include <asm-generic/mman-common.h>
-
 #define MAP_32BIT       0x40            /* only give out 32bit addresses */
 
-#define MAP_GROWSDOWN   0x0100          /* stack-like segment */
-#define MAP_DENYWRITE   0x0800          /* ETXTBSY */
-#define MAP_EXECUTABLE  0x1000          /* mark it as an executable */
-#define MAP_LOCKED      0x2000          /* pages are locked */
-#define MAP_NORESERVE   0x4000          /* don't check for reservations */
-#define MAP_POPULATE    0x8000          /* populate (prefault) pagetables */
-#define MAP_NONBLOCK    0x10000         /* do not block on IO */
-#define MAP_STACK       0x20000         /* give out an address that is best suited for process/thread stacks */
-
-#define MCL_CURRENT     1               /* lock all current mappings */
-#define MCL_FUTURE      2               /* lock all future mappings */
+#include <asm-generic/mman.h>
 
 #endif /* _ASM_X86_MMAN_H */
diff --git a/arch/x86/include/asm/mmu_context.h b/arch/x86/include/asm/mmu_context.h
index f923203dc39a..4a2d4e0c18d9 100644
--- a/arch/x86/include/asm/mmu_context.h
+++ b/arch/x86/include/asm/mmu_context.h
@@ -37,12 +37,12 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
 
         if (likely(prev != next)) {
                 /* stop flush ipis for the previous mm */
-                cpu_clear(cpu, prev->cpu_vm_mask);
+                cpumask_clear_cpu(cpu, mm_cpumask(prev));
 #ifdef CONFIG_SMP
                 percpu_write(cpu_tlbstate.state, TLBSTATE_OK);
                 percpu_write(cpu_tlbstate.active_mm, next);
 #endif
-                cpu_set(cpu, next->cpu_vm_mask);
+                cpumask_set_cpu(cpu, mm_cpumask(next));
 
                 /* Re-load page tables */
                 load_cr3(next->pgd);
@@ -58,7 +58,7 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
                 percpu_write(cpu_tlbstate.state, TLBSTATE_OK);
                 BUG_ON(percpu_read(cpu_tlbstate.active_mm) != next);
 
-                if (!cpu_test_and_set(cpu, next->cpu_vm_mask)) {
+                if (!cpumask_test_and_set_cpu(cpu, mm_cpumask(next))) {
                         /* We were in lazy tlb mode and leave_mm disabled
                          * tlb flush IPI delivery. We must reload CR3
                          * to make sure to use no freed page tables.
diff --git a/arch/x86/include/asm/module.h b/arch/x86/include/asm/module.h
index 47d62743c4d5..3e2ce58a31a3 100644
--- a/arch/x86/include/asm/module.h
+++ b/arch/x86/include/asm/module.h
@@ -1,18 +1,7 @@
 #ifndef _ASM_X86_MODULE_H
 #define _ASM_X86_MODULE_H
 
-/* x86_32/64 are simple */
-struct mod_arch_specific {};
-
-#ifdef CONFIG_X86_32
-# define Elf_Shdr Elf32_Shdr
-# define Elf_Sym Elf32_Sym
-# define Elf_Ehdr Elf32_Ehdr
-#else
-# define Elf_Shdr Elf64_Shdr
-# define Elf_Sym Elf64_Sym
-# define Elf_Ehdr Elf64_Ehdr
-#endif
+#include <asm-generic/module.h>
 
 #ifdef CONFIG_X86_64
 /* X86_64 does not define MODULE_PROC_FAMILY */
@@ -28,6 +17,8 @@ struct mod_arch_specific {};
 #define MODULE_PROC_FAMILY "586MMX "
 #elif defined CONFIG_MCORE2
 #define MODULE_PROC_FAMILY "CORE2 "
+#elif defined CONFIG_MATOM
+#define MODULE_PROC_FAMILY "ATOM "
 #elif defined CONFIG_M686
 #define MODULE_PROC_FAMILY "686 "
 #elif defined CONFIG_MPENTIUMII
diff --git a/arch/x86/include/asm/mpspec.h b/arch/x86/include/asm/mpspec.h
index e2a1bb6d71ea..79c94500c0bb 100644
--- a/arch/x86/include/asm/mpspec.h
+++ b/arch/x86/include/asm/mpspec.h
@@ -4,6 +4,7 @@
 #include <linux/init.h>
 
 #include <asm/mpspec_def.h>
+#include <asm/x86_init.h>
 
 extern int apic_version[MAX_APICS];
 extern int pic_mode;
@@ -41,9 +42,6 @@ extern int quad_local_to_mp_bus_id [NR_CPUS/4][4];
 
 #endif /* CONFIG_X86_64 */
 
-extern void early_find_smp_config(void);
-extern void early_get_smp_config(void);
-
 #if defined(CONFIG_MCA) || defined(CONFIG_EISA)
 extern int mp_bus_id_to_type[MAX_MP_BUSSES];
 #endif
@@ -52,20 +50,55 @@ extern DECLARE_BITMAP(mp_bus_not_pci, MAX_MP_BUSSES);
 
 extern unsigned int boot_cpu_physical_apicid;
 extern unsigned int max_physical_apicid;
-extern int smp_found_config;
 extern int mpc_default_type;
 extern unsigned long mp_lapic_addr;
 
-extern void get_smp_config(void);
+#ifdef CONFIG_X86_LOCAL_APIC
+extern int smp_found_config;
+#else
+# define smp_found_config 0
+#endif
+
+static inline void get_smp_config(void)
+{
+        x86_init.mpparse.get_smp_config(0);
+}
+
+static inline void early_get_smp_config(void)
+{
+        x86_init.mpparse.get_smp_config(1);
+}
+
+static inline void find_smp_config(void)
+{
+        x86_init.mpparse.find_smp_config(1);
+}
+
+static inline void early_find_smp_config(void)
+{
+        x86_init.mpparse.find_smp_config(0);
+}
 
 #ifdef CONFIG_X86_MPPARSE
-extern void find_smp_config(void);
 extern void early_reserve_e820_mpc_new(void);
 extern int enable_update_mptable;
+extern int default_mpc_apic_id(struct mpc_cpu *m);
+extern void default_smp_read_mpc_oem(struct mpc_table *mpc);
+# ifdef CONFIG_X86_IO_APIC
+extern void default_mpc_oem_bus_info(struct mpc_bus *m, char *str);
+# else
+#  define default_mpc_oem_bus_info NULL
+# endif
+extern void default_find_smp_config(unsigned int reserve);
+extern void default_get_smp_config(unsigned int early);
 #else
-static inline void find_smp_config(void) { }
 static inline void early_reserve_e820_mpc_new(void) { }
 #define enable_update_mptable 0
+#define default_mpc_apic_id NULL
+#define default_smp_read_mpc_oem NULL
+#define default_mpc_oem_bus_info NULL
+#define default_find_smp_config x86_init_uint_noop
+#define default_get_smp_config x86_init_uint_noop
 #endif
 
 void __cpuinit generic_processor_info(int apicid, int version);
diff --git a/arch/x86/include/asm/msgbuf.h b/arch/x86/include/asm/msgbuf.h
index 7e4e9481f51c..809134c644a6 100644
--- a/arch/x86/include/asm/msgbuf.h
+++ b/arch/x86/include/asm/msgbuf.h
@@ -1,39 +1 @@
-#ifndef _ASM_X86_MSGBUF_H
-#define _ASM_X86_MSGBUF_H
-
-/*
- * The msqid64_ds structure for i386 architecture.
- * Note extra padding because this structure is passed back and forth
- * between kernel and user space.
- *
- * Pad space on i386 is left for:
- * - 64-bit time_t to solve y2038 problem
- * - 2 miscellaneous 32-bit values
- *
- * Pad space on x8664 is left for:
- * - 2 miscellaneous 64-bit values
- */
-struct msqid64_ds {
-        struct ipc64_perm msg_perm;
-        __kernel_time_t msg_stime;      /* last msgsnd time */
-#ifdef __i386__
-        unsigned long   __unused1;
-#endif
-        __kernel_time_t msg_rtime;      /* last msgrcv time */
-#ifdef __i386__
-        unsigned long   __unused2;
-#endif
-        __kernel_time_t msg_ctime;      /* last change time */
-#ifdef __i386__
-        unsigned long   __unused3;
-#endif
-        unsigned long  msg_cbytes;      /* current number of bytes on queue */
-        unsigned long  msg_qnum;        /* number of messages in queue */
-        unsigned long  msg_qbytes;      /* max number of bytes on queue */
-        __kernel_pid_t msg_lspid;       /* pid of last msgsnd */
-        __kernel_pid_t msg_lrpid;       /* last receive pid */
-        unsigned long  __unused4;
-        unsigned long  __unused5;
-};
-
-#endif /* _ASM_X86_MSGBUF_H */
+#include <asm-generic/msgbuf.h>
diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h
index 6be7fc254b59..4ffe09b2ad75 100644
--- a/arch/x86/include/asm/msr-index.h
+++ b/arch/x86/include/asm/msr-index.h
@@ -81,8 +81,15 @@
 #define MSR_IA32_MC0_ADDR               0x00000402
 #define MSR_IA32_MC0_MISC               0x00000403
 
+#define MSR_IA32_MCx_CTL(x)             (MSR_IA32_MC0_CTL + 4*(x))
+#define MSR_IA32_MCx_STATUS(x)          (MSR_IA32_MC0_STATUS + 4*(x))
+#define MSR_IA32_MCx_ADDR(x)            (MSR_IA32_MC0_ADDR + 4*(x))
+#define MSR_IA32_MCx_MISC(x)            (MSR_IA32_MC0_MISC + 4*(x))
+
 /* These are consecutive and not in the normal 4er MCE bank block */
 #define MSR_IA32_MC0_CTL2               0x00000280
+#define MSR_IA32_MCx_CTL2(x)            (MSR_IA32_MC0_CTL2 + (x))
+
 #define CMCI_EN                 (1ULL << 30)
 #define CMCI_THRESHOLD_MASK             0xffffULL
 
@@ -215,6 +222,10 @@
 
 #define THERM_STATUS_PROCHOT            (1 << 0)
 
+#define MSR_THERM2_CTL                  0x0000019d
+
+#define MSR_THERM2_CTL_TM_SELECT        (1ULL << 16)
+
 #define MSR_IA32_MISC_ENABLE            0x000001a0
 
 /* MISC_ENABLE bits: architectural */
@@ -374,6 +385,7 @@
 /* AMD-V MSRs */
 
 #define MSR_VM_CR                       0xc0010114
+#define MSR_VM_IGNNE                    0xc0010115
 #define MSR_VM_HSAVE_PA                 0xc0010117
 
 #endif /* _ASM_X86_MSR_INDEX_H */
diff --git a/arch/x86/include/asm/msr.h b/arch/x86/include/asm/msr.h
index 48ad9d29484a..7e2b6ba962ff 100644
--- a/arch/x86/include/asm/msr.h
+++ b/arch/x86/include/asm/msr.h
@@ -3,10 +3,16 @@
 
 #include <asm/msr-index.h>
 
-#ifdef __KERNEL__
 #ifndef __ASSEMBLY__
 
 #include <linux/types.h>
+#include <linux/ioctl.h>
+
+#define X86_IOC_RDMSR_REGS      _IOWR('c', 0xA0, __u32[8])
+#define X86_IOC_WRMSR_REGS      _IOWR('c', 0xA1, __u32[8])
+
+#ifdef __KERNEL__
+
 #include <asm/asm.h>
 #include <asm/errno.h>
 #include <asm/cpumask.h>
@@ -67,23 +73,7 @@ static inline unsigned long long native_read_msr_safe(unsigned int msr,
                      ".previous\n\t"
                      _ASM_EXTABLE(2b, 3b)
                      : [err] "=r" (*err), EAX_EDX_RET(val, low, high)
-                     : "c" (msr), [fault] "i" (-EFAULT));
-        return EAX_EDX_VAL(val, low, high);
-}
-
-static inline unsigned long long native_read_msr_amd_safe(unsigned int msr,
-                                                      int *err)
-{
-        DECLARE_ARGS(val, low, high);
-
-        asm volatile("2: rdmsr ; xor %0,%0\n"
-                     "1:\n\t"
-                     ".section .fixup,\"ax\"\n\t"
-                     "3:  mov %3,%0 ; jmp 1b\n\t"
-                     ".previous\n\t"
-                     _ASM_EXTABLE(2b, 3b)
-                     : "=r" (*err), EAX_EDX_RET(val, low, high)
-                     : "c" (msr), "D" (0x9c5a203a), "i" (-EFAULT));
+                     : "c" (msr), [fault] "i" (-EIO));
         return EAX_EDX_VAL(val, low, high);
 }
 
@@ -106,13 +96,16 @@ notrace static inline int native_write_msr_safe(unsigned int msr,
                      _ASM_EXTABLE(2b, 3b)
                      : [err] "=a" (err)
                      : "c" (msr), "0" (low), "d" (high),
-                       [fault] "i" (-EFAULT)
+                       [fault] "i" (-EIO)
                      : "memory");
         return err;
 }
 
 extern unsigned long long native_read_tsc(void);
 
+extern int native_rdmsr_safe_regs(u32 regs[8]);
+extern int native_wrmsr_safe_regs(u32 regs[8]);
+
 static __always_inline unsigned long long __native_read_tsc(void)
 {
         DECLARE_ARGS(val, low, high);
@@ -181,14 +174,44 @@ static inline int rdmsrl_safe(unsigned msr, unsigned long long *p)
         *p = native_read_msr_safe(msr, &err);
         return err;
 }
+
 static inline int rdmsrl_amd_safe(unsigned msr, unsigned long long *p)
 {
+        u32 gprs[8] = { 0 };
         int err;
 
-        *p = native_read_msr_amd_safe(msr, &err);
+        gprs[1] = msr;
+        gprs[7] = 0x9c5a203a;
+
+        err = native_rdmsr_safe_regs(gprs);
+
+        *p = gprs[0] | ((u64)gprs[2] << 32);
+
         return err;
 }
 
+static inline int wrmsrl_amd_safe(unsigned msr, unsigned long long val)
+{
+        u32 gprs[8] = { 0 };
+
+        gprs[0] = (u32)val;
+        gprs[1] = msr;
+        gprs[2] = val >> 32;
+        gprs[7] = 0x9c5a203a;
+
+        return native_wrmsr_safe_regs(gprs);
+}
+
+static inline int rdmsr_safe_regs(u32 regs[8])
+{
+        return native_rdmsr_safe_regs(regs);
+}
+
+static inline int wrmsr_safe_regs(u32 regs[8])
+{
+        return native_wrmsr_safe_regs(regs);
+}
+
 #define rdtscl(low)                                             \
         ((low) = (u32)__native_read_tsc())
 
@@ -228,6 +251,8 @@ void rdmsr_on_cpus(const cpumask_t *mask, u32 msr_no, struct msr *msrs);
 void wrmsr_on_cpus(const cpumask_t *mask, u32 msr_no, struct msr *msrs);
 int rdmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h);
 int wrmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h);
+int rdmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8]);
+int wrmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8]);
 #else  /*  CONFIG_SMP  */
 static inline int rdmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h)
 {
@@ -258,7 +283,15 @@ static inline int wrmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h)
 {
         return wrmsr_safe(msr_no, l, h);
 }
+static inline int rdmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8])
+{
+        return rdmsr_safe_regs(regs);
+}
+static inline int wrmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8])
+{
+        return wrmsr_safe_regs(regs);
+}
 #endif  /* CONFIG_SMP */
-#endif /* __ASSEMBLY__ */
 #endif /* __KERNEL__ */
+#endif /* __ASSEMBLY__ */
 #endif /* _ASM_X86_MSR_H */
diff --git a/arch/x86/include/asm/mtrr.h b/arch/x86/include/asm/mtrr.h
index a51ada8467de..4365ffdb461f 100644
--- a/arch/x86/include/asm/mtrr.h
+++ b/arch/x86/include/asm/mtrr.h
@@ -121,6 +121,9 @@ extern int mtrr_del_page(int reg, unsigned long base, unsigned long size);
 extern void mtrr_centaur_report_mcr(int mcr, u32 lo, u32 hi);
 extern void mtrr_ap_init(void);
 extern void mtrr_bp_init(void);
+extern void set_mtrr_aps_delayed_init(void);
+extern void mtrr_aps_init(void);
+extern void mtrr_bp_restore(void);
 extern int mtrr_trim_uncached_memory(unsigned long end_pfn);
 extern int amd_special_default_mtrr(void);
 #  else
@@ -161,6 +164,9 @@ static inline void mtrr_centaur_report_mcr(int mcr, u32 lo, u32 hi)
 
 #define mtrr_ap_init() do {} while (0)
 #define mtrr_bp_init() do {} while (0)
+#define set_mtrr_aps_delayed_init() do {} while (0)
+#define mtrr_aps_init() do {} while (0)
+#define mtrr_bp_restore() do {} while (0)
 #  endif
 
 #ifdef CONFIG_COMPAT
diff --git a/arch/x86/include/asm/nmi.h b/arch/x86/include/asm/nmi.h
index c86e5ed4af51..139d4c1a33a7 100644
--- a/arch/x86/include/asm/nmi.h
+++ b/arch/x86/include/asm/nmi.h
@@ -40,13 +40,12 @@ extern unsigned int nmi_watchdog;
 #define NMI_INVALID     3
 
 struct ctl_table;
-struct file;
-extern int proc_nmi_enabled(struct ctl_table *, int , struct file *,
+extern int proc_nmi_enabled(struct ctl_table *, int ,
                         void __user *, size_t *, loff_t *);
 extern int unknown_nmi_panic;
 
-void __trigger_all_cpu_backtrace(void);
-#define trigger_all_cpu_backtrace() __trigger_all_cpu_backtrace()
+void arch_trigger_all_cpu_backtrace(void);
+#define arch_trigger_all_cpu_backtrace arch_trigger_all_cpu_backtrace
 
 static inline void localise_nmi_watchdog(void)
 {
diff --git a/arch/x86/include/asm/nops.h b/arch/x86/include/asm/nops.h
index ad2668ee1aa7..6d8723a766cc 100644
--- a/arch/x86/include/asm/nops.h
+++ b/arch/x86/include/asm/nops.h
@@ -65,6 +65,8 @@
    6: osp nopl 0x00(%eax,%eax,1)
    7: nopl 0x00000000(%eax)
    8: nopl 0x00000000(%eax,%eax,1)
+   Note: All the above are assumed to be a single instruction.
+        There is kernel code that depends on this.
 */
 #define P6_NOP1 GENERIC_NOP1
 #define P6_NOP2 ".byte 0x66,0x90\n"
diff --git a/arch/x86/include/asm/param.h b/arch/x86/include/asm/param.h
index 6f0d0422f4ca..965d45427975 100644
--- a/arch/x86/include/asm/param.h
+++ b/arch/x86/include/asm/param.h
@@ -1,22 +1 @@
-#ifndef _ASM_X86_PARAM_H
-#define _ASM_X86_PARAM_H
-
-#ifdef __KERNEL__
-# define HZ             CONFIG_HZ       /* Internal kernel timer frequency */
-# define USER_HZ        100             /* some user interfaces are */
-# define CLOCKS_PER_SEC (USER_HZ)       /* in "ticks" like times() */
-#endif
-
-#ifndef HZ
-#define HZ 100
-#endif
-
-#define EXEC_PAGESIZE   4096
-
-#ifndef NOGROUP
-#define NOGROUP         (-1)
-#endif
-
-#define MAXHOSTNAMELEN  64      /* max length of hostname */
-
-#endif /* _ASM_X86_PARAM_H */
+#include <asm-generic/param.h>
diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h
index 4fb37c8a0832..8aebcc41041d 100644
--- a/arch/x86/include/asm/paravirt.h
+++ b/arch/x86/include/asm/paravirt.h
@@ -7,689 +7,11 @@
 #include <asm/pgtable_types.h>
 #include <asm/asm.h>
 
-/* Bitmask of what can be clobbered: usually at least eax. */
-#define CLBR_NONE 0
-#define CLBR_EAX  (1 << 0)
-#define CLBR_ECX  (1 << 1)
-#define CLBR_EDX  (1 << 2)
-#define CLBR_EDI  (1 << 3)
-
-#ifdef CONFIG_X86_32
-/* CLBR_ANY should match all regs platform has. For i386, that's just it */
-#define CLBR_ANY  ((1 << 4) - 1)
-
-#define CLBR_ARG_REGS   (CLBR_EAX | CLBR_EDX | CLBR_ECX)
-#define CLBR_RET_REG    (CLBR_EAX | CLBR_EDX)
-#define CLBR_SCRATCH    (0)
-#else
-#define CLBR_RAX  CLBR_EAX
-#define CLBR_RCX  CLBR_ECX
-#define CLBR_RDX  CLBR_EDX
-#define CLBR_RDI  CLBR_EDI
-#define CLBR_RSI  (1 << 4)
-#define CLBR_R8   (1 << 5)
-#define CLBR_R9   (1 << 6)
-#define CLBR_R10  (1 << 7)
-#define CLBR_R11  (1 << 8)
-
-#define CLBR_ANY  ((1 << 9) - 1)
-
-#define CLBR_ARG_REGS   (CLBR_RDI | CLBR_RSI | CLBR_RDX | \
-                         CLBR_RCX | CLBR_R8 | CLBR_R9)
-#define CLBR_RET_REG    (CLBR_RAX)
-#define CLBR_SCRATCH    (CLBR_R10 | CLBR_R11)
-
-#include <asm/desc_defs.h>
-#endif /* X86_64 */
-
-#define CLBR_CALLEE_SAVE ((CLBR_ARG_REGS | CLBR_SCRATCH) & ~CLBR_RET_REG)
+#include <asm/paravirt_types.h>
 
 #ifndef __ASSEMBLY__
 #include <linux/types.h>
 #include <linux/cpumask.h>
-#include <asm/kmap_types.h>
-#include <asm/desc_defs.h>
-
-struct page;
-struct thread_struct;
-struct desc_ptr;
-struct tss_struct;
-struct mm_struct;
-struct desc_struct;
-struct task_struct;
-
-/*
- * Wrapper type for pointers to code which uses the non-standard
- * calling convention.  See PV_CALL_SAVE_REGS_THUNK below.
- */
-struct paravirt_callee_save {
-        void *func;
-};
-
-/* general info */
-struct pv_info {
-        unsigned int kernel_rpl;
-        int shared_kernel_pmd;
-        int paravirt_enabled;
-        const char *name;
-};
-
-struct pv_init_ops {
-        /*
-         * Patch may replace one of the defined code sequences with
-         * arbitrary code, subject to the same register constraints.
-         * This generally means the code is not free to clobber any
-         * registers other than EAX.  The patch function should return
-         * the number of bytes of code generated, as we nop pad the
-         * rest in generic code.
-         */
-        unsigned (*patch)(u8 type, u16 clobber, void *insnbuf,
-                          unsigned long addr, unsigned len);
-
-        /* Basic arch-specific setup */
-        void (*arch_setup)(void);
-        char *(*memory_setup)(void);
-        void (*post_allocator_init)(void);
-
-        /* Print a banner to identify the environment */
-        void (*banner)(void);
-};
-
-
-struct pv_lazy_ops {
-        /* Set deferred update mode, used for batching operations. */
-        void (*enter)(void);
-        void (*leave)(void);
-};
-
-struct pv_time_ops {
-        void (*time_init)(void);
-
-        /* Set and set time of day */
-        unsigned long (*get_wallclock)(void);
-        int (*set_wallclock)(unsigned long);
-
-        unsigned long long (*sched_clock)(void);
-        unsigned long (*get_tsc_khz)(void);
-};
-
-struct pv_cpu_ops {
-        /* hooks for various privileged instructions */
-        unsigned long (*get_debugreg)(int regno);
-        void (*set_debugreg)(int regno, unsigned long value);
-
-        void (*clts)(void);
-
-        unsigned long (*read_cr0)(void);
-        void (*write_cr0)(unsigned long);
-
-        unsigned long (*read_cr4_safe)(void);
-        unsigned long (*read_cr4)(void);
-        void (*write_cr4)(unsigned long);
-
-#ifdef CONFIG_X86_64
-        unsigned long (*read_cr8)(void);
-        void (*write_cr8)(unsigned long);
-#endif
-
-        /* Segment descriptor handling */
-        void (*load_tr_desc)(void);
-        void (*load_gdt)(const struct desc_ptr *);
-        void (*load_idt)(const struct desc_ptr *);
-        void (*store_gdt)(struct desc_ptr *);
-        void (*store_idt)(struct desc_ptr *);
-        void (*set_ldt)(const void *desc, unsigned entries);
-        unsigned long (*store_tr)(void);
-        void (*load_tls)(struct thread_struct *t, unsigned int cpu);
-#ifdef CONFIG_X86_64
-        void (*load_gs_index)(unsigned int idx);
-#endif
-        void (*write_ldt_entry)(struct desc_struct *ldt, int entrynum,
-                                const void *desc);
-        void (*write_gdt_entry)(struct desc_struct *,
-                                int entrynum, const void *desc, int size);
-        void (*write_idt_entry)(gate_desc *,
-                                int entrynum, const gate_desc *gate);
-        void (*alloc_ldt)(struct desc_struct *ldt, unsigned entries);
-        void (*free_ldt)(struct desc_struct *ldt, unsigned entries);
-
-        void (*load_sp0)(struct tss_struct *tss, struct thread_struct *t);
-
-        void (*set_iopl_mask)(unsigned mask);
-
-        void (*wbinvd)(void);
-        void (*io_delay)(void);
-
-        /* cpuid emulation, mostly so that caps bits can be disabled */
-        void (*cpuid)(unsigned int *eax, unsigned int *ebx,
-                      unsigned int *ecx, unsigned int *edx);
-
-        /* MSR, PMC and TSR operations.
-           err = 0/-EFAULT.  wrmsr returns 0/-EFAULT. */
-        u64 (*read_msr_amd)(unsigned int msr, int *err);
-        u64 (*read_msr)(unsigned int msr, int *err);
-        int (*write_msr)(unsigned int msr, unsigned low, unsigned high);
-
-        u64 (*read_tsc)(void);
-        u64 (*read_pmc)(int counter);
-        unsigned long long (*read_tscp)(unsigned int *aux);
-
-        /*
-         * Atomically enable interrupts and return to userspace.  This
-         * is only ever used to return to 32-bit processes; in a
-         * 64-bit kernel, it's used for 32-on-64 compat processes, but
-         * never native 64-bit processes.  (Jump, not call.)
-         */
-        void (*irq_enable_sysexit)(void);
-
-        /*
-         * Switch to usermode gs and return to 64-bit usermode using
-         * sysret.  Only used in 64-bit kernels to return to 64-bit
-         * processes.  Usermode register state, including %rsp, must
-         * already be restored.
-         */
-        void (*usergs_sysret64)(void);
-
-        /*
-         * Switch to usermode gs and return to 32-bit usermode using
-         * sysret.  Used to return to 32-on-64 compat processes.
-         * Other usermode register state, including %esp, must already
-         * be restored.
-         */
-        void (*usergs_sysret32)(void);
-
-        /* Normal iret.  Jump to this with the standard iret stack
-           frame set up. */
-        void (*iret)(void);
-
-        void (*swapgs)(void);
-
-        void (*start_context_switch)(struct task_struct *prev);
-        void (*end_context_switch)(struct task_struct *next);
-};
-
-struct pv_irq_ops {
-        void (*init_IRQ)(void);
-
-        /*
-         * Get/set interrupt state.  save_fl and restore_fl are only
-         * expected to use X86_EFLAGS_IF; all other bits
-         * returned from save_fl are undefined, and may be ignored by
-         * restore_fl.
-         *
-         * NOTE: These functions callers expect the callee to preserve
-         * more registers than the standard C calling convention.
-         */
-        struct paravirt_callee_save save_fl;
-        struct paravirt_callee_save restore_fl;
-        struct paravirt_callee_save irq_disable;
-        struct paravirt_callee_save irq_enable;
-
-        void (*safe_halt)(void);
-        void (*halt)(void);
-
-#ifdef CONFIG_X86_64
-        void (*adjust_exception_frame)(void);
-#endif
-};
-
-struct pv_apic_ops {
-#ifdef CONFIG_X86_LOCAL_APIC
-        void (*setup_boot_clock)(void);
-        void (*setup_secondary_clock)(void);
-
-        void (*startup_ipi_hook)(int phys_apicid,
-                                 unsigned long start_eip,
-                                 unsigned long start_esp);
-#endif
-};
-
-struct pv_mmu_ops {
-        /*
-         * Called before/after init_mm pagetable setup. setup_start
-         * may reset %cr3, and may pre-install parts of the pagetable;
-         * pagetable setup is expected to preserve any existing
-         * mapping.
-         */
-        void (*pagetable_setup_start)(pgd_t *pgd_base);
-        void (*pagetable_setup_done)(pgd_t *pgd_base);
-
-        unsigned long (*read_cr2)(void);
-        void (*write_cr2)(unsigned long);
-
-        unsigned long (*read_cr3)(void);
-        void (*write_cr3)(unsigned long);
-
-        /*
-         * Hooks for intercepting the creation/use/destruction of an
-         * mm_struct.
-         */
-        void (*activate_mm)(struct mm_struct *prev,
-                            struct mm_struct *next);
-        void (*dup_mmap)(struct mm_struct *oldmm,
-                         struct mm_struct *mm);
-        void (*exit_mmap)(struct mm_struct *mm);
-
-
-        /* TLB operations */
-        void (*flush_tlb_user)(void);
-        void (*flush_tlb_kernel)(void);
-        void (*flush_tlb_single)(unsigned long addr);
-        void (*flush_tlb_others)(const struct cpumask *cpus,
-                                 struct mm_struct *mm,
-                                 unsigned long va);
-
-        /* Hooks for allocating and freeing a pagetable top-level */
-        int  (*pgd_alloc)(struct mm_struct *mm);
-        void (*pgd_free)(struct mm_struct *mm, pgd_t *pgd);
-
-        /*
-         * Hooks for allocating/releasing pagetable pages when they're
-         * attached to a pagetable
-         */
-        void (*alloc_pte)(struct mm_struct *mm, unsigned long pfn);
-        void (*alloc_pmd)(struct mm_struct *mm, unsigned long pfn);
-        void (*alloc_pmd_clone)(unsigned long pfn, unsigned long clonepfn, unsigned long start, unsigned long count);
-        void (*alloc_pud)(struct mm_struct *mm, unsigned long pfn);
-        void (*release_pte)(unsigned long pfn);
-        void (*release_pmd)(unsigned long pfn);
-        void (*release_pud)(unsigned long pfn);
-
-        /* Pagetable manipulation functions */
-        void (*set_pte)(pte_t *ptep, pte_t pteval);
-        void (*set_pte_at)(struct mm_struct *mm, unsigned long addr,
-                           pte_t *ptep, pte_t pteval);
-        void (*set_pmd)(pmd_t *pmdp, pmd_t pmdval);
-        void (*pte_update)(struct mm_struct *mm, unsigned long addr,
-                           pte_t *ptep);
-        void (*pte_update_defer)(struct mm_struct *mm,
-                                 unsigned long addr, pte_t *ptep);
-
-        pte_t (*ptep_modify_prot_start)(struct mm_struct *mm, unsigned long addr,
-                                        pte_t *ptep);
-        void (*ptep_modify_prot_commit)(struct mm_struct *mm, unsigned long addr,
-                                        pte_t *ptep, pte_t pte);
-
-        struct paravirt_callee_save pte_val;
-        struct paravirt_callee_save make_pte;
-
-        struct paravirt_callee_save pgd_val;
-        struct paravirt_callee_save make_pgd;
-
-#if PAGETABLE_LEVELS >= 3
-#ifdef CONFIG_X86_PAE
-        void (*set_pte_atomic)(pte_t *ptep, pte_t pteval);
-        void (*pte_clear)(struct mm_struct *mm, unsigned long addr,
-                          pte_t *ptep);
-        void (*pmd_clear)(pmd_t *pmdp);
-
-#endif  /* CONFIG_X86_PAE */
-
-        void (*set_pud)(pud_t *pudp, pud_t pudval);
-
-        struct paravirt_callee_save pmd_val;
-        struct paravirt_callee_save make_pmd;
-
-#if PAGETABLE_LEVELS == 4
-        struct paravirt_callee_save pud_val;
-        struct paravirt_callee_save make_pud;
-
-        void (*set_pgd)(pgd_t *pudp, pgd_t pgdval);
-#endif  /* PAGETABLE_LEVELS == 4 */
-#endif  /* PAGETABLE_LEVELS >= 3 */
-
-#ifdef CONFIG_HIGHPTE
-        void *(*kmap_atomic_pte)(struct page *page, enum km_type type);
-#endif
-
-        struct pv_lazy_ops lazy_mode;
-
-        /* dom0 ops */
-
-        /* Sometimes the physical address is a pfn, and sometimes its
-           an mfn.  We can tell which is which from the index. */
-        void (*set_fixmap)(unsigned /* enum fixed_addresses */ idx,
-                           phys_addr_t phys, pgprot_t flags);
-};
-
-struct raw_spinlock;
-struct pv_lock_ops {
-        int (*spin_is_locked)(struct raw_spinlock *lock);
-        int (*spin_is_contended)(struct raw_spinlock *lock);
-        void (*spin_lock)(struct raw_spinlock *lock);
-        void (*spin_lock_flags)(struct raw_spinlock *lock, unsigned long flags);
-        int (*spin_trylock)(struct raw_spinlock *lock);
-        void (*spin_unlock)(struct raw_spinlock *lock);
-};
-
-/* This contains all the paravirt structures: we get a convenient
- * number for each function using the offset which we use to indicate
- * what to patch. */
-struct paravirt_patch_template {
-        struct pv_init_ops pv_init_ops;
-        struct pv_time_ops pv_time_ops;
-        struct pv_cpu_ops pv_cpu_ops;
-        struct pv_irq_ops pv_irq_ops;
-        struct pv_apic_ops pv_apic_ops;
-        struct pv_mmu_ops pv_mmu_ops;
-        struct pv_lock_ops pv_lock_ops;
-};
-
-extern struct pv_info pv_info;
-extern struct pv_init_ops pv_init_ops;
-extern struct pv_time_ops pv_time_ops;
-extern struct pv_cpu_ops pv_cpu_ops;
-extern struct pv_irq_ops pv_irq_ops;
-extern struct pv_apic_ops pv_apic_ops;
-extern struct pv_mmu_ops pv_mmu_ops;
-extern struct pv_lock_ops pv_lock_ops;
-
-#define PARAVIRT_PATCH(x)                                       \
-        (offsetof(struct paravirt_patch_template, x) / sizeof(void *))
-
-#define paravirt_type(op)                               \
-        [paravirt_typenum] "i" (PARAVIRT_PATCH(op)),    \
-        [paravirt_opptr] "i" (&(op))
-#define paravirt_clobber(clobber)               \
-        [paravirt_clobber] "i" (clobber)
-
-/*
- * Generate some code, and mark it as patchable by the
- * apply_paravirt() alternate instruction patcher.
- */
-#define _paravirt_alt(insn_string, type, clobber)       \
-        "771:\n\t" insn_string "\n" "772:\n"            \
-        ".pushsection .parainstructions,\"a\"\n"        \
-        _ASM_ALIGN "\n"                                 \
-        _ASM_PTR " 771b\n"                              \
-        "  .byte " type "\n"                            \
-        "  .byte 772b-771b\n"                           \
-        "  .short " clobber "\n"                        \
-        ".popsection\n"
-
-/* Generate patchable code, with the default asm parameters. */
-#define paravirt_alt(insn_string)                                       \
-        _paravirt_alt(insn_string, "%c[paravirt_typenum]", "%c[paravirt_clobber]")
-
-/* Simple instruction patching code. */
-#define DEF_NATIVE(ops, name, code)                                     \
-        extern const char start_##ops##_##name[], end_##ops##_##name[]; \
-        asm("start_" #ops "_" #name ": " code "; end_" #ops "_" #name ":")
-
-unsigned paravirt_patch_nop(void);
-unsigned paravirt_patch_ident_32(void *insnbuf, unsigned len);
-unsigned paravirt_patch_ident_64(void *insnbuf, unsigned len);
-unsigned paravirt_patch_ignore(unsigned len);
-unsigned paravirt_patch_call(void *insnbuf,
-                             const void *target, u16 tgt_clobbers,
-                             unsigned long addr, u16 site_clobbers,
-                             unsigned len);
-unsigned paravirt_patch_jmp(void *insnbuf, const void *target,
-                            unsigned long addr, unsigned len);
-unsigned paravirt_patch_default(u8 type, u16 clobbers, void *insnbuf,
-                                unsigned long addr, unsigned len);
-
-unsigned paravirt_patch_insns(void *insnbuf, unsigned len,
-                              const char *start, const char *end);
-
-unsigned native_patch(u8 type, u16 clobbers, void *ibuf,
-                      unsigned long addr, unsigned len);
-
-int paravirt_disable_iospace(void);
-
-/*
- * This generates an indirect call based on the operation type number.
- * The type number, computed in PARAVIRT_PATCH, is derived from the
- * offset into the paravirt_patch_template structure, and can therefore be
- * freely converted back into a structure offset.
- */
-#define PARAVIRT_CALL   "call *%c[paravirt_opptr];"
-
-/*
- * These macros are intended to wrap calls through one of the paravirt
- * ops structs, so that they can be later identified and patched at
- * runtime.
- *
- * Normally, a call to a pv_op function is a simple indirect call:
- * (pv_op_struct.operations)(args...).
- *
- * Unfortunately, this is a relatively slow operation for modern CPUs,
- * because it cannot necessarily determine what the destination
- * address is.  In this case, the address is a runtime constant, so at
- * the very least we can patch the call to e a simple direct call, or
- * ideally, patch an inline implementation into the callsite.  (Direct
- * calls are essentially free, because the call and return addresses
- * are completely predictable.)
- *
- * For i386, these macros rely on the standard gcc "regparm(3)" calling
- * convention, in which the first three arguments are placed in %eax,
- * %edx, %ecx (in that order), and the remaining arguments are placed
- * on the stack.  All caller-save registers (eax,edx,ecx) are expected
- * to be modified (either clobbered or used for return values).
- * X86_64, on the other hand, already specifies a register-based calling
- * conventions, returning at %rax, with parameteres going on %rdi, %rsi,
- * %rdx, and %rcx. Note that for this reason, x86_64 does not need any
- * special handling for dealing with 4 arguments, unlike i386.
- * However, x86_64 also have to clobber all caller saved registers, which
- * unfortunately, are quite a bit (r8 - r11)
- *
- * The call instruction itself is marked by placing its start address
- * and size into the .parainstructions section, so that
- * apply_paravirt() in arch/i386/kernel/alternative.c can do the
- * appropriate patching under the control of the backend pv_init_ops
- * implementation.
- *
- * Unfortunately there's no way to get gcc to generate the args setup
- * for the call, and then allow the call itself to be generated by an
- * inline asm.  Because of this, we must do the complete arg setup and
- * return value handling from within these macros.  This is fairly
- * cumbersome.
- *
- * There are 5 sets of PVOP_* macros for dealing with 0-4 arguments.
- * It could be extended to more arguments, but there would be little
- * to be gained from that.  For each number of arguments, there are
- * the two VCALL and CALL variants for void and non-void functions.
- *
- * When there is a return value, the invoker of the macro must specify
- * the return type.  The macro then uses sizeof() on that type to
- * determine whether its a 32 or 64 bit value, and places the return
- * in the right register(s) (just %eax for 32-bit, and %edx:%eax for
- * 64-bit). For x86_64 machines, it just returns at %rax regardless of
- * the return value size.
- *
- * 64-bit arguments are passed as a pair of adjacent 32-bit arguments
- * i386 also passes 64-bit arguments as a pair of adjacent 32-bit arguments
- * in low,high order
- *
- * Small structures are passed and returned in registers.  The macro
- * calling convention can't directly deal with this, so the wrapper
- * functions must do this.
- *
- * These PVOP_* macros are only defined within this header.  This
- * means that all uses must be wrapped in inline functions.  This also
- * makes sure the incoming and outgoing types are always correct.
- */
-#ifdef CONFIG_X86_32
-#define PVOP_VCALL_ARGS                         \
-        unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx
-#define PVOP_CALL_ARGS                  PVOP_VCALL_ARGS
-
-#define PVOP_CALL_ARG1(x)               "a" ((unsigned long)(x))
-#define PVOP_CALL_ARG2(x)               "d" ((unsigned long)(x))
-#define PVOP_CALL_ARG3(x)               "c" ((unsigned long)(x))
-
-#define PVOP_VCALL_CLOBBERS             "=a" (__eax), "=d" (__edx),     \
-                                        "=c" (__ecx)
-#define PVOP_CALL_CLOBBERS              PVOP_VCALL_CLOBBERS
-
-#define PVOP_VCALLEE_CLOBBERS           "=a" (__eax), "=d" (__edx)
-#define PVOP_CALLEE_CLOBBERS            PVOP_VCALLEE_CLOBBERS
-
-#define EXTRA_CLOBBERS
-#define VEXTRA_CLOBBERS
-#else  /* CONFIG_X86_64 */
-#define PVOP_VCALL_ARGS                                 \
-        unsigned long __edi = __edi, __esi = __esi,     \
-                __edx = __edx, __ecx = __ecx
-#define PVOP_CALL_ARGS          PVOP_VCALL_ARGS, __eax
-
-#define PVOP_CALL_ARG1(x)               "D" ((unsigned long)(x))
-#define PVOP_CALL_ARG2(x)               "S" ((unsigned long)(x))
-#define PVOP_CALL_ARG3(x)               "d" ((unsigned long)(x))
-#define PVOP_CALL_ARG4(x)               "c" ((unsigned long)(x))
-
-#define PVOP_VCALL_CLOBBERS     "=D" (__edi),                           \
-                                "=S" (__esi), "=d" (__edx),             \
-                                "=c" (__ecx)
-#define PVOP_CALL_CLOBBERS      PVOP_VCALL_CLOBBERS, "=a" (__eax)
-
-#define PVOP_VCALLEE_CLOBBERS   "=a" (__eax)
-#define PVOP_CALLEE_CLOBBERS    PVOP_VCALLEE_CLOBBERS
-
-#define EXTRA_CLOBBERS   , "r8", "r9", "r10", "r11"
-#define VEXTRA_CLOBBERS  , "rax", "r8", "r9", "r10", "r11"
-#endif  /* CONFIG_X86_32 */
-
-#ifdef CONFIG_PARAVIRT_DEBUG
-#define PVOP_TEST_NULL(op)      BUG_ON(op == NULL)
-#else
-#define PVOP_TEST_NULL(op)      ((void)op)
-#endif
-
-#define ____PVOP_CALL(rettype, op, clbr, call_clbr, extra_clbr,         \
-                      pre, post, ...)                                   \
-        ({                                                              \
-                rettype __ret;                                          \
-                PVOP_CALL_ARGS;                                         \
-                PVOP_TEST_NULL(op);                                     \
-                /* This is 32-bit specific, but is okay in 64-bit */    \
-                /* since this condition will never hold */              \
-                if (sizeof(rettype) > sizeof(unsigned long)) {          \
-                        asm volatile(pre                                \
-                                     paravirt_alt(PARAVIRT_CALL)        \
-                                     post                               \
-                                     : call_clbr                        \
-                                     : paravirt_type(op),               \
-                                       paravirt_clobber(clbr),          \
-                                       ##__VA_ARGS__                    \
-                                     : "memory", "cc" extra_clbr);      \
-                        __ret = (rettype)((((u64)__edx) << 32) | __eax); \
-                } else {                                                \
-                        asm volatile(pre                                \
-                                     paravirt_alt(PARAVIRT_CALL)        \
-                                     post                               \
-                                     : call_clbr                        \
-                                     : paravirt_type(op),               \
-                                       paravirt_clobber(clbr),          \
-                                       ##__VA_ARGS__                    \
-                                     : "memory", "cc" extra_clbr);      \
-                        __ret = (rettype)__eax;                         \
-                }                                                       \
-                __ret;                                                  \
-        })
-
-#define __PVOP_CALL(rettype, op, pre, post, ...)                        \
-        ____PVOP_CALL(rettype, op, CLBR_ANY, PVOP_CALL_CLOBBERS,        \
-                      EXTRA_CLOBBERS, pre, post, ##__VA_ARGS__)
-
-#define __PVOP_CALLEESAVE(rettype, op, pre, post, ...)                  \
-        ____PVOP_CALL(rettype, op.func, CLBR_RET_REG,                   \
-                      PVOP_CALLEE_CLOBBERS, ,                           \
-                      pre, post, ##__VA_ARGS__)
-
-
-#define ____PVOP_VCALL(op, clbr, call_clbr, extra_clbr, pre, post, ...) \
-        ({                                                              \
-                PVOP_VCALL_ARGS;                                        \
-                PVOP_TEST_NULL(op);                                     \
-                asm volatile(pre                                        \
-                             paravirt_alt(PARAVIRT_CALL)                \
-                             post                                       \
-                             : call_clbr                                \
-                             : paravirt_type(op),                       \
-                               paravirt_clobber(clbr),                  \
-                               ##__VA_ARGS__                            \
-                             : "memory", "cc" extra_clbr);              \
-        })
-
-#define __PVOP_VCALL(op, pre, post, ...)                                \
-        ____PVOP_VCALL(op, CLBR_ANY, PVOP_VCALL_CLOBBERS,               \
-                       VEXTRA_CLOBBERS,                                 \
-                       pre, post, ##__VA_ARGS__)
-
-#define __PVOP_VCALLEESAVE(rettype, op, pre, post, ...)                 \
-        ____PVOP_CALL(rettype, op.func, CLBR_RET_REG,                   \
-                      PVOP_VCALLEE_CLOBBERS, ,                          \
-                      pre, post, ##__VA_ARGS__)
-
-
-
-#define PVOP_CALL0(rettype, op)                                         \
-        __PVOP_CALL(rettype, op, "", "")
-#define PVOP_VCALL0(op)                                                 \
-        __PVOP_VCALL(op, "", "")
-
-#define PVOP_CALLEE0(rettype, op)                                       \
-        __PVOP_CALLEESAVE(rettype, op, "", "")
-#define PVOP_VCALLEE0(op)                                               \
-        __PVOP_VCALLEESAVE(op, "", "")
-
-
-#define PVOP_CALL1(rettype, op, arg1)                                   \
-        __PVOP_CALL(rettype, op, "", "", PVOP_CALL_ARG1(arg1))
-#define PVOP_VCALL1(op, arg1)                                           \
-        __PVOP_VCALL(op, "", "", PVOP_CALL_ARG1(arg1))
-
-#define PVOP_CALLEE1(rettype, op, arg1)                                 \
-        __PVOP_CALLEESAVE(rettype, op, "", "", PVOP_CALL_ARG1(arg1))
-#define PVOP_VCALLEE1(op, arg1)                                         \
-        __PVOP_VCALLEESAVE(op, "", "", PVOP_CALL_ARG1(arg1))
-
-
-#define PVOP_CALL2(rettype, op, arg1, arg2)                             \
-        __PVOP_CALL(rettype, op, "", "", PVOP_CALL_ARG1(arg1),          \
-                    PVOP_CALL_ARG2(arg2))
-#define PVOP_VCALL2(op, arg1, arg2)                                     \
-        __PVOP_VCALL(op, "", "", PVOP_CALL_ARG1(arg1),                  \
-                     PVOP_CALL_ARG2(arg2))
-
-#define PVOP_CALLEE2(rettype, op, arg1, arg2)                           \
-        __PVOP_CALLEESAVE(rettype, op, "", "", PVOP_CALL_ARG1(arg1),    \
-                          PVOP_CALL_ARG2(arg2))
-#define PVOP_VCALLEE2(op, arg1, arg2)                                   \
-        __PVOP_VCALLEESAVE(op, "", "", PVOP_CALL_ARG1(arg1),            \
-                           PVOP_CALL_ARG2(arg2))
-
-
-#define PVOP_CALL3(rettype, op, arg1, arg2, arg3)                       \
-        __PVOP_CALL(rettype, op, "", "", PVOP_CALL_ARG1(arg1),          \
-                    PVOP_CALL_ARG2(arg2), PVOP_CALL_ARG3(arg3))
-#define PVOP_VCALL3(op, arg1, arg2, arg3)                               \
-        __PVOP_VCALL(op, "", "", PVOP_CALL_ARG1(arg1),                  \
-                     PVOP_CALL_ARG2(arg2), PVOP_CALL_ARG3(arg3))
-
-/* This is the only difference in x86_64. We can make it much simpler */
-#ifdef CONFIG_X86_32
-#define PVOP_CALL4(rettype, op, arg1, arg2, arg3, arg4)                 \
-        __PVOP_CALL(rettype, op,                                        \
-                    "push %[_arg4];", "lea 4(%%esp),%%esp;",            \
-                    PVOP_CALL_ARG1(arg1), PVOP_CALL_ARG2(arg2),         \
-                    PVOP_CALL_ARG3(arg3), [_arg4] "mr" ((u32)(arg4)))
-#define PVOP_VCALL4(op, arg1, arg2, arg3, arg4)                         \
-        __PVOP_VCALL(op,                                                \
-                    "push %[_arg4];", "lea 4(%%esp),%%esp;",            \
-                    "0" ((u32)(arg1)), "1" ((u32)(arg2)),               \
-                    "2" ((u32)(arg3)), [_arg4] "mr" ((u32)(arg4)))
-#else
-#define PVOP_CALL4(rettype, op, arg1, arg2, arg3, arg4)                 \
-        __PVOP_CALL(rettype, op, "", "",                                \
-                    PVOP_CALL_ARG1(arg1), PVOP_CALL_ARG2(arg2),         \
-                    PVOP_CALL_ARG3(arg3), PVOP_CALL_ARG4(arg4))
-#define PVOP_VCALL4(op, arg1, arg2, arg3, arg4)                         \
-        __PVOP_VCALL(op, "", "",                                        \
-                     PVOP_CALL_ARG1(arg1), PVOP_CALL_ARG2(arg2),        \
-                     PVOP_CALL_ARG3(arg3), PVOP_CALL_ARG4(arg4))
-#endif
 
 static inline int paravirt_enabled(void)
 {
@@ -702,22 +24,6 @@ static inline void load_sp0(struct tss_struct *tss,
         PVOP_VCALL2(pv_cpu_ops.load_sp0, tss, thread);
 }
 
-#define ARCH_SETUP                      pv_init_ops.arch_setup();
-static inline unsigned long get_wallclock(void)
-{
-        return PVOP_CALL0(unsigned long, pv_time_ops.get_wallclock);
-}
-
-static inline int set_wallclock(unsigned long nowtime)
-{
-        return PVOP_CALL1(int, pv_time_ops.set_wallclock, nowtime);
-}
-
-static inline void (*choose_time_init(void))(void)
-{
-        return pv_time_ops.time_init;
-}
-
 /* The paravirtualized CPUID instruction. */
 static inline void __cpuid(unsigned int *eax, unsigned int *ebx,
                            unsigned int *ecx, unsigned int *edx)
@@ -820,15 +126,22 @@ static inline u64 paravirt_read_msr(unsigned msr, int *err)
 {
         return PVOP_CALL2(u64, pv_cpu_ops.read_msr, msr, err);
 }
-static inline u64 paravirt_read_msr_amd(unsigned msr, int *err)
+
+static inline int paravirt_rdmsr_regs(u32 *regs)
 {
-        return PVOP_CALL2(u64, pv_cpu_ops.read_msr_amd, msr, err);
+        return PVOP_CALL1(int, pv_cpu_ops.rdmsr_regs, regs);
 }
+
 static inline int paravirt_write_msr(unsigned msr, unsigned low, unsigned high)
 {
         return PVOP_CALL3(int, pv_cpu_ops.write_msr, msr, low, high);
 }
 
+static inline int paravirt_wrmsr_regs(u32 *regs)
+{
+        return PVOP_CALL1(int, pv_cpu_ops.wrmsr_regs, regs);
+}
+
 /* These should all do BUG_ON(_err), but our headers are too tangled. */
 #define rdmsr(msr, val1, val2)                  \
 do {                                            \
@@ -862,6 +175,9 @@ do {						\
         _err;                                   \
 })
 
+#define rdmsr_safe_regs(regs)   paravirt_rdmsr_regs(regs)
+#define wrmsr_safe_regs(regs)   paravirt_wrmsr_regs(regs)
+
 static inline int rdmsrl_safe(unsigned msr, unsigned long long *p)
 {
         int err;
@@ -871,12 +187,31 @@ static inline int rdmsrl_safe(unsigned msr, unsigned long long *p)
 }
 static inline int rdmsrl_amd_safe(unsigned msr, unsigned long long *p)
 {
+        u32 gprs[8] = { 0 };
         int err;
 
-        *p = paravirt_read_msr_amd(msr, &err);
+        gprs[1] = msr;
+        gprs[7] = 0x9c5a203a;
+
+        err = paravirt_rdmsr_regs(gprs);
+
+        *p = gprs[0] | ((u64)gprs[2] << 32);
+
         return err;
 }
 
+static inline int wrmsrl_amd_safe(unsigned msr, unsigned long long val)
+{
+        u32 gprs[8] = { 0 };
+
+        gprs[0] = (u32)val;
+        gprs[1] = msr;
+        gprs[2] = val >> 32;
+        gprs[7] = 0x9c5a203a;
+
+        return paravirt_wrmsr_regs(gprs);
+}
+
 static inline u64 paravirt_read_tsc(void)
 {
         return PVOP_CALL0(u64, pv_cpu_ops.read_tsc);
@@ -894,7 +229,6 @@ static inline unsigned long long paravirt_sched_clock(void)
 {
         return PVOP_CALL0(unsigned long long, pv_time_ops.sched_clock);
 }
-#define calibrate_tsc() (pv_time_ops.get_tsc_khz())
 
 static inline unsigned long long paravirt_read_pmc(int counter)
 {
@@ -1012,34 +346,6 @@ static inline void slow_down_io(void)
 #endif
 }
 
-#ifdef CONFIG_X86_LOCAL_APIC
-static inline void setup_boot_clock(void)
-{
-        PVOP_VCALL0(pv_apic_ops.setup_boot_clock);
-}
-
-static inline void setup_secondary_clock(void)
-{
-        PVOP_VCALL0(pv_apic_ops.setup_secondary_clock);
-}
-#endif
-
-static inline void paravirt_post_allocator_init(void)
-{
-        if (pv_init_ops.post_allocator_init)
-                (*pv_init_ops.post_allocator_init)();
-}
-
-static inline void paravirt_pagetable_setup_start(pgd_t *base)
-{
-        (*pv_mmu_ops.pagetable_setup_start)(base);
-}
-
-static inline void paravirt_pagetable_setup_done(pgd_t *base)
-{
-        (*pv_mmu_ops.pagetable_setup_done)(base);
-}
-
 #ifdef CONFIG_SMP
 static inline void startup_ipi_hook(int phys_apicid, unsigned long start_eip,
                                     unsigned long start_esp)
@@ -1393,20 +699,6 @@ static inline void pmd_clear(pmd_t *pmdp)
 }
 #endif  /* CONFIG_X86_PAE */
 
-/* Lazy mode for batching updates / context switch */
-enum paravirt_lazy_mode {
-        PARAVIRT_LAZY_NONE,
-        PARAVIRT_LAZY_MMU,
-        PARAVIRT_LAZY_CPU,
-};
-
-enum paravirt_lazy_mode paravirt_get_lazy_mode(void);
-void paravirt_start_context_switch(struct task_struct *prev);
-void paravirt_end_context_switch(struct task_struct *next);
-
-void paravirt_enter_lazy_mmu(void);
-void paravirt_leave_lazy_mmu(void);
-
 #define  __HAVE_ARCH_START_CONTEXT_SWITCH
 static inline void arch_start_context_switch(struct task_struct *prev)
 {
@@ -1437,12 +729,6 @@ static inline void __set_fixmap(unsigned /* enum fixed_addresses */ idx,
         pv_mmu_ops.set_fixmap(idx, phys, flags);
 }
 
-void _paravirt_nop(void);
-u32 _paravirt_ident_32(u32);
-u64 _paravirt_ident_64(u64);
-
-#define paravirt_nop    ((void *)_paravirt_nop)
-
 #if defined(CONFIG_SMP) && defined(CONFIG_PARAVIRT_SPINLOCKS)
 
 static inline int __raw_spin_is_locked(struct raw_spinlock *lock)
@@ -1479,17 +765,6 @@ static __always_inline void __raw_spin_unlock(struct raw_spinlock *lock)
 
 #endif
 
-/* These all sit in the .parainstructions section to tell us what to patch. */
-struct paravirt_patch_site {
-        u8 *instr;              /* original instructions */
-        u8 instrtype;           /* type of this instruction */
-        u8 len;                 /* length of original instruction */
-        u16 clobbers;           /* what registers you may clobber */
-};
-
-extern struct paravirt_patch_site __parainstructions[],
-        __parainstructions_end[];
-
 #ifdef CONFIG_X86_32
 #define PV_SAVE_REGS "pushl %ecx; pushl %edx;"
 #define PV_RESTORE_REGS "popl %edx; popl %ecx;"
@@ -1628,6 +903,8 @@ static inline unsigned long __raw_local_irq_save(void)
 #undef PVOP_VCALL4
 #undef PVOP_CALL4
 
+extern void default_banner(void);
+
 #else  /* __ASSEMBLY__ */
 
 #define _PVSITE(ptype, clobbers, ops, word, algn)       \
@@ -1768,5 +1045,7 @@ static inline unsigned long __raw_local_irq_save(void)
 #endif  /* CONFIG_X86_32 */
 
 #endif /* __ASSEMBLY__ */
-#endif /* CONFIG_PARAVIRT */
+#else  /* CONFIG_PARAVIRT */
+# define default_banner x86_init_noop
+#endif /* !CONFIG_PARAVIRT */
 #endif /* _ASM_X86_PARAVIRT_H */
diff --git a/arch/x86/include/asm/paravirt_types.h b/arch/x86/include/asm/paravirt_types.h
new file mode 100644
index 000000000000..dd0f5b32489d
--- /dev/null
+++ b/arch/x86/include/asm/paravirt_types.h
@@ -0,0 +1,693 @@
+#ifndef _ASM_X86_PARAVIRT_TYPES_H
+#define _ASM_X86_PARAVIRT_TYPES_H
+
+/* Bitmask of what can be clobbered: usually at least eax. */
+#define CLBR_NONE 0
+#define CLBR_EAX  (1 << 0)
+#define CLBR_ECX  (1 << 1)
+#define CLBR_EDX  (1 << 2)
+#define CLBR_EDI  (1 << 3)
+
+#ifdef CONFIG_X86_32
+/* CLBR_ANY should match all regs platform has. For i386, that's just it */
+#define CLBR_ANY  ((1 << 4) - 1)
+
+#define CLBR_ARG_REGS   (CLBR_EAX | CLBR_EDX | CLBR_ECX)
+#define CLBR_RET_REG    (CLBR_EAX | CLBR_EDX)
+#define CLBR_SCRATCH    (0)
+#else
+#define CLBR_RAX  CLBR_EAX
+#define CLBR_RCX  CLBR_ECX
+#define CLBR_RDX  CLBR_EDX
+#define CLBR_RDI  CLBR_EDI
+#define CLBR_RSI  (1 << 4)
+#define CLBR_R8   (1 << 5)
+#define CLBR_R9   (1 << 6)
+#define CLBR_R10  (1 << 7)
+#define CLBR_R11  (1 << 8)
+
+#define CLBR_ANY  ((1 << 9) - 1)
+
+#define CLBR_ARG_REGS   (CLBR_RDI | CLBR_RSI | CLBR_RDX | \
+                         CLBR_RCX | CLBR_R8 | CLBR_R9)
+#define CLBR_RET_REG    (CLBR_RAX)
+#define CLBR_SCRATCH    (CLBR_R10 | CLBR_R11)
+
+#endif /* X86_64 */
+
+#define CLBR_CALLEE_SAVE ((CLBR_ARG_REGS | CLBR_SCRATCH) & ~CLBR_RET_REG)
+
+#ifndef __ASSEMBLY__
+
+#include <asm/desc_defs.h>
+#include <asm/kmap_types.h>
+
+struct page;
+struct thread_struct;
+struct desc_ptr;
+struct tss_struct;
+struct mm_struct;
+struct desc_struct;
+struct task_struct;
+struct cpumask;
+
+/*
+ * Wrapper type for pointers to code which uses the non-standard
+ * calling convention.  See PV_CALL_SAVE_REGS_THUNK below.
+ */
+struct paravirt_callee_save {
+        void *func;
+};
+
+/* general info */
+struct pv_info {
+        unsigned int kernel_rpl;
+        int shared_kernel_pmd;
+        int paravirt_enabled;
+        const char *name;
+};
+
+struct pv_init_ops {
+        /*
+         * Patch may replace one of the defined code sequences with
+         * arbitrary code, subject to the same register constraints.
+         * This generally means the code is not free to clobber any
+         * registers other than EAX.  The patch function should return
+         * the number of bytes of code generated, as we nop pad the
+         * rest in generic code.
+         */
+        unsigned (*patch)(u8 type, u16 clobber, void *insnbuf,
+                          unsigned long addr, unsigned len);
+};
+
+
+struct pv_lazy_ops {
+        /* Set deferred update mode, used for batching operations. */
+        void (*enter)(void);
+        void (*leave)(void);
+};
+
+struct pv_time_ops {
+        unsigned long long (*sched_clock)(void);
+        unsigned long (*get_tsc_khz)(void);
+};
+
+struct pv_cpu_ops {
+        /* hooks for various privileged instructions */
+        unsigned long (*get_debugreg)(int regno);
+        void (*set_debugreg)(int regno, unsigned long value);
+
+        void (*clts)(void);
+
+        unsigned long (*read_cr0)(void);
+        void (*write_cr0)(unsigned long);
+
+        unsigned long (*read_cr4_safe)(void);
+        unsigned long (*read_cr4)(void);
+        void (*write_cr4)(unsigned long);
+
+#ifdef CONFIG_X86_64
+        unsigned long (*read_cr8)(void);
+        void (*write_cr8)(unsigned long);
+#endif
+
+        /* Segment descriptor handling */
+        void (*load_tr_desc)(void);
+        void (*load_gdt)(const struct desc_ptr *);
+        void (*load_idt)(const struct desc_ptr *);
+        void (*store_gdt)(struct desc_ptr *);
+        void (*store_idt)(struct desc_ptr *);
+        void (*set_ldt)(const void *desc, unsigned entries);
+        unsigned long (*store_tr)(void);
+        void (*load_tls)(struct thread_struct *t, unsigned int cpu);
+#ifdef CONFIG_X86_64
+        void (*load_gs_index)(unsigned int idx);
+#endif
+        void (*write_ldt_entry)(struct desc_struct *ldt, int entrynum,
+                                const void *desc);
+        void (*write_gdt_entry)(struct desc_struct *,
+                                int entrynum, const void *desc, int size);
+        void (*write_idt_entry)(gate_desc *,
+                                int entrynum, const gate_desc *gate);
+        void (*alloc_ldt)(struct desc_struct *ldt, unsigned entries);
+        void (*free_ldt)(struct desc_struct *ldt, unsigned entries);
+
+        void (*load_sp0)(struct tss_struct *tss, struct thread_struct *t);
+
+        void (*set_iopl_mask)(unsigned mask);
+
+        void (*wbinvd)(void);
+        void (*io_delay)(void);
+
+        /* cpuid emulation, mostly so that caps bits can be disabled */
+        void (*cpuid)(unsigned int *eax, unsigned int *ebx,
+                      unsigned int *ecx, unsigned int *edx);
+
+        /* MSR, PMC and TSR operations.
+           err = 0/-EFAULT.  wrmsr returns 0/-EFAULT. */
+        u64 (*read_msr)(unsigned int msr, int *err);
+        int (*rdmsr_regs)(u32 *regs);
+        int (*write_msr)(unsigned int msr, unsigned low, unsigned high);
+        int (*wrmsr_regs)(u32 *regs);
+
+        u64 (*read_tsc)(void);
+        u64 (*read_pmc)(int counter);
+        unsigned long long (*read_tscp)(unsigned int *aux);
+
+        /*
+         * Atomically enable interrupts and return to userspace.  This
+         * is only ever used to return to 32-bit processes; in a
+         * 64-bit kernel, it's used for 32-on-64 compat processes, but
+         * never native 64-bit processes.  (Jump, not call.)
+         */
+        void (*irq_enable_sysexit)(void);
+
+        /*
+         * Switch to usermode gs and return to 64-bit usermode using
+         * sysret.  Only used in 64-bit kernels to return to 64-bit
+         * processes.  Usermode register state, including %rsp, must
+         * already be restored.
+         */
+        void (*usergs_sysret64)(void);
+
+        /*
+         * Switch to usermode gs and return to 32-bit usermode using
+         * sysret.  Used to return to 32-on-64 compat processes.
+         * Other usermode register state, including %esp, must already
+         * be restored.
+         */
+        void (*usergs_sysret32)(void);
+
+        /* Normal iret.  Jump to this with the standard iret stack
+           frame set up. */
+        void (*iret)(void);
+
+        void (*swapgs)(void);
+
+        void (*start_context_switch)(struct task_struct *prev);
+        void (*end_context_switch)(struct task_struct *next);
+};
+
+struct pv_irq_ops {
+        /*
+         * Get/set interrupt state.  save_fl and restore_fl are only
+         * expected to use X86_EFLAGS_IF; all other bits
+         * returned from save_fl are undefined, and may be ignored by
+         * restore_fl.
+         *
+         * NOTE: These functions callers expect the callee to preserve
+         * more registers than the standard C calling convention.
+         */
+        struct paravirt_callee_save save_fl;
+        struct paravirt_callee_save restore_fl;
+        struct paravirt_callee_save irq_disable;
+        struct paravirt_callee_save irq_enable;
+
+        void (*safe_halt)(void);
+        void (*halt)(void);
+
+#ifdef CONFIG_X86_64
+        void (*adjust_exception_frame)(void);
+#endif
+};
+
+struct pv_apic_ops {
+#ifdef CONFIG_X86_LOCAL_APIC
+        void (*startup_ipi_hook)(int phys_apicid,
+                                 unsigned long start_eip,
+                                 unsigned long start_esp);
+#endif
+};
+
+struct pv_mmu_ops {
+        unsigned long (*read_cr2)(void);
+        void (*write_cr2)(unsigned long);
+
+        unsigned long (*read_cr3)(void);
+        void (*write_cr3)(unsigned long);
+
+        /*
+         * Hooks for intercepting the creation/use/destruction of an
+         * mm_struct.
+         */
+        void (*activate_mm)(struct mm_struct *prev,
+                            struct mm_struct *next);
+        void (*dup_mmap)(struct mm_struct *oldmm,
+                         struct mm_struct *mm);
+        void (*exit_mmap)(struct mm_struct *mm);
+
+
+        /* TLB operations */
+        void (*flush_tlb_user)(void);
+        void (*flush_tlb_kernel)(void);
+        void (*flush_tlb_single)(unsigned long addr);
+        void (*flush_tlb_others)(const struct cpumask *cpus,
+                                 struct mm_struct *mm,
+                                 unsigned long va);
+
+        /* Hooks for allocating and freeing a pagetable top-level */
+        int  (*pgd_alloc)(struct mm_struct *mm);
+        void (*pgd_free)(struct mm_struct *mm, pgd_t *pgd);
+
+        /*
+         * Hooks for allocating/releasing pagetable pages when they're
+         * attached to a pagetable
+         */
+        void (*alloc_pte)(struct mm_struct *mm, unsigned long pfn);
+        void (*alloc_pmd)(struct mm_struct *mm, unsigned long pfn);
+        void (*alloc_pmd_clone)(unsigned long pfn, unsigned long clonepfn, unsigned long start, unsigned long count);
+        void (*alloc_pud)(struct mm_struct *mm, unsigned long pfn);
+        void (*release_pte)(unsigned long pfn);
+        void (*release_pmd)(unsigned long pfn);
+        void (*release_pud)(unsigned long pfn);
+
+        /* Pagetable manipulation functions */
+        void (*set_pte)(pte_t *ptep, pte_t pteval);
+        void (*set_pte_at)(struct mm_struct *mm, unsigned long addr,
+                           pte_t *ptep, pte_t pteval);
+        void (*set_pmd)(pmd_t *pmdp, pmd_t pmdval);
+        void (*pte_update)(struct mm_struct *mm, unsigned long addr,
+                           pte_t *ptep);
+        void (*pte_update_defer)(struct mm_struct *mm,
+                                 unsigned long addr, pte_t *ptep);
+
+        pte_t (*ptep_modify_prot_start)(struct mm_struct *mm, unsigned long addr,
+                                        pte_t *ptep);
+        void (*ptep_modify_prot_commit)(struct mm_struct *mm, unsigned long addr,
+                                        pte_t *ptep, pte_t pte);
+
+        struct paravirt_callee_save pte_val;
+        struct paravirt_callee_save make_pte;
+
+        struct paravirt_callee_save pgd_val;
+        struct paravirt_callee_save make_pgd;
+
+#if PAGETABLE_LEVELS >= 3
+#ifdef CONFIG_X86_PAE
+        void (*set_pte_atomic)(pte_t *ptep, pte_t pteval);
+        void (*pte_clear)(struct mm_struct *mm, unsigned long addr,
+                          pte_t *ptep);
+        void (*pmd_clear)(pmd_t *pmdp);
+
+#endif  /* CONFIG_X86_PAE */
+
+        void (*set_pud)(pud_t *pudp, pud_t pudval);
+
+        struct paravirt_callee_save pmd_val;
+        struct paravirt_callee_save make_pmd;
+
+#if PAGETABLE_LEVELS == 4
+        struct paravirt_callee_save pud_val;
+        struct paravirt_callee_save make_pud;
+
+        void (*set_pgd)(pgd_t *pudp, pgd_t pgdval);
+#endif  /* PAGETABLE_LEVELS == 4 */
+#endif  /* PAGETABLE_LEVELS >= 3 */
+
+#ifdef CONFIG_HIGHPTE
+        void *(*kmap_atomic_pte)(struct page *page, enum km_type type);
+#endif
+
+        struct pv_lazy_ops lazy_mode;
+
+        /* dom0 ops */
+
+        /* Sometimes the physical address is a pfn, and sometimes its
+           an mfn.  We can tell which is which from the index. */
+        void (*set_fixmap)(unsigned /* enum fixed_addresses */ idx,
+                           phys_addr_t phys, pgprot_t flags);
+};
+
+struct raw_spinlock;
+struct pv_lock_ops {
+        int (*spin_is_locked)(struct raw_spinlock *lock);
+        int (*spin_is_contended)(struct raw_spinlock *lock);
+        void (*spin_lock)(struct raw_spinlock *lock);
+        void (*spin_lock_flags)(struct raw_spinlock *lock, unsigned long flags);
+        int (*spin_trylock)(struct raw_spinlock *lock);
+        void (*spin_unlock)(struct raw_spinlock *lock);
+};
+
+/* This contains all the paravirt structures: we get a convenient
+ * number for each function using the offset which we use to indicate
+ * what to patch. */
+struct paravirt_patch_template {
+        struct pv_init_ops pv_init_ops;
+        struct pv_time_ops pv_time_ops;
+        struct pv_cpu_ops pv_cpu_ops;
+        struct pv_irq_ops pv_irq_ops;
+        struct pv_apic_ops pv_apic_ops;
+        struct pv_mmu_ops pv_mmu_ops;
+        struct pv_lock_ops pv_lock_ops;
+};
+
+extern struct pv_info pv_info;
+extern struct pv_init_ops pv_init_ops;
+extern struct pv_time_ops pv_time_ops;
+extern struct pv_cpu_ops pv_cpu_ops;
+extern struct pv_irq_ops pv_irq_ops;
+extern struct pv_apic_ops pv_apic_ops;
+extern struct pv_mmu_ops pv_mmu_ops;
+extern struct pv_lock_ops pv_lock_ops;
+
+#define PARAVIRT_PATCH(x)                                       \
+        (offsetof(struct paravirt_patch_template, x) / sizeof(void *))
+
+#define paravirt_type(op)                               \
+        [paravirt_typenum] "i" (PARAVIRT_PATCH(op)),    \
+        [paravirt_opptr] "i" (&(op))
+#define paravirt_clobber(clobber)               \
+        [paravirt_clobber] "i" (clobber)
+
+/*
+ * Generate some code, and mark it as patchable by the
+ * apply_paravirt() alternate instruction patcher.
+ */
+#define _paravirt_alt(insn_string, type, clobber)       \
+        "771:\n\t" insn_string "\n" "772:\n"            \
+        ".pushsection .parainstructions,\"a\"\n"        \
+        _ASM_ALIGN "\n"                                 \
+        _ASM_PTR " 771b\n"                              \
+        "  .byte " type "\n"                            \
+        "  .byte 772b-771b\n"                           \
+        "  .short " clobber "\n"                        \
+        ".popsection\n"
+
+/* Generate patchable code, with the default asm parameters. */
+#define paravirt_alt(insn_string)                                       \
+        _paravirt_alt(insn_string, "%c[paravirt_typenum]", "%c[paravirt_clobber]")
+
+/* Simple instruction patching code. */
+#define DEF_NATIVE(ops, name, code)                                     \
+        extern const char start_##ops##_##name[], end_##ops##_##name[]; \
+        asm("start_" #ops "_" #name ": " code "; end_" #ops "_" #name ":")
+
+unsigned paravirt_patch_nop(void);
+unsigned paravirt_patch_ident_32(void *insnbuf, unsigned len);
+unsigned paravirt_patch_ident_64(void *insnbuf, unsigned len);
+unsigned paravirt_patch_ignore(unsigned len);
+unsigned paravirt_patch_call(void *insnbuf,
+                             const void *target, u16 tgt_clobbers,
+                             unsigned long addr, u16 site_clobbers,
+                             unsigned len);
+unsigned paravirt_patch_jmp(void *insnbuf, const void *target,
+                            unsigned long addr, unsigned len);
+unsigned paravirt_patch_default(u8 type, u16 clobbers, void *insnbuf,
+                                unsigned long addr, unsigned len);
+
+unsigned paravirt_patch_insns(void *insnbuf, unsigned len,
+                              const char *start, const char *end);
+
+unsigned native_patch(u8 type, u16 clobbers, void *ibuf,
+                      unsigned long addr, unsigned len);
+
+int paravirt_disable_iospace(void);
+
+/*
+ * This generates an indirect call based on the operation type number.
+ * The type number, computed in PARAVIRT_PATCH, is derived from the
+ * offset into the paravirt_patch_template structure, and can therefore be
+ * freely converted back into a structure offset.
+ */
+#define PARAVIRT_CALL   "call *%c[paravirt_opptr];"
+
+/*
+ * These macros are intended to wrap calls through one of the paravirt
+ * ops structs, so that they can be later identified and patched at
+ * runtime.
+ *
+ * Normally, a call to a pv_op function is a simple indirect call:
+ * (pv_op_struct.operations)(args...).
+ *
+ * Unfortunately, this is a relatively slow operation for modern CPUs,
+ * because it cannot necessarily determine what the destination
+ * address is.  In this case, the address is a runtime constant, so at
+ * the very least we can patch the call to e a simple direct call, or
+ * ideally, patch an inline implementation into the callsite.  (Direct
+ * calls are essentially free, because the call and return addresses
+ * are completely predictable.)
+ *
+ * For i386, these macros rely on the standard gcc "regparm(3)" calling
+ * convention, in which the first three arguments are placed in %eax,
+ * %edx, %ecx (in that order), and the remaining arguments are placed
+ * on the stack.  All caller-save registers (eax,edx,ecx) are expected
+ * to be modified (either clobbered or used for return values).
+ * X86_64, on the other hand, already specifies a register-based calling
+ * conventions, returning at %rax, with parameteres going on %rdi, %rsi,
+ * %rdx, and %rcx. Note that for this reason, x86_64 does not need any
+ * special handling for dealing with 4 arguments, unlike i386.
+ * However, x86_64 also have to clobber all caller saved registers, which
+ * unfortunately, are quite a bit (r8 - r11)
+ *
+ * The call instruction itself is marked by placing its start address
+ * and size into the .parainstructions section, so that
+ * apply_paravirt() in arch/i386/kernel/alternative.c can do the
+ * appropriate patching under the control of the backend pv_init_ops
+ * implementation.
+ *
+ * Unfortunately there's no way to get gcc to generate the args setup
+ * for the call, and then allow the call itself to be generated by an
+ * inline asm.  Because of this, we must do the complete arg setup and
+ * return value handling from within these macros.  This is fairly
+ * cumbersome.
+ *
+ * There are 5 sets of PVOP_* macros for dealing with 0-4 arguments.
+ * It could be extended to more arguments, but there would be little
+ * to be gained from that.  For each number of arguments, there are
+ * the two VCALL and CALL variants for void and non-void functions.
+ *
+ * When there is a return value, the invoker of the macro must specify
+ * the return type.  The macro then uses sizeof() on that type to
+ * determine whether its a 32 or 64 bit value, and places the return
+ * in the right register(s) (just %eax for 32-bit, and %edx:%eax for
+ * 64-bit). For x86_64 machines, it just returns at %rax regardless of
+ * the return value size.
+ *
+ * 64-bit arguments are passed as a pair of adjacent 32-bit arguments
+ * i386 also passes 64-bit arguments as a pair of adjacent 32-bit arguments
+ * in low,high order
+ *
+ * Small structures are passed and returned in registers.  The macro
+ * calling convention can't directly deal with this, so the wrapper
+ * functions must do this.
+ *
+ * These PVOP_* macros are only defined within this header.  This
+ * means that all uses must be wrapped in inline functions.  This also
+ * makes sure the incoming and outgoing types are always correct.
+ */
+#ifdef CONFIG_X86_32
+#define PVOP_VCALL_ARGS                         \
+        unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx
+#define PVOP_CALL_ARGS                  PVOP_VCALL_ARGS
+
+#define PVOP_CALL_ARG1(x)               "a" ((unsigned long)(x))
+#define PVOP_CALL_ARG2(x)               "d" ((unsigned long)(x))
+#define PVOP_CALL_ARG3(x)               "c" ((unsigned long)(x))
+
+#define PVOP_VCALL_CLOBBERS             "=a" (__eax), "=d" (__edx),     \
+                                        "=c" (__ecx)
+#define PVOP_CALL_CLOBBERS              PVOP_VCALL_CLOBBERS
+
+#define PVOP_VCALLEE_CLOBBERS           "=a" (__eax), "=d" (__edx)
+#define PVOP_CALLEE_CLOBBERS            PVOP_VCALLEE_CLOBBERS
+
+#define EXTRA_CLOBBERS
+#define VEXTRA_CLOBBERS
+#else  /* CONFIG_X86_64 */
+#define PVOP_VCALL_ARGS                                 \
+        unsigned long __edi = __edi, __esi = __esi,     \
+                __edx = __edx, __ecx = __ecx
+#define PVOP_CALL_ARGS          PVOP_VCALL_ARGS, __eax
+
+#define PVOP_CALL_ARG1(x)               "D" ((unsigned long)(x))
+#define PVOP_CALL_ARG2(x)               "S" ((unsigned long)(x))
+#define PVOP_CALL_ARG3(x)               "d" ((unsigned long)(x))
+#define PVOP_CALL_ARG4(x)               "c" ((unsigned long)(x))
+
+#define PVOP_VCALL_CLOBBERS     "=D" (__edi),                           \
+                                "=S" (__esi), "=d" (__edx),             \
+                                "=c" (__ecx)
+#define PVOP_CALL_CLOBBERS      PVOP_VCALL_CLOBBERS, "=a" (__eax)
+
+#define PVOP_VCALLEE_CLOBBERS   "=a" (__eax)
+#define PVOP_CALLEE_CLOBBERS    PVOP_VCALLEE_CLOBBERS
+
+#define EXTRA_CLOBBERS   , "r8", "r9", "r10", "r11"
+#define VEXTRA_CLOBBERS  , "rax", "r8", "r9", "r10", "r11"
+#endif  /* CONFIG_X86_32 */
+
+#ifdef CONFIG_PARAVIRT_DEBUG
+#define PVOP_TEST_NULL(op)      BUG_ON(op == NULL)
+#else
+#define PVOP_TEST_NULL(op)      ((void)op)
+#endif
+
+#define ____PVOP_CALL(rettype, op, clbr, call_clbr, extra_clbr,         \
+                      pre, post, ...)                                   \
+        ({                                                              \
+                rettype __ret;                                          \
+                PVOP_CALL_ARGS;                                         \
+                PVOP_TEST_NULL(op);                                     \
+                /* This is 32-bit specific, but is okay in 64-bit */    \
+                /* since this condition will never hold */              \
+                if (sizeof(rettype) > sizeof(unsigned long)) {          \
+                        asm volatile(pre                                \
+                                     paravirt_alt(PARAVIRT_CALL)        \
+                                     post                               \
+                                     : call_clbr                        \
+                                     : paravirt_type(op),               \
+                                       paravirt_clobber(clbr),          \
+                                       ##__VA_ARGS__                    \
+                                     : "memory", "cc" extra_clbr);      \
+                        __ret = (rettype)((((u64)__edx) << 32) | __eax); \
+                } else {                                                \
+                        asm volatile(pre                                \
+                                     paravirt_alt(PARAVIRT_CALL)        \
+                                     post                               \
+                                     : call_clbr                        \
+                                     : paravirt_type(op),               \
+                                       paravirt_clobber(clbr),          \
+                                       ##__VA_ARGS__                    \
+                                     : "memory", "cc" extra_clbr);      \
+                        __ret = (rettype)__eax;                         \
+                }                                                       \
+                __ret;                                                  \
+        })
+
+#define __PVOP_CALL(rettype, op, pre, post, ...)                        \
+        ____PVOP_CALL(rettype, op, CLBR_ANY, PVOP_CALL_CLOBBERS,        \
+                      EXTRA_CLOBBERS, pre, post, ##__VA_ARGS__)
+
+#define __PVOP_CALLEESAVE(rettype, op, pre, post, ...)                  \
+        ____PVOP_CALL(rettype, op.func, CLBR_RET_REG,                   \
+                      PVOP_CALLEE_CLOBBERS, ,                           \
+                      pre, post, ##__VA_ARGS__)
+
+
+#define ____PVOP_VCALL(op, clbr, call_clbr, extra_clbr, pre, post, ...) \
+        ({                                                              \
+                PVOP_VCALL_ARGS;                                        \
+                PVOP_TEST_NULL(op);                                     \
+                asm volatile(pre                                        \
+                             paravirt_alt(PARAVIRT_CALL)                \
+                             post                                       \
+                             : call_clbr                                \
+                             : paravirt_type(op),                       \
+                               paravirt_clobber(clbr),                  \
+                               ##__VA_ARGS__                            \
+                             : "memory", "cc" extra_clbr);              \
+        })
+
+#define __PVOP_VCALL(op, pre, post, ...)                                \
+        ____PVOP_VCALL(op, CLBR_ANY, PVOP_VCALL_CLOBBERS,               \
+                       VEXTRA_CLOBBERS,                                 \
+                       pre, post, ##__VA_ARGS__)
+
+#define __PVOP_VCALLEESAVE(rettype, op, pre, post, ...)                 \
+        ____PVOP_CALL(rettype, op.func, CLBR_RET_REG,                   \
+                      PVOP_VCALLEE_CLOBBERS, ,                          \
+                      pre, post, ##__VA_ARGS__)
+
+
+
+#define PVOP_CALL0(rettype, op)                                         \
+        __PVOP_CALL(rettype, op, "", "")
+#define PVOP_VCALL0(op)                                                 \
+        __PVOP_VCALL(op, "", "")
+
+#define PVOP_CALLEE0(rettype, op)                                       \
+        __PVOP_CALLEESAVE(rettype, op, "", "")
+#define PVOP_VCALLEE0(op)                                               \
+        __PVOP_VCALLEESAVE(op, "", "")
+
+
+#define PVOP_CALL1(rettype, op, arg1)                                   \
+        __PVOP_CALL(rettype, op, "", "", PVOP_CALL_ARG1(arg1))
+#define PVOP_VCALL1(op, arg1)                                           \
+        __PVOP_VCALL(op, "", "", PVOP_CALL_ARG1(arg1))
+
+#define PVOP_CALLEE1(rettype, op, arg1)                                 \
+        __PVOP_CALLEESAVE(rettype, op, "", "", PVOP_CALL_ARG1(arg1))
+#define PVOP_VCALLEE1(op, arg1)                                         \
+        __PVOP_VCALLEESAVE(op, "", "", PVOP_CALL_ARG1(arg1))
+
+
+#define PVOP_CALL2(rettype, op, arg1, arg2)                             \
+        __PVOP_CALL(rettype, op, "", "", PVOP_CALL_ARG1(arg1),          \
+                    PVOP_CALL_ARG2(arg2))
+#define PVOP_VCALL2(op, arg1, arg2)                                     \
+        __PVOP_VCALL(op, "", "", PVOP_CALL_ARG1(arg1),                  \
+                     PVOP_CALL_ARG2(arg2))
+
+#define PVOP_CALLEE2(rettype, op, arg1, arg2)                           \
+        __PVOP_CALLEESAVE(rettype, op, "", "", PVOP_CALL_ARG1(arg1),    \
+                          PVOP_CALL_ARG2(arg2))
+#define PVOP_VCALLEE2(op, arg1, arg2)                                   \
+        __PVOP_VCALLEESAVE(op, "", "", PVOP_CALL_ARG1(arg1),            \
+                           PVOP_CALL_ARG2(arg2))
+
+
+#define PVOP_CALL3(rettype, op, arg1, arg2, arg3)                       \
+        __PVOP_CALL(rettype, op, "", "", PVOP_CALL_ARG1(arg1),          \
+                    PVOP_CALL_ARG2(arg2), PVOP_CALL_ARG3(arg3))
+#define PVOP_VCALL3(op, arg1, arg2, arg3)                               \
+        __PVOP_VCALL(op, "", "", PVOP_CALL_ARG1(arg1),                  \
+                     PVOP_CALL_ARG2(arg2), PVOP_CALL_ARG3(arg3))
+
+/* This is the only difference in x86_64. We can make it much simpler */
+#ifdef CONFIG_X86_32
+#define PVOP_CALL4(rettype, op, arg1, arg2, arg3, arg4)                 \
+        __PVOP_CALL(rettype, op,                                        \
+                    "push %[_arg4];", "lea 4(%%esp),%%esp;",            \
+                    PVOP_CALL_ARG1(arg1), PVOP_CALL_ARG2(arg2),         \
+                    PVOP_CALL_ARG3(arg3), [_arg4] "mr" ((u32)(arg4)))
+#define PVOP_VCALL4(op, arg1, arg2, arg3, arg4)                         \
+        __PVOP_VCALL(op,                                                \
+                    "push %[_arg4];", "lea 4(%%esp),%%esp;",            \
+                    "0" ((u32)(arg1)), "1" ((u32)(arg2)),               \
+                    "2" ((u32)(arg3)), [_arg4] "mr" ((u32)(arg4)))
+#else
+#define PVOP_CALL4(rettype, op, arg1, arg2, arg3, arg4)                 \
+        __PVOP_CALL(rettype, op, "", "",                                \
+                    PVOP_CALL_ARG1(arg1), PVOP_CALL_ARG2(arg2),         \
+                    PVOP_CALL_ARG3(arg3), PVOP_CALL_ARG4(arg4))
+#define PVOP_VCALL4(op, arg1, arg2, arg3, arg4)                         \
+        __PVOP_VCALL(op, "", "",                                        \
+                     PVOP_CALL_ARG1(arg1), PVOP_CALL_ARG2(arg2),        \
+                     PVOP_CALL_ARG3(arg3), PVOP_CALL_ARG4(arg4))
+#endif
+
+/* Lazy mode for batching updates / context switch */
+enum paravirt_lazy_mode {
+        PARAVIRT_LAZY_NONE,
+        PARAVIRT_LAZY_MMU,
+        PARAVIRT_LAZY_CPU,
+};
+
+enum paravirt_lazy_mode paravirt_get_lazy_mode(void);
+void paravirt_start_context_switch(struct task_struct *prev);
+void paravirt_end_context_switch(struct task_struct *next);
+
+void paravirt_enter_lazy_mmu(void);
+void paravirt_leave_lazy_mmu(void);
+
+void _paravirt_nop(void);
+u32 _paravirt_ident_32(u32);
+u64 _paravirt_ident_64(u64);
+
+#define paravirt_nop    ((void *)_paravirt_nop)
+
+/* These all sit in the .parainstructions section to tell us what to patch. */
+struct paravirt_patch_site {
+        u8 *instr;              /* original instructions */
+        u8 instrtype;           /* type of this instruction */
+        u8 len;                 /* length of original instruction */
+        u16 clobbers;           /* what registers you may clobber */
+};
+
+extern struct paravirt_patch_site __parainstructions[],
+        __parainstructions_end[];
+
+#endif  /* __ASSEMBLY__ */
+
+#endif  /* _ASM_X86_PARAVIRT_TYPES_H */
diff --git a/arch/x86/include/asm/pat.h b/arch/x86/include/asm/pat.h
index 7af14e512f97..e2c1668dde7a 100644
--- a/arch/x86/include/asm/pat.h
+++ b/arch/x86/include/asm/pat.h
@@ -19,4 +19,9 @@ extern int free_memtype(u64 start, u64 end);
 extern int kernel_map_sync_memtype(u64 base, unsigned long size,
                 unsigned long flag);
 
+int io_reserve_memtype(resource_size_t start, resource_size_t end,
+                        unsigned long *type);
+
+void io_free_memtype(resource_size_t start, resource_size_t end);
+
 #endif /* _ASM_X86_PAT_H */
diff --git a/arch/x86/include/asm/pci.h b/arch/x86/include/asm/pci.h
index 1ff685ca221c..ada8c201d513 100644
--- a/arch/x86/include/asm/pci.h
+++ b/arch/x86/include/asm/pci.h
@@ -48,7 +48,6 @@ extern unsigned int pcibios_assign_all_busses(void);
 #else
 #define pcibios_assign_all_busses()     0
 #endif
-#define pcibios_scan_all_fns(a, b)      0
 
 extern unsigned long pci_mem_start;
 #define PCIBIOS_MIN_IO          0x1000
@@ -144,7 +143,11 @@ static inline int __pcibus_to_node(const struct pci_bus *bus)
 static inline const struct cpumask *
 cpumask_of_pcibus(const struct pci_bus *bus)
 {
-        return cpumask_of_node(__pcibus_to_node(bus));
+        int node;
+
+        node = __pcibus_to_node(bus);
+        return (node == -1) ? cpu_online_mask :
+                              cpumask_of_node(node);
 }
 #endif
 
diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index 103f1ddb0d85..b65a36defeb7 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -49,7 +49,7 @@
 #define __percpu_arg(x)         "%%"__stringify(__percpu_seg)":%P" #x
 #define __my_cpu_offset         percpu_read(this_cpu_off)
 #else
-#define __percpu_arg(x)         "%" #x
+#define __percpu_arg(x)         "%P" #x
 #endif
 
 /*
@@ -104,36 +104,48 @@ do {							\
         }                                               \
 } while (0)
 
-#define percpu_from_op(op, var)                         \
+#define percpu_from_op(op, var, constraint)             \
 ({                                                      \
         typeof(var) ret__;                              \
         switch (sizeof(var)) {                          \
         case 1:                                         \
                 asm(op "b "__percpu_arg(1)",%0"         \
                     : "=q" (ret__)                      \
-                    : "m" (var));                       \
+                    : constraint);                      \
                 break;                                  \
         case 2:                                         \
                 asm(op "w "__percpu_arg(1)",%0"         \
                     : "=r" (ret__)                      \
-                    : "m" (var));                       \
+                    : constraint);                      \
                 break;                                  \
         case 4:                                         \
                 asm(op "l "__percpu_arg(1)",%0"         \
                     : "=r" (ret__)                      \
-                    : "m" (var));                       \
+                    : constraint);                      \
                 break;                                  \
         case 8:                                         \
                 asm(op "q "__percpu_arg(1)",%0"         \
                     : "=r" (ret__)                      \
-                    : "m" (var));                       \
+                    : constraint);                      \
                 break;                                  \
         default: __bad_percpu_size();                   \
         }                                               \
         ret__;                                          \
 })
 
-#define percpu_read(var)        percpu_from_op("mov", per_cpu__##var)
+/*
+ * percpu_read() makes gcc load the percpu variable every time it is
+ * accessed while percpu_read_stable() allows the value to be cached.
+ * percpu_read_stable() is more efficient and can be used if its value
+ * is guaranteed to be valid across cpus.  The current users include
+ * get_current() and get_thread_info() both of which are actually
+ * per-thread variables implemented as per-cpu variables and thus
+ * stable for the duration of the respective task.
+ */
+#define percpu_read(var)        percpu_from_op("mov", per_cpu__##var,   \
+                                               "m" (per_cpu__##var))
+#define percpu_read_stable(var) percpu_from_op("mov", per_cpu__##var,   \
+                                               "p" (&per_cpu__##var))
 #define percpu_write(var, val)  percpu_to_op("mov", per_cpu__##var, val)
 #define percpu_add(var, val)    percpu_to_op("add", per_cpu__##var, val)
 #define percpu_sub(var, val)    percpu_to_op("sub", per_cpu__##var, val)
@@ -156,15 +168,6 @@ do {							\
 /* We can use this directly for local CPU (faster). */
 DECLARE_PER_CPU(unsigned long, this_cpu_off);
 
-#ifdef CONFIG_NEED_MULTIPLE_NODES
-void *pcpu_lpage_remapped(void *kaddr);
-#else
-static inline void *pcpu_lpage_remapped(void *kaddr)
-{
-        return NULL;
-}
-#endif
-
 #endif /* !__ASSEMBLY__ */
 
 #ifdef CONFIG_SMP
diff --git a/arch/x86/include/asm/perf_counter.h b/arch/x86/include/asm/perf_event.h
index fa64e401589d..ad7ce3fd5065 100644
--- a/arch/x86/include/asm/perf_counter.h
+++ b/arch/x86/include/asm/perf_event.h
@@ -1,8 +1,8 @@
-#ifndef _ASM_X86_PERF_COUNTER_H
-#define _ASM_X86_PERF_COUNTER_H
+#ifndef _ASM_X86_PERF_EVENT_H
+#define _ASM_X86_PERF_EVENT_H
 
 /*
- * Performance counter hw details:
+ * Performance event hw details:
  */
 
 #define X86_PMC_MAX_GENERIC                                     8
@@ -43,7 +43,7 @@
 union cpuid10_eax {
         struct {
                 unsigned int version_id:8;
-                unsigned int num_counters:8;
+                unsigned int num_events:8;
                 unsigned int bit_width:8;
                 unsigned int mask_length:8;
         } split;
@@ -52,7 +52,7 @@ union cpuid10_eax {
 
 union cpuid10_edx {
         struct {
-                unsigned int num_counters_fixed:4;
+                unsigned int num_events_fixed:4;
                 unsigned int reserved:28;
         } split;
         unsigned int full;
@@ -60,7 +60,7 @@ union cpuid10_edx {
 
 
 /*
- * Fixed-purpose performance counters:
+ * Fixed-purpose performance events:
  */
 
 /*
@@ -84,15 +84,25 @@ union cpuid10_edx {
 #define MSR_ARCH_PERFMON_FIXED_CTR2                     0x30b
 #define X86_PMC_IDX_FIXED_BUS_CYCLES                    (X86_PMC_IDX_FIXED + 2)
 
-#ifdef CONFIG_PERF_COUNTERS
-extern void init_hw_perf_counters(void);
-extern void perf_counters_lapic_init(void);
+/*
+ * We model BTS tracing as another fixed-mode PMC.
+ *
+ * We choose a value in the middle of the fixed event range, since lower
+ * values are used by actual fixed events and higher values are used
+ * to indicate other overflow conditions in the PERF_GLOBAL_STATUS msr.
+ */
+#define X86_PMC_IDX_FIXED_BTS                           (X86_PMC_IDX_FIXED + 16)
+
+
+#ifdef CONFIG_PERF_EVENTS
+extern void init_hw_perf_events(void);
+extern void perf_events_lapic_init(void);
 
-#define PERF_COUNTER_INDEX_OFFSET                       0
+#define PERF_EVENT_INDEX_OFFSET                 0
 
 #else
-static inline void init_hw_perf_counters(void)          { }
-static inline void perf_counters_lapic_init(void)       { }
+static inline void init_hw_perf_events(void)            { }
+static inline void perf_events_lapic_init(void) { }
 #endif
 
-#endif /* _ASM_X86_PERF_COUNTER_H */
+#endif /* _ASM_X86_PERF_EVENT_H */
diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h
index 16748077559a..af6fd360ab35 100644
--- a/arch/x86/include/asm/pgtable.h
+++ b/arch/x86/include/asm/pgtable.h
@@ -56,16 +56,6 @@ extern struct list_head pgd_list;
 #define pte_update(mm, addr, ptep)              do { } while (0)
 #define pte_update_defer(mm, addr, ptep)        do { } while (0)
 
-static inline void __init paravirt_pagetable_setup_start(pgd_t *base)
-{
-        native_pagetable_setup_start(base);
-}
-
-static inline void __init paravirt_pagetable_setup_done(pgd_t *base)
-{
-        native_pagetable_setup_done(base);
-}
-
 #define pgd_val(x)      native_pgd_val(x)
 #define __pgd(x)        native_make_pgd(x)
 
@@ -135,6 +125,11 @@ static inline unsigned long pte_pfn(pte_t pte)
         return (pte_val(pte) & PTE_PFN_MASK) >> PAGE_SHIFT;
 }
 
+static inline unsigned long pmd_pfn(pmd_t pmd)
+{
+        return (pmd_val(pmd) & PTE_PFN_MASK) >> PAGE_SHIFT;
+}
+
 #define pte_page(pte)   pfn_to_page(pte_pfn(pte))
 
 static inline int pmd_large(pmd_t pte)
@@ -359,7 +354,7 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd)
  * this macro returns the index of the entry in the pmd page which would
  * control the given virtual address
  */
-static inline unsigned pmd_index(unsigned long address)
+static inline unsigned long pmd_index(unsigned long address)
 {
         return (address >> PMD_SHIFT) & (PTRS_PER_PMD - 1);
 }
@@ -379,7 +374,7 @@ static inline unsigned pmd_index(unsigned long address)
  * this function returns the index of the entry in the pte page which would
  * control the given virtual address
  */
-static inline unsigned pte_index(unsigned long address)
+static inline unsigned long pte_index(unsigned long address)
 {
         return (address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1);
 }
@@ -430,11 +425,6 @@ static inline pmd_t *pmd_offset(pud_t *pud, unsigned long address)
         return (pmd_t *)pud_page_vaddr(*pud) + pmd_index(address);
 }
 
-static inline unsigned long pmd_pfn(pmd_t pmd)
-{
-        return (pmd_val(pmd) & PTE_PFN_MASK) >> PAGE_SHIFT;
-}
-
 static inline int pud_large(pud_t pud)
 {
         return (pud_val(pud) & (_PAGE_PSE | _PAGE_PRESENT)) ==
@@ -470,7 +460,7 @@ static inline unsigned long pgd_page_vaddr(pgd_t pgd)
 #define pgd_page(pgd)           pfn_to_page(pgd_val(pgd) >> PAGE_SHIFT)
 
 /* to find an entry in a page-table-directory. */
-static inline unsigned pud_index(unsigned long address)
+static inline unsigned long pud_index(unsigned long address)
 {
         return (address >> PUD_SHIFT) & (PTRS_PER_PUD - 1);
 }
diff --git a/arch/x86/include/asm/pgtable_types.h b/arch/x86/include/asm/pgtable_types.h
index 54cb697f4900..7b467bf3c680 100644
--- a/arch/x86/include/asm/pgtable_types.h
+++ b/arch/x86/include/asm/pgtable_types.h
@@ -299,8 +299,8 @@ void set_pte_vaddr(unsigned long vaddr, pte_t pte);
 extern void native_pagetable_setup_start(pgd_t *base);
 extern void native_pagetable_setup_done(pgd_t *base);
 #else
-static inline void native_pagetable_setup_start(pgd_t *base) {}
-static inline void native_pagetable_setup_done(pgd_t *base) {}
+#define native_pagetable_setup_start x86_init_pgd_noop
+#define native_pagetable_setup_done  x86_init_pgd_noop
 #endif
 
 struct seq_file;
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index c7768269b1cf..c3429e8b2424 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -27,6 +27,7 @@ struct mm_struct;
 #include <linux/cpumask.h>
 #include <linux/cache.h>
 #include <linux/threads.h>
+#include <linux/math64.h>
 #include <linux/init.h>
 
 /*
@@ -403,7 +404,17 @@ extern unsigned long kernel_eflags;
 extern asmlinkage void ignore_sysret(void);
 #else   /* X86_64 */
 #ifdef CONFIG_CC_STACKPROTECTOR
-DECLARE_PER_CPU(unsigned long, stack_canary);
+/*
+ * Make sure stack canary segment base is cached-aligned:
+ *   "For Intel Atom processors, avoid non zero segment base address
+ *    that is not aligned to cache line boundary at all cost."
+ * (Optim Ref Manual Assembly/Compiler Coding Rule 15.)
+ */
+struct stack_canary {
+        char __pad[20];         /* canary at %gs:20 */
+        unsigned long canary;
+};
+DECLARE_PER_CPU_ALIGNED(struct stack_canary, stack_canary);
 #endif
 #endif  /* X86_64 */
 
@@ -703,13 +714,23 @@ static inline void cpu_relax(void)
         rep_nop();
 }
 
-/* Stop speculative execution: */
+/* Stop speculative execution and prefetching of modified code. */
 static inline void sync_core(void)
 {
         int tmp;
 
-        asm volatile("cpuid" : "=a" (tmp) : "0" (1)
-                     : "ebx", "ecx", "edx", "memory");
+#if defined(CONFIG_M386) || defined(CONFIG_M486)
+        if (boot_cpu_data.x86 < 5)
+                /* There is no speculative execution.
+                 * jmp is a barrier to prefetching. */
+                asm volatile("jmp 1f\n1:\n" ::: "memory");
+        else
+#endif
+                /* cpuid is a barrier to speculative execution.
+                 * Prefetched instructions are automatically
+                 * invalidated when modified. */
+                asm volatile("cpuid" : "=a" (tmp) : "0" (1)
+                             : "ebx", "ecx", "edx", "memory");
 }
 
 static inline void __monitor(const void *eax, unsigned long ecx,
@@ -1000,4 +1021,35 @@ extern void start_thread(struct pt_regs *regs, unsigned long new_ip,
 extern int get_tsc_mode(unsigned long adr);
 extern int set_tsc_mode(unsigned int val);
 
+extern int amd_get_nb_id(int cpu);
+
+struct aperfmperf {
+        u64 aperf, mperf;
+};
+
+static inline void get_aperfmperf(struct aperfmperf *am)
+{
+        WARN_ON_ONCE(!boot_cpu_has(X86_FEATURE_APERFMPERF));
+
+        rdmsrl(MSR_IA32_APERF, am->aperf);
+        rdmsrl(MSR_IA32_MPERF, am->mperf);
+}
+
+#define APERFMPERF_SHIFT 10
+
+static inline
+unsigned long calc_aperfmperf_ratio(struct aperfmperf *old,
+                                    struct aperfmperf *new)
+{
+        u64 aperf = new->aperf - old->aperf;
+        u64 mperf = new->mperf - old->mperf;
+        unsigned long ratio = aperf;
+
+        mperf >>= APERFMPERF_SHIFT;
+        if (mperf)
+                ratio = div64_u64(aperf, mperf);
+
+        return ratio;
+}
+
 #endif /* _ASM_X86_PROCESSOR_H */
diff --git a/arch/x86/include/asm/scatterlist.h b/arch/x86/include/asm/scatterlist.h
index 263d397d2eef..75af592677ec 100644
--- a/arch/x86/include/asm/scatterlist.h
+++ b/arch/x86/include/asm/scatterlist.h
@@ -1,33 +1,8 @@
 #ifndef _ASM_X86_SCATTERLIST_H
 #define _ASM_X86_SCATTERLIST_H
 
-#include <asm/types.h>
-
-struct scatterlist {
-#ifdef CONFIG_DEBUG_SG
-        unsigned long   sg_magic;
-#endif
-        unsigned long   page_link;
-        unsigned int    offset;
-        unsigned int    length;
-        dma_addr_t      dma_address;
-        unsigned int    dma_length;
-};
-
-#define ARCH_HAS_SG_CHAIN
 #define ISA_DMA_THRESHOLD (0x00ffffff)
 
-/*
- * These macros should be used after a pci_map_sg call has been done
- * to get bus addresses of each of the SG entries and their lengths.
- * You should only work with the number of sg entries pci_map_sg
- * returns.
- */
-#define sg_dma_address(sg)      ((sg)->dma_address)
-#ifdef CONFIG_X86_32
-# define sg_dma_len(sg)         ((sg)->length)
-#else
-# define sg_dma_len(sg)         ((sg)->dma_length)
-#endif
+#include <asm-generic/scatterlist.h>
 
 #endif /* _ASM_X86_SCATTERLIST_H */
diff --git a/arch/x86/include/asm/setup.h b/arch/x86/include/asm/setup.h
index 4093d1ed6db2..18e496c98ff0 100644
--- a/arch/x86/include/asm/setup.h
+++ b/arch/x86/include/asm/setup.h
@@ -5,43 +5,6 @@
 
 #define COMMAND_LINE_SIZE 2048
 
-#ifndef __ASSEMBLY__
-
-/*
- * Any setup quirks to be performed?
- */
-struct mpc_cpu;
-struct mpc_bus;
-struct mpc_oemtable;
-
-struct x86_quirks {
-        int (*arch_pre_time_init)(void);
-        int (*arch_time_init)(void);
-        int (*arch_pre_intr_init)(void);
-        int (*arch_intr_init)(void);
-        int (*arch_trap_init)(void);
-        char * (*arch_memory_setup)(void);
-        int (*mach_get_smp_config)(unsigned int early);
-        int (*mach_find_smp_config)(unsigned int reserve);
-
-        int *mpc_record;
-        int (*mpc_apic_id)(struct mpc_cpu *m);
-        void (*mpc_oem_bus_info)(struct mpc_bus *m, char *name);
-        void (*mpc_oem_pci_bus)(struct mpc_bus *m);
-        void (*smp_read_mpc_oem)(struct mpc_oemtable *oemtable,
-                                unsigned short oemsize);
-        int (*setup_ioapic_ids)(void);
-};
-
-extern void x86_quirk_intr_init(void);
-
-extern void x86_quirk_trap_init(void);
-
-extern void x86_quirk_pre_time_init(void);
-extern void x86_quirk_time_init(void);
-
-#endif /* __ASSEMBLY__ */
-
 #ifdef __i386__
 
 #include <linux/pfn.h>
@@ -61,6 +24,7 @@ extern void x86_quirk_time_init(void);
 
 #ifndef __ASSEMBLY__
 #include <asm/bootparam.h>
+#include <asm/x86_init.h>
 
 /* Interrupt control for vSMPowered x86_64 systems */
 #ifdef CONFIG_X86_64
@@ -79,11 +43,16 @@ static inline void visws_early_detect(void) { }
 static inline int is_visws_box(void) { return 0; }
 #endif
 
-extern struct x86_quirks *x86_quirks;
 extern unsigned long saved_video_mode;
 
-#ifndef CONFIG_PARAVIRT
-#define paravirt_post_allocator_init()  do {} while (0)
+extern void reserve_standard_io_resources(void);
+extern void i386_reserve_resources(void);
+extern void setup_default_timer_irq(void);
+
+#ifdef CONFIG_X86_MRST
+extern void x86_mrst_early_setup(void);
+#else
+static inline void x86_mrst_early_setup(void) { }
 #endif
 
 #ifndef _SETUP
diff --git a/arch/x86/include/asm/shmbuf.h b/arch/x86/include/asm/shmbuf.h
index b51413b74971..83c05fc2de38 100644
--- a/arch/x86/include/asm/shmbuf.h
+++ b/arch/x86/include/asm/shmbuf.h
@@ -1,51 +1 @@
-#ifndef _ASM_X86_SHMBUF_H
-#define _ASM_X86_SHMBUF_H
-
-/*
- * The shmid64_ds structure for x86 architecture.
- * Note extra padding because this structure is passed back and forth
- * between kernel and user space.
- *
- * Pad space on 32 bit is left for:
- * - 64-bit time_t to solve y2038 problem
- * - 2 miscellaneous 32-bit values
- *
- * Pad space on 64 bit is left for:
- * - 2 miscellaneous 64-bit values
- */
-
-struct shmid64_ds {
-        struct ipc64_perm       shm_perm;       /* operation perms */
-        size_t                  shm_segsz;      /* size of segment (bytes) */
-        __kernel_time_t         shm_atime;      /* last attach time */
-#ifdef __i386__
-        unsigned long           __unused1;
-#endif
-        __kernel_time_t         shm_dtime;      /* last detach time */
-#ifdef __i386__
-        unsigned long           __unused2;
-#endif
-        __kernel_time_t         shm_ctime;      /* last change time */
-#ifdef __i386__
-        unsigned long           __unused3;
-#endif
-        __kernel_pid_t          shm_cpid;       /* pid of creator */
-        __kernel_pid_t          shm_lpid;       /* pid of last operator */
-        unsigned long           shm_nattch;     /* no. of current attaches */
-        unsigned long           __unused4;
-        unsigned long           __unused5;
-};
-
-struct shminfo64 {
-        unsigned long   shmmax;
-        unsigned long   shmmin;
-        unsigned long   shmmni;
-        unsigned long   shmseg;
-        unsigned long   shmall;
-        unsigned long   __unused1;
-        unsigned long   __unused2;
-        unsigned long   __unused3;
-        unsigned long   __unused4;
-};
-
-#endif /* _ASM_X86_SHMBUF_H */
+#include <asm-generic/shmbuf.h>
diff --git a/arch/x86/include/asm/smp.h b/arch/x86/include/asm/smp.h
index 6a84ed166aec..1e796782cd7b 100644
--- a/arch/x86/include/asm/smp.h
+++ b/arch/x86/include/asm/smp.h
@@ -121,7 +121,6 @@ static inline void arch_send_call_function_single_ipi(int cpu)
         smp_ops.send_call_func_single_ipi(cpu);
 }
 
-#define arch_send_call_function_ipi_mask arch_send_call_function_ipi_mask
 static inline void arch_send_call_function_ipi_mask(const struct cpumask *mask)
 {
         smp_ops.send_call_func_ipi(mask);
diff --git a/arch/x86/include/asm/socket.h b/arch/x86/include/asm/socket.h
index ca8bf2cd0ba9..6b71384b9d8b 100644
--- a/arch/x86/include/asm/socket.h
+++ b/arch/x86/include/asm/socket.h
@@ -1,60 +1 @@
-#ifndef _ASM_X86_SOCKET_H
-#define _ASM_X86_SOCKET_H
-
-#include <asm/sockios.h>
-
-/* For setsockopt(2) */
-#define SOL_SOCKET      1
-
-#define SO_DEBUG        1
-#define SO_REUSEADDR    2
-#define SO_TYPE         3
-#define SO_ERROR        4
-#define SO_DONTROUTE    5
-#define SO_BROADCAST    6
-#define SO_SNDBUF       7
-#define SO_RCVBUF       8
-#define SO_SNDBUFFORCE  32
-#define SO_RCVBUFFORCE  33
-#define SO_KEEPALIVE    9
-#define SO_OOBINLINE    10
-#define SO_NO_CHECK     11
-#define SO_PRIORITY     12
-#define SO_LINGER       13
-#define SO_BSDCOMPAT    14
-/* To add :#define SO_REUSEPORT 15 */
-#define SO_PASSCRED     16
-#define SO_PEERCRED     17
-#define SO_RCVLOWAT     18
-#define SO_SNDLOWAT     19
-#define SO_RCVTIMEO     20
-#define SO_SNDTIMEO     21
-
-/* Security levels - as per NRL IPv6 - don't actually do anything */
-#define SO_SECURITY_AUTHENTICATION              22
-#define SO_SECURITY_ENCRYPTION_TRANSPORT        23
-#define SO_SECURITY_ENCRYPTION_NETWORK          24
-
-#define SO_BINDTODEVICE 25
-
-/* Socket filtering */
-#define SO_ATTACH_FILTER        26
-#define SO_DETACH_FILTER        27
-
-#define SO_PEERNAME             28
-#define SO_TIMESTAMP            29
-#define SCM_TIMESTAMP           SO_TIMESTAMP
-
-#define SO_ACCEPTCONN           30
-
-#define SO_PEERSEC              31
-#define SO_PASSSEC              34
-#define SO_TIMESTAMPNS          35
-#define SCM_TIMESTAMPNS         SO_TIMESTAMPNS
-
-#define SO_MARK                 36
-
-#define SO_TIMESTAMPING         37
-#define SCM_TIMESTAMPING        SO_TIMESTAMPING
-
-#endif /* _ASM_X86_SOCKET_H */
+#include <asm-generic/socket.h>
diff --git a/arch/x86/include/asm/sockios.h b/arch/x86/include/asm/sockios.h
index 49cc72b5d3c9..def6d4746ee7 100644
--- a/arch/x86/include/asm/sockios.h
+++ b/arch/x86/include/asm/sockios.h
@@ -1,13 +1 @@
-#ifndef _ASM_X86_SOCKIOS_H
-#define _ASM_X86_SOCKIOS_H
-
-/* Socket-level I/O control calls. */
-#define FIOSETOWN       0x8901
-#define SIOCSPGRP       0x8902
-#define FIOGETOWN       0x8903
-#define SIOCGPGRP       0x8904
-#define SIOCATMARK      0x8905
-#define SIOCGSTAMP      0x8906          /* Get stamp (timeval) */
-#define SIOCGSTAMPNS    0x8907          /* Get stamp (timespec) */
-
-#endif /* _ASM_X86_SOCKIOS_H */
+#include <asm-generic/sockios.h>
diff --git a/arch/x86/include/asm/stackprotector.h b/arch/x86/include/asm/stackprotector.h
index c2d742c6e15f..157517763565 100644
--- a/arch/x86/include/asm/stackprotector.h
+++ b/arch/x86/include/asm/stackprotector.h
@@ -48,7 +48,7 @@
  * head_32 for boot CPU and setup_per_cpu_areas() for others.
  */
 #define GDT_STACK_CANARY_INIT                                           \
-        [GDT_ENTRY_STACK_CANARY] = { { { 0x00000018, 0x00409000 } } },
+        [GDT_ENTRY_STACK_CANARY] = GDT_ENTRY_INIT(0x4090, 0, 0x18),
 
 /*
  * Initialize the stackprotector canary value.
@@ -78,21 +78,19 @@ static __always_inline void boot_init_stack_canary(void)
 #ifdef CONFIG_X86_64
         percpu_write(irq_stack_union.stack_canary, canary);
 #else
-        percpu_write(stack_canary, canary);
+        percpu_write(stack_canary.canary, canary);
 #endif
 }
 
 static inline void setup_stack_canary_segment(int cpu)
 {
 #ifdef CONFIG_X86_32
-        unsigned long canary = (unsigned long)&per_cpu(stack_canary, cpu) - 20;
+        unsigned long canary = (unsigned long)&per_cpu(stack_canary, cpu);
         struct desc_struct *gdt_table = get_cpu_gdt_table(cpu);
         struct desc_struct desc;
 
         desc = gdt_table[GDT_ENTRY_STACK_CANARY];
-        desc.base0 = canary & 0xffff;
-        desc.base1 = (canary >> 16) & 0xff;
-        desc.base2 = (canary >> 24) & 0xff;
+        set_desc_base(&desc, canary);
         write_gdt_entry(gdt_table, GDT_ENTRY_STACK_CANARY, &desc, DESCTYPE_S);
 #endif
 }
diff --git a/arch/x86/include/asm/string_32.h b/arch/x86/include/asm/string_32.h
index c86f452256de..ae907e617181 100644
--- a/arch/x86/include/asm/string_32.h
+++ b/arch/x86/include/asm/string_32.h
@@ -65,7 +65,6 @@ static __always_inline void *__constant_memcpy(void *to, const void *from,
         case 4:
                 *(int *)to = *(int *)from;
                 return to;
-
         case 3:
                 *(short *)to = *(short *)from;
                 *((char *)to + 2) = *((char *)from + 2);
diff --git a/arch/x86/include/asm/syscall.h b/arch/x86/include/asm/syscall.h
index d82f39bb7905..8d33bc5462d1 100644
--- a/arch/x86/include/asm/syscall.h
+++ b/arch/x86/include/asm/syscall.h
@@ -1,7 +1,7 @@
 /*
  * Access to user system call parameters and results
  *
- * Copyright (C) 2008 Red Hat, Inc.  All rights reserved.
+ * Copyright (C) 2008-2009 Red Hat, Inc.  All rights reserved.
  *
  * This copyrighted material is made available to anyone wishing to use,
  * modify, copy, or redistribute it subject to the terms and conditions
@@ -16,13 +16,13 @@
 #include <linux/sched.h>
 #include <linux/err.h>
 
-static inline long syscall_get_nr(struct task_struct *task,
-                                  struct pt_regs *regs)
+/*
+ * Only the low 32 bits of orig_ax are meaningful, so we return int.
+ * This importantly ignores the high bits on 64-bit, so comparisons
+ * sign-extend the low 32 bits.
+ */
+static inline int syscall_get_nr(struct task_struct *task, struct pt_regs *regs)
 {
-        /*
-         * We always sign-extend a -1 value being set here,
-         * so this is always either -1L or a syscall number.
-         */
         return regs->orig_ax;
 }
 
diff --git a/arch/x86/include/asm/system.h b/arch/x86/include/asm/system.h
index 643c59b4bc6e..f08f97374892 100644
--- a/arch/x86/include/asm/system.h
+++ b/arch/x86/include/asm/system.h
@@ -31,7 +31,7 @@ void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p,
         "movl %P[task_canary](%[next]), %%ebx\n\t"                      \
         "movl %%ebx, "__percpu_arg([stack_canary])"\n\t"
 #define __switch_canary_oparam                                          \
-        , [stack_canary] "=m" (per_cpu_var(stack_canary))
+        , [stack_canary] "=m" (per_cpu_var(stack_canary.canary))
 #define __switch_canary_iparam                                          \
         , [task_canary] "i" (offsetof(struct task_struct, stack_canary))
 #else   /* CC_STACKPROTECTOR */
@@ -150,33 +150,6 @@ do {									\
 #endif
 
 #ifdef __KERNEL__
-#define _set_base(addr, base) do { unsigned long __pr; \
-__asm__ __volatile__ ("movw %%dx,%1\n\t" \
-        "rorl $16,%%edx\n\t" \
-        "movb %%dl,%2\n\t" \
-        "movb %%dh,%3" \
-        :"=&d" (__pr) \
-        :"m" (*((addr)+2)), \
-         "m" (*((addr)+4)), \
-         "m" (*((addr)+7)), \
-         "0" (base) \
-        ); } while (0)
-
-#define _set_limit(addr, limit) do { unsigned long __lr; \
-__asm__ __volatile__ ("movw %%dx,%1\n\t" \
-        "rorl $16,%%edx\n\t" \
-        "movb %2,%%dh\n\t" \
-        "andb $0xf0,%%dh\n\t" \
-        "orb %%dh,%%dl\n\t" \
-        "movb %%dl,%2" \
-        :"=&d" (__lr) \
-        :"m" (*(addr)), \
-         "m" (*((addr)+6)), \
-         "0" (limit) \
-        ); } while (0)
-
-#define set_base(ldt, base) _set_base(((char *)&(ldt)) , (base))
-#define set_limit(ldt, limit) _set_limit(((char *)&(ldt)) , ((limit)-1))
 
 extern void native_load_gs_index(unsigned);
 
diff --git a/arch/x86/include/asm/termbits.h b/arch/x86/include/asm/termbits.h
index af1b70ea440f..3935b106de79 100644
--- a/arch/x86/include/asm/termbits.h
+++ b/arch/x86/include/asm/termbits.h
@@ -1,198 +1 @@
-#ifndef _ASM_X86_TERMBITS_H
-#define _ASM_X86_TERMBITS_H
-
-#include <linux/posix_types.h>
-
-typedef unsigned char   cc_t;
-typedef unsigned int    speed_t;
-typedef unsigned int    tcflag_t;
-
-#define NCCS 19
-struct termios {
-        tcflag_t c_iflag;               /* input mode flags */
-        tcflag_t c_oflag;               /* output mode flags */
-        tcflag_t c_cflag;               /* control mode flags */
-        tcflag_t c_lflag;               /* local mode flags */
-        cc_t c_line;                    /* line discipline */
-        cc_t c_cc[NCCS];                /* control characters */
-};
-
-struct termios2 {
-        tcflag_t c_iflag;               /* input mode flags */
-        tcflag_t c_oflag;               /* output mode flags */
-        tcflag_t c_cflag;               /* control mode flags */
-        tcflag_t c_lflag;               /* local mode flags */
-        cc_t c_line;                    /* line discipline */
-        cc_t c_cc[NCCS];                /* control characters */
-        speed_t c_ispeed;               /* input speed */
-        speed_t c_ospeed;               /* output speed */
-};
-
-struct ktermios {
-        tcflag_t c_iflag;               /* input mode flags */
-        tcflag_t c_oflag;               /* output mode flags */
-        tcflag_t c_cflag;               /* control mode flags */
-        tcflag_t c_lflag;               /* local mode flags */
-        cc_t c_line;                    /* line discipline */
-        cc_t c_cc[NCCS];                /* control characters */
-        speed_t c_ispeed;               /* input speed */
-        speed_t c_ospeed;               /* output speed */
-};
-
-/* c_cc characters */
-#define VINTR 0
-#define VQUIT 1
-#define VERASE 2
-#define VKILL 3
-#define VEOF 4
-#define VTIME 5
-#define VMIN 6
-#define VSWTC 7
-#define VSTART 8
-#define VSTOP 9
-#define VSUSP 10
-#define VEOL 11
-#define VREPRINT 12
-#define VDISCARD 13
-#define VWERASE 14
-#define VLNEXT 15
-#define VEOL2 16
-
-/* c_iflag bits */
-#define IGNBRK  0000001
-#define BRKINT  0000002
-#define IGNPAR  0000004
-#define PARMRK  0000010
-#define INPCK   0000020
-#define ISTRIP  0000040
-#define INLCR   0000100
-#define IGNCR   0000200
-#define ICRNL   0000400
-#define IUCLC   0001000
-#define IXON    0002000
-#define IXANY   0004000
-#define IXOFF   0010000
-#define IMAXBEL 0020000
-#define IUTF8   0040000
-
-/* c_oflag bits */
-#define OPOST   0000001
-#define OLCUC   0000002
-#define ONLCR   0000004
-#define OCRNL   0000010
-#define ONOCR   0000020
-#define ONLRET  0000040
-#define OFILL   0000100
-#define OFDEL   0000200
-#define NLDLY   0000400
-#define   NL0   0000000
-#define   NL1   0000400
-#define CRDLY   0003000
-#define   CR0   0000000
-#define   CR1   0001000
-#define   CR2   0002000
-#define   CR3   0003000
-#define TABDLY  0014000
-#define   TAB0  0000000
-#define   TAB1  0004000
-#define   TAB2  0010000
-#define   TAB3  0014000
-#define   XTABS 0014000
-#define BSDLY   0020000
-#define   BS0   0000000
-#define   BS1   0020000
-#define VTDLY   0040000
-#define   VT0   0000000
-#define   VT1   0040000
-#define FFDLY   0100000
-#define   FF0   0000000
-#define   FF1   0100000
-
-/* c_cflag bit meaning */
-#define CBAUD   0010017
-#define  B0     0000000         /* hang up */
-#define  B50    0000001
-#define  B75    0000002
-#define  B110   0000003
-#define  B134   0000004
-#define  B150   0000005
-#define  B200   0000006
-#define  B300   0000007
-#define  B600   0000010
-#define  B1200  0000011
-#define  B1800  0000012
-#define  B2400  0000013
-#define  B4800  0000014
-#define  B9600  0000015
-#define  B19200 0000016
-#define  B38400 0000017
-#define EXTA B19200
-#define EXTB B38400
-#define CSIZE   0000060
-#define   CS5   0000000
-#define   CS6   0000020
-#define   CS7   0000040
-#define   CS8   0000060
-#define CSTOPB  0000100
-#define CREAD   0000200
-#define PARENB  0000400
-#define PARODD  0001000
-#define HUPCL   0002000
-#define CLOCAL  0004000
-#define CBAUDEX 0010000
-#define    BOTHER 0010000               /* non standard rate */
-#define    B57600 0010001
-#define   B115200 0010002
-#define   B230400 0010003
-#define   B460800 0010004
-#define   B500000 0010005
-#define   B576000 0010006
-#define   B921600 0010007
-#define  B1000000 0010010
-#define  B1152000 0010011
-#define  B1500000 0010012
-#define  B2000000 0010013
-#define  B2500000 0010014
-#define  B3000000 0010015
-#define  B3500000 0010016
-#define  B4000000 0010017
-#define CIBAUD    002003600000  /* input baud rate */
-#define CMSPAR    010000000000  /* mark or space (stick) parity */
-#define CRTSCTS   020000000000  /* flow control */
-
-#define IBSHIFT   16            /* Shift from CBAUD to CIBAUD */
-
-/* c_lflag bits */
-#define ISIG    0000001
-#define ICANON  0000002
-#define XCASE   0000004
-#define ECHO    0000010
-#define ECHOE   0000020
-#define ECHOK   0000040
-#define ECHONL  0000100
-#define NOFLSH  0000200
-#define TOSTOP  0000400
-#define ECHOCTL 0001000
-#define ECHOPRT 0002000
-#define ECHOKE  0004000
-#define FLUSHO  0010000
-#define PENDIN  0040000
-#define IEXTEN  0100000
-
-/* tcflow() and TCXONC use these */
-#define TCOOFF          0
-#define TCOON           1
-#define TCIOFF          2
-#define TCION           3
-
-/* tcflush() and TCFLSH use these */
-#define TCIFLUSH        0
-#define TCOFLUSH        1
-#define TCIOFLUSH       2
-
-/* tcsetattr uses these */
-#define TCSANOW         0
-#define TCSADRAIN       1
-#define TCSAFLUSH       2
-
-#endif /* _ASM_X86_TERMBITS_H */
+#include <asm-generic/termbits.h>
diff --git a/arch/x86/include/asm/termios.h b/arch/x86/include/asm/termios.h
index c4ee8056baca..280d78a9d966 100644
--- a/arch/x86/include/asm/termios.h
+++ b/arch/x86/include/asm/termios.h
@@ -1,114 +1 @@
-#ifndef _ASM_X86_TERMIOS_H
-#define _ASM_X86_TERMIOS_H
-
-#include <asm/termbits.h>
-#include <asm/ioctls.h>
-
-struct winsize {
-        unsigned short ws_row;
-        unsigned short ws_col;
-        unsigned short ws_xpixel;
-        unsigned short ws_ypixel;
-};
-
-#define NCC 8
-struct termio {
-        unsigned short c_iflag;         /* input mode flags */
-        unsigned short c_oflag;         /* output mode flags */
-        unsigned short c_cflag;         /* control mode flags */
-        unsigned short c_lflag;         /* local mode flags */
-        unsigned char c_line;           /* line discipline */
-        unsigned char c_cc[NCC];        /* control characters */
-};
-
-/* modem lines */
-#define TIOCM_LE        0x001
-#define TIOCM_DTR       0x002
-#define TIOCM_RTS       0x004
-#define TIOCM_ST        0x008
-#define TIOCM_SR        0x010
-#define TIOCM_CTS       0x020
-#define TIOCM_CAR       0x040
-#define TIOCM_RNG       0x080
-#define TIOCM_DSR       0x100
-#define TIOCM_CD        TIOCM_CAR
-#define TIOCM_RI        TIOCM_RNG
-#define TIOCM_OUT1      0x2000
-#define TIOCM_OUT2      0x4000
-#define TIOCM_LOOP      0x8000
-
-/* ioctl (fd, TIOCSERGETLSR, &result) where result may be as below */
-
-#ifdef __KERNEL__
-
-#include <asm/uaccess.h>
-
-/*      intr=^C         quit=^\         erase=del       kill=^U
-        eof=^D          vtime=\0        vmin=\1         sxtc=\0
-        start=^Q        stop=^S         susp=^Z         eol=\0
-        reprint=^R      discard=^U      werase=^W       lnext=^V
-        eol2=\0
-*/
-#define INIT_C_CC "\003\034\177\025\004\0\1\0\021\023\032\0\022\017\027\026\0"
-
-/*
- * Translate a "termio" structure into a "termios". Ugh.
- */
-#define SET_LOW_TERMIOS_BITS(termios, termio, x) { \
-        unsigned short __tmp; \
-        get_user(__tmp,&(termio)->x); \
-        *(unsigned short *) &(termios)->x = __tmp; \
-}
-
-static inline int user_termio_to_kernel_termios(struct ktermios *termios,
-                                                struct termio __user *termio)
-{
-        SET_LOW_TERMIOS_BITS(termios, termio, c_iflag);
-        SET_LOW_TERMIOS_BITS(termios, termio, c_oflag);
-        SET_LOW_TERMIOS_BITS(termios, termio, c_cflag);
-        SET_LOW_TERMIOS_BITS(termios, termio, c_lflag);
-        get_user(termios->c_line, &termio->c_line);
-        return copy_from_user(termios->c_cc, termio->c_cc, NCC);
-}
-
-/*
- * Translate a "termios" structure into a "termio". Ugh.
- */
-static inline int kernel_termios_to_user_termio(struct termio __user *termio,
-                                            struct ktermios *termios)
-{
-        put_user((termios)->c_iflag, &(termio)->c_iflag);
-        put_user((termios)->c_oflag, &(termio)->c_oflag);
-        put_user((termios)->c_cflag, &(termio)->c_cflag);
-        put_user((termios)->c_lflag, &(termio)->c_lflag);
-        put_user((termios)->c_line,  &(termio)->c_line);
-        return copy_to_user((termio)->c_cc, (termios)->c_cc, NCC);
-}
-
-static inline int user_termios_to_kernel_termios(struct ktermios *k,
-                                                 struct termios2 __user *u)
-{
-        return copy_from_user(k, u, sizeof(struct termios2));
-}
-
-static inline int kernel_termios_to_user_termios(struct termios2 __user *u,
-                                                 struct ktermios *k)
-{
-        return copy_to_user(u, k, sizeof(struct termios2));
-}
-
-static inline int user_termios_to_kernel_termios_1(struct ktermios *k,
-                                                   struct termios __user *u)
-{
-        return copy_from_user(k, u, sizeof(struct termios));
-}
-
-static inline int kernel_termios_to_user_termios_1(struct termios __user *u,
-                                                   struct ktermios *k)
-{
-        return copy_to_user(u, k, sizeof(struct termios));
-}
-
-#endif  /* __KERNEL__ */
-
-#endif /* _ASM_X86_TERMIOS_H */
+#include <asm-generic/termios.h>
diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h
index fad7d40b75f8..d27d0a2fec4c 100644
--- a/arch/x86/include/asm/thread_info.h
+++ b/arch/x86/include/asm/thread_info.h
@@ -95,7 +95,7 @@ struct thread_info {
 #define TIF_DEBUGCTLMSR         25      /* uses thread_struct.debugctlmsr */
 #define TIF_DS_AREA_MSR         26      /* uses thread_struct.ds_area_msr */
 #define TIF_LAZY_MMU_UPDATES    27      /* task is updating the mmu lazily */
-#define TIF_SYSCALL_FTRACE      28      /* for ftrace syscall instrumentation */
+#define TIF_SYSCALL_TRACEPOINT  28      /* syscall tracepoint instrumentation */
 
 #define _TIF_SYSCALL_TRACE      (1 << TIF_SYSCALL_TRACE)
 #define _TIF_NOTIFY_RESUME      (1 << TIF_NOTIFY_RESUME)
@@ -118,17 +118,17 @@ struct thread_info {
 #define _TIF_DEBUGCTLMSR        (1 << TIF_DEBUGCTLMSR)
 #define _TIF_DS_AREA_MSR        (1 << TIF_DS_AREA_MSR)
 #define _TIF_LAZY_MMU_UPDATES   (1 << TIF_LAZY_MMU_UPDATES)
-#define _TIF_SYSCALL_FTRACE     (1 << TIF_SYSCALL_FTRACE)
+#define _TIF_SYSCALL_TRACEPOINT (1 << TIF_SYSCALL_TRACEPOINT)
 
 /* work to do in syscall_trace_enter() */
 #define _TIF_WORK_SYSCALL_ENTRY \
-        (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_EMU | _TIF_SYSCALL_FTRACE |  \
-         _TIF_SYSCALL_AUDIT | _TIF_SECCOMP | _TIF_SINGLESTEP)
+        (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_EMU | _TIF_SYSCALL_AUDIT |   \
+         _TIF_SECCOMP | _TIF_SINGLESTEP | _TIF_SYSCALL_TRACEPOINT)
 
 /* work to do in syscall_trace_leave() */
 #define _TIF_WORK_SYSCALL_EXIT  \
         (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | _TIF_SINGLESTEP |    \
-         _TIF_SYSCALL_FTRACE)
+         _TIF_SYSCALL_TRACEPOINT)
 
 /* work to do on interrupt/exception return */
 #define _TIF_WORK_MASK                                                  \
@@ -137,7 +137,8 @@ struct thread_info {
            _TIF_SINGLESTEP|_TIF_SECCOMP|_TIF_SYSCALL_EMU))
 
 /* work to do on any return to user space */
-#define _TIF_ALLWORK_MASK ((0x0000FFFF & ~_TIF_SECCOMP) | _TIF_SYSCALL_FTRACE)
+#define _TIF_ALLWORK_MASK                                               \
+        ((0x0000FFFF & ~_TIF_SECCOMP) | _TIF_SYSCALL_TRACEPOINT)
 
 /* Only used for 64 bit */
 #define _TIF_DO_NOTIFY_MASK                                             \
@@ -213,7 +214,7 @@ DECLARE_PER_CPU(unsigned long, kernel_stack);
 static inline struct thread_info *current_thread_info(void)
 {
         struct thread_info *ti;
-        ti = (void *)(percpu_read(kernel_stack) +
+        ti = (void *)(percpu_read_stable(kernel_stack) +
                       KERNEL_STACK_OFFSET - THREAD_SIZE);
         return ti;
 }
diff --git a/arch/x86/include/asm/time.h b/arch/x86/include/asm/time.h
index 50c733aac421..7bdec4e9b739 100644
--- a/arch/x86/include/asm/time.h
+++ b/arch/x86/include/asm/time.h
@@ -4,60 +4,7 @@
 extern void hpet_time_init(void);
 
 #include <asm/mc146818rtc.h>
-#ifdef CONFIG_X86_32
-#include <linux/efi.h>
-
-static inline unsigned long native_get_wallclock(void)
-{
-        unsigned long retval;
-
-        if (efi_enabled)
-                retval = efi_get_time();
-        else
-                retval = mach_get_cmos_time();
-
-        return retval;
-}
-
-static inline int native_set_wallclock(unsigned long nowtime)
-{
-        int retval;
-
-        if (efi_enabled)
-                retval = efi_set_rtc_mmss(nowtime);
-        else
-                retval = mach_set_rtc_mmss(nowtime);
-
-        return retval;
-}
-
-#else
-extern void native_time_init_hook(void);
-
-static inline unsigned long native_get_wallclock(void)
-{
-        return mach_get_cmos_time();
-}
-
-static inline int native_set_wallclock(unsigned long nowtime)
-{
-        return mach_set_rtc_mmss(nowtime);
-}
-
-#endif
 
 extern void time_init(void);
 
-#ifdef CONFIG_PARAVIRT
-#include <asm/paravirt.h>
-#else /* !CONFIG_PARAVIRT */
-
-#define get_wallclock() native_get_wallclock()
-#define set_wallclock(x) native_set_wallclock(x)
-#define choose_time_init() hpet_time_init
-
-#endif /* CONFIG_PARAVIRT */
-
-extern unsigned long __init calibrate_cpu(void);
-
 #endif /* _ASM_X86_TIME_H */
diff --git a/arch/x86/include/asm/timer.h b/arch/x86/include/asm/timer.h
index 20ca9c4d4686..5469630b27f5 100644
--- a/arch/x86/include/asm/timer.h
+++ b/arch/x86/include/asm/timer.h
@@ -8,20 +8,16 @@
 #define TICK_SIZE (tick_nsec / 1000)
 
 unsigned long long native_sched_clock(void);
-unsigned long native_calibrate_tsc(void);
+extern int recalibrate_cpu_khz(void);
 
-#ifdef CONFIG_X86_32
+#if defined(CONFIG_X86_32) && defined(CONFIG_X86_IO_APIC)
 extern int timer_ack;
-extern irqreturn_t timer_interrupt(int irq, void *dev_id);
-#endif /* CONFIG_X86_32 */
-extern int recalibrate_cpu_khz(void);
+#else
+# define timer_ack (0)
+#endif
 
 extern int no_timer_check;
 
-#ifndef CONFIG_PARAVIRT
-#define calibrate_tsc() native_calibrate_tsc()
-#endif
-
 /* Accelerators for sched_clock()
  * convert from cycles(64bits) => nanoseconds (64bits)
  *  basic equation:
diff --git a/arch/x86/include/asm/topology.h b/arch/x86/include/asm/topology.h
index 066ef590d7e0..6f0695d744bf 100644
--- a/arch/x86/include/asm/topology.h
+++ b/arch/x86/include/asm/topology.h
@@ -116,38 +116,41 @@ extern unsigned long node_remap_size[];
 
 # define SD_CACHE_NICE_TRIES    1
 # define SD_IDLE_IDX            1
-# define SD_NEWIDLE_IDX         2
-# define SD_FORKEXEC_IDX        0
 
 #else
 
 # define SD_CACHE_NICE_TRIES    2
 # define SD_IDLE_IDX            2
-# define SD_NEWIDLE_IDX         2
-# define SD_FORKEXEC_IDX        1
 
 #endif
 
 /* sched_domains SD_NODE_INIT for NUMA machines */
-#define SD_NODE_INIT (struct sched_domain) {            \
-        .min_interval           = 8,                    \
-        .max_interval           = 32,                   \
-        .busy_factor            = 32,                   \
-        .imbalance_pct          = 125,                  \
-        .cache_nice_tries       = SD_CACHE_NICE_TRIES,  \
-        .busy_idx               = 3,                    \
-        .idle_idx               = SD_IDLE_IDX,          \
-        .newidle_idx            = SD_NEWIDLE_IDX,       \
-        .wake_idx               = 1,                    \
-        .forkexec_idx           = SD_FORKEXEC_IDX,      \
-        .flags                  = SD_LOAD_BALANCE       \
-                                | SD_BALANCE_EXEC       \
-                                | SD_BALANCE_FORK       \
-                                | SD_WAKE_AFFINE        \
-                                | SD_WAKE_BALANCE       \
-                                | SD_SERIALIZE,         \
-        .last_balance           = jiffies,              \
-        .balance_interval       = 1,                    \
+#define SD_NODE_INIT (struct sched_domain) {                            \
+        .min_interval           = 8,                                    \
+        .max_interval           = 32,                                   \
+        .busy_factor            = 32,                                   \
+        .imbalance_pct          = 125,                                  \
+        .cache_nice_tries       = SD_CACHE_NICE_TRIES,                  \
+        .busy_idx               = 3,                                    \
+        .idle_idx               = SD_IDLE_IDX,                          \
+        .newidle_idx            = 0,                                    \
+        .wake_idx               = 0,                                    \
+        .forkexec_idx           = 0,                                    \
+                                                                        \
+        .flags                  = 1*SD_LOAD_BALANCE                     \
+                                | 1*SD_BALANCE_NEWIDLE                  \
+                                | 1*SD_BALANCE_EXEC                     \
+                                | 1*SD_BALANCE_FORK                     \
+                                | 0*SD_BALANCE_WAKE                     \
+                                | 1*SD_WAKE_AFFINE                      \
+                                | 0*SD_SHARE_CPUPOWER                   \
+                                | 0*SD_POWERSAVINGS_BALANCE             \
+                                | 0*SD_SHARE_PKG_RESOURCES              \
+                                | 1*SD_SERIALIZE                        \
+                                | 0*SD_PREFER_SIBLING                   \
+                                ,                                       \
+        .last_balance           = jiffies,                              \
+        .balance_interval       = 1,                                    \
 }
 
 #ifdef CONFIG_X86_64_ACPI_NUMA
diff --git a/arch/x86/include/asm/traps.h b/arch/x86/include/asm/traps.h
index bfd74c032fca..4da91ad69e0d 100644
--- a/arch/x86/include/asm/traps.h
+++ b/arch/x86/include/asm/traps.h
@@ -81,9 +81,7 @@ extern int panic_on_unrecovered_nmi;
 
 void math_error(void __user *);
 void math_emulate(struct math_emu_info *);
-#ifdef CONFIG_X86_32
-unsigned long patch_espfix_desc(unsigned long, unsigned long);
-#else
+#ifndef CONFIG_X86_32
 asmlinkage void smp_thermal_interrupt(void);
 asmlinkage void mce_threshold_interrupt(void);
 #endif
diff --git a/arch/x86/include/asm/tsc.h b/arch/x86/include/asm/tsc.h
index 38ae163cc91b..c0427295e8f5 100644
--- a/arch/x86/include/asm/tsc.h
+++ b/arch/x86/include/asm/tsc.h
@@ -48,7 +48,8 @@ static __always_inline cycles_t vget_cycles(void)
 extern void tsc_init(void);
 extern void mark_tsc_unstable(char *reason);
 extern int unsynchronized_tsc(void);
-int check_tsc_unstable(void);
+extern int check_tsc_unstable(void);
+extern unsigned long native_calibrate_tsc(void);
 
 /*
  * Boot-time check whether the TSCs are synchronized across
diff --git a/arch/x86/include/asm/types.h b/arch/x86/include/asm/types.h
index 09b97745772f..df1da20f4534 100644
--- a/arch/x86/include/asm/types.h
+++ b/arch/x86/include/asm/types.h
@@ -1,19 +1,11 @@
 #ifndef _ASM_X86_TYPES_H
 #define _ASM_X86_TYPES_H
 
-#include <asm-generic/int-ll64.h>
+#define dma_addr_t      dma_addr_t
 
-#ifndef __ASSEMBLY__
-
-typedef unsigned short umode_t;
+#include <asm-generic/types.h>
 
-#endif /* __ASSEMBLY__ */
-
-/*
- * These aren't exported outside the kernel to avoid name space clashes
- */
 #ifdef __KERNEL__
-
 #ifndef __ASSEMBLY__
 
 typedef u64 dma64_addr_t;
diff --git a/arch/x86/include/asm/uaccess_32.h b/arch/x86/include/asm/uaccess_32.h
index 5e06259e90e5..632fb44b4cb5 100644
--- a/arch/x86/include/asm/uaccess_32.h
+++ b/arch/x86/include/asm/uaccess_32.h
@@ -33,7 +33,7 @@ unsigned long __must_check __copy_from_user_ll_nocache_nozero
  * Copy data from kernel space to user space.  Caller must check
  * the specified block with access_ok() before calling this function.
  * The caller should also make sure he pins the user space address
- * so that the we don't result in page fault and sleep.
+ * so that we don't result in page fault and sleep.
  *
  * Here we special-case 1, 2 and 4-byte copy_*_user invocations.  On a fault
  * we return the initial request size (1, 2 or 4), as copy_*_user should do.
diff --git a/arch/x86/include/asm/ucontext.h b/arch/x86/include/asm/ucontext.h
index 87324cf439d9..b7c29c8017f2 100644
--- a/arch/x86/include/asm/ucontext.h
+++ b/arch/x86/include/asm/ucontext.h
@@ -7,12 +7,6 @@
                                  * sigcontext struct (uc_mcontext).
                                  */
 
-struct ucontext {
-        unsigned long     uc_flags;
-        struct ucontext  *uc_link;
-        stack_t           uc_stack;
-        struct sigcontext uc_mcontext;
-        sigset_t          uc_sigmask;   /* mask last for extensibility */
-};
+#include <asm-generic/ucontext.h>
 
 #endif /* _ASM_X86_UCONTEXT_H */
diff --git a/arch/x86/include/asm/unistd_32.h b/arch/x86/include/asm/unistd_32.h
index 732a30706153..6fb3c209a7e3 100644
--- a/arch/x86/include/asm/unistd_32.h
+++ b/arch/x86/include/asm/unistd_32.h
@@ -341,10 +341,12 @@
 #define __NR_preadv             333
 #define __NR_pwritev            334
 #define __NR_rt_tgsigqueueinfo  335
-#define __NR_perf_counter_open  336
+#define __NR_perf_event_open    336
 
 #ifdef __KERNEL__
 
+#define NR_syscalls 337
+
 #define __ARCH_WANT_IPC_PARSE_VERSION
 #define __ARCH_WANT_OLD_READDIR
 #define __ARCH_WANT_OLD_STAT
diff --git a/arch/x86/include/asm/unistd_64.h b/arch/x86/include/asm/unistd_64.h
index 900e1617e672..8d3ad0adbc68 100644
--- a/arch/x86/include/asm/unistd_64.h
+++ b/arch/x86/include/asm/unistd_64.h
@@ -659,8 +659,8 @@ __SYSCALL(__NR_preadv, sys_preadv)
 __SYSCALL(__NR_pwritev, sys_pwritev)
 #define __NR_rt_tgsigqueueinfo                  297
 __SYSCALL(__NR_rt_tgsigqueueinfo, sys_rt_tgsigqueueinfo)
-#define __NR_perf_counter_open                  298
-__SYSCALL(__NR_perf_counter_open, sys_perf_counter_open)
+#define __NR_perf_event_open                    298
+__SYSCALL(__NR_perf_event_open, sys_perf_event_open)
 
 #ifndef __NO_STUBS
 #define __ARCH_WANT_OLD_READDIR
@@ -688,6 +688,12 @@ __SYSCALL(__NR_perf_counter_open, sys_perf_counter_open)
 #endif  /* __NO_STUBS */
 
 #ifdef __KERNEL__
+
+#ifndef COMPILE_OFFSETS
+#include <asm/asm-offsets.h>
+#define NR_syscalls (__NR_syscall_max + 1)
+#endif
+
 /*
  * "Conditional" syscalls
  *
diff --git a/arch/x86/include/asm/uv/uv_hub.h b/arch/x86/include/asm/uv/uv_hub.h
index 77a68505419a..04eb6c958b9d 100644
--- a/arch/x86/include/asm/uv/uv_hub.h
+++ b/arch/x86/include/asm/uv/uv_hub.h
@@ -15,6 +15,7 @@
 #include <linux/numa.h>
 #include <linux/percpu.h>
 #include <linux/timer.h>
+#include <linux/io.h>
 #include <asm/types.h>
 #include <asm/percpu.h>
 #include <asm/uv/uv_mmrs.h>
@@ -258,13 +259,13 @@ static inline unsigned long *uv_global_mmr32_address(int pnode,
 static inline void uv_write_global_mmr32(int pnode, unsigned long offset,
                                  unsigned long val)
 {
-        *uv_global_mmr32_address(pnode, offset) = val;
+        writeq(val, uv_global_mmr32_address(pnode, offset));
 }
 
 static inline unsigned long uv_read_global_mmr32(int pnode,
                                                  unsigned long offset)
 {
-        return *uv_global_mmr32_address(pnode, offset);
+        return readq(uv_global_mmr32_address(pnode, offset));
 }
 
 /*
@@ -281,13 +282,13 @@ static inline unsigned long *uv_global_mmr64_address(int pnode,
 static inline void uv_write_global_mmr64(int pnode, unsigned long offset,
                                 unsigned long val)
 {
-        *uv_global_mmr64_address(pnode, offset) = val;
+        writeq(val, uv_global_mmr64_address(pnode, offset));
 }
 
 static inline unsigned long uv_read_global_mmr64(int pnode,
                                                  unsigned long offset)
 {
-        return *uv_global_mmr64_address(pnode, offset);
+        return readq(uv_global_mmr64_address(pnode, offset));
 }
 
 /*
@@ -301,22 +302,22 @@ static inline unsigned long *uv_local_mmr_address(unsigned long offset)
 
 static inline unsigned long uv_read_local_mmr(unsigned long offset)
 {
-        return *uv_local_mmr_address(offset);
+        return readq(uv_local_mmr_address(offset));
 }
 
 static inline void uv_write_local_mmr(unsigned long offset, unsigned long val)
 {
-        *uv_local_mmr_address(offset) = val;
+        writeq(val, uv_local_mmr_address(offset));
 }
 
 static inline unsigned char uv_read_local_mmr8(unsigned long offset)
 {
-        return *((unsigned char *)uv_local_mmr_address(offset));
+        return readb(uv_local_mmr_address(offset));
 }
 
 static inline void uv_write_local_mmr8(unsigned long offset, unsigned char val)
 {
-        *((unsigned char *)uv_local_mmr_address(offset)) = val;
+        writeb(val, uv_local_mmr_address(offset));
 }
 
 /*
@@ -422,7 +423,7 @@ static inline void uv_hub_send_ipi(int pnode, int apicid, int vector)
         unsigned long val;
 
         val = (1UL << UVH_IPI_INT_SEND_SHFT) |
-                        ((apicid & 0x3f) << UVH_IPI_INT_APIC_ID_SHFT) |
+                        ((apicid) << UVH_IPI_INT_APIC_ID_SHFT) |
                         (vector << UVH_IPI_INT_VECTOR_SHFT);
         uv_write_global_mmr64(pnode, UVH_IPI_INT, val);
 }
diff --git a/arch/x86/include/asm/vgtod.h b/arch/x86/include/asm/vgtod.h
index dc27a69e5d2a..3d61e204826f 100644
--- a/arch/x86/include/asm/vgtod.h
+++ b/arch/x86/include/asm/vgtod.h
@@ -21,6 +21,7 @@ struct vsyscall_gtod_data {
                 u32     shift;
         } clock;
         struct timespec wall_to_monotonic;
+        struct timespec wall_time_coarse;
 };
 extern struct vsyscall_gtod_data __vsyscall_gtod_data
 __section_vsyscall_gtod_data;
diff --git a/arch/x86/include/asm/vmware.h b/arch/x86/include/asm/vmware.h
index c11b7e100d83..e49ed6d2fd4e 100644
--- a/arch/x86/include/asm/vmware.h
+++ b/arch/x86/include/asm/vmware.h
@@ -20,7 +20,7 @@
 #ifndef ASM_X86__VMWARE_H
 #define ASM_X86__VMWARE_H
 
-extern unsigned long vmware_get_tsc_khz(void);
+extern void vmware_platform_setup(void);
 extern int vmware_platform(void);
 extern void vmware_set_feature_bits(struct cpuinfo_x86 *c);
 
diff --git a/arch/x86/include/asm/vmx.h b/arch/x86/include/asm/vmx.h
index 11be5ad2e0e9..272514c2d456 100644
--- a/arch/x86/include/asm/vmx.h
+++ b/arch/x86/include/asm/vmx.h
@@ -55,6 +55,7 @@
 #define SECONDARY_EXEC_ENABLE_EPT               0x00000002
 #define SECONDARY_EXEC_ENABLE_VPID              0x00000020
 #define SECONDARY_EXEC_WBINVD_EXITING           0x00000040
+#define SECONDARY_EXEC_UNRESTRICTED_GUEST       0x00000080
 
 
 #define PIN_BASED_EXT_INTR_MASK                 0x00000001
@@ -351,9 +352,16 @@ enum vmcs_field {
 #define VMX_EPT_EXTENT_INDIVIDUAL_ADDR          0
 #define VMX_EPT_EXTENT_CONTEXT                  1
 #define VMX_EPT_EXTENT_GLOBAL                   2
+
+#define VMX_EPT_EXECUTE_ONLY_BIT                (1ull)
+#define VMX_EPT_PAGE_WALK_4_BIT                 (1ull << 6)
+#define VMX_EPTP_UC_BIT                         (1ull << 8)
+#define VMX_EPTP_WB_BIT                         (1ull << 14)
+#define VMX_EPT_2MB_PAGE_BIT                    (1ull << 16)
 #define VMX_EPT_EXTENT_INDIVIDUAL_BIT           (1ull << 24)
 #define VMX_EPT_EXTENT_CONTEXT_BIT              (1ull << 25)
 #define VMX_EPT_EXTENT_GLOBAL_BIT               (1ull << 26)
+
 #define VMX_EPT_DEFAULT_GAW                     3
 #define VMX_EPT_MAX_GAW                         0x4
 #define VMX_EPT_MT_EPTE_SHIFT                   3
diff --git a/arch/x86/include/asm/x86_init.h b/arch/x86/include/asm/x86_init.h
new file mode 100644
index 000000000000..2c756fd4ab0e
--- /dev/null
+++ b/arch/x86/include/asm/x86_init.h
@@ -0,0 +1,133 @@
+#ifndef _ASM_X86_PLATFORM_H
+#define _ASM_X86_PLATFORM_H
+
+#include <asm/pgtable_types.h>
+#include <asm/bootparam.h>
+
+struct mpc_bus;
+struct mpc_cpu;
+struct mpc_table;
+
+/**
+ * struct x86_init_mpparse - platform specific mpparse ops
+ * @mpc_record:                 platform specific mpc record accounting
+ * @setup_ioapic_ids:           platform specific ioapic id override
+ * @mpc_apic_id:                platform specific mpc apic id assignment
+ * @smp_read_mpc_oem:           platform specific oem mpc table setup
+ * @mpc_oem_pci_bus:            platform specific pci bus setup (default NULL)
+ * @mpc_oem_bus_info:           platform specific mpc bus info
+ * @find_smp_config:            find the smp configuration
+ * @get_smp_config:             get the smp configuration
+ */
+struct x86_init_mpparse {
+        void (*mpc_record)(unsigned int mode);
+        void (*setup_ioapic_ids)(void);
+        int (*mpc_apic_id)(struct mpc_cpu *m);
+        void (*smp_read_mpc_oem)(struct mpc_table *mpc);
+        void (*mpc_oem_pci_bus)(struct mpc_bus *m);
+        void (*mpc_oem_bus_info)(struct mpc_bus *m, char *name);
+        void (*find_smp_config)(unsigned int reserve);
+        void (*get_smp_config)(unsigned int early);
+};
+
+/**
+ * struct x86_init_resources - platform specific resource related ops
+ * @probe_roms:                 probe BIOS roms
+ * @reserve_resources:          reserve the standard resources for the
+ *                              platform
+ * @memory_setup:               platform specific memory setup
+ *
+ */
+struct x86_init_resources {
+        void (*probe_roms)(void);
+        void (*reserve_resources)(void);
+        char *(*memory_setup)(void);
+};
+
+/**
+ * struct x86_init_irqs - platform specific interrupt setup
+ * @pre_vector_init:            init code to run before interrupt vectors
+ *                              are set up.
+ * @intr_init:                  interrupt init code
+ * @trap_init:                  platform specific trap setup
+ */
+struct x86_init_irqs {
+        void (*pre_vector_init)(void);
+        void (*intr_init)(void);
+        void (*trap_init)(void);
+};
+
+/**
+ * struct x86_init_oem - oem platform specific customizing functions
+ * @arch_setup:                 platform specific architecure setup
+ * @banner:                     print a platform specific banner
+ */
+struct x86_init_oem {
+        void (*arch_setup)(void);
+        void (*banner)(void);
+};
+
+/**
+ * struct x86_init_paging - platform specific paging functions
+ * @pagetable_setup_start:      platform specific pre paging_init() call
+ * @pagetable_setup_done:       platform specific post paging_init() call
+ */
+struct x86_init_paging {
+        void (*pagetable_setup_start)(pgd_t *base);
+        void (*pagetable_setup_done)(pgd_t *base);
+};
+
+/**
+ * struct x86_init_timers - platform specific timer setup
+ * @setup_perpcu_clockev:       set up the per cpu clock event device for the
+ *                              boot cpu
+ * @tsc_pre_init:               platform function called before TSC init
+ * @timer_init:                 initialize the platform timer (default PIT/HPET)
+ */
+struct x86_init_timers {
+        void (*setup_percpu_clockev)(void);
+        void (*tsc_pre_init)(void);
+        void (*timer_init)(void);
+};
+
+/**
+ * struct x86_init_ops - functions for platform specific setup
+ *
+ */
+struct x86_init_ops {
+        struct x86_init_resources       resources;
+        struct x86_init_mpparse         mpparse;
+        struct x86_init_irqs            irqs;
+        struct x86_init_oem             oem;
+        struct x86_init_paging          paging;
+        struct x86_init_timers          timers;
+};
+
+/**
+ * struct x86_cpuinit_ops - platform specific cpu hotplug setups
+ * @setup_percpu_clockev:       set up the per cpu clock event device
+ */
+struct x86_cpuinit_ops {
+        void (*setup_percpu_clockev)(void);
+};
+
+/**
+ * struct x86_platform_ops - platform specific runtime functions
+ * @calibrate_tsc:              calibrate TSC
+ * @get_wallclock:              get time from HW clock like RTC etc.
+ * @set_wallclock:              set time back to HW clock
+ */
+struct x86_platform_ops {
+        unsigned long (*calibrate_tsc)(void);
+        unsigned long (*get_wallclock)(void);
+        int (*set_wallclock)(unsigned long nowtime);
+};
+
+extern struct x86_init_ops x86_init;
+extern struct x86_cpuinit_ops x86_cpuinit;
+extern struct x86_platform_ops x86_platform;
+
+extern void x86_init_noop(void);
+extern void x86_init_uint_noop(unsigned int unused);
+
+#endif
diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile
index 430d5b24af7b..d8e5d0cdd678 100644
--- a/arch/x86/kernel/Makefile
+++ b/arch/x86/kernel/Makefile
@@ -31,8 +31,8 @@ GCOV_PROFILE_paravirt.o		:= n
 
 obj-y                   := process_$(BITS).o signal.o entry_$(BITS).o
 obj-y                   += traps.o irq.o irq_$(BITS).o dumpstack_$(BITS).o
-obj-y                   += time_$(BITS).o ioport.o ldt.o dumpstack.o
-obj-y                   += setup.o i8259.o irqinit.o
+obj-y                   += time.o ioport.o ldt.o dumpstack.o
+obj-y                   += setup.o x86_init.o i8259.o irqinit.o
 obj-$(CONFIG_X86_VISWS) += visws_quirks.o
 obj-$(CONFIG_X86_32)    += probe_roms_32.o
 obj-$(CONFIG_X86_32)    += sys_i386_32.o i386_ksyms_32.o
@@ -52,9 +52,11 @@ obj-$(CONFIG_X86_DS_SELFTEST)		+= ds_selftest.o
 obj-$(CONFIG_X86_32)            += tls.o
 obj-$(CONFIG_IA32_EMULATION)    += tls.o
 obj-y                           += step.o
+obj-$(CONFIG_INTEL_TXT)         += tboot.o
 obj-$(CONFIG_STACKTRACE)        += stacktrace.o
 obj-y                           += cpu/
 obj-y                           += acpi/
+obj-$(CONFIG_SFI)               += sfi.o
 obj-y                           += reboot.o
 obj-$(CONFIG_MCA)               += mca_32.o
 obj-$(CONFIG_X86_MSR)           += msr.o
@@ -104,6 +106,7 @@ obj-$(CONFIG_SCx200)		+= scx200.o
 scx200-y                        += scx200_32.o
 
 obj-$(CONFIG_OLPC)              += olpc.o
+obj-$(CONFIG_X86_MRST)          += mrst.o
 
 microcode-y                             := microcode_core.o
 microcode-$(CONFIG_MICROCODE_INTEL)     += microcode_intel.o
diff --git a/arch/x86/kernel/acpi/boot.c b/arch/x86/kernel/acpi/boot.c
index 6b8ca3a0285d..67e929b89875 100644
--- a/arch/x86/kernel/acpi/boot.c
+++ b/arch/x86/kernel/acpi/boot.c
@@ -833,106 +833,6 @@ static int __init acpi_parse_madt_lapic_entries(void)
 extern int es7000_plat;
 #endif
 
-static struct {
-        int gsi_base;
-        int gsi_end;
-} mp_ioapic_routing[MAX_IO_APICS];
-
-int mp_find_ioapic(int gsi)
-{
-        int i = 0;
-
-        /* Find the IOAPIC that manages this GSI. */
-        for (i = 0; i < nr_ioapics; i++) {
-                if ((gsi >= mp_ioapic_routing[i].gsi_base)
-                    && (gsi <= mp_ioapic_routing[i].gsi_end))
-                        return i;
-        }
-
-        printk(KERN_ERR "ERROR: Unable to locate IOAPIC for GSI %d\n", gsi);
-        return -1;
-}
-
-int mp_find_ioapic_pin(int ioapic, int gsi)
-{
-        if (WARN_ON(ioapic == -1))
-                return -1;
-        if (WARN_ON(gsi > mp_ioapic_routing[ioapic].gsi_end))
-                return -1;
-
-        return gsi - mp_ioapic_routing[ioapic].gsi_base;
-}
-
-static u8 __init uniq_ioapic_id(u8 id)
-{
-#ifdef CONFIG_X86_32
-        if ((boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) &&
-            !APIC_XAPIC(apic_version[boot_cpu_physical_apicid]))
-                return io_apic_get_unique_id(nr_ioapics, id);
-        else
-                return id;
-#else
-        int i;
-        DECLARE_BITMAP(used, 256);
-        bitmap_zero(used, 256);
-        for (i = 0; i < nr_ioapics; i++) {
-                struct mpc_ioapic *ia = &mp_ioapics[i];
-                __set_bit(ia->apicid, used);
-        }
-        if (!test_bit(id, used))
-                return id;
-        return find_first_zero_bit(used, 256);
-#endif
-}
-
-static int bad_ioapic(unsigned long address)
-{
-        if (nr_ioapics >= MAX_IO_APICS) {
-                printk(KERN_ERR "ERROR: Max # of I/O APICs (%d) exceeded "
-                       "(found %d)\n", MAX_IO_APICS, nr_ioapics);
-                panic("Recompile kernel with bigger MAX_IO_APICS!\n");
-        }
-        if (!address) {
-                printk(KERN_ERR "WARNING: Bogus (zero) I/O APIC address"
-                       " found in table, skipping!\n");
-                return 1;
-        }
-        return 0;
-}
-
-void __init mp_register_ioapic(int id, u32 address, u32 gsi_base)
-{
-        int idx = 0;
-
-        if (bad_ioapic(address))
-                return;
-
-        idx = nr_ioapics;
-
-        mp_ioapics[idx].type = MP_IOAPIC;
-        mp_ioapics[idx].flags = MPC_APIC_USABLE;
-        mp_ioapics[idx].apicaddr = address;
-
-        set_fixmap_nocache(FIX_IO_APIC_BASE_0 + idx, address);
-        mp_ioapics[idx].apicid = uniq_ioapic_id(id);
-        mp_ioapics[idx].apicver = io_apic_get_version(idx);
-
-        /*
-         * Build basic GSI lookup table to facilitate gsi->io_apic lookups
-         * and to prevent reprogramming of IOAPIC pins (PCI GSIs).
-         */
-        mp_ioapic_routing[idx].gsi_base = gsi_base;
-        mp_ioapic_routing[idx].gsi_end = gsi_base +
-            io_apic_get_redir_entries(idx);
-
-        printk(KERN_INFO "IOAPIC[%d]: apic_id %d, version %d, address 0x%x, "
-               "GSI %d-%d\n", idx, mp_ioapics[idx].apicid,
-               mp_ioapics[idx].apicver, mp_ioapics[idx].apicaddr,
-               mp_ioapic_routing[idx].gsi_base, mp_ioapic_routing[idx].gsi_end);
-
-        nr_ioapics++;
-}
-
 int __init acpi_probe_gsi(void)
 {
         int idx;
@@ -947,7 +847,7 @@ int __init acpi_probe_gsi(void)
 
         max_gsi = 0;
         for (idx = 0; idx < nr_ioapics; idx++) {
-                gsi = mp_ioapic_routing[idx].gsi_end;
+                gsi = mp_gsi_routing[idx].gsi_end;
 
                 if (gsi > max_gsi)
                         max_gsi = gsi;
@@ -1179,9 +1079,8 @@ static int __init acpi_parse_madt_ioapic_entries(void)
          * If MPS is present, it will handle them,
          * otherwise the system will stay in PIC mode
          */
-        if (acpi_disabled || acpi_noirq) {
+        if (acpi_disabled || acpi_noirq)
                 return -ENODEV;
-        }
 
         if (!cpu_has_apic)
                 return -ENODEV;
diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c
index f57658702571..de7353c0ce9c 100644
--- a/arch/x86/kernel/alternative.c
+++ b/arch/x86/kernel/alternative.c
@@ -2,6 +2,7 @@
 #include <linux/sched.h>
 #include <linux/mutex.h>
 #include <linux/list.h>
+#include <linux/stringify.h>
 #include <linux/kprobes.h>
 #include <linux/mm.h>
 #include <linux/vmalloc.h>
@@ -32,7 +33,7 @@ __setup("smp-alt-boot", bootonly);
 #define smp_alt_once 1
 #endif
 
-static int debug_alternative;
+static int __initdata_or_module debug_alternative;
 
 static int __init debug_alt(char *str)
 {
@@ -51,7 +52,7 @@ static int __init setup_noreplace_smp(char *str)
 __setup("noreplace-smp", setup_noreplace_smp);
 
 #ifdef CONFIG_PARAVIRT
-static int noreplace_paravirt = 0;
+static int __initdata_or_module noreplace_paravirt = 0;
 
 static int __init setup_noreplace_paravirt(char *str)
 {
@@ -64,16 +65,17 @@ __setup("noreplace-paravirt", setup_noreplace_paravirt);
 #define DPRINTK(fmt, args...) if (debug_alternative) \
         printk(KERN_DEBUG fmt, args)
 
-#ifdef GENERIC_NOP1
+#if defined(GENERIC_NOP1) && !defined(CONFIG_X86_64)
 /* Use inline assembly to define this because the nops are defined
    as inline assembly strings in the include files and we cannot
    get them easily into strings. */
-asm("\t.section .rodata, \"a\"\nintelnops: "
+asm("\t" __stringify(__INITRODATA_OR_MODULE) "\nintelnops: "
         GENERIC_NOP1 GENERIC_NOP2 GENERIC_NOP3 GENERIC_NOP4 GENERIC_NOP5 GENERIC_NOP6
         GENERIC_NOP7 GENERIC_NOP8
     "\t.previous");
 extern const unsigned char intelnops[];
-static const unsigned char *const intel_nops[ASM_NOP_MAX+1] = {
+static const unsigned char *const __initconst_or_module
+intel_nops[ASM_NOP_MAX+1] = {
         NULL,
         intelnops,
         intelnops + 1,
@@ -87,12 +89,13 @@ static const unsigned char *const intel_nops[ASM_NOP_MAX+1] = {
 #endif
 
 #ifdef K8_NOP1
-asm("\t.section .rodata, \"a\"\nk8nops: "
+asm("\t" __stringify(__INITRODATA_OR_MODULE) "\nk8nops: "
         K8_NOP1 K8_NOP2 K8_NOP3 K8_NOP4 K8_NOP5 K8_NOP6
         K8_NOP7 K8_NOP8
     "\t.previous");
 extern const unsigned char k8nops[];
-static const unsigned char *const k8_nops[ASM_NOP_MAX+1] = {
+static const unsigned char *const __initconst_or_module
+k8_nops[ASM_NOP_MAX+1] = {
         NULL,
         k8nops,
         k8nops + 1,
@@ -105,13 +108,14 @@ static const unsigned char *const k8_nops[ASM_NOP_MAX+1] = {
 };
 #endif
 
-#ifdef K7_NOP1
-asm("\t.section .rodata, \"a\"\nk7nops: "
+#if defined(K7_NOP1) && !defined(CONFIG_X86_64)
+asm("\t" __stringify(__INITRODATA_OR_MODULE) "\nk7nops: "
         K7_NOP1 K7_NOP2 K7_NOP3 K7_NOP4 K7_NOP5 K7_NOP6
         K7_NOP7 K7_NOP8
     "\t.previous");
 extern const unsigned char k7nops[];
-static const unsigned char *const k7_nops[ASM_NOP_MAX+1] = {
+static const unsigned char *const __initconst_or_module
+k7_nops[ASM_NOP_MAX+1] = {
         NULL,
         k7nops,
         k7nops + 1,
@@ -125,12 +129,13 @@ static const unsigned char *const k7_nops[ASM_NOP_MAX+1] = {
 #endif
 
 #ifdef P6_NOP1
-asm("\t.section .rodata, \"a\"\np6nops: "
+asm("\t" __stringify(__INITRODATA_OR_MODULE) "\np6nops: "
         P6_NOP1 P6_NOP2 P6_NOP3 P6_NOP4 P6_NOP5 P6_NOP6
         P6_NOP7 P6_NOP8
     "\t.previous");
 extern const unsigned char p6nops[];
-static const unsigned char *const p6_nops[ASM_NOP_MAX+1] = {
+static const unsigned char *const __initconst_or_module
+p6_nops[ASM_NOP_MAX+1] = {
         NULL,
         p6nops,
         p6nops + 1,
@@ -146,7 +151,7 @@ static const unsigned char *const p6_nops[ASM_NOP_MAX+1] = {
 #ifdef CONFIG_X86_64
 
 extern char __vsyscall_0;
-const unsigned char *const *find_nop_table(void)
+static const unsigned char *const *__init_or_module find_nop_table(void)
 {
         if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL &&
             boot_cpu_has(X86_FEATURE_NOPL))
@@ -157,7 +162,7 @@ const unsigned char *const *find_nop_table(void)
 
 #else /* CONFIG_X86_64 */
 
-const unsigned char *const *find_nop_table(void)
+static const unsigned char *const *__init_or_module find_nop_table(void)
 {
         if (boot_cpu_has(X86_FEATURE_K8))
                 return k8_nops;
@@ -172,7 +177,7 @@ const unsigned char *const *find_nop_table(void)
 #endif /* CONFIG_X86_64 */
 
 /* Use this to add nops to a buffer, then text_poke the whole buffer. */
-void add_nops(void *insns, unsigned int len)
+static void __init_or_module add_nops(void *insns, unsigned int len)
 {
         const unsigned char *const *noptable = find_nop_table();
 
@@ -185,10 +190,10 @@ void add_nops(void *insns, unsigned int len)
                 len -= noplen;
         }
 }
-EXPORT_SYMBOL_GPL(add_nops);
 
 extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
 extern u8 *__smp_locks[], *__smp_locks_end[];
+static void *text_poke_early(void *addr, const void *opcode, size_t len);
 
 /* Replace instructions with better alternatives for this CPU type.
    This runs before SMP is initialized to avoid SMP problems with
@@ -196,7 +201,8 @@ extern u8 *__smp_locks[], *__smp_locks_end[];
    APs have less capabilities than the boot processor are not handled.
    Tough. Make sure you disable such features by hand. */
 
-void apply_alternatives(struct alt_instr *start, struct alt_instr *end)
+void __init_or_module apply_alternatives(struct alt_instr *start,
+                                         struct alt_instr *end)
 {
         struct alt_instr *a;
         char insnbuf[MAX_PATCH_LEN];
@@ -279,9 +285,10 @@ static LIST_HEAD(smp_alt_modules);
 static DEFINE_MUTEX(smp_alt);
 static int smp_mode = 1;        /* protected by smp_alt */
 
-void alternatives_smp_module_add(struct module *mod, char *name,
-                                 void *locks, void *locks_end,
-                                 void *text,  void *text_end)
+void __init_or_module alternatives_smp_module_add(struct module *mod,
+                                                  char *name,
+                                                  void *locks, void *locks_end,
+                                                  void *text,  void *text_end)
 {
         struct smp_alt_module *smp;
 
@@ -317,7 +324,7 @@ void alternatives_smp_module_add(struct module *mod, char *name,
         mutex_unlock(&smp_alt);
 }
 
-void alternatives_smp_module_del(struct module *mod)
+void __init_or_module alternatives_smp_module_del(struct module *mod)
 {
         struct smp_alt_module *item;
 
@@ -386,8 +393,8 @@ void alternatives_smp_switch(int smp)
 #endif
 
 #ifdef CONFIG_PARAVIRT
-void apply_paravirt(struct paravirt_patch_site *start,
-                    struct paravirt_patch_site *end)
+void __init_or_module apply_paravirt(struct paravirt_patch_site *start,
+                                     struct paravirt_patch_site *end)
 {
         struct paravirt_patch_site *p;
         char insnbuf[MAX_PATCH_LEN];
@@ -485,13 +492,14 @@ void __init alternative_instructions(void)
  * instructions. And on the local CPU you need to be protected again NMI or MCE
  * handlers seeing an inconsistent instruction while you patch.
  */
-void *text_poke_early(void *addr, const void *opcode, size_t len)
+static void *__init_or_module text_poke_early(void *addr, const void *opcode,
+                                              size_t len)
 {
         unsigned long flags;
         local_irq_save(flags);
         memcpy(addr, opcode, len);
-        local_irq_restore(flags);
         sync_core();
+        local_irq_restore(flags);
         /* Could also do a CLFLUSH here to speed up CPU recovery; but
            that causes hangs on some VIA CPUs. */
         return addr;
diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c
index 6c99f5037801..98f230f6a28d 100644
--- a/arch/x86/kernel/amd_iommu.c
+++ b/arch/x86/kernel/amd_iommu.c
@@ -41,9 +41,13 @@ static DEFINE_RWLOCK(amd_iommu_devtable_lock);
 static LIST_HEAD(iommu_pd_list);
 static DEFINE_SPINLOCK(iommu_pd_list_lock);
 
-#ifdef CONFIG_IOMMU_API
+/*
+ * Domain for untranslated devices - only allocated
+ * if iommu=pt passed on kernel cmd line.
+ */
+static struct protection_domain *pt_domain;
+
 static struct iommu_ops amd_iommu_ops;
-#endif
 
 /*
  * general struct to manage commands send to an IOMMU
@@ -55,16 +59,16 @@ struct iommu_cmd {
 static int dma_ops_unity_map(struct dma_ops_domain *dma_dom,
                              struct unity_map_entry *e);
 static struct dma_ops_domain *find_protection_domain(u16 devid);
-static u64* alloc_pte(struct protection_domain *dom,
-                      unsigned long address, u64
-                      **pte_page, gfp_t gfp);
+static u64 *alloc_pte(struct protection_domain *domain,
+                      unsigned long address, int end_lvl,
+                      u64 **pte_page, gfp_t gfp);
 static void dma_ops_reserve_addresses(struct dma_ops_domain *dom,
                                       unsigned long start_page,
                                       unsigned int pages);
-
-#ifndef BUS_NOTIFY_UNBOUND_DRIVER
-#define BUS_NOTIFY_UNBOUND_DRIVER 0x0005
-#endif
+static void reset_iommu_command_buffer(struct amd_iommu *iommu);
+static u64 *fetch_pte(struct protection_domain *domain,
+                      unsigned long address, int map_size);
+static void update_domain(struct protection_domain *domain);
 
 #ifdef CONFIG_AMD_IOMMU_STATS
 
@@ -138,7 +142,25 @@ static int iommu_has_npcache(struct amd_iommu *iommu)
  *
  ****************************************************************************/
 
-static void iommu_print_event(void *__evt)
+static void dump_dte_entry(u16 devid)
+{
+        int i;
+
+        for (i = 0; i < 8; ++i)
+                pr_err("AMD-Vi: DTE[%d]: %08x\n", i,
+                        amd_iommu_dev_table[devid].data[i]);
+}
+
+static void dump_command(unsigned long phys_addr)
+{
+        struct iommu_cmd *cmd = phys_to_virt(phys_addr);
+        int i;
+
+        for (i = 0; i < 4; ++i)
+                pr_err("AMD-Vi: CMD[%d]: %08x\n", i, cmd->data[i]);
+}
+
+static void iommu_print_event(struct amd_iommu *iommu, void *__evt)
 {
         u32 *event = __evt;
         int type  = (event[1] >> EVENT_TYPE_SHIFT)  & EVENT_TYPE_MASK;
@@ -147,7 +169,7 @@ static void iommu_print_event(void *__evt)
         int flags = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK;
         u64 address = (u64)(((u64)event[3]) << 32) | event[2];
 
-        printk(KERN_ERR "AMD IOMMU: Event logged [");
+        printk(KERN_ERR "AMD-Vi: Event logged [");
 
         switch (type) {
         case EVENT_TYPE_ILL_DEV:
@@ -155,6 +177,7 @@ static void iommu_print_event(void *__evt)
                        "address=0x%016llx flags=0x%04x]\n",
                        PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
                        address, flags);
+                dump_dte_entry(devid);
                 break;
         case EVENT_TYPE_IO_FAULT:
                 printk("IO_PAGE_FAULT device=%02x:%02x.%x "
@@ -176,6 +199,8 @@ static void iommu_print_event(void *__evt)
                 break;
         case EVENT_TYPE_ILL_CMD:
                 printk("ILLEGAL_COMMAND_ERROR address=0x%016llx]\n", address);
+                reset_iommu_command_buffer(iommu);
+                dump_command(address);
                 break;
         case EVENT_TYPE_CMD_HARD_ERR:
                 printk("COMMAND_HARDWARE_ERROR address=0x%016llx "
@@ -209,7 +234,7 @@ static void iommu_poll_events(struct amd_iommu *iommu)
         tail = readl(iommu->mmio_base + MMIO_EVT_TAIL_OFFSET);
 
         while (head != tail) {
-                iommu_print_event(iommu->evt_buf + head);
+                iommu_print_event(iommu, iommu->evt_buf + head);
                 head = (head + EVENT_ENTRY_SIZE) % iommu->evt_buf_size;
         }
 
@@ -296,8 +321,11 @@ static void __iommu_wait_for_completion(struct amd_iommu *iommu)
         status &= ~MMIO_STATUS_COM_WAIT_INT_MASK;
         writel(status, iommu->mmio_base + MMIO_STATUS_OFFSET);
 
-        if (unlikely(i == EXIT_LOOP_COUNT))
-                panic("AMD IOMMU: Completion wait loop failed\n");
+        if (unlikely(i == EXIT_LOOP_COUNT)) {
+                spin_unlock(&iommu->lock);
+                reset_iommu_command_buffer(iommu);
+                spin_lock(&iommu->lock);
+        }
 }
 
 /*
@@ -445,47 +473,78 @@ static void iommu_flush_tlb_pde(struct amd_iommu *iommu, u16 domid)
 }
 
 /*
+ * This function flushes one domain on one IOMMU
+ */
+static void flush_domain_on_iommu(struct amd_iommu *iommu, u16 domid)
+{
+        struct iommu_cmd cmd;
+        unsigned long flags;
+
+        __iommu_build_inv_iommu_pages(&cmd, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,
+                                      domid, 1, 1);
+
+        spin_lock_irqsave(&iommu->lock, flags);
+        __iommu_queue_command(iommu, &cmd);
+        __iommu_completion_wait(iommu);
+        __iommu_wait_for_completion(iommu);
+        spin_unlock_irqrestore(&iommu->lock, flags);
+}
+
+static void flush_all_domains_on_iommu(struct amd_iommu *iommu)
+{
+        int i;
+
+        for (i = 1; i < MAX_DOMAIN_ID; ++i) {
+                if (!test_bit(i, amd_iommu_pd_alloc_bitmap))
+                        continue;
+                flush_domain_on_iommu(iommu, i);
+        }
+
+}
+
+/*
  * This function is used to flush the IO/TLB for a given protection domain
  * on every IOMMU in the system
  */
 static void iommu_flush_domain(u16 domid)
 {
-        unsigned long flags;
         struct amd_iommu *iommu;
-        struct iommu_cmd cmd;
 
         INC_STATS_COUNTER(domain_flush_all);
 
-        __iommu_build_inv_iommu_pages(&cmd, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,
-                                      domid, 1, 1);
-
-        for_each_iommu(iommu) {
-                spin_lock_irqsave(&iommu->lock, flags);
-                __iommu_queue_command(iommu, &cmd);
-                __iommu_completion_wait(iommu);
-                __iommu_wait_for_completion(iommu);
-                spin_unlock_irqrestore(&iommu->lock, flags);
-        }
+        for_each_iommu(iommu)
+                flush_domain_on_iommu(iommu, domid);
 }
 
 void amd_iommu_flush_all_domains(void)
 {
+        struct amd_iommu *iommu;
+
+        for_each_iommu(iommu)
+                flush_all_domains_on_iommu(iommu);
+}
+
+static void flush_all_devices_for_iommu(struct amd_iommu *iommu)
+{
         int i;
 
-        for (i = 1; i < MAX_DOMAIN_ID; ++i) {
-                if (!test_bit(i, amd_iommu_pd_alloc_bitmap))
+        for (i = 0; i <= amd_iommu_last_bdf; ++i) {
+                if (iommu != amd_iommu_rlookup_table[i])
                         continue;
-                iommu_flush_domain(i);
+
+                iommu_queue_inv_dev_entry(iommu, i);
+                iommu_completion_wait(iommu);
         }
 }
 
-void amd_iommu_flush_all_devices(void)
+static void flush_devices_by_domain(struct protection_domain *domain)
 {
         struct amd_iommu *iommu;
         int i;
 
         for (i = 0; i <= amd_iommu_last_bdf; ++i) {
-                if (amd_iommu_pd_table[i] == NULL)
+                if ((domain == NULL && amd_iommu_pd_table[i] == NULL) ||
+                    (amd_iommu_pd_table[i] != domain))
                         continue;
 
                 iommu = amd_iommu_rlookup_table[i];
@@ -497,6 +556,27 @@ void amd_iommu_flush_all_devices(void)
         }
 }
 
+static void reset_iommu_command_buffer(struct amd_iommu *iommu)
+{
+        pr_err("AMD-Vi: Resetting IOMMU command buffer\n");
+
+        if (iommu->reset_in_progress)
+                panic("AMD-Vi: ILLEGAL_COMMAND_ERROR while resetting command buffer\n");
+
+        iommu->reset_in_progress = true;
+
+        amd_iommu_reset_cmd_buffer(iommu);
+        flush_all_devices_for_iommu(iommu);
+        flush_all_domains_on_iommu(iommu);
+
+        iommu->reset_in_progress = false;
+}
+
+void amd_iommu_flush_all_devices(void)
+{
+        flush_devices_by_domain(NULL);
+}
+
 /****************************************************************************
  *
  * The functions below are used the create the page table mappings for
@@ -514,18 +594,21 @@ void amd_iommu_flush_all_devices(void)
 static int iommu_map_page(struct protection_domain *dom,
                           unsigned long bus_addr,
                           unsigned long phys_addr,
-                          int prot)
+                          int prot,
+                          int map_size)
 {
         u64 __pte, *pte;
 
         bus_addr  = PAGE_ALIGN(bus_addr);
         phys_addr = PAGE_ALIGN(phys_addr);
 
-        /* only support 512GB address spaces for now */
-        if (bus_addr > IOMMU_MAP_SIZE_L3 || !(prot & IOMMU_PROT_MASK))
+        BUG_ON(!PM_ALIGNED(map_size, bus_addr));
+        BUG_ON(!PM_ALIGNED(map_size, phys_addr));
+
+        if (!(prot & IOMMU_PROT_MASK))
                 return -EINVAL;
 
-        pte = alloc_pte(dom, bus_addr, NULL, GFP_KERNEL);
+        pte = alloc_pte(dom, bus_addr, map_size, NULL, GFP_KERNEL);
 
         if (IOMMU_PTE_PRESENT(*pte))
                 return -EBUSY;
@@ -538,29 +621,18 @@ static int iommu_map_page(struct protection_domain *dom,
 
         *pte = __pte;
 
+        update_domain(dom);
+
         return 0;
 }
 
 static void iommu_unmap_page(struct protection_domain *dom,
-                             unsigned long bus_addr)
+                             unsigned long bus_addr, int map_size)
 {
-        u64 *pte;
-
-        pte = &dom->pt_root[IOMMU_PTE_L2_INDEX(bus_addr)];
-
-        if (!IOMMU_PTE_PRESENT(*pte))
-                return;
-
-        pte = IOMMU_PTE_PAGE(*pte);
-        pte = &pte[IOMMU_PTE_L1_INDEX(bus_addr)];
+        u64 *pte = fetch_pte(dom, bus_addr, map_size);
 
-        if (!IOMMU_PTE_PRESENT(*pte))
-                return;
-
-        pte = IOMMU_PTE_PAGE(*pte);
-        pte = &pte[IOMMU_PTE_L1_INDEX(bus_addr)];
-
-        *pte = 0;
+        if (pte)
+                *pte = 0;
 }
 
 /*
@@ -615,7 +687,8 @@ static int dma_ops_unity_map(struct dma_ops_domain *dma_dom,
 
         for (addr = e->address_start; addr < e->address_end;
              addr += PAGE_SIZE) {
-                ret = iommu_map_page(&dma_dom->domain, addr, addr, e->prot);
+                ret = iommu_map_page(&dma_dom->domain, addr, addr, e->prot,
+                                     PM_MAP_4k);
                 if (ret)
                         return ret;
                 /*
@@ -670,24 +743,29 @@ static int init_unity_mappings_for_device(struct dma_ops_domain *dma_dom,
  * This function checks if there is a PTE for a given dma address. If
  * there is one, it returns the pointer to it.
  */
-static u64* fetch_pte(struct protection_domain *domain,
-                      unsigned long address)
+static u64 *fetch_pte(struct protection_domain *domain,
+                      unsigned long address, int map_size)
 {
+        int level;
         u64 *pte;
 
-        pte = &domain->pt_root[IOMMU_PTE_L2_INDEX(address)];
+        level =  domain->mode - 1;
+        pte   = &domain->pt_root[PM_LEVEL_INDEX(level, address)];
 
-        if (!IOMMU_PTE_PRESENT(*pte))
-                return NULL;
+        while (level > map_size) {
+                if (!IOMMU_PTE_PRESENT(*pte))
+                        return NULL;
 
-        pte = IOMMU_PTE_PAGE(*pte);
-        pte = &pte[IOMMU_PTE_L1_INDEX(address)];
+                level -= 1;
 
-        if (!IOMMU_PTE_PRESENT(*pte))
-                return NULL;
+                pte = IOMMU_PTE_PAGE(*pte);
+                pte = &pte[PM_LEVEL_INDEX(level, address)];
 
-        pte = IOMMU_PTE_PAGE(*pte);
-        pte = &pte[IOMMU_PTE_L0_INDEX(address)];
+                if ((PM_PTE_LEVEL(*pte) == 0) && level != map_size) {
+                        pte = NULL;
+                        break;
+                }
+        }
 
         return pte;
 }
@@ -727,7 +805,7 @@ static int alloc_new_range(struct amd_iommu *iommu,
                 u64 *pte, *pte_page;
 
                 for (i = 0; i < num_ptes; ++i) {
-                        pte = alloc_pte(&dma_dom->domain, address,
+                        pte = alloc_pte(&dma_dom->domain, address, PM_MAP_4k,
                                         &pte_page, gfp);
                         if (!pte)
                                 goto out_free;
@@ -760,16 +838,20 @@ static int alloc_new_range(struct amd_iommu *iommu,
         for (i = dma_dom->aperture[index]->offset;
              i < dma_dom->aperture_size;
              i += PAGE_SIZE) {
-                u64 *pte = fetch_pte(&dma_dom->domain, i);
+                u64 *pte = fetch_pte(&dma_dom->domain, i, PM_MAP_4k);
                 if (!pte || !IOMMU_PTE_PRESENT(*pte))
                         continue;
 
                 dma_ops_reserve_addresses(dma_dom, i << PAGE_SHIFT, 1);
         }
 
+        update_domain(&dma_dom->domain);
+
         return 0;
 
 out_free:
+        update_domain(&dma_dom->domain);
+
         free_page((unsigned long)dma_dom->aperture[index]->bitmap);
 
         kfree(dma_dom->aperture[index]);
@@ -1009,7 +1091,7 @@ static struct dma_ops_domain *dma_ops_domain_alloc(struct amd_iommu *iommu)
         dma_dom->domain.id = domain_id_alloc();
         if (dma_dom->domain.id == 0)
                 goto free_dma_dom;
-        dma_dom->domain.mode = PAGE_MODE_3_LEVEL;
+        dma_dom->domain.mode = PAGE_MODE_2_LEVEL;
         dma_dom->domain.pt_root = (void *)get_zeroed_page(GFP_KERNEL);
         dma_dom->domain.flags = PD_DMA_OPS_MASK;
         dma_dom->domain.priv = dma_dom;
@@ -1063,6 +1145,41 @@ static struct protection_domain *domain_for_device(u16 devid)
         return dom;
 }
 
+static void set_dte_entry(u16 devid, struct protection_domain *domain)
+{
+        u64 pte_root = virt_to_phys(domain->pt_root);
+
+        pte_root |= (domain->mode & DEV_ENTRY_MODE_MASK)
+                    << DEV_ENTRY_MODE_SHIFT;
+        pte_root |= IOMMU_PTE_IR | IOMMU_PTE_IW | IOMMU_PTE_P | IOMMU_PTE_TV;
+
+        amd_iommu_dev_table[devid].data[2] = domain->id;
+        amd_iommu_dev_table[devid].data[1] = upper_32_bits(pte_root);
+        amd_iommu_dev_table[devid].data[0] = lower_32_bits(pte_root);
+
+        amd_iommu_pd_table[devid] = domain;
+}
+
+/*
+ * If a device is not yet associated with a domain, this function does
+ * assigns it visible for the hardware
+ */
+static void __attach_device(struct amd_iommu *iommu,
+                            struct protection_domain *domain,
+                            u16 devid)
+{
+        /* lock domain */
+        spin_lock(&domain->lock);
+
+        /* update DTE entry */
+        set_dte_entry(devid, domain);
+
+        domain->dev_cnt += 1;
+
+        /* ready */
+        spin_unlock(&domain->lock);
+}
+
 /*
  * If a device is not yet associated with a domain, this function does
  * assigns it visible for the hardware
@@ -1072,27 +1189,16 @@ static void attach_device(struct amd_iommu *iommu,
                           u16 devid)
 {
         unsigned long flags;
-        u64 pte_root = virt_to_phys(domain->pt_root);
-
-        domain->dev_cnt += 1;
-
-        pte_root |= (domain->mode & DEV_ENTRY_MODE_MASK)
-                    << DEV_ENTRY_MODE_SHIFT;
-        pte_root |= IOMMU_PTE_IR | IOMMU_PTE_IW | IOMMU_PTE_P | IOMMU_PTE_TV;
 
         write_lock_irqsave(&amd_iommu_devtable_lock, flags);
-        amd_iommu_dev_table[devid].data[0] = lower_32_bits(pte_root);
-        amd_iommu_dev_table[devid].data[1] = upper_32_bits(pte_root);
-        amd_iommu_dev_table[devid].data[2] = domain->id;
-
-        amd_iommu_pd_table[devid] = domain;
+        __attach_device(iommu, domain, devid);
         write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
 
-       /*
-        * We might boot into a crash-kernel here. The crashed kernel
-        * left the caches in the IOMMU dirty. So we have to flush
-        * here to evict all dirty stuff.
-        */
+        /*
+         * We might boot into a crash-kernel here. The crashed kernel
+         * left the caches in the IOMMU dirty. So we have to flush
+         * here to evict all dirty stuff.
+         */
         iommu_queue_inv_dev_entry(iommu, devid);
         iommu_flush_tlb_pde(iommu, domain->id);
 }
@@ -1119,6 +1225,15 @@ static void __detach_device(struct protection_domain *domain, u16 devid)
 
         /* ready */
         spin_unlock(&domain->lock);
+
+        /*
+         * If we run in passthrough mode the device must be assigned to the
+         * passthrough domain if it is detached from any other domain
+         */
+        if (iommu_pass_through) {
+                struct amd_iommu *iommu = amd_iommu_rlookup_table[devid];
+                __attach_device(iommu, pt_domain, devid);
+        }
 }
 
 /*
@@ -1164,6 +1279,8 @@ static int device_change_notifier(struct notifier_block *nb,
         case BUS_NOTIFY_UNBOUND_DRIVER:
                 if (!domain)
                         goto out;
+                if (iommu_pass_through)
+                        break;
                 detach_device(domain, devid);
                 break;
         case BUS_NOTIFY_ADD_DEVICE:
@@ -1292,39 +1409,91 @@ static int get_device_resources(struct device *dev,
         return 1;
 }
 
+static void update_device_table(struct protection_domain *domain)
+{
+        unsigned long flags;
+        int i;
+
+        for (i = 0; i <= amd_iommu_last_bdf; ++i) {
+                if (amd_iommu_pd_table[i] != domain)
+                        continue;
+                write_lock_irqsave(&amd_iommu_devtable_lock, flags);
+                set_dte_entry(i, domain);
+                write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
+        }
+}
+
+static void update_domain(struct protection_domain *domain)
+{
+        if (!domain->updated)
+                return;
+
+        update_device_table(domain);
+        flush_devices_by_domain(domain);
+        iommu_flush_domain(domain->id);
+
+        domain->updated = false;
+}
+
 /*
- * If the pte_page is not yet allocated this function is called
+ * This function is used to add another level to an IO page table. Adding
+ * another level increases the size of the address space by 9 bits to a size up
+ * to 64 bits.
  */
-static u64* alloc_pte(struct protection_domain *dom,
-                      unsigned long address, u64 **pte_page, gfp_t gfp)
+static bool increase_address_space(struct protection_domain *domain,
+                                   gfp_t gfp)
+{
+        u64 *pte;
+
+        if (domain->mode == PAGE_MODE_6_LEVEL)
+                /* address space already 64 bit large */
+                return false;
+
+        pte = (void *)get_zeroed_page(gfp);
+        if (!pte)
+                return false;
+
+        *pte             = PM_LEVEL_PDE(domain->mode,
+                                        virt_to_phys(domain->pt_root));
+        domain->pt_root  = pte;
+        domain->mode    += 1;
+        domain->updated  = true;
+
+        return true;
+}
+
+static u64 *alloc_pte(struct protection_domain *domain,
+                      unsigned long address,
+                      int end_lvl,
+                      u64 **pte_page,
+                      gfp_t gfp)
 {
         u64 *pte, *page;
+        int level;
 
-        pte = &dom->pt_root[IOMMU_PTE_L2_INDEX(address)];
+        while (address > PM_LEVEL_SIZE(domain->mode))
+                increase_address_space(domain, gfp);
 
-        if (!IOMMU_PTE_PRESENT(*pte)) {
-                page = (u64 *)get_zeroed_page(gfp);
-                if (!page)
-                        return NULL;
-                *pte = IOMMU_L2_PDE(virt_to_phys(page));
-        }
+        level =  domain->mode - 1;
+        pte   = &domain->pt_root[PM_LEVEL_INDEX(level, address)];
 
-        pte = IOMMU_PTE_PAGE(*pte);
-        pte = &pte[IOMMU_PTE_L1_INDEX(address)];
+        while (level > end_lvl) {
+                if (!IOMMU_PTE_PRESENT(*pte)) {
+                        page = (u64 *)get_zeroed_page(gfp);
+                        if (!page)
+                                return NULL;
+                        *pte = PM_LEVEL_PDE(level, virt_to_phys(page));
+                }
 
-        if (!IOMMU_PTE_PRESENT(*pte)) {
-                page = (u64 *)get_zeroed_page(gfp);
-                if (!page)
-                        return NULL;
-                *pte = IOMMU_L1_PDE(virt_to_phys(page));
-        }
+                level -= 1;
 
-        pte = IOMMU_PTE_PAGE(*pte);
+                pte = IOMMU_PTE_PAGE(*pte);
 
-        if (pte_page)
-                *pte_page = pte;
+                if (pte_page && level == end_lvl)
+                        *pte_page = pte;
 
-        pte = &pte[IOMMU_PTE_L0_INDEX(address)];
+                pte = &pte[PM_LEVEL_INDEX(level, address)];
+        }
 
         return pte;
 }
@@ -1344,10 +1513,13 @@ static u64* dma_ops_get_pte(struct dma_ops_domain *dom,
 
         pte = aperture->pte_pages[APERTURE_PAGE_INDEX(address)];
         if (!pte) {
-                pte = alloc_pte(&dom->domain, address, &pte_page, GFP_ATOMIC);
+                pte = alloc_pte(&dom->domain, address, PM_MAP_4k, &pte_page,
+                                GFP_ATOMIC);
                 aperture->pte_pages[APERTURE_PAGE_INDEX(address)] = pte_page;
         } else
-                pte += IOMMU_PTE_L0_INDEX(address);
+                pte += PM_LEVEL_INDEX(0, address);
+
+        update_domain(&dom->domain);
 
         return pte;
 }
@@ -1409,7 +1581,7 @@ static void dma_ops_domain_unmap(struct amd_iommu *iommu,
         if (!pte)
                 return;
 
-        pte += IOMMU_PTE_L0_INDEX(address);
+        pte += PM_LEVEL_INDEX(0, address);
 
         WARN_ON(!*pte);
 
@@ -1988,19 +2160,47 @@ static void cleanup_domain(struct protection_domain *domain)
         write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
 }
 
-static int amd_iommu_domain_init(struct iommu_domain *dom)
+static void protection_domain_free(struct protection_domain *domain)
+{
+        if (!domain)
+                return;
+
+        if (domain->id)
+                domain_id_free(domain->id);
+
+        kfree(domain);
+}
+
+static struct protection_domain *protection_domain_alloc(void)
 {
         struct protection_domain *domain;
 
         domain = kzalloc(sizeof(*domain), GFP_KERNEL);
         if (!domain)
-                return -ENOMEM;
+                return NULL;
 
         spin_lock_init(&domain->lock);
-        domain->mode = PAGE_MODE_3_LEVEL;
         domain->id = domain_id_alloc();
         if (!domain->id)
+                goto out_err;
+
+        return domain;
+
+out_err:
+        kfree(domain);
+
+        return NULL;
+}
+
+static int amd_iommu_domain_init(struct iommu_domain *dom)
+{
+        struct protection_domain *domain;
+
+        domain = protection_domain_alloc();
+        if (!domain)
                 goto out_free;
+
+        domain->mode    = PAGE_MODE_3_LEVEL;
         domain->pt_root = (void *)get_zeroed_page(GFP_KERNEL);
         if (!domain->pt_root)
                 goto out_free;
@@ -2010,7 +2210,7 @@ static int amd_iommu_domain_init(struct iommu_domain *dom)
         return 0;
 
 out_free:
-        kfree(domain);
+        protection_domain_free(domain);
 
         return -ENOMEM;
 }
@@ -2115,7 +2315,7 @@ static int amd_iommu_map_range(struct iommu_domain *dom,
         paddr &= PAGE_MASK;
 
         for (i = 0; i < npages; ++i) {
-                ret = iommu_map_page(domain, iova, paddr, prot);
+                ret = iommu_map_page(domain, iova, paddr, prot, PM_MAP_4k);
                 if (ret)
                         return ret;
 
@@ -2136,7 +2336,7 @@ static void amd_iommu_unmap_range(struct iommu_domain *dom,
         iova  &= PAGE_MASK;
 
         for (i = 0; i < npages; ++i) {
-                iommu_unmap_page(domain, iova);
+                iommu_unmap_page(domain, iova, PM_MAP_4k);
                 iova  += PAGE_SIZE;
         }
 
@@ -2151,21 +2351,9 @@ static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom,
         phys_addr_t paddr;
         u64 *pte;
 
-        pte = &domain->pt_root[IOMMU_PTE_L2_INDEX(iova)];
-
-        if (!IOMMU_PTE_PRESENT(*pte))
-                return 0;
-
-        pte = IOMMU_PTE_PAGE(*pte);
-        pte = &pte[IOMMU_PTE_L1_INDEX(iova)];
-
-        if (!IOMMU_PTE_PRESENT(*pte))
-                return 0;
-
-        pte = IOMMU_PTE_PAGE(*pte);
-        pte = &pte[IOMMU_PTE_L0_INDEX(iova)];
+        pte = fetch_pte(domain, iova, PM_MAP_4k);
 
-        if (!IOMMU_PTE_PRESENT(*pte))
+        if (!pte || !IOMMU_PTE_PRESENT(*pte))
                 return 0;
 
         paddr  = *pte & IOMMU_PAGE_MASK;
@@ -2191,3 +2379,46 @@ static struct iommu_ops amd_iommu_ops = {
         .domain_has_cap = amd_iommu_domain_has_cap,
 };
 
+/*****************************************************************************
+ *
+ * The next functions do a basic initialization of IOMMU for pass through
+ * mode
+ *
+ * In passthrough mode the IOMMU is initialized and enabled but not used for
+ * DMA-API translation.
+ *
+ *****************************************************************************/
+
+int __init amd_iommu_init_passthrough(void)
+{
+        struct pci_dev *dev = NULL;
+        u16 devid, devid2;
+
+        /* allocate passthroug domain */
+        pt_domain = protection_domain_alloc();
+        if (!pt_domain)
+                return -ENOMEM;
+
+        pt_domain->mode |= PAGE_MODE_NONE;
+
+        while ((dev = pci_get_device(PCI_ANY_ID, PCI_ANY_ID, dev)) != NULL) {
+                struct amd_iommu *iommu;
+
+                devid = calc_devid(dev->bus->number, dev->devfn);
+                if (devid > amd_iommu_last_bdf)
+                        continue;
+
+                devid2 = amd_iommu_alias_table[devid];
+
+                iommu = amd_iommu_rlookup_table[devid2];
+                if (!iommu)
+                        continue;
+
+                __attach_device(iommu, pt_domain, devid);
+                __attach_device(iommu, pt_domain, devid2);
+        }
+
+        pr_info("AMD-Vi: Initialized for Passthrough Mode\n");
+
+        return 0;
+}
diff --git a/arch/x86/kernel/amd_iommu_init.c b/arch/x86/kernel/amd_iommu_init.c
index c1b17e97252e..b4b61d462dcc 100644
--- a/arch/x86/kernel/amd_iommu_init.c
+++ b/arch/x86/kernel/amd_iommu_init.c
@@ -252,7 +252,7 @@ static void __init iommu_feature_disable(struct amd_iommu *iommu, u8 bit)
 /* Function to enable the hardware */
 static void iommu_enable(struct amd_iommu *iommu)
 {
-        printk(KERN_INFO "AMD IOMMU: Enabling IOMMU at %s cap 0x%hx\n",
+        printk(KERN_INFO "AMD-Vi: Enabling IOMMU at %s cap 0x%hx\n",
                dev_name(&iommu->dev->dev), iommu->cap_ptr);
 
         iommu_feature_enable(iommu, CONTROL_IOMMU_EN);
@@ -435,6 +435,20 @@ static u8 * __init alloc_command_buffer(struct amd_iommu *iommu)
 }
 
 /*
+ * This function resets the command buffer if the IOMMU stopped fetching
+ * commands from it.
+ */
+void amd_iommu_reset_cmd_buffer(struct amd_iommu *iommu)
+{
+        iommu_feature_disable(iommu, CONTROL_CMDBUF_EN);
+
+        writel(0x00, iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);
+        writel(0x00, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
+
+        iommu_feature_enable(iommu, CONTROL_CMDBUF_EN);
+}
+
+/*
  * This function writes the command buffer address to the hardware and
  * enables it.
  */
@@ -450,11 +464,7 @@ static void iommu_enable_command_buffer(struct amd_iommu *iommu)
         memcpy_toio(iommu->mmio_base + MMIO_CMD_BUF_OFFSET,
                     &entry, sizeof(entry));
 
-        /* set head and tail to zero manually */
-        writel(0x00, iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);
-        writel(0x00, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
-
-        iommu_feature_enable(iommu, CONTROL_CMDBUF_EN);
+        amd_iommu_reset_cmd_buffer(iommu);
 }
 
 static void __init free_command_buffer(struct amd_iommu *iommu)
@@ -858,7 +868,7 @@ static int __init init_iommu_all(struct acpi_table_header *table)
                 switch (*p) {
                 case ACPI_IVHD_TYPE:
 
-                        DUMP_printk("IOMMU: device: %02x:%02x.%01x cap: %04x "
+                        DUMP_printk("device: %02x:%02x.%01x cap: %04x "
                                     "seg: %d flags: %01x info %04x\n",
                                     PCI_BUS(h->devid), PCI_SLOT(h->devid),
                                     PCI_FUNC(h->devid), h->cap_ptr,
@@ -902,7 +912,7 @@ static int __init iommu_setup_msi(struct amd_iommu *iommu)
 
         r = request_irq(iommu->dev->irq, amd_iommu_int_handler,
                         IRQF_SAMPLE_RANDOM,
-                        "AMD IOMMU",
+                        "AMD-Vi",
                         NULL);
 
         if (r) {
@@ -1150,7 +1160,7 @@ int __init amd_iommu_init(void)
 
 
         if (no_iommu) {
-                printk(KERN_INFO "AMD IOMMU disabled by kernel command line\n");
+                printk(KERN_INFO "AMD-Vi disabled by kernel command line\n");
                 return 0;
         }
 
@@ -1242,22 +1252,28 @@ int __init amd_iommu_init(void)
         if (ret)
                 goto free;
 
-        ret = amd_iommu_init_dma_ops();
+        if (iommu_pass_through)
+                ret = amd_iommu_init_passthrough();
+        else
+                ret = amd_iommu_init_dma_ops();
         if (ret)
                 goto free;
 
         enable_iommus();
 
-        printk(KERN_INFO "AMD IOMMU: device isolation ");
+        if (iommu_pass_through)
+                goto out;
+
+        printk(KERN_INFO "AMD-Vi: device isolation ");
         if (amd_iommu_isolate)
                 printk("enabled\n");
         else
                 printk("disabled\n");
 
         if (amd_iommu_unmap_flush)
-                printk(KERN_INFO "AMD IOMMU: IO/TLB flush on unmap enabled\n");
+                printk(KERN_INFO "AMD-Vi: IO/TLB flush on unmap enabled\n");
         else
-                printk(KERN_INFO "AMD IOMMU: Lazy IO/TLB flushing enabled\n");
+                printk(KERN_INFO "AMD-Vi: Lazy IO/TLB flushing enabled\n");
 
 out:
         return ret;
diff --git a/arch/x86/kernel/aperture_64.c b/arch/x86/kernel/aperture_64.c
index 676debfc1702..128111d8ffe0 100644
--- a/arch/x86/kernel/aperture_64.c
+++ b/arch/x86/kernel/aperture_64.c
@@ -20,6 +20,7 @@
 #include <linux/bitops.h>
 #include <linux/ioport.h>
 #include <linux/suspend.h>
+#include <linux/kmemleak.h>
 #include <asm/e820.h>
 #include <asm/io.h>
 #include <asm/iommu.h>
@@ -94,6 +95,11 @@ static u32 __init allocate_aperture(void)
          * code for safe
          */
         p = __alloc_bootmem_nopanic(aper_size, aper_size, 512ULL<<20);
+        /*
+         * Kmemleak should not scan this block as it may not be mapped via the
+         * kernel direct mapping.
+         */
+        kmemleak_ignore(p);
         if (!p || __pa(p)+aper_size > 0xffffffff) {
                 printk(KERN_ERR
                         "Cannot allocate aperture memory hole (%p,%uK)\n",
diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c
index 0a1c2830ec66..894aa97f0717 100644
--- a/arch/x86/kernel/apic/apic.c
+++ b/arch/x86/kernel/apic/apic.c
@@ -14,7 +14,7 @@
  *      Mikael Pettersson       :       PM converted to driver model.
  */
 
-#include <linux/perf_counter.h>
+#include <linux/perf_event.h>
 #include <linux/kernel_stat.h>
 #include <linux/mc146818rtc.h>
 #include <linux/acpi_pmtmr.h>
@@ -35,7 +35,8 @@
 #include <linux/smp.h>
 #include <linux/mm.h>
 
-#include <asm/perf_counter.h>
+#include <asm/perf_event.h>
+#include <asm/x86_init.h>
 #include <asm/pgalloc.h>
 #include <asm/atomic.h>
 #include <asm/mpspec.h>
@@ -49,6 +50,7 @@
 #include <asm/mtrr.h>
 #include <asm/smp.h>
 #include <asm/mce.h>
+#include <asm/kvm_para.h>
 
 unsigned int num_processors;
 
@@ -60,7 +62,7 @@ unsigned int boot_cpu_physical_apicid = -1U;
 /*
  * The highest APIC ID seen during enumeration.
  *
- * This determines the messaging protocol we can use: if all APIC IDs
+ * On AMD, this determines the messaging protocol we can use: if all APIC IDs
  * are in the 0 ... 7 range, then we can use logical addressing which
  * has some performance advantages (better broadcasting).
  *
@@ -977,7 +979,7 @@ void lapic_shutdown(void)
 {
         unsigned long flags;
 
-        if (!cpu_has_apic)
+        if (!cpu_has_apic && !apic_from_smp_config())
                 return;
 
         local_irq_save(flags);
@@ -1187,7 +1189,7 @@ void __cpuinit setup_local_APIC(void)
                 apic_write(APIC_ESR, 0);
         }
 #endif
-        perf_counters_lapic_init();
+        perf_events_lapic_init();
 
         preempt_disable();
 
@@ -1195,8 +1197,7 @@ void __cpuinit setup_local_APIC(void)
          * Double-check whether this APIC is really registered.
          * This is meaningless in clustered apic mode, so we skip it.
          */
-        if (!apic->apic_id_registered())
-                BUG();
+        BUG_ON(!apic->apic_id_registered());
 
         /*
          * Intel recommends to set DFR, LDR and TPR before enabling
@@ -1361,52 +1362,80 @@ void enable_x2apic(void)
 }
 #endif /* CONFIG_X86_X2APIC */
 
-void __init enable_IR_x2apic(void)
+int __init enable_IR(void)
 {
 #ifdef CONFIG_INTR_REMAP
-        int ret;
-        unsigned long flags;
-        struct IO_APIC_route_entry **ioapic_entries = NULL;
-
-        ret = dmar_table_init();
-        if (ret) {
-                pr_debug("dmar_table_init() failed with %d:\n", ret);
-                goto ir_failed;
-        }
-
         if (!intr_remapping_supported()) {
                 pr_debug("intr-remapping not supported\n");
-                goto ir_failed;
+                return 0;
         }
 
-
         if (!x2apic_preenabled && skip_ioapic_setup) {
                 pr_info("Skipped enabling intr-remap because of skipping "
                         "io-apic setup\n");
-                return;
+                return 0;
         }
 
+        if (enable_intr_remapping(x2apic_supported()))
+                return 0;
+
+        pr_info("Enabled Interrupt-remapping\n");
+
+        return 1;
+
+#endif
+        return 0;
+}
+
+void __init enable_IR_x2apic(void)
+{
+        unsigned long flags;
+        struct IO_APIC_route_entry **ioapic_entries = NULL;
+        int ret, x2apic_enabled = 0;
+        int dmar_table_init_ret = 0;
+
+#ifdef CONFIG_INTR_REMAP
+        dmar_table_init_ret = dmar_table_init();
+        if (dmar_table_init_ret)
+                pr_debug("dmar_table_init() failed with %d:\n",
+                                dmar_table_init_ret);
+#endif
+
         ioapic_entries = alloc_ioapic_entries();
         if (!ioapic_entries) {
-                pr_info("Allocate ioapic_entries failed: %d\n", ret);
-                goto end;
+                pr_err("Allocate ioapic_entries failed\n");
+                goto out;
         }
 
         ret = save_IO_APIC_setup(ioapic_entries);
         if (ret) {
                 pr_info("Saving IO-APIC state failed: %d\n", ret);
-                goto end;
+                goto out;
         }
 
         local_irq_save(flags);
-        mask_IO_APIC_setup(ioapic_entries);
         mask_8259A();
+        mask_IO_APIC_setup(ioapic_entries);
 
-        ret = enable_intr_remapping(x2apic_supported());
-        if (ret)
-                goto end_restore;
+        if (dmar_table_init_ret)
+                ret = 0;
+        else
+                ret = enable_IR();
 
-        pr_info("Enabled Interrupt-remapping\n");
+        if (!ret) {
+                /* IR is required if there is APIC ID > 255 even when running
+                 * under KVM
+                 */
+                if (max_physical_apicid > 255 || !kvm_para_available())
+                        goto nox2apic;
+                /*
+                 * without IR all CPUs can be addressed by IOAPIC/MSI
+                 * only in physical mode
+                 */
+                x2apic_force_phys();
+        }
+
+        x2apic_enabled = 1;
 
         if (x2apic_supported() && !x2apic_mode) {
                 x2apic_mode = 1;
@@ -1414,41 +1443,25 @@ void __init enable_IR_x2apic(void)
                 pr_info("Enabled x2apic\n");
         }
 
-end_restore:
-        if (ret)
-                /*
-                 * IR enabling failed
-                 */
+nox2apic:
+        if (!ret) /* IR enabling failed */
                 restore_IO_APIC_setup(ioapic_entries);
-
         unmask_8259A();
         local_irq_restore(flags);
 
-end:
+out:
         if (ioapic_entries)
                 free_ioapic_entries(ioapic_entries);
 
-        if (!ret)
+        if (x2apic_enabled)
                 return;
 
-ir_failed:
         if (x2apic_preenabled)
-                panic("x2apic enabled by bios. But IR enabling failed");
+                panic("x2apic: enabled by BIOS but kernel init failed.");
         else if (cpu_has_x2apic)
-                pr_info("Not enabling x2apic,Intr-remapping\n");
-#else
-        if (!cpu_has_x2apic)
-                return;
-
-        if (x2apic_preenabled)
-                panic("x2apic enabled prior OS handover,"
-                      " enable CONFIG_X86_X2APIC, CONFIG_INTR_REMAP");
-#endif
-
-        return;
+                pr_info("Not enabling x2apic, Intr-remapping init failed.\n");
 }
 
-
 #ifdef CONFIG_X86_64
 /*
  * Detect and enable local APICs on non-SMP boards.
@@ -1549,8 +1562,6 @@ no_apic:
 #ifdef CONFIG_X86_64
 void __init early_init_lapic_mapping(void)
 {
-        unsigned long phys_addr;
-
         /*
          * If no local APIC can be found then go out
          * : it means there is no mpatable and MADT
@@ -1558,11 +1569,9 @@ void __init early_init_lapic_mapping(void)
         if (!smp_found_config)
                 return;
 
-        phys_addr = mp_lapic_addr;
-
-        set_fixmap_nocache(FIX_APIC_BASE, phys_addr);
+        set_fixmap_nocache(FIX_APIC_BASE, mp_lapic_addr);
         apic_printk(APIC_VERBOSE, "mapped APIC to %16lx (%16lx)\n",
-                    APIC_BASE, phys_addr);
+                    APIC_BASE, mp_lapic_addr);
 
         /*
          * Fetch the APIC ID of the BSP in case we have a
@@ -1651,7 +1660,6 @@ int __init APIC_init_uniprocessor(void)
             APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid])) {
                 pr_err("BIOS bug, local APIC 0x%x not detected!...\n",
                         boot_cpu_physical_apicid);
-                clear_cpu_cap(&boot_cpu_data, X86_FEATURE_APIC);
                 return -1;
         }
 #endif
@@ -1701,7 +1709,7 @@ int __init APIC_init_uniprocessor(void)
         localise_nmi_watchdog();
 #endif
 
-        setup_boot_clock();
+        x86_init.timers.setup_percpu_clockev();
 #ifdef CONFIG_X86_64
         check_nmi_watchdog();
 #endif
@@ -1908,24 +1916,14 @@ void __cpuinit generic_processor_info(int apicid, int version)
                 max_physical_apicid = apicid;
 
 #ifdef CONFIG_X86_32
-        /*
-         * Would be preferable to switch to bigsmp when CONFIG_HOTPLUG_CPU=y
-         * but we need to work other dependencies like SMP_SUSPEND etc
-         * before this can be done without some confusion.
-         * if (CPU_HOTPLUG_ENABLED || num_processors > 8)
-         *       - Ashok Raj <ashok.raj@intel.com>
-         */
-        if (max_physical_apicid >= 8) {
-                switch (boot_cpu_data.x86_vendor) {
-                case X86_VENDOR_INTEL:
-                        if (!APIC_XAPIC(version)) {
-                                def_to_bigsmp = 0;
-                                break;
-                        }
-                        /* If P4 and above fall through */
-                case X86_VENDOR_AMD:
+        switch (boot_cpu_data.x86_vendor) {
+        case X86_VENDOR_INTEL:
+                if (num_processors > 8)
+                        def_to_bigsmp = 1;
+                break;
+        case X86_VENDOR_AMD:
+                if (max_physical_apicid >= 8)
                         def_to_bigsmp = 1;
-                }
         }
 #endif
 
diff --git a/arch/x86/kernel/apic/bigsmp_32.c b/arch/x86/kernel/apic/bigsmp_32.c
index 676cdac385c0..77a06413b6b2 100644
--- a/arch/x86/kernel/apic/bigsmp_32.c
+++ b/arch/x86/kernel/apic/bigsmp_32.c
@@ -112,7 +112,7 @@ static physid_mask_t bigsmp_ioapic_phys_id_map(physid_mask_t phys_map)
         return physids_promote(0xFFL);
 }
 
-static int bigsmp_check_phys_apicid_present(int boot_cpu_physical_apicid)
+static int bigsmp_check_phys_apicid_present(int phys_apicid)
 {
         return 1;
 }
diff --git a/arch/x86/kernel/apic/es7000_32.c b/arch/x86/kernel/apic/es7000_32.c
index 8952a5890281..89174f847b49 100644
--- a/arch/x86/kernel/apic/es7000_32.c
+++ b/arch/x86/kernel/apic/es7000_32.c
@@ -167,7 +167,7 @@ static int es7000_apic_is_cluster(void)
 {
         /* MPENTIUMIII */
         if (boot_cpu_data.x86 == 6 &&
-            (boot_cpu_data.x86_model >= 7 || boot_cpu_data.x86_model <= 11))
+            (boot_cpu_data.x86_model >= 7 && boot_cpu_data.x86_model <= 11))
                 return 1;
 
         return 0;
diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c
index d2ed6c5ddc80..dc69f28489f5 100644
--- a/arch/x86/kernel/apic/io_apic.c
+++ b/arch/x86/kernel/apic/io_apic.c
@@ -66,6 +66,8 @@
 #include <asm/apic.h>
 
 #define __apicdebuginit(type) static type __init
+#define for_each_irq_pin(entry, head) \
+        for (entry = head; entry; entry = entry->next)
 
 /*
  *      Is the SiS APIC rmw bug present ?
@@ -85,12 +87,20 @@ int nr_ioapic_registers[MAX_IO_APICS];
 struct mpc_ioapic mp_ioapics[MAX_IO_APICS];
 int nr_ioapics;
 
+/* IO APIC gsi routing info */
+struct mp_ioapic_gsi  mp_gsi_routing[MAX_IO_APICS];
+
 /* MP IRQ source entries */
 struct mpc_intsrc mp_irqs[MAX_IRQ_SOURCES];
 
 /* # of MP IRQ source entries */
 int mp_irq_entries;
 
+/* Number of legacy interrupts */
+static int nr_legacy_irqs __read_mostly = NR_IRQS_LEGACY;
+/* GSI interrupts */
+static int nr_irqs_gsi = NR_IRQS_LEGACY;
+
 #if defined (CONFIG_MCA) || defined (CONFIG_EISA)
 int mp_bus_id_to_type[MAX_MP_BUSSES];
 #endif
@@ -116,15 +126,6 @@ static int __init parse_noapic(char *str)
 }
 early_param("noapic", parse_noapic);
 
-struct irq_pin_list;
-
-/*
- * This is performance-critical, we want to do it O(1)
- *
- * the indexing order of this array favors 1:1 mappings
- * between pins and IRQs.
- */
-
 struct irq_pin_list {
         int apic, pin;
         struct irq_pin_list *next;
@@ -139,6 +140,11 @@ static struct irq_pin_list *get_one_free_irq_2_pin(int node)
         return pin;
 }
 
+/*
+ * This is performance-critical, we want to do it O(1)
+ *
+ * Most irqs are mapped 1:1 with pins.
+ */
 struct irq_cfg {
         struct irq_pin_list *irq_2_pin;
         cpumask_var_t domain;
@@ -172,6 +178,12 @@ static struct irq_cfg irq_cfgx[NR_IRQS] = {
         [15] = { .vector = IRQ15_VECTOR, },
 };
 
+void __init io_apic_disable_legacy(void)
+{
+        nr_legacy_irqs = 0;
+        nr_irqs_gsi = 0;
+}
+
 int __init arch_early_irq_init(void)
 {
         struct irq_cfg *cfg;
@@ -189,7 +201,7 @@ int __init arch_early_irq_init(void)
                 desc->chip_data = &cfg[i];
                 zalloc_cpumask_var_node(&cfg[i].domain, GFP_NOWAIT, node);
                 zalloc_cpumask_var_node(&cfg[i].old_domain, GFP_NOWAIT, node);
-                if (i < NR_IRQS_LEGACY)
+                if (i < nr_legacy_irqs)
                         cpumask_setall(cfg[i].domain);
         }
 
@@ -215,17 +227,14 @@ static struct irq_cfg *get_one_free_irq_cfg(int node)
 
         cfg = kzalloc_node(sizeof(*cfg), GFP_ATOMIC, node);
         if (cfg) {
-                if (!alloc_cpumask_var_node(&cfg->domain, GFP_ATOMIC, node)) {
+                if (!zalloc_cpumask_var_node(&cfg->domain, GFP_ATOMIC, node)) {
                         kfree(cfg);
                         cfg = NULL;
-                } else if (!alloc_cpumask_var_node(&cfg->old_domain,
+                } else if (!zalloc_cpumask_var_node(&cfg->old_domain,
                                                           GFP_ATOMIC, node)) {
                         free_cpumask_var(cfg->domain);
                         kfree(cfg);
                         cfg = NULL;
-                } else {
-                        cpumask_clear(cfg->domain);
-                        cpumask_clear(cfg->old_domain);
                 }
         }
 
@@ -414,13 +423,10 @@ static bool io_apic_level_ack_pending(struct irq_cfg *cfg)
         unsigned long flags;
 
         spin_lock_irqsave(&ioapic_lock, flags);
-        entry = cfg->irq_2_pin;
-        for (;;) {
+        for_each_irq_pin(entry, cfg->irq_2_pin) {
                 unsigned int reg;
                 int pin;
 
-                if (!entry)
-                        break;
                 pin = entry->pin;
                 reg = io_apic_read(entry->apic, 0x10 + pin*2);
                 /* Is the remote IRR bit set? */
@@ -428,9 +434,6 @@ static bool io_apic_level_ack_pending(struct irq_cfg *cfg)
                         spin_unlock_irqrestore(&ioapic_lock, flags);
                         return true;
                 }
-                if (!entry->next)
-                        break;
-                entry = entry->next;
         }
         spin_unlock_irqrestore(&ioapic_lock, flags);
 
@@ -498,72 +501,68 @@ static void ioapic_mask_entry(int apic, int pin)
  * shared ISA-space IRQs, so we have to support them. We are super
  * fast in the common case, and fast for shared ISA-space IRQs.
  */
-static void add_pin_to_irq_node(struct irq_cfg *cfg, int node, int apic, int pin)
+static int
+add_pin_to_irq_node_nopanic(struct irq_cfg *cfg, int node, int apic, int pin)
 {
-        struct irq_pin_list *entry;
+        struct irq_pin_list **last, *entry;
 
-        entry = cfg->irq_2_pin;
-        if (!entry) {
-                entry = get_one_free_irq_2_pin(node);
-                if (!entry) {
-                        printk(KERN_ERR "can not alloc irq_2_pin to add %d - %d\n",
-                                        apic, pin);
-                        return;
-                }
-                cfg->irq_2_pin = entry;
-                entry->apic = apic;
-                entry->pin = pin;
-                return;
-        }
-
-        while (entry->next) {
-                /* not again, please */
+        /* don't allow duplicates */
+        last = &cfg->irq_2_pin;
+        for_each_irq_pin(entry, cfg->irq_2_pin) {
                 if (entry->apic == apic && entry->pin == pin)
-                        return;
-
-                entry = entry->next;
+                        return 0;
+                last = &entry->next;
         }
 
-        entry->next = get_one_free_irq_2_pin(node);
-        entry = entry->next;
+        entry = get_one_free_irq_2_pin(node);
+        if (!entry) {
+                printk(KERN_ERR "can not alloc irq_pin_list (%d,%d,%d)\n",
+                                node, apic, pin);
+                return -ENOMEM;
+        }
         entry->apic = apic;
         entry->pin = pin;
+
+        *last = entry;
+        return 0;
+}
+
+static void add_pin_to_irq_node(struct irq_cfg *cfg, int node, int apic, int pin)
+{
+        if (add_pin_to_irq_node_nopanic(cfg, node, apic, pin))
+                panic("IO-APIC: failed to add irq-pin. Can not proceed\n");
 }
 
 /*
  * Reroute an IRQ to a different pin.
  */
 static void __init replace_pin_at_irq_node(struct irq_cfg *cfg, int node,
-                                      int oldapic, int oldpin,
-                                      int newapic, int newpin)
+                                           int oldapic, int oldpin,
+                                           int newapic, int newpin)
 {
-        struct irq_pin_list *entry = cfg->irq_2_pin;
-        int replaced = 0;
+        struct irq_pin_list *entry;
 
-        while (entry) {
+        for_each_irq_pin(entry, cfg->irq_2_pin) {
                 if (entry->apic == oldapic && entry->pin == oldpin) {
                         entry->apic = newapic;
                         entry->pin = newpin;
-                        replaced = 1;
                         /* every one is different, right? */
-                        break;
+                        return;
                 }
-                entry = entry->next;
         }
 
-        /* why? call replace before add? */
-        if (!replaced)
-                add_pin_to_irq_node(cfg, node, newapic, newpin);
+        /* old apic/pin didn't exist, so just add new ones */
+        add_pin_to_irq_node(cfg, node, newapic, newpin);
 }
 
-static inline void io_apic_modify_irq(struct irq_cfg *cfg,
-                                int mask_and, int mask_or,
-                                void (*final)(struct irq_pin_list *entry))
+static void io_apic_modify_irq(struct irq_cfg *cfg,
+                               int mask_and, int mask_or,
+                               void (*final)(struct irq_pin_list *entry))
 {
         int pin;
         struct irq_pin_list *entry;
 
-        for (entry = cfg->irq_2_pin; entry != NULL; entry = entry->next) {
+        for_each_irq_pin(entry, cfg->irq_2_pin) {
                 unsigned int reg;
                 pin = entry->pin;
                 reg = io_apic_read(entry->apic, 0x10 + pin * 2);
@@ -580,7 +579,6 @@ static void __unmask_IO_APIC_irq(struct irq_cfg *cfg)
         io_apic_modify_irq(cfg, ~IO_APIC_REDIR_MASKED, 0, NULL);
 }
 
-#ifdef CONFIG_X86_64
 static void io_apic_sync(struct irq_pin_list *entry)
 {
         /*
@@ -596,11 +594,6 @@ static void __mask_IO_APIC_irq(struct irq_cfg *cfg)
 {
         io_apic_modify_irq(cfg, ~0, IO_APIC_REDIR_MASKED, &io_apic_sync);
 }
-#else /* CONFIG_X86_32 */
-static void __mask_IO_APIC_irq(struct irq_cfg *cfg)
-{
-        io_apic_modify_irq(cfg, ~0, IO_APIC_REDIR_MASKED, NULL);
-}
 
 static void __mask_and_edge_IO_APIC_irq(struct irq_cfg *cfg)
 {
@@ -613,7 +606,6 @@ static void __unmask_and_level_IO_APIC_irq(struct irq_cfg *cfg)
         io_apic_modify_irq(cfg, ~IO_APIC_REDIR_MASKED,
                         IO_APIC_REDIR_LEVEL_TRIGGER, NULL);
 }
-#endif /* CONFIG_X86_32 */
 
 static void mask_IO_APIC_irq_desc(struct irq_desc *desc)
 {
@@ -883,7 +875,7 @@ static int __init find_isa_irq_apic(int irq, int type)
  */
 static int EISA_ELCR(unsigned int irq)
 {
-        if (irq < NR_IRQS_LEGACY) {
+        if (irq < nr_legacy_irqs) {
                 unsigned int port = 0x4d0 + (irq >> 3);
                 return (inb(port) >> (irq & 7)) & 1;
         }
@@ -1480,7 +1472,7 @@ static void setup_IO_APIC_irq(int apic_id, int pin, unsigned int irq, struct irq
         }
 
         ioapic_register_intr(irq, desc, trigger);
-        if (irq < NR_IRQS_LEGACY)
+        if (irq < nr_legacy_irqs)
                 disable_8259A_irq(irq);
 
         ioapic_write_entry(apic_id, pin, entry);
@@ -1702,12 +1694,8 @@ __apicdebuginit(void) print_IO_APIC(void)
                 if (!entry)
                         continue;
                 printk(KERN_DEBUG "IRQ%d ", irq);
-                for (;;) {
+                for_each_irq_pin(entry, cfg->irq_2_pin)
                         printk("-> %d:%d", entry->apic, entry->pin);
-                        if (!entry->next)
-                                break;
-                        entry = entry->next;
-                }
                 printk("\n");
         }
 
@@ -1851,7 +1839,7 @@ __apicdebuginit(void) print_PIC(void)
         unsigned int v;
         unsigned long flags;
 
-        if (apic_verbosity == APIC_QUIET)
+        if (apic_verbosity == APIC_QUIET || !nr_legacy_irqs)
                 return;
 
         printk(KERN_DEBUG "\nprinting PIC contents\n");
@@ -1883,7 +1871,7 @@ __apicdebuginit(int) print_all_ICs(void)
         print_PIC();
 
         /* don't print out if apic is not there */
-        if (!cpu_has_apic || disable_apic)
+        if (!cpu_has_apic && !apic_from_smp_config())
                 return 0;
 
         print_all_local_APICs();
@@ -1914,6 +1902,10 @@ void __init enable_IO_APIC(void)
                 spin_unlock_irqrestore(&ioapic_lock, flags);
                 nr_ioapic_registers[apic] = reg_01.bits.entries+1;
         }
+
+        if (!nr_legacy_irqs)
+                return;
+
         for(apic = 0; apic < nr_ioapics; apic++) {
                 int pin;
                 /* See if any of the pins is in ExtINT mode */
@@ -1968,6 +1960,9 @@ void disable_IO_APIC(void)
          */
         clear_IO_APIC();
 
+        if (!nr_legacy_irqs)
+                return;
+
         /*
          * If the i8259 is routed through an IOAPIC
          * Put that IOAPIC in virtual wire mode
@@ -2001,7 +1996,7 @@ void disable_IO_APIC(void)
         /*
          * Use virtual wire A mode when interrupt remapping is enabled.
          */
-        if (cpu_has_apic)
+        if (cpu_has_apic || apic_from_smp_config())
                 disconnect_bsp_APIC(!intr_remapping_enabled &&
                                 ioapic_i8259.pin != -1);
 }
@@ -2014,7 +2009,7 @@ void disable_IO_APIC(void)
  * by Matt Domsch <Matt_Domsch@dell.com>  Tue Dec 21 12:25:05 CST 1999
  */
 
-static void __init setup_ioapic_ids_from_mpc(void)
+void __init setup_ioapic_ids_from_mpc(void)
 {
         union IO_APIC_reg_00 reg_00;
         physid_mask_t phys_id_present_map;
@@ -2023,9 +2018,8 @@ static void __init setup_ioapic_ids_from_mpc(void)
         unsigned char old_id;
         unsigned long flags;
 
-        if (x86_quirks->setup_ioapic_ids && x86_quirks->setup_ioapic_ids())
+        if (acpi_ioapic)
                 return;
-
         /*
          * Don't check I/O APIC IDs for xAPIC systems.  They have
          * no meaning without the serial APIC bus.
@@ -2199,7 +2193,7 @@ static unsigned int startup_ioapic_irq(unsigned int irq)
         struct irq_cfg *cfg;
 
         spin_lock_irqsave(&ioapic_lock, flags);
-        if (irq < NR_IRQS_LEGACY) {
+        if (irq < nr_legacy_irqs) {
                 disable_8259A_irq(irq);
                 if (i8259A_irq_pending(irq))
                         was_pending = 1;
@@ -2211,7 +2205,6 @@ static unsigned int startup_ioapic_irq(unsigned int irq)
         return was_pending;
 }
 
-#ifdef CONFIG_X86_64
 static int ioapic_retrigger_irq(unsigned int irq)
 {
 
@@ -2224,14 +2217,6 @@ static int ioapic_retrigger_irq(unsigned int irq)
 
         return 1;
 }
-#else
-static int ioapic_retrigger_irq(unsigned int irq)
-{
-        apic->send_IPI_self(irq_cfg(irq)->vector);
-
-        return 1;
-}
-#endif
 
 /*
  * Level and edge triggered IO-APIC interrupts need different handling,
@@ -2269,13 +2254,9 @@ static void __target_IO_APIC_irq(unsigned int irq, unsigned int dest, struct irq
         struct irq_pin_list *entry;
         u8 vector = cfg->vector;
 
-        entry = cfg->irq_2_pin;
-        for (;;) {
+        for_each_irq_pin(entry, cfg->irq_2_pin) {
                 unsigned int reg;
 
-                if (!entry)
-                        break;
-
                 apic = entry->apic;
                 pin = entry->pin;
                 /*
@@ -2288,9 +2269,6 @@ static void __target_IO_APIC_irq(unsigned int irq, unsigned int dest, struct irq
                 reg &= ~IO_APIC_REDIR_VECTOR_MASK;
                 reg |= vector;
                 io_apic_modify(apic, 0x10 + pin*2, reg);
-                if (!entry->next)
-                        break;
-                entry = entry->next;
         }
 }
 
@@ -2515,11 +2493,8 @@ atomic_t irq_mis_count;
 static void ack_apic_level(unsigned int irq)
 {
         struct irq_desc *desc = irq_to_desc(irq);
-
-#ifdef CONFIG_X86_32
         unsigned long v;
         int i;
-#endif
         struct irq_cfg *cfg;
         int do_unmask_irq = 0;
 
@@ -2532,31 +2507,28 @@ static void ack_apic_level(unsigned int irq)
         }
 #endif
 
-#ifdef CONFIG_X86_32
         /*
-        * It appears there is an erratum which affects at least version 0x11
-        * of I/O APIC (that's the 82093AA and cores integrated into various
-        * chipsets).  Under certain conditions a level-triggered interrupt is
-        * erroneously delivered as edge-triggered one but the respective IRR
-        * bit gets set nevertheless.  As a result the I/O unit expects an EOI
-        * message but it will never arrive and further interrupts are blocked
-        * from the source.  The exact reason is so far unknown, but the
-        * phenomenon was observed when two consecutive interrupt requests
-        * from a given source get delivered to the same CPU and the source is
-        * temporarily disabled in between.
-        *
-        * A workaround is to simulate an EOI message manually.  We achieve it
-        * by setting the trigger mode to edge and then to level when the edge
-        * trigger mode gets detected in the TMR of a local APIC for a
-        * level-triggered interrupt.  We mask the source for the time of the
-        * operation to prevent an edge-triggered interrupt escaping meanwhile.
-        * The idea is from Manfred Spraul.  --macro
-        */
+         * It appears there is an erratum which affects at least version 0x11
+         * of I/O APIC (that's the 82093AA and cores integrated into various
+         * chipsets).  Under certain conditions a level-triggered interrupt is
+         * erroneously delivered as edge-triggered one but the respective IRR
+         * bit gets set nevertheless.  As a result the I/O unit expects an EOI
+         * message but it will never arrive and further interrupts are blocked
+         * from the source.  The exact reason is so far unknown, but the
+         * phenomenon was observed when two consecutive interrupt requests
+         * from a given source get delivered to the same CPU and the source is
+         * temporarily disabled in between.
+         *
+         * A workaround is to simulate an EOI message manually.  We achieve it
+         * by setting the trigger mode to edge and then to level when the edge
+         * trigger mode gets detected in the TMR of a local APIC for a
+         * level-triggered interrupt.  We mask the source for the time of the
+         * operation to prevent an edge-triggered interrupt escaping meanwhile.
+         * The idea is from Manfred Spraul.  --macro
+         */
         cfg = desc->chip_data;
         i = cfg->vector;
-
         v = apic_read(APIC_TMR + ((i & ~0x1f) >> 1));
-#endif
 
         /*
          * We must acknowledge the irq before we move it or the acknowledge will
@@ -2598,7 +2570,7 @@ static void ack_apic_level(unsigned int irq)
                 unmask_IO_APIC_irq_desc(desc);
         }
 
-#ifdef CONFIG_X86_32
+        /* Tail end of version 0x11 I/O APIC bug workaround */
         if (!(v & (1 << (i & 0x1f)))) {
                 atomic_inc(&irq_mis_count);
                 spin_lock(&ioapic_lock);
@@ -2606,26 +2578,15 @@ static void ack_apic_level(unsigned int irq)
                 __unmask_and_level_IO_APIC_irq(cfg);
                 spin_unlock(&ioapic_lock);
         }
-#endif
 }
 
 #ifdef CONFIG_INTR_REMAP
 static void __eoi_ioapic_irq(unsigned int irq, struct irq_cfg *cfg)
 {
-        int apic, pin;
         struct irq_pin_list *entry;
 
-        entry = cfg->irq_2_pin;
-        for (;;) {
-
-                if (!entry)
-                        break;
-
-                apic = entry->apic;
-                pin = entry->pin;
-                io_apic_eoi(apic, pin);
-                entry = entry->next;
-        }
+        for_each_irq_pin(entry, cfg->irq_2_pin)
+                io_apic_eoi(entry->apic, entry->pin);
 }
 
 static void
@@ -2710,7 +2671,7 @@ static inline void init_IO_APIC_traps(void)
                          * so default to an old-fashioned 8259
                          * interrupt if we can..
                          */
-                        if (irq < NR_IRQS_LEGACY)
+                        if (irq < nr_legacy_irqs)
                                 make_8259A_irq(irq);
                         else
                                 /* Strange. Oh, well.. */
@@ -3046,7 +3007,7 @@ out:
  * the I/O APIC in all cases now.  No actual device should request
  * it anyway.  --macro
  */
-#define PIC_IRQS        (1 << PIC_CASCADE_IR)
+#define PIC_IRQS        (1UL << PIC_CASCADE_IR)
 
 void __init setup_IO_APIC(void)
 {
@@ -3054,21 +3015,19 @@ void __init setup_IO_APIC(void)
         /*
          * calling enable_IO_APIC() is moved to setup_local_APIC for BP
          */
-
-        io_apic_irqs = ~PIC_IRQS;
+        io_apic_irqs = nr_legacy_irqs ? ~PIC_IRQS : ~0UL;
 
         apic_printk(APIC_VERBOSE, "ENABLING IO-APIC IRQs\n");
         /*
          * Set up IO-APIC IRQ routing.
          */
-#ifdef CONFIG_X86_32
-        if (!acpi_ioapic)
-                setup_ioapic_ids_from_mpc();
-#endif
+        x86_init.mpparse.setup_ioapic_ids();
+
         sync_Arb_IDs();
         setup_IO_APIC_irqs();
         init_IO_APIC_traps();
-        check_timer();
+        if (nr_legacy_irqs)
+                check_timer();
 }
 
 /*
@@ -3169,7 +3128,6 @@ static int __init ioapic_init_sysfs(void)
 
 device_initcall(ioapic_init_sysfs);
 
-static int nr_irqs_gsi = NR_IRQS_LEGACY;
 /*
  * Dynamic irq allocate and deallocation
  */
@@ -3241,8 +3199,7 @@ void destroy_irq(unsigned int irq)
         cfg = desc->chip_data;
         dynamic_irq_cleanup(irq);
         /* connect back irq_cfg */
-        if (desc)
-                desc->chip_data = cfg;
+        desc->chip_data = cfg;
 
         free_irte(irq);
         spin_lock_irqsave(&vector_lock, flags);
@@ -3910,9 +3867,13 @@ static int __io_apic_set_pci_routing(struct device *dev, int irq,
         /*
          * IRQs < 16 are already in the irq_2_pin[] map
          */
-        if (irq >= NR_IRQS_LEGACY) {
+        if (irq >= nr_legacy_irqs) {
                 cfg = desc->chip_data;
-                add_pin_to_irq_node(cfg, node, ioapic, pin);
+                if (add_pin_to_irq_node_nopanic(cfg, node, ioapic, pin)) {
+                        printk(KERN_INFO "can not add pin %d for irq %d\n",
+                                pin, irq);
+                        return 0;
+                }
         }
 
         setup_IO_APIC_irq(ioapic, pin, irq, desc, trigger, polarity);
@@ -3941,11 +3902,28 @@ int io_apic_set_pci_routing(struct device *dev, int irq,
         return __io_apic_set_pci_routing(dev, irq, irq_attr);
 }
 
-/* --------------------------------------------------------------------------
-                          ACPI-based IOAPIC Configuration
-   -------------------------------------------------------------------------- */
+u8 __init io_apic_unique_id(u8 id)
+{
+#ifdef CONFIG_X86_32
+        if ((boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) &&
+            !APIC_XAPIC(apic_version[boot_cpu_physical_apicid]))
+                return io_apic_get_unique_id(nr_ioapics, id);
+        else
+                return id;
+#else
+        int i;
+        DECLARE_BITMAP(used, 256);
 
-#ifdef CONFIG_ACPI
+        bitmap_zero(used, 256);
+        for (i = 0; i < nr_ioapics; i++) {
+                struct mpc_ioapic *ia = &mp_ioapics[i];
+                __set_bit(ia->apicid, used);
+        }
+        if (!test_bit(id, used))
+                return id;
+        return find_first_zero_bit(used, 256);
+#endif
+}
 
 #ifdef CONFIG_X86_32
 int __init io_apic_get_unique_id(int ioapic, int apic_id)
@@ -4054,8 +4032,6 @@ int acpi_get_override_irq(int bus_irq, int *trigger, int *polarity)
         return 0;
 }
 
-#endif /* CONFIG_ACPI */
-
 /*
  * This function currently is only a helper for the i386 smp boot process where
  * we need to reprogram the ioredtbls to cater for the cpus which have come online
@@ -4109,7 +4085,7 @@ void __init setup_ioapic_dest(void)
 
 static struct resource *ioapic_resources;
 
-static struct resource * __init ioapic_setup_resources(void)
+static struct resource * __init ioapic_setup_resources(int nr_ioapics)
 {
         unsigned long n;
         struct resource *res;
@@ -4125,15 +4101,13 @@ static struct resource * __init ioapic_setup_resources(void)
         mem = alloc_bootmem(n);
         res = (void *)mem;
 
-        if (mem != NULL) {
-                mem += sizeof(struct resource) * nr_ioapics;
+        mem += sizeof(struct resource) * nr_ioapics;
 
-                for (i = 0; i < nr_ioapics; i++) {
-                        res[i].name = mem;
-                        res[i].flags = IORESOURCE_MEM | IORESOURCE_BUSY;
-                        sprintf(mem,  "IOAPIC %u", i);
-                        mem += IOAPIC_RESOURCE_NAME_SIZE;
-                }
+        for (i = 0; i < nr_ioapics; i++) {
+                res[i].name = mem;
+                res[i].flags = IORESOURCE_MEM | IORESOURCE_BUSY;
+                sprintf(mem,  "IOAPIC %u", i);
+                mem += IOAPIC_RESOURCE_NAME_SIZE;
         }
 
         ioapic_resources = res;
@@ -4147,7 +4121,7 @@ void __init ioapic_init_mappings(void)
         struct resource *ioapic_res;
         int i;
 
-        ioapic_res = ioapic_setup_resources();
+        ioapic_res = ioapic_setup_resources(nr_ioapics);
         for (i = 0; i < nr_ioapics; i++) {
                 if (smp_found_config) {
                         ioapic_phys = mp_ioapics[i].apicaddr;
@@ -4176,11 +4150,9 @@ fake_ioapic_page:
                             __fix_to_virt(idx), ioapic_phys);
                 idx++;
 
-                if (ioapic_res != NULL) {
-                        ioapic_res->start = ioapic_phys;
-                        ioapic_res->end = ioapic_phys + (4 * 1024) - 1;
-                        ioapic_res++;
-                }
+                ioapic_res->start = ioapic_phys;
+                ioapic_res->end = ioapic_phys + (4 * 1024) - 1;
+                ioapic_res++;
         }
 }
 
@@ -4201,3 +4173,76 @@ void __init ioapic_insert_resources(void)
                 r++;
         }
 }
+
+int mp_find_ioapic(int gsi)
+{
+        int i = 0;
+
+        /* Find the IOAPIC that manages this GSI. */
+        for (i = 0; i < nr_ioapics; i++) {
+                if ((gsi >= mp_gsi_routing[i].gsi_base)
+                    && (gsi <= mp_gsi_routing[i].gsi_end))
+                        return i;
+        }
+
+        printk(KERN_ERR "ERROR: Unable to locate IOAPIC for GSI %d\n", gsi);
+        return -1;
+}
+
+int mp_find_ioapic_pin(int ioapic, int gsi)
+{
+        if (WARN_ON(ioapic == -1))
+                return -1;
+        if (WARN_ON(gsi > mp_gsi_routing[ioapic].gsi_end))
+                return -1;
+
+        return gsi - mp_gsi_routing[ioapic].gsi_base;
+}
+
+static int bad_ioapic(unsigned long address)
+{
+        if (nr_ioapics >= MAX_IO_APICS) {
+                printk(KERN_WARNING "WARING: Max # of I/O APICs (%d) exceeded "
+                       "(found %d), skipping\n", MAX_IO_APICS, nr_ioapics);
+                return 1;
+        }
+        if (!address) {
+                printk(KERN_WARNING "WARNING: Bogus (zero) I/O APIC address"
+                       " found in table, skipping!\n");
+                return 1;
+        }
+        return 0;
+}
+
+void __init mp_register_ioapic(int id, u32 address, u32 gsi_base)
+{
+        int idx = 0;
+
+        if (bad_ioapic(address))
+                return;
+
+        idx = nr_ioapics;
+
+        mp_ioapics[idx].type = MP_IOAPIC;
+        mp_ioapics[idx].flags = MPC_APIC_USABLE;
+        mp_ioapics[idx].apicaddr = address;
+
+        set_fixmap_nocache(FIX_IO_APIC_BASE_0 + idx, address);
+        mp_ioapics[idx].apicid = io_apic_unique_id(id);
+        mp_ioapics[idx].apicver = io_apic_get_version(idx);
+
+        /*
+         * Build basic GSI lookup table to facilitate gsi->io_apic lookups
+         * and to prevent reprogramming of IOAPIC pins (PCI GSIs).
+         */
+        mp_gsi_routing[idx].gsi_base = gsi_base;
+        mp_gsi_routing[idx].gsi_end = gsi_base +
+            io_apic_get_redir_entries(idx);
+
+        printk(KERN_INFO "IOAPIC[%d]: apic_id %d, version %d, address 0x%x, "
+               "GSI %d-%d\n", idx, mp_ioapics[idx].apicid,
+               mp_ioapics[idx].apicver, mp_ioapics[idx].apicaddr,
+               mp_gsi_routing[idx].gsi_base, mp_gsi_routing[idx].gsi_end);
+
+        nr_ioapics++;
+}
diff --git a/arch/x86/kernel/apic/ipi.c b/arch/x86/kernel/apic/ipi.c
index 6ef00ba4c886..08385e090a6f 100644
--- a/arch/x86/kernel/apic/ipi.c
+++ b/arch/x86/kernel/apic/ipi.c
@@ -153,7 +153,7 @@ int safe_smp_processor_id(void)
 {
         int apicid, cpuid;
 
-        if (!boot_cpu_has(X86_FEATURE_APIC))
+        if (!cpu_has_apic)
                 return 0;
 
         apicid = hard_smp_processor_id();
diff --git a/arch/x86/kernel/apic/nmi.c b/arch/x86/kernel/apic/nmi.c
index b3025b43b63a..7ff61d6a188a 100644
--- a/arch/x86/kernel/apic/nmi.c
+++ b/arch/x86/kernel/apic/nmi.c
@@ -39,7 +39,7 @@
 int unknown_nmi_panic;
 int nmi_watchdog_enabled;
 
-static cpumask_var_t backtrace_mask;
+static cpumask_t backtrace_mask __read_mostly;
 
 /* nmi_active:
  * >0: the lapic NMI watchdog is active, but can be disabled
@@ -66,7 +66,7 @@ static inline unsigned int get_nmi_count(int cpu)
 
 static inline int mce_in_progress(void)
 {
-#if defined(CONFIG_X86_NEW_MCE)
+#if defined(CONFIG_X86_MCE)
         return atomic_read(&mce_entry) > 0;
 #endif
         return 0;
@@ -138,7 +138,6 @@ int __init check_nmi_watchdog(void)
         if (!prev_nmi_count)
                 goto error;
 
-        alloc_cpumask_var(&backtrace_mask, GFP_KERNEL|__GFP_ZERO);
         printk(KERN_INFO "Testing NMI watchdog ... ");
 
 #ifdef CONFIG_SMP
@@ -415,14 +414,17 @@ nmi_watchdog_tick(struct pt_regs *regs, unsigned reason)
         }
 
         /* We can be called before check_nmi_watchdog, hence NULL check. */
-        if (backtrace_mask != NULL && cpumask_test_cpu(cpu, backtrace_mask)) {
+        if (cpumask_test_cpu(cpu, &backtrace_mask)) {
                 static DEFINE_SPINLOCK(lock);   /* Serialise the printks */
 
                 spin_lock(&lock);
                 printk(KERN_WARNING "NMI backtrace for cpu %d\n", cpu);
+                show_regs(regs);
                 dump_stack();
                 spin_unlock(&lock);
-                cpumask_clear_cpu(cpu, backtrace_mask);
+                cpumask_clear_cpu(cpu, &backtrace_mask);
+
+                rc = 1;
         }
 
         /* Could check oops_in_progress here too, but it's safer not to */
@@ -506,14 +508,14 @@ static int unknown_nmi_panic_callback(struct pt_regs *regs, int cpu)
 /*
  * proc handler for /proc/sys/kernel/nmi
  */
-int proc_nmi_enabled(struct ctl_table *table, int write, struct file *file,
+int proc_nmi_enabled(struct ctl_table *table, int write,
                         void __user *buffer, size_t *length, loff_t *ppos)
 {
         int old_state;
 
         nmi_watchdog_enabled = (atomic_read(&nmi_active) > 0) ? 1 : 0;
         old_state = nmi_watchdog_enabled;
-        proc_dointvec(table, write, file, buffer, length, ppos);
+        proc_dointvec(table, write, buffer, length, ppos);
         if (!!old_state == !!nmi_watchdog_enabled)
                 return 0;
 
@@ -552,14 +554,18 @@ int do_nmi_callback(struct pt_regs *regs, int cpu)
         return 0;
 }
 
-void __trigger_all_cpu_backtrace(void)
+void arch_trigger_all_cpu_backtrace(void)
 {
         int i;
 
-        cpumask_copy(backtrace_mask, cpu_online_mask);
+        cpumask_copy(&backtrace_mask, cpu_online_mask);
+
+        printk(KERN_INFO "sending NMI to all CPUs:\n");
+        apic->send_IPI_all(NMI_VECTOR);
+
         /* Wait for up to 10 seconds for all CPUs to do the backtrace */
         for (i = 0; i < 10 * 1000; i++) {
-                if (cpumask_empty(backtrace_mask))
+                if (cpumask_empty(&backtrace_mask))
                         break;
                 mdelay(1);
         }
diff --git a/arch/x86/kernel/apic/numaq_32.c b/arch/x86/kernel/apic/numaq_32.c
index ca96e68f0d23..efa00e2b8505 100644
--- a/arch/x86/kernel/apic/numaq_32.c
+++ b/arch/x86/kernel/apic/numaq_32.c
@@ -66,7 +66,6 @@ struct mpc_trans {
         unsigned short                  trans_reserved;
 };
 
-/* x86_quirks member */
 static int                              mpc_record;
 
 static struct mpc_trans                 *translation_table[MAX_MPC_ENTRY];
@@ -130,10 +129,9 @@ void __cpuinit numaq_tsc_disable(void)
         }
 }
 
-static int __init numaq_pre_time_init(void)
+static void __init numaq_tsc_init(void)
 {
         numaq_tsc_disable();
-        return 0;
 }
 
 static inline int generate_logical_apicid(int quad, int phys_apicid)
@@ -177,6 +175,19 @@ static void mpc_oem_pci_bus(struct mpc_bus *m)
         quad_local_to_mp_bus_id[quad][local] = m->busid;
 }
 
+/*
+ * Called from mpparse code.
+ * mode = 0: prescan
+ * mode = 1: one mpc entry scanned
+ */
+static void numaq_mpc_record(unsigned int mode)
+{
+        if (!mode)
+                mpc_record = 0;
+        else
+                mpc_record++;
+}
+
 static void __init MP_translation_info(struct mpc_trans *m)
 {
         printk(KERN_INFO
@@ -206,9 +217,9 @@ static int __init mpf_checksum(unsigned char *mp, int len)
 /*
  * Read/parse the MPC oem tables
  */
-static void __init
- smp_read_mpc_oem(struct mpc_oemtable *oemtable, unsigned short oemsize)
+static void __init smp_read_mpc_oem(struct mpc_table *mpc)
 {
+        struct mpc_oemtable *oemtable = (void *)(long)mpc->oemptr;
         int count = sizeof(*oemtable);  /* the header size */
         unsigned char *oemptr = ((unsigned char *)oemtable) + count;
 
@@ -250,29 +261,6 @@ static void __init
         }
 }
 
-static int __init numaq_setup_ioapic_ids(void)
-{
-        /* so can skip it */
-        return 1;
-}
-
-static struct x86_quirks numaq_x86_quirks __initdata = {
-        .arch_pre_time_init             = numaq_pre_time_init,
-        .arch_time_init                 = NULL,
-        .arch_pre_intr_init             = NULL,
-        .arch_memory_setup              = NULL,
-        .arch_intr_init                 = NULL,
-        .arch_trap_init                 = NULL,
-        .mach_get_smp_config            = NULL,
-        .mach_find_smp_config           = NULL,
-        .mpc_record                     = &mpc_record,
-        .mpc_apic_id                    = mpc_apic_id,
-        .mpc_oem_bus_info               = mpc_oem_bus_info,
-        .mpc_oem_pci_bus                = mpc_oem_pci_bus,
-        .smp_read_mpc_oem               = smp_read_mpc_oem,
-        .setup_ioapic_ids               = numaq_setup_ioapic_ids,
-};
-
 static __init void early_check_numaq(void)
 {
         /*
@@ -286,8 +274,15 @@ static __init void early_check_numaq(void)
         if (smp_found_config)
                 early_get_smp_config();
 
-        if (found_numaq)
-                x86_quirks = &numaq_x86_quirks;
+        if (found_numaq) {
+                x86_init.mpparse.mpc_record = numaq_mpc_record;
+                x86_init.mpparse.setup_ioapic_ids = x86_init_noop;
+                x86_init.mpparse.mpc_apic_id = mpc_apic_id;
+                x86_init.mpparse.smp_read_mpc_oem = smp_read_mpc_oem;
+                x86_init.mpparse.mpc_oem_pci_bus = mpc_oem_pci_bus;
+                x86_init.mpparse.mpc_oem_bus_info = mpc_oem_bus_info;
+                x86_init.timers.tsc_pre_init = numaq_tsc_init;
+        }
 }
 
 int __init get_memcfg_numaq(void)
@@ -418,7 +413,7 @@ static inline physid_mask_t numaq_apicid_to_cpu_present(int logical_apicid)
 /* Where the IO area was mapped on multiquad, always 0 otherwise */
 void *xquad_portio;
 
-static inline int numaq_check_phys_apicid_present(int boot_cpu_physical_apicid)
+static inline int numaq_check_phys_apicid_present(int phys_apicid)
 {
         return 1;
 }
diff --git a/arch/x86/kernel/apic/probe_64.c b/arch/x86/kernel/apic/probe_64.c
index fcec2f1d34a1..c4cbd3080c1c 100644
--- a/arch/x86/kernel/apic/probe_64.c
+++ b/arch/x86/kernel/apic/probe_64.c
@@ -55,25 +55,32 @@ static int apicid_phys_pkg_id(int initial_apic_id, int index_msb)
 void __init default_setup_apic_routing(void)
 {
 #ifdef CONFIG_X86_X2APIC
-        if (x2apic_mode && (apic != &apic_x2apic_phys &&
+        if (x2apic_mode
 #ifdef CONFIG_X86_UV
-                       apic != &apic_x2apic_uv_x &&
+                       && apic != &apic_x2apic_uv_x
 #endif
-                       apic != &apic_x2apic_cluster)) {
+                       ) {
                 if (x2apic_phys)
                         apic = &apic_x2apic_phys;
                 else
                         apic = &apic_x2apic_cluster;
-                printk(KERN_INFO "Setting APIC routing to %s\n", apic->name);
         }
 #endif
 
         if (apic == &apic_flat) {
-                if (max_physical_apicid >= 8)
-                        apic = &apic_physflat;
-                printk(KERN_INFO "Setting APIC routing to %s\n", apic->name);
+                switch (boot_cpu_data.x86_vendor) {
+                case X86_VENDOR_INTEL:
+                        if (num_processors > 8)
+                                apic = &apic_physflat;
+                        break;
+                case X86_VENDOR_AMD:
+                        if (max_physical_apicid >= 8)
+                                apic = &apic_physflat;
+                }
         }
 
+        printk(KERN_INFO "Setting APIC routing to %s\n", apic->name);
+
         if (is_vsmp_box()) {
                 /* need to update phys_pkg_id */
                 apic->phys_pkg_id = apicid_phys_pkg_id;
diff --git a/arch/x86/kernel/apic/summit_32.c b/arch/x86/kernel/apic/summit_32.c
index eafdfbd1ea95..645ecc4ff0be 100644
--- a/arch/x86/kernel/apic/summit_32.c
+++ b/arch/x86/kernel/apic/summit_32.c
@@ -272,7 +272,7 @@ static physid_mask_t summit_apicid_to_cpu_present(int apicid)
         return physid_mask_of_physid(0);
 }
 
-static int summit_check_phys_apicid_present(int boot_cpu_physical_apicid)
+static int summit_check_phys_apicid_present(int physical_apicid)
 {
         return 1;
 }
diff --git a/arch/x86/kernel/apic/x2apic_uv_x.c b/arch/x86/kernel/apic/x2apic_uv_x.c
index 601159374e87..f5f5886a6b53 100644
--- a/arch/x86/kernel/apic/x2apic_uv_x.c
+++ b/arch/x86/kernel/apic/x2apic_uv_x.c
@@ -389,6 +389,16 @@ static __init void map_gru_high(int max_pnode)
                 map_high("GRU", gru.s.base, shift, max_pnode, map_wb);
 }
 
+static __init void map_mmr_high(int max_pnode)
+{
+        union uvh_rh_gam_mmr_overlay_config_mmr_u mmr;
+        int shift = UVH_RH_GAM_MMR_OVERLAY_CONFIG_MMR_BASE_SHFT;
+
+        mmr.v = uv_read_local_mmr(UVH_RH_GAM_MMR_OVERLAY_CONFIG_MMR);
+        if (mmr.s.enable)
+                map_high("MMR", mmr.s.base, shift, max_pnode, map_uc);
+}
+
 static __init void map_mmioh_high(int max_pnode)
 {
         union uvh_rh_gam_mmioh_overlay_config_mmr_u mmioh;
@@ -643,6 +653,7 @@ void __init uv_system_init(void)
         }
 
         map_gru_high(max_pnode);
+        map_mmr_high(max_pnode);
         map_mmioh_high(max_pnode);
 
         uv_cpu_init();
diff --git a/arch/x86/kernel/apm_32.c b/arch/x86/kernel/apm_32.c
index 442b5508893f..151ace69a5aa 100644
--- a/arch/x86/kernel/apm_32.c
+++ b/arch/x86/kernel/apm_32.c
@@ -403,7 +403,15 @@ static DECLARE_WAIT_QUEUE_HEAD(apm_waitqueue);
 static DECLARE_WAIT_QUEUE_HEAD(apm_suspend_waitqueue);
 static struct apm_user *user_list;
 static DEFINE_SPINLOCK(user_list_lock);
-static const struct desc_struct bad_bios_desc = { { { 0, 0x00409200 } } };
+
+/*
+ * Set up a segment that references the real mode segment 0x40
+ * that extends up to the end of page zero (that we have reserved).
+ * This is for buggy BIOS's that refer to (real mode) segment 0x40
+ * even though they are called in protected mode.
+ */
+static struct desc_struct bad_bios_desc = GDT_ENTRY_INIT(0x4092,
+                        (unsigned long)__va(0x400UL), PAGE_SIZE - 0x400 - 1);
 
 static const char driver_version[] = "1.16ac";  /* no spaces */
 
@@ -2332,15 +2340,6 @@ static int __init apm_init(void)
         pm_flags |= PM_APM;
 
         /*
-         * Set up a segment that references the real mode segment 0x40
-         * that extends up to the end of page zero (that we have reserved).
-         * This is for buggy BIOS's that refer to (real mode) segment 0x40
-         * even though they are called in protected mode.
-         */
-        set_base(bad_bios_desc, __va((unsigned long)0x40 << 4));
-        _set_limit((char *)&bad_bios_desc, 4095 - (0x40 << 4));
-
-        /*
          * Set up the long jump entry point to the APM BIOS, which is called
          * from inline assembly.
          */
@@ -2358,12 +2357,12 @@ static int __init apm_init(void)
          * code to that CPU.
          */
         gdt = get_cpu_gdt_table(0);
-        set_base(gdt[APM_CS >> 3],
-                 __va((unsigned long)apm_info.bios.cseg << 4));
-        set_base(gdt[APM_CS_16 >> 3],
-                 __va((unsigned long)apm_info.bios.cseg_16 << 4));
-        set_base(gdt[APM_DS >> 3],
-                 __va((unsigned long)apm_info.bios.dseg << 4));
+        set_desc_base(&gdt[APM_CS >> 3],
+                 (unsigned long)__va((unsigned long)apm_info.bios.cseg << 4));
+        set_desc_base(&gdt[APM_CS_16 >> 3],
+                 (unsigned long)__va((unsigned long)apm_info.bios.cseg_16 << 4));
+        set_desc_base(&gdt[APM_DS >> 3],
+                 (unsigned long)__va((unsigned long)apm_info.bios.dseg << 4));
 
         proc_create("apm", 0, NULL, &apm_file_ops);
 
diff --git a/arch/x86/kernel/asm-offsets_64.c b/arch/x86/kernel/asm-offsets_64.c
index 898ecc47e129..4a6aeedcd965 100644
--- a/arch/x86/kernel/asm-offsets_64.c
+++ b/arch/x86/kernel/asm-offsets_64.c
@@ -3,6 +3,7 @@
  * This code generates raw asm output which is post-processed to extract
  * and format the required data.
  */
+#define COMPILE_OFFSETS
 
 #include <linux/crypto.h>
 #include <linux/sched.h> 
diff --git a/arch/x86/kernel/cpu/Makefile b/arch/x86/kernel/cpu/Makefile
index c1f253dac155..68537e957a9b 100644
--- a/arch/x86/kernel/cpu/Makefile
+++ b/arch/x86/kernel/cpu/Makefile
@@ -13,7 +13,7 @@ CFLAGS_common.o		:= $(nostackp)
 
 obj-y                   := intel_cacheinfo.o addon_cpuid_features.o
 obj-y                   += proc.o capflags.o powerflags.o common.o
-obj-y                   += vmware.o hypervisor.o
+obj-y                   += vmware.o hypervisor.o sched.o
 
 obj-$(CONFIG_X86_32)    += bugs.o cmpxchg.o
 obj-$(CONFIG_X86_64)    += bugs_64.o
@@ -27,7 +27,7 @@ obj-$(CONFIG_CPU_SUP_CENTAUR)		+= centaur.o
 obj-$(CONFIG_CPU_SUP_TRANSMETA_32)      += transmeta.o
 obj-$(CONFIG_CPU_SUP_UMC_32)            += umc.o
 
-obj-$(CONFIG_PERF_COUNTERS)             += perf_counter.o
+obj-$(CONFIG_PERF_EVENTS)               += perf_event.o
 
 obj-$(CONFIG_X86_MCE)                   += mcheck/
 obj-$(CONFIG_MTRR)                      += mtrr/
diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c
index 63fddcd082cd..c910a716a71c 100644
--- a/arch/x86/kernel/cpu/amd.c
+++ b/arch/x86/kernel/cpu/amd.c
@@ -2,7 +2,7 @@
 #include <linux/bitops.h>
 #include <linux/mm.h>
 
-#include <asm/io.h>
+#include <linux/io.h>
 #include <asm/processor.h>
 #include <asm/apic.h>
 #include <asm/cpu.h>
@@ -45,8 +45,8 @@ static void __cpuinit init_amd_k5(struct cpuinfo_x86 *c)
 #define CBAR_ENB        (0x80000000)
 #define CBAR_KEY        (0X000000CB)
         if (c->x86_model == 9 || c->x86_model == 10) {
-                if (inl (CBAR) & CBAR_ENB)
-                        outl (0 | CBAR_KEY, CBAR);
+                if (inl(CBAR) & CBAR_ENB)
+                        outl(0 | CBAR_KEY, CBAR);
         }
 }
 
@@ -87,9 +87,10 @@ static void __cpuinit init_amd_k6(struct cpuinfo_x86 *c)
                 d = d2-d;
 
                 if (d > 20*K6_BUG_LOOP)
-                        printk("system stability may be impaired when more than 32 MB are used.\n");
+                        printk(KERN_CONT
+                                "system stability may be impaired when more than 32 MB are used.\n");
                 else
-                        printk("probably OK (after B9730xxxx).\n");
+                        printk(KERN_CONT "probably OK (after B9730xxxx).\n");
                 printk(KERN_INFO "Please see http://membres.lycos.fr/poulot/k6bug.html\n");
         }
 
@@ -183,7 +184,7 @@ static void __cpuinit amd_k7_smp_check(struct cpuinfo_x86 *c)
          * approved Athlon
          */
         WARN_ONCE(1, "WARNING: This combination of AMD"
-                "processors is not suitable for SMP.\n");
+                " processors is not suitable for SMP.\n");
         if (!test_taint(TAINT_UNSAFE_SMP))
                 add_taint(TAINT_UNSAFE_SMP);
 
@@ -219,8 +220,9 @@ static void __cpuinit init_amd_k7(struct cpuinfo_x86 *c)
         if ((c->x86_model == 8 && c->x86_mask >= 1) || (c->x86_model > 8)) {
                 rdmsr(MSR_K7_CLK_CTL, l, h);
                 if ((l & 0xfff00000) != 0x20000000) {
-                        printk ("CPU: CLK_CTL MSR was %x. Reprogramming to %x\n", l,
-                                ((l & 0x000fffff)|0x20000000));
+                        printk(KERN_INFO
+                            "CPU: CLK_CTL MSR was %x. Reprogramming to %x\n",
+                                        l, ((l & 0x000fffff)|0x20000000));
                         wrmsr(MSR_K7_CLK_CTL, (l & 0x000fffff)|0x20000000, h);
                 }
         }
@@ -251,6 +253,64 @@ static int __cpuinit nearby_node(int apicid)
 #endif
 
 /*
+ * Fixup core topology information for AMD multi-node processors.
+ * Assumption 1: Number of cores in each internal node is the same.
+ * Assumption 2: Mixed systems with both single-node and dual-node
+ *               processors are not supported.
+ */
+#ifdef CONFIG_X86_HT
+static void __cpuinit amd_fixup_dcm(struct cpuinfo_x86 *c)
+{
+#ifdef CONFIG_PCI
+        u32 t, cpn;
+        u8 n, n_id;
+        int cpu = smp_processor_id();
+
+        /* fixup topology information only once for a core */
+        if (cpu_has(c, X86_FEATURE_AMD_DCM))
+                return;
+
+        /* check for multi-node processor on boot cpu */
+        t = read_pci_config(0, 24, 3, 0xe8);
+        if (!(t & (1 << 29)))
+                return;
+
+        set_cpu_cap(c, X86_FEATURE_AMD_DCM);
+
+        /* cores per node: each internal node has half the number of cores */
+        cpn = c->x86_max_cores >> 1;
+
+        /* even-numbered NB_id of this dual-node processor */
+        n = c->phys_proc_id << 1;
+
+        /*
+         * determine internal node id and assign cores fifty-fifty to
+         * each node of the dual-node processor
+         */
+        t = read_pci_config(0, 24 + n, 3, 0xe8);
+        n = (t>>30) & 0x3;
+        if (n == 0) {
+                if (c->cpu_core_id < cpn)
+                        n_id = 0;
+                else
+                        n_id = 1;
+        } else {
+                if (c->cpu_core_id < cpn)
+                        n_id = 1;
+                else
+                        n_id = 0;
+        }
+
+        /* compute entire NodeID, use llc_shared_map to store sibling info */
+        per_cpu(cpu_llc_id, cpu) = (c->phys_proc_id << 1) + n_id;
+
+        /* fixup core id to be in range from 0 to cpn */
+        c->cpu_core_id = c->cpu_core_id % cpn;
+#endif
+}
+#endif
+
+/*
  * On a AMD dual core setup the lower bits of the APIC id distingush the cores.
  * Assumes number of cores is a power of two.
  */
@@ -267,17 +327,31 @@ static void __cpuinit amd_detect_cmp(struct cpuinfo_x86 *c)
         c->phys_proc_id = c->initial_apicid >> bits;
         /* use socket ID also for last level cache */
         per_cpu(cpu_llc_id, cpu) = c->phys_proc_id;
+        /* fixup topology information on multi-node processors */
+        if ((c->x86 == 0x10) && (c->x86_model == 9))
+                amd_fixup_dcm(c);
 #endif
 }
 
+int amd_get_nb_id(int cpu)
+{
+        int id = 0;
+#ifdef CONFIG_SMP
+        id = per_cpu(cpu_llc_id, cpu);
+#endif
+        return id;
+}
+EXPORT_SYMBOL_GPL(amd_get_nb_id);
+
 static void __cpuinit srat_detect_node(struct cpuinfo_x86 *c)
 {
 #if defined(CONFIG_NUMA) && defined(CONFIG_X86_64)
         int cpu = smp_processor_id();
         int node;
-        unsigned apicid = cpu_has_apic ? hard_smp_processor_id() : c->apicid;
+        unsigned apicid = c->apicid;
+
+        node = per_cpu(cpu_llc_id, cpu);
 
-        node = c->phys_proc_id;
         if (apicid_to_node[apicid] != NUMA_NO_NODE)
                 node = apicid_to_node[apicid];
         if (!node_online(node)) {
@@ -398,18 +472,30 @@ static void __cpuinit init_amd(struct cpuinfo_x86 *c)
                 u32 level;
 
                 level = cpuid_eax(1);
-                if((level >= 0x0f48 && level < 0x0f50) || level >= 0x0f58)
+                if ((level >= 0x0f48 && level < 0x0f50) || level >= 0x0f58)
                         set_cpu_cap(c, X86_FEATURE_REP_GOOD);
 
                 /*
                  * Some BIOSes incorrectly force this feature, but only K8
                  * revision D (model = 0x14) and later actually support it.
+                 * (AMD Erratum #110, docId: 25759).
                  */
-                if (c->x86_model < 0x14)
+                if (c->x86_model < 0x14 && cpu_has(c, X86_FEATURE_LAHF_LM)) {
+                        u64 val;
+
                         clear_cpu_cap(c, X86_FEATURE_LAHF_LM);
+                        if (!rdmsrl_amd_safe(0xc001100d, &val)) {
+                                val &= ~(1ULL << 32);
+                                wrmsrl_amd_safe(0xc001100d, val);
+                        }
+                }
+
         }
         if (c->x86 == 0x10 || c->x86 == 0x11)
                 set_cpu_cap(c, X86_FEATURE_REP_GOOD);
+
+        /* get apicid instead of initial apic id from cpuid */
+        c->apicid = hard_smp_processor_id();
 #else
 
         /*
@@ -494,27 +580,30 @@ static void __cpuinit init_amd(struct cpuinfo_x86 *c)
                  * benefit in doing so.
                  */
                 if (!rdmsrl_safe(MSR_K8_TSEG_ADDR, &tseg)) {
-                    printk(KERN_DEBUG "tseg: %010llx\n", tseg);
-                    if ((tseg>>PMD_SHIFT) <
+                        printk(KERN_DEBUG "tseg: %010llx\n", tseg);
+                        if ((tseg>>PMD_SHIFT) <
                                 (max_low_pfn_mapped>>(PMD_SHIFT-PAGE_SHIFT)) ||
-                        ((tseg>>PMD_SHIFT) <
+                                ((tseg>>PMD_SHIFT) <
                                 (max_pfn_mapped>>(PMD_SHIFT-PAGE_SHIFT)) &&
-                         (tseg>>PMD_SHIFT) >= (1ULL<<(32 - PMD_SHIFT))))
-                        set_memory_4k((unsigned long)__va(tseg), 1);
+                                (tseg>>PMD_SHIFT) >= (1ULL<<(32 - PMD_SHIFT))))
+                                set_memory_4k((unsigned long)__va(tseg), 1);
                 }
         }
 #endif
 }
 
 #ifdef CONFIG_X86_32
-static unsigned int __cpuinit amd_size_cache(struct cpuinfo_x86 *c, unsigned int size)
+static unsigned int __cpuinit amd_size_cache(struct cpuinfo_x86 *c,
+                                                        unsigned int size)
 {
         /* AMD errata T13 (order #21922) */
         if ((c->x86 == 6)) {
-                if (c->x86_model == 3 && c->x86_mask == 0)      /* Duron Rev A0 */
+                /* Duron Rev A0 */
+                if (c->x86_model == 3 && c->x86_mask == 0)
                         size = 64;
+                /* Tbird rev A1/A2 */
                 if (c->x86_model == 4 &&
-                    (c->x86_mask == 0 || c->x86_mask == 1))     /* Tbird rev A1/A2 */
+                        (c->x86_mask == 0 || c->x86_mask == 1))
                         size = 256;
         }
         return size;
diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index c8e315f1aa83..01a265212395 100644
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -81,7 +81,7 @@ static void __init check_fpu(void)
 
         boot_cpu_data.fdiv_bug = fdiv_bug;
         if (boot_cpu_data.fdiv_bug)
-                printk("Hmm, FPU with FDIV bug.\n");
+                printk(KERN_WARNING "Hmm, FPU with FDIV bug.\n");
 }
 
 static void __init check_hlt(void)
@@ -98,7 +98,7 @@ static void __init check_hlt(void)
         halt();
         halt();
         halt();
-        printk("OK.\n");
+        printk(KERN_CONT "OK.\n");
 }
 
 /*
@@ -122,9 +122,9 @@ static void __init check_popad(void)
          * CPU hard. Too bad.
          */
         if (res != 12345678)
-                printk("Buggy.\n");
+                printk(KERN_CONT "Buggy.\n");
         else
-                printk("OK.\n");
+                printk(KERN_CONT "OK.\n");
 #endif
 }
 
@@ -156,7 +156,7 @@ void __init check_bugs(void)
 {
         identify_boot_cpu();
 #ifndef CONFIG_SMP
-        printk("CPU: ");
+        printk(KERN_INFO "CPU: ");
         print_cpu_info(&boot_cpu_data);
 #endif
         check_config();
diff --git a/arch/x86/kernel/cpu/bugs_64.c b/arch/x86/kernel/cpu/bugs_64.c
index 9a3ed0649d4e..04f0fe5af83e 100644
--- a/arch/x86/kernel/cpu/bugs_64.c
+++ b/arch/x86/kernel/cpu/bugs_64.c
@@ -15,7 +15,7 @@ void __init check_bugs(void)
 {
         identify_boot_cpu();
 #if !defined(CONFIG_SMP)
-        printk("CPU: ");
+        printk(KERN_INFO "CPU: ");
         print_cpu_info(&boot_cpu_data);
 #endif
         alternative_instructions();
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 5ce60a88027b..cc25c2b4a567 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -13,13 +13,13 @@
 #include <linux/io.h>
 
 #include <asm/stackprotector.h>
-#include <asm/perf_counter.h>
+#include <asm/perf_event.h>
 #include <asm/mmu_context.h>
 #include <asm/hypervisor.h>
 #include <asm/processor.h>
 #include <asm/sections.h>
-#include <asm/topology.h>
-#include <asm/cpumask.h>
+#include <linux/topology.h>
+#include <linux/cpumask.h>
 #include <asm/pgtable.h>
 #include <asm/atomic.h>
 #include <asm/proto.h>
@@ -28,13 +28,12 @@
 #include <asm/desc.h>
 #include <asm/i387.h>
 #include <asm/mtrr.h>
-#include <asm/numa.h>
+#include <linux/numa.h>
 #include <asm/asm.h>
 #include <asm/cpu.h>
 #include <asm/mce.h>
 #include <asm/msr.h>
 #include <asm/pat.h>
-#include <asm/smp.h>
 
 #ifdef CONFIG_X86_LOCAL_APIC
 #include <asm/uv/uv.h>
@@ -94,45 +93,45 @@ DEFINE_PER_CPU_PAGE_ALIGNED(struct gdt_page, gdt_page) = { .gdt = {
          * TLS descriptors are currently at a different place compared to i386.
          * Hopefully nobody expects them at a fixed place (Wine?)
          */
-        [GDT_ENTRY_KERNEL32_CS]         = { { { 0x0000ffff, 0x00cf9b00 } } },
-        [GDT_ENTRY_KERNEL_CS]           = { { { 0x0000ffff, 0x00af9b00 } } },
-        [GDT_ENTRY_KERNEL_DS]           = { { { 0x0000ffff, 0x00cf9300 } } },
-        [GDT_ENTRY_DEFAULT_USER32_CS]   = { { { 0x0000ffff, 0x00cffb00 } } },
-        [GDT_ENTRY_DEFAULT_USER_DS]     = { { { 0x0000ffff, 0x00cff300 } } },
-        [GDT_ENTRY_DEFAULT_USER_CS]     = { { { 0x0000ffff, 0x00affb00 } } },
+        [GDT_ENTRY_KERNEL32_CS]         = GDT_ENTRY_INIT(0xc09b, 0, 0xfffff),
+        [GDT_ENTRY_KERNEL_CS]           = GDT_ENTRY_INIT(0xa09b, 0, 0xfffff),
+        [GDT_ENTRY_KERNEL_DS]           = GDT_ENTRY_INIT(0xc093, 0, 0xfffff),
+        [GDT_ENTRY_DEFAULT_USER32_CS]   = GDT_ENTRY_INIT(0xc0fb, 0, 0xfffff),
+        [GDT_ENTRY_DEFAULT_USER_DS]     = GDT_ENTRY_INIT(0xc0f3, 0, 0xfffff),
+        [GDT_ENTRY_DEFAULT_USER_CS]     = GDT_ENTRY_INIT(0xa0fb, 0, 0xfffff),
 #else
-        [GDT_ENTRY_KERNEL_CS]           = { { { 0x0000ffff, 0x00cf9a00 } } },
-        [GDT_ENTRY_KERNEL_DS]           = { { { 0x0000ffff, 0x00cf9200 } } },
-        [GDT_ENTRY_DEFAULT_USER_CS]     = { { { 0x0000ffff, 0x00cffa00 } } },
-        [GDT_ENTRY_DEFAULT_USER_DS]     = { { { 0x0000ffff, 0x00cff200 } } },
+        [GDT_ENTRY_KERNEL_CS]           = GDT_ENTRY_INIT(0xc09a, 0, 0xfffff),
+        [GDT_ENTRY_KERNEL_DS]           = GDT_ENTRY_INIT(0xc092, 0, 0xfffff),
+        [GDT_ENTRY_DEFAULT_USER_CS]     = GDT_ENTRY_INIT(0xc0fa, 0, 0xfffff),
+        [GDT_ENTRY_DEFAULT_USER_DS]     = GDT_ENTRY_INIT(0xc0f2, 0, 0xfffff),
         /*
          * Segments used for calling PnP BIOS have byte granularity.
          * They code segments and data segments have fixed 64k limits,
          * the transfer segment sizes are set at run time.
          */
         /* 32-bit code */
-        [GDT_ENTRY_PNPBIOS_CS32]        = { { { 0x0000ffff, 0x00409a00 } } },
+        [GDT_ENTRY_PNPBIOS_CS32]        = GDT_ENTRY_INIT(0x409a, 0, 0xffff),
         /* 16-bit code */
-        [GDT_ENTRY_PNPBIOS_CS16]        = { { { 0x0000ffff, 0x00009a00 } } },
+        [GDT_ENTRY_PNPBIOS_CS16]        = GDT_ENTRY_INIT(0x009a, 0, 0xffff),
         /* 16-bit data */
-        [GDT_ENTRY_PNPBIOS_DS]          = { { { 0x0000ffff, 0x00009200 } } },
+        [GDT_ENTRY_PNPBIOS_DS]          = GDT_ENTRY_INIT(0x0092, 0, 0xffff),
         /* 16-bit data */
-        [GDT_ENTRY_PNPBIOS_TS1]         = { { { 0x00000000, 0x00009200 } } },
+        [GDT_ENTRY_PNPBIOS_TS1]         = GDT_ENTRY_INIT(0x0092, 0, 0),
         /* 16-bit data */
-        [GDT_ENTRY_PNPBIOS_TS2]         = { { { 0x00000000, 0x00009200 } } },
+        [GDT_ENTRY_PNPBIOS_TS2]         = GDT_ENTRY_INIT(0x0092, 0, 0),
         /*
          * The APM segments have byte granularity and their bases
          * are set at run time.  All have 64k limits.
          */
         /* 32-bit code */
-        [GDT_ENTRY_APMBIOS_BASE]        = { { { 0x0000ffff, 0x00409a00 } } },
+        [GDT_ENTRY_APMBIOS_BASE]        = GDT_ENTRY_INIT(0x409a, 0, 0xffff),
         /* 16-bit code */
-        [GDT_ENTRY_APMBIOS_BASE+1]      = { { { 0x0000ffff, 0x00009a00 } } },
+        [GDT_ENTRY_APMBIOS_BASE+1]      = GDT_ENTRY_INIT(0x009a, 0, 0xffff),
         /* data */
-        [GDT_ENTRY_APMBIOS_BASE+2]      = { { { 0x0000ffff, 0x00409200 } } },
+        [GDT_ENTRY_APMBIOS_BASE+2]      = GDT_ENTRY_INIT(0x4092, 0, 0xffff),
 
-        [GDT_ENTRY_ESPFIX_SS]           = { { { 0x0000ffff, 0x00cf9200 } } },
-        [GDT_ENTRY_PERCPU]              = { { { 0x0000ffff, 0x00cf9200 } } },
+        [GDT_ENTRY_ESPFIX_SS]           = GDT_ENTRY_INIT(0xc092, 0, 0xfffff),
+        [GDT_ENTRY_PERCPU]              = GDT_ENTRY_INIT(0xc092, 0, 0xfffff),
         GDT_STACK_CANARY_INIT
 #endif
 } };
@@ -870,7 +869,7 @@ void __init identify_boot_cpu(void)
 #else
         vgetcpu_set_mode();
 #endif
-        init_hw_perf_counters();
+        init_hw_perf_events();
 }
 
 void __cpuinit identify_secondary_cpu(struct cpuinfo_x86 *c)
@@ -982,18 +981,26 @@ static __init int setup_disablecpuid(char *arg)
 __setup("clearcpuid=", setup_disablecpuid);
 
 #ifdef CONFIG_X86_64
-struct desc_ptr idt_descr = { 256 * 16 - 1, (unsigned long) idt_table };
+struct desc_ptr idt_descr = { NR_VECTORS * 16 - 1, (unsigned long) idt_table };
 
 DEFINE_PER_CPU_FIRST(union irq_stack_union,
                      irq_stack_union) __aligned(PAGE_SIZE);
 
-DEFINE_PER_CPU(char *, irq_stack_ptr) =
-        init_per_cpu_var(irq_stack_union.irq_stack) + IRQ_STACK_SIZE - 64;
+/*
+ * The following four percpu variables are hot.  Align current_task to
+ * cacheline size such that all four fall in the same cacheline.
+ */
+DEFINE_PER_CPU(struct task_struct *, current_task) ____cacheline_aligned =
+        &init_task;
+EXPORT_PER_CPU_SYMBOL(current_task);
 
 DEFINE_PER_CPU(unsigned long, kernel_stack) =
         (unsigned long)&init_thread_union - KERNEL_STACK_OFFSET + THREAD_SIZE;
 EXPORT_PER_CPU_SYMBOL(kernel_stack);
 
+DEFINE_PER_CPU(char *, irq_stack_ptr) =
+        init_per_cpu_var(irq_stack_union.irq_stack) + IRQ_STACK_SIZE - 64;
+
 DEFINE_PER_CPU(unsigned int, irq_count) = -1;
 
 /*
@@ -1008,8 +1015,7 @@ static const unsigned int exception_stack_sizes[N_EXCEPTION_STACKS] = {
 };
 
 static DEFINE_PER_CPU_PAGE_ALIGNED(char, exception_stacks
-        [(N_EXCEPTION_STACKS - 1) * EXCEPTION_STKSZ + DEBUG_STKSZ])
-        __aligned(PAGE_SIZE);
+        [(N_EXCEPTION_STACKS - 1) * EXCEPTION_STKSZ + DEBUG_STKSZ]);
 
 /* May not be marked __init: used by software suspend */
 void syscall_init(void)
@@ -1042,8 +1048,11 @@ DEFINE_PER_CPU(struct orig_ist, orig_ist);
 
 #else   /* CONFIG_X86_64 */
 
+DEFINE_PER_CPU(struct task_struct *, current_task) = &init_task;
+EXPORT_PER_CPU_SYMBOL(current_task);
+
 #ifdef CONFIG_CC_STACKPROTECTOR
-DEFINE_PER_CPU(unsigned long, stack_canary);
+DEFINE_PER_CPU_ALIGNED(struct stack_canary, stack_canary);
 #endif
 
 /* Make sure %fs and %gs are initialized properly in idle threads */
diff --git a/arch/x86/kernel/cpu/cpu_debug.c b/arch/x86/kernel/cpu/cpu_debug.c
index 6b2a52dd0403..dca325c03999 100644
--- a/arch/x86/kernel/cpu/cpu_debug.c
+++ b/arch/x86/kernel/cpu/cpu_debug.c
@@ -30,8 +30,8 @@
 #include <asm/apic.h>
 #include <asm/desc.h>
 
-static DEFINE_PER_CPU(struct cpu_cpuX_base, cpu_arr[CPU_REG_ALL_BIT]);
-static DEFINE_PER_CPU(struct cpu_private *, priv_arr[MAX_CPU_FILES]);
+static DEFINE_PER_CPU(struct cpu_cpuX_base [CPU_REG_ALL_BIT], cpu_arr);
+static DEFINE_PER_CPU(struct cpu_private * [MAX_CPU_FILES], priv_arr);
 static DEFINE_PER_CPU(int, cpu_priv_count);
 
 static DEFINE_MUTEX(cpu_debug_lock);
diff --git a/arch/x86/kernel/cpu/cpufreq/acpi-cpufreq.c b/arch/x86/kernel/cpu/cpufreq/acpi-cpufreq.c
index ae9b503220ca..7d5c3b0ea8da 100644
--- a/arch/x86/kernel/cpu/cpufreq/acpi-cpufreq.c
+++ b/arch/x86/kernel/cpu/cpufreq/acpi-cpufreq.c
@@ -33,7 +33,7 @@
 #include <linux/cpufreq.h>
 #include <linux/compiler.h>
 #include <linux/dmi.h>
-#include <trace/power.h>
+#include <trace/events/power.h>
 
 #include <linux/acpi.h>
 #include <linux/io.h>
@@ -60,7 +60,6 @@ enum {
 };
 
 #define INTEL_MSR_RANGE         (0xffff)
-#define CPUID_6_ECX_APERFMPERF_CAPABILITY       (0x1)
 
 struct acpi_cpufreq_data {
         struct acpi_processor_performance *acpi_data;
@@ -71,13 +70,7 @@ struct acpi_cpufreq_data {
 
 static DEFINE_PER_CPU(struct acpi_cpufreq_data *, drv_data);
 
-struct acpi_msr_data {
-        u64 saved_aperf, saved_mperf;
-};
-
-static DEFINE_PER_CPU(struct acpi_msr_data, msr_data);
-
-DEFINE_TRACE(power_mark);
+static DEFINE_PER_CPU(struct aperfmperf, old_perf);
 
 /* acpi_perf_data is a pointer to percpu data. */
 static struct acpi_processor_performance *acpi_perf_data;
@@ -244,23 +237,12 @@ static u32 get_cur_val(const struct cpumask *mask)
         return cmd.val;
 }
 
-struct perf_pair {
-        union {
-                struct {
-                        u32 lo;
-                        u32 hi;
-                } split;
-                u64 whole;
-        } aperf, mperf;
-};
-
 /* Called via smp_call_function_single(), on the target CPU */
 static void read_measured_perf_ctrs(void *_cur)
 {
-        struct perf_pair *cur = _cur;
+        struct aperfmperf *am = _cur;
 
-        rdmsr(MSR_IA32_APERF, cur->aperf.split.lo, cur->aperf.split.hi);
-        rdmsr(MSR_IA32_MPERF, cur->mperf.split.lo, cur->mperf.split.hi);
+        get_aperfmperf(am);
 }
 
 /*
@@ -279,63 +261,17 @@ static void read_measured_perf_ctrs(void *_cur)
 static unsigned int get_measured_perf(struct cpufreq_policy *policy,
                                       unsigned int cpu)
 {
-        struct perf_pair readin, cur;
-        unsigned int perf_percent;
+        struct aperfmperf perf;
+        unsigned long ratio;
         unsigned int retval;
 
-        if (smp_call_function_single(cpu, read_measured_perf_ctrs, &readin, 1))
+        if (smp_call_function_single(cpu, read_measured_perf_ctrs, &perf, 1))
                 return 0;
 
-        cur.aperf.whole = readin.aperf.whole -
-                                per_cpu(msr_data, cpu).saved_aperf;
-        cur.mperf.whole = readin.mperf.whole -
-                                per_cpu(msr_data, cpu).saved_mperf;
-        per_cpu(msr_data, cpu).saved_aperf = readin.aperf.whole;
-        per_cpu(msr_data, cpu).saved_mperf = readin.mperf.whole;
-
-#ifdef __i386__
-        /*
-         * We dont want to do 64 bit divide with 32 bit kernel
-         * Get an approximate value. Return failure in case we cannot get
-         * an approximate value.
-         */
-        if (unlikely(cur.aperf.split.hi || cur.mperf.split.hi)) {
-                int shift_count;
-                u32 h;
-
-                h = max_t(u32, cur.aperf.split.hi, cur.mperf.split.hi);
-                shift_count = fls(h);
-
-                cur.aperf.whole >>= shift_count;
-                cur.mperf.whole >>= shift_count;
-        }
-
-        if (((unsigned long)(-1) / 100) < cur.aperf.split.lo) {
-                int shift_count = 7;
-                cur.aperf.split.lo >>= shift_count;
-                cur.mperf.split.lo >>= shift_count;
-        }
-
-        if (cur.aperf.split.lo && cur.mperf.split.lo)
-                perf_percent = (cur.aperf.split.lo * 100) / cur.mperf.split.lo;
-        else
-                perf_percent = 0;
-
-#else
-        if (unlikely(((unsigned long)(-1) / 100) < cur.aperf.whole)) {
-                int shift_count = 7;
-                cur.aperf.whole >>= shift_count;
-                cur.mperf.whole >>= shift_count;
-        }
-
-        if (cur.aperf.whole && cur.mperf.whole)
-                perf_percent = (cur.aperf.whole * 100) / cur.mperf.whole;
-        else
-                perf_percent = 0;
-
-#endif
+        ratio = calc_aperfmperf_ratio(&per_cpu(old_perf, cpu), &perf);
+        per_cpu(old_perf, cpu) = perf;
 
-        retval = (policy->cpuinfo.max_freq * perf_percent) / 100;
+        retval = (policy->cpuinfo.max_freq * ratio) >> APERFMPERF_SHIFT;
 
         return retval;
 }
@@ -394,7 +330,6 @@ static int acpi_cpufreq_target(struct cpufreq_policy *policy,
         unsigned int next_perf_state = 0; /* Index into perf table */
         unsigned int i;
         int result = 0;
-        struct power_trace it;
 
         dprintk("acpi_cpufreq_target %d (%d)\n", target_freq, policy->cpu);
 
@@ -426,7 +361,7 @@ static int acpi_cpufreq_target(struct cpufreq_policy *policy,
                 }
         }
 
-        trace_power_mark(&it, POWER_PSTATE, next_perf_state);
+        trace_power_frequency(POWER_PSTATE, data->freq_table[next_state].frequency);
 
         switch (data->cpu_feature) {
         case SYSTEM_INTEL_MSR_CAPABLE:
@@ -588,6 +523,21 @@ static const struct dmi_system_id sw_any_bug_dmi_table[] = {
         },
         { }
 };
+
+static int acpi_cpufreq_blacklist(struct cpuinfo_x86 *c)
+{
+        /* http://www.intel.com/Assets/PDF/specupdate/314554.pdf
+         * AL30: A Machine Check Exception (MCE) Occurring during an
+         * Enhanced Intel SpeedStep Technology Ratio Change May Cause
+         * Both Processor Cores to Lock Up when HT is enabled*/
+        if (c->x86_vendor == X86_VENDOR_INTEL) {
+                if ((c->x86 == 15) &&
+                    (c->x86_model == 6) &&
+                    (c->x86_mask == 8) && smt_capable())
+                        return -ENODEV;
+                }
+        return 0;
+}
 #endif
 
 static int acpi_cpufreq_cpu_init(struct cpufreq_policy *policy)
@@ -602,6 +552,12 @@ static int acpi_cpufreq_cpu_init(struct cpufreq_policy *policy)
 
         dprintk("acpi_cpufreq_cpu_init\n");
 
+#ifdef CONFIG_SMP
+        result = acpi_cpufreq_blacklist(c);
+        if (result)
+                return result;
+#endif
+
         data = kzalloc(sizeof(struct acpi_cpufreq_data), GFP_KERNEL);
         if (!data)
                 return -ENOMEM;
@@ -731,12 +687,8 @@ static int acpi_cpufreq_cpu_init(struct cpufreq_policy *policy)
         acpi_processor_notify_smm(THIS_MODULE);
 
         /* Check for APERF/MPERF support in hardware */
-        if (c->x86_vendor == X86_VENDOR_INTEL && c->cpuid_level >= 6) {
-                unsigned int ecx;
-                ecx = cpuid_ecx(6);
-                if (ecx & CPUID_6_ECX_APERFMPERF_CAPABILITY)
-                        acpi_cpufreq_driver.getavg = get_measured_perf;
-        }
+        if (cpu_has(c, X86_FEATURE_APERFMPERF))
+                acpi_cpufreq_driver.getavg = get_measured_perf;
 
         dprintk("CPU%u - ACPI performance management activated.\n", cpu);
         for (i = 0; i < perf->state_count; i++)
diff --git a/arch/x86/kernel/cpu/cpufreq/powernow-k8.c b/arch/x86/kernel/cpu/cpufreq/powernow-k8.c
index 2a50ef891000..6394aa5c7985 100644
--- a/arch/x86/kernel/cpu/cpufreq/powernow-k8.c
+++ b/arch/x86/kernel/cpu/cpufreq/powernow-k8.c
@@ -605,9 +605,10 @@ static int check_pst_table(struct powernow_k8_data *data, struct pst_s *pst,
         return 0;
 }
 
-static void invalidate_entry(struct powernow_k8_data *data, unsigned int entry)
+static void invalidate_entry(struct cpufreq_frequency_table *powernow_table,
+                unsigned int entry)
 {
-        data->powernow_table[entry].frequency = CPUFREQ_ENTRY_INVALID;
+        powernow_table[entry].frequency = CPUFREQ_ENTRY_INVALID;
 }
 
 static void print_basics(struct powernow_k8_data *data)
@@ -854,6 +855,10 @@ static int powernow_k8_cpu_init_acpi(struct powernow_k8_data *data)
                 goto err_out;
         }
 
+        /* fill in data */
+        data->numps = data->acpi_data.state_count;
+        powernow_k8_acpi_pst_values(data, 0);
+
         if (cpu_family == CPU_HW_PSTATE)
                 ret_val = fill_powernow_table_pstate(data, powernow_table);
         else
@@ -866,11 +871,8 @@ static int powernow_k8_cpu_init_acpi(struct powernow_k8_data *data)
         powernow_table[data->acpi_data.state_count].index = 0;
         data->powernow_table = powernow_table;
 
-        /* fill in data */
-        data->numps = data->acpi_data.state_count;
         if (cpumask_first(cpu_core_mask(data->cpu)) == data->cpu)
                 print_basics(data);
-        powernow_k8_acpi_pst_values(data, 0);
 
         /* notify BIOS that we exist */
         acpi_processor_notify_smm(THIS_MODULE);
@@ -914,13 +916,13 @@ static int fill_powernow_table_pstate(struct powernow_k8_data *data,
                                         "bad value %d.\n", i, index);
                         printk(KERN_ERR PFX "Please report to BIOS "
                                         "manufacturer\n");
-                        invalidate_entry(data, i);
+                        invalidate_entry(powernow_table, i);
                         continue;
                 }
                 rdmsr(MSR_PSTATE_DEF_BASE + index, lo, hi);
                 if (!(hi & HW_PSTATE_VALID_MASK)) {
                         dprintk("invalid pstate %d, ignoring\n", index);
-                        invalidate_entry(data, i);
+                        invalidate_entry(powernow_table, i);
                         continue;
                 }
 
@@ -941,7 +943,6 @@ static int fill_powernow_table_fidvid(struct powernow_k8_data *data,
                 struct cpufreq_frequency_table *powernow_table)
 {
         int i;
-        int cntlofreq = 0;
 
         for (i = 0; i < data->acpi_data.state_count; i++) {
                 u32 fid;
@@ -970,7 +971,7 @@ static int fill_powernow_table_fidvid(struct powernow_k8_data *data,
                 /* verify frequency is OK */
                 if ((freq > (MAX_FREQ * 1000)) || (freq < (MIN_FREQ * 1000))) {
                         dprintk("invalid freq %u kHz, ignoring\n", freq);
-                        invalidate_entry(data, i);
+                        invalidate_entry(powernow_table, i);
                         continue;
                 }
 
@@ -978,38 +979,17 @@ static int fill_powernow_table_fidvid(struct powernow_k8_data *data,
                  * BIOSs are using "off" to indicate invalid */
                 if (vid == VID_OFF) {
                         dprintk("invalid vid %u, ignoring\n", vid);
-                        invalidate_entry(data, i);
+                        invalidate_entry(powernow_table, i);
                         continue;
                 }
 
-                /* verify only 1 entry from the lo frequency table */
-                if (fid < HI_FID_TABLE_BOTTOM) {
-                        if (cntlofreq) {
-                                /* if both entries are the same,
-                                 * ignore this one ... */
-                                if ((freq != powernow_table[cntlofreq].frequency) ||
-                                    (index != powernow_table[cntlofreq].index)) {
-                                        printk(KERN_ERR PFX
-                                                "Too many lo freq table "
-                                                "entries\n");
-                                        return 1;
-                                }
-
-                                dprintk("double low frequency table entry, "
-                                                "ignoring it.\n");
-                                invalidate_entry(data, i);
-                                continue;
-                        } else
-                                cntlofreq = i;
-                }
-
                 if (freq != (data->acpi_data.states[i].core_frequency * 1000)) {
                         printk(KERN_INFO PFX "invalid freq entries "
                                 "%u kHz vs. %u kHz\n", freq,
                                 (unsigned int)
                                 (data->acpi_data.states[i].core_frequency
                                  * 1000));
-                        invalidate_entry(data, i);
+                        invalidate_entry(powernow_table, i);
                         continue;
                 }
         }
diff --git a/arch/x86/kernel/cpu/cyrix.c b/arch/x86/kernel/cpu/cyrix.c
index 593171e967ef..19807b89f058 100644
--- a/arch/x86/kernel/cpu/cyrix.c
+++ b/arch/x86/kernel/cpu/cyrix.c
@@ -3,10 +3,10 @@
 #include <linux/delay.h>
 #include <linux/pci.h>
 #include <asm/dma.h>
-#include <asm/io.h>
+#include <linux/io.h>
 #include <asm/processor-cyrix.h>
 #include <asm/processor-flags.h>
-#include <asm/timer.h>
+#include <linux/timer.h>
 #include <asm/pci-direct.h>
 #include <asm/tsc.h>
 
@@ -282,7 +282,8 @@ static void __cpuinit init_cyrix(struct cpuinfo_x86 *c)
                  *  The 5510/5520 companion chips have a funky PIT.
                  */
                 if (vendor == PCI_VENDOR_ID_CYRIX &&
-         (device == PCI_DEVICE_ID_CYRIX_5510 || device == PCI_DEVICE_ID_CYRIX_5520))
+                        (device == PCI_DEVICE_ID_CYRIX_5510 ||
+                                        device == PCI_DEVICE_ID_CYRIX_5520))
                         mark_tsc_unstable("cyrix 5510/5520 detected");
         }
 #endif
@@ -299,7 +300,8 @@ static void __cpuinit init_cyrix(struct cpuinfo_x86 *c)
                          *  ?  : 0x7x
                          * GX1 : 0x8x          GX1  datasheet 56
                          */
-                        if ((0x30 <= dir1 && dir1 <= 0x6f) || (0x80 <= dir1 && dir1 <= 0x8f))
+                        if ((0x30 <= dir1 && dir1 <= 0x6f) ||
+                                        (0x80 <= dir1 && dir1 <= 0x8f))
                                 geode_configure();
                         return;
                 } else { /* MediaGX */
@@ -427,9 +429,12 @@ static void __cpuinit cyrix_identify(struct cpuinfo_x86 *c)
                         printk(KERN_INFO "Enabling CPUID on Cyrix processor.\n");
                         local_irq_save(flags);
                         ccr3 = getCx86(CX86_CCR3);
-                        setCx86(CX86_CCR3, (ccr3 & 0x0f) | 0x10);       /* enable MAPEN  */
-                        setCx86_old(CX86_CCR4, getCx86_old(CX86_CCR4) | 0x80);  /* enable cpuid  */
-                        setCx86(CX86_CCR3, ccr3);                       /* disable MAPEN */
+                        /* enable MAPEN  */
+                        setCx86(CX86_CCR3, (ccr3 & 0x0f) | 0x10);
+                        /* enable cpuid  */
+                        setCx86_old(CX86_CCR4, getCx86_old(CX86_CCR4) | 0x80);
+                        /* disable MAPEN */
+                        setCx86(CX86_CCR3, ccr3);
                         local_irq_restore(flags);
                 }
         }
diff --git a/arch/x86/kernel/cpu/hypervisor.c b/arch/x86/kernel/cpu/hypervisor.c
index fb5b86af0b01..08be922de33a 100644
--- a/arch/x86/kernel/cpu/hypervisor.c
+++ b/arch/x86/kernel/cpu/hypervisor.c
@@ -28,18 +28,10 @@
 static inline void __cpuinit
 detect_hypervisor_vendor(struct cpuinfo_x86 *c)
 {
-        if (vmware_platform()) {
+        if (vmware_platform())
                 c->x86_hyper_vendor = X86_HYPER_VENDOR_VMWARE;
-        } else {
+        else
                 c->x86_hyper_vendor = X86_HYPER_VENDOR_NONE;
-        }
-}
-
-unsigned long get_hypervisor_tsc_freq(void)
-{
-        if (boot_cpu_data.x86_hyper_vendor == X86_HYPER_VENDOR_VMWARE)
-                return vmware_get_tsc_khz();
-        return 0;
 }
 
 static inline void __cpuinit
@@ -56,3 +48,10 @@ void __cpuinit init_hypervisor(struct cpuinfo_x86 *c)
         detect_hypervisor_vendor(c);
         hypervisor_set_feature_bits(c);
 }
+
+void __init init_hypervisor_platform(void)
+{
+        init_hypervisor(&boot_cpu_data);
+        if (boot_cpu_data.x86_hyper_vendor == X86_HYPER_VENDOR_VMWARE)
+                vmware_platform_setup();
+}
diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c
index 3260ab044996..40e1835b35e8 100644
--- a/arch/x86/kernel/cpu/intel.c
+++ b/arch/x86/kernel/cpu/intel.c
@@ -7,17 +7,17 @@
 #include <linux/sched.h>
 #include <linux/thread_info.h>
 #include <linux/module.h>
+#include <linux/uaccess.h>
 
 #include <asm/processor.h>
 #include <asm/pgtable.h>
 #include <asm/msr.h>
-#include <asm/uaccess.h>
 #include <asm/ds.h>
 #include <asm/bugs.h>
 #include <asm/cpu.h>
 
 #ifdef CONFIG_X86_64
-#include <asm/topology.h>
+#include <linux/topology.h>
 #include <asm/numa_64.h>
 #endif
 
@@ -174,7 +174,8 @@ static void __cpuinit intel_workarounds(struct cpuinfo_x86 *c)
 #ifdef CONFIG_X86_F00F_BUG
         /*
          * All current models of Pentium and Pentium with MMX technology CPUs
-         * have the F0 0F bug, which lets nonprivileged users lock up the system.
+         * have the F0 0F bug, which lets nonprivileged users lock up the
+         * system.
          * Note that the workaround only should be initialized once...
          */
         c->f00f_bug = 0;
@@ -207,7 +208,7 @@ static void __cpuinit intel_workarounds(struct cpuinfo_x86 *c)
                         printk (KERN_INFO "CPU: C0 stepping P4 Xeon detected.\n");
                         printk (KERN_INFO "CPU: Disabling hardware prefetching (Errata 037)\n");
                         lo |= MSR_IA32_MISC_ENABLE_PREFETCH_DISABLE;
-                        wrmsr (MSR_IA32_MISC_ENABLE, lo, hi);
+                        wrmsr(MSR_IA32_MISC_ENABLE, lo, hi);
                 }
         }
 
@@ -283,7 +284,7 @@ static int __cpuinit intel_num_cpu_cores(struct cpuinfo_x86 *c)
         /* Intel has a non-standard dependency on %ecx for this CPUID level. */
         cpuid_count(4, 0, &eax, &ebx, &ecx, &edx);
         if (eax & 0x1f)
-                return ((eax >> 26) + 1);
+                return (eax >> 26) + 1;
         else
                 return 1;
 }
@@ -349,6 +350,12 @@ static void __cpuinit init_intel(struct cpuinfo_x86 *c)
                         set_cpu_cap(c, X86_FEATURE_ARCH_PERFMON);
         }
 
+        if (c->cpuid_level > 6) {
+                unsigned ecx = cpuid_ecx(6);
+                if (ecx & 0x01)
+                        set_cpu_cap(c, X86_FEATURE_APERFMPERF);
+        }
+
         if (cpu_has_xmm2)
                 set_cpu_cap(c, X86_FEATURE_LFENCE_RDTSC);
         if (cpu_has_ds) {
diff --git a/arch/x86/kernel/cpu/intel_cacheinfo.c b/arch/x86/kernel/cpu/intel_cacheinfo.c
index 789efe217e1a..804c40e2bc3e 100644
--- a/arch/x86/kernel/cpu/intel_cacheinfo.c
+++ b/arch/x86/kernel/cpu/intel_cacheinfo.c
@@ -3,7 +3,7 @@
  *
  *      Changes:
  *      Venkatesh Pallipadi     : Adding cache identification through cpuid(4)
- *              Ashok Raj <ashok.raj@intel.com>: Work with CPU hotplug infrastructure.
+ *      Ashok Raj <ashok.raj@intel.com>: Work with CPU hotplug infrastructure.
  *      Andi Kleen / Andreas Herrmann   : CPUID4 emulation on AMD.
  */
 
@@ -16,7 +16,7 @@
 #include <linux/pci.h>
 
 #include <asm/processor.h>
-#include <asm/smp.h>
+#include <linux/smp.h>
 #include <asm/k8.h>
 
 #define LVL_1_INST      1
@@ -25,14 +25,15 @@
 #define LVL_3           4
 #define LVL_TRACE       5
 
-struct _cache_table
-{
+struct _cache_table {
         unsigned char descriptor;
         char cache_type;
         short size;
 };
 
-/* all the cache descriptor types we care about (no TLB or trace cache entries) */
+/* All the cache descriptor types we care about (no TLB or
+   trace cache entries) */
+
 static const struct _cache_table __cpuinitconst cache_table[] =
 {
         { 0x06, LVL_1_INST, 8 },        /* 4-way set assoc, 32 byte line size */
@@ -105,8 +106,7 @@ static const struct _cache_table __cpuinitconst cache_table[] =
 };
 
 
-enum _cache_type
-{
+enum _cache_type {
         CACHE_TYPE_NULL = 0,
         CACHE_TYPE_DATA = 1,
         CACHE_TYPE_INST = 2,
@@ -170,31 +170,31 @@ unsigned short			num_cache_leaves;
    Maybe later */
 union l1_cache {
         struct {
-                unsigned line_size : 8;
-                unsigned lines_per_tag : 8;
-                unsigned assoc : 8;
-                unsigned size_in_kb : 8;
+                unsigned line_size:8;
+                unsigned lines_per_tag:8;
+                unsigned assoc:8;
+                unsigned size_in_kb:8;
         };
         unsigned val;
 };
 
 union l2_cache {
         struct {
-                unsigned line_size : 8;
-                unsigned lines_per_tag : 4;
-                unsigned assoc : 4;
-                unsigned size_in_kb : 16;
+                unsigned line_size:8;
+                unsigned lines_per_tag:4;
+                unsigned assoc:4;
+                unsigned size_in_kb:16;
         };
         unsigned val;
 };
 
 union l3_cache {
         struct {
-                unsigned line_size : 8;
-                unsigned lines_per_tag : 4;
-                unsigned assoc : 4;
-                unsigned res : 2;
-                unsigned size_encoded : 14;
+                unsigned line_size:8;
+                unsigned lines_per_tag:4;
+                unsigned assoc:4;
+                unsigned res:2;
+                unsigned size_encoded:14;
         };
         unsigned val;
 };
@@ -241,7 +241,7 @@ amd_cpuid4(int leaf, union _cpuid4_leaf_eax *eax,
         case 0:
                 if (!l1->val)
                         return;
-                assoc = l1->assoc;
+                assoc = assocs[l1->assoc];
                 line_size = l1->line_size;
                 lines_per_tag = l1->lines_per_tag;
                 size_in_kb = l1->size_in_kb;
@@ -249,7 +249,7 @@ amd_cpuid4(int leaf, union _cpuid4_leaf_eax *eax,
         case 2:
                 if (!l2.val)
                         return;
-                assoc = l2.assoc;
+                assoc = assocs[l2.assoc];
                 line_size = l2.line_size;
                 lines_per_tag = l2.lines_per_tag;
                 /* cpu_data has errata corrections for K7 applied */
@@ -258,10 +258,14 @@ amd_cpuid4(int leaf, union _cpuid4_leaf_eax *eax,
         case 3:
                 if (!l3.val)
                         return;
-                assoc = l3.assoc;
+                assoc = assocs[l3.assoc];
                 line_size = l3.line_size;
                 lines_per_tag = l3.lines_per_tag;
                 size_in_kb = l3.size_encoded * 512;
+                if (boot_cpu_has(X86_FEATURE_AMD_DCM)) {
+                        size_in_kb = size_in_kb >> 1;
+                        assoc = assoc >> 1;
+                }
                 break;
         default:
                 return;
@@ -270,18 +274,14 @@ amd_cpuid4(int leaf, union _cpuid4_leaf_eax *eax,
         eax->split.is_self_initializing = 1;
         eax->split.type = types[leaf];
         eax->split.level = levels[leaf];
-        if (leaf == 3)
-                eax->split.num_threads_sharing =
-                        current_cpu_data.x86_max_cores - 1;
-        else
-                eax->split.num_threads_sharing = 0;
+        eax->split.num_threads_sharing = 0;
         eax->split.num_cores_on_die = current_cpu_data.x86_max_cores - 1;
 
 
-        if (assoc == 0xf)
+        if (assoc == 0xffff)
                 eax->split.is_fully_associative = 1;
         ebx->split.coherency_line_size = line_size - 1;
-        ebx->split.ways_of_associativity = assocs[assoc] - 1;
+        ebx->split.ways_of_associativity = assoc - 1;
         ebx->split.physical_line_partition = lines_per_tag - 1;
         ecx->split.number_of_sets = (size_in_kb * 1024) / line_size /
                 (ebx->split.ways_of_associativity + 1) - 1;
@@ -350,7 +350,8 @@ static int __cpuinit find_num_cache_leaves(void)
 
 unsigned int __cpuinit init_intel_cacheinfo(struct cpuinfo_x86 *c)
 {
-        unsigned int trace = 0, l1i = 0, l1d = 0, l2 = 0, l3 = 0; /* Cache sizes */
+        /* Cache sizes */
+        unsigned int trace = 0, l1i = 0, l1d = 0, l2 = 0, l3 = 0;
         unsigned int new_l1d = 0, new_l1i = 0; /* Cache sizes from cpuid(4) */
         unsigned int new_l2 = 0, new_l3 = 0, i; /* Cache sizes from cpuid(4) */
         unsigned int l2_id = 0, l3_id = 0, num_threads_sharing, index_msb;
@@ -377,8 +378,8 @@ unsigned int __cpuinit init_intel_cacheinfo(struct cpuinfo_x86 *c)
 
                         retval = cpuid4_cache_lookup_regs(i, &this_leaf);
                         if (retval >= 0) {
-                                switch(this_leaf.eax.split.level) {
-                                    case 1:
+                                switch (this_leaf.eax.split.level) {
+                                case 1:
                                         if (this_leaf.eax.split.type ==
                                                         CACHE_TYPE_DATA)
                                                 new_l1d = this_leaf.size/1024;
@@ -386,19 +387,20 @@ unsigned int __cpuinit init_intel_cacheinfo(struct cpuinfo_x86 *c)
                                                         CACHE_TYPE_INST)
                                                 new_l1i = this_leaf.size/1024;
                                         break;
-                                    case 2:
+                                case 2:
                                         new_l2 = this_leaf.size/1024;
                                         num_threads_sharing = 1 + this_leaf.eax.split.num_threads_sharing;
                                         index_msb = get_count_order(num_threads_sharing);
                                         l2_id = c->apicid >> index_msb;
                                         break;
-                                    case 3:
+                                case 3:
                                         new_l3 = this_leaf.size/1024;
                                         num_threads_sharing = 1 + this_leaf.eax.split.num_threads_sharing;
-                                        index_msb = get_count_order(num_threads_sharing);
+                                        index_msb = get_count_order(
+                                                        num_threads_sharing);
                                         l3_id = c->apicid >> index_msb;
                                         break;
-                                    default:
+                                default:
                                         break;
                                 }
                         }
@@ -421,22 +423,21 @@ unsigned int __cpuinit init_intel_cacheinfo(struct cpuinfo_x86 *c)
                 /* Number of times to iterate */
                 n = cpuid_eax(2) & 0xFF;
 
-                for ( i = 0 ; i < n ; i++ ) {
+                for (i = 0 ; i < n ; i++) {
                         cpuid(2, &regs[0], &regs[1], &regs[2], &regs[3]);
 
                         /* If bit 31 is set, this is an unknown format */
-                        for ( j = 0 ; j < 3 ; j++ ) {
-                                if (regs[j] & (1 << 31)) regs[j] = 0;
-                        }
+                        for (j = 0 ; j < 3 ; j++)
+                                if (regs[j] & (1 << 31))
+                                        regs[j] = 0;
 
                         /* Byte 0 is level count, not a descriptor */
-                        for ( j = 1 ; j < 16 ; j++ ) {
+                        for (j = 1 ; j < 16 ; j++) {
                                 unsigned char des = dp[j];
                                 unsigned char k = 0;
 
                                 /* look up this descriptor in the table */
-                                while (cache_table[k].descriptor != 0)
-                                {
+                                while (cache_table[k].descriptor != 0) {
                                         if (cache_table[k].descriptor == des) {
                                                 if (only_trace && cache_table[k].cache_type != LVL_TRACE)
                                                         break;
@@ -488,14 +489,14 @@ unsigned int __cpuinit init_intel_cacheinfo(struct cpuinfo_x86 *c)
         }
 
         if (trace)
-                printk (KERN_INFO "CPU: Trace cache: %dK uops", trace);
-        else if ( l1i )
-                printk (KERN_INFO "CPU: L1 I cache: %dK", l1i);
+                printk(KERN_INFO "CPU: Trace cache: %dK uops", trace);
+        else if (l1i)
+                printk(KERN_INFO "CPU: L1 I cache: %dK", l1i);
 
         if (l1d)
-                printk(", L1 D cache: %dK\n", l1d);
+                printk(KERN_CONT ", L1 D cache: %dK\n", l1d);
         else
-                printk("\n");
+                printk(KERN_CONT "\n");
 
         if (l2)
                 printk(KERN_INFO "CPU: L2 cache: %dK\n", l2);
@@ -522,6 +523,18 @@ static void __cpuinit cache_shared_cpu_map_setup(unsigned int cpu, int index)
         int index_msb, i;
         struct cpuinfo_x86 *c = &cpu_data(cpu);
 
+        if ((index == 3) && (c->x86_vendor == X86_VENDOR_AMD)) {
+                struct cpuinfo_x86 *d;
+                for_each_online_cpu(i) {
+                        if (!per_cpu(cpuid4_info, i))
+                                continue;
+                        d = &cpu_data(i);
+                        this_leaf = CPUID4_INFO_IDX(i, index);
+                        cpumask_copy(to_cpumask(this_leaf->shared_cpu_map),
+                                     d->llc_shared_map);
+                }
+                return;
+        }
         this_leaf = CPUID4_INFO_IDX(cpu, index);
         num_threads_sharing = 1 + this_leaf->eax.split.num_threads_sharing;
 
@@ -558,8 +571,13 @@ static void __cpuinit cache_remove_shared_cpu_map(unsigned int cpu, int index)
         }
 }
 #else
-static void __cpuinit cache_shared_cpu_map_setup(unsigned int cpu, int index) {}
-static void __cpuinit cache_remove_shared_cpu_map(unsigned int cpu, int index) {}
+static void __cpuinit cache_shared_cpu_map_setup(unsigned int cpu, int index)
+{
+}
+
+static void __cpuinit cache_remove_shared_cpu_map(unsigned int cpu, int index)
+{
+}
 #endif
 
 static void __cpuinit free_cache_attributes(unsigned int cpu)
@@ -645,7 +663,7 @@ static DEFINE_PER_CPU(struct _index_kobject *, index_kobject);
 static ssize_t show_##file_name                                         \
                         (struct _cpuid4_info *this_leaf, char *buf)     \
 {                                                                       \
-        return sprintf (buf, "%lu\n", (unsigned long)this_leaf->object + val); \
+        return sprintf(buf, "%lu\n", (unsigned long)this_leaf->object + val); \
 }
 
 show_one_plus(level, eax.split.level, 0);
@@ -656,7 +674,7 @@ show_one_plus(number_of_sets, ecx.split.number_of_sets, 1);
 
 static ssize_t show_size(struct _cpuid4_info *this_leaf, char *buf)
 {
-        return sprintf (buf, "%luK\n", this_leaf->size / 1024);
+        return sprintf(buf, "%luK\n", this_leaf->size / 1024);
 }
 
 static ssize_t show_shared_cpu_map_func(struct _cpuid4_info *this_leaf,
@@ -669,7 +687,7 @@ static ssize_t show_shared_cpu_map_func(struct _cpuid4_info *this_leaf,
                 const struct cpumask *mask;
 
                 mask = to_cpumask(this_leaf->shared_cpu_map);
-                n = type?
+                n = type ?
                         cpulist_scnprintf(buf, len-2, mask) :
                         cpumask_scnprintf(buf, len-2, mask);
                 buf[n++] = '\n';
@@ -800,7 +818,7 @@ static struct _cache_attr cache_disable_0 = __ATTR(cache_disable_0, 0644,
 static struct _cache_attr cache_disable_1 = __ATTR(cache_disable_1, 0644,
                 show_cache_disable_1, store_cache_disable_1);
 
-static struct attribute * default_attrs[] = {
+static struct attribute *default_attrs[] = {
         &type.attr,
         &level.attr,
         &coherency_line_size.attr,
@@ -815,7 +833,7 @@ static struct attribute * default_attrs[] = {
         NULL
 };
 
-static ssize_t show(struct kobject * kobj, struct attribute * attr, char * buf)
+static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
 {
         struct _cache_attr *fattr = to_attr(attr);
         struct _index_kobject *this_leaf = to_object(kobj);
@@ -828,8 +846,8 @@ static ssize_t show(struct kobject * kobj, struct attribute * attr, char * buf)
         return ret;
 }
 
-static ssize_t store(struct kobject * kobj, struct attribute * attr,
-                     const char * buf, size_t count)
+static ssize_t store(struct kobject *kobj, struct attribute *attr,
+                     const char *buf, size_t count)
 {
         struct _cache_attr *fattr = to_attr(attr);
         struct _index_kobject *this_leaf = to_object(kobj);
@@ -883,7 +901,7 @@ static int __cpuinit cpuid4_cache_sysfs_init(unsigned int cpu)
                 goto err_out;
 
         per_cpu(index_kobject, cpu) = kzalloc(
-            sizeof(struct _index_kobject ) * num_cache_leaves, GFP_KERNEL);
+            sizeof(struct _index_kobject) * num_cache_leaves, GFP_KERNEL);
         if (unlikely(per_cpu(index_kobject, cpu) == NULL))
                 goto err_out;
 
@@ -917,7 +935,7 @@ static int __cpuinit cache_add_dev(struct sys_device * sys_dev)
         }
 
         for (i = 0; i < num_cache_leaves; i++) {
-                this_object = INDEX_KOBJECT_PTR(cpu,i);
+                this_object = INDEX_KOBJECT_PTR(cpu, i);
                 this_object->cpu = cpu;
                 this_object->index = i;
                 retval = kobject_init_and_add(&(this_object->kobj),
@@ -925,9 +943,8 @@ static int __cpuinit cache_add_dev(struct sys_device * sys_dev)
                                               per_cpu(cache_kobject, cpu),
                                               "index%1lu", i);
                 if (unlikely(retval)) {
-                        for (j = 0; j < i; j++) {
-                                kobject_put(&(INDEX_KOBJECT_PTR(cpu,j)->kobj));
-                        }
+                        for (j = 0; j < i; j++)
+                                kobject_put(&(INDEX_KOBJECT_PTR(cpu, j)->kobj));
                         kobject_put(per_cpu(cache_kobject, cpu));
                         cpuid4_cache_sysfs_exit(cpu);
                         return retval;
@@ -952,7 +969,7 @@ static void __cpuinit cache_remove_dev(struct sys_device * sys_dev)
         cpumask_clear_cpu(cpu, to_cpumask(cache_dev_map));
 
         for (i = 0; i < num_cache_leaves; i++)
-                kobject_put(&(INDEX_KOBJECT_PTR(cpu,i)->kobj));
+                kobject_put(&(INDEX_KOBJECT_PTR(cpu, i)->kobj));
         kobject_put(per_cpu(cache_kobject, cpu));
         cpuid4_cache_sysfs_exit(cpu);
 }
@@ -977,8 +994,7 @@ static int __cpuinit cacheinfo_cpu_callback(struct notifier_block *nfb,
         return NOTIFY_OK;
 }
 
-static struct notifier_block __cpuinitdata cacheinfo_cpu_notifier =
-{
+static struct notifier_block __cpuinitdata cacheinfo_cpu_notifier = {
         .notifier_call = cacheinfo_cpu_callback,
 };
 
diff --git a/arch/x86/kernel/cpu/mcheck/Makefile b/arch/x86/kernel/cpu/mcheck/Makefile
index 188a1ca5ad2b..4ac6d48fe11b 100644
--- a/arch/x86/kernel/cpu/mcheck/Makefile
+++ b/arch/x86/kernel/cpu/mcheck/Makefile
@@ -1,11 +1,8 @@
-obj-y                           =  mce.o
+obj-y                           =  mce.o mce-severity.o
 
-obj-$(CONFIG_X86_NEW_MCE)       += mce-severity.o
-obj-$(CONFIG_X86_OLD_MCE)       += k7.o p4.o p6.o
 obj-$(CONFIG_X86_ANCIENT_MCE)   += winchip.o p5.o
 obj-$(CONFIG_X86_MCE_INTEL)     += mce_intel.o
 obj-$(CONFIG_X86_MCE_AMD)       += mce_amd.o
-obj-$(CONFIG_X86_MCE_NONFATAL)  += non-fatal.o
 obj-$(CONFIG_X86_MCE_THRESHOLD) += threshold.o
 obj-$(CONFIG_X86_MCE_INJECT)    += mce-inject.o
 
diff --git a/arch/x86/kernel/cpu/mcheck/k7.c b/arch/x86/kernel/cpu/mcheck/k7.c
deleted file mode 100644
index b945d5dbc609..000000000000
--- a/arch/x86/kernel/cpu/mcheck/k7.c
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Athlon specific Machine Check Exception Reporting
- * (C) Copyright 2002 Dave Jones <davej@redhat.com>
- */
-#include <linux/interrupt.h>
-#include <linux/kernel.h>
-#include <linux/types.h>
-#include <linux/init.h>
-#include <linux/smp.h>
-
-#include <asm/processor.h>
-#include <asm/system.h>
-#include <asm/mce.h>
-#include <asm/msr.h>
-
-/* Machine Check Handler For AMD Athlon/Duron: */
-static void k7_machine_check(struct pt_regs *regs, long error_code)
-{
-        u32 alow, ahigh, high, low;
-        u32 mcgstl, mcgsth;
-        int recover = 1;
-        int i;
-
-        rdmsr(MSR_IA32_MCG_STATUS, mcgstl, mcgsth);
-        if (mcgstl & (1<<0))    /* Recoverable ? */
-                recover = 0;
-
-        printk(KERN_EMERG "CPU %d: Machine Check Exception: %08x%08x\n",
-                smp_processor_id(), mcgsth, mcgstl);
-
-        for (i = 1; i < nr_mce_banks; i++) {
-                rdmsr(MSR_IA32_MC0_STATUS+i*4, low, high);
-                if (high & (1<<31)) {
-                        char misc[20];
-                        char addr[24];
-
-                        misc[0] = '\0';
-                        addr[0] = '\0';
-
-                        if (high & (1<<29))
-                                recover |= 1;
-                        if (high & (1<<25))
-                                recover |= 2;
-                        high &= ~(1<<31);
-
-                        if (high & (1<<27)) {
-                                rdmsr(MSR_IA32_MC0_MISC+i*4, alow, ahigh);
-                                snprintf(misc, 20, "[%08x%08x]", ahigh, alow);
-                        }
-                        if (high & (1<<26)) {
-                                rdmsr(MSR_IA32_MC0_ADDR+i*4, alow, ahigh);
-                                snprintf(addr, 24, " at %08x%08x", ahigh, alow);
-                        }
-
-                        printk(KERN_EMERG "CPU %d: Bank %d: %08x%08x%s%s\n",
-                                smp_processor_id(), i, high, low, misc, addr);
-
-                        /* Clear it: */
-                        wrmsr(MSR_IA32_MC0_STATUS+i*4, 0UL, 0UL);
-                        /* Serialize: */
-                        wmb();
-                        add_taint(TAINT_MACHINE_CHECK);
-                }
-        }
-
-        if (recover & 2)
-                panic("CPU context corrupt");
-        if (recover & 1)
-                panic("Unable to continue");
-
-        printk(KERN_EMERG "Attempting to continue.\n");
-
-        mcgstl &= ~(1<<2);
-        wrmsr(MSR_IA32_MCG_STATUS, mcgstl, mcgsth);
-}
-
-
-/* AMD K7 machine check is Intel like: */
-void amd_mcheck_init(struct cpuinfo_x86 *c)
-{
-        u32 l, h;
-        int i;
-
-        if (!cpu_has(c, X86_FEATURE_MCE))
-                return;
-
-        machine_check_vector = k7_machine_check;
-        /* Make sure the vector pointer is visible before we enable MCEs: */
-        wmb();
-
-        printk(KERN_INFO "Intel machine check architecture supported.\n");
-
-        rdmsr(MSR_IA32_MCG_CAP, l, h);
-        if (l & (1<<8)) /* Control register present ? */
-                wrmsr(MSR_IA32_MCG_CTL, 0xffffffff, 0xffffffff);
-        nr_mce_banks = l & 0xff;
-
-        /*
-         * Clear status for MC index 0 separately, we don't touch CTL,
-         * as some K7 Athlons cause spurious MCEs when its enabled:
-         */
-        if (boot_cpu_data.x86 == 6) {
-                wrmsr(MSR_IA32_MC0_STATUS, 0x0, 0x0);
-                i = 1;
-        } else
-                i = 0;
-
-        for (; i < nr_mce_banks; i++) {
-                wrmsr(MSR_IA32_MC0_CTL+4*i, 0xffffffff, 0xffffffff);
-                wrmsr(MSR_IA32_MC0_STATUS+4*i, 0x0, 0x0);
-        }
-
-        set_in_cr4(X86_CR4_MCE);
-        printk(KERN_INFO "Intel machine check reporting enabled on CPU#%d.\n",
-                smp_processor_id());
-}
diff --git a/arch/x86/kernel/cpu/mcheck/mce-inject.c b/arch/x86/kernel/cpu/mcheck/mce-inject.c
index a3a235a53f09..7029f0e2acad 100644
--- a/arch/x86/kernel/cpu/mcheck/mce-inject.c
+++ b/arch/x86/kernel/cpu/mcheck/mce-inject.c
@@ -18,7 +18,12 @@
 #include <linux/string.h>
 #include <linux/fs.h>
 #include <linux/smp.h>
+#include <linux/notifier.h>
+#include <linux/kdebug.h>
+#include <linux/cpu.h>
+#include <linux/sched.h>
 #include <asm/mce.h>
+#include <asm/apic.h>
 
 /* Update fake mce registers on current CPU. */
 static void inject_mce(struct mce *m)
@@ -39,44 +44,141 @@ static void inject_mce(struct mce *m)
         i->finished = 1;
 }
 
-struct delayed_mce {
-        struct timer_list timer;
-        struct mce m;
-};
+static void raise_poll(struct mce *m)
+{
+        unsigned long flags;
+        mce_banks_t b;
 
-/* Inject mce on current CPU */
-static void raise_mce(unsigned long data)
+        memset(&b, 0xff, sizeof(mce_banks_t));
+        local_irq_save(flags);
+        machine_check_poll(0, &b);
+        local_irq_restore(flags);
+        m->finished = 0;
+}
+
+static void raise_exception(struct mce *m, struct pt_regs *pregs)
 {
-        struct delayed_mce *dm = (struct delayed_mce *)data;
-        struct mce *m = &dm->m;
-        int cpu = m->extcpu;
+        struct pt_regs regs;
+        unsigned long flags;
 
-        inject_mce(m);
-        if (m->status & MCI_STATUS_UC) {
-                struct pt_regs regs;
+        if (!pregs) {
                 memset(&regs, 0, sizeof(struct pt_regs));
                 regs.ip = m->ip;
                 regs.cs = m->cs;
+                pregs = &regs;
+        }
+        /* in mcheck exeception handler, irq will be disabled */
+        local_irq_save(flags);
+        do_machine_check(pregs, 0);
+        local_irq_restore(flags);
+        m->finished = 0;
+}
+
+static cpumask_t mce_inject_cpumask;
+
+static int mce_raise_notify(struct notifier_block *self,
+                            unsigned long val, void *data)
+{
+        struct die_args *args = (struct die_args *)data;
+        int cpu = smp_processor_id();
+        struct mce *m = &__get_cpu_var(injectm);
+        if (val != DIE_NMI_IPI || !cpu_isset(cpu, mce_inject_cpumask))
+                return NOTIFY_DONE;
+        cpu_clear(cpu, mce_inject_cpumask);
+        if (m->inject_flags & MCJ_EXCEPTION)
+                raise_exception(m, args->regs);
+        else if (m->status)
+                raise_poll(m);
+        return NOTIFY_STOP;
+}
+
+static struct notifier_block mce_raise_nb = {
+        .notifier_call = mce_raise_notify,
+        .priority = 1000,
+};
+
+/* Inject mce on current CPU */
+static int raise_local(struct mce *m)
+{
+        int context = MCJ_CTX(m->inject_flags);
+        int ret = 0;
+        int cpu = m->extcpu;
+
+        if (m->inject_flags & MCJ_EXCEPTION) {
                 printk(KERN_INFO "Triggering MCE exception on CPU %d\n", cpu);
-                do_machine_check(&regs, 0);
+                switch (context) {
+                case MCJ_CTX_IRQ:
+                        /*
+                         * Could do more to fake interrupts like
+                         * calling irq_enter, but the necessary
+                         * machinery isn't exported currently.
+                         */
+                        /*FALL THROUGH*/
+                case MCJ_CTX_PROCESS:
+                        raise_exception(m, NULL);
+                        break;
+                default:
+                        printk(KERN_INFO "Invalid MCE context\n");
+                        ret = -EINVAL;
+                }
                 printk(KERN_INFO "MCE exception done on CPU %d\n", cpu);
-        } else {
-                mce_banks_t b;
-                memset(&b, 0xff, sizeof(mce_banks_t));
+        } else if (m->status) {
                 printk(KERN_INFO "Starting machine check poll CPU %d\n", cpu);
-                machine_check_poll(0, &b);
+                raise_poll(m);
                 mce_notify_irq();
-                printk(KERN_INFO "Finished machine check poll on CPU %d\n",
-                       cpu);
-        }
-        kfree(dm);
+                printk(KERN_INFO "Machine check poll done on CPU %d\n", cpu);
+        } else
+                m->finished = 0;
+
+        return ret;
+}
+
+static void raise_mce(struct mce *m)
+{
+        int context = MCJ_CTX(m->inject_flags);
+
+        inject_mce(m);
+
+        if (context == MCJ_CTX_RANDOM)
+                return;
+
+#ifdef CONFIG_X86_LOCAL_APIC
+        if (m->inject_flags & MCJ_NMI_BROADCAST) {
+                unsigned long start;
+                int cpu;
+                get_online_cpus();
+                mce_inject_cpumask = cpu_online_map;
+                cpu_clear(get_cpu(), mce_inject_cpumask);
+                for_each_online_cpu(cpu) {
+                        struct mce *mcpu = &per_cpu(injectm, cpu);
+                        if (!mcpu->finished ||
+                            MCJ_CTX(mcpu->inject_flags) != MCJ_CTX_RANDOM)
+                                cpu_clear(cpu, mce_inject_cpumask);
+                }
+                if (!cpus_empty(mce_inject_cpumask))
+                        apic->send_IPI_mask(&mce_inject_cpumask, NMI_VECTOR);
+                start = jiffies;
+                while (!cpus_empty(mce_inject_cpumask)) {
+                        if (!time_before(jiffies, start + 2*HZ)) {
+                                printk(KERN_ERR
+                                "Timeout waiting for mce inject NMI %lx\n",
+                                        *cpus_addr(mce_inject_cpumask));
+                                break;
+                        }
+                        cpu_relax();
+                }
+                raise_local(m);
+                put_cpu();
+                put_online_cpus();
+        } else
+#endif
+                raise_local(m);
 }
 
 /* Error injection interface */
 static ssize_t mce_write(struct file *filp, const char __user *ubuf,
                          size_t usize, loff_t *off)
 {
-        struct delayed_mce *dm;
         struct mce m;
 
         if (!capable(CAP_SYS_ADMIN))
@@ -96,19 +198,12 @@ static ssize_t mce_write(struct file *filp, const char __user *ubuf,
         if (m.extcpu >= num_possible_cpus() || !cpu_online(m.extcpu))
                 return -EINVAL;
 
-        dm = kmalloc(sizeof(struct delayed_mce), GFP_KERNEL);
-        if (!dm)
-                return -ENOMEM;
-
         /*
          * Need to give user space some time to set everything up,
          * so do it a jiffie or two later everywhere.
-         * Should we use a hrtimer here for better synchronization?
          */
-        memcpy(&dm->m, &m, sizeof(struct mce));
-        setup_timer(&dm->timer, raise_mce, (unsigned long)dm);
-        dm->timer.expires = jiffies + 2;
-        add_timer_on(&dm->timer, m.extcpu);
+        schedule_timeout(2);
+        raise_mce(&m);
         return usize;
 }
 
@@ -116,6 +211,7 @@ static int inject_init(void)
 {
         printk(KERN_INFO "Machine check injector initialized\n");
         mce_chrdev_ops.write = mce_write;
+        register_die_notifier(&mce_raise_nb);
         return 0;
 }
 
diff --git a/arch/x86/kernel/cpu/mcheck/mce-internal.h b/arch/x86/kernel/cpu/mcheck/mce-internal.h
index 54dcb8ff12e5..32996f9fab67 100644
--- a/arch/x86/kernel/cpu/mcheck/mce-internal.h
+++ b/arch/x86/kernel/cpu/mcheck/mce-internal.h
@@ -1,3 +1,4 @@
+#include <linux/sysdev.h>
 #include <asm/mce.h>
 
 enum severity_level {
@@ -10,6 +11,20 @@ enum severity_level {
         MCE_PANIC_SEVERITY,
 };
 
+#define ATTR_LEN                16
+
+/* One object for each MCE bank, shared by all CPUs */
+struct mce_bank {
+        u64                     ctl;                    /* subevents to enable */
+        unsigned char init;                             /* initialise bank? */
+        struct sysdev_attribute attr;                   /* sysdev attribute */
+        char                    attrname[ATTR_LEN];     /* attribute name */
+};
+
 int mce_severity(struct mce *a, int tolerant, char **msg);
+struct dentry *mce_get_debugfs_dir(void);
 
 extern int mce_ser;
+
+extern struct mce_bank *mce_banks;
+
diff --git a/arch/x86/kernel/cpu/mcheck/mce-severity.c b/arch/x86/kernel/cpu/mcheck/mce-severity.c
index ff0807f97056..8a85dd1b1aa1 100644
--- a/arch/x86/kernel/cpu/mcheck/mce-severity.c
+++ b/arch/x86/kernel/cpu/mcheck/mce-severity.c
@@ -139,6 +139,7 @@ int mce_severity(struct mce *a, int tolerant, char **msg)
         }
 }
 
+#ifdef CONFIG_DEBUG_FS
 static void *s_start(struct seq_file *f, loff_t *pos)
 {
         if (*pos >= ARRAY_SIZE(severities))
@@ -197,7 +198,7 @@ static int __init severities_debugfs_init(void)
 {
         struct dentry *dmce = NULL, *fseverities_coverage = NULL;
 
-        dmce = debugfs_create_dir("mce", NULL);
+        dmce = mce_get_debugfs_dir();
         if (dmce == NULL)
                 goto err_out;
         fseverities_coverage = debugfs_create_file("severities-coverage",
@@ -209,10 +210,7 @@ static int __init severities_debugfs_init(void)
         return 0;
 
 err_out:
-        if (fseverities_coverage)
-                debugfs_remove(fseverities_coverage);
-        if (dmce)
-                debugfs_remove(dmce);
         return -ENOMEM;
 }
 late_initcall(severities_debugfs_init);
+#endif
diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c
index 01213048f62f..2f5aab26320e 100644
--- a/arch/x86/kernel/cpu/mcheck/mce.c
+++ b/arch/x86/kernel/cpu/mcheck/mce.c
@@ -34,6 +34,7 @@
 #include <linux/smp.h>
 #include <linux/fs.h>
 #include <linux/mm.h>
+#include <linux/debugfs.h>
 
 #include <asm/processor.h>
 #include <asm/hw_irq.h>
@@ -45,21 +46,8 @@
 
 #include "mce-internal.h"
 
-/* Handle unconfigured int18 (should never happen) */
-static void unexpected_machine_check(struct pt_regs *regs, long error_code)
-{
-        printk(KERN_ERR "CPU#%d: Unexpected int18 (Machine Check).\n",
-               smp_processor_id());
-}
-
-/* Call the installed machine check handler for this CPU setup. */
-void (*machine_check_vector)(struct pt_regs *, long error_code) =
-                                                unexpected_machine_check;
-
 int mce_disabled __read_mostly;
 
-#ifdef CONFIG_X86_NEW_MCE
-
 #define MISC_MCELOG_MINOR       227
 
 #define SPINUNIT 100    /* 100ns */
@@ -77,7 +65,6 @@ DEFINE_PER_CPU(unsigned, mce_exception_count);
  */
 static int                      tolerant                __read_mostly = 1;
 static int                      banks                   __read_mostly;
-static u64                      *bank                   __read_mostly;
 static int                      rip_msr                 __read_mostly;
 static int                      mce_bootlog             __read_mostly = -1;
 static int                      monarch_timeout         __read_mostly = -1;
@@ -87,13 +74,13 @@ int				mce_cmci_disabled	__read_mostly;
 int                             mce_ignore_ce           __read_mostly;
 int                             mce_ser                 __read_mostly;
 
+struct mce_bank                *mce_banks               __read_mostly;
+
 /* User mode helper program triggered by machine check event */
 static unsigned long            mce_need_notify;
 static char                     mce_helper[128];
 static char                     *mce_helper_argv[2] = { mce_helper, NULL };
 
-static unsigned long            dont_init_banks;
-
 static DECLARE_WAIT_QUEUE_HEAD(mce_wait);
 static DEFINE_PER_CPU(struct mce, mces_seen);
 static int                      cpu_missing;
@@ -104,11 +91,6 @@ DEFINE_PER_CPU(mce_banks_t, mce_poll_banks) = {
         [0 ... BITS_TO_LONGS(MAX_NR_BANKS)-1] = ~0UL
 };
 
-static inline int skip_bank_init(int i)
-{
-        return i < BITS_PER_LONG && test_bit(i, &dont_init_banks);
-}
-
 static DEFINE_PER_CPU(struct work_struct, mce_work);
 
 /* Do initial initialization of a struct mce */
@@ -183,6 +165,11 @@ void mce_log(struct mce *mce)
         set_bit(0, &mce_need_notify);
 }
 
+void __weak decode_mce(struct mce *m)
+{
+        return;
+}
+
 static void print_mce(struct mce *m)
 {
         printk(KERN_EMERG
@@ -205,6 +192,8 @@ static void print_mce(struct mce *m)
         printk(KERN_EMERG "PROCESSOR %u:%x TIME %llu SOCKET %u APIC %x\n",
                         m->cpuvendor, m->cpuid, m->time, m->socketid,
                         m->apicid);
+
+        decode_mce(m);
 }
 
 static void print_mce_head(void)
@@ -215,13 +204,19 @@ static void print_mce_head(void)
 static void print_mce_tail(void)
 {
         printk(KERN_EMERG "This is not a software problem!\n"
-               "Run through mcelog --ascii to decode and contact your hardware vendor\n");
+#if (!defined(CONFIG_EDAC) || !defined(CONFIG_CPU_SUP_AMD))
+               "Run through mcelog --ascii to decode and contact your hardware vendor\n"
+#endif
+               );
 }
 
 #define PANIC_TIMEOUT 5 /* 5 seconds */
 
 static atomic_t mce_paniced;
 
+static int fake_panic;
+static atomic_t mce_fake_paniced;
+
 /* Panic in progress. Enable interrupts and wait for final IPI */
 static void wait_for_panic(void)
 {
@@ -239,15 +234,21 @@ static void mce_panic(char *msg, struct mce *final, char *exp)
 {
         int i;
 
-        /*
-         * Make sure only one CPU runs in machine check panic
-         */
-        if (atomic_add_return(1, &mce_paniced) > 1)
-                wait_for_panic();
-        barrier();
+        if (!fake_panic) {
+                /*
+                 * Make sure only one CPU runs in machine check panic
+                 */
+                if (atomic_inc_return(&mce_paniced) > 1)
+                        wait_for_panic();
+                barrier();
 
-        bust_spinlocks(1);
-        console_verbose();
+                bust_spinlocks(1);
+                console_verbose();
+        } else {
+                /* Don't log too much for fake panic */
+                if (atomic_inc_return(&mce_fake_paniced) > 1)
+                        return;
+        }
         print_mce_head();
         /* First print corrected ones that are still unlogged */
         for (i = 0; i < MCE_LOG_LEN; i++) {
@@ -274,9 +275,12 @@ static void mce_panic(char *msg, struct mce *final, char *exp)
         print_mce_tail();
         if (exp)
                 printk(KERN_EMERG "Machine check: %s\n", exp);
-        if (panic_timeout == 0)
-                panic_timeout = mce_panic_timeout;
-        panic(msg);
+        if (!fake_panic) {
+                if (panic_timeout == 0)
+                        panic_timeout = mce_panic_timeout;
+                panic(msg);
+        } else
+                printk(KERN_EMERG "Fake kernel panic: %s\n", msg);
 }
 
 /* Support code for software error injection */
@@ -286,11 +290,11 @@ static int msr_to_offset(u32 msr)
         unsigned bank = __get_cpu_var(injectm.bank);
         if (msr == rip_msr)
                 return offsetof(struct mce, ip);
-        if (msr == MSR_IA32_MC0_STATUS + bank*4)
+        if (msr == MSR_IA32_MCx_STATUS(bank))
                 return offsetof(struct mce, status);
-        if (msr == MSR_IA32_MC0_ADDR + bank*4)
+        if (msr == MSR_IA32_MCx_ADDR(bank))
                 return offsetof(struct mce, addr);
-        if (msr == MSR_IA32_MC0_MISC + bank*4)
+        if (msr == MSR_IA32_MCx_MISC(bank))
                 return offsetof(struct mce, misc);
         if (msr == MSR_IA32_MCG_STATUS)
                 return offsetof(struct mce, mcgstatus);
@@ -495,7 +499,7 @@ void machine_check_poll(enum mcp_flags flags, mce_banks_t *b)
 
         m.mcgstatus = mce_rdmsrl(MSR_IA32_MCG_STATUS);
         for (i = 0; i < banks; i++) {
-                if (!bank[i] || !test_bit(i, *b))
+                if (!mce_banks[i].ctl || !test_bit(i, *b))
                         continue;
 
                 m.misc = 0;
@@ -504,7 +508,7 @@ void machine_check_poll(enum mcp_flags flags, mce_banks_t *b)
                 m.tsc = 0;
 
                 barrier();
-                m.status = mce_rdmsrl(MSR_IA32_MC0_STATUS + i*4);
+                m.status = mce_rdmsrl(MSR_IA32_MCx_STATUS(i));
                 if (!(m.status & MCI_STATUS_VAL))
                         continue;
 
@@ -519,9 +523,9 @@ void machine_check_poll(enum mcp_flags flags, mce_banks_t *b)
                         continue;
 
                 if (m.status & MCI_STATUS_MISCV)
-                        m.misc = mce_rdmsrl(MSR_IA32_MC0_MISC + i*4);
+                        m.misc = mce_rdmsrl(MSR_IA32_MCx_MISC(i));
                 if (m.status & MCI_STATUS_ADDRV)
-                        m.addr = mce_rdmsrl(MSR_IA32_MC0_ADDR + i*4);
+                        m.addr = mce_rdmsrl(MSR_IA32_MCx_ADDR(i));
 
                 if (!(flags & MCP_TIMESTAMP))
                         m.tsc = 0;
@@ -537,7 +541,7 @@ void machine_check_poll(enum mcp_flags flags, mce_banks_t *b)
                 /*
                  * Clear state for this bank.
                  */
-                mce_wrmsrl(MSR_IA32_MC0_STATUS+4*i, 0);
+                mce_wrmsrl(MSR_IA32_MCx_STATUS(i), 0);
         }
 
         /*
@@ -558,7 +562,7 @@ static int mce_no_way_out(struct mce *m, char **msg)
         int i;
 
         for (i = 0; i < banks; i++) {
-                m->status = mce_rdmsrl(MSR_IA32_MC0_STATUS + i*4);
+                m->status = mce_rdmsrl(MSR_IA32_MCx_STATUS(i));
                 if (mce_severity(m, tolerant, msg) >= MCE_PANIC_SEVERITY)
                         return 1;
         }
@@ -618,7 +622,7 @@ out:
  * This way we prevent any potential data corruption in a unrecoverable case
  * and also makes sure always all CPU's errors are examined.
  *
- * Also this detects the case of an machine check event coming from outer
+ * Also this detects the case of a machine check event coming from outer
  * space (not detected by any CPUs) In this case some external agent wants
  * us to shut down, so panic too.
  *
@@ -671,7 +675,7 @@ static void mce_reign(void)
          * No machine check event found. Must be some external
          * source or one CPU is hung. Panic.
          */
-        if (!m && tolerant < 3)
+        if (global_worst <= MCE_KEEP_SEVERITY && tolerant < 3)
                 mce_panic("Machine check from unknown source", NULL, NULL);
 
         /*
@@ -705,7 +709,7 @@ static int mce_start(int *no_way_out)
          * global_nwo should be updated before mce_callin
          */
         smp_wmb();
-        order = atomic_add_return(1, &mce_callin);
+        order = atomic_inc_return(&mce_callin);
 
         /*
          * Wait for everyone.
@@ -842,7 +846,7 @@ static void mce_clear_state(unsigned long *toclear)
 
         for (i = 0; i < banks; i++) {
                 if (test_bit(i, toclear))
-                        mce_wrmsrl(MSR_IA32_MC0_STATUS+4*i, 0);
+                        mce_wrmsrl(MSR_IA32_MCx_STATUS(i), 0);
         }
 }
 
@@ -895,11 +899,11 @@ void do_machine_check(struct pt_regs *regs, long error_code)
         mce_setup(&m);
 
         m.mcgstatus = mce_rdmsrl(MSR_IA32_MCG_STATUS);
-        no_way_out = mce_no_way_out(&m, &msg);
-
         final = &__get_cpu_var(mces_seen);
         *final = m;
 
+        no_way_out = mce_no_way_out(&m, &msg);
+
         barrier();
 
         /*
@@ -916,14 +920,14 @@ void do_machine_check(struct pt_regs *regs, long error_code)
         order = mce_start(&no_way_out);
         for (i = 0; i < banks; i++) {
                 __clear_bit(i, toclear);
-                if (!bank[i])
+                if (!mce_banks[i].ctl)
                         continue;
 
                 m.misc = 0;
                 m.addr = 0;
                 m.bank = i;
 
-                m.status = mce_rdmsrl(MSR_IA32_MC0_STATUS + i*4);
+                m.status = mce_rdmsrl(MSR_IA32_MCx_STATUS(i));
                 if ((m.status & MCI_STATUS_VAL) == 0)
                         continue;
 
@@ -964,9 +968,9 @@ void do_machine_check(struct pt_regs *regs, long error_code)
                         kill_it = 1;
 
                 if (m.status & MCI_STATUS_MISCV)
-                        m.misc = mce_rdmsrl(MSR_IA32_MC0_MISC + i*4);
+                        m.misc = mce_rdmsrl(MSR_IA32_MCx_MISC(i));
                 if (m.status & MCI_STATUS_ADDRV)
-                        m.addr = mce_rdmsrl(MSR_IA32_MC0_ADDR + i*4);
+                        m.addr = mce_rdmsrl(MSR_IA32_MCx_ADDR(i));
 
                 /*
                  * Action optional error. Queue address for later processing.
@@ -1091,7 +1095,7 @@ void mce_log_therm_throt_event(__u64 status)
  */
 static int check_interval = 5 * 60; /* 5 minutes */
 
-static DEFINE_PER_CPU(int, next_interval); /* in jiffies */
+static DEFINE_PER_CPU(int, mce_next_interval); /* in jiffies */
 static DEFINE_PER_CPU(struct timer_list, mce_timer);
 
 static void mcheck_timer(unsigned long data)
@@ -1110,7 +1114,7 @@ static void mcheck_timer(unsigned long data)
          * Alert userspace if needed.  If we logged an MCE, reduce the
          * polling interval, otherwise increase the polling interval.
          */
-        n = &__get_cpu_var(next_interval);
+        n = &__get_cpu_var(mce_next_interval);
         if (mce_notify_irq())
                 *n = max(*n/2, HZ/100);
         else
@@ -1159,10 +1163,25 @@ int mce_notify_irq(void)
 }
 EXPORT_SYMBOL_GPL(mce_notify_irq);
 
+static int mce_banks_init(void)
+{
+        int i;
+
+        mce_banks = kzalloc(banks * sizeof(struct mce_bank), GFP_KERNEL);
+        if (!mce_banks)
+                return -ENOMEM;
+        for (i = 0; i < banks; i++) {
+                struct mce_bank *b = &mce_banks[i];
+                b->ctl = -1ULL;
+                b->init = 1;
+        }
+        return 0;
+}
+
 /*
  * Initialize Machine Checks for a CPU.
  */
-static int mce_cap_init(void)
+static int __cpuinit mce_cap_init(void)
 {
         unsigned b;
         u64 cap;
@@ -1182,11 +1201,10 @@ static int mce_cap_init(void)
         /* Don't support asymmetric configurations today */
         WARN_ON(banks != 0 && b != banks);
         banks = b;
-        if (!bank) {
-                bank = kmalloc(banks * sizeof(u64), GFP_KERNEL);
-                if (!bank)
-                        return -ENOMEM;
-                memset(bank, 0xff, banks * sizeof(u64));
+        if (!mce_banks) {
+                int err = mce_banks_init();
+                if (err)
+                        return err;
         }
 
         /* Use accurate RIP reporting if available. */
@@ -1218,15 +1236,16 @@ static void mce_init(void)
                 wrmsr(MSR_IA32_MCG_CTL, 0xffffffff, 0xffffffff);
 
         for (i = 0; i < banks; i++) {
-                if (skip_bank_init(i))
+                struct mce_bank *b = &mce_banks[i];
+                if (!b->init)
                         continue;
-                wrmsrl(MSR_IA32_MC0_CTL+4*i, bank[i]);
-                wrmsrl(MSR_IA32_MC0_STATUS+4*i, 0);
+                wrmsrl(MSR_IA32_MCx_CTL(i), b->ctl);
+                wrmsrl(MSR_IA32_MCx_STATUS(i), 0);
         }
 }
 
 /* Add per CPU specific workarounds here */
-static int mce_cpu_quirks(struct cpuinfo_x86 *c)
+static int __cpuinit mce_cpu_quirks(struct cpuinfo_x86 *c)
 {
         if (c->x86_vendor == X86_VENDOR_UNKNOWN) {
                 pr_info("MCE: unknown CPU type - not enabling MCE support.\n");
@@ -1241,7 +1260,7 @@ static int mce_cpu_quirks(struct cpuinfo_x86 *c)
                          * trips off incorrectly with the IOMMU & 3ware
                          * & Cerberus:
                          */
-                        clear_bit(10, (unsigned long *)&bank[4]);
+                        clear_bit(10, (unsigned long *)&mce_banks[4].ctl);
                 }
                 if (c->x86 <= 17 && mce_bootlog < 0) {
                         /*
@@ -1255,7 +1274,7 @@ static int mce_cpu_quirks(struct cpuinfo_x86 *c)
                  * by default.
                  */
                  if (c->x86 == 6 && banks > 0)
-                        bank[0] = 0;
+                        mce_banks[0].ctl = 0;
         }
 
         if (c->x86_vendor == X86_VENDOR_INTEL) {
@@ -1268,8 +1287,8 @@ static int mce_cpu_quirks(struct cpuinfo_x86 *c)
                  * valid event later, merely don't write CTL0.
                  */
 
-                if (c->x86 == 6 && c->x86_model < 0x1A)
-                        __set_bit(0, &dont_init_banks);
+                if (c->x86 == 6 && c->x86_model < 0x1A && banks > 0)
+                        mce_banks[0].init = 0;
 
                 /*
                  * All newer Intel systems support MCE broadcasting. Enable
@@ -1325,7 +1344,7 @@ static void mce_cpu_features(struct cpuinfo_x86 *c)
 static void mce_init_timer(void)
 {
         struct timer_list *t = &__get_cpu_var(mce_timer);
-        int *n = &__get_cpu_var(next_interval);
+        int *n = &__get_cpu_var(mce_next_interval);
 
         if (mce_ignore_ce)
                 return;
@@ -1338,6 +1357,17 @@ static void mce_init_timer(void)
         add_timer_on(t, smp_processor_id());
 }
 
+/* Handle unconfigured int18 (should never happen) */
+static void unexpected_machine_check(struct pt_regs *regs, long error_code)
+{
+        printk(KERN_ERR "CPU#%d: Unexpected int18 (Machine Check).\n",
+               smp_processor_id());
+}
+
+/* Call the installed machine check handler for this CPU setup. */
+void (*machine_check_vector)(struct pt_regs *, long error_code) =
+                                                unexpected_machine_check;
+
 /*
  * Called for each booted CPU to set up machine checks.
  * Must be called with preempt off:
@@ -1551,8 +1581,10 @@ static struct miscdevice mce_log_device = {
  */
 static int __init mcheck_enable(char *str)
 {
-        if (*str == 0)
+        if (*str == 0) {
                 enable_p5_mce();
+                return 1;
+        }
         if (*str == '=')
                 str++;
         if (!strcmp(str, "off"))
@@ -1593,8 +1625,9 @@ static int mce_disable(void)
         int i;
 
         for (i = 0; i < banks; i++) {
-                if (!skip_bank_init(i))
-                        wrmsrl(MSR_IA32_MC0_CTL + i*4, 0);
+                struct mce_bank *b = &mce_banks[i];
+                if (b->init)
+                        wrmsrl(MSR_IA32_MCx_CTL(i), 0);
         }
         return 0;
 }
@@ -1669,14 +1702,15 @@ DEFINE_PER_CPU(struct sys_device, mce_dev);
 __cpuinitdata
 void (*threshold_cpu_callback)(unsigned long action, unsigned int cpu);
 
-static struct sysdev_attribute *bank_attrs;
+static inline struct mce_bank *attr_to_bank(struct sysdev_attribute *attr)
+{
+        return container_of(attr, struct mce_bank, attr);
+}
 
 static ssize_t show_bank(struct sys_device *s, struct sysdev_attribute *attr,
                          char *buf)
 {
-        u64 b = bank[attr - bank_attrs];
-
-        return sprintf(buf, "%llx\n", b);
+        return sprintf(buf, "%llx\n", attr_to_bank(attr)->ctl);
 }
 
 static ssize_t set_bank(struct sys_device *s, struct sysdev_attribute *attr,
@@ -1687,7 +1721,7 @@ static ssize_t set_bank(struct sys_device *s, struct sysdev_attribute *attr,
         if (strict_strtoull(buf, 0, &new) < 0)
                 return -EINVAL;
 
-        bank[attr - bank_attrs] = new;
+        attr_to_bank(attr)->ctl = new;
         mce_restart();
 
         return size;
@@ -1829,7 +1863,7 @@ static __cpuinit int mce_create_device(unsigned int cpu)
         }
         for (j = 0; j < banks; j++) {
                 err = sysdev_create_file(&per_cpu(mce_dev, cpu),
-                                        &bank_attrs[j]);
+                                        &mce_banks[j].attr);
                 if (err)
                         goto error2;
         }
@@ -1838,10 +1872,10 @@ static __cpuinit int mce_create_device(unsigned int cpu)
         return 0;
 error2:
         while (--j >= 0)
-                sysdev_remove_file(&per_cpu(mce_dev, cpu), &bank_attrs[j]);
+                sysdev_remove_file(&per_cpu(mce_dev, cpu), &mce_banks[j].attr);
 error:
         while (--i >= 0)
-                sysdev_remove_file(&per_cpu(mce_dev, cpu), mce_attrs[i]);
+                sysdev_remove_file(&per_cpu(mce_dev, cpu), &mce_banks[i].attr);
 
         sysdev_unregister(&per_cpu(mce_dev, cpu));
 
@@ -1859,7 +1893,7 @@ static __cpuinit void mce_remove_device(unsigned int cpu)
                 sysdev_remove_file(&per_cpu(mce_dev, cpu), mce_attrs[i]);
 
         for (i = 0; i < banks; i++)
-                sysdev_remove_file(&per_cpu(mce_dev, cpu), &bank_attrs[i]);
+                sysdev_remove_file(&per_cpu(mce_dev, cpu), &mce_banks[i].attr);
 
         sysdev_unregister(&per_cpu(mce_dev, cpu));
         cpumask_clear_cpu(cpu, mce_dev_initialized);
@@ -1876,8 +1910,9 @@ static void mce_disable_cpu(void *h)
         if (!(action & CPU_TASKS_FROZEN))
                 cmci_clear();
         for (i = 0; i < banks; i++) {
-                if (!skip_bank_init(i))
-                        wrmsrl(MSR_IA32_MC0_CTL + i*4, 0);
+                struct mce_bank *b = &mce_banks[i];
+                if (b->init)
+                        wrmsrl(MSR_IA32_MCx_CTL(i), 0);
         }
 }
 
@@ -1892,8 +1927,9 @@ static void mce_reenable_cpu(void *h)
         if (!(action & CPU_TASKS_FROZEN))
                 cmci_reenable();
         for (i = 0; i < banks; i++) {
-                if (!skip_bank_init(i))
-                        wrmsrl(MSR_IA32_MC0_CTL + i*4, bank[i]);
+                struct mce_bank *b = &mce_banks[i];
+                if (b->init)
+                        wrmsrl(MSR_IA32_MCx_CTL(i), b->ctl);
         }
 }
 
@@ -1925,7 +1961,7 @@ mce_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu)
         case CPU_DOWN_FAILED:
         case CPU_DOWN_FAILED_FROZEN:
                 t->expires = round_jiffies(jiffies +
-                                                __get_cpu_var(next_interval));
+                                           __get_cpu_var(mce_next_interval));
                 add_timer_on(t, cpu);
                 smp_call_function_single(cpu, mce_reenable_cpu, &action, 1);
                 break;
@@ -1941,35 +1977,21 @@ static struct notifier_block mce_cpu_notifier __cpuinitdata = {
         .notifier_call = mce_cpu_callback,
 };
 
-static __init int mce_init_banks(void)
+static __init void mce_init_banks(void)
 {
         int i;
 
-        bank_attrs = kzalloc(sizeof(struct sysdev_attribute) * banks,
-                                GFP_KERNEL);
-        if (!bank_attrs)
-                return -ENOMEM;
-
         for (i = 0; i < banks; i++) {
-                struct sysdev_attribute *a = &bank_attrs[i];
+                struct mce_bank *b = &mce_banks[i];
+                struct sysdev_attribute *a = &b->attr;
 
-                a->attr.name    = kasprintf(GFP_KERNEL, "bank%d", i);
-                if (!a->attr.name)
-                        goto nomem;
+                a->attr.name    = b->attrname;
+                snprintf(b->attrname, ATTR_LEN, "bank%d", i);
 
                 a->attr.mode    = 0644;
                 a->show         = show_bank;
                 a->store        = set_bank;
         }
-        return 0;
-
-nomem:
-        while (--i >= 0)
-                kfree(bank_attrs[i].attr.name);
-        kfree(bank_attrs);
-        bank_attrs = NULL;
-
-        return -ENOMEM;
 }
 
 static __init int mce_init_device(void)
@@ -1982,9 +2004,7 @@ static __init int mce_init_device(void)
 
         zalloc_cpumask_var(&mce_dev_initialized, GFP_KERNEL);
 
-        err = mce_init_banks();
-        if (err)
-                return err;
+        mce_init_banks();
 
         err = sysdev_class_register(&mce_sysclass);
         if (err)
@@ -2004,57 +2024,65 @@ static __init int mce_init_device(void)
 
 device_initcall(mce_init_device);
 
-#else /* CONFIG_X86_OLD_MCE: */
-
-int nr_mce_banks;
-EXPORT_SYMBOL_GPL(nr_mce_banks);        /* non-fatal.o */
+/*
+ * Old style boot options parsing. Only for compatibility.
+ */
+static int __init mcheck_disable(char *str)
+{
+        mce_disabled = 1;
+        return 1;
+}
+__setup("nomce", mcheck_disable);
 
-/* This has to be run for each processor */
-void mcheck_init(struct cpuinfo_x86 *c)
+#ifdef CONFIG_DEBUG_FS
+struct dentry *mce_get_debugfs_dir(void)
 {
-        if (mce_disabled)
-                return;
+        static struct dentry *dmce;
 
-        switch (c->x86_vendor) {
-        case X86_VENDOR_AMD:
-                amd_mcheck_init(c);
-                break;
+        if (!dmce)
+                dmce = debugfs_create_dir("mce", NULL);
 
-        case X86_VENDOR_INTEL:
-                if (c->x86 == 5)
-                        intel_p5_mcheck_init(c);
-                if (c->x86 == 6)
-                        intel_p6_mcheck_init(c);
-                if (c->x86 == 15)
-                        intel_p4_mcheck_init(c);
-                break;
+        return dmce;
+}
 
-        case X86_VENDOR_CENTAUR:
-                if (c->x86 == 5)
-                        winchip_mcheck_init(c);
-                break;
+static void mce_reset(void)
+{
+        cpu_missing = 0;
+        atomic_set(&mce_fake_paniced, 0);
+        atomic_set(&mce_executing, 0);
+        atomic_set(&mce_callin, 0);
+        atomic_set(&global_nwo, 0);
+}
 
-        default:
-                break;
-        }
-        printk(KERN_INFO "mce: CPU supports %d MCE banks\n", nr_mce_banks);
+static int fake_panic_get(void *data, u64 *val)
+{
+        *val = fake_panic;
+        return 0;
 }
 
-static int __init mcheck_enable(char *str)
+static int fake_panic_set(void *data, u64 val)
 {
-        mce_p5_enabled = 1;
-        return 1;
+        mce_reset();
+        fake_panic = val;
+        return 0;
 }
-__setup("mce", mcheck_enable);
 
-#endif /* CONFIG_X86_OLD_MCE */
+DEFINE_SIMPLE_ATTRIBUTE(fake_panic_fops, fake_panic_get,
+                        fake_panic_set, "%llu\n");
 
-/*
- * Old style boot options parsing. Only for compatibility.
- */
-static int __init mcheck_disable(char *str)
+static int __init mce_debugfs_init(void)
 {
-        mce_disabled = 1;
-        return 1;
+        struct dentry *dmce, *ffake_panic;
+
+        dmce = mce_get_debugfs_dir();
+        if (!dmce)
+                return -ENOMEM;
+        ffake_panic = debugfs_create_file("fake_panic", 0444, dmce, NULL,
+                                          &fake_panic_fops);
+        if (!ffake_panic)
+                return -ENOMEM;
+
+        return 0;
 }
-__setup("nomce", mcheck_disable);
+late_initcall(mce_debugfs_init);
+#endif
diff --git a/arch/x86/kernel/cpu/mcheck/mce_amd.c b/arch/x86/kernel/cpu/mcheck/mce_amd.c
index ddae21620bda..83a3d1f4efca 100644
--- a/arch/x86/kernel/cpu/mcheck/mce_amd.c
+++ b/arch/x86/kernel/cpu/mcheck/mce_amd.c
@@ -69,7 +69,7 @@ struct threshold_bank {
         struct threshold_block  *blocks;
         cpumask_var_t           cpus;
 };
-static DEFINE_PER_CPU(struct threshold_bank *, threshold_banks[NR_BANKS]);
+static DEFINE_PER_CPU(struct threshold_bank * [NR_BANKS], threshold_banks);
 
 #ifdef CONFIG_SMP
 static unsigned char shared_bank[NR_BANKS] = {
@@ -489,12 +489,15 @@ static __cpuinit int threshold_create_bank(unsigned int cpu, unsigned int bank)
         int i, err = 0;
         struct threshold_bank *b = NULL;
         char name[32];
+#ifdef CONFIG_SMP
+        struct cpuinfo_x86 *c = &cpu_data(cpu);
+#endif
 
         sprintf(name, "threshold_bank%i", bank);
 
 #ifdef CONFIG_SMP
         if (cpu_data(cpu).cpu_core_id && shared_bank[bank]) {   /* symlink */
-                i = cpumask_first(cpu_core_mask(cpu));
+                i = cpumask_first(c->llc_shared_map);
 
                 /* first core not up yet */
                 if (cpu_data(i).cpu_core_id)
@@ -514,7 +517,7 @@ static __cpuinit int threshold_create_bank(unsigned int cpu, unsigned int bank)
                 if (err)
                         goto out;
 
-                cpumask_copy(b->cpus, cpu_core_mask(cpu));
+                cpumask_copy(b->cpus, c->llc_shared_map);
                 per_cpu(threshold_banks, cpu)[bank] = b;
 
                 goto out;
@@ -539,7 +542,7 @@ static __cpuinit int threshold_create_bank(unsigned int cpu, unsigned int bank)
 #ifndef CONFIG_SMP
         cpumask_setall(b->cpus);
 #else
-        cpumask_copy(b->cpus, cpu_core_mask(cpu));
+        cpumask_copy(b->cpus, c->llc_shared_map);
 #endif
 
         per_cpu(threshold_banks, cpu)[bank] = b;
diff --git a/arch/x86/kernel/cpu/mcheck/mce_intel.c b/arch/x86/kernel/cpu/mcheck/mce_intel.c
index e1acec0f7a32..889f665fe93d 100644
--- a/arch/x86/kernel/cpu/mcheck/mce_intel.c
+++ b/arch/x86/kernel/cpu/mcheck/mce_intel.c
@@ -90,7 +90,7 @@ static void cmci_discover(int banks, int boot)
                 if (test_bit(i, owned))
                         continue;
 
-                rdmsrl(MSR_IA32_MC0_CTL2 + i, val);
+                rdmsrl(MSR_IA32_MCx_CTL2(i), val);
 
                 /* Already owned by someone else? */
                 if (val & CMCI_EN) {
@@ -101,8 +101,8 @@ static void cmci_discover(int banks, int boot)
                 }
 
                 val |= CMCI_EN | CMCI_THRESHOLD;
-                wrmsrl(MSR_IA32_MC0_CTL2 + i, val);
-                rdmsrl(MSR_IA32_MC0_CTL2 + i, val);
+                wrmsrl(MSR_IA32_MCx_CTL2(i), val);
+                rdmsrl(MSR_IA32_MCx_CTL2(i), val);
 
                 /* Did the enable bit stick? -- the bank supports CMCI */
                 if (val & CMCI_EN) {
@@ -152,9 +152,9 @@ void cmci_clear(void)
                 if (!test_bit(i, __get_cpu_var(mce_banks_owned)))
                         continue;
                 /* Disable CMCI */
-                rdmsrl(MSR_IA32_MC0_CTL2 + i, val);
+                rdmsrl(MSR_IA32_MCx_CTL2(i), val);
                 val &= ~(CMCI_EN|CMCI_THRESHOLD_MASK);
-                wrmsrl(MSR_IA32_MC0_CTL2 + i, val);
+                wrmsrl(MSR_IA32_MCx_CTL2(i), val);
                 __clear_bit(i, __get_cpu_var(mce_banks_owned));
         }
         spin_unlock_irqrestore(&cmci_discover_lock, flags);
diff --git a/arch/x86/kernel/cpu/mcheck/non-fatal.c b/arch/x86/kernel/cpu/mcheck/non-fatal.c
deleted file mode 100644
index f5f2d6f71fb6..000000000000
--- a/arch/x86/kernel/cpu/mcheck/non-fatal.c
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Non Fatal Machine Check Exception Reporting
- *
- * (C) Copyright 2002 Dave Jones. <davej@redhat.com>
- *
- * This file contains routines to check for non-fatal MCEs every 15s
- *
- */
-#include <linux/interrupt.h>
-#include <linux/workqueue.h>
-#include <linux/jiffies.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/types.h>
-#include <linux/init.h>
-#include <linux/smp.h>
-
-#include <asm/processor.h>
-#include <asm/system.h>
-#include <asm/mce.h>
-#include <asm/msr.h>
-
-static int              firstbank;
-
-#define MCE_RATE        (15*HZ) /* timer rate is 15s */
-
-static void mce_checkregs(void *info)
-{
-        u32 low, high;
-        int i;
-
-        for (i = firstbank; i < nr_mce_banks; i++) {
-                rdmsr(MSR_IA32_MC0_STATUS+i*4, low, high);
-
-                if (!(high & (1<<31)))
-                        continue;
-
-                printk(KERN_INFO "MCE: The hardware reports a non fatal, "
-                        "correctable incident occurred on CPU %d.\n",
-                                smp_processor_id());
-
-                printk(KERN_INFO "Bank %d: %08x%08x\n", i, high, low);
-
-                /*
-                 * Scrub the error so we don't pick it up in MCE_RATE
-                 * seconds time:
-                 */
-                wrmsr(MSR_IA32_MC0_STATUS+i*4, 0UL, 0UL);
-
-                /* Serialize: */
-                wmb();
-                add_taint(TAINT_MACHINE_CHECK);
-        }
-}
-
-static void mce_work_fn(struct work_struct *work);
-static DECLARE_DELAYED_WORK(mce_work, mce_work_fn);
-
-static void mce_work_fn(struct work_struct *work)
-{
-        on_each_cpu(mce_checkregs, NULL, 1);
-        schedule_delayed_work(&mce_work, round_jiffies_relative(MCE_RATE));
-}
-
-static int __init init_nonfatal_mce_checker(void)
-{
-        struct cpuinfo_x86 *c = &boot_cpu_data;
-
-        /* Check for MCE support */
-        if (!cpu_has(c, X86_FEATURE_MCE))
-                return -ENODEV;
-
-        /* Check for PPro style MCA */
-        if (!cpu_has(c, X86_FEATURE_MCA))
-                return -ENODEV;
-
-        /* Some Athlons misbehave when we frob bank 0 */
-        if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD &&
-                                                boot_cpu_data.x86 == 6)
-                firstbank = 1;
-        else
-                firstbank = 0;
-
-        /*
-         * Check for non-fatal errors every MCE_RATE s
-         */
-        schedule_delayed_work(&mce_work, round_jiffies_relative(MCE_RATE));
-        printk(KERN_INFO "Machine check exception polling timer started.\n");
-
-        return 0;
-}
-module_init(init_nonfatal_mce_checker);
-
-MODULE_LICENSE("GPL");
diff --git a/arch/x86/kernel/cpu/mcheck/p4.c b/arch/x86/kernel/cpu/mcheck/p4.c
deleted file mode 100644
index 4482aea9aa2e..000000000000
--- a/arch/x86/kernel/cpu/mcheck/p4.c
+++ /dev/null
@@ -1,163 +0,0 @@
-/*
- * P4 specific Machine Check Exception Reporting
- */
-#include <linux/kernel.h>
-#include <linux/types.h>
-#include <linux/init.h>
-#include <linux/smp.h>
-
-#include <asm/processor.h>
-#include <asm/mce.h>
-#include <asm/msr.h>
-
-/* as supported by the P4/Xeon family */
-struct intel_mce_extended_msrs {
-        u32 eax;
-        u32 ebx;
-        u32 ecx;
-        u32 edx;
-        u32 esi;
-        u32 edi;
-        u32 ebp;
-        u32 esp;
-        u32 eflags;
-        u32 eip;
-        /* u32 *reserved[]; */
-};
-
-static int mce_num_extended_msrs;
-
-/* P4/Xeon Extended MCE MSR retrieval, return 0 if unsupported */
-static void intel_get_extended_msrs(struct intel_mce_extended_msrs *r)
-{
-        u32 h;
-
-        rdmsr(MSR_IA32_MCG_EAX, r->eax, h);
-        rdmsr(MSR_IA32_MCG_EBX, r->ebx, h);
-        rdmsr(MSR_IA32_MCG_ECX, r->ecx, h);
-        rdmsr(MSR_IA32_MCG_EDX, r->edx, h);
-        rdmsr(MSR_IA32_MCG_ESI, r->esi, h);
-        rdmsr(MSR_IA32_MCG_EDI, r->edi, h);
-        rdmsr(MSR_IA32_MCG_EBP, r->ebp, h);
-        rdmsr(MSR_IA32_MCG_ESP, r->esp, h);
-        rdmsr(MSR_IA32_MCG_EFLAGS, r->eflags, h);
-        rdmsr(MSR_IA32_MCG_EIP, r->eip, h);
-}
-
-static void intel_machine_check(struct pt_regs *regs, long error_code)
-{
-        u32 alow, ahigh, high, low;
-        u32 mcgstl, mcgsth;
-        int recover = 1;
-        int i;
-
-        rdmsr(MSR_IA32_MCG_STATUS, mcgstl, mcgsth);
-        if (mcgstl & (1<<0))    /* Recoverable ? */
-                recover = 0;
-
-        printk(KERN_EMERG "CPU %d: Machine Check Exception: %08x%08x\n",
-                smp_processor_id(), mcgsth, mcgstl);
-
-        if (mce_num_extended_msrs > 0) {
-                struct intel_mce_extended_msrs dbg;
-
-                intel_get_extended_msrs(&dbg);
-
-                printk(KERN_DEBUG "CPU %d: EIP: %08x EFLAGS: %08x\n"
-                        "\teax: %08x ebx: %08x ecx: %08x edx: %08x\n"
-                        "\tesi: %08x edi: %08x ebp: %08x esp: %08x\n",
-                        smp_processor_id(), dbg.eip, dbg.eflags,
-                        dbg.eax, dbg.ebx, dbg.ecx, dbg.edx,
-                        dbg.esi, dbg.edi, dbg.ebp, dbg.esp);
-        }
-
-        for (i = 0; i < nr_mce_banks; i++) {
-                rdmsr(MSR_IA32_MC0_STATUS+i*4, low, high);
-                if (high & (1<<31)) {
-                        char misc[20];
-                        char addr[24];
-
-                        misc[0] = addr[0] = '\0';
-                        if (high & (1<<29))
-                                recover |= 1;
-                        if (high & (1<<25))
-                                recover |= 2;
-                        high &= ~(1<<31);
-                        if (high & (1<<27)) {
-                                rdmsr(MSR_IA32_MC0_MISC+i*4, alow, ahigh);
-                                snprintf(misc, 20, "[%08x%08x]", ahigh, alow);
-                        }
-                        if (high & (1<<26)) {
-                                rdmsr(MSR_IA32_MC0_ADDR+i*4, alow, ahigh);
-                                snprintf(addr, 24, " at %08x%08x", ahigh, alow);
-                        }
-                        printk(KERN_EMERG "CPU %d: Bank %d: %08x%08x%s%s\n",
-                                smp_processor_id(), i, high, low, misc, addr);
-                }
-        }
-
-        if (recover & 2)
-                panic("CPU context corrupt");
-        if (recover & 1)
-                panic("Unable to continue");
-
-        printk(KERN_EMERG "Attempting to continue.\n");
-
-        /*
-         * Do not clear the MSR_IA32_MCi_STATUS if the error is not
-         * recoverable/continuable.This will allow BIOS to look at the MSRs
-         * for errors if the OS could not log the error.
-         */
-        for (i = 0; i < nr_mce_banks; i++) {
-                u32 msr;
-                msr = MSR_IA32_MC0_STATUS+i*4;
-                rdmsr(msr, low, high);
-                if (high&(1<<31)) {
-                        /* Clear it */
-                        wrmsr(msr, 0UL, 0UL);
-                        /* Serialize */
-                        wmb();
-                        add_taint(TAINT_MACHINE_CHECK);
-                }
-        }
-        mcgstl &= ~(1<<2);
-        wrmsr(MSR_IA32_MCG_STATUS, mcgstl, mcgsth);
-}
-
-void intel_p4_mcheck_init(struct cpuinfo_x86 *c)
-{
-        u32 l, h;
-        int i;
-
-        machine_check_vector = intel_machine_check;
-        wmb();
-
-        printk(KERN_INFO "Intel machine check architecture supported.\n");
-        rdmsr(MSR_IA32_MCG_CAP, l, h);
-        if (l & (1<<8)) /* Control register present ? */
-                wrmsr(MSR_IA32_MCG_CTL, 0xffffffff, 0xffffffff);
-        nr_mce_banks = l & 0xff;
-
-        for (i = 0; i < nr_mce_banks; i++) {
-                wrmsr(MSR_IA32_MC0_CTL+4*i, 0xffffffff, 0xffffffff);
-                wrmsr(MSR_IA32_MC0_STATUS+4*i, 0x0, 0x0);
-        }
-
-        set_in_cr4(X86_CR4_MCE);
-        printk(KERN_INFO "Intel machine check reporting enabled on CPU#%d.\n",
-                smp_processor_id());
-
-        /* Check for P4/Xeon extended MCE MSRs */
-        rdmsr(MSR_IA32_MCG_CAP, l, h);
-        if (l & (1<<9)) {/* MCG_EXT_P */
-                mce_num_extended_msrs = (l >> 16) & 0xff;
-                printk(KERN_INFO "CPU%d: Intel P4/Xeon Extended MCE MSRs (%d)"
-                                " available\n",
-                        smp_processor_id(), mce_num_extended_msrs);
-
-#ifdef CONFIG_X86_MCE_P4THERMAL
-                /* Check for P4/Xeon Thermal monitor */
-                intel_init_thermal(c);
-#endif
-        }
-}
diff --git a/arch/x86/kernel/cpu/mcheck/p6.c b/arch/x86/kernel/cpu/mcheck/p6.c
deleted file mode 100644
index 01e4f8178183..000000000000
--- a/arch/x86/kernel/cpu/mcheck/p6.c
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- * P6 specific Machine Check Exception Reporting
- * (C) Copyright 2002 Alan Cox <alan@lxorguk.ukuu.org.uk>
- */
-#include <linux/interrupt.h>
-#include <linux/kernel.h>
-#include <linux/types.h>
-#include <linux/init.h>
-#include <linux/smp.h>
-
-#include <asm/processor.h>
-#include <asm/system.h>
-#include <asm/mce.h>
-#include <asm/msr.h>
-
-/* Machine Check Handler For PII/PIII */
-static void intel_machine_check(struct pt_regs *regs, long error_code)
-{
-        u32 alow, ahigh, high, low;
-        u32 mcgstl, mcgsth;
-        int recover = 1;
-        int i;
-
-        rdmsr(MSR_IA32_MCG_STATUS, mcgstl, mcgsth);
-        if (mcgstl & (1<<0))    /* Recoverable ? */
-                recover = 0;
-
-        printk(KERN_EMERG "CPU %d: Machine Check Exception: %08x%08x\n",
-                smp_processor_id(), mcgsth, mcgstl);
-
-        for (i = 0; i < nr_mce_banks; i++) {
-                rdmsr(MSR_IA32_MC0_STATUS+i*4, low, high);
-                if (high & (1<<31)) {
-                        char misc[20];
-                        char addr[24];
-
-                        misc[0] = '\0';
-                        addr[0] = '\0';
-
-                        if (high & (1<<29))
-                                recover |= 1;
-                        if (high & (1<<25))
-                                recover |= 2;
-                        high &= ~(1<<31);
-
-                        if (high & (1<<27)) {
-                                rdmsr(MSR_IA32_MC0_MISC+i*4, alow, ahigh);
-                                snprintf(misc, 20, "[%08x%08x]", ahigh, alow);
-                        }
-                        if (high & (1<<26)) {
-                                rdmsr(MSR_IA32_MC0_ADDR+i*4, alow, ahigh);
-                                snprintf(addr, 24, " at %08x%08x", ahigh, alow);
-                        }
-
-                        printk(KERN_EMERG "CPU %d: Bank %d: %08x%08x%s%s\n",
-                                smp_processor_id(), i, high, low, misc, addr);
-                }
-        }
-
-        if (recover & 2)
-                panic("CPU context corrupt");
-        if (recover & 1)
-                panic("Unable to continue");
-
-        printk(KERN_EMERG "Attempting to continue.\n");
-        /*
-         * Do not clear the MSR_IA32_MCi_STATUS if the error is not
-         * recoverable/continuable.This will allow BIOS to look at the MSRs
-         * for errors if the OS could not log the error:
-         */
-        for (i = 0; i < nr_mce_banks; i++) {
-                unsigned int msr;
-
-                msr = MSR_IA32_MC0_STATUS+i*4;
-                rdmsr(msr, low, high);
-                if (high & (1<<31)) {
-                        /* Clear it: */
-                        wrmsr(msr, 0UL, 0UL);
-                        /* Serialize: */
-                        wmb();
-                        add_taint(TAINT_MACHINE_CHECK);
-                }
-        }
-        mcgstl &= ~(1<<2);
-        wrmsr(MSR_IA32_MCG_STATUS, mcgstl, mcgsth);
-}
-
-/* Set up machine check reporting for processors with Intel style MCE: */
-void intel_p6_mcheck_init(struct cpuinfo_x86 *c)
-{
-        u32 l, h;
-        int i;
-
-        /* Check for MCE support */
-        if (!cpu_has(c, X86_FEATURE_MCE))
-                return;
-
-        /* Check for PPro style MCA */
-        if (!cpu_has(c, X86_FEATURE_MCA))
-                return;
-
-        /* Ok machine check is available */
-        machine_check_vector = intel_machine_check;
-        /* Make sure the vector pointer is visible before we enable MCEs: */
-        wmb();
-
-        printk(KERN_INFO "Intel machine check architecture supported.\n");
-        rdmsr(MSR_IA32_MCG_CAP, l, h);
-        if (l & (1<<8)) /* Control register present ? */
-                wrmsr(MSR_IA32_MCG_CTL, 0xffffffff, 0xffffffff);
-        nr_mce_banks = l & 0xff;
-
-        /*
-         * Following the example in IA-32 SDM Vol 3:
-         * - MC0_CTL should not be written
-         * - Status registers on all banks should be cleared on reset
-         */
-        for (i = 1; i < nr_mce_banks; i++)
-                wrmsr(MSR_IA32_MC0_CTL+4*i, 0xffffffff, 0xffffffff);
-
-        for (i = 0; i < nr_mce_banks; i++)
-                wrmsr(MSR_IA32_MC0_STATUS+4*i, 0x0, 0x0);
-
-        set_in_cr4(X86_CR4_MCE);
-        printk(KERN_INFO "Intel machine check reporting enabled on CPU#%d.\n",
-                smp_processor_id());
-}
diff --git a/arch/x86/kernel/cpu/mcheck/therm_throt.c b/arch/x86/kernel/cpu/mcheck/therm_throt.c
index 5957a93e5173..63a56d147e4a 100644
--- a/arch/x86/kernel/cpu/mcheck/therm_throt.c
+++ b/arch/x86/kernel/cpu/mcheck/therm_throt.c
@@ -260,9 +260,6 @@ void intel_init_thermal(struct cpuinfo_x86 *c)
                 return;
         }
 
-        if (cpu_has(c, X86_FEATURE_TM2) && (l & MSR_IA32_MISC_ENABLE_TM2))
-                tm2 = 1;
-
         /* Check whether a vector already exists */
         if (h & APIC_VECTOR_MASK) {
                 printk(KERN_DEBUG
@@ -271,6 +268,16 @@ void intel_init_thermal(struct cpuinfo_x86 *c)
                 return;
         }
 
+        /* early Pentium M models use different method for enabling TM2 */
+        if (cpu_has(c, X86_FEATURE_TM2)) {
+                if (c->x86 == 6 && (c->x86_model == 9 || c->x86_model == 13)) {
+                        rdmsr(MSR_THERM2_CTL, l, h);
+                        if (l & MSR_THERM2_CTL_TM_SELECT)
+                                tm2 = 1;
+                } else if (l & MSR_IA32_MISC_ENABLE_TM2)
+                        tm2 = 1;
+        }
+
         /* We'll mask the thermal vector in the lapic till we're ready: */
         h = THERMAL_APIC_VECTOR | APIC_DM_FIXED | APIC_LVT_MASKED;
         apic_write(APIC_LVTTHMR, h);
diff --git a/arch/x86/kernel/cpu/mtrr/amd.c b/arch/x86/kernel/cpu/mtrr/amd.c
index ee2331b0e58f..33af14110dfd 100644
--- a/arch/x86/kernel/cpu/mtrr/amd.c
+++ b/arch/x86/kernel/cpu/mtrr/amd.c
@@ -7,15 +7,15 @@
 
 static void
 amd_get_mtrr(unsigned int reg, unsigned long *base,
-             unsigned long *size, mtrr_type * type)
+             unsigned long *size, mtrr_type *type)
 {
         unsigned long low, high;
 
         rdmsr(MSR_K6_UWCCR, low, high);
-        /*  Upper dword is region 1, lower is region 0  */
+        /* Upper dword is region 1, lower is region 0 */
         if (reg == 1)
                 low = high;
-        /*  The base masks off on the right alignment  */
+        /* The base masks off on the right alignment */
         *base = (low & 0xFFFE0000) >> PAGE_SHIFT;
         *type = 0;
         if (low & 1)
@@ -27,74 +27,81 @@ amd_get_mtrr(unsigned int reg, unsigned long *base,
                 return;
         }
         /*
-         *  This needs a little explaining. The size is stored as an
-         *  inverted mask of bits of 128K granularity 15 bits long offset
-         *  2 bits
+         * This needs a little explaining. The size is stored as an
+         * inverted mask of bits of 128K granularity 15 bits long offset
+         * 2 bits.
          *
-         *  So to get a size we do invert the mask and add 1 to the lowest
-         *  mask bit (4 as its 2 bits in). This gives us a size we then shift
-         *  to turn into 128K blocks
+         * So to get a size we do invert the mask and add 1 to the lowest
+         * mask bit (4 as its 2 bits in). This gives us a size we then shift
+         * to turn into 128K blocks.
          *
-         *  eg              111 1111 1111 1100      is 512K
+         * eg              111 1111 1111 1100      is 512K
          *
-         *  invert          000 0000 0000 0011
-         *  +1              000 0000 0000 0100
-         *  *128K   ...
+         * invert          000 0000 0000 0011
+         * +1              000 0000 0000 0100
+         * *128K   ...
          */
         low = (~low) & 0x1FFFC;
         *size = (low + 4) << (15 - PAGE_SHIFT);
-        return;
 }
 
-static void amd_set_mtrr(unsigned int reg, unsigned long base,
-                         unsigned long size, mtrr_type type)
-/*  [SUMMARY] Set variable MTRR register on the local CPU.
-    <reg> The register to set.
-    <base> The base address of the region.
-    <size> The size of the region. If this is 0 the region is disabled.
-    <type> The type of the region.
-    [RETURNS] Nothing.
-*/
+/**
+ * amd_set_mtrr - Set variable MTRR register on the local CPU.
+ *
+ * @reg The register to set.
+ * @base The base address of the region.
+ * @size The size of the region. If this is 0 the region is disabled.
+ * @type The type of the region.
+ *
+ * Returns nothing.
+ */
+static void
+amd_set_mtrr(unsigned int reg, unsigned long base, unsigned long size, mtrr_type type)
 {
         u32 regs[2];
 
         /*
-         *  Low is MTRR0 , High MTRR 1
+         * Low is MTRR0, High MTRR 1
          */
         rdmsr(MSR_K6_UWCCR, regs[0], regs[1]);
         /*
-         *  Blank to disable
+         * Blank to disable
          */
-        if (size == 0)
+        if (size == 0) {
                 regs[reg] = 0;
-        else
-                /* Set the register to the base, the type (off by one) and an
-                   inverted bitmask of the size The size is the only odd
-                   bit. We are fed say 512K We invert this and we get 111 1111
-                   1111 1011 but if you subtract one and invert you get the   
-                   desired 111 1111 1111 1100 mask
-
-                   But ~(x - 1) == ~x + 1 == -x. Two's complement rocks!  */
+        } else {
+                /*
+                 * Set the register to the base, the type (off by one) and an
+                 * inverted bitmask of the size The size is the only odd
+                 * bit. We are fed say 512K We invert this and we get 111 1111
+                 * 1111 1011 but if you subtract one and invert you get the
+                 * desired 111 1111 1111 1100 mask
+                 *
+                 *  But ~(x - 1) == ~x + 1 == -x. Two's complement rocks!
+                 */
                 regs[reg] = (-size >> (15 - PAGE_SHIFT) & 0x0001FFFC)
                     | (base << PAGE_SHIFT) | (type + 1);
+        }
 
         /*
-         *  The writeback rule is quite specific. See the manual. Its
-         *  disable local interrupts, write back the cache, set the mtrr
+         * The writeback rule is quite specific. See the manual. Its
+         * disable local interrupts, write back the cache, set the mtrr
          */
         wbinvd();
         wrmsr(MSR_K6_UWCCR, regs[0], regs[1]);
 }
 
-static int amd_validate_add_page(unsigned long base, unsigned long size, unsigned int type)
+static int
+amd_validate_add_page(unsigned long base, unsigned long size, unsigned int type)
 {
-        /* Apply the K6 block alignment and size rules
-           In order
-           o Uncached or gathering only
-           o 128K or bigger block
-           o Power of 2 block
-           o base suitably aligned to the power
-        */
+        /*
+         * Apply the K6 block alignment and size rules
+         * In order
+         * o Uncached or gathering only
+         * o 128K or bigger block
+         * o Power of 2 block
+         * o base suitably aligned to the power
+         */
         if (type > MTRR_TYPE_WRCOMB || size < (1 << (17 - PAGE_SHIFT))
             || (size & ~(size - 1)) - size || (base & (size - 1)))
                 return -EINVAL;
@@ -115,5 +122,3 @@ int __init amd_init_mtrr(void)
         set_mtrr_ops(&amd_mtrr_ops);
         return 0;
 }
-
-//arch_initcall(amd_mtrr_init);
diff --git a/arch/x86/kernel/cpu/mtrr/centaur.c b/arch/x86/kernel/cpu/mtrr/centaur.c
index cb9aa3a7a7ab..de89f14eff3a 100644
--- a/arch/x86/kernel/cpu/mtrr/centaur.c
+++ b/arch/x86/kernel/cpu/mtrr/centaur.c
@@ -1,7 +1,9 @@
 #include <linux/init.h>
 #include <linux/mm.h>
+
 #include <asm/mtrr.h>
 #include <asm/msr.h>
+
 #include "mtrr.h"
 
 static struct {
@@ -12,25 +14,25 @@ static struct {
 static u8 centaur_mcr_reserved;
 static u8 centaur_mcr_type;     /* 0 for winchip, 1 for winchip2 */
 
-/*
- *      Report boot time MCR setups 
+/**
+ * centaur_get_free_region - Get a free MTRR.
+ *
+ * @base: The starting (base) address of the region.
+ * @size: The size (in bytes) of the region.
+ *
+ * Returns: the index of the region on success, else -1 on error.
  */
-
 static int
 centaur_get_free_region(unsigned long base, unsigned long size, int replace_reg)
-/*  [SUMMARY] Get a free MTRR.
-    <base> The starting (base) address of the region.
-    <size> The size (in bytes) of the region.
-    [RETURNS] The index of the region on success, else -1 on error.
-*/
 {
-        int i, max;
-        mtrr_type ltype;
         unsigned long lbase, lsize;
+        mtrr_type ltype;
+        int i, max;
 
         max = num_var_ranges;
         if (replace_reg >= 0 && replace_reg < max)
                 return replace_reg;
+
         for (i = 0; i < max; ++i) {
                 if (centaur_mcr_reserved & (1 << i))
                         continue;
@@ -38,11 +40,14 @@ centaur_get_free_region(unsigned long base, unsigned long size, int replace_reg)
                 if (lsize == 0)
                         return i;
         }
+
         return -ENOSPC;
 }
 
-void
-mtrr_centaur_report_mcr(int mcr, u32 lo, u32 hi)
+/*
+ * Report boot time MCR setups
+ */
+void mtrr_centaur_report_mcr(int mcr, u32 lo, u32 hi)
 {
         centaur_mcr[mcr].low = lo;
         centaur_mcr[mcr].high = hi;
@@ -54,33 +59,35 @@ centaur_get_mcr(unsigned int reg, unsigned long *base,
 {
         *base = centaur_mcr[reg].high >> PAGE_SHIFT;
         *size = -(centaur_mcr[reg].low & 0xfffff000) >> PAGE_SHIFT;
-        *type = MTRR_TYPE_WRCOMB;       /*  If it is there, it is write-combining  */
+        *type = MTRR_TYPE_WRCOMB;               /* write-combining  */
+
         if (centaur_mcr_type == 1 && ((centaur_mcr[reg].low & 31) & 2))
                 *type = MTRR_TYPE_UNCACHABLE;
         if (centaur_mcr_type == 1 && (centaur_mcr[reg].low & 31) == 25)
                 *type = MTRR_TYPE_WRBACK;
         if (centaur_mcr_type == 0 && (centaur_mcr[reg].low & 31) == 31)
                 *type = MTRR_TYPE_WRBACK;
-
 }
 
-static void centaur_set_mcr(unsigned int reg, unsigned long base,
-                            unsigned long size, mtrr_type type)
+static void
+centaur_set_mcr(unsigned int reg, unsigned long base,
+                unsigned long size, mtrr_type type)
 {
         unsigned long low, high;
 
         if (size == 0) {
-                /*  Disable  */
+                /* Disable */
                 high = low = 0;
         } else {
                 high = base << PAGE_SHIFT;
-                if (centaur_mcr_type == 0)
-                        low = -size << PAGE_SHIFT | 0x1f;       /* only support write-combining... */
-                else {
+                if (centaur_mcr_type == 0) {
+                        /* Only support write-combining... */
+                        low = -size << PAGE_SHIFT | 0x1f;
+                } else {
                         if (type == MTRR_TYPE_UNCACHABLE)
-                                low = -size << PAGE_SHIFT | 0x02;       /* NC */
+                                low = -size << PAGE_SHIFT | 0x02; /* NC */
                         else
-                                low = -size << PAGE_SHIFT | 0x09;       /* WWO,WC */
+                                low = -size << PAGE_SHIFT | 0x09; /* WWO, WC */
                 }
         }
         centaur_mcr[reg].high = high;
@@ -88,118 +95,16 @@ static void centaur_set_mcr(unsigned int reg, unsigned long base,
         wrmsr(MSR_IDT_MCR0 + reg, low, high);
 }
 
-#if 0
-/*
- *      Initialise the later (saner) Winchip MCR variant. In this version
- *      the BIOS can pass us the registers it has used (but not their values)
- *      and the control register is read/write
- */
-
-static void __init
-centaur_mcr1_init(void)
-{
-        unsigned i;
-        u32 lo, hi;
-
-        /* Unfortunately, MCR's are read-only, so there is no way to
-         * find out what the bios might have done.
-         */
-
-        rdmsr(MSR_IDT_MCR_CTRL, lo, hi);
-        if (((lo >> 17) & 7) == 1) {    /* Type 1 Winchip2 MCR */
-                lo &= ~0x1C0;   /* clear key */
-                lo |= 0x040;    /* set key to 1 */
-                wrmsr(MSR_IDT_MCR_CTRL, lo, hi);        /* unlock MCR */
-        }
-
-        centaur_mcr_type = 1;
-
-        /*
-         *  Clear any unconfigured MCR's.
-         */
-
-        for (i = 0; i < 8; ++i) {
-                if (centaur_mcr[i].high == 0 && centaur_mcr[i].low == 0) {
-                        if (!(lo & (1 << (9 + i))))
-                                wrmsr(MSR_IDT_MCR0 + i, 0, 0);
-                        else
-                                /*
-                                 *      If the BIOS set up an MCR we cannot see it
-                                 *      but we don't wish to obliterate it
-                                 */
-                                centaur_mcr_reserved |= (1 << i);
-                }
-        }
-        /*  
-         *  Throw the main write-combining switch... 
-         *  However if OOSTORE is enabled then people have already done far
-         *  cleverer things and we should behave. 
-         */
-
-        lo |= 15;               /* Write combine enables */
-        wrmsr(MSR_IDT_MCR_CTRL, lo, hi);
-}
-
-/*
- *      Initialise the original winchip with read only MCR registers
- *      no used bitmask for the BIOS to pass on and write only control
- */
-
-static void __init
-centaur_mcr0_init(void)
-{
-        unsigned i;
-
-        /* Unfortunately, MCR's are read-only, so there is no way to
-         * find out what the bios might have done.
-         */
-
-        /* Clear any unconfigured MCR's.
-         * This way we are sure that the centaur_mcr array contains the actual
-         * values. The disadvantage is that any BIOS tweaks are thus undone.
-         *
-         */
-        for (i = 0; i < 8; ++i) {
-                if (centaur_mcr[i].high == 0 && centaur_mcr[i].low == 0)
-                        wrmsr(MSR_IDT_MCR0 + i, 0, 0);
-        }
-
-        wrmsr(MSR_IDT_MCR_CTRL, 0x01F0001F, 0); /* Write only */
-}
-
-/*
- *      Initialise Winchip series MCR registers
- */
-
-static void __init
-centaur_mcr_init(void)
-{
-        struct set_mtrr_context ctxt;
-
-        set_mtrr_prepare_save(&ctxt);
-        set_mtrr_cache_disable(&ctxt);
-
-        if (boot_cpu_data.x86_model == 4)
-                centaur_mcr0_init();
-        else if (boot_cpu_data.x86_model == 8 || boot_cpu_data.x86_model == 9)
-                centaur_mcr1_init();
-
-        set_mtrr_done(&ctxt);
-}
-#endif
-
-static int centaur_validate_add_page(unsigned long base, 
-                                     unsigned long size, unsigned int type)
+static int
+centaur_validate_add_page(unsigned long base, unsigned long size, unsigned int type)
 {
         /*
-         *  FIXME: Winchip2 supports uncached
+         * FIXME: Winchip2 supports uncached
          */
-        if (type != MTRR_TYPE_WRCOMB && 
+        if (type != MTRR_TYPE_WRCOMB &&
             (centaur_mcr_type == 0 || type != MTRR_TYPE_UNCACHABLE)) {
-                printk(KERN_WARNING
-                       "mtrr: only write-combining%s supported\n",
-                       centaur_mcr_type ? " and uncacheable are"
-                       : " is");
+                pr_warning("mtrr: only write-combining%s supported\n",
+                           centaur_mcr_type ? " and uncacheable are" : " is");
                 return -EINVAL;
         }
         return 0;
@@ -207,7 +112,6 @@ static int centaur_validate_add_page(unsigned long base,
 
 static struct mtrr_ops centaur_mtrr_ops = {
         .vendor            = X86_VENDOR_CENTAUR,
-//      .init              = centaur_mcr_init,
         .set               = centaur_set_mcr,
         .get               = centaur_get_mcr,
         .get_free_region   = centaur_get_free_region,
@@ -220,5 +124,3 @@ int __init centaur_init_mtrr(void)
         set_mtrr_ops(&centaur_mtrr_ops);
         return 0;
 }
-
-//arch_initcall(centaur_init_mtrr);
diff --git a/arch/x86/kernel/cpu/mtrr/cleanup.c b/arch/x86/kernel/cpu/mtrr/cleanup.c
index 1d584a18a50d..315738c74aad 100644
--- a/arch/x86/kernel/cpu/mtrr/cleanup.c
+++ b/arch/x86/kernel/cpu/mtrr/cleanup.c
@@ -1,51 +1,75 @@
-/*  MTRR (Memory Type Range Register) cleanup
-
-    Copyright (C) 2009 Yinghai Lu
-
-    This library is free software; you can redistribute it and/or
-    modify it under the terms of the GNU Library General Public
-    License as published by the Free Software Foundation; either
-    version 2 of the License, or (at your option) any later version.
-
-    This library is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-    Library General Public License for more details.
-
-    You should have received a copy of the GNU Library General Public
-    License along with this library; if not, write to the Free
-    Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-*/
-
+/*
+ * MTRR (Memory Type Range Register) cleanup
+ *
+ *  Copyright (C) 2009 Yinghai Lu
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free
+ * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
 #include <linux/module.h>
 #include <linux/init.h>
 #include <linux/pci.h>
 #include <linux/smp.h>
 #include <linux/cpu.h>
-#include <linux/mutex.h>
 #include <linux/sort.h>
+#include <linux/mutex.h>
+#include <linux/uaccess.h>
+#include <linux/kvm_para.h>
 
+#include <asm/processor.h>
 #include <asm/e820.h>
 #include <asm/mtrr.h>
-#include <asm/uaccess.h>
-#include <asm/processor.h>
 #include <asm/msr.h>
-#include <asm/kvm_para.h>
-#include "mtrr.h"
 
-/* should be related to MTRR_VAR_RANGES nums */
-#define RANGE_NUM 256
+#include "mtrr.h"
 
 struct res_range {
-        unsigned long start;
-        unsigned long end;
+        unsigned long   start;
+        unsigned long   end;
+};
+
+struct var_mtrr_range_state {
+        unsigned long   base_pfn;
+        unsigned long   size_pfn;
+        mtrr_type       type;
+};
+
+struct var_mtrr_state {
+        unsigned long   range_startk;
+        unsigned long   range_sizek;
+        unsigned long   chunk_sizek;
+        unsigned long   gran_sizek;
+        unsigned int    reg;
 };
 
+/* Should be related to MTRR_VAR_RANGES nums */
+#define RANGE_NUM                               256
+
+static struct res_range __initdata              range[RANGE_NUM];
+static int __initdata                           nr_range;
+
+static struct var_mtrr_range_state __initdata   range_state[RANGE_NUM];
+
+static int __initdata debug_print;
+#define Dprintk(x...) do { if (debug_print) printk(KERN_DEBUG x); } while (0)
+
+
 static int __init
-add_range(struct res_range *range, int nr_range, unsigned long start,
-                              unsigned long end)
+add_range(struct res_range *range, int nr_range,
+          unsigned long start, unsigned long end)
 {
-        /* out of slots */
+        /* Out of slots: */
         if (nr_range >= RANGE_NUM)
                 return nr_range;
 
@@ -58,12 +82,12 @@ add_range(struct res_range *range, int nr_range, unsigned long start,
 }
 
 static int __init
-add_range_with_merge(struct res_range *range, int nr_range, unsigned long start,
-                              unsigned long end)
+add_range_with_merge(struct res_range *range, int nr_range,
+                     unsigned long start, unsigned long end)
 {
         int i;
 
-        /* try to merge it with old one */
+        /* Try to merge it with old one: */
         for (i = 0; i < nr_range; i++) {
                 unsigned long final_start, final_end;
                 unsigned long common_start, common_end;
@@ -84,7 +108,7 @@ add_range_with_merge(struct res_range *range, int nr_range, unsigned long start,
                 return nr_range;
         }
 
-        /* need to add that */
+        /* Need to add it: */
         return add_range(range, nr_range, start, end);
 }
 
@@ -117,7 +141,7 @@ subtract_range(struct res_range *range, unsigned long start, unsigned long end)
                 }
 
                 if (start > range[j].start && end < range[j].end) {
-                        /* find the new spare */
+                        /* Find the new spare: */
                         for (i = 0; i < RANGE_NUM; i++) {
                                 if (range[i].end == 0)
                                         break;
@@ -146,14 +170,8 @@ static int __init cmp_range(const void *x1, const void *x2)
         return start1 - start2;
 }
 
-struct var_mtrr_range_state {
-        unsigned long base_pfn;
-        unsigned long size_pfn;
-        mtrr_type type;
-};
-
-static struct var_mtrr_range_state __initdata range_state[RANGE_NUM];
-static int __initdata debug_print;
+#define BIOS_BUG_MSG KERN_WARNING \
+        "WARNING: BIOS bug: VAR MTRR %d contains strange UC entry under 1M, check with your system vendor!\n"
 
 static int __init
 x86_get_mtrr_mem_range(struct res_range *range, int nr_range,
@@ -180,7 +198,7 @@ x86_get_mtrr_mem_range(struct res_range *range, int nr_range,
                                  range[i].start, range[i].end + 1);
         }
 
-        /* take out UC ranges */
+        /* Take out UC ranges: */
         for (i = 0; i < num_var_ranges; i++) {
                 type = range_state[i].type;
                 if (type != MTRR_TYPE_UNCACHABLE &&
@@ -193,9 +211,7 @@ x86_get_mtrr_mem_range(struct res_range *range, int nr_range,
                 if (base < (1<<(20-PAGE_SHIFT)) && mtrr_state.have_fixed &&
                     (mtrr_state.enabled & 1)) {
                         /* Var MTRR contains UC entry below 1M? Skip it: */
-                        printk(KERN_WARNING "WARNING: BIOS bug: VAR MTRR %d "
-                                "contains strange UC entry under 1M, check "
-                                "with your system vendor!\n", i);
+                        printk(BIOS_BUG_MSG, i);
                         if (base + size <= (1<<(20-PAGE_SHIFT)))
                                 continue;
                         size -= (1<<(20-PAGE_SHIFT)) - base;
@@ -237,17 +253,13 @@ x86_get_mtrr_mem_range(struct res_range *range, int nr_range,
         return nr_range;
 }
 
-static struct res_range __initdata range[RANGE_NUM];
-static int __initdata nr_range;
-
 #ifdef CONFIG_MTRR_SANITIZER
 
 static unsigned long __init sum_ranges(struct res_range *range, int nr_range)
 {
-        unsigned long sum;
+        unsigned long sum = 0;
         int i;
 
-        sum = 0;
         for (i = 0; i < nr_range; i++)
                 sum += range[i].end + 1 - range[i].start;
 
@@ -278,17 +290,9 @@ static int __init mtrr_cleanup_debug_setup(char *str)
 }
 early_param("mtrr_cleanup_debug", mtrr_cleanup_debug_setup);
 
-struct var_mtrr_state {
-        unsigned long   range_startk;
-        unsigned long   range_sizek;
-        unsigned long   chunk_sizek;
-        unsigned long   gran_sizek;
-        unsigned int    reg;
-};
-
 static void __init
 set_var_mtrr(unsigned int reg, unsigned long basek, unsigned long sizek,
-                unsigned char type, unsigned int address_bits)
+             unsigned char type, unsigned int address_bits)
 {
         u32 base_lo, base_hi, mask_lo, mask_hi;
         u64 base, mask;
@@ -301,7 +305,7 @@ set_var_mtrr(unsigned int reg, unsigned long basek, unsigned long sizek,
         mask = (1ULL << address_bits) - 1;
         mask &= ~((((u64)sizek) << 10) - 1);
 
-        base  = ((u64)basek) << 10;
+        base = ((u64)basek) << 10;
 
         base |= type;
         mask |= 0x800;
@@ -317,15 +321,14 @@ set_var_mtrr(unsigned int reg, unsigned long basek, unsigned long sizek,
 
 static void __init
 save_var_mtrr(unsigned int reg, unsigned long basek, unsigned long sizek,
-                unsigned char type)
+              unsigned char type)
 {
         range_state[reg].base_pfn = basek >> (PAGE_SHIFT - 10);
         range_state[reg].size_pfn = sizek >> (PAGE_SHIFT - 10);
         range_state[reg].type = type;
 }
 
-static void __init
-set_var_mtrr_all(unsigned int address_bits)
+static void __init set_var_mtrr_all(unsigned int address_bits)
 {
         unsigned long basek, sizek;
         unsigned char type;
@@ -342,11 +345,11 @@ set_var_mtrr_all(unsigned int address_bits)
 
 static unsigned long to_size_factor(unsigned long sizek, char *factorp)
 {
-        char factor;
         unsigned long base = sizek;
+        char factor;
 
         if (base & ((1<<10) - 1)) {
-                /* not MB alignment */
+                /* Not MB-aligned: */
                 factor = 'K';
         } else if (base & ((1<<20) - 1)) {
                 factor = 'M';
@@ -372,11 +375,12 @@ range_to_mtrr(unsigned int reg, unsigned long range_startk,
                 unsigned long max_align, align;
                 unsigned long sizek;
 
-                /* Compute the maximum size I can make a range */
+                /* Compute the maximum size with which we can make a range: */
                 if (range_startk)
                         max_align = ffs(range_startk) - 1;
                 else
                         max_align = 32;
+
                 align = fls(range_sizek) - 1;
                 if (align > max_align)
                         align = max_align;
@@ -386,11 +390,10 @@ range_to_mtrr(unsigned int reg, unsigned long range_startk,
                         char start_factor = 'K', size_factor = 'K';
                         unsigned long start_base, size_base;
 
-                        start_base = to_size_factor(range_startk,
-                                                         &start_factor),
-                        size_base = to_size_factor(sizek, &size_factor),
+                        start_base = to_size_factor(range_startk, &start_factor);
+                        size_base = to_size_factor(sizek, &size_factor);
 
-                        printk(KERN_DEBUG "Setting variable MTRR %d, "
+                        Dprintk("Setting variable MTRR %d, "
                                 "base: %ld%cB, range: %ld%cB, type %s\n",
                                 reg, start_base, start_factor,
                                 size_base, size_factor,
@@ -425,10 +428,11 @@ range_to_mtrr_with_hole(struct var_mtrr_state *state, unsigned long basek,
         chunk_sizek = state->chunk_sizek;
         gran_sizek = state->gran_sizek;
 
-        /* align with gran size, prevent small block used up MTRRs */
+        /* Align with gran size, prevent small block used up MTRRs: */
         range_basek = ALIGN(state->range_startk, gran_sizek);
         if ((range_basek > basek) && basek)
                 return second_sizek;
+
         state->range_sizek -= (range_basek - state->range_startk);
         range_sizek = ALIGN(state->range_sizek, gran_sizek);
 
@@ -439,22 +443,21 @@ range_to_mtrr_with_hole(struct var_mtrr_state *state, unsigned long basek,
         }
         state->range_sizek = range_sizek;
 
-        /* try to append some small hole */
+        /* Try to append some small hole: */
         range0_basek = state->range_startk;
         range0_sizek = ALIGN(state->range_sizek, chunk_sizek);
 
-        /* no increase */
+        /* No increase: */
         if (range0_sizek == state->range_sizek) {
-                if (debug_print)
-                        printk(KERN_DEBUG "rangeX: %016lx - %016lx\n",
-                                range0_basek<<10,
-                                (range0_basek + state->range_sizek)<<10);
+                Dprintk("rangeX: %016lx - %016lx\n",
+                        range0_basek<<10,
+                        (range0_basek + state->range_sizek)<<10);
                 state->reg = range_to_mtrr(state->reg, range0_basek,
                                 state->range_sizek, MTRR_TYPE_WRBACK);
                 return 0;
         }
 
-        /* only cut back, when it is not the last */
+        /* Only cut back when it is not the last: */
         if (sizek) {
                 while (range0_basek + range0_sizek > (basek + sizek)) {
                         if (range0_sizek >= chunk_sizek)
@@ -470,16 +473,16 @@ range_to_mtrr_with_hole(struct var_mtrr_state *state, unsigned long basek,
 second_try:
         range_basek = range0_basek + range0_sizek;
 
-        /* one hole in the middle */
+        /* One hole in the middle: */
         if (range_basek > basek && range_basek <= (basek + sizek))
                 second_sizek = range_basek - basek;
 
         if (range0_sizek > state->range_sizek) {
 
-                /* one hole in middle or at end */
+                /* One hole in middle or at the end: */
                 hole_sizek = range0_sizek - state->range_sizek - second_sizek;
 
-                /* hole size should be less than half of range0 size */
+                /* Hole size should be less than half of range0 size: */
                 if (hole_sizek >= (range0_sizek >> 1) &&
                     range0_sizek >= chunk_sizek) {
                         range0_sizek -= chunk_sizek;
@@ -491,32 +494,30 @@ second_try:
         }
 
         if (range0_sizek) {
-                if (debug_print)
-                        printk(KERN_DEBUG "range0: %016lx - %016lx\n",
-                                range0_basek<<10,
-                                (range0_basek + range0_sizek)<<10);
+                Dprintk("range0: %016lx - %016lx\n",
+                        range0_basek<<10,
+                        (range0_basek + range0_sizek)<<10);
                 state->reg = range_to_mtrr(state->reg, range0_basek,
                                 range0_sizek, MTRR_TYPE_WRBACK);
         }
 
         if (range0_sizek < state->range_sizek) {
-                /* need to handle left over */
+                /* Need to handle left over range: */
                 range_sizek = state->range_sizek - range0_sizek;
 
-                if (debug_print)
-                        printk(KERN_DEBUG "range: %016lx - %016lx\n",
-                                 range_basek<<10,
-                                 (range_basek + range_sizek)<<10);
+                Dprintk("range: %016lx - %016lx\n",
+                         range_basek<<10,
+                         (range_basek + range_sizek)<<10);
+
                 state->reg = range_to_mtrr(state->reg, range_basek,
                                  range_sizek, MTRR_TYPE_WRBACK);
         }
 
         if (hole_sizek) {
                 hole_basek = range_basek - hole_sizek - second_sizek;
-                if (debug_print)
-                        printk(KERN_DEBUG "hole: %016lx - %016lx\n",
-                                 hole_basek<<10,
-                                 (hole_basek + hole_sizek)<<10);
+                Dprintk("hole: %016lx - %016lx\n",
+                         hole_basek<<10,
+                         (hole_basek + hole_sizek)<<10);
                 state->reg = range_to_mtrr(state->reg, hole_basek,
                                  hole_sizek, MTRR_TYPE_UNCACHABLE);
         }
@@ -537,23 +538,23 @@ set_var_mtrr_range(struct var_mtrr_state *state, unsigned long base_pfn,
         basek = base_pfn << (PAGE_SHIFT - 10);
         sizek = size_pfn << (PAGE_SHIFT - 10);
 
-        /* See if I can merge with the last range */
+        /* See if I can merge with the last range: */
         if ((basek <= 1024) ||
             (state->range_startk + state->range_sizek == basek)) {
                 unsigned long endk = basek + sizek;
                 state->range_sizek = endk - state->range_startk;
                 return;
         }
-        /* Write the range mtrrs */
+        /* Write the range mtrrs: */
         if (state->range_sizek != 0)
                 second_sizek = range_to_mtrr_with_hole(state, basek, sizek);
 
-        /* Allocate an msr */
+        /* Allocate an msr: */
         state->range_startk = basek + second_sizek;
         state->range_sizek  = sizek - second_sizek;
 }
 
-/* mininum size of mtrr block that can take hole */
+/* Mininum size of mtrr block that can take hole: */
 static u64 mtrr_chunk_size __initdata = (256ULL<<20);
 
 static int __init parse_mtrr_chunk_size_opt(char *p)
@@ -565,7 +566,7 @@ static int __init parse_mtrr_chunk_size_opt(char *p)
 }
 early_param("mtrr_chunk_size", parse_mtrr_chunk_size_opt);
 
-/* granity of mtrr of block */
+/* Granularity of mtrr of block: */
 static u64 mtrr_gran_size __initdata;
 
 static int __init parse_mtrr_gran_size_opt(char *p)
@@ -577,7 +578,7 @@ static int __init parse_mtrr_gran_size_opt(char *p)
 }
 early_param("mtrr_gran_size", parse_mtrr_gran_size_opt);
 
-static int nr_mtrr_spare_reg __initdata =
+static unsigned long nr_mtrr_spare_reg __initdata =
                                  CONFIG_MTRR_SANITIZER_SPARE_REG_NR_DEFAULT;
 
 static int __init parse_mtrr_spare_reg(char *arg)
@@ -586,7 +587,6 @@ static int __init parse_mtrr_spare_reg(char *arg)
                 nr_mtrr_spare_reg = simple_strtoul(arg, NULL, 0);
         return 0;
 }
-
 early_param("mtrr_spare_reg_nr", parse_mtrr_spare_reg);
 
 static int __init
@@ -594,8 +594,8 @@ x86_setup_var_mtrrs(struct res_range *range, int nr_range,
                     u64 chunk_size, u64 gran_size)
 {
         struct var_mtrr_state var_state;
-        int i;
         int num_reg;
+        int i;
 
         var_state.range_startk  = 0;
         var_state.range_sizek   = 0;
@@ -605,17 +605,18 @@ x86_setup_var_mtrrs(struct res_range *range, int nr_range,
 
         memset(range_state, 0, sizeof(range_state));
 
-        /* Write the range etc */
-        for (i = 0; i < nr_range; i++)
+        /* Write the range: */
+        for (i = 0; i < nr_range; i++) {
                 set_var_mtrr_range(&var_state, range[i].start,
                                    range[i].end - range[i].start + 1);
+        }
 
-        /* Write the last range */
+        /* Write the last range: */
         if (var_state.range_sizek != 0)
                 range_to_mtrr_with_hole(&var_state, 0, 0);
 
         num_reg = var_state.reg;
-        /* Clear out the extra MTRR's */
+        /* Clear out the extra MTRR's: */
         while (var_state.reg < num_var_ranges) {
                 save_var_mtrr(var_state.reg, 0, 0, 0);
                 var_state.reg++;
@@ -625,11 +626,11 @@ x86_setup_var_mtrrs(struct res_range *range, int nr_range,
 }
 
 struct mtrr_cleanup_result {
-        unsigned long gran_sizek;
-        unsigned long chunk_sizek;
-        unsigned long lose_cover_sizek;
-        unsigned int num_reg;
-        int bad;
+        unsigned long   gran_sizek;
+        unsigned long   chunk_sizek;
+        unsigned long   lose_cover_sizek;
+        unsigned int    num_reg;
+        int             bad;
 };
 
 /*
@@ -645,10 +646,10 @@ static unsigned long __initdata min_loss_pfn[RANGE_NUM];
 
 static void __init print_out_mtrr_range_state(void)
 {
-        int i;
         char start_factor = 'K', size_factor = 'K';
         unsigned long start_base, size_base;
         mtrr_type type;
+        int i;
 
         for (i = 0; i < num_var_ranges; i++) {
 
@@ -676,10 +677,10 @@ static int __init mtrr_need_cleanup(void)
         int i;
         mtrr_type type;
         unsigned long size;
-        /* extra one for all 0 */
+        /* Extra one for all 0: */
         int num[MTRR_NUM_TYPES + 1];
 
-        /* check entries number */
+        /* Check entries number: */
         memset(num, 0, sizeof(num));
         for (i = 0; i < num_var_ranges; i++) {
                 type = range_state[i].type;
@@ -693,88 +694,86 @@ static int __init mtrr_need_cleanup(void)
                 num[type]++;
         }
 
-        /* check if we got UC entries */
+        /* Check if we got UC entries: */
         if (!num[MTRR_TYPE_UNCACHABLE])
                 return 0;
 
-        /* check if we only had WB and UC */
+        /* Check if we only had WB and UC */
         if (num[MTRR_TYPE_WRBACK] + num[MTRR_TYPE_UNCACHABLE] !=
-                num_var_ranges - num[MTRR_NUM_TYPES])
+            num_var_ranges - num[MTRR_NUM_TYPES])
                 return 0;
 
         return 1;
 }
 
 static unsigned long __initdata range_sums;
-static void __init mtrr_calc_range_state(u64 chunk_size, u64 gran_size,
-                                         unsigned long extra_remove_base,
-                                         unsigned long extra_remove_size,
-                                         int i)
+
+static void __init
+mtrr_calc_range_state(u64 chunk_size, u64 gran_size,
+                      unsigned long x_remove_base,
+                      unsigned long x_remove_size, int i)
 {
-        int num_reg;
         static struct res_range range_new[RANGE_NUM];
-        static int nr_range_new;
         unsigned long range_sums_new;
+        static int nr_range_new;
+        int num_reg;
 
-        /* convert ranges to var ranges state */
-        num_reg = x86_setup_var_mtrrs(range, nr_range,
-                                                chunk_size, gran_size);
+        /* Convert ranges to var ranges state: */
+        num_reg = x86_setup_var_mtrrs(range, nr_range, chunk_size, gran_size);
 
-        /* we got new setting in range_state, check it */
+        /* We got new setting in range_state, check it: */
         memset(range_new, 0, sizeof(range_new));
         nr_range_new = x86_get_mtrr_mem_range(range_new, 0,
-                                extra_remove_base, extra_remove_size);
+                                x_remove_base, x_remove_size);
         range_sums_new = sum_ranges(range_new, nr_range_new);
 
         result[i].chunk_sizek = chunk_size >> 10;
         result[i].gran_sizek = gran_size >> 10;
         result[i].num_reg = num_reg;
+
         if (range_sums < range_sums_new) {
-                result[i].lose_cover_sizek =
-                        (range_sums_new - range_sums) << PSHIFT;
+                result[i].lose_cover_sizek = (range_sums_new - range_sums) << PSHIFT;
                 result[i].bad = 1;
-        } else
-                result[i].lose_cover_sizek =
-                        (range_sums - range_sums_new) << PSHIFT;
+        } else {
+                result[i].lose_cover_sizek = (range_sums - range_sums_new) << PSHIFT;
+        }
 
-        /* double check it */
+        /* Double check it: */
         if (!result[i].bad && !result[i].lose_cover_sizek) {
-                if (nr_range_new != nr_range ||
-                        memcmp(range, range_new, sizeof(range)))
-                                result[i].bad = 1;
+                if (nr_range_new != nr_range || memcmp(range, range_new, sizeof(range)))
+                        result[i].bad = 1;
         }
 
-        if (!result[i].bad && (range_sums - range_sums_new <
-                                min_loss_pfn[num_reg])) {
-                min_loss_pfn[num_reg] =
-                        range_sums - range_sums_new;
-        }
+        if (!result[i].bad && (range_sums - range_sums_new < min_loss_pfn[num_reg]))
+                min_loss_pfn[num_reg] = range_sums - range_sums_new;
 }
 
 static void __init mtrr_print_out_one_result(int i)
 {
-        char gran_factor, chunk_factor, lose_factor;
         unsigned long gran_base, chunk_base, lose_base;
+        char gran_factor, chunk_factor, lose_factor;
 
         gran_base = to_size_factor(result[i].gran_sizek, &gran_factor),
         chunk_base = to_size_factor(result[i].chunk_sizek, &chunk_factor),
         lose_base = to_size_factor(result[i].lose_cover_sizek, &lose_factor),
-        printk(KERN_INFO "%sgran_size: %ld%c \tchunk_size: %ld%c \t",
-                        result[i].bad ? "*BAD*" : " ",
-                        gran_base, gran_factor, chunk_base, chunk_factor);
-        printk(KERN_CONT "num_reg: %d  \tlose cover RAM: %s%ld%c\n",
-                        result[i].num_reg, result[i].bad ? "-" : "",
-                        lose_base, lose_factor);
+
+        pr_info("%sgran_size: %ld%c \tchunk_size: %ld%c \t",
+                result[i].bad ? "*BAD*" : " ",
+                gran_base, gran_factor, chunk_base, chunk_factor);
+        pr_cont("num_reg: %d  \tlose cover RAM: %s%ld%c\n",
+                result[i].num_reg, result[i].bad ? "-" : "",
+                lose_base, lose_factor);
 }
 
 static int __init mtrr_search_optimal_index(void)
 {
-        int i;
         int num_reg_good;
         int index_good;
+        int i;
 
         if (nr_mtrr_spare_reg >= num_var_ranges)
                 nr_mtrr_spare_reg = num_var_ranges - 1;
+
         num_reg_good = -1;
         for (i = num_var_ranges - nr_mtrr_spare_reg; i > 0; i--) {
                 if (!min_loss_pfn[i])
@@ -796,24 +795,24 @@ static int __init mtrr_search_optimal_index(void)
         return index_good;
 }
 
-
 int __init mtrr_cleanup(unsigned address_bits)
 {
-        unsigned long extra_remove_base, extra_remove_size;
+        unsigned long x_remove_base, x_remove_size;
         unsigned long base, size, def, dummy;
-        mtrr_type type;
         u64 chunk_size, gran_size;
+        mtrr_type type;
         int index_good;
         int i;
 
         if (!is_cpu(INTEL) || enable_mtrr_cleanup < 1)
                 return 0;
+
         rdmsr(MSR_MTRRdefType, def, dummy);
         def &= 0xff;
         if (def != MTRR_TYPE_UNCACHABLE)
                 return 0;
 
-        /* get it and store it aside */
+        /* Get it and store it aside: */
         memset(range_state, 0, sizeof(range_state));
         for (i = 0; i < num_var_ranges; i++) {
                 mtrr_if->get(i, &base, &size, &type);
@@ -822,29 +821,28 @@ int __init mtrr_cleanup(unsigned address_bits)
                 range_state[i].type = type;
         }
 
-        /* check if we need handle it and can handle it */
+        /* Check if we need handle it and can handle it: */
         if (!mtrr_need_cleanup())
                 return 0;
 
-        /* print original var MTRRs at first, for debugging: */
+        /* Print original var MTRRs at first, for debugging: */
         printk(KERN_DEBUG "original variable MTRRs\n");
         print_out_mtrr_range_state();
 
         memset(range, 0, sizeof(range));
-        extra_remove_size = 0;
-        extra_remove_base = 1 << (32 - PAGE_SHIFT);
+        x_remove_size = 0;
+        x_remove_base = 1 << (32 - PAGE_SHIFT);
         if (mtrr_tom2)
-                extra_remove_size =
-                        (mtrr_tom2 >> PAGE_SHIFT) - extra_remove_base;
-        nr_range = x86_get_mtrr_mem_range(range, 0, extra_remove_base,
-                                          extra_remove_size);
+                x_remove_size = (mtrr_tom2 >> PAGE_SHIFT) - x_remove_base;
+
+        nr_range = x86_get_mtrr_mem_range(range, 0, x_remove_base, x_remove_size);
         /*
-         * [0, 1M) should always be coverred by var mtrr with WB
-         * and fixed mtrrs should take effective before var mtrr for it
+         * [0, 1M) should always be covered by var mtrr with WB
+         * and fixed mtrrs should take effect before var mtrr for it:
          */
         nr_range = add_range_with_merge(range, nr_range, 0,
                                         (1ULL<<(20 - PAGE_SHIFT)) - 1);
-        /* sort the ranges */
+        /* Sort the ranges: */
         sort(range, nr_range, sizeof(struct res_range), cmp_range, NULL);
 
         range_sums = sum_ranges(range, nr_range);
@@ -854,7 +852,7 @@ int __init mtrr_cleanup(unsigned address_bits)
         if (mtrr_chunk_size && mtrr_gran_size) {
                 i = 0;
                 mtrr_calc_range_state(mtrr_chunk_size, mtrr_gran_size,
-                                      extra_remove_base, extra_remove_size, i);
+                                      x_remove_base, x_remove_size, i);
 
                 mtrr_print_out_one_result(i);
 
@@ -880,7 +878,7 @@ int __init mtrr_cleanup(unsigned address_bits)
                                 continue;
 
                         mtrr_calc_range_state(chunk_size, gran_size,
-                                      extra_remove_base, extra_remove_size, i);
+                                      x_remove_base, x_remove_size, i);
                         if (debug_print) {
                                 mtrr_print_out_one_result(i);
                                 printk(KERN_INFO "\n");
@@ -890,7 +888,7 @@ int __init mtrr_cleanup(unsigned address_bits)
                 }
         }
 
-        /* try to find the optimal index */
+        /* Try to find the optimal index: */
         index_good = mtrr_search_optimal_index();
 
         if (index_good != -1) {
@@ -898,7 +896,7 @@ int __init mtrr_cleanup(unsigned address_bits)
                 i = index_good;
                 mtrr_print_out_one_result(i);
 
-                /* convert ranges to var ranges state */
+                /* Convert ranges to var ranges state: */
                 chunk_size = result[i].chunk_sizek;
                 chunk_size <<= 10;
                 gran_size = result[i].gran_sizek;
@@ -941,8 +939,8 @@ early_param("disable_mtrr_trim", disable_mtrr_trim_setup);
  * Note this won't check if the MTRRs < 4GB where the magic bit doesn't
  * apply to are wrong, but so far we don't know of any such case in the wild.
  */
-#define Tom2Enabled (1U << 21)
-#define Tom2ForceMemTypeWB (1U << 22)
+#define Tom2Enabled             (1U << 21)
+#define Tom2ForceMemTypeWB      (1U << 22)
 
 int __init amd_special_default_mtrr(void)
 {
@@ -952,7 +950,7 @@ int __init amd_special_default_mtrr(void)
                 return 0;
         if (boot_cpu_data.x86 < 0xf || boot_cpu_data.x86 > 0x11)
                 return 0;
-        /* In case some hypervisor doesn't pass SYSCFG through */
+        /* In case some hypervisor doesn't pass SYSCFG through: */
         if (rdmsr_safe(MSR_K8_SYSCFG, &l, &h) < 0)
                 return 0;
         /*
@@ -965,19 +963,21 @@ int __init amd_special_default_mtrr(void)
         return 0;
 }
 
-static u64 __init real_trim_memory(unsigned long start_pfn,
-                                   unsigned long limit_pfn)
+static u64 __init
+real_trim_memory(unsigned long start_pfn, unsigned long limit_pfn)
 {
         u64 trim_start, trim_size;
+
         trim_start = start_pfn;
         trim_start <<= PAGE_SHIFT;
+
         trim_size = limit_pfn;
         trim_size <<= PAGE_SHIFT;
         trim_size -= trim_start;
 
-        return e820_update_range(trim_start, trim_size, E820_RAM,
-                                E820_RESERVED);
+        return e820_update_range(trim_start, trim_size, E820_RAM, E820_RESERVED);
 }
+
 /**
  * mtrr_trim_uncached_memory - trim RAM not covered by MTRRs
  * @end_pfn: ending page frame number
@@ -985,7 +985,7 @@ static u64 __init real_trim_memory(unsigned long start_pfn,
  * Some buggy BIOSes don't setup the MTRRs properly for systems with certain
  * memory configurations.  This routine checks that the highest MTRR matches
  * the end of memory, to make sure the MTRRs having a write back type cover
- * all of the memory the kernel is intending to use. If not, it'll trim any
+ * all of the memory the kernel is intending to use.  If not, it'll trim any
  * memory off the end by adjusting end_pfn, removing it from the kernel's
  * allocation pools, warning the user with an obnoxious message.
  */
@@ -994,21 +994,22 @@ int __init mtrr_trim_uncached_memory(unsigned long end_pfn)
         unsigned long i, base, size, highest_pfn = 0, def, dummy;
         mtrr_type type;
         u64 total_trim_size;
-
         /* extra one for all 0 */
         int num[MTRR_NUM_TYPES + 1];
+
         /*
          * Make sure we only trim uncachable memory on machines that
          * support the Intel MTRR architecture:
          */
         if (!is_cpu(INTEL) || disable_mtrr_trim)
                 return 0;
+
         rdmsr(MSR_MTRRdefType, def, dummy);
         def &= 0xff;
         if (def != MTRR_TYPE_UNCACHABLE)
                 return 0;
 
-        /* get it and store it aside */
+        /* Get it and store it aside: */
         memset(range_state, 0, sizeof(range_state));
         for (i = 0; i < num_var_ranges; i++) {
                 mtrr_if->get(i, &base, &size, &type);
@@ -1017,7 +1018,7 @@ int __init mtrr_trim_uncached_memory(unsigned long end_pfn)
                 range_state[i].type = type;
         }
 
-        /* Find highest cached pfn */
+        /* Find highest cached pfn: */
         for (i = 0; i < num_var_ranges; i++) {
                 type = range_state[i].type;
                 if (type != MTRR_TYPE_WRBACK)
@@ -1028,13 +1029,13 @@ int __init mtrr_trim_uncached_memory(unsigned long end_pfn)
                         highest_pfn = base + size;
         }
 
-        /* kvm/qemu doesn't have mtrr set right, don't trim them all */
+        /* kvm/qemu doesn't have mtrr set right, don't trim them all: */
         if (!highest_pfn) {
                 printk(KERN_INFO "CPU MTRRs all blank - virtualized system.\n");
                 return 0;
         }
 
-        /* check entries number */
+        /* Check entries number: */
         memset(num, 0, sizeof(num));
         for (i = 0; i < num_var_ranges; i++) {
                 type = range_state[i].type;
@@ -1046,11 +1047,11 @@ int __init mtrr_trim_uncached_memory(unsigned long end_pfn)
                 num[type]++;
         }
 
-        /* no entry for WB? */
+        /* No entry for WB? */
         if (!num[MTRR_TYPE_WRBACK])
                 return 0;
 
-        /* check if we only had WB and UC */
+        /* Check if we only had WB and UC: */
         if (num[MTRR_TYPE_WRBACK] + num[MTRR_TYPE_UNCACHABLE] !=
                 num_var_ranges - num[MTRR_NUM_TYPES])
                 return 0;
@@ -1066,31 +1067,31 @@ int __init mtrr_trim_uncached_memory(unsigned long end_pfn)
         }
         nr_range = x86_get_mtrr_mem_range(range, nr_range, 0, 0);
 
+        /* Check the head: */
         total_trim_size = 0;
-        /* check the head */
         if (range[0].start)
                 total_trim_size += real_trim_memory(0, range[0].start);
-        /* check the holes */
+
+        /* Check the holes: */
         for (i = 0; i < nr_range - 1; i++) {
                 if (range[i].end + 1 < range[i+1].start)
                         total_trim_size += real_trim_memory(range[i].end + 1,
                                                             range[i+1].start);
         }
-        /* check the top */
+
+        /* Check the top: */
         i = nr_range - 1;
         if (range[i].end + 1 < end_pfn)
                 total_trim_size += real_trim_memory(range[i].end + 1,
                                                          end_pfn);
 
         if (total_trim_size) {
-                printk(KERN_WARNING "WARNING: BIOS bug: CPU MTRRs don't cover"
-                        " all of memory, losing %lluMB of RAM.\n",
-                        total_trim_size >> 20);
+                pr_warning("WARNING: BIOS bug: CPU MTRRs don't cover all of memory, losing %lluMB of RAM.\n", total_trim_size >> 20);
 
                 if (!changed_by_mtrr_cleanup)
                         WARN_ON(1);
 
-                printk(KERN_INFO "update e820 for mtrr\n");
+                pr_info("update e820 for mtrr\n");
                 update_e820();
 
                 return 1;
@@ -1098,4 +1099,3 @@ int __init mtrr_trim_uncached_memory(unsigned long end_pfn)
 
         return 0;
 }
-
diff --git a/arch/x86/kernel/cpu/mtrr/cyrix.c b/arch/x86/kernel/cpu/mtrr/cyrix.c
index ff14c320040c..228d982ce09c 100644
--- a/arch/x86/kernel/cpu/mtrr/cyrix.c
+++ b/arch/x86/kernel/cpu/mtrr/cyrix.c
@@ -1,38 +1,40 @@
 #include <linux/init.h>
+#include <linux/io.h>
 #include <linux/mm.h>
-#include <asm/mtrr.h>
-#include <asm/msr.h>
-#include <asm/io.h>
+
 #include <asm/processor-cyrix.h>
 #include <asm/processor-flags.h>
+#include <asm/mtrr.h>
+#include <asm/msr.h>
+
 #include "mtrr.h"
 
 static void
 cyrix_get_arr(unsigned int reg, unsigned long *base,
               unsigned long *size, mtrr_type * type)
 {
-        unsigned long flags;
         unsigned char arr, ccr3, rcr, shift;
+        unsigned long flags;
 
         arr = CX86_ARR_BASE + (reg << 1) + reg; /* avoid multiplication by 3 */
 
-        /* Save flags and disable interrupts */
         local_irq_save(flags);
 
         ccr3 = getCx86(CX86_CCR3);
         setCx86(CX86_CCR3, (ccr3 & 0x0f) | 0x10);       /* enable MAPEN */
-        ((unsigned char *) base)[3] = getCx86(arr);
-        ((unsigned char *) base)[2] = getCx86(arr + 1);
-        ((unsigned char *) base)[1] = getCx86(arr + 2);
+        ((unsigned char *)base)[3] = getCx86(arr);
+        ((unsigned char *)base)[2] = getCx86(arr + 1);
+        ((unsigned char *)base)[1] = getCx86(arr + 2);
         rcr = getCx86(CX86_RCR_BASE + reg);
-        setCx86(CX86_CCR3, ccr3);       /* disable MAPEN */
+        setCx86(CX86_CCR3, ccr3);                       /* disable MAPEN */
 
-        /* Enable interrupts if it was enabled previously */
         local_irq_restore(flags);
+
         shift = ((unsigned char *) base)[1] & 0x0f;
         *base >>= PAGE_SHIFT;
 
-        /* Power of two, at least 4K on ARR0-ARR6, 256K on ARR7
+        /*
+         * Power of two, at least 4K on ARR0-ARR6, 256K on ARR7
          * Note: shift==0xf means 4G, this is unsupported.
          */
         if (shift)
@@ -76,17 +78,20 @@ cyrix_get_arr(unsigned int reg, unsigned long *base,
         }
 }
 
+/*
+ * cyrix_get_free_region - get a free ARR.
+ *
+ * @base: the starting (base) address of the region.
+ * @size: the size (in bytes) of the region.
+ *
+ * Returns: the index of the region on success, else -1 on error.
+*/
 static int
 cyrix_get_free_region(unsigned long base, unsigned long size, int replace_reg)
-/*  [SUMMARY] Get a free ARR.
-    <base> The starting (base) address of the region.
-    <size> The size (in bytes) of the region.
-    [RETURNS] The index of the region on success, else -1 on error.
-*/
 {
-        int i;
-        mtrr_type ltype;
         unsigned long lbase, lsize;
+        mtrr_type ltype;
+        int i;
 
         switch (replace_reg) {
         case 7:
@@ -107,14 +112,17 @@ cyrix_get_free_region(unsigned long base, unsigned long size, int replace_reg)
                 cyrix_get_arr(7, &lbase, &lsize, &ltype);
                 if (lsize == 0)
                         return 7;
-                /*  Else try ARR0-ARR6 first  */
+                /* Else try ARR0-ARR6 first  */
         } else {
                 for (i = 0; i < 7; i++) {
                         cyrix_get_arr(i, &lbase, &lsize, &ltype);
                         if (lsize == 0)
                                 return i;
                 }
-                /* ARR0-ARR6 isn't free, try ARR7 but its size must be at least 256K */
+                /*
+                 * ARR0-ARR6 isn't free
+                 * try ARR7 but its size must be at least 256K
+                 */
                 cyrix_get_arr(i, &lbase, &lsize, &ltype);
                 if ((lsize == 0) && (size >= 0x40))
                         return i;
@@ -122,21 +130,22 @@ cyrix_get_free_region(unsigned long base, unsigned long size, int replace_reg)
         return -ENOSPC;
 }
 
-static u32 cr4 = 0;
-static u32 ccr3;
+static u32 cr4, ccr3;
 
 static void prepare_set(void)
 {
         u32 cr0;
 
         /*  Save value of CR4 and clear Page Global Enable (bit 7)  */
-        if ( cpu_has_pge ) {
+        if (cpu_has_pge) {
                 cr4 = read_cr4();
                 write_cr4(cr4 & ~X86_CR4_PGE);
         }
 
-        /*  Disable and flush caches. Note that wbinvd flushes the TLBs as
-            a side-effect  */
+        /*
+         * Disable and flush caches.
+         * Note that wbinvd flushes the TLBs as a side-effect
+         */
         cr0 = read_cr0() | X86_CR0_CD;
         wbinvd();
         write_cr0(cr0);
@@ -147,22 +156,21 @@ static void prepare_set(void)
 
         /* Cyrix ARRs - everything else was excluded at the top */
         setCx86(CX86_CCR3, (ccr3 & 0x0f) | 0x10);
-
 }
 
 static void post_set(void)
 {
-        /*  Flush caches and TLBs  */
+        /* Flush caches and TLBs */
         wbinvd();
 
         /* Cyrix ARRs - everything else was excluded at the top */
         setCx86(CX86_CCR3, ccr3);
-                
-        /*  Enable caches  */
+
+        /* Enable caches */
         write_cr0(read_cr0() & 0xbfffffff);
 
-        /*  Restore value of CR4  */
-        if ( cpu_has_pge )
+        /* Restore value of CR4 */
+        if (cpu_has_pge)
                 write_cr4(cr4);
 }
 
@@ -178,7 +186,8 @@ static void cyrix_set_arr(unsigned int reg, unsigned long base,
                 size >>= 6;
 
         size &= 0x7fff;         /* make sure arr_size <= 14 */
-        for (arr_size = 0; size; arr_size++, size >>= 1) ;
+        for (arr_size = 0; size; arr_size++, size >>= 1)
+                ;
 
         if (reg < 7) {
                 switch (type) {
@@ -215,18 +224,18 @@ static void cyrix_set_arr(unsigned int reg, unsigned long base,
         prepare_set();
 
         base <<= PAGE_SHIFT;
-        setCx86(arr, ((unsigned char *) &base)[3]);
-        setCx86(arr + 1, ((unsigned char *) &base)[2]);
-        setCx86(arr + 2, (((unsigned char *) &base)[1]) | arr_size);
+        setCx86(arr + 0,  ((unsigned char *)&base)[3]);
+        setCx86(arr + 1,  ((unsigned char *)&base)[2]);
+        setCx86(arr + 2, (((unsigned char *)&base)[1]) | arr_size);
         setCx86(CX86_RCR_BASE + reg, arr_type);
 
         post_set();
 }
 
 typedef struct {
-        unsigned long base;
-        unsigned long size;
-        mtrr_type type;
+        unsigned long   base;
+        unsigned long   size;
+        mtrr_type       type;
 } arr_state_t;
 
 static arr_state_t arr_state[8] = {
@@ -247,16 +256,17 @@ static void cyrix_set_all(void)
                 setCx86(CX86_CCR0 + i, ccr_state[i]);
         for (; i < 7; i++)
                 setCx86(CX86_CCR4 + i, ccr_state[i]);
-        for (i = 0; i < 8; i++)
-                cyrix_set_arr(i, arr_state[i].base, 
+
+        for (i = 0; i < 8; i++) {
+                cyrix_set_arr(i, arr_state[i].base,
                               arr_state[i].size, arr_state[i].type);
+        }
 
         post_set();
 }
 
 static struct mtrr_ops cyrix_mtrr_ops = {
         .vendor            = X86_VENDOR_CYRIX,
-//      .init              = cyrix_arr_init,
         .set_all           = cyrix_set_all,
         .set               = cyrix_set_arr,
         .get               = cyrix_get_arr,
@@ -270,5 +280,3 @@ int __init cyrix_init_mtrr(void)
         set_mtrr_ops(&cyrix_mtrr_ops);
         return 0;
 }
-
-//arch_initcall(cyrix_init_mtrr);
diff --git a/arch/x86/kernel/cpu/mtrr/generic.c b/arch/x86/kernel/cpu/mtrr/generic.c
index 0543f69f0b27..55da0c5f68dd 100644
--- a/arch/x86/kernel/cpu/mtrr/generic.c
+++ b/arch/x86/kernel/cpu/mtrr/generic.c
@@ -1,28 +1,34 @@
-/* This only handles 32bit MTRR on 32bit hosts. This is strictly wrong
-   because MTRRs can span upto 40 bits (36bits on most modern x86) */ 
+/*
+ * This only handles 32bit MTRR on 32bit hosts. This is strictly wrong
+ * because MTRRs can span upto 40 bits (36bits on most modern x86)
+ */
+#define DEBUG
+
+#include <linux/module.h>
 #include <linux/init.h>
 #include <linux/slab.h>
+#include <linux/io.h>
 #include <linux/mm.h>
-#include <linux/module.h>
-#include <asm/io.h>
-#include <asm/mtrr.h>
-#include <asm/msr.h>
-#include <asm/system.h>
-#include <asm/cpufeature.h>
+
 #include <asm/processor-flags.h>
+#include <asm/cpufeature.h>
 #include <asm/tlbflush.h>
+#include <asm/system.h>
+#include <asm/mtrr.h>
+#include <asm/msr.h>
 #include <asm/pat.h>
+
 #include "mtrr.h"
 
 struct fixed_range_block {
-        int base_msr; /* start address of an MTRR block */
-        int ranges;   /* number of MTRRs in this block  */
+        int base_msr;           /* start address of an MTRR block */
+        int ranges;             /* number of MTRRs in this block  */
 };
 
 static struct fixed_range_block fixed_range_blocks[] = {
-        { MSR_MTRRfix64K_00000, 1 }, /* one  64k MTRR  */
-        { MSR_MTRRfix16K_80000, 2 }, /* two  16k MTRRs */
-        { MSR_MTRRfix4K_C0000,  8 }, /* eight 4k MTRRs */
+        { MSR_MTRRfix64K_00000, 1 }, /* one   64k MTRR  */
+        { MSR_MTRRfix16K_80000, 2 }, /* two   16k MTRRs */
+        { MSR_MTRRfix4K_C0000,  8 }, /* eight  4k MTRRs */
         {}
 };
 
@@ -30,10 +36,10 @@ static unsigned long smp_changes_mask;
 static int mtrr_state_set;
 u64 mtrr_tom2;
 
-struct mtrr_state_type mtrr_state = {};
+struct mtrr_state_type mtrr_state;
 EXPORT_SYMBOL_GPL(mtrr_state);
 
-/**
+/*
  * BIOS is expected to clear MtrrFixDramModEn bit, see for example
  * "BIOS and Kernel Developer's Guide for the AMD Athlon 64 and AMD
  * Opteron Processors" (26094 Rev. 3.30 February 2006), section
@@ -104,9 +110,8 @@ u8 mtrr_type_lookup(u64 start, u64 end)
          * Look of multiple ranges matching this address and pick type
          * as per MTRR precedence
          */
-        if (!(mtrr_state.enabled & 2)) {
+        if (!(mtrr_state.enabled & 2))
                 return mtrr_state.def_type;
-        }
 
         prev_match = 0xFF;
         for (i = 0; i < num_var_ranges; ++i) {
@@ -125,9 +130,8 @@ u8 mtrr_type_lookup(u64 start, u64 end)
                 if (start_state != end_state)
                         return 0xFE;
 
-                if ((start & mask) != (base & mask)) {
+                if ((start & mask) != (base & mask))
                         continue;
-                }
 
                 curr_match = mtrr_state.var_ranges[i].base_lo & 0xff;
                 if (prev_match == 0xFF) {
@@ -148,9 +152,8 @@ u8 mtrr_type_lookup(u64 start, u64 end)
                         curr_match = MTRR_TYPE_WRTHROUGH;
                 }
 
-                if (prev_match != curr_match) {
+                if (prev_match != curr_match)
                         return MTRR_TYPE_UNCACHABLE;
-                }
         }
 
         if (mtrr_tom2) {
@@ -164,7 +167,7 @@ u8 mtrr_type_lookup(u64 start, u64 end)
         return mtrr_state.def_type;
 }
 
-/*  Get the MSR pair relating to a var range  */
+/* Get the MSR pair relating to a var range */
 static void
 get_mtrr_var_range(unsigned int index, struct mtrr_var_range *vr)
 {
@@ -172,7 +175,7 @@ get_mtrr_var_range(unsigned int index, struct mtrr_var_range *vr)
         rdmsr(MTRRphysMask_MSR(index), vr->mask_lo, vr->mask_hi);
 }
 
-/*  fill the MSR pair relating to a var range  */
+/* Fill the MSR pair relating to a var range */
 void fill_mtrr_var_range(unsigned int index,
                 u32 base_lo, u32 base_hi, u32 mask_lo, u32 mask_hi)
 {
@@ -186,10 +189,9 @@ void fill_mtrr_var_range(unsigned int index,
         vr[index].mask_hi = mask_hi;
 }
 
-static void
-get_fixed_ranges(mtrr_type * frs)
+static void get_fixed_ranges(mtrr_type *frs)
 {
-        unsigned int *p = (unsigned int *) frs;
+        unsigned int *p = (unsigned int *)frs;
         int i;
 
         k8_check_syscfg_dram_mod_en();
@@ -217,22 +219,22 @@ static void __init print_fixed_last(void)
         if (!last_fixed_end)
                 return;
 
-        printk(KERN_DEBUG "  %05X-%05X %s\n", last_fixed_start,
-                last_fixed_end - 1, mtrr_attrib_to_str(last_fixed_type));
+        pr_debug("  %05X-%05X %s\n", last_fixed_start,
+                 last_fixed_end - 1, mtrr_attrib_to_str(last_fixed_type));
 
         last_fixed_end = 0;
 }
 
 static void __init update_fixed_last(unsigned base, unsigned end,
-                                       mtrr_type type)
+                                     mtrr_type type)
 {
         last_fixed_start = base;
         last_fixed_end = end;
         last_fixed_type = type;
 }
 
-static void __init print_fixed(unsigned base, unsigned step,
-                               const mtrr_type *types)
+static void __init
+print_fixed(unsigned base, unsigned step, const mtrr_type *types)
 {
         unsigned i;
 
@@ -259,54 +261,55 @@ static void __init print_mtrr_state(void)
         unsigned int i;
         int high_width;
 
-        printk(KERN_DEBUG "MTRR default type: %s\n",
-                         mtrr_attrib_to_str(mtrr_state.def_type));
+        pr_debug("MTRR default type: %s\n",
+                 mtrr_attrib_to_str(mtrr_state.def_type));
         if (mtrr_state.have_fixed) {
-                printk(KERN_DEBUG "MTRR fixed ranges %sabled:\n",
-                       mtrr_state.enabled & 1 ? "en" : "dis");
+                pr_debug("MTRR fixed ranges %sabled:\n",
+                         mtrr_state.enabled & 1 ? "en" : "dis");
                 print_fixed(0x00000, 0x10000, mtrr_state.fixed_ranges + 0);
                 for (i = 0; i < 2; ++i)
-                        print_fixed(0x80000 + i * 0x20000, 0x04000, mtrr_state.fixed_ranges + (i + 1) * 8);
+                        print_fixed(0x80000 + i * 0x20000, 0x04000,
+                                    mtrr_state.fixed_ranges + (i + 1) * 8);
                 for (i = 0; i < 8; ++i)
-                        print_fixed(0xC0000 + i * 0x08000, 0x01000, mtrr_state.fixed_ranges + (i + 3) * 8);
+                        print_fixed(0xC0000 + i * 0x08000, 0x01000,
+                                    mtrr_state.fixed_ranges + (i + 3) * 8);
 
                 /* tail */
                 print_fixed_last();
         }
-        printk(KERN_DEBUG "MTRR variable ranges %sabled:\n",
-               mtrr_state.enabled & 2 ? "en" : "dis");
+        pr_debug("MTRR variable ranges %sabled:\n",
+                 mtrr_state.enabled & 2 ? "en" : "dis");
         if (size_or_mask & 0xffffffffUL)
                 high_width = ffs(size_or_mask & 0xffffffffUL) - 1;
         else
                 high_width = ffs(size_or_mask>>32) + 32 - 1;
         high_width = (high_width - (32 - PAGE_SHIFT) + 3) / 4;
+
         for (i = 0; i < num_var_ranges; ++i) {
                 if (mtrr_state.var_ranges[i].mask_lo & (1 << 11))
-                        printk(KERN_DEBUG "  %u base %0*X%05X000 mask %0*X%05X000 %s\n",
-                               i,
-                               high_width,
-                               mtrr_state.var_ranges[i].base_hi,
-                               mtrr_state.var_ranges[i].base_lo >> 12,
-                               high_width,
-                               mtrr_state.var_ranges[i].mask_hi,
-                               mtrr_state.var_ranges[i].mask_lo >> 12,
-                               mtrr_attrib_to_str(mtrr_state.var_ranges[i].base_lo & 0xff));
+                        pr_debug("  %u base %0*X%05X000 mask %0*X%05X000 %s\n",
+                                 i,
+                                 high_width,
+                                 mtrr_state.var_ranges[i].base_hi,
+                                 mtrr_state.var_ranges[i].base_lo >> 12,
+                                 high_width,
+                                 mtrr_state.var_ranges[i].mask_hi,
+                                 mtrr_state.var_ranges[i].mask_lo >> 12,
+                                 mtrr_attrib_to_str(mtrr_state.var_ranges[i].base_lo & 0xff));
                 else
-                        printk(KERN_DEBUG "  %u disabled\n", i);
-        }
-        if (mtrr_tom2) {
-                printk(KERN_DEBUG "TOM2: %016llx aka %lldM\n",
-                                  mtrr_tom2, mtrr_tom2>>20);
+                        pr_debug("  %u disabled\n", i);
         }
+        if (mtrr_tom2)
+                pr_debug("TOM2: %016llx aka %lldM\n", mtrr_tom2, mtrr_tom2>>20);
 }
 
-/*  Grab all of the MTRR state for this CPU into *state  */
+/* Grab all of the MTRR state for this CPU into *state */
 void __init get_mtrr_state(void)
 {
-        unsigned int i;
         struct mtrr_var_range *vrs;
-        unsigned lo, dummy;
         unsigned long flags;
+        unsigned lo, dummy;
+        unsigned int i;
 
         vrs = mtrr_state.var_ranges;
 
@@ -324,6 +327,7 @@ void __init get_mtrr_state(void)
 
         if (amd_special_default_mtrr()) {
                 unsigned low, high;
+
                 /* TOP_MEM2 */
                 rdmsr(MSR_K8_TOP_MEM2, low, high);
                 mtrr_tom2 = high;
@@ -344,10 +348,9 @@ void __init get_mtrr_state(void)
 
         post_set();
         local_irq_restore(flags);
-
 }
 
-/*  Some BIOS's are fucked and don't set all MTRRs the same!  */
+/* Some BIOS's are messed up and don't set all MTRRs the same! */
 void __init mtrr_state_warn(void)
 {
         unsigned long mask = smp_changes_mask;
@@ -355,28 +358,33 @@ void __init mtrr_state_warn(void)
         if (!mask)
                 return;
         if (mask & MTRR_CHANGE_MASK_FIXED)
-                printk(KERN_WARNING "mtrr: your CPUs had inconsistent fixed MTRR settings\n");
+                pr_warning("mtrr: your CPUs had inconsistent fixed MTRR settings\n");
         if (mask & MTRR_CHANGE_MASK_VARIABLE)
-                printk(KERN_WARNING "mtrr: your CPUs had inconsistent variable MTRR settings\n");
+                pr_warning("mtrr: your CPUs had inconsistent variable MTRR settings\n");
         if (mask & MTRR_CHANGE_MASK_DEFTYPE)
-                printk(KERN_WARNING "mtrr: your CPUs had inconsistent MTRRdefType settings\n");
+                pr_warning("mtrr: your CPUs had inconsistent MTRRdefType settings\n");
+
         printk(KERN_INFO "mtrr: probably your BIOS does not setup all CPUs.\n");
         printk(KERN_INFO "mtrr: corrected configuration.\n");
 }
 
-/* Doesn't attempt to pass an error out to MTRR users
-   because it's quite complicated in some cases and probably not
-   worth it because the best error handling is to ignore it. */
+/*
+ * Doesn't attempt to pass an error out to MTRR users
+ * because it's quite complicated in some cases and probably not
+ * worth it because the best error handling is to ignore it.
+ */
 void mtrr_wrmsr(unsigned msr, unsigned a, unsigned b)
 {
-        if (wrmsr_safe(msr, a, b) < 0)
+        if (wrmsr_safe(msr, a, b) < 0) {
                 printk(KERN_ERR
                         "MTRR: CPU %u: Writing MSR %x to %x:%x failed\n",
                         smp_processor_id(), msr, a, b);
+        }
 }
 
 /**
- * set_fixed_range - checks & updates a fixed-range MTRR if it differs from the value it should have
+ * set_fixed_range - checks & updates a fixed-range MTRR if it
+ *                   differs from the value it should have
  * @msr: MSR address of the MTTR which should be checked and updated
  * @changed: pointer which indicates whether the MTRR needed to be changed
  * @msrwords: pointer to the MSR values which the MSR should have
@@ -401,20 +409,23 @@ static void set_fixed_range(int msr, bool *changed, unsigned int *msrwords)
  *
  * Returns: The index of the region on success, else negative on error.
  */
-int generic_get_free_region(unsigned long base, unsigned long size, int replace_reg)
+int
+generic_get_free_region(unsigned long base, unsigned long size, int replace_reg)
 {
-        int i, max;
-        mtrr_type ltype;
         unsigned long lbase, lsize;
+        mtrr_type ltype;
+        int i, max;
 
         max = num_var_ranges;
         if (replace_reg >= 0 && replace_reg < max)
                 return replace_reg;
+
         for (i = 0; i < max; ++i) {
                 mtrr_if->get(i, &lbase, &lsize, &ltype);
                 if (lsize == 0)
                         return i;
         }
+
         return -ENOSPC;
 }
 
@@ -434,7 +445,7 @@ static void generic_get_mtrr(unsigned int reg, unsigned long *base,
         rdmsr(MTRRphysMask_MSR(reg), mask_lo, mask_hi);
 
         if ((mask_lo & 0x800) == 0) {
-                /*  Invalid (i.e. free) range  */
+                /*  Invalid (i.e. free) range */
                 *base = 0;
                 *size = 0;
                 *type = 0;
@@ -471,27 +482,31 @@ out_put_cpu:
 }
 
 /**
- * set_fixed_ranges - checks & updates the fixed-range MTRRs if they differ from the saved set
+ * set_fixed_ranges - checks & updates the fixed-range MTRRs if they
+ *                    differ from the saved set
  * @frs: pointer to fixed-range MTRR values, saved by get_fixed_ranges()
  */
-static int set_fixed_ranges(mtrr_type * frs)
+static int set_fixed_ranges(mtrr_type *frs)
 {
-        unsigned long long *saved = (unsigned long long *) frs;
+        unsigned long long *saved = (unsigned long long *)frs;
         bool changed = false;
-        int block=-1, range;
+        int block = -1, range;
 
         k8_check_syscfg_dram_mod_en();
 
-        while (fixed_range_blocks[++block].ranges)
-            for (range=0; range < fixed_range_blocks[block].ranges; range++)
-                set_fixed_range(fixed_range_blocks[block].base_msr + range,
-                    &changed, (unsigned int *) saved++);
+        while (fixed_range_blocks[++block].ranges) {
+                for (range = 0; range < fixed_range_blocks[block].ranges; range++)
+                        set_fixed_range(fixed_range_blocks[block].base_msr + range,
+                                        &changed, (unsigned int *)saved++);
+        }
 
         return changed;
 }
 
-/*  Set the MSR pair relating to a var range. Returns TRUE if
-    changes are made  */
+/*
+ * Set the MSR pair relating to a var range.
+ * Returns true if changes are made.
+ */
 static bool set_mtrr_var_ranges(unsigned int index, struct mtrr_var_range *vr)
 {
         unsigned int lo, hi;
@@ -501,6 +516,7 @@ static bool set_mtrr_var_ranges(unsigned int index, struct mtrr_var_range *vr)
         if ((vr->base_lo & 0xfffff0ffUL) != (lo & 0xfffff0ffUL)
             || (vr->base_hi & (size_and_mask >> (32 - PAGE_SHIFT))) !=
                 (hi & (size_and_mask >> (32 - PAGE_SHIFT)))) {
+
                 mtrr_wrmsr(MTRRphysBase_MSR(index), vr->base_lo, vr->base_hi);
                 changed = true;
         }
@@ -526,21 +542,26 @@ static u32 deftype_lo, deftype_hi;
  */
 static unsigned long set_mtrr_state(void)
 {
-        unsigned int i;
         unsigned long change_mask = 0;
+        unsigned int i;
 
-        for (i = 0; i < num_var_ranges; i++)
+        for (i = 0; i < num_var_ranges; i++) {
                 if (set_mtrr_var_ranges(i, &mtrr_state.var_ranges[i]))
                         change_mask |= MTRR_CHANGE_MASK_VARIABLE;
+        }
 
         if (mtrr_state.have_fixed && set_fixed_ranges(mtrr_state.fixed_ranges))
                 change_mask |= MTRR_CHANGE_MASK_FIXED;
 
-        /*  Set_mtrr_restore restores the old value of MTRRdefType,
-           so to set it we fiddle with the saved value  */
+        /*
+         * Set_mtrr_restore restores the old value of MTRRdefType,
+         * so to set it we fiddle with the saved value:
+         */
         if ((deftype_lo & 0xff) != mtrr_state.def_type
             || ((deftype_lo & 0xc00) >> 10) != mtrr_state.enabled) {
-                deftype_lo = (deftype_lo & ~0xcff) | mtrr_state.def_type | (mtrr_state.enabled << 10);
+
+                deftype_lo = (deftype_lo & ~0xcff) | mtrr_state.def_type |
+                             (mtrr_state.enabled << 10);
                 change_mask |= MTRR_CHANGE_MASK_DEFTYPE;
         }
 
@@ -548,33 +569,36 @@ static unsigned long set_mtrr_state(void)
 }
 
 
-static unsigned long cr4 = 0;
+static unsigned long cr4;
 static DEFINE_SPINLOCK(set_atomicity_lock);
 
 /*
- * Since we are disabling the cache don't allow any interrupts - they
- * would run extremely slow and would only increase the pain.  The caller must
- * ensure that local interrupts are disabled and are reenabled after post_set()
- * has been called.
+ * Since we are disabling the cache don't allow any interrupts,
+ * they would run extremely slow and would only increase the pain.
+ *
+ * The caller must ensure that local interrupts are disabled and
+ * are reenabled after post_set() has been called.
  */
-
 static void prepare_set(void) __acquires(set_atomicity_lock)
 {
         unsigned long cr0;
 
-        /*  Note that this is not ideal, since the cache is only flushed/disabled
-           for this CPU while the MTRRs are changed, but changing this requires
-           more invasive changes to the way the kernel boots  */
+        /*
+         * Note that this is not ideal
+         * since the cache is only flushed/disabled for this CPU while the
+         * MTRRs are changed, but changing this requires more invasive
+         * changes to the way the kernel boots
+         */
 
         spin_lock(&set_atomicity_lock);
 
-        /*  Enter the no-fill (CD=1, NW=0) cache mode and flush caches. */
+        /* Enter the no-fill (CD=1, NW=0) cache mode and flush caches. */
         cr0 = read_cr0() | X86_CR0_CD;
         write_cr0(cr0);
         wbinvd();
 
-        /*  Save value of CR4 and clear Page Global Enable (bit 7)  */
-        if ( cpu_has_pge ) {
+        /* Save value of CR4 and clear Page Global Enable (bit 7) */
+        if (cpu_has_pge) {
                 cr4 = read_cr4();
                 write_cr4(cr4 & ~X86_CR4_PGE);
         }
@@ -582,26 +606,26 @@ static void prepare_set(void) __acquires(set_atomicity_lock)
         /* Flush all TLBs via a mov %cr3, %reg; mov %reg, %cr3 */
         __flush_tlb();
 
-        /*  Save MTRR state */
+        /* Save MTRR state */
         rdmsr(MSR_MTRRdefType, deftype_lo, deftype_hi);
 
-        /*  Disable MTRRs, and set the default type to uncached  */
+        /* Disable MTRRs, and set the default type to uncached */
         mtrr_wrmsr(MSR_MTRRdefType, deftype_lo & ~0xcff, deftype_hi);
 }
 
 static void post_set(void) __releases(set_atomicity_lock)
 {
-        /*  Flush TLBs (no need to flush caches - they are disabled)  */
+        /* Flush TLBs (no need to flush caches - they are disabled) */
         __flush_tlb();
 
         /* Intel (P6) standard MTRRs */
         mtrr_wrmsr(MSR_MTRRdefType, deftype_lo, deftype_hi);
-                
-        /*  Enable caches  */
+
+        /* Enable caches */
         write_cr0(read_cr0() & 0xbfffffff);
 
-        /*  Restore value of CR4  */
-        if ( cpu_has_pge )
+        /* Restore value of CR4 */
+        if (cpu_has_pge)
                 write_cr4(cr4);
         spin_unlock(&set_atomicity_lock);
 }
@@ -623,24 +647,27 @@ static void generic_set_all(void)
         post_set();
         local_irq_restore(flags);
 
-        /*  Use the atomic bitops to update the global mask  */
+        /* Use the atomic bitops to update the global mask */
         for (count = 0; count < sizeof mask * 8; ++count) {
                 if (mask & 0x01)
                         set_bit(count, &smp_changes_mask);
                 mask >>= 1;
         }
-        
+
 }
 
+/**
+ * generic_set_mtrr - set variable MTRR register on the local CPU.
+ *
+ * @reg: The register to set.
+ * @base: The base address of the region.
+ * @size: The size of the region. If this is 0 the region is disabled.
+ * @type: The type of the region.
+ *
+ * Returns nothing.
+ */
 static void generic_set_mtrr(unsigned int reg, unsigned long base,
                              unsigned long size, mtrr_type type)
-/*  [SUMMARY] Set variable MTRR register on the local CPU.
-    <reg> The register to set.
-    <base> The base address of the region.
-    <size> The size of the region. If this is 0 the region is disabled.
-    <type> The type of the region.
-    [RETURNS] Nothing.
-*/
 {
         unsigned long flags;
         struct mtrr_var_range *vr;
@@ -651,8 +678,10 @@ static void generic_set_mtrr(unsigned int reg, unsigned long base,
         prepare_set();
 
         if (size == 0) {
-                /* The invalid bit is kept in the mask, so we simply clear the
-                   relevant mask register to disable a range. */
+                /*
+                 * The invalid bit is kept in the mask, so we simply
+                 * clear the relevant mask register to disable a range.
+                 */
                 mtrr_wrmsr(MTRRphysMask_MSR(reg), 0, 0);
                 memset(vr, 0, sizeof(struct mtrr_var_range));
         } else {
@@ -669,46 +698,50 @@ static void generic_set_mtrr(unsigned int reg, unsigned long base,
         local_irq_restore(flags);
 }
 
-int generic_validate_add_page(unsigned long base, unsigned long size, unsigned int type)
+int generic_validate_add_page(unsigned long base, unsigned long size,
+                              unsigned int type)
 {
         unsigned long lbase, last;
 
-        /*  For Intel PPro stepping <= 7, must be 4 MiB aligned 
-            and not touch 0x70000000->0x7003FFFF */
+        /*
+         * For Intel PPro stepping <= 7
+         * must be 4 MiB aligned and not touch 0x70000000 -> 0x7003FFFF
+         */
         if (is_cpu(INTEL) && boot_cpu_data.x86 == 6 &&
             boot_cpu_data.x86_model == 1 &&
             boot_cpu_data.x86_mask <= 7) {
                 if (base & ((1 << (22 - PAGE_SHIFT)) - 1)) {
-                        printk(KERN_WARNING "mtrr: base(0x%lx000) is not 4 MiB aligned\n", base);
+                        pr_warning("mtrr: base(0x%lx000) is not 4 MiB aligned\n", base);
                         return -EINVAL;
                 }
                 if (!(base + size < 0x70000 || base > 0x7003F) &&
                     (type == MTRR_TYPE_WRCOMB
                      || type == MTRR_TYPE_WRBACK)) {
-                        printk(KERN_WARNING "mtrr: writable mtrr between 0x70000000 and 0x7003FFFF may hang the CPU.\n");
+                        pr_warning("mtrr: writable mtrr between 0x70000000 and 0x7003FFFF may hang the CPU.\n");
                         return -EINVAL;
                 }
         }
 
-        /*  Check upper bits of base and last are equal and lower bits are 0
-            for base and 1 for last  */
+        /*
+         * Check upper bits of base and last are equal and lower bits are 0
+         * for base and 1 for last
+         */
         last = base + size - 1;
         for (lbase = base; !(lbase & 1) && (last & 1);
-             lbase = lbase >> 1, last = last >> 1) ;
+             lbase = lbase >> 1, last = last >> 1)
+                ;
         if (lbase != last) {
-                printk(KERN_WARNING "mtrr: base(0x%lx000) is not aligned on a size(0x%lx000) boundary\n",
-                       base, size);
+                pr_warning("mtrr: base(0x%lx000) is not aligned on a size(0x%lx000) boundary\n", base, size);
                 return -EINVAL;
         }
         return 0;
 }
 
-
 static int generic_have_wrcomb(void)
 {
         unsigned long config, dummy;
         rdmsr(MSR_MTRRcap, config, dummy);
-        return (config & (1 << 10));
+        return config & (1 << 10);
 }
 
 int positive_have_wrcomb(void)
@@ -716,14 +749,15 @@ int positive_have_wrcomb(void)
         return 1;
 }
 
-/* generic structure...
+/*
+ * Generic structure...
  */
 struct mtrr_ops generic_mtrr_ops = {
-        .use_intel_if      = 1,
-        .set_all           = generic_set_all,
-        .get               = generic_get_mtrr,
-        .get_free_region   = generic_get_free_region,
-        .set               = generic_set_mtrr,
-        .validate_add_page = generic_validate_add_page,
-        .have_wrcomb       = generic_have_wrcomb,
+        .use_intel_if           = 1,
+        .set_all                = generic_set_all,
+        .get                    = generic_get_mtrr,
+        .get_free_region        = generic_get_free_region,
+        .set                    = generic_set_mtrr,
+        .validate_add_page      = generic_validate_add_page,
+        .have_wrcomb            = generic_have_wrcomb,
 };
diff --git a/arch/x86/kernel/cpu/mtrr/if.c b/arch/x86/kernel/cpu/mtrr/if.c
index fb73a52913a4..f04e72527604 100644
--- a/arch/x86/kernel/cpu/mtrr/if.c
+++ b/arch/x86/kernel/cpu/mtrr/if.c
@@ -1,27 +1,28 @@
-#include <linux/init.h>
-#include <linux/proc_fs.h>
 #include <linux/capability.h>
-#include <linux/ctype.h>
-#include <linux/module.h>
 #include <linux/seq_file.h>
-#include <asm/uaccess.h>
+#include <linux/uaccess.h>
+#include <linux/proc_fs.h>
+#include <linux/module.h>
+#include <linux/ctype.h>
+#include <linux/init.h>
 
 #define LINE_SIZE 80
 
 #include <asm/mtrr.h>
+
 #include "mtrr.h"
 
 #define FILE_FCOUNT(f) (((struct seq_file *)((f)->private_data))->private)
 
 static const char *const mtrr_strings[MTRR_NUM_TYPES] =
 {
-    "uncachable",               /* 0 */
-    "write-combining",          /* 1 */
-    "?",                        /* 2 */
-    "?",                        /* 3 */
-    "write-through",            /* 4 */
-    "write-protect",            /* 5 */
-    "write-back",               /* 6 */
+        "uncachable",           /* 0 */
+        "write-combining",      /* 1 */
+        "?",                    /* 2 */
+        "?",                    /* 3 */
+        "write-through",        /* 4 */
+        "write-protect",        /* 5 */
+        "write-back",           /* 6 */
 };
 
 const char *mtrr_attrib_to_str(int x)
@@ -35,8 +36,8 @@ static int
 mtrr_file_add(unsigned long base, unsigned long size,
               unsigned int type, bool increment, struct file *file, int page)
 {
+        unsigned int *fcount = FILE_FCOUNT(file);
         int reg, max;
-        unsigned int *fcount = FILE_FCOUNT(file); 
 
         max = num_var_ranges;
         if (fcount == NULL) {
@@ -61,8 +62,8 @@ static int
 mtrr_file_del(unsigned long base, unsigned long size,
               struct file *file, int page)
 {
-        int reg;
         unsigned int *fcount = FILE_FCOUNT(file);
+        int reg;
 
         if (!page) {
                 if ((base & (PAGE_SIZE - 1)) || (size & (PAGE_SIZE - 1)))
@@ -81,13 +82,14 @@ mtrr_file_del(unsigned long base, unsigned long size,
         return reg;
 }
 
-/* RED-PEN: seq_file can seek now. this is ignored. */
+/*
+ * seq_file can seek but we ignore it.
+ *
+ * Format of control line:
+ *    "base=%Lx size=%Lx type=%s" or "disable=%d"
+ */
 static ssize_t
 mtrr_write(struct file *file, const char __user *buf, size_t len, loff_t * ppos)
-/*  Format of control line:
-    "base=%Lx size=%Lx type=%s"     OR:
-    "disable=%d"
-*/
 {
         int i, err;
         unsigned long reg;
@@ -100,15 +102,18 @@ mtrr_write(struct file *file, const char __user *buf, size_t len, loff_t * ppos)
                 return -EPERM;
         if (!len)
                 return -EINVAL;
+
         memset(line, 0, LINE_SIZE);
         if (len > LINE_SIZE)
                 len = LINE_SIZE;
         if (copy_from_user(line, buf, len - 1))
                 return -EFAULT;
+
         linelen = strlen(line);
         ptr = line + linelen - 1;
         if (linelen && *ptr == '\n')
                 *ptr = '\0';
+
         if (!strncmp(line, "disable=", 8)) {
                 reg = simple_strtoul(line + 8, &ptr, 0);
                 err = mtrr_del_page(reg, 0, 0);
@@ -116,28 +121,35 @@ mtrr_write(struct file *file, const char __user *buf, size_t len, loff_t * ppos)
                         return err;
                 return len;
         }
+
         if (strncmp(line, "base=", 5))
                 return -EINVAL;
+
         base = simple_strtoull(line + 5, &ptr, 0);
-        for (; isspace(*ptr); ++ptr) ;
+        while (isspace(*ptr))
+                ptr++;
+
         if (strncmp(ptr, "size=", 5))
                 return -EINVAL;
+
         size = simple_strtoull(ptr + 5, &ptr, 0);
         if ((base & 0xfff) || (size & 0xfff))
                 return -EINVAL;
-        for (; isspace(*ptr); ++ptr) ;
+        while (isspace(*ptr))
+                ptr++;
+
         if (strncmp(ptr, "type=", 5))
                 return -EINVAL;
         ptr += 5;
-        for (; isspace(*ptr); ++ptr) ;
+        while (isspace(*ptr))
+                ptr++;
+
         for (i = 0; i < MTRR_NUM_TYPES; ++i) {
                 if (strcmp(ptr, mtrr_strings[i]))
                         continue;
                 base >>= PAGE_SHIFT;
                 size >>= PAGE_SHIFT;
-                err =
-                    mtrr_add_page((unsigned long) base, (unsigned long) size, i,
-                                  true);
+                err = mtrr_add_page((unsigned long)base, (unsigned long)size, i, true);
                 if (err < 0)
                         return err;
                 return len;
@@ -181,7 +193,9 @@ mtrr_ioctl(struct file *file, unsigned int cmd, unsigned long __arg)
         case MTRRIOC32_SET_PAGE_ENTRY:
         case MTRRIOC32_DEL_PAGE_ENTRY:
         case MTRRIOC32_KILL_PAGE_ENTRY: {
-                struct mtrr_sentry32 __user *s32 = (struct mtrr_sentry32 __user *)__arg;
+                struct mtrr_sentry32 __user *s32;
+
+                s32 = (struct mtrr_sentry32 __user *)__arg;
                 err = get_user(sentry.base, &s32->base);
                 err |= get_user(sentry.size, &s32->size);
                 err |= get_user(sentry.type, &s32->type);
@@ -191,7 +205,9 @@ mtrr_ioctl(struct file *file, unsigned int cmd, unsigned long __arg)
         }
         case MTRRIOC32_GET_ENTRY:
         case MTRRIOC32_GET_PAGE_ENTRY: {
-                struct mtrr_gentry32 __user *g32 = (struct mtrr_gentry32 __user *)__arg;
+                struct mtrr_gentry32 __user *g32;
+
+                g32 = (struct mtrr_gentry32 __user *)__arg;
                 err = get_user(gentry.regnum, &g32->regnum);
                 err |= get_user(gentry.base, &g32->base);
                 err |= get_user(gentry.size, &g32->size);
@@ -314,7 +330,7 @@ mtrr_ioctl(struct file *file, unsigned int cmd, unsigned long __arg)
         if (err)
                 return err;
 
-        switch(cmd) {
+        switch (cmd) {
         case MTRRIOC_GET_ENTRY:
         case MTRRIOC_GET_PAGE_ENTRY:
                 if (copy_to_user(arg, &gentry, sizeof gentry))
@@ -323,7 +339,9 @@ mtrr_ioctl(struct file *file, unsigned int cmd, unsigned long __arg)
 #ifdef CONFIG_COMPAT
         case MTRRIOC32_GET_ENTRY:
         case MTRRIOC32_GET_PAGE_ENTRY: {
-                struct mtrr_gentry32 __user *g32 = (struct mtrr_gentry32 __user *)__arg;
+                struct mtrr_gentry32 __user *g32;
+
+                g32 = (struct mtrr_gentry32 __user *)__arg;
                 err = put_user(gentry.base, &g32->base);
                 err |= put_user(gentry.size, &g32->size);
                 err |= put_user(gentry.regnum, &g32->regnum);
@@ -335,11 +353,10 @@ mtrr_ioctl(struct file *file, unsigned int cmd, unsigned long __arg)
         return err;
 }
 
-static int
-mtrr_close(struct inode *ino, struct file *file)
+static int mtrr_close(struct inode *ino, struct file *file)
 {
-        int i, max;
         unsigned int *fcount = FILE_FCOUNT(file);
+        int i, max;
 
         if (fcount != NULL) {
                 max = num_var_ranges;
@@ -359,22 +376,22 @@ static int mtrr_seq_show(struct seq_file *seq, void *offset);
 
 static int mtrr_open(struct inode *inode, struct file *file)
 {
-        if (!mtrr_if) 
+        if (!mtrr_if)
                 return -EIO;
-        if (!mtrr_if->get) 
-                return -ENXIO; 
+        if (!mtrr_if->get)
+                return -ENXIO;
         return single_open(file, mtrr_seq_show, NULL);
 }
 
 static const struct file_operations mtrr_fops = {
-        .owner   = THIS_MODULE,
-        .open    = mtrr_open, 
-        .read    = seq_read,
-        .llseek  = seq_lseek,
-        .write   = mtrr_write,
-        .unlocked_ioctl = mtrr_ioctl,
-        .compat_ioctl = mtrr_ioctl,
-        .release = mtrr_close,
+        .owner                  = THIS_MODULE,
+        .open                   = mtrr_open,
+        .read                   = seq_read,
+        .llseek                 = seq_lseek,
+        .write                  = mtrr_write,
+        .unlocked_ioctl         = mtrr_ioctl,
+        .compat_ioctl           = mtrr_ioctl,
+        .release                = mtrr_close,
 };
 
 static int mtrr_seq_show(struct seq_file *seq, void *offset)
@@ -388,23 +405,24 @@ static int mtrr_seq_show(struct seq_file *seq, void *offset)
         max = num_var_ranges;
         for (i = 0; i < max; i++) {
                 mtrr_if->get(i, &base, &size, &type);
-                if (size == 0)
+                if (size == 0) {
                         mtrr_usage_table[i] = 0;
-                else {
-                        if (size < (0x100000 >> PAGE_SHIFT)) {
-                                /* less than 1MB */
-                                factor = 'K';
-                                size <<= PAGE_SHIFT - 10;
-                        } else {
-                                factor = 'M';
-                                size >>= 20 - PAGE_SHIFT;
-                        }
-                        /* RED-PEN: base can be > 32bit */ 
-                        len += seq_printf(seq, 
-                                   "reg%02i: base=0x%06lx000 (%5luMB), size=%5lu%cB, count=%d: %s\n",
-                             i, base, base >> (20 - PAGE_SHIFT), size, factor,
-                             mtrr_usage_table[i], mtrr_attrib_to_str(type));
+                        continue;
                 }
+                if (size < (0x100000 >> PAGE_SHIFT)) {
+                        /* less than 1MB */
+                        factor = 'K';
+                        size <<= PAGE_SHIFT - 10;
+                } else {
+                        factor = 'M';
+                        size >>= 20 - PAGE_SHIFT;
+                }
+                /* Base can be > 32bit */
+                len += seq_printf(seq, "reg%02i: base=0x%06lx000 "
+                        "(%5luMB), size=%5lu%cB, count=%d: %s\n",
+                        i, base, base >> (20 - PAGE_SHIFT), size,
+                        factor, mtrr_usage_table[i],
+                        mtrr_attrib_to_str(type));
         }
         return 0;
 }
@@ -422,6 +440,5 @@ static int __init mtrr_if_init(void)
         proc_create("mtrr", S_IWUSR | S_IRUGO, NULL, &mtrr_fops);
         return 0;
 }
-
 arch_initcall(mtrr_if_init);
 #endif                  /*  CONFIG_PROC_FS  */
diff --git a/arch/x86/kernel/cpu/mtrr/main.c b/arch/x86/kernel/cpu/mtrr/main.c
index 8fc248b5aeaf..84e83de54575 100644
--- a/arch/x86/kernel/cpu/mtrr/main.c
+++ b/arch/x86/kernel/cpu/mtrr/main.c
@@ -25,43 +25,49 @@
     Operating System Writer's Guide" (Intel document number 242692),
     section 11.11.7
 
-    This was cleaned and made readable by Patrick Mochel <mochel@osdl.org> 
-    on 6-7 March 2002. 
-    Source: Intel Architecture Software Developers Manual, Volume 3: 
+    This was cleaned and made readable by Patrick Mochel <mochel@osdl.org>
+    on 6-7 March 2002.
+    Source: Intel Architecture Software Developers Manual, Volume 3:
     System Programming Guide; Section 9.11. (1997 edition - PPro).
 */
 
+#define DEBUG
+
+#include <linux/types.h> /* FIXME: kvm_para.h needs this */
+
+#include <linux/kvm_para.h>
+#include <linux/uaccess.h>
 #include <linux/module.h>
+#include <linux/mutex.h>
 #include <linux/init.h>
+#include <linux/sort.h>
+#include <linux/cpu.h>
 #include <linux/pci.h>
 #include <linux/smp.h>
-#include <linux/cpu.h>
-#include <linux/mutex.h>
-#include <linux/sort.h>
 
+#include <asm/processor.h>
 #include <asm/e820.h>
 #include <asm/mtrr.h>
-#include <asm/uaccess.h>
-#include <asm/processor.h>
 #include <asm/msr.h>
-#include <asm/kvm_para.h>
+
 #include "mtrr.h"
 
-u32 num_var_ranges = 0;
+u32 num_var_ranges;
 
 unsigned int mtrr_usage_table[MTRR_MAX_VAR_RANGES];
 static DEFINE_MUTEX(mtrr_mutex);
 
 u64 size_or_mask, size_and_mask;
+static bool mtrr_aps_delayed_init;
 
-static struct mtrr_ops * mtrr_ops[X86_VENDOR_NUM] = {};
+static struct mtrr_ops *mtrr_ops[X86_VENDOR_NUM];
 
-struct mtrr_ops * mtrr_if = NULL;
+struct mtrr_ops *mtrr_if;
 
 static void set_mtrr(unsigned int reg, unsigned long base,
                      unsigned long size, mtrr_type type);
 
-void set_mtrr_ops(struct mtrr_ops * ops)
+void set_mtrr_ops(struct mtrr_ops *ops)
 {
         if (ops->vendor && ops->vendor < X86_VENDOR_NUM)
                 mtrr_ops[ops->vendor] = ops;
@@ -72,30 +78,36 @@ static int have_wrcomb(void)
 {
         struct pci_dev *dev;
         u8 rev;
-        
-        if ((dev = pci_get_class(PCI_CLASS_BRIDGE_HOST << 8, NULL)) != NULL) {
-                /* ServerWorks LE chipsets < rev 6 have problems with write-combining
-                   Don't allow it and leave room for other chipsets to be tagged */
+
+        dev = pci_get_class(PCI_CLASS_BRIDGE_HOST << 8, NULL);
+        if (dev != NULL) {
+                /*
+                 * ServerWorks LE chipsets < rev 6 have problems with
+                 * write-combining. Don't allow it and leave room for other
+                 * chipsets to be tagged
+                 */
                 if (dev->vendor == PCI_VENDOR_ID_SERVERWORKS &&
                     dev->device == PCI_DEVICE_ID_SERVERWORKS_LE) {
                         pci_read_config_byte(dev, PCI_CLASS_REVISION, &rev);
                         if (rev <= 5) {
-                                printk(KERN_INFO "mtrr: Serverworks LE rev < 6 detected. Write-combining disabled.\n");
+                                pr_info("mtrr: Serverworks LE rev < 6 detected. Write-combining disabled.\n");
                                 pci_dev_put(dev);
                                 return 0;
                         }
                 }
-                /* Intel 450NX errata # 23. Non ascending cacheline evictions to
-                   write combining memory may resulting in data corruption */
+                /*
+                 * Intel 450NX errata # 23. Non ascending cacheline evictions to
+                 * write combining memory may resulting in data corruption
+                 */
                 if (dev->vendor == PCI_VENDOR_ID_INTEL &&
                     dev->device == PCI_DEVICE_ID_INTEL_82451NX) {
-                        printk(KERN_INFO "mtrr: Intel 450NX MMC detected. Write-combining disabled.\n");
+                        pr_info("mtrr: Intel 450NX MMC detected. Write-combining disabled.\n");
                         pci_dev_put(dev);
                         return 0;
                 }
                 pci_dev_put(dev);
-        }               
-        return (mtrr_if->have_wrcomb ? mtrr_if->have_wrcomb() : 0);
+        }
+        return mtrr_if->have_wrcomb ? mtrr_if->have_wrcomb() : 0;
 }
 
 /*  This function returns the number of variable MTRRs  */
@@ -103,12 +115,13 @@ static void __init set_num_var_ranges(void)
 {
         unsigned long config = 0, dummy;
 
-        if (use_intel()) {
+        if (use_intel())
                 rdmsr(MSR_MTRRcap, config, dummy);
-        } else if (is_cpu(AMD))
+        else if (is_cpu(AMD))
                 config = 2;
         else if (is_cpu(CYRIX) || is_cpu(CENTAUR))
                 config = 8;
+
         num_var_ranges = config & 0xff;
 }
 
@@ -130,10 +143,12 @@ struct set_mtrr_data {
         mtrr_type       smp_type;
 };
 
+/**
+ * ipi_handler - Synchronisation handler. Executed by "other" CPUs.
+ *
+ * Returns nothing.
+ */
 static void ipi_handler(void *info)
-/*  [SUMMARY] Synchronisation handler. Executed by "other" CPUs.
-    [RETURNS] Nothing.
-*/
 {
 #ifdef CONFIG_SMP
         struct set_mtrr_data *data = info;
@@ -142,18 +157,22 @@ static void ipi_handler(void *info)
         local_irq_save(flags);
 
         atomic_dec(&data->count);
-        while(!atomic_read(&data->gate))
+        while (!atomic_read(&data->gate))
                 cpu_relax();
 
         /*  The master has cleared me to execute  */
-        if (data->smp_reg != ~0U) 
-                mtrr_if->set(data->smp_reg, data->smp_base, 
+        if (data->smp_reg != ~0U) {
+                mtrr_if->set(data->smp_reg, data->smp_base,
                              data->smp_size, data->smp_type);
-        else
+        } else if (mtrr_aps_delayed_init) {
+                /*
+                 * Initialize the MTRRs inaddition to the synchronisation.
+                 */
                 mtrr_if->set_all();
+        }
 
         atomic_dec(&data->count);
-        while(atomic_read(&data->gate))
+        while (atomic_read(&data->gate))
                 cpu_relax();
 
         atomic_dec(&data->count);
@@ -161,7 +180,8 @@ static void ipi_handler(void *info)
 #endif
 }
 
-static inline int types_compatible(mtrr_type type1, mtrr_type type2) {
+static inline int types_compatible(mtrr_type type1, mtrr_type type2)
+{
         return type1 == MTRR_TYPE_UNCACHABLE ||
                type2 == MTRR_TYPE_UNCACHABLE ||
                (type1 == MTRR_TYPE_WRTHROUGH && type2 == MTRR_TYPE_WRBACK) ||
@@ -176,10 +196,10 @@ static inline int types_compatible(mtrr_type type1, mtrr_type type2) {
  * @type:       mtrr type
  *
  * This is kinda tricky, but fortunately, Intel spelled it out for us cleanly:
- * 
+ *
  * 1. Send IPI to do the following:
  * 2. Disable Interrupts
- * 3. Wait for all procs to do so 
+ * 3. Wait for all procs to do so
  * 4. Enter no-fill cache mode
  * 5. Flush caches
  * 6. Clear PGE bit
@@ -189,26 +209,27 @@ static inline int types_compatible(mtrr_type type1, mtrr_type type2) {
  * 10. Enable all range registers
  * 11. Flush all TLBs and caches again
  * 12. Enter normal cache mode and reenable caching
- * 13. Set PGE 
+ * 13. Set PGE
  * 14. Wait for buddies to catch up
  * 15. Enable interrupts.
- * 
+ *
  * What does that mean for us? Well, first we set data.count to the number
  * of CPUs. As each CPU disables interrupts, it'll decrement it once. We wait
  * until it hits 0 and proceed. We set the data.gate flag and reset data.count.
- * Meanwhile, they are waiting for that flag to be set. Once it's set, each 
- * CPU goes through the transition of updating MTRRs. The CPU vendors may each do it 
- * differently, so we call mtrr_if->set() callback and let them take care of it.
- * When they're done, they again decrement data->count and wait for data.gate to 
- * be reset. 
- * When we finish, we wait for data.count to hit 0 and toggle the data.gate flag.
+ * Meanwhile, they are waiting for that flag to be set. Once it's set, each
+ * CPU goes through the transition of updating MTRRs.
+ * The CPU vendors may each do it differently,
+ * so we call mtrr_if->set() callback and let them take care of it.
+ * When they're done, they again decrement data->count and wait for data.gate
+ * to be reset.
+ * When we finish, we wait for data.count to hit 0 and toggle the data.gate flag
  * Everyone then enables interrupts and we all continue on.
  *
  * Note that the mechanism is the same for UP systems, too; all the SMP stuff
  * becomes nops.
  */
-static void set_mtrr(unsigned int reg, unsigned long base,
-                     unsigned long size, mtrr_type type)
+static void
+set_mtrr(unsigned int reg, unsigned long base, unsigned long size, mtrr_type type)
 {
         struct set_mtrr_data data;
         unsigned long flags;
@@ -218,121 +239,124 @@ static void set_mtrr(unsigned int reg, unsigned long base,
         data.smp_size = size;
         data.smp_type = type;
         atomic_set(&data.count, num_booting_cpus() - 1);
-        /* make sure data.count is visible before unleashing other CPUs */
+
+        /* Make sure data.count is visible before unleashing other CPUs */
         smp_wmb();
-        atomic_set(&data.gate,0);
+        atomic_set(&data.gate, 0);
 
-        /*  Start the ball rolling on other CPUs  */
+        /* Start the ball rolling on other CPUs */
         if (smp_call_function(ipi_handler, &data, 0) != 0)
                 panic("mtrr: timed out waiting for other CPUs\n");
 
         local_irq_save(flags);
 
-        while(atomic_read(&data.count))
+        while (atomic_read(&data.count))
                 cpu_relax();
 
-        /* ok, reset count and toggle gate */
+        /* Ok, reset count and toggle gate */
         atomic_set(&data.count, num_booting_cpus() - 1);
         smp_wmb();
-        atomic_set(&data.gate,1);
+        atomic_set(&data.gate, 1);
 
-        /* do our MTRR business */
+        /* Do our MTRR business */
 
-        /* HACK!
+        /*
+         * HACK!
          * We use this same function to initialize the mtrrs on boot.
          * The state of the boot cpu's mtrrs has been saved, and we want
-         * to replicate across all the APs. 
+         * to replicate across all the APs.
          * If we're doing that @reg is set to something special...
          */
-        if (reg != ~0U) 
-                mtrr_if->set(reg,base,size,type);
+        if (reg != ~0U)
+                mtrr_if->set(reg, base, size, type);
+        else if (!mtrr_aps_delayed_init)
+                mtrr_if->set_all();
 
-        /* wait for the others */
-        while(atomic_read(&data.count))
+        /* Wait for the others */
+        while (atomic_read(&data.count))
                 cpu_relax();
 
         atomic_set(&data.count, num_booting_cpus() - 1);
         smp_wmb();
-        atomic_set(&data.gate,0);
+        atomic_set(&data.gate, 0);
 
         /*
          * Wait here for everyone to have seen the gate change
          * So we're the last ones to touch 'data'
          */
-        while(atomic_read(&data.count))
+        while (atomic_read(&data.count))
                 cpu_relax();
 
         local_irq_restore(flags);
 }
 
 /**
- *      mtrr_add_page - Add a memory type region
- *      @base: Physical base address of region in pages (in units of 4 kB!)
- *      @size: Physical size of region in pages (4 kB)
- *      @type: Type of MTRR desired
- *      @increment: If this is true do usage counting on the region
+ * mtrr_add_page - Add a memory type region
+ * @base: Physical base address of region in pages (in units of 4 kB!)
+ * @size: Physical size of region in pages (4 kB)
+ * @type: Type of MTRR desired
+ * @increment: If this is true do usage counting on the region
  *
- *      Memory type region registers control the caching on newer Intel and
- *      non Intel processors. This function allows drivers to request an
- *      MTRR is added. The details and hardware specifics of each processor's
- *      implementation are hidden from the caller, but nevertheless the 
- *      caller should expect to need to provide a power of two size on an
- *      equivalent power of two boundary.
+ * Memory type region registers control the caching on newer Intel and
+ * non Intel processors. This function allows drivers to request an
+ * MTRR is added. The details and hardware specifics of each processor's
+ * implementation are hidden from the caller, but nevertheless the
+ * caller should expect to need to provide a power of two size on an
+ * equivalent power of two boundary.
  *
- *      If the region cannot be added either because all regions are in use
- *      or the CPU cannot support it a negative value is returned. On success
- *      the register number for this entry is returned, but should be treated
- *      as a cookie only.
+ * If the region cannot be added either because all regions are in use
+ * or the CPU cannot support it a negative value is returned. On success
+ * the register number for this entry is returned, but should be treated
+ * as a cookie only.
  *
- *      On a multiprocessor machine the changes are made to all processors.
- *      This is required on x86 by the Intel processors.
+ * On a multiprocessor machine the changes are made to all processors.
+ * This is required on x86 by the Intel processors.
  *
- *      The available types are
+ * The available types are
  *
- *      %MTRR_TYPE_UNCACHABLE   -       No caching
+ * %MTRR_TYPE_UNCACHABLE - No caching
  *
- *      %MTRR_TYPE_WRBACK       -       Write data back in bursts whenever
+ * %MTRR_TYPE_WRBACK - Write data back in bursts whenever
  *
- *      %MTRR_TYPE_WRCOMB       -       Write data back soon but allow bursts
+ * %MTRR_TYPE_WRCOMB - Write data back soon but allow bursts
  *
- *      %MTRR_TYPE_WRTHROUGH    -       Cache reads but not writes
+ * %MTRR_TYPE_WRTHROUGH - Cache reads but not writes
  *
- *      BUGS: Needs a quiet flag for the cases where drivers do not mind
- *      failures and do not wish system log messages to be sent.
+ * BUGS: Needs a quiet flag for the cases where drivers do not mind
+ * failures and do not wish system log messages to be sent.
  */
-
-int mtrr_add_page(unsigned long base, unsigned long size, 
+int mtrr_add_page(unsigned long base, unsigned long size,
                   unsigned int type, bool increment)
 {
+        unsigned long lbase, lsize;
         int i, replace, error;
         mtrr_type ltype;
-        unsigned long lbase, lsize;
 
         if (!mtrr_if)
                 return -ENXIO;
-                
-        if ((error = mtrr_if->validate_add_page(base,size,type)))
+
+        error = mtrr_if->validate_add_page(base, size, type);
+        if (error)
                 return error;
 
         if (type >= MTRR_NUM_TYPES) {
-                printk(KERN_WARNING "mtrr: type: %u invalid\n", type);
+                pr_warning("mtrr: type: %u invalid\n", type);
                 return -EINVAL;
         }
 
-        /*  If the type is WC, check that this processor supports it  */
+        /* If the type is WC, check that this processor supports it */
         if ((type == MTRR_TYPE_WRCOMB) && !have_wrcomb()) {
-                printk(KERN_WARNING
-                       "mtrr: your processor doesn't support write-combining\n");
+                pr_warning("mtrr: your processor doesn't support write-combining\n");
                 return -ENOSYS;
         }
 
         if (!size) {
-                printk(KERN_WARNING "mtrr: zero sized request\n");
+                pr_warning("mtrr: zero sized request\n");
                 return -EINVAL;
         }
 
         if (base & size_or_mask || size & size_or_mask) {
-                printk(KERN_WARNING "mtrr: base or size exceeds the MTRR width\n");
+                pr_warning("mtrr: base or size exceeds the MTRR width\n");
                 return -EINVAL;
         }
 
@@ -341,36 +365,40 @@ int mtrr_add_page(unsigned long base, unsigned long size,
 
         /* No CPU hotplug when we change MTRR entries */
         get_online_cpus();
-        /*  Search for existing MTRR  */
+
+        /* Search for existing MTRR  */
         mutex_lock(&mtrr_mutex);
         for (i = 0; i < num_var_ranges; ++i) {
                 mtrr_if->get(i, &lbase, &lsize, &ltype);
-                if (!lsize || base > lbase + lsize - 1 || base + size - 1 < lbase)
+                if (!lsize || base > lbase + lsize - 1 ||
+                    base + size - 1 < lbase)
                         continue;
-                /*  At this point we know there is some kind of overlap/enclosure  */
+                /*
+                 * At this point we know there is some kind of
+                 * overlap/enclosure
+                 */
                 if (base < lbase || base + size - 1 > lbase + lsize - 1) {
-                        if (base <= lbase && base + size - 1 >= lbase + lsize - 1) {
+                        if (base <= lbase &&
+                            base + size - 1 >= lbase + lsize - 1) {
                                 /*  New region encloses an existing region  */
                                 if (type == ltype) {
                                         replace = replace == -1 ? i : -2;
                                         continue;
-                                }
-                                else if (types_compatible(type, ltype))
+                                } else if (types_compatible(type, ltype))
                                         continue;
                         }
-                        printk(KERN_WARNING
-                               "mtrr: 0x%lx000,0x%lx000 overlaps existing"
-                               " 0x%lx000,0x%lx000\n", base, size, lbase,
-                               lsize);
+                        pr_warning("mtrr: 0x%lx000,0x%lx000 overlaps existing"
+                                " 0x%lx000,0x%lx000\n", base, size, lbase,
+                                lsize);
                         goto out;
                 }
-                /*  New region is enclosed by an existing region  */
+                /* New region is enclosed by an existing region */
                 if (ltype != type) {
                         if (types_compatible(type, ltype))
                                 continue;
-                        printk (KERN_WARNING "mtrr: type mismatch for %lx000,%lx000 old: %s new: %s\n",
-                             base, size, mtrr_attrib_to_str(ltype),
-                             mtrr_attrib_to_str(type));
+                        pr_warning("mtrr: type mismatch for %lx000,%lx000 old: %s new: %s\n",
+                                base, size, mtrr_attrib_to_str(ltype),
+                                mtrr_attrib_to_str(type));
                         goto out;
                 }
                 if (increment)
@@ -378,7 +406,7 @@ int mtrr_add_page(unsigned long base, unsigned long size,
                 error = i;
                 goto out;
         }
-        /*  Search for an empty MTRR  */
+        /* Search for an empty MTRR */
         i = mtrr_if->get_free_region(base, size, replace);
         if (i >= 0) {
                 set_mtrr(i, base, size, type);
@@ -393,8 +421,9 @@ int mtrr_add_page(unsigned long base, unsigned long size,
                                 mtrr_usage_table[replace] = 0;
                         }
                 }
-        } else
-                printk(KERN_INFO "mtrr: no more MTRRs available\n");
+        } else {
+                pr_info("mtrr: no more MTRRs available\n");
+        }
         error = i;
  out:
         mutex_unlock(&mtrr_mutex);
@@ -405,10 +434,8 @@ int mtrr_add_page(unsigned long base, unsigned long size,
 static int mtrr_check(unsigned long base, unsigned long size)
 {
         if ((base & (PAGE_SIZE - 1)) || (size & (PAGE_SIZE - 1))) {
-                printk(KERN_WARNING
-                        "mtrr: size and base must be multiples of 4 kiB\n");
-                printk(KERN_DEBUG
-                        "mtrr: size: 0x%lx  base: 0x%lx\n", size, base);
+                pr_warning("mtrr: size and base must be multiples of 4 kiB\n");
+                pr_debug("mtrr: size: 0x%lx  base: 0x%lx\n", size, base);
                 dump_stack();
                 return -1;
         }
@@ -416,66 +443,64 @@ static int mtrr_check(unsigned long base, unsigned long size)
 }
 
 /**
- *      mtrr_add - Add a memory type region
- *      @base: Physical base address of region
- *      @size: Physical size of region
- *      @type: Type of MTRR desired
- *      @increment: If this is true do usage counting on the region
+ * mtrr_add - Add a memory type region
+ * @base: Physical base address of region
+ * @size: Physical size of region
+ * @type: Type of MTRR desired
+ * @increment: If this is true do usage counting on the region
  *
- *      Memory type region registers control the caching on newer Intel and
- *      non Intel processors. This function allows drivers to request an
- *      MTRR is added. The details and hardware specifics of each processor's
- *      implementation are hidden from the caller, but nevertheless the 
- *      caller should expect to need to provide a power of two size on an
- *      equivalent power of two boundary.
+ * Memory type region registers control the caching on newer Intel and
+ * non Intel processors. This function allows drivers to request an
+ * MTRR is added. The details and hardware specifics of each processor's
+ * implementation are hidden from the caller, but nevertheless the
+ * caller should expect to need to provide a power of two size on an
+ * equivalent power of two boundary.
  *
- *      If the region cannot be added either because all regions are in use
- *      or the CPU cannot support it a negative value is returned. On success
- *      the register number for this entry is returned, but should be treated
- *      as a cookie only.
+ * If the region cannot be added either because all regions are in use
+ * or the CPU cannot support it a negative value is returned. On success
+ * the register number for this entry is returned, but should be treated
+ * as a cookie only.
  *
- *      On a multiprocessor machine the changes are made to all processors.
- *      This is required on x86 by the Intel processors.
+ * On a multiprocessor machine the changes are made to all processors.
+ * This is required on x86 by the Intel processors.
  *
- *      The available types are
+ * The available types are
  *
- *      %MTRR_TYPE_UNCACHABLE   -       No caching
+ * %MTRR_TYPE_UNCACHABLE - No caching
  *
- *      %MTRR_TYPE_WRBACK       -       Write data back in bursts whenever
+ * %MTRR_TYPE_WRBACK - Write data back in bursts whenever
  *
- *      %MTRR_TYPE_WRCOMB       -       Write data back soon but allow bursts
+ * %MTRR_TYPE_WRCOMB - Write data back soon but allow bursts
  *
- *      %MTRR_TYPE_WRTHROUGH    -       Cache reads but not writes
+ * %MTRR_TYPE_WRTHROUGH - Cache reads but not writes
  *
- *      BUGS: Needs a quiet flag for the cases where drivers do not mind
- *      failures and do not wish system log messages to be sent.
+ * BUGS: Needs a quiet flag for the cases where drivers do not mind
+ * failures and do not wish system log messages to be sent.
  */
-
-int
-mtrr_add(unsigned long base, unsigned long size, unsigned int type,
-         bool increment)
+int mtrr_add(unsigned long base, unsigned long size, unsigned int type,
+             bool increment)
 {
         if (mtrr_check(base, size))
                 return -EINVAL;
         return mtrr_add_page(base >> PAGE_SHIFT, size >> PAGE_SHIFT, type,
                              increment);
 }
+EXPORT_SYMBOL(mtrr_add);
 
 /**
- *      mtrr_del_page - delete a memory type region
- *      @reg: Register returned by mtrr_add
- *      @base: Physical base address
- *      @size: Size of region
+ * mtrr_del_page - delete a memory type region
+ * @reg: Register returned by mtrr_add
+ * @base: Physical base address
+ * @size: Size of region
  *
- *      If register is supplied then base and size are ignored. This is
- *      how drivers should call it.
+ * If register is supplied then base and size are ignored. This is
+ * how drivers should call it.
  *
- *      Releases an MTRR region. If the usage count drops to zero the 
- *      register is freed and the region returns to default state.
- *      On success the register is returned, on failure a negative error
- *      code.
+ * Releases an MTRR region. If the usage count drops to zero the
+ * register is freed and the region returns to default state.
+ * On success the register is returned, on failure a negative error
+ * code.
  */
-
 int mtrr_del_page(int reg, unsigned long base, unsigned long size)
 {
         int i, max;
@@ -500,22 +525,22 @@ int mtrr_del_page(int reg, unsigned long base, unsigned long size)
                         }
                 }
                 if (reg < 0) {
-                        printk(KERN_DEBUG "mtrr: no MTRR for %lx000,%lx000 found\n", base,
-                               size);
+                        pr_debug("mtrr: no MTRR for %lx000,%lx000 found\n",
+                                 base, size);
                         goto out;
                 }
         }
         if (reg >= max) {
-                printk(KERN_WARNING "mtrr: register: %d too big\n", reg);
+                pr_warning("mtrr: register: %d too big\n", reg);
                 goto out;
         }
         mtrr_if->get(reg, &lbase, &lsize, &ltype);
         if (lsize < 1) {
-                printk(KERN_WARNING "mtrr: MTRR %d not used\n", reg);
+                pr_warning("mtrr: MTRR %d not used\n", reg);
                 goto out;
         }
         if (mtrr_usage_table[reg] < 1) {
-                printk(KERN_WARNING "mtrr: reg: %d has count=0\n", reg);
+                pr_warning("mtrr: reg: %d has count=0\n", reg);
                 goto out;
         }
         if (--mtrr_usage_table[reg] < 1)
@@ -526,33 +551,31 @@ int mtrr_del_page(int reg, unsigned long base, unsigned long size)
         put_online_cpus();
         return error;
 }
+
 /**
- *      mtrr_del - delete a memory type region
- *      @reg: Register returned by mtrr_add
- *      @base: Physical base address
- *      @size: Size of region
+ * mtrr_del - delete a memory type region
+ * @reg: Register returned by mtrr_add
+ * @base: Physical base address
+ * @size: Size of region
  *
- *      If register is supplied then base and size are ignored. This is
- *      how drivers should call it.
+ * If register is supplied then base and size are ignored. This is
+ * how drivers should call it.
  *
- *      Releases an MTRR region. If the usage count drops to zero the 
- *      register is freed and the region returns to default state.
- *      On success the register is returned, on failure a negative error
- *      code.
+ * Releases an MTRR region. If the usage count drops to zero the
+ * register is freed and the region returns to default state.
+ * On success the register is returned, on failure a negative error
+ * code.
  */
-
-int
-mtrr_del(int reg, unsigned long base, unsigned long size)
+int mtrr_del(int reg, unsigned long base, unsigned long size)
 {
         if (mtrr_check(base, size))
                 return -EINVAL;
         return mtrr_del_page(reg, base >> PAGE_SHIFT, size >> PAGE_SHIFT);
 }
-
-EXPORT_SYMBOL(mtrr_add);
 EXPORT_SYMBOL(mtrr_del);
 
-/* HACK ALERT!
+/*
+ * HACK ALERT!
  * These should be called implicitly, but we can't yet until all the initcall
  * stuff is done...
  */
@@ -576,29 +599,28 @@ struct mtrr_value {
 
 static struct mtrr_value mtrr_value[MTRR_MAX_VAR_RANGES];
 
-static int mtrr_save(struct sys_device * sysdev, pm_message_t state)
+static int mtrr_save(struct sys_device *sysdev, pm_message_t state)
 {
         int i;
 
         for (i = 0; i < num_var_ranges; i++) {
-                mtrr_if->get(i,
-                             &mtrr_value[i].lbase,
-                             &mtrr_value[i].lsize,
-                             &mtrr_value[i].ltype);
+                mtrr_if->get(i, &mtrr_value[i].lbase,
+                                &mtrr_value[i].lsize,
+                                &mtrr_value[i].ltype);
         }
         return 0;
 }
 
-static int mtrr_restore(struct sys_device * sysdev)
+static int mtrr_restore(struct sys_device *sysdev)
 {
         int i;
 
         for (i = 0; i < num_var_ranges; i++) {
-                if (mtrr_value[i].lsize)
-                        set_mtrr(i,
-                                 mtrr_value[i].lbase,
-                                 mtrr_value[i].lsize,
-                                 mtrr_value[i].ltype);
+                if (mtrr_value[i].lsize) {
+                        set_mtrr(i, mtrr_value[i].lbase,
+                                    mtrr_value[i].lsize,
+                                    mtrr_value[i].ltype);
+                }
         }
         return 0;
 }
@@ -615,26 +637,29 @@ int __initdata changed_by_mtrr_cleanup;
 /**
  * mtrr_bp_init - initialize mtrrs on the boot CPU
  *
- * This needs to be called early; before any of the other CPUs are 
+ * This needs to be called early; before any of the other CPUs are
  * initialized (i.e. before smp_init()).
- * 
+ *
  */
 void __init mtrr_bp_init(void)
 {
         u32 phys_addr;
+
         init_ifs();
 
         phys_addr = 32;
 
         if (cpu_has_mtrr) {
                 mtrr_if = &generic_mtrr_ops;
-                size_or_mask = 0xff000000;      /* 36 bits */
+                size_or_mask = 0xff000000;                      /* 36 bits */
                 size_and_mask = 0x00f00000;
                 phys_addr = 36;
 
-                /* This is an AMD specific MSR, but we assume(hope?) that
-                   Intel will implement it to when they extend the address
-                   bus of the Xeon. */
+                /*
+                 * This is an AMD specific MSR, but we assume(hope?) that
+                 * Intel will implement it to when they extend the address
+                 * bus of the Xeon.
+                 */
                 if (cpuid_eax(0x80000000) >= 0x80000008) {
                         phys_addr = cpuid_eax(0x80000008) & 0xff;
                         /* CPUID workaround for Intel 0F33/0F34 CPU */
@@ -649,9 +674,11 @@ void __init mtrr_bp_init(void)
                         size_and_mask = ~size_or_mask & 0xfffff00000ULL;
                 } else if (boot_cpu_data.x86_vendor == X86_VENDOR_CENTAUR &&
                            boot_cpu_data.x86 == 6) {
-                        /* VIA C* family have Intel style MTRRs, but
-                           don't support PAE */
-                        size_or_mask = 0xfff00000;      /* 32 bits */
+                        /*
+                         * VIA C* family have Intel style MTRRs,
+                         * but don't support PAE
+                         */
+                        size_or_mask = 0xfff00000;              /* 32 bits */
                         size_and_mask = 0;
                         phys_addr = 32;
                 }
@@ -694,30 +721,28 @@ void __init mtrr_bp_init(void)
                                 changed_by_mtrr_cleanup = 1;
                                 mtrr_if->set_all();
                         }
-
                 }
         }
 }
 
 void mtrr_ap_init(void)
 {
-        unsigned long flags;
-
-        if (!mtrr_if || !use_intel())
+        if (!use_intel() || mtrr_aps_delayed_init)
                 return;
         /*
-         * Ideally we should hold mtrr_mutex here to avoid mtrr entries changed,
-         * but this routine will be called in cpu boot time, holding the lock
-         * breaks it. This routine is called in two cases: 1.very earily time
-         * of software resume, when there absolutely isn't mtrr entry changes;
-         * 2.cpu hotadd time. We let mtrr_add/del_page hold cpuhotplug lock to
-         * prevent mtrr entry changes
+         * Ideally we should hold mtrr_mutex here to avoid mtrr entries
+         * changed, but this routine will be called in cpu boot time,
+         * holding the lock breaks it.
+         *
+         * This routine is called in two cases:
+         *
+         *   1. very earily time of software resume, when there absolutely
+         *      isn't mtrr entry changes;
+         *
+         *   2. cpu hotadd time. We let mtrr_add/del_page hold cpuhotplug
+         *      lock to prevent mtrr entry changes
          */
-        local_irq_save(flags);
-
-        mtrr_if->set_all();
-
-        local_irq_restore(flags);
+        set_mtrr(~0U, 0, 0, 0);
 }
 
 /**
@@ -728,23 +753,55 @@ void mtrr_save_state(void)
         smp_call_function_single(0, mtrr_save_fixed_ranges, NULL, 1);
 }
 
+void set_mtrr_aps_delayed_init(void)
+{
+        if (!use_intel())
+                return;
+
+        mtrr_aps_delayed_init = true;
+}
+
+/*
+ * MTRR initialization for all AP's
+ */
+void mtrr_aps_init(void)
+{
+        if (!use_intel())
+                return;
+
+        set_mtrr(~0U, 0, 0, 0);
+        mtrr_aps_delayed_init = false;
+}
+
+void mtrr_bp_restore(void)
+{
+        if (!use_intel())
+                return;
+
+        mtrr_if->set_all();
+}
+
 static int __init mtrr_init_finialize(void)
 {
         if (!mtrr_if)
                 return 0;
+
         if (use_intel()) {
                 if (!changed_by_mtrr_cleanup)
                         mtrr_state_warn();
-        } else {
-                /* The CPUs haven't MTRR and seem to not support SMP. They have
-                 * specific drivers, we use a tricky method to support
-                 * suspend/resume for them.
-                 * TBD: is there any system with such CPU which supports
-                 * suspend/resume?  if no, we should remove the code.
-                 */
-                sysdev_driver_register(&cpu_sysdev_class,
-                        &mtrr_sysdev_driver);
+                return 0;
         }
+
+        /*
+         * The CPU has no MTRR and seems to not support SMP. They have
+         * specific drivers, we use a tricky method to support
+         * suspend/resume for them.
+         *
+         * TBD: is there any system with such CPU which supports
+         * suspend/resume? If no, we should remove the code.
+         */
+        sysdev_driver_register(&cpu_sysdev_class, &mtrr_sysdev_driver);
+
         return 0;
 }
 subsys_initcall(mtrr_init_finialize);
diff --git a/arch/x86/kernel/cpu/mtrr/mtrr.h b/arch/x86/kernel/cpu/mtrr/mtrr.h
index 7538b767f206..a501dee9a87a 100644
--- a/arch/x86/kernel/cpu/mtrr/mtrr.h
+++ b/arch/x86/kernel/cpu/mtrr/mtrr.h
@@ -1,5 +1,5 @@
 /*
- * local mtrr defines.
+ * local MTRR defines.
  */
 
 #include <linux/types.h>
@@ -14,13 +14,12 @@ extern unsigned int mtrr_usage_table[MTRR_MAX_VAR_RANGES];
 struct mtrr_ops {
         u32     vendor;
         u32     use_intel_if;
-//      void    (*init)(void);
         void    (*set)(unsigned int reg, unsigned long base,
                        unsigned long size, mtrr_type type);
         void    (*set_all)(void);
 
         void    (*get)(unsigned int reg, unsigned long *base,
-                       unsigned long *size, mtrr_type * type);
+                       unsigned long *size, mtrr_type *type);
         int     (*get_free_region)(unsigned long base, unsigned long size,
                                    int replace_reg);
         int     (*validate_add_page)(unsigned long base, unsigned long size,
@@ -39,11 +38,11 @@ extern int positive_have_wrcomb(void);
 
 /* library functions for processor-specific routines */
 struct set_mtrr_context {
-        unsigned long flags;
-        unsigned long cr4val;
-        u32 deftype_lo;
-        u32 deftype_hi;
-        u32 ccr3;
+        unsigned long   flags;
+        unsigned long   cr4val;
+        u32             deftype_lo;
+        u32             deftype_hi;
+        u32             ccr3;
 };
 
 void set_mtrr_done(struct set_mtrr_context *ctxt);
@@ -54,10 +53,10 @@ void fill_mtrr_var_range(unsigned int index,
                 u32 base_lo, u32 base_hi, u32 mask_lo, u32 mask_hi);
 void get_mtrr_state(void);
 
-extern void set_mtrr_ops(struct mtrr_ops * ops);
+extern void set_mtrr_ops(struct mtrr_ops *ops);
 
 extern u64 size_or_mask, size_and_mask;
-extern struct mtrr_ops * mtrr_if;
+extern struct mtrr_ops *mtrr_if;
 
 #define is_cpu(vnd)     (mtrr_if && mtrr_if->vendor == X86_VENDOR_##vnd)
 #define use_intel()     (mtrr_if && mtrr_if->use_intel_if == 1)
diff --git a/arch/x86/kernel/cpu/mtrr/state.c b/arch/x86/kernel/cpu/mtrr/state.c
index 1f5fb1588d1f..dfc80b4e6b0d 100644
--- a/arch/x86/kernel/cpu/mtrr/state.c
+++ b/arch/x86/kernel/cpu/mtrr/state.c
@@ -1,24 +1,25 @@
-#include <linux/mm.h>
 #include <linux/init.h>
-#include <asm/io.h>
-#include <asm/mtrr.h>
-#include <asm/msr.h>
+#include <linux/io.h>
+#include <linux/mm.h>
+
 #include <asm/processor-cyrix.h>
 #include <asm/processor-flags.h>
-#include "mtrr.h"
+#include <asm/mtrr.h>
+#include <asm/msr.h>
 
+#include "mtrr.h"
 
-/*  Put the processor into a state where MTRRs can be safely set  */
+/* Put the processor into a state where MTRRs can be safely set */
 void set_mtrr_prepare_save(struct set_mtrr_context *ctxt)
 {
         unsigned int cr0;
 
-        /*  Disable interrupts locally  */
+        /* Disable interrupts locally */
         local_irq_save(ctxt->flags);
 
         if (use_intel() || is_cpu(CYRIX)) {
 
-                /*  Save value of CR4 and clear Page Global Enable (bit 7)  */
+                /* Save value of CR4 and clear Page Global Enable (bit 7) */
                 if (cpu_has_pge) {
                         ctxt->cr4val = read_cr4();
                         write_cr4(ctxt->cr4val & ~X86_CR4_PGE);
@@ -33,50 +34,61 @@ void set_mtrr_prepare_save(struct set_mtrr_context *ctxt)
                 write_cr0(cr0);
                 wbinvd();
 
-                if (use_intel())
-                        /*  Save MTRR state */
+                if (use_intel()) {
+                        /* Save MTRR state */
                         rdmsr(MSR_MTRRdefType, ctxt->deftype_lo, ctxt->deftype_hi);
-                else
-                        /* Cyrix ARRs - everything else were excluded at the top */
+                } else {
+                        /*
+                         * Cyrix ARRs -
+                         * everything else were excluded at the top
+                         */
                         ctxt->ccr3 = getCx86(CX86_CCR3);
+                }
         }
 }
 
 void set_mtrr_cache_disable(struct set_mtrr_context *ctxt)
 {
-        if (use_intel())
-                /*  Disable MTRRs, and set the default type to uncached  */
+        if (use_intel()) {
+                /* Disable MTRRs, and set the default type to uncached */
                 mtrr_wrmsr(MSR_MTRRdefType, ctxt->deftype_lo & 0xf300UL,
                       ctxt->deftype_hi);
-        else if (is_cpu(CYRIX))
-                /* Cyrix ARRs - everything else were excluded at the top */
-                setCx86(CX86_CCR3, (ctxt->ccr3 & 0x0f) | 0x10);
+        } else {
+                if (is_cpu(CYRIX)) {
+                        /* Cyrix ARRs - everything else were excluded at the top */
+                        setCx86(CX86_CCR3, (ctxt->ccr3 & 0x0f) | 0x10);
+                }
+        }
 }
 
-/*  Restore the processor after a set_mtrr_prepare  */
+/* Restore the processor after a set_mtrr_prepare */
 void set_mtrr_done(struct set_mtrr_context *ctxt)
 {
         if (use_intel() || is_cpu(CYRIX)) {
 
-                /*  Flush caches and TLBs  */
+                /* Flush caches and TLBs */
                 wbinvd();
 
-                /*  Restore MTRRdefType  */
-                if (use_intel())
+                /* Restore MTRRdefType */
+                if (use_intel()) {
                         /* Intel (P6) standard MTRRs */
-                        mtrr_wrmsr(MSR_MTRRdefType, ctxt->deftype_lo, ctxt->deftype_hi);
-                else
-                        /* Cyrix ARRs - everything else was excluded at the top */
+                        mtrr_wrmsr(MSR_MTRRdefType, ctxt->deftype_lo,
+                                   ctxt->deftype_hi);
+                } else {
+                        /*
+                         * Cyrix ARRs -
+                         * everything else was excluded at the top
+                         */
                         setCx86(CX86_CCR3, ctxt->ccr3);
+                }
 
-                /*  Enable caches  */
+                /* Enable caches */
                 write_cr0(read_cr0() & 0xbfffffff);
 
-                /*  Restore value of CR4  */
+                /* Restore value of CR4 */
                 if (cpu_has_pge)
                         write_cr4(ctxt->cr4val);
         }
-        /*  Re-enable interrupts locally (if enabled previously)  */
+        /* Re-enable interrupts locally (if enabled previously) */
         local_irq_restore(ctxt->flags);
 }
-
diff --git a/arch/x86/kernel/cpu/perf_counter.c b/arch/x86/kernel/cpu/perf_event.c
index 900332b800f8..a3c7adb06b78 100644
--- a/arch/x86/kernel/cpu/perf_counter.c
+++ b/arch/x86/kernel/cpu/perf_event.c
@@ -1,16 +1,17 @@
 /*
- * Performance counter x86 architecture code
+ * Performance events x86 architecture code
  *
  *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
  *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
  *  Copyright (C) 2009 Jaswinder Singh Rajput
  *  Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
  *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
+ *  Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com>
  *
  *  For licencing details see kernel-base/COPYING
  */
 
-#include <linux/perf_counter.h>
+#include <linux/perf_event.h>
 #include <linux/capability.h>
 #include <linux/notifier.h>
 #include <linux/hardirq.h>
@@ -20,19 +21,60 @@
 #include <linux/sched.h>
 #include <linux/uaccess.h>
 #include <linux/highmem.h>
+#include <linux/cpu.h>
 
 #include <asm/apic.h>
 #include <asm/stacktrace.h>
 #include <asm/nmi.h>
 
-static u64 perf_counter_mask __read_mostly;
+static u64 perf_event_mask __read_mostly;
 
-struct cpu_hw_counters {
-        struct perf_counter     *counters[X86_PMC_IDX_MAX];
+/* The maximal number of PEBS events: */
+#define MAX_PEBS_EVENTS 4
+
+/* The size of a BTS record in bytes: */
+#define BTS_RECORD_SIZE         24
+
+/* The size of a per-cpu BTS buffer in bytes: */
+#define BTS_BUFFER_SIZE         (BTS_RECORD_SIZE * 2048)
+
+/* The BTS overflow threshold in bytes from the end of the buffer: */
+#define BTS_OVFL_TH             (BTS_RECORD_SIZE * 128)
+
+
+/*
+ * Bits in the debugctlmsr controlling branch tracing.
+ */
+#define X86_DEBUGCTL_TR                 (1 << 6)
+#define X86_DEBUGCTL_BTS                (1 << 7)
+#define X86_DEBUGCTL_BTINT              (1 << 8)
+#define X86_DEBUGCTL_BTS_OFF_OS         (1 << 9)
+#define X86_DEBUGCTL_BTS_OFF_USR        (1 << 10)
+
+/*
+ * A debug store configuration.
+ *
+ * We only support architectures that use 64bit fields.
+ */
+struct debug_store {
+        u64     bts_buffer_base;
+        u64     bts_index;
+        u64     bts_absolute_maximum;
+        u64     bts_interrupt_threshold;
+        u64     pebs_buffer_base;
+        u64     pebs_index;
+        u64     pebs_absolute_maximum;
+        u64     pebs_interrupt_threshold;
+        u64     pebs_event_reset[MAX_PEBS_EVENTS];
+};
+
+struct cpu_hw_events {
+        struct perf_event       *events[X86_PMC_IDX_MAX];
         unsigned long           used_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
         unsigned long           active_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
         unsigned long           interrupts;
         int                     enabled;
+        struct debug_store      *ds;
 };
 
 /*
@@ -44,25 +86,27 @@ struct x86_pmu {
         int             (*handle_irq)(struct pt_regs *);
         void            (*disable_all)(void);
         void            (*enable_all)(void);
-        void            (*enable)(struct hw_perf_counter *, int);
-        void            (*disable)(struct hw_perf_counter *, int);
+        void            (*enable)(struct hw_perf_event *, int);
+        void            (*disable)(struct hw_perf_event *, int);
         unsigned        eventsel;
         unsigned        perfctr;
         u64             (*event_map)(int);
         u64             (*raw_event)(u64);
         int             max_events;
-        int             num_counters;
-        int             num_counters_fixed;
-        int             counter_bits;
-        u64             counter_mask;
+        int             num_events;
+        int             num_events_fixed;
+        int             event_bits;
+        u64             event_mask;
         int             apic;
         u64             max_period;
         u64             intel_ctrl;
+        void            (*enable_bts)(u64 config);
+        void            (*disable_bts)(void);
 };
 
 static struct x86_pmu x86_pmu __read_mostly;
 
-static DEFINE_PER_CPU(struct cpu_hw_counters, cpu_hw_counters) = {
+static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = {
         .enabled = 1,
 };
 
@@ -80,35 +124,35 @@ static const u64 p6_perfmon_event_map[] =
   [PERF_COUNT_HW_BUS_CYCLES]            = 0x0062,
 };
 
-static u64 p6_pmu_event_map(int event)
+static u64 p6_pmu_event_map(int hw_event)
 {
-        return p6_perfmon_event_map[event];
+        return p6_perfmon_event_map[hw_event];
 }
 
 /*
- * Counter setting that is specified not to count anything.
+ * Event setting that is specified not to count anything.
  * We use this to effectively disable a counter.
  *
  * L2_RQSTS with 0 MESI unit mask.
  */
-#define P6_NOP_COUNTER                  0x0000002EULL
+#define P6_NOP_EVENT                    0x0000002EULL
 
-static u64 p6_pmu_raw_event(u64 event)
+static u64 p6_pmu_raw_event(u64 hw_event)
 {
 #define P6_EVNTSEL_EVENT_MASK           0x000000FFULL
 #define P6_EVNTSEL_UNIT_MASK            0x0000FF00ULL
 #define P6_EVNTSEL_EDGE_MASK            0x00040000ULL
 #define P6_EVNTSEL_INV_MASK             0x00800000ULL
-#define P6_EVNTSEL_COUNTER_MASK         0xFF000000ULL
+#define P6_EVNTSEL_REG_MASK             0xFF000000ULL
 
 #define P6_EVNTSEL_MASK                 \
         (P6_EVNTSEL_EVENT_MASK |        \
          P6_EVNTSEL_UNIT_MASK  |        \
          P6_EVNTSEL_EDGE_MASK  |        \
          P6_EVNTSEL_INV_MASK   |        \
-         P6_EVNTSEL_COUNTER_MASK)
+         P6_EVNTSEL_REG_MASK)
 
-        return event & P6_EVNTSEL_MASK;
+        return hw_event & P6_EVNTSEL_MASK;
 }
 
 
@@ -126,16 +170,16 @@ static const u64 intel_perfmon_event_map[] =
   [PERF_COUNT_HW_BUS_CYCLES]            = 0x013c,
 };
 
-static u64 intel_pmu_event_map(int event)
+static u64 intel_pmu_event_map(int hw_event)
 {
-        return intel_perfmon_event_map[event];
+        return intel_perfmon_event_map[hw_event];
 }
 
 /*
- * Generalized hw caching related event table, filled
+ * Generalized hw caching related hw_event table, filled
  * in on a per model basis. A value of 0 means
- * 'not supported', -1 means 'event makes no sense on
- * this CPU', any other value means the raw event
+ * 'not supported', -1 means 'hw_event makes no sense on
+ * this CPU', any other value means the raw hw_event
  * ID.
  */
 
@@ -419,22 +463,22 @@ static const u64 atom_hw_cache_event_ids
  },
 };
 
-static u64 intel_pmu_raw_event(u64 event)
+static u64 intel_pmu_raw_event(u64 hw_event)
 {
 #define CORE_EVNTSEL_EVENT_MASK         0x000000FFULL
 #define CORE_EVNTSEL_UNIT_MASK          0x0000FF00ULL
 #define CORE_EVNTSEL_EDGE_MASK          0x00040000ULL
 #define CORE_EVNTSEL_INV_MASK           0x00800000ULL
-#define CORE_EVNTSEL_COUNTER_MASK       0xFF000000ULL
+#define CORE_EVNTSEL_REG_MASK   0xFF000000ULL
 
 #define CORE_EVNTSEL_MASK               \
         (CORE_EVNTSEL_EVENT_MASK |      \
          CORE_EVNTSEL_UNIT_MASK  |      \
          CORE_EVNTSEL_EDGE_MASK  |      \
          CORE_EVNTSEL_INV_MASK  |       \
-         CORE_EVNTSEL_COUNTER_MASK)
+         CORE_EVNTSEL_REG_MASK)
 
-        return event & CORE_EVNTSEL_MASK;
+        return hw_event & CORE_EVNTSEL_MASK;
 }
 
 static const u64 amd_hw_cache_event_ids
@@ -541,52 +585,55 @@ static const u64 amd_perfmon_event_map[] =
   [PERF_COUNT_HW_BRANCH_MISSES]         = 0x00c5,
 };
 
-static u64 amd_pmu_event_map(int event)
+static u64 amd_pmu_event_map(int hw_event)
 {
-        return amd_perfmon_event_map[event];
+        return amd_perfmon_event_map[hw_event];
 }
 
-static u64 amd_pmu_raw_event(u64 event)
+static u64 amd_pmu_raw_event(u64 hw_event)
 {
 #define K7_EVNTSEL_EVENT_MASK   0x7000000FFULL
 #define K7_EVNTSEL_UNIT_MASK    0x00000FF00ULL
 #define K7_EVNTSEL_EDGE_MASK    0x000040000ULL
 #define K7_EVNTSEL_INV_MASK     0x000800000ULL
-#define K7_EVNTSEL_COUNTER_MASK 0x0FF000000ULL
+#define K7_EVNTSEL_REG_MASK     0x0FF000000ULL
 
 #define K7_EVNTSEL_MASK                 \
         (K7_EVNTSEL_EVENT_MASK |        \
          K7_EVNTSEL_UNIT_MASK  |        \
          K7_EVNTSEL_EDGE_MASK  |        \
          K7_EVNTSEL_INV_MASK   |        \
-         K7_EVNTSEL_COUNTER_MASK)
+         K7_EVNTSEL_REG_MASK)
 
-        return event & K7_EVNTSEL_MASK;
+        return hw_event & K7_EVNTSEL_MASK;
 }
 
 /*
- * Propagate counter elapsed time into the generic counter.
- * Can only be executed on the CPU where the counter is active.
+ * Propagate event elapsed time into the generic event.
+ * Can only be executed on the CPU where the event is active.
  * Returns the delta events processed.
  */
 static u64
-x86_perf_counter_update(struct perf_counter *counter,
-                        struct hw_perf_counter *hwc, int idx)
+x86_perf_event_update(struct perf_event *event,
+                        struct hw_perf_event *hwc, int idx)
 {
-        int shift = 64 - x86_pmu.counter_bits;
+        int shift = 64 - x86_pmu.event_bits;
         u64 prev_raw_count, new_raw_count;
         s64 delta;
 
+        if (idx == X86_PMC_IDX_FIXED_BTS)
+                return 0;
+
         /*
-         * Careful: an NMI might modify the previous counter value.
+         * Careful: an NMI might modify the previous event value.
          *
          * Our tactic to handle this is to first atomically read and
          * exchange a new raw count - then add that new-prev delta
-         * count to the generic counter atomically:
+         * count to the generic event atomically:
          */
 again:
         prev_raw_count = atomic64_read(&hwc->prev_count);
-        rdmsrl(hwc->counter_base + idx, new_raw_count);
+        rdmsrl(hwc->event_base + idx, new_raw_count);
 
         if (atomic64_cmpxchg(&hwc->prev_count, prev_raw_count,
                                         new_raw_count) != prev_raw_count)
@@ -595,7 +642,7 @@ again:
         /*
          * Now we have the new raw value and have updated the prev
          * timestamp already. We can now calculate the elapsed delta
-         * (counter-)time and add that to the generic counter.
+         * (event-)time and add that to the generic event.
          *
          * Careful, not all hw sign-extends above the physical width
          * of the count.
@@ -603,13 +650,13 @@ again:
         delta = (new_raw_count << shift) - (prev_raw_count << shift);
         delta >>= shift;
 
-        atomic64_add(delta, &counter->count);
+        atomic64_add(delta, &event->count);
         atomic64_sub(delta, &hwc->period_left);
 
         return new_raw_count;
 }
 
-static atomic_t active_counters;
+static atomic_t active_events;
 static DEFINE_MUTEX(pmc_reserve_mutex);
 
 static bool reserve_pmc_hardware(void)
@@ -620,12 +667,12 @@ static bool reserve_pmc_hardware(void)
         if (nmi_watchdog == NMI_LOCAL_APIC)
                 disable_lapic_nmi_watchdog();
 
-        for (i = 0; i < x86_pmu.num_counters; i++) {
+        for (i = 0; i < x86_pmu.num_events; i++) {
                 if (!reserve_perfctr_nmi(x86_pmu.perfctr + i))
                         goto perfctr_fail;
         }
 
-        for (i = 0; i < x86_pmu.num_counters; i++) {
+        for (i = 0; i < x86_pmu.num_events; i++) {
                 if (!reserve_evntsel_nmi(x86_pmu.eventsel + i))
                         goto eventsel_fail;
         }
@@ -638,7 +685,7 @@ eventsel_fail:
         for (i--; i >= 0; i--)
                 release_evntsel_nmi(x86_pmu.eventsel + i);
 
-        i = x86_pmu.num_counters;
+        i = x86_pmu.num_events;
 
 perfctr_fail:
         for (i--; i >= 0; i--)
@@ -656,7 +703,7 @@ static void release_pmc_hardware(void)
 #ifdef CONFIG_X86_LOCAL_APIC
         int i;
 
-        for (i = 0; i < x86_pmu.num_counters; i++) {
+        for (i = 0; i < x86_pmu.num_events; i++) {
                 release_perfctr_nmi(x86_pmu.perfctr + i);
                 release_evntsel_nmi(x86_pmu.eventsel + i);
         }
@@ -666,10 +713,110 @@ static void release_pmc_hardware(void)
 #endif
 }
 
-static void hw_perf_counter_destroy(struct perf_counter *counter)
+static inline bool bts_available(void)
 {
-        if (atomic_dec_and_mutex_lock(&active_counters, &pmc_reserve_mutex)) {
+        return x86_pmu.enable_bts != NULL;
+}
+
+static inline void init_debug_store_on_cpu(int cpu)
+{
+        struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
+
+        if (!ds)
+                return;
+
+        wrmsr_on_cpu(cpu, MSR_IA32_DS_AREA,
+                     (u32)((u64)(unsigned long)ds),
+                     (u32)((u64)(unsigned long)ds >> 32));
+}
+
+static inline void fini_debug_store_on_cpu(int cpu)
+{
+        if (!per_cpu(cpu_hw_events, cpu).ds)
+                return;
+
+        wrmsr_on_cpu(cpu, MSR_IA32_DS_AREA, 0, 0);
+}
+
+static void release_bts_hardware(void)
+{
+        int cpu;
+
+        if (!bts_available())
+                return;
+
+        get_online_cpus();
+
+        for_each_online_cpu(cpu)
+                fini_debug_store_on_cpu(cpu);
+
+        for_each_possible_cpu(cpu) {
+                struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
+
+                if (!ds)
+                        continue;
+
+                per_cpu(cpu_hw_events, cpu).ds = NULL;
+
+                kfree((void *)(unsigned long)ds->bts_buffer_base);
+                kfree(ds);
+        }
+
+        put_online_cpus();
+}
+
+static int reserve_bts_hardware(void)
+{
+        int cpu, err = 0;
+
+        if (!bts_available())
+                return 0;
+
+        get_online_cpus();
+
+        for_each_possible_cpu(cpu) {
+                struct debug_store *ds;
+                void *buffer;
+
+                err = -ENOMEM;
+                buffer = kzalloc(BTS_BUFFER_SIZE, GFP_KERNEL);
+                if (unlikely(!buffer))
+                        break;
+
+                ds = kzalloc(sizeof(*ds), GFP_KERNEL);
+                if (unlikely(!ds)) {
+                        kfree(buffer);
+                        break;
+                }
+
+                ds->bts_buffer_base = (u64)(unsigned long)buffer;
+                ds->bts_index = ds->bts_buffer_base;
+                ds->bts_absolute_maximum =
+                        ds->bts_buffer_base + BTS_BUFFER_SIZE;
+                ds->bts_interrupt_threshold =
+                        ds->bts_absolute_maximum - BTS_OVFL_TH;
+
+                per_cpu(cpu_hw_events, cpu).ds = ds;
+                err = 0;
+        }
+
+        if (err)
+                release_bts_hardware();
+        else {
+                for_each_online_cpu(cpu)
+                        init_debug_store_on_cpu(cpu);
+        }
+
+        put_online_cpus();
+
+        return err;
+}
+
+static void hw_perf_event_destroy(struct perf_event *event)
+{
+        if (atomic_dec_and_mutex_lock(&active_events, &pmc_reserve_mutex)) {
                 release_pmc_hardware();
+                release_bts_hardware();
                 mutex_unlock(&pmc_reserve_mutex);
         }
 }
@@ -680,7 +827,7 @@ static inline int x86_pmu_initialized(void)
 }
 
 static inline int
-set_ext_hw_attr(struct hw_perf_counter *hwc, struct perf_counter_attr *attr)
+set_ext_hw_attr(struct hw_perf_event *hwc, struct perf_event_attr *attr)
 {
         unsigned int cache_type, cache_op, cache_result;
         u64 config, val;
@@ -712,13 +859,49 @@ set_ext_hw_attr(struct hw_perf_counter *hwc, struct perf_counter_attr *attr)
         return 0;
 }
 
+static void intel_pmu_enable_bts(u64 config)
+{
+        unsigned long debugctlmsr;
+
+        debugctlmsr = get_debugctlmsr();
+
+        debugctlmsr |= X86_DEBUGCTL_TR;
+        debugctlmsr |= X86_DEBUGCTL_BTS;
+        debugctlmsr |= X86_DEBUGCTL_BTINT;
+
+        if (!(config & ARCH_PERFMON_EVENTSEL_OS))
+                debugctlmsr |= X86_DEBUGCTL_BTS_OFF_OS;
+
+        if (!(config & ARCH_PERFMON_EVENTSEL_USR))
+                debugctlmsr |= X86_DEBUGCTL_BTS_OFF_USR;
+
+        update_debugctlmsr(debugctlmsr);
+}
+
+static void intel_pmu_disable_bts(void)
+{
+        struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+        unsigned long debugctlmsr;
+
+        if (!cpuc->ds)
+                return;
+
+        debugctlmsr = get_debugctlmsr();
+
+        debugctlmsr &=
+                ~(X86_DEBUGCTL_TR | X86_DEBUGCTL_BTS | X86_DEBUGCTL_BTINT |
+                  X86_DEBUGCTL_BTS_OFF_OS | X86_DEBUGCTL_BTS_OFF_USR);
+
+        update_debugctlmsr(debugctlmsr);
+}
+
 /*
  * Setup the hardware configuration for a given attr_type
  */
-static int __hw_perf_counter_init(struct perf_counter *counter)
+static int __hw_perf_event_init(struct perf_event *event)
 {
-        struct perf_counter_attr *attr = &counter->attr;
-        struct hw_perf_counter *hwc = &counter->hw;
+        struct perf_event_attr *attr = &event->attr;
+        struct hw_perf_event *hwc = &event->hw;
         u64 config;
         int err;
 
@@ -726,17 +909,23 @@ static int __hw_perf_counter_init(struct perf_counter *counter)
                 return -ENODEV;
 
         err = 0;
-        if (!atomic_inc_not_zero(&active_counters)) {
+        if (!atomic_inc_not_zero(&active_events)) {
                 mutex_lock(&pmc_reserve_mutex);
-                if (atomic_read(&active_counters) == 0 && !reserve_pmc_hardware())
-                        err = -EBUSY;
-                else
-                        atomic_inc(&active_counters);
+                if (atomic_read(&active_events) == 0) {
+                        if (!reserve_pmc_hardware())
+                                err = -EBUSY;
+                        else
+                                err = reserve_bts_hardware();
+                }
+                if (!err)
+                        atomic_inc(&active_events);
                 mutex_unlock(&pmc_reserve_mutex);
         }
         if (err)
                 return err;
 
+        event->destroy = hw_perf_event_destroy;
+
         /*
          * Generate PMC IRQs:
          * (keep 'enabled' bit clear for now)
@@ -759,17 +948,15 @@ static int __hw_perf_counter_init(struct perf_counter *counter)
                 /*
                  * If we have a PMU initialized but no APIC
                  * interrupts, we cannot sample hardware
-                 * counters (user-space has to fall back and
-                 * sample via a hrtimer based software counter):
+                 * events (user-space has to fall back and
+                 * sample via a hrtimer based software event):
                  */
                 if (!x86_pmu.apic)
                         return -EOPNOTSUPP;
         }
 
-        counter->destroy = hw_perf_counter_destroy;
-
         /*
-         * Raw event type provide the config in the event structure
+         * Raw hw_event type provide the config in the hw_event structure
          */
         if (attr->type == PERF_TYPE_RAW) {
                 hwc->config |= x86_pmu.raw_event(attr->config);
@@ -793,6 +980,20 @@ static int __hw_perf_counter_init(struct perf_counter *counter)
         if (config == -1LL)
                 return -EINVAL;
 
+        /*
+         * Branch tracing:
+         */
+        if ((attr->config == PERF_COUNT_HW_BRANCH_INSTRUCTIONS) &&
+            (hwc->sample_period == 1)) {
+                /* BTS is not supported by this architecture. */
+                if (!bts_available())
+                        return -EOPNOTSUPP;
+
+                /* BTS is currently only allowed for user-mode. */
+                if (hwc->config & ARCH_PERFMON_EVENTSEL_OS)
+                        return -EOPNOTSUPP;
+        }
+
         hwc->config |= config;
 
         return 0;
@@ -800,7 +1001,7 @@ static int __hw_perf_counter_init(struct perf_counter *counter)
 
 static void p6_pmu_disable_all(void)
 {
-        struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
+        struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
         u64 val;
 
         if (!cpuc->enabled)
@@ -817,12 +1018,23 @@ static void p6_pmu_disable_all(void)
 
 static void intel_pmu_disable_all(void)
 {
+        struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+
+        if (!cpuc->enabled)
+                return;
+
+        cpuc->enabled = 0;
+        barrier();
+
         wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0);
+
+        if (test_bit(X86_PMC_IDX_FIXED_BTS, cpuc->active_mask))
+                intel_pmu_disable_bts();
 }
 
 static void amd_pmu_disable_all(void)
 {
-        struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
+        struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
         int idx;
 
         if (!cpuc->enabled)
@@ -831,12 +1043,12 @@ static void amd_pmu_disable_all(void)
         cpuc->enabled = 0;
         /*
          * ensure we write the disable before we start disabling the
-         * counters proper, so that amd_pmu_enable_counter() does the
+         * events proper, so that amd_pmu_enable_event() does the
          * right thing.
          */
         barrier();
 
-        for (idx = 0; idx < x86_pmu.num_counters; idx++) {
+        for (idx = 0; idx < x86_pmu.num_events; idx++) {
                 u64 val;
 
                 if (!test_bit(idx, cpuc->active_mask))
@@ -858,7 +1070,7 @@ void hw_perf_disable(void)
 
 static void p6_pmu_enable_all(void)
 {
-        struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
+        struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
         unsigned long val;
 
         if (cpuc->enabled)
@@ -875,12 +1087,30 @@ static void p6_pmu_enable_all(void)
 
 static void intel_pmu_enable_all(void)
 {
+        struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+
+        if (cpuc->enabled)
+                return;
+
+        cpuc->enabled = 1;
+        barrier();
+
         wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, x86_pmu.intel_ctrl);
+
+        if (test_bit(X86_PMC_IDX_FIXED_BTS, cpuc->active_mask)) {
+                struct perf_event *event =
+                        cpuc->events[X86_PMC_IDX_FIXED_BTS];
+
+                if (WARN_ON_ONCE(!event))
+                        return;
+
+                intel_pmu_enable_bts(event->hw.config);
+        }
 }
 
 static void amd_pmu_enable_all(void)
 {
-        struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
+        struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
         int idx;
 
         if (cpuc->enabled)
@@ -889,14 +1119,14 @@ static void amd_pmu_enable_all(void)
         cpuc->enabled = 1;
         barrier();
 
-        for (idx = 0; idx < x86_pmu.num_counters; idx++) {
-                struct perf_counter *counter = cpuc->counters[idx];
+        for (idx = 0; idx < x86_pmu.num_events; idx++) {
+                struct perf_event *event = cpuc->events[idx];
                 u64 val;
 
                 if (!test_bit(idx, cpuc->active_mask))
                         continue;
 
-                val = counter->hw.config;
+                val = event->hw.config;
                 val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
                 wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
         }
@@ -923,19 +1153,19 @@ static inline void intel_pmu_ack_status(u64 ack)
         wrmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, ack);
 }
 
-static inline void x86_pmu_enable_counter(struct hw_perf_counter *hwc, int idx)
+static inline void x86_pmu_enable_event(struct hw_perf_event *hwc, int idx)
 {
         (void)checking_wrmsrl(hwc->config_base + idx,
                               hwc->config | ARCH_PERFMON_EVENTSEL0_ENABLE);
 }
 
-static inline void x86_pmu_disable_counter(struct hw_perf_counter *hwc, int idx)
+static inline void x86_pmu_disable_event(struct hw_perf_event *hwc, int idx)
 {
         (void)checking_wrmsrl(hwc->config_base + idx, hwc->config);
 }
 
 static inline void
-intel_pmu_disable_fixed(struct hw_perf_counter *hwc, int __idx)
+intel_pmu_disable_fixed(struct hw_perf_event *hwc, int __idx)
 {
         int idx = __idx - X86_PMC_IDX_FIXED;
         u64 ctrl_val, mask;
@@ -948,10 +1178,10 @@ intel_pmu_disable_fixed(struct hw_perf_counter *hwc, int __idx)
 }
 
 static inline void
-p6_pmu_disable_counter(struct hw_perf_counter *hwc, int idx)
+p6_pmu_disable_event(struct hw_perf_event *hwc, int idx)
 {
-        struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
-        u64 val = P6_NOP_COUNTER;
+        struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+        u64 val = P6_NOP_EVENT;
 
         if (cpuc->enabled)
                 val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
@@ -960,36 +1190,44 @@ p6_pmu_disable_counter(struct hw_perf_counter *hwc, int idx)
 }
 
 static inline void
-intel_pmu_disable_counter(struct hw_perf_counter *hwc, int idx)
+intel_pmu_disable_event(struct hw_perf_event *hwc, int idx)
 {
+        if (unlikely(idx == X86_PMC_IDX_FIXED_BTS)) {
+                intel_pmu_disable_bts();
+                return;
+        }
+
         if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
                 intel_pmu_disable_fixed(hwc, idx);
                 return;
         }
 
-        x86_pmu_disable_counter(hwc, idx);
+        x86_pmu_disable_event(hwc, idx);
 }
 
 static inline void
-amd_pmu_disable_counter(struct hw_perf_counter *hwc, int idx)
+amd_pmu_disable_event(struct hw_perf_event *hwc, int idx)
 {
-        x86_pmu_disable_counter(hwc, idx);
+        x86_pmu_disable_event(hwc, idx);
 }
 
-static DEFINE_PER_CPU(u64, prev_left[X86_PMC_IDX_MAX]);
+static DEFINE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left);
 
 /*
  * Set the next IRQ period, based on the hwc->period_left value.
- * To be called with the counter disabled in hw:
+ * To be called with the event disabled in hw:
  */
 static int
-x86_perf_counter_set_period(struct perf_counter *counter,
-                             struct hw_perf_counter *hwc, int idx)
+x86_perf_event_set_period(struct perf_event *event,
+                             struct hw_perf_event *hwc, int idx)
 {
         s64 left = atomic64_read(&hwc->period_left);
         s64 period = hwc->sample_period;
         int err, ret = 0;
 
+        if (idx == X86_PMC_IDX_FIXED_BTS)
+                return 0;
+
         /*
          * If we are way outside a reasoable range then just skip forward:
          */
@@ -1007,7 +1245,7 @@ x86_perf_counter_set_period(struct perf_counter *counter,
                 ret = 1;
         }
         /*
-         * Quirk: certain CPUs dont like it if just 1 event is left:
+         * Quirk: certain CPUs dont like it if just 1 hw_event is left:
          */
         if (unlikely(left < 2))
                 left = 2;
@@ -1015,24 +1253,24 @@ x86_perf_counter_set_period(struct perf_counter *counter,
         if (left > x86_pmu.max_period)
                 left = x86_pmu.max_period;
 
-        per_cpu(prev_left[idx], smp_processor_id()) = left;
+        per_cpu(pmc_prev_left[idx], smp_processor_id()) = left;
 
         /*
-         * The hw counter starts counting from this counter offset,
+         * The hw event starts counting from this event offset,
          * mark it to be able to extra future deltas:
          */
         atomic64_set(&hwc->prev_count, (u64)-left);
 
-        err = checking_wrmsrl(hwc->counter_base + idx,
-                             (u64)(-left) & x86_pmu.counter_mask);
+        err = checking_wrmsrl(hwc->event_base + idx,
+                             (u64)(-left) & x86_pmu.event_mask);
 
-        perf_counter_update_userpage(counter);
+        perf_event_update_userpage(event);
 
         return ret;
 }
 
 static inline void
-intel_pmu_enable_fixed(struct hw_perf_counter *hwc, int __idx)
+intel_pmu_enable_fixed(struct hw_perf_event *hwc, int __idx)
 {
         int idx = __idx - X86_PMC_IDX_FIXED;
         u64 ctrl_val, bits, mask;
@@ -1057,9 +1295,9 @@ intel_pmu_enable_fixed(struct hw_perf_counter *hwc, int __idx)
         err = checking_wrmsrl(hwc->config_base, ctrl_val);
 }
 
-static void p6_pmu_enable_counter(struct hw_perf_counter *hwc, int idx)
+static void p6_pmu_enable_event(struct hw_perf_event *hwc, int idx)
 {
-        struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
+        struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
         u64 val;
 
         val = hwc->config;
@@ -1070,128 +1308,149 @@ static void p6_pmu_enable_counter(struct hw_perf_counter *hwc, int idx)
 }
 
 
-static void intel_pmu_enable_counter(struct hw_perf_counter *hwc, int idx)
+static void intel_pmu_enable_event(struct hw_perf_event *hwc, int idx)
 {
+        if (unlikely(idx == X86_PMC_IDX_FIXED_BTS)) {
+                if (!__get_cpu_var(cpu_hw_events).enabled)
+                        return;
+
+                intel_pmu_enable_bts(hwc->config);
+                return;
+        }
+
         if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
                 intel_pmu_enable_fixed(hwc, idx);
                 return;
         }
 
-        x86_pmu_enable_counter(hwc, idx);
+        x86_pmu_enable_event(hwc, idx);
 }
 
-static void amd_pmu_enable_counter(struct hw_perf_counter *hwc, int idx)
+static void amd_pmu_enable_event(struct hw_perf_event *hwc, int idx)
 {
-        struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
+        struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 
         if (cpuc->enabled)
-                x86_pmu_enable_counter(hwc, idx);
+                x86_pmu_enable_event(hwc, idx);
 }
 
 static int
-fixed_mode_idx(struct perf_counter *counter, struct hw_perf_counter *hwc)
+fixed_mode_idx(struct perf_event *event, struct hw_perf_event *hwc)
 {
-        unsigned int event;
+        unsigned int hw_event;
 
-        if (!x86_pmu.num_counters_fixed)
-                return -1;
+        hw_event = hwc->config & ARCH_PERFMON_EVENT_MASK;
 
-        event = hwc->config & ARCH_PERFMON_EVENT_MASK;
+        if (unlikely((hw_event ==
+                      x86_pmu.event_map(PERF_COUNT_HW_BRANCH_INSTRUCTIONS)) &&
+                     (hwc->sample_period == 1)))
+                return X86_PMC_IDX_FIXED_BTS;
+
+        if (!x86_pmu.num_events_fixed)
+                return -1;
 
-        if (unlikely(event == x86_pmu.event_map(PERF_COUNT_HW_INSTRUCTIONS)))
+        if (unlikely(hw_event == x86_pmu.event_map(PERF_COUNT_HW_INSTRUCTIONS)))
                 return X86_PMC_IDX_FIXED_INSTRUCTIONS;
-        if (unlikely(event == x86_pmu.event_map(PERF_COUNT_HW_CPU_CYCLES)))
+        if (unlikely(hw_event == x86_pmu.event_map(PERF_COUNT_HW_CPU_CYCLES)))
                 return X86_PMC_IDX_FIXED_CPU_CYCLES;
-        if (unlikely(event == x86_pmu.event_map(PERF_COUNT_HW_BUS_CYCLES)))
+        if (unlikely(hw_event == x86_pmu.event_map(PERF_COUNT_HW_BUS_CYCLES)))
                 return X86_PMC_IDX_FIXED_BUS_CYCLES;
 
         return -1;
 }
 
 /*
- * Find a PMC slot for the freshly enabled / scheduled in counter:
+ * Find a PMC slot for the freshly enabled / scheduled in event:
  */
-static int x86_pmu_enable(struct perf_counter *counter)
+static int x86_pmu_enable(struct perf_event *event)
 {
-        struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
-        struct hw_perf_counter *hwc = &counter->hw;
+        struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+        struct hw_perf_event *hwc = &event->hw;
         int idx;
 
-        idx = fixed_mode_idx(counter, hwc);
-        if (idx >= 0) {
+        idx = fixed_mode_idx(event, hwc);
+        if (idx == X86_PMC_IDX_FIXED_BTS) {
+                /* BTS is already occupied. */
+                if (test_and_set_bit(idx, cpuc->used_mask))
+                        return -EAGAIN;
+
+                hwc->config_base        = 0;
+                hwc->event_base = 0;
+                hwc->idx                = idx;
+        } else if (idx >= 0) {
                 /*
-                 * Try to get the fixed counter, if that is already taken
-                 * then try to get a generic counter:
+                 * Try to get the fixed event, if that is already taken
+                 * then try to get a generic event:
                  */
                 if (test_and_set_bit(idx, cpuc->used_mask))
                         goto try_generic;
 
                 hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
                 /*
-                 * We set it so that counter_base + idx in wrmsr/rdmsr maps to
+                 * We set it so that event_base + idx in wrmsr/rdmsr maps to
                  * MSR_ARCH_PERFMON_FIXED_CTR0 ... CTR2:
                  */
-                hwc->counter_base =
+                hwc->event_base =
                         MSR_ARCH_PERFMON_FIXED_CTR0 - X86_PMC_IDX_FIXED;
                 hwc->idx = idx;
         } else {
                 idx = hwc->idx;
-                /* Try to get the previous generic counter again */
+                /* Try to get the previous generic event again */
                 if (test_and_set_bit(idx, cpuc->used_mask)) {
 try_generic:
                         idx = find_first_zero_bit(cpuc->used_mask,
-                                                  x86_pmu.num_counters);
-                        if (idx == x86_pmu.num_counters)
+                                                  x86_pmu.num_events);
+                        if (idx == x86_pmu.num_events)
                                 return -EAGAIN;
 
                         set_bit(idx, cpuc->used_mask);
                         hwc->idx = idx;
                 }
                 hwc->config_base  = x86_pmu.eventsel;
-                hwc->counter_base = x86_pmu.perfctr;
+                hwc->event_base = x86_pmu.perfctr;
         }
 
-        perf_counters_lapic_init();
+        perf_events_lapic_init();
 
         x86_pmu.disable(hwc, idx);
 
-        cpuc->counters[idx] = counter;
+        cpuc->events[idx] = event;
         set_bit(idx, cpuc->active_mask);
 
-        x86_perf_counter_set_period(counter, hwc, idx);
+        x86_perf_event_set_period(event, hwc, idx);
         x86_pmu.enable(hwc, idx);
 
-        perf_counter_update_userpage(counter);
+        perf_event_update_userpage(event);
 
         return 0;
 }
 
-static void x86_pmu_unthrottle(struct perf_counter *counter)
+static void x86_pmu_unthrottle(struct perf_event *event)
 {
-        struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
-        struct hw_perf_counter *hwc = &counter->hw;
+        struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+        struct hw_perf_event *hwc = &event->hw;
 
         if (WARN_ON_ONCE(hwc->idx >= X86_PMC_IDX_MAX ||
-                                cpuc->counters[hwc->idx] != counter))
+                                cpuc->events[hwc->idx] != event))
                 return;
 
         x86_pmu.enable(hwc, hwc->idx);
 }
 
-void perf_counter_print_debug(void)
+void perf_event_print_debug(void)
 {
         u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed;
-        struct cpu_hw_counters *cpuc;
+        struct cpu_hw_events *cpuc;
         unsigned long flags;
         int cpu, idx;
 
-        if (!x86_pmu.num_counters)
+        if (!x86_pmu.num_events)
                 return;
 
         local_irq_save(flags);
 
         cpu = smp_processor_id();
-        cpuc = &per_cpu(cpu_hw_counters, cpu);
+        cpuc = &per_cpu(cpu_hw_events, cpu);
 
         if (x86_pmu.version >= 2) {
                 rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
@@ -1207,11 +1466,11 @@ void perf_counter_print_debug(void)
         }
         pr_info("CPU#%d: used:       %016llx\n", cpu, *(u64 *)cpuc->used_mask);
 
-        for (idx = 0; idx < x86_pmu.num_counters; idx++) {
+        for (idx = 0; idx < x86_pmu.num_events; idx++) {
                 rdmsrl(x86_pmu.eventsel + idx, pmc_ctrl);
                 rdmsrl(x86_pmu.perfctr  + idx, pmc_count);
 
-                prev_left = per_cpu(prev_left[idx], cpu);
+                prev_left = per_cpu(pmc_prev_left[idx], cpu);
 
                 pr_info("CPU#%d:   gen-PMC%d ctrl:  %016llx\n",
                         cpu, idx, pmc_ctrl);
@@ -1220,7 +1479,7 @@ void perf_counter_print_debug(void)
                 pr_info("CPU#%d:   gen-PMC%d left:  %016llx\n",
                         cpu, idx, prev_left);
         }
-        for (idx = 0; idx < x86_pmu.num_counters_fixed; idx++) {
+        for (idx = 0; idx < x86_pmu.num_events_fixed; idx++) {
                 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count);
 
                 pr_info("CPU#%d: fixed-PMC%d count: %016llx\n",
@@ -1229,10 +1488,69 @@ void perf_counter_print_debug(void)
         local_irq_restore(flags);
 }
 
-static void x86_pmu_disable(struct perf_counter *counter)
+static void intel_pmu_drain_bts_buffer(struct cpu_hw_events *cpuc)
+{
+        struct debug_store *ds = cpuc->ds;
+        struct bts_record {
+                u64     from;
+                u64     to;
+                u64     flags;
+        };
+        struct perf_event *event = cpuc->events[X86_PMC_IDX_FIXED_BTS];
+        struct bts_record *at, *top;
+        struct perf_output_handle handle;
+        struct perf_event_header header;
+        struct perf_sample_data data;
+        struct pt_regs regs;
+
+        if (!event)
+                return;
+
+        if (!ds)
+                return;
+
+        at  = (struct bts_record *)(unsigned long)ds->bts_buffer_base;
+        top = (struct bts_record *)(unsigned long)ds->bts_index;
+
+        if (top <= at)
+                return;
+
+        ds->bts_index = ds->bts_buffer_base;
+
+
+        data.period     = event->hw.last_period;
+        data.addr       = 0;
+        regs.ip         = 0;
+
+        /*
+         * Prepare a generic sample, i.e. fill in the invariant fields.
+         * We will overwrite the from and to address before we output
+         * the sample.
+         */
+        perf_prepare_sample(&header, &data, event, &regs);
+
+        if (perf_output_begin(&handle, event,
+                              header.size * (top - at), 1, 1))
+                return;
+
+        for (; at < top; at++) {
+                data.ip         = at->from;
+                data.addr       = at->to;
+
+                perf_output_sample(&handle, &header, &data, event);
+        }
+
+        perf_output_end(&handle);
+
+        /* There's new data available. */
+        event->hw.interrupts++;
+        event->pending_kill = POLL_IN;
+}
+
+static void x86_pmu_disable(struct perf_event *event)
 {
-        struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
-        struct hw_perf_counter *hwc = &counter->hw;
+        struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+        struct hw_perf_event *hwc = &event->hw;
         int idx = hwc->idx;
 
         /*
@@ -1244,59 +1562,67 @@ static void x86_pmu_disable(struct perf_counter *counter)
 
         /*
          * Make sure the cleared pointer becomes visible before we
-         * (potentially) free the counter:
+         * (potentially) free the event:
          */
         barrier();
 
         /*
-         * Drain the remaining delta count out of a counter
+         * Drain the remaining delta count out of a event
          * that we are disabling:
          */
-        x86_perf_counter_update(counter, hwc, idx);
-        cpuc->counters[idx] = NULL;
+        x86_perf_event_update(event, hwc, idx);
+
+        /* Drain the remaining BTS records. */
+        if (unlikely(idx == X86_PMC_IDX_FIXED_BTS))
+                intel_pmu_drain_bts_buffer(cpuc);
+
+        cpuc->events[idx] = NULL;
         clear_bit(idx, cpuc->used_mask);
 
-        perf_counter_update_userpage(counter);
+        perf_event_update_userpage(event);
 }
 
 /*
- * Save and restart an expired counter. Called by NMI contexts,
- * so it has to be careful about preempting normal counter ops:
+ * Save and restart an expired event. Called by NMI contexts,
+ * so it has to be careful about preempting normal event ops:
  */
-static int intel_pmu_save_and_restart(struct perf_counter *counter)
+static int intel_pmu_save_and_restart(struct perf_event *event)
 {
-        struct hw_perf_counter *hwc = &counter->hw;
+        struct hw_perf_event *hwc = &event->hw;
         int idx = hwc->idx;
         int ret;
 
-        x86_perf_counter_update(counter, hwc, idx);
-        ret = x86_perf_counter_set_period(counter, hwc, idx);
+        x86_perf_event_update(event, hwc, idx);
+        ret = x86_perf_event_set_period(event, hwc, idx);
 
-        if (counter->state == PERF_COUNTER_STATE_ACTIVE)
-                intel_pmu_enable_counter(hwc, idx);
+        if (event->state == PERF_EVENT_STATE_ACTIVE)
+                intel_pmu_enable_event(hwc, idx);
 
         return ret;
 }
 
 static void intel_pmu_reset(void)
 {
+        struct debug_store *ds = __get_cpu_var(cpu_hw_events).ds;
         unsigned long flags;
         int idx;
 
-        if (!x86_pmu.num_counters)
+        if (!x86_pmu.num_events)
                 return;
 
         local_irq_save(flags);
 
         printk("clearing PMU state on CPU#%d\n", smp_processor_id());
 
-        for (idx = 0; idx < x86_pmu.num_counters; idx++) {
+        for (idx = 0; idx < x86_pmu.num_events; idx++) {
                 checking_wrmsrl(x86_pmu.eventsel + idx, 0ull);
                 checking_wrmsrl(x86_pmu.perfctr  + idx, 0ull);
         }
-        for (idx = 0; idx < x86_pmu.num_counters_fixed; idx++) {
+        for (idx = 0; idx < x86_pmu.num_events_fixed; idx++) {
                 checking_wrmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, 0ull);
         }
+        if (ds)
+                ds->bts_index = ds->bts_buffer_base;
 
         local_irq_restore(flags);
 }
@@ -1304,39 +1630,38 @@ static void intel_pmu_reset(void)
 static int p6_pmu_handle_irq(struct pt_regs *regs)
 {
         struct perf_sample_data data;
-        struct cpu_hw_counters *cpuc;
-        struct perf_counter *counter;
-        struct hw_perf_counter *hwc;
+        struct cpu_hw_events *cpuc;
+        struct perf_event *event;
+        struct hw_perf_event *hwc;
         int idx, handled = 0;
         u64 val;
 
-        data.regs = regs;
         data.addr = 0;
 
-        cpuc = &__get_cpu_var(cpu_hw_counters);
+        cpuc = &__get_cpu_var(cpu_hw_events);
 
-        for (idx = 0; idx < x86_pmu.num_counters; idx++) {
+        for (idx = 0; idx < x86_pmu.num_events; idx++) {
                 if (!test_bit(idx, cpuc->active_mask))
                         continue;
 
-                counter = cpuc->counters[idx];
-                hwc = &counter->hw;
+                event = cpuc->events[idx];
+                hwc = &event->hw;
 
-                val = x86_perf_counter_update(counter, hwc, idx);
-                if (val & (1ULL << (x86_pmu.counter_bits - 1)))
+                val = x86_perf_event_update(event, hwc, idx);
+                if (val & (1ULL << (x86_pmu.event_bits - 1)))
                         continue;
 
                 /*
-                 * counter overflow
+                 * event overflow
                  */
                 handled         = 1;
-                data.period     = counter->hw.last_period;
+                data.period     = event->hw.last_period;
 
-                if (!x86_perf_counter_set_period(counter, hwc, idx))
+                if (!x86_perf_event_set_period(event, hwc, idx))
                         continue;
 
-                if (perf_counter_overflow(counter, 1, &data))
-                        p6_pmu_disable_counter(hwc, idx);
+                if (perf_event_overflow(event, 1, &data, regs))
+                        p6_pmu_disable_event(hwc, idx);
         }
 
         if (handled)
@@ -1352,16 +1677,16 @@ static int p6_pmu_handle_irq(struct pt_regs *regs)
 static int intel_pmu_handle_irq(struct pt_regs *regs)
 {
         struct perf_sample_data data;
-        struct cpu_hw_counters *cpuc;
+        struct cpu_hw_events *cpuc;
         int bit, loops;
         u64 ack, status;
 
-        data.regs = regs;
         data.addr = 0;
 
-        cpuc = &__get_cpu_var(cpu_hw_counters);
+        cpuc = &__get_cpu_var(cpu_hw_events);
 
         perf_disable();
+        intel_pmu_drain_bts_buffer(cpuc);
         status = intel_pmu_get_status();
         if (!status) {
                 perf_enable();
@@ -1371,8 +1696,8 @@ static int intel_pmu_handle_irq(struct pt_regs *regs)
         loops = 0;
 again:
         if (++loops > 100) {
-                WARN_ONCE(1, "perfcounters: irq loop stuck!\n");
-                perf_counter_print_debug();
+                WARN_ONCE(1, "perfevents: irq loop stuck!\n");
+                perf_event_print_debug();
                 intel_pmu_reset();
                 perf_enable();
                 return 1;
@@ -1381,19 +1706,19 @@ again:
         inc_irq_stat(apic_perf_irqs);
         ack = status;
         for_each_bit(bit, (unsigned long *)&status, X86_PMC_IDX_MAX) {
-                struct perf_counter *counter = cpuc->counters[bit];
+                struct perf_event *event = cpuc->events[bit];
 
                 clear_bit(bit, (unsigned long *) &status);
                 if (!test_bit(bit, cpuc->active_mask))
                         continue;
 
-                if (!intel_pmu_save_and_restart(counter))
+                if (!intel_pmu_save_and_restart(event))
                         continue;
 
-                data.period = counter->hw.last_period;
+                data.period = event->hw.last_period;
 
-                if (perf_counter_overflow(counter, 1, &data))
-                        intel_pmu_disable_counter(&counter->hw, bit);
+                if (perf_event_overflow(event, 1, &data, regs))
+                        intel_pmu_disable_event(&event->hw, bit);
         }
 
         intel_pmu_ack_status(ack);
@@ -1413,39 +1738,38 @@ again:
 static int amd_pmu_handle_irq(struct pt_regs *regs)
 {
         struct perf_sample_data data;
-        struct cpu_hw_counters *cpuc;
-        struct perf_counter *counter;
-        struct hw_perf_counter *hwc;
+        struct cpu_hw_events *cpuc;
+        struct perf_event *event;
+        struct hw_perf_event *hwc;
         int idx, handled = 0;
         u64 val;
 
-        data.regs = regs;
         data.addr = 0;
 
-        cpuc = &__get_cpu_var(cpu_hw_counters);
+        cpuc = &__get_cpu_var(cpu_hw_events);
 
-        for (idx = 0; idx < x86_pmu.num_counters; idx++) {
+        for (idx = 0; idx < x86_pmu.num_events; idx++) {
                 if (!test_bit(idx, cpuc->active_mask))
                         continue;
 
-                counter = cpuc->counters[idx];
-                hwc = &counter->hw;
+                event = cpuc->events[idx];
+                hwc = &event->hw;
 
-                val = x86_perf_counter_update(counter, hwc, idx);
-                if (val & (1ULL << (x86_pmu.counter_bits - 1)))
+                val = x86_perf_event_update(event, hwc, idx);
+                if (val & (1ULL << (x86_pmu.event_bits - 1)))
                         continue;
 
                 /*
-                 * counter overflow
+                 * event overflow
                  */
                 handled         = 1;
-                data.period     = counter->hw.last_period;
+                data.period     = event->hw.last_period;
 
-                if (!x86_perf_counter_set_period(counter, hwc, idx))
+                if (!x86_perf_event_set_period(event, hwc, idx))
                         continue;
 
-                if (perf_counter_overflow(counter, 1, &data))
-                        amd_pmu_disable_counter(hwc, idx);
+                if (perf_event_overflow(event, 1, &data, regs))
+                        amd_pmu_disable_event(hwc, idx);
         }
 
         if (handled)
@@ -1459,18 +1783,18 @@ void smp_perf_pending_interrupt(struct pt_regs *regs)
         irq_enter();
         ack_APIC_irq();
         inc_irq_stat(apic_pending_irqs);
-        perf_counter_do_pending();
+        perf_event_do_pending();
         irq_exit();
 }
 
-void set_perf_counter_pending(void)
+void set_perf_event_pending(void)
 {
 #ifdef CONFIG_X86_LOCAL_APIC
         apic->send_IPI_self(LOCAL_PENDING_VECTOR);
 #endif
 }
 
-void perf_counters_lapic_init(void)
+void perf_events_lapic_init(void)
 {
 #ifdef CONFIG_X86_LOCAL_APIC
         if (!x86_pmu.apic || !x86_pmu_initialized())
@@ -1484,13 +1808,13 @@ void perf_counters_lapic_init(void)
 }
 
 static int __kprobes
-perf_counter_nmi_handler(struct notifier_block *self,
+perf_event_nmi_handler(struct notifier_block *self,
                          unsigned long cmd, void *__args)
 {
         struct die_args *args = __args;
         struct pt_regs *regs;
 
-        if (!atomic_read(&active_counters))
+        if (!atomic_read(&active_events))
                 return NOTIFY_DONE;
 
         switch (cmd) {
@@ -1509,7 +1833,7 @@ perf_counter_nmi_handler(struct notifier_block *self,
 #endif
         /*
          * Can't rely on the handled return value to say it was our NMI, two
-         * counters could trigger 'simultaneously' raising two back-to-back NMIs.
+         * events could trigger 'simultaneously' raising two back-to-back NMIs.
          *
          * If the first NMI handles both, the latter will be empty and daze
          * the CPU.
@@ -1519,8 +1843,8 @@ perf_counter_nmi_handler(struct notifier_block *self,
         return NOTIFY_STOP;
 }
 
-static __read_mostly struct notifier_block perf_counter_nmi_notifier = {
-        .notifier_call          = perf_counter_nmi_handler,
+static __read_mostly struct notifier_block perf_event_nmi_notifier = {
+        .notifier_call          = perf_event_nmi_handler,
         .next                   = NULL,
         .priority               = 1
 };
@@ -1530,8 +1854,8 @@ static struct x86_pmu p6_pmu = {
         .handle_irq             = p6_pmu_handle_irq,
         .disable_all            = p6_pmu_disable_all,
         .enable_all             = p6_pmu_enable_all,
-        .enable                 = p6_pmu_enable_counter,
-        .disable                = p6_pmu_disable_counter,
+        .enable                 = p6_pmu_enable_event,
+        .disable                = p6_pmu_disable_event,
         .eventsel               = MSR_P6_EVNTSEL0,
         .perfctr                = MSR_P6_PERFCTR0,
         .event_map              = p6_pmu_event_map,
@@ -1540,16 +1864,16 @@ static struct x86_pmu p6_pmu = {
         .apic                   = 1,
         .max_period             = (1ULL << 31) - 1,
         .version                = 0,
-        .num_counters           = 2,
+        .num_events             = 2,
         /*
-         * Counters have 40 bits implemented. However they are designed such
+         * Events have 40 bits implemented. However they are designed such
          * that bits [32-39] are sign extensions of bit 31. As such the
-         * effective width of a counter for P6-like PMU is 32 bits only.
+         * effective width of a event for P6-like PMU is 32 bits only.
          *
          * See IA-32 Intel Architecture Software developer manual Vol 3B
          */
-        .counter_bits           = 32,
-        .counter_mask           = (1ULL << 32) - 1,
+        .event_bits             = 32,
+        .event_mask             = (1ULL << 32) - 1,
 };
 
 static struct x86_pmu intel_pmu = {
@@ -1557,8 +1881,8 @@ static struct x86_pmu intel_pmu = {
         .handle_irq             = intel_pmu_handle_irq,
         .disable_all            = intel_pmu_disable_all,
         .enable_all             = intel_pmu_enable_all,
-        .enable                 = intel_pmu_enable_counter,
-        .disable                = intel_pmu_disable_counter,
+        .enable                 = intel_pmu_enable_event,
+        .disable                = intel_pmu_disable_event,
         .eventsel               = MSR_ARCH_PERFMON_EVENTSEL0,
         .perfctr                = MSR_ARCH_PERFMON_PERFCTR0,
         .event_map              = intel_pmu_event_map,
@@ -1568,9 +1892,11 @@ static struct x86_pmu intel_pmu = {
         /*
          * Intel PMCs cannot be accessed sanely above 32 bit width,
          * so we install an artificial 1<<31 period regardless of
-         * the generic counter period:
+         * the generic event period:
          */
         .max_period             = (1ULL << 31) - 1,
+        .enable_bts             = intel_pmu_enable_bts,
+        .disable_bts            = intel_pmu_disable_bts,
 };
 
 static struct x86_pmu amd_pmu = {
@@ -1578,16 +1904,16 @@ static struct x86_pmu amd_pmu = {
         .handle_irq             = amd_pmu_handle_irq,
         .disable_all            = amd_pmu_disable_all,
         .enable_all             = amd_pmu_enable_all,
-        .enable                 = amd_pmu_enable_counter,
-        .disable                = amd_pmu_disable_counter,
+        .enable                 = amd_pmu_enable_event,
+        .disable                = amd_pmu_disable_event,
         .eventsel               = MSR_K7_EVNTSEL0,
         .perfctr                = MSR_K7_PERFCTR0,
         .event_map              = amd_pmu_event_map,
         .raw_event              = amd_pmu_raw_event,
         .max_events             = ARRAY_SIZE(amd_perfmon_event_map),
-        .num_counters           = 4,
-        .counter_bits           = 48,
-        .counter_mask           = (1ULL << 48) - 1,
+        .num_events             = 4,
+        .event_bits             = 48,
+        .event_mask             = (1ULL << 48) - 1,
         .apic                   = 1,
         /* use highest bit to detect overflow */
         .max_period             = (1ULL << 47) - 1,
@@ -1644,7 +1970,7 @@ static int intel_pmu_init(void)
 
         /*
          * Check whether the Architectural PerfMon supports
-         * Branch Misses Retired Event or not.
+         * Branch Misses Retired hw_event or not.
          */
         cpuid(10, &eax.full, &ebx, &unused, &edx.full);
         if (eax.split.mask_length <= ARCH_PERFMON_BRANCH_MISSES_RETIRED)
@@ -1656,15 +1982,15 @@ static int intel_pmu_init(void)
 
         x86_pmu                         = intel_pmu;
         x86_pmu.version                 = version;
-        x86_pmu.num_counters            = eax.split.num_counters;
-        x86_pmu.counter_bits            = eax.split.bit_width;
-        x86_pmu.counter_mask            = (1ULL << eax.split.bit_width) - 1;
+        x86_pmu.num_events              = eax.split.num_events;
+        x86_pmu.event_bits              = eax.split.bit_width;
+        x86_pmu.event_mask              = (1ULL << eax.split.bit_width) - 1;
 
         /*
-         * Quirk: v2 perfmon does not report fixed-purpose counters, so
-         * assume at least 3 counters:
+         * Quirk: v2 perfmon does not report fixed-purpose events, so
+         * assume at least 3 events:
          */
-        x86_pmu.num_counters_fixed      = max((int)edx.split.num_counters_fixed, 3);
+        x86_pmu.num_events_fixed        = max((int)edx.split.num_events_fixed, 3);
 
         /*
          * Install the hw-cache-events table:
@@ -1711,11 +2037,11 @@ static int amd_pmu_init(void)
         return 0;
 }
 
-void __init init_hw_perf_counters(void)
+void __init init_hw_perf_events(void)
 {
         int err;
 
-        pr_info("Performance Counters: ");
+        pr_info("Performance Events: ");
 
         switch (boot_cpu_data.x86_vendor) {
         case X86_VENDOR_INTEL:
@@ -1728,45 +2054,45 @@ void __init init_hw_perf_counters(void)
                 return;
         }
         if (err != 0) {
-                pr_cont("no PMU driver, software counters only.\n");
+                pr_cont("no PMU driver, software events only.\n");
                 return;
         }
 
         pr_cont("%s PMU driver.\n", x86_pmu.name);
 
-        if (x86_pmu.num_counters > X86_PMC_MAX_GENERIC) {
-                WARN(1, KERN_ERR "hw perf counters %d > max(%d), clipping!",
-                     x86_pmu.num_counters, X86_PMC_MAX_GENERIC);
-                x86_pmu.num_counters = X86_PMC_MAX_GENERIC;
+        if (x86_pmu.num_events > X86_PMC_MAX_GENERIC) {
+                WARN(1, KERN_ERR "hw perf events %d > max(%d), clipping!",
+                     x86_pmu.num_events, X86_PMC_MAX_GENERIC);
+                x86_pmu.num_events = X86_PMC_MAX_GENERIC;
         }
-        perf_counter_mask = (1 << x86_pmu.num_counters) - 1;
-        perf_max_counters = x86_pmu.num_counters;
+        perf_event_mask = (1 << x86_pmu.num_events) - 1;
+        perf_max_events = x86_pmu.num_events;
 
-        if (x86_pmu.num_counters_fixed > X86_PMC_MAX_FIXED) {
-                WARN(1, KERN_ERR "hw perf counters fixed %d > max(%d), clipping!",
-                     x86_pmu.num_counters_fixed, X86_PMC_MAX_FIXED);
-                x86_pmu.num_counters_fixed = X86_PMC_MAX_FIXED;
+        if (x86_pmu.num_events_fixed > X86_PMC_MAX_FIXED) {
+                WARN(1, KERN_ERR "hw perf events fixed %d > max(%d), clipping!",
+                     x86_pmu.num_events_fixed, X86_PMC_MAX_FIXED);
+                x86_pmu.num_events_fixed = X86_PMC_MAX_FIXED;
         }
 
-        perf_counter_mask |=
-                ((1LL << x86_pmu.num_counters_fixed)-1) << X86_PMC_IDX_FIXED;
-        x86_pmu.intel_ctrl = perf_counter_mask;
+        perf_event_mask |=
+                ((1LL << x86_pmu.num_events_fixed)-1) << X86_PMC_IDX_FIXED;
+        x86_pmu.intel_ctrl = perf_event_mask;
 
-        perf_counters_lapic_init();
-        register_die_notifier(&perf_counter_nmi_notifier);
+        perf_events_lapic_init();
+        register_die_notifier(&perf_event_nmi_notifier);
 
-        pr_info("... version:                 %d\n",     x86_pmu.version);
-        pr_info("... bit width:               %d\n",     x86_pmu.counter_bits);
-        pr_info("... generic counters:        %d\n",     x86_pmu.num_counters);
-        pr_info("... value mask:              %016Lx\n", x86_pmu.counter_mask);
-        pr_info("... max period:              %016Lx\n", x86_pmu.max_period);
-        pr_info("... fixed-purpose counters:  %d\n",     x86_pmu.num_counters_fixed);
-        pr_info("... counter mask:            %016Lx\n", perf_counter_mask);
+        pr_info("... version:                %d\n",     x86_pmu.version);
+        pr_info("... bit width:              %d\n",     x86_pmu.event_bits);
+        pr_info("... generic registers:      %d\n",     x86_pmu.num_events);
+        pr_info("... value mask:             %016Lx\n", x86_pmu.event_mask);
+        pr_info("... max period:             %016Lx\n", x86_pmu.max_period);
+        pr_info("... fixed-purpose events:   %d\n",     x86_pmu.num_events_fixed);
+        pr_info("... event mask:             %016Lx\n", perf_event_mask);
 }
 
-static inline void x86_pmu_read(struct perf_counter *counter)
+static inline void x86_pmu_read(struct perf_event *event)
 {
-        x86_perf_counter_update(counter, &counter->hw, counter->hw.idx);
+        x86_perf_event_update(event, &event->hw, event->hw.idx);
 }
 
 static const struct pmu pmu = {
@@ -1776,13 +2102,16 @@ static const struct pmu pmu = {
         .unthrottle     = x86_pmu_unthrottle,
 };
 
-const struct pmu *hw_perf_counter_init(struct perf_counter *counter)
+const struct pmu *hw_perf_event_init(struct perf_event *event)
 {
         int err;
 
-        err = __hw_perf_counter_init(counter);
-        if (err)
+        err = __hw_perf_event_init(event);
+        if (err) {
+                if (event->destroy)
+                        event->destroy(event);
                 return ERR_PTR(err);
+        }
 
         return &pmu;
 }
@@ -1798,8 +2127,8 @@ void callchain_store(struct perf_callchain_entry *entry, u64 ip)
                 entry->ip[entry->nr++] = ip;
 }
 
-static DEFINE_PER_CPU(struct perf_callchain_entry, irq_entry);
-static DEFINE_PER_CPU(struct perf_callchain_entry, nmi_entry);
+static DEFINE_PER_CPU(struct perf_callchain_entry, pmc_irq_entry);
+static DEFINE_PER_CPU(struct perf_callchain_entry, pmc_nmi_entry);
 static DEFINE_PER_CPU(int, in_nmi_frame);
 
 
@@ -1952,9 +2281,9 @@ struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
         struct perf_callchain_entry *entry;
 
         if (in_nmi())
-                entry = &__get_cpu_var(nmi_entry);
+                entry = &__get_cpu_var(pmc_nmi_entry);
         else
-                entry = &__get_cpu_var(irq_entry);
+                entry = &__get_cpu_var(pmc_irq_entry);
 
         entry->nr = 0;
 
@@ -1962,3 +2291,8 @@ struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
 
         return entry;
 }
+
+void hw_perf_event_setup_online(int cpu)
+{
+        init_debug_store_on_cpu(cpu);
+}
diff --git a/arch/x86/kernel/cpu/perfctr-watchdog.c b/arch/x86/kernel/cpu/perfctr-watchdog.c
index e60ed740d2b3..fab786f60ed6 100644
--- a/arch/x86/kernel/cpu/perfctr-watchdog.c
+++ b/arch/x86/kernel/cpu/perfctr-watchdog.c
@@ -20,7 +20,7 @@
 #include <linux/kprobes.h>
 
 #include <asm/apic.h>
-#include <asm/perf_counter.h>
+#include <asm/perf_event.h>
 
 struct nmi_watchdog_ctlblk {
         unsigned int cccr_msr;
@@ -68,16 +68,16 @@ static inline unsigned int nmi_perfctr_msr_to_bit(unsigned int msr)
         /* returns the bit offset of the performance counter register */
         switch (boot_cpu_data.x86_vendor) {
         case X86_VENDOR_AMD:
-                return (msr - MSR_K7_PERFCTR0);
+                return msr - MSR_K7_PERFCTR0;
         case X86_VENDOR_INTEL:
                 if (cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON))
-                        return (msr - MSR_ARCH_PERFMON_PERFCTR0);
+                        return msr - MSR_ARCH_PERFMON_PERFCTR0;
 
                 switch (boot_cpu_data.x86) {
                 case 6:
-                        return (msr - MSR_P6_PERFCTR0);
+                        return msr - MSR_P6_PERFCTR0;
                 case 15:
-                        return (msr - MSR_P4_BPU_PERFCTR0);
+                        return msr - MSR_P4_BPU_PERFCTR0;
                 }
         }
         return 0;
@@ -92,16 +92,16 @@ static inline unsigned int nmi_evntsel_msr_to_bit(unsigned int msr)
         /* returns the bit offset of the event selection register */
         switch (boot_cpu_data.x86_vendor) {
         case X86_VENDOR_AMD:
-                return (msr - MSR_K7_EVNTSEL0);
+                return msr - MSR_K7_EVNTSEL0;
         case X86_VENDOR_INTEL:
                 if (cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON))
-                        return (msr - MSR_ARCH_PERFMON_EVENTSEL0);
+                        return msr - MSR_ARCH_PERFMON_EVENTSEL0;
 
                 switch (boot_cpu_data.x86) {
                 case 6:
-                        return (msr - MSR_P6_EVNTSEL0);
+                        return msr - MSR_P6_EVNTSEL0;
                 case 15:
-                        return (msr - MSR_P4_BSU_ESCR0);
+                        return msr - MSR_P4_BSU_ESCR0;
                 }
         }
         return 0;
@@ -113,7 +113,7 @@ int avail_to_resrv_perfctr_nmi_bit(unsigned int counter)
 {
         BUG_ON(counter > NMI_MAX_COUNTER_BITS);
 
-        return (!test_bit(counter, perfctr_nmi_owner));
+        return !test_bit(counter, perfctr_nmi_owner);
 }
 
 /* checks the an msr for availability */
@@ -124,7 +124,7 @@ int avail_to_resrv_perfctr_nmi(unsigned int msr)
         counter = nmi_perfctr_msr_to_bit(msr);
         BUG_ON(counter > NMI_MAX_COUNTER_BITS);
 
-        return (!test_bit(counter, perfctr_nmi_owner));
+        return !test_bit(counter, perfctr_nmi_owner);
 }
 EXPORT_SYMBOL(avail_to_resrv_perfctr_nmi_bit);
 
@@ -237,7 +237,7 @@ static unsigned int adjust_for_32bit_ctr(unsigned int hz)
          */
         counter_val = (u64)cpu_khz * 1000;
         do_div(counter_val, retval);
-        if (counter_val > 0x7fffffffULL) {
+        if (counter_val > 0x7fffffffULL) {
                 u64 count = (u64)cpu_khz * 1000;
                 do_div(count, 0x7fffffffUL);
                 retval = count + 1;
@@ -251,7 +251,7 @@ static void write_watchdog_counter(unsigned int perfctr_msr,
         u64 count = (u64)cpu_khz * 1000;
 
         do_div(count, nmi_hz);
-        if(descr)
+        if (descr)
                 pr_debug("setting %s to -0x%08Lx\n", descr, count);
         wrmsrl(perfctr_msr, 0 - count);
 }
@@ -262,7 +262,7 @@ static void write_watchdog_counter32(unsigned int perfctr_msr,
         u64 count = (u64)cpu_khz * 1000;
 
         do_div(count, nmi_hz);
-        if(descr)
+        if (descr)
                 pr_debug("setting %s to -0x%08Lx\n", descr, count);
         wrmsr(perfctr_msr, (u32)(-count), 0);
 }
@@ -296,7 +296,7 @@ static int setup_k7_watchdog(unsigned nmi_hz)
 
         /* setup the timer */
         wrmsr(evntsel_msr, evntsel, 0);
-        write_watchdog_counter(perfctr_msr, "K7_PERFCTR0",nmi_hz);
+        write_watchdog_counter(perfctr_msr, "K7_PERFCTR0", nmi_hz);
 
         /* initialize the wd struct before enabling */
         wd->perfctr_msr = perfctr_msr;
@@ -387,7 +387,7 @@ static int setup_p6_watchdog(unsigned nmi_hz)
         /* setup the timer */
         wrmsr(evntsel_msr, evntsel, 0);
         nmi_hz = adjust_for_32bit_ctr(nmi_hz);
-        write_watchdog_counter32(perfctr_msr, "P6_PERFCTR0",nmi_hz);
+        write_watchdog_counter32(perfctr_msr, "P6_PERFCTR0", nmi_hz);
 
         /* initialize the wd struct before enabling */
         wd->perfctr_msr = perfctr_msr;
@@ -415,7 +415,7 @@ static void __kprobes p6_rearm(struct nmi_watchdog_ctlblk *wd, unsigned nmi_hz)
         apic_write(APIC_LVTPC, APIC_DM_NMI);
 
         /* P6/ARCH_PERFMON has 32 bit counter write */
-        write_watchdog_counter32(wd->perfctr_msr, NULL,nmi_hz);
+        write_watchdog_counter32(wd->perfctr_msr, NULL, nmi_hz);
 }
 
 static const struct wd_ops p6_wd_ops = {
@@ -490,9 +490,9 @@ static int setup_p4_watchdog(unsigned nmi_hz)
         if (smp_num_siblings == 2) {
                 unsigned int ebx, apicid;
 
-                ebx = cpuid_ebx(1);
-                apicid = (ebx >> 24) & 0xff;
-                ht_num = apicid & 1;
+                ebx = cpuid_ebx(1);
+                apicid = (ebx >> 24) & 0xff;
+                ht_num = apicid & 1;
         } else
 #endif
                 ht_num = 0;
@@ -544,7 +544,7 @@ static int setup_p4_watchdog(unsigned nmi_hz)
         }
 
         evntsel = P4_ESCR_EVENT_SELECT(0x3F)
-                | P4_ESCR_OS
+                | P4_ESCR_OS
                 | P4_ESCR_USR;
 
         cccr_val |= P4_CCCR_THRESHOLD(15)
@@ -612,7 +612,7 @@ static void __kprobes p4_rearm(struct nmi_watchdog_ctlblk *wd, unsigned nmi_hz)
 {
         unsigned dummy;
         /*
-         * P4 quirks:
+         * P4 quirks:
          * - An overflown perfctr will assert its interrupt
          *   until the OVF flag in its CCCR is cleared.
          * - LVTPC is masked on interrupt and must be
@@ -662,7 +662,8 @@ static int setup_intel_arch_watchdog(unsigned nmi_hz)
          * NOTE: Corresponding bit = 0 in ebx indicates event present.
          */
         cpuid(10, &(eax.full), &ebx, &unused, &unused);
-        if ((eax.split.mask_length < (ARCH_PERFMON_UNHALTED_CORE_CYCLES_INDEX+1)) ||
+        if ((eax.split.mask_length <
+                        (ARCH_PERFMON_UNHALTED_CORE_CYCLES_INDEX+1)) ||
             (ebx & ARCH_PERFMON_UNHALTED_CORE_CYCLES_PRESENT))
                 return 0;
 
diff --git a/arch/x86/kernel/cpu/proc.c b/arch/x86/kernel/cpu/proc.c
index d5e30397246b..62ac8cb6ba27 100644
--- a/arch/x86/kernel/cpu/proc.c
+++ b/arch/x86/kernel/cpu/proc.c
@@ -116,11 +116,9 @@ static int show_cpuinfo(struct seq_file *m, void *v)
                 seq_printf(m, "TLB size\t: %d 4K pages\n", c->x86_tlbsize);
 #endif
         seq_printf(m, "clflush size\t: %u\n", c->x86_clflush_size);
-#ifdef CONFIG_X86_64
         seq_printf(m, "cache_alignment\t: %d\n", c->x86_cache_alignment);
         seq_printf(m, "address sizes\t: %u bits physical, %u bits virtual\n",
                    c->x86_phys_bits, c->x86_virt_bits);
-#endif
 
         seq_printf(m, "power management:");
         for (i = 0; i < 32; i++) {
@@ -128,7 +126,7 @@ static int show_cpuinfo(struct seq_file *m, void *v)
                         if (i < ARRAY_SIZE(x86_power_flags) &&
                             x86_power_flags[i])
                                 seq_printf(m, "%s%s",
-                                           x86_power_flags[i][0]?" ":"",
+                                           x86_power_flags[i][0] ? " " : "",
                                            x86_power_flags[i]);
                         else
                                 seq_printf(m, " [%d]", i);
diff --git a/arch/x86/kernel/cpu/sched.c b/arch/x86/kernel/cpu/sched.c
new file mode 100644
index 000000000000..a640ae5ad201
--- /dev/null
+++ b/arch/x86/kernel/cpu/sched.c
@@ -0,0 +1,55 @@
+#include <linux/sched.h>
+#include <linux/math64.h>
+#include <linux/percpu.h>
+#include <linux/irqflags.h>
+
+#include <asm/cpufeature.h>
+#include <asm/processor.h>
+
+#ifdef CONFIG_SMP
+
+static DEFINE_PER_CPU(struct aperfmperf, old_perf_sched);
+
+static unsigned long scale_aperfmperf(void)
+{
+        struct aperfmperf val, *old = &__get_cpu_var(old_perf_sched);
+        unsigned long ratio, flags;
+
+        local_irq_save(flags);
+        get_aperfmperf(&val);
+        local_irq_restore(flags);
+
+        ratio = calc_aperfmperf_ratio(old, &val);
+        *old = val;
+
+        return ratio;
+}
+
+unsigned long arch_scale_freq_power(struct sched_domain *sd, int cpu)
+{
+        /*
+         * do aperf/mperf on the cpu level because it includes things
+         * like turbo mode, which are relevant to full cores.
+         */
+        if (boot_cpu_has(X86_FEATURE_APERFMPERF))
+                return scale_aperfmperf();
+
+        /*
+         * maybe have something cpufreq here
+         */
+
+        return default_scale_freq_power(sd, cpu);
+}
+
+unsigned long arch_scale_smt_power(struct sched_domain *sd, int cpu)
+{
+        /*
+         * aperf/mperf already includes the smt gain
+         */
+        if (boot_cpu_has(X86_FEATURE_APERFMPERF))
+                return SCHED_LOAD_SCALE;
+
+        return default_scale_smt_power(sd, cpu);
+}
+
+#endif
diff --git a/arch/x86/kernel/cpu/vmware.c b/arch/x86/kernel/cpu/vmware.c
index 284c399e3234..1cbed97b59cf 100644
--- a/arch/x86/kernel/cpu/vmware.c
+++ b/arch/x86/kernel/cpu/vmware.c
@@ -24,6 +24,7 @@
 #include <linux/dmi.h>
 #include <asm/div64.h>
 #include <asm/vmware.h>
+#include <asm/x86_init.h>
 
 #define CPUID_VMWARE_INFO_LEAF  0x40000000
 #define VMWARE_HYPERVISOR_MAGIC 0x564D5868
@@ -47,19 +48,33 @@ static inline int __vmware_platform(void)
         return eax != (uint32_t)-1 && ebx == VMWARE_HYPERVISOR_MAGIC;
 }
 
-static unsigned long __vmware_get_tsc_khz(void)
+static unsigned long vmware_get_tsc_khz(void)
 {
-        uint64_t tsc_hz;
-        uint32_t eax, ebx, ecx, edx;
+        uint64_t tsc_hz;
+        uint32_t eax, ebx, ecx, edx;
+
+        VMWARE_PORT(GETHZ, eax, ebx, ecx, edx);
+
+        tsc_hz = eax | (((uint64_t)ebx) << 32);
+        do_div(tsc_hz, 1000);
+        BUG_ON(tsc_hz >> 32);
+        printk(KERN_INFO "TSC freq read from hypervisor : %lu.%03lu MHz\n",
+                         (unsigned long) tsc_hz / 1000,
+                         (unsigned long) tsc_hz % 1000);
+        return tsc_hz;
+}
+
+void __init vmware_platform_setup(void)
+{
+        uint32_t eax, ebx, ecx, edx;
 
-        VMWARE_PORT(GETHZ, eax, ebx, ecx, edx);
+        VMWARE_PORT(GETHZ, eax, ebx, ecx, edx);
 
-        if (ebx == UINT_MAX)
-                return 0;
-        tsc_hz = eax | (((uint64_t)ebx) << 32);
-        do_div(tsc_hz, 1000);
-        BUG_ON(tsc_hz >> 32);
-        return tsc_hz;
+        if (ebx != UINT_MAX)
+                x86_platform.calibrate_tsc = vmware_get_tsc_khz;
+        else
+                printk(KERN_WARNING
+                       "Failed to get TSC freq from the hypervisor\n");
 }
 
 /*
@@ -87,12 +102,6 @@ int vmware_platform(void)
         return 0;
 }
 
-unsigned long vmware_get_tsc_khz(void)
-{
-        BUG_ON(!vmware_platform());
-        return __vmware_get_tsc_khz();
-}
-
 /*
  * VMware hypervisor takes care of exporting a reliable TSC to the guest.
  * Still, due to timing difference when running on virtual cpus, the TSC can
diff --git a/arch/x86/kernel/cpuid.c b/arch/x86/kernel/cpuid.c
index b07af8861244..6a52d4b36a30 100644
--- a/arch/x86/kernel/cpuid.c
+++ b/arch/x86/kernel/cpuid.c
@@ -182,7 +182,7 @@ static struct notifier_block __refdata cpuid_class_cpu_notifier =
         .notifier_call = cpuid_class_cpu_callback,
 };
 
-static char *cpuid_nodename(struct device *dev)
+static char *cpuid_devnode(struct device *dev, mode_t *mode)
 {
         return kasprintf(GFP_KERNEL, "cpu/%u/cpuid", MINOR(dev->devt));
 }
@@ -203,7 +203,7 @@ static int __init cpuid_init(void)
                 err = PTR_ERR(cpuid_class);
                 goto out_chrdev;
         }
-        cpuid_class->nodename = cpuid_nodename;
+        cpuid_class->devnode = cpuid_devnode;
         for_each_online_cpu(i) {
                 err = cpuid_device_create(i);
                 if (err != 0)
diff --git a/arch/x86/kernel/doublefault_32.c b/arch/x86/kernel/doublefault_32.c
index b4f14c6c09d9..37250fe490b1 100644
--- a/arch/x86/kernel/doublefault_32.c
+++ b/arch/x86/kernel/doublefault_32.c
@@ -27,9 +27,7 @@ static void doublefault_fn(void)
 
         if (ptr_ok(gdt)) {
                 gdt += GDT_ENTRY_TSS << 3;
-                tss = *(u16 *)(gdt+2);
-                tss += *(u8 *)(gdt+4) << 16;
-                tss += *(u8 *)(gdt+7) << 24;
+                tss = get_desc_base((struct desc_struct *)gdt);
                 printk(KERN_EMERG "double fault, tss at %08lx\n", tss);
 
                 if (ptr_ok(tss)) {
diff --git a/arch/x86/kernel/ds.c b/arch/x86/kernel/ds.c
index 48bfe1386038..ef42a038f1a6 100644
--- a/arch/x86/kernel/ds.c
+++ b/arch/x86/kernel/ds.c
@@ -509,15 +509,15 @@ enum bts_field {
         bts_escape              = ((unsigned long)-1 & ~bts_qual_mask)
 };
 
-static inline unsigned long bts_get(const char *base, enum bts_field field)
+static inline unsigned long bts_get(const char *base, unsigned long field)
 {
         base += (ds_cfg.sizeof_ptr_field * field);
         return *(unsigned long *)base;
 }
 
-static inline void bts_set(char *base, enum bts_field field, unsigned long val)
+static inline void bts_set(char *base, unsigned long field, unsigned long val)
 {
-        base += (ds_cfg.sizeof_ptr_field * field);;
+        base += (ds_cfg.sizeof_ptr_field * field);
         (*(unsigned long *)base) = val;
 }
 
diff --git a/arch/x86/kernel/dumpstack.c b/arch/x86/kernel/dumpstack.c
index c8405718a4c3..2d8a371d4339 100644
--- a/arch/x86/kernel/dumpstack.c
+++ b/arch/x86/kernel/dumpstack.c
@@ -15,7 +15,6 @@
 #include <linux/bug.h>
 #include <linux/nmi.h>
 #include <linux/sysfs.h>
-#include <linux/ftrace.h>
 
 #include <asm/stacktrace.h>
 
diff --git a/arch/x86/kernel/dumpstack_32.c b/arch/x86/kernel/dumpstack_32.c
index bca5fba91c9e..f7dd2a7c3bf4 100644
--- a/arch/x86/kernel/dumpstack_32.c
+++ b/arch/x86/kernel/dumpstack_32.c
@@ -5,7 +5,6 @@
 #include <linux/kallsyms.h>
 #include <linux/kprobes.h>
 #include <linux/uaccess.h>
-#include <linux/utsname.h>
 #include <linux/hardirq.h>
 #include <linux/kdebug.h>
 #include <linux/module.h>
diff --git a/arch/x86/kernel/dumpstack_64.c b/arch/x86/kernel/dumpstack_64.c
index 54b0a3276766..a071e6be177e 100644
--- a/arch/x86/kernel/dumpstack_64.c
+++ b/arch/x86/kernel/dumpstack_64.c
@@ -5,7 +5,6 @@
 #include <linux/kallsyms.h>
 #include <linux/kprobes.h>
 #include <linux/uaccess.h>
-#include <linux/utsname.h>
 #include <linux/hardirq.h>
 #include <linux/kdebug.h>
 #include <linux/module.h>
diff --git a/arch/x86/kernel/e820.c b/arch/x86/kernel/e820.c
index 5cb5725b2bae..85419bb7d4ab 100644
--- a/arch/x86/kernel/e820.c
+++ b/arch/x86/kernel/e820.c
@@ -115,7 +115,7 @@ static void __init __e820_add_region(struct e820map *e820x, u64 start, u64 size,
 {
         int x = e820x->nr_map;
 
-        if (x == ARRAY_SIZE(e820x->map)) {
+        if (x >= ARRAY_SIZE(e820x->map)) {
                 printk(KERN_ERR "Ooops! Too many entries in the memory map!\n");
                 return;
         }
@@ -1331,7 +1331,7 @@ void __init e820_reserve_resources(void)
         struct resource *res;
         u64 end;
 
-        res = alloc_bootmem_low(sizeof(struct resource) * e820.nr_map);
+        res = alloc_bootmem(sizeof(struct resource) * e820.nr_map);
         e820_res = res;
         for (i = 0; i < e820.nr_map; i++) {
                 end = e820.map[i].addr + e820.map[i].size - 1;
@@ -1455,28 +1455,11 @@ char *__init default_machine_specific_memory_setup(void)
         return who;
 }
 
-char *__init __attribute__((weak)) machine_specific_memory_setup(void)
-{
-        if (x86_quirks->arch_memory_setup) {
-                char *who = x86_quirks->arch_memory_setup();
-
-                if (who)
-                        return who;
-        }
-        return default_machine_specific_memory_setup();
-}
-
-/* Overridden in paravirt.c if CONFIG_PARAVIRT */
-char * __init __attribute__((weak)) memory_setup(void)
-{
-        return machine_specific_memory_setup();
-}
-
 void __init setup_memory_map(void)
 {
         char *who;
 
-        who = memory_setup();
+        who = x86_init.resources.memory_setup();
         memcpy(&e820_saved, &e820, sizeof(struct e820map));
         printk(KERN_INFO "BIOS-provided physical RAM map:\n");
         e820_print_map(who);
diff --git a/arch/x86/kernel/early_printk.c b/arch/x86/kernel/early_printk.c
index 335f049d110f..2acfd3fdc0cc 100644
--- a/arch/x86/kernel/early_printk.c
+++ b/arch/x86/kernel/early_printk.c
@@ -160,721 +160,6 @@ static struct console early_serial_console = {
         .index =        -1,
 };
 
-#ifdef CONFIG_EARLY_PRINTK_DBGP
-
-static struct ehci_caps __iomem *ehci_caps;
-static struct ehci_regs __iomem *ehci_regs;
-static struct ehci_dbg_port __iomem *ehci_debug;
-static unsigned int dbgp_endpoint_out;
-
-struct ehci_dev {
-        u32 bus;
-        u32 slot;
-        u32 func;
-};
-
-static struct ehci_dev ehci_dev;
-
-#define USB_DEBUG_DEVNUM 127
-
-#define DBGP_DATA_TOGGLE        0x8800
-
-static inline u32 dbgp_pid_update(u32 x, u32 tok)
-{
-        return ((x ^ DBGP_DATA_TOGGLE) & 0xffff00) | (tok & 0xff);
-}
-
-static inline u32 dbgp_len_update(u32 x, u32 len)
-{
-        return (x & ~0x0f) | (len & 0x0f);
-}
-
-/*
- * USB Packet IDs (PIDs)
- */
-
-/* token */
-#define USB_PID_OUT             0xe1
-#define USB_PID_IN              0x69
-#define USB_PID_SOF             0xa5
-#define USB_PID_SETUP           0x2d
-/* handshake */
-#define USB_PID_ACK             0xd2
-#define USB_PID_NAK             0x5a
-#define USB_PID_STALL           0x1e
-#define USB_PID_NYET            0x96
-/* data */
-#define USB_PID_DATA0           0xc3
-#define USB_PID_DATA1           0x4b
-#define USB_PID_DATA2           0x87
-#define USB_PID_MDATA           0x0f
-/* Special */
-#define USB_PID_PREAMBLE        0x3c
-#define USB_PID_ERR             0x3c
-#define USB_PID_SPLIT           0x78
-#define USB_PID_PING            0xb4
-#define USB_PID_UNDEF_0         0xf0
-
-#define USB_PID_DATA_TOGGLE     0x88
-#define DBGP_CLAIM (DBGP_OWNER | DBGP_ENABLED | DBGP_INUSE)
-
-#define PCI_CAP_ID_EHCI_DEBUG   0xa
-
-#define HUB_ROOT_RESET_TIME     50      /* times are in msec */
-#define HUB_SHORT_RESET_TIME    10
-#define HUB_LONG_RESET_TIME     200
-#define HUB_RESET_TIMEOUT       500
-
-#define DBGP_MAX_PACKET         8
-
-static int dbgp_wait_until_complete(void)
-{
-        u32 ctrl;
-        int loop = 0x100000;
-
-        do {
-                ctrl = readl(&ehci_debug->control);
-                /* Stop when the transaction is finished */
-                if (ctrl & DBGP_DONE)
-                        break;
-        } while (--loop > 0);
-
-        if (!loop)
-                return -1;
-
-        /*
-         * Now that we have observed the completed transaction,
-         * clear the done bit.
-         */
-        writel(ctrl | DBGP_DONE, &ehci_debug->control);
-        return (ctrl & DBGP_ERROR) ? -DBGP_ERRCODE(ctrl) : DBGP_LEN(ctrl);
-}
-
-static void __init dbgp_mdelay(int ms)
-{
-        int i;
-
-        while (ms--) {
-                for (i = 0; i < 1000; i++)
-                        outb(0x1, 0x80);
-        }
-}
-
-static void dbgp_breath(void)
-{
-        /* Sleep to give the debug port a chance to breathe */
-}
-
-static int dbgp_wait_until_done(unsigned ctrl)
-{
-        u32 pids, lpid;
-        int ret;
-        int loop = 3;
-
-retry:
-        writel(ctrl | DBGP_GO, &ehci_debug->control);
-        ret = dbgp_wait_until_complete();
-        pids = readl(&ehci_debug->pids);
-        lpid = DBGP_PID_GET(pids);
-
-        if (ret < 0)
-                return ret;
-
-        /*
-         * If the port is getting full or it has dropped data
-         * start pacing ourselves, not necessary but it's friendly.
-         */
-        if ((lpid == USB_PID_NAK) || (lpid == USB_PID_NYET))
-                dbgp_breath();
-
-        /* If I get a NACK reissue the transmission */
-        if (lpid == USB_PID_NAK) {
-                if (--loop > 0)
-                        goto retry;
-        }
-
-        return ret;
-}
-
-static void dbgp_set_data(const void *buf, int size)
-{
-        const unsigned char *bytes = buf;
-        u32 lo, hi;
-        int i;
-
-        lo = hi = 0;
-        for (i = 0; i < 4 && i < size; i++)
-                lo |= bytes[i] << (8*i);
-        for (; i < 8 && i < size; i++)
-                hi |= bytes[i] << (8*(i - 4));
-        writel(lo, &ehci_debug->data03);
-        writel(hi, &ehci_debug->data47);
-}
-
-static void __init dbgp_get_data(void *buf, int size)
-{
-        unsigned char *bytes = buf;
-        u32 lo, hi;
-        int i;
-
-        lo = readl(&ehci_debug->data03);
-        hi = readl(&ehci_debug->data47);
-        for (i = 0; i < 4 && i < size; i++)
-                bytes[i] = (lo >> (8*i)) & 0xff;
-        for (; i < 8 && i < size; i++)
-                bytes[i] = (hi >> (8*(i - 4))) & 0xff;
-}
-
-static int dbgp_bulk_write(unsigned devnum, unsigned endpoint,
-                         const char *bytes, int size)
-{
-        u32 pids, addr, ctrl;
-        int ret;
-
-        if (size > DBGP_MAX_PACKET)
-                return -1;
-
-        addr = DBGP_EPADDR(devnum, endpoint);
-
-        pids = readl(&ehci_debug->pids);
-        pids = dbgp_pid_update(pids, USB_PID_OUT);
-
-        ctrl = readl(&ehci_debug->control);
-        ctrl = dbgp_len_update(ctrl, size);
-        ctrl |= DBGP_OUT;
-        ctrl |= DBGP_GO;
-
-        dbgp_set_data(bytes, size);
-        writel(addr, &ehci_debug->address);
-        writel(pids, &ehci_debug->pids);
-
-        ret = dbgp_wait_until_done(ctrl);
-        if (ret < 0)
-                return ret;
-
-        return ret;
-}
-
-static int __init dbgp_bulk_read(unsigned devnum, unsigned endpoint, void *data,
-                                 int size)
-{
-        u32 pids, addr, ctrl;
-        int ret;
-
-        if (size > DBGP_MAX_PACKET)
-                return -1;
-
-        addr = DBGP_EPADDR(devnum, endpoint);
-
-        pids = readl(&ehci_debug->pids);
-        pids = dbgp_pid_update(pids, USB_PID_IN);
-
-        ctrl = readl(&ehci_debug->control);
-        ctrl = dbgp_len_update(ctrl, size);
-        ctrl &= ~DBGP_OUT;
-        ctrl |= DBGP_GO;
-
-        writel(addr, &ehci_debug->address);
-        writel(pids, &ehci_debug->pids);
-        ret = dbgp_wait_until_done(ctrl);
-        if (ret < 0)
-                return ret;
-
-        if (size > ret)
-                size = ret;
-        dbgp_get_data(data, size);
-        return ret;
-}
-
-static int __init dbgp_control_msg(unsigned devnum, int requesttype,
-        int request, int value, int index, void *data, int size)
-{
-        u32 pids, addr, ctrl;
-        struct usb_ctrlrequest req;
-        int read;
-        int ret;
-
-        read = (requesttype & USB_DIR_IN) != 0;
-        if (size > (read ? DBGP_MAX_PACKET:0))
-                return -1;
-
-        /* Compute the control message */
-        req.bRequestType = requesttype;
-        req.bRequest = request;
-        req.wValue = cpu_to_le16(value);
-        req.wIndex = cpu_to_le16(index);
-        req.wLength = cpu_to_le16(size);
-
-        pids = DBGP_PID_SET(USB_PID_DATA0, USB_PID_SETUP);
-        addr = DBGP_EPADDR(devnum, 0);
-
-        ctrl = readl(&ehci_debug->control);
-        ctrl = dbgp_len_update(ctrl, sizeof(req));
-        ctrl |= DBGP_OUT;
-        ctrl |= DBGP_GO;
-
-        /* Send the setup message */
-        dbgp_set_data(&req, sizeof(req));
-        writel(addr, &ehci_debug->address);
-        writel(pids, &ehci_debug->pids);
-        ret = dbgp_wait_until_done(ctrl);
-        if (ret < 0)
-                return ret;
-
-        /* Read the result */
-        return dbgp_bulk_read(devnum, 0, data, size);
-}
-
-
-/* Find a PCI capability */
-static u32 __init find_cap(u32 num, u32 slot, u32 func, int cap)
-{
-        u8 pos;
-        int bytes;
-
-        if (!(read_pci_config_16(num, slot, func, PCI_STATUS) &
-                PCI_STATUS_CAP_LIST))
-                return 0;
-
-        pos = read_pci_config_byte(num, slot, func, PCI_CAPABILITY_LIST);
-        for (bytes = 0; bytes < 48 && pos >= 0x40; bytes++) {
-                u8 id;
-
-                pos &= ~3;
-                id = read_pci_config_byte(num, slot, func, pos+PCI_CAP_LIST_ID);
-                if (id == 0xff)
-                        break;
-                if (id == cap)
-                        return pos;
-
-                pos = read_pci_config_byte(num, slot, func,
-                                                 pos+PCI_CAP_LIST_NEXT);
-        }
-        return 0;
-}
-
-static u32 __init __find_dbgp(u32 bus, u32 slot, u32 func)
-{
-        u32 class;
-
-        class = read_pci_config(bus, slot, func, PCI_CLASS_REVISION);
-        if ((class >> 8) != PCI_CLASS_SERIAL_USB_EHCI)
-                return 0;
-
-        return find_cap(bus, slot, func, PCI_CAP_ID_EHCI_DEBUG);
-}
-
-static u32 __init find_dbgp(int ehci_num, u32 *rbus, u32 *rslot, u32 *rfunc)
-{
-        u32 bus, slot, func;
-
-        for (bus = 0; bus < 256; bus++) {
-                for (slot = 0; slot < 32; slot++) {
-                        for (func = 0; func < 8; func++) {
-                                unsigned cap;
-
-                                cap = __find_dbgp(bus, slot, func);
-
-                                if (!cap)
-                                        continue;
-                                if (ehci_num-- != 0)
-                                        continue;
-                                *rbus = bus;
-                                *rslot = slot;
-                                *rfunc = func;
-                                return cap;
-                        }
-                }
-        }
-        return 0;
-}
-
-static int __init ehci_reset_port(int port)
-{
-        u32 portsc;
-        u32 delay_time, delay;
-        int loop;
-
-        /* Reset the usb debug port */
-        portsc = readl(&ehci_regs->port_status[port - 1]);
-        portsc &= ~PORT_PE;
-        portsc |= PORT_RESET;
-        writel(portsc, &ehci_regs->port_status[port - 1]);
-
-        delay = HUB_ROOT_RESET_TIME;
-        for (delay_time = 0; delay_time < HUB_RESET_TIMEOUT;
-             delay_time += delay) {
-                dbgp_mdelay(delay);
-
-                portsc = readl(&ehci_regs->port_status[port - 1]);
-                if (portsc & PORT_RESET) {
-                        /* force reset to complete */
-                        loop = 2;
-                        writel(portsc & ~(PORT_RWC_BITS | PORT_RESET),
-                                &ehci_regs->port_status[port - 1]);
-                        do {
-                                portsc = readl(&ehci_regs->port_status[port-1]);
-                        } while ((portsc & PORT_RESET) && (--loop > 0));
-                }
-
-                /* Device went away? */
-                if (!(portsc & PORT_CONNECT))
-                        return -ENOTCONN;
-
-                /* bomb out completely if something weird happend */
-                if ((portsc & PORT_CSC))
-                        return -EINVAL;
-
-                /* If we've finished resetting, then break out of the loop */
-                if (!(portsc & PORT_RESET) && (portsc & PORT_PE))
-                        return 0;
-        }
-        return -EBUSY;
-}
-
-static int __init ehci_wait_for_port(int port)
-{
-        u32 status;
-        int ret, reps;
-
-        for (reps = 0; reps < 3; reps++) {
-                dbgp_mdelay(100);
-                status = readl(&ehci_regs->status);
-                if (status & STS_PCD) {
-                        ret = ehci_reset_port(port);
-                        if (ret == 0)
-                                return 0;
-                }
-        }
-        return -ENOTCONN;
-}
-
-#ifdef DBGP_DEBUG
-# define dbgp_printk early_printk
-#else
-static inline void dbgp_printk(const char *fmt, ...) { }
-#endif
-
-typedef void (*set_debug_port_t)(int port);
-
-static void __init default_set_debug_port(int port)
-{
-}
-
-static set_debug_port_t __initdata set_debug_port = default_set_debug_port;
-
-static void __init nvidia_set_debug_port(int port)
-{
-        u32 dword;
-        dword = read_pci_config(ehci_dev.bus, ehci_dev.slot, ehci_dev.func,
-                                 0x74);
-        dword &= ~(0x0f<<12);
-        dword |= ((port & 0x0f)<<12);
-        write_pci_config(ehci_dev.bus, ehci_dev.slot, ehci_dev.func, 0x74,
-                                 dword);
-        dbgp_printk("set debug port to %d\n", port);
-}
-
-static void __init detect_set_debug_port(void)
-{
-        u32 vendorid;
-
-        vendorid = read_pci_config(ehci_dev.bus, ehci_dev.slot, ehci_dev.func,
-                 0x00);
-
-        if ((vendorid & 0xffff) == 0x10de) {
-                dbgp_printk("using nvidia set_debug_port\n");
-                set_debug_port = nvidia_set_debug_port;
-        }
-}
-
-static int __init ehci_setup(void)
-{
-        struct usb_debug_descriptor dbgp_desc;
-        u32 cmd, ctrl, status, portsc, hcs_params;
-        u32 debug_port, new_debug_port = 0, n_ports;
-        u32  devnum;
-        int ret, i;
-        int loop;
-        int port_map_tried;
-        int playtimes = 3;
-
-try_next_time:
-        port_map_tried = 0;
-
-try_next_port:
-
-        hcs_params = readl(&ehci_caps->hcs_params);
-        debug_port = HCS_DEBUG_PORT(hcs_params);
-        n_ports    = HCS_N_PORTS(hcs_params);
-
-        dbgp_printk("debug_port: %d\n", debug_port);
-        dbgp_printk("n_ports:    %d\n", n_ports);
-
-        for (i = 1; i <= n_ports; i++) {
-                portsc = readl(&ehci_regs->port_status[i-1]);
-                dbgp_printk("portstatus%d: %08x\n", i, portsc);
-        }
-
-        if (port_map_tried && (new_debug_port != debug_port)) {
-                if (--playtimes) {
-                        set_debug_port(new_debug_port);
-                        goto try_next_time;
-                }
-                return -1;
-        }
-
-        loop = 10;
-        /* Reset the EHCI controller */
-        cmd = readl(&ehci_regs->command);
-        cmd |= CMD_RESET;
-        writel(cmd, &ehci_regs->command);
-        do {
-                cmd = readl(&ehci_regs->command);
-        } while ((cmd & CMD_RESET) && (--loop > 0));
-
-        if (!loop) {
-                dbgp_printk("can not reset ehci\n");
-                return -1;
-        }
-        dbgp_printk("ehci reset done\n");
-
-        /* Claim ownership, but do not enable yet */
-        ctrl = readl(&ehci_debug->control);
-        ctrl |= DBGP_OWNER;
-        ctrl &= ~(DBGP_ENABLED | DBGP_INUSE);
-        writel(ctrl, &ehci_debug->control);
-
-        /* Start the ehci running */
-        cmd = readl(&ehci_regs->command);
-        cmd &= ~(CMD_LRESET | CMD_IAAD | CMD_PSE | CMD_ASE | CMD_RESET);
-        cmd |= CMD_RUN;
-        writel(cmd, &ehci_regs->command);
-
-        /* Ensure everything is routed to the EHCI */
-        writel(FLAG_CF, &ehci_regs->configured_flag);
-
-        /* Wait until the controller is no longer halted */
-        loop = 10;
-        do {
-                status = readl(&ehci_regs->status);
-        } while ((status & STS_HALT) && (--loop > 0));
-
-        if (!loop) {
-                dbgp_printk("ehci can be started\n");
-                return -1;
-        }
-        dbgp_printk("ehci started\n");
-
-        /* Wait for a device to show up in the debug port */
-        ret = ehci_wait_for_port(debug_port);
-        if (ret < 0) {
-                dbgp_printk("No device found in debug port\n");
-                goto next_debug_port;
-        }
-        dbgp_printk("ehci wait for port done\n");
-
-        /* Enable the debug port */
-        ctrl = readl(&ehci_debug->control);
-        ctrl |= DBGP_CLAIM;
-        writel(ctrl, &ehci_debug->control);
-        ctrl = readl(&ehci_debug->control);
-        if ((ctrl & DBGP_CLAIM) != DBGP_CLAIM) {
-                dbgp_printk("No device in debug port\n");
-                writel(ctrl & ~DBGP_CLAIM, &ehci_debug->control);
-                goto err;
-        }
-        dbgp_printk("debug ported enabled\n");
-
-        /* Completely transfer the debug device to the debug controller */
-        portsc = readl(&ehci_regs->port_status[debug_port - 1]);
-        portsc &= ~PORT_PE;
-        writel(portsc, &ehci_regs->port_status[debug_port - 1]);
-
-        dbgp_mdelay(100);
-
-        /* Find the debug device and make it device number 127 */
-        for (devnum = 0; devnum <= 127; devnum++) {
-                ret = dbgp_control_msg(devnum,
-                        USB_DIR_IN | USB_TYPE_STANDARD | USB_RECIP_DEVICE,
-                        USB_REQ_GET_DESCRIPTOR, (USB_DT_DEBUG << 8), 0,
-                        &dbgp_desc, sizeof(dbgp_desc));
-                if (ret > 0)
-                        break;
-        }
-        if (devnum > 127) {
-                dbgp_printk("Could not find attached debug device\n");
-                goto err;
-        }
-        if (ret < 0) {
-                dbgp_printk("Attached device is not a debug device\n");
-                goto err;
-        }
-        dbgp_endpoint_out = dbgp_desc.bDebugOutEndpoint;
-
-        /* Move the device to 127 if it isn't already there */
-        if (devnum != USB_DEBUG_DEVNUM) {
-                ret = dbgp_control_msg(devnum,
-                        USB_DIR_OUT | USB_TYPE_STANDARD | USB_RECIP_DEVICE,
-                        USB_REQ_SET_ADDRESS, USB_DEBUG_DEVNUM, 0, NULL, 0);
-                if (ret < 0) {
-                        dbgp_printk("Could not move attached device to %d\n",
-                                USB_DEBUG_DEVNUM);
-                        goto err;
-                }
-                devnum = USB_DEBUG_DEVNUM;
-                dbgp_printk("debug device renamed to 127\n");
-        }
-
-        /* Enable the debug interface */
-        ret = dbgp_control_msg(USB_DEBUG_DEVNUM,
-                USB_DIR_OUT | USB_TYPE_STANDARD | USB_RECIP_DEVICE,
-                USB_REQ_SET_FEATURE, USB_DEVICE_DEBUG_MODE, 0, NULL, 0);
-        if (ret < 0) {
-                dbgp_printk(" Could not enable the debug device\n");
-                goto err;
-        }
-        dbgp_printk("debug interface enabled\n");
-
-        /* Perform a small write to get the even/odd data state in sync
-         */
-        ret = dbgp_bulk_write(USB_DEBUG_DEVNUM, dbgp_endpoint_out, " ", 1);
-        if (ret < 0) {
-                dbgp_printk("dbgp_bulk_write failed: %d\n", ret);
-                goto err;
-        }
-        dbgp_printk("small write doned\n");
-
-        return 0;
-err:
-        /* Things didn't work so remove my claim */
-        ctrl = readl(&ehci_debug->control);
-        ctrl &= ~(DBGP_CLAIM | DBGP_OUT);
-        writel(ctrl, &ehci_debug->control);
-        return -1;
-
-next_debug_port:
-        port_map_tried |= (1<<(debug_port - 1));
-        new_debug_port = ((debug_port-1+1)%n_ports) + 1;
-        if (port_map_tried != ((1<<n_ports) - 1)) {
-                set_debug_port(new_debug_port);
-                goto try_next_port;
-        }
-        if (--playtimes) {
-                set_debug_port(new_debug_port);
-                goto try_next_time;
-        }
-
-        return -1;
-}
-
-static int __init early_dbgp_init(char *s)
-{
-        u32 debug_port, bar, offset;
-        u32 bus, slot, func, cap;
-        void __iomem *ehci_bar;
-        u32 dbgp_num;
-        u32 bar_val;
-        char *e;
-        int ret;
-        u8 byte;
-
-        if (!early_pci_allowed())
-                return -1;
-
-        dbgp_num = 0;
-        if (*s)
-                dbgp_num = simple_strtoul(s, &e, 10);
-        dbgp_printk("dbgp_num: %d\n", dbgp_num);
-
-        cap = find_dbgp(dbgp_num, &bus, &slot, &func);
-        if (!cap)
-                return -1;
-
-        dbgp_printk("Found EHCI debug port on %02x:%02x.%1x\n", bus, slot,
-                         func);
-
-        debug_port = read_pci_config(bus, slot, func, cap);
-        bar = (debug_port >> 29) & 0x7;
-        bar = (bar * 4) + 0xc;
-        offset = (debug_port >> 16) & 0xfff;
-        dbgp_printk("bar: %02x offset: %03x\n", bar, offset);
-        if (bar != PCI_BASE_ADDRESS_0) {
-                dbgp_printk("only debug ports on bar 1 handled.\n");
-
-                return -1;
-        }
-
-        bar_val = read_pci_config(bus, slot, func, PCI_BASE_ADDRESS_0);
-        dbgp_printk("bar_val: %02x offset: %03x\n", bar_val, offset);
-        if (bar_val & ~PCI_BASE_ADDRESS_MEM_MASK) {
-                dbgp_printk("only simple 32bit mmio bars supported\n");
-
-                return -1;
-        }
-
-        /* double check if the mem space is enabled */
-        byte = read_pci_config_byte(bus, slot, func, 0x04);
-        if (!(byte & 0x2)) {
-                byte  |= 0x02;
-                write_pci_config_byte(bus, slot, func, 0x04, byte);
-                dbgp_printk("mmio for ehci enabled\n");
-        }
-
-        /*
-         * FIXME I don't have the bar size so just guess PAGE_SIZE is more
-         * than enough.  1K is the biggest I have seen.
-         */
-        set_fixmap_nocache(FIX_DBGP_BASE, bar_val & PAGE_MASK);
-        ehci_bar = (void __iomem *)__fix_to_virt(FIX_DBGP_BASE);
-        ehci_bar += bar_val & ~PAGE_MASK;
-        dbgp_printk("ehci_bar: %p\n", ehci_bar);
-
-        ehci_caps  = ehci_bar;
-        ehci_regs  = ehci_bar + HC_LENGTH(readl(&ehci_caps->hc_capbase));
-        ehci_debug = ehci_bar + offset;
-        ehci_dev.bus = bus;
-        ehci_dev.slot = slot;
-        ehci_dev.func = func;
-
-        detect_set_debug_port();
-
-        ret = ehci_setup();
-        if (ret < 0) {
-                dbgp_printk("ehci_setup failed\n");
-                ehci_debug = NULL;
-
-                return -1;
-        }
-
-        return 0;
-}
-
-static void early_dbgp_write(struct console *con, const char *str, u32 n)
-{
-        int chunk, ret;
-
-        if (!ehci_debug)
-                return;
-        while (n > 0) {
-                chunk = n;
-                if (chunk > DBGP_MAX_PACKET)
-                        chunk = DBGP_MAX_PACKET;
-                ret = dbgp_bulk_write(USB_DEBUG_DEVNUM,
-                        dbgp_endpoint_out, str, chunk);
-                str += chunk;
-                n -= chunk;
-        }
-}
-
-static struct console early_dbgp_console = {
-        .name =         "earlydbg",
-        .write =        early_dbgp_write,
-        .flags =        CON_PRINTBUFFER,
-        .index =        -1,
-};
-#endif
-
 /* Direct interface for emergencies */
 static struct console *early_console = &early_vga_console;
 static int __initdata early_console_initialized;
@@ -891,10 +176,19 @@ asmlinkage void early_printk(const char *fmt, ...)
         va_end(ap);
 }
 
+static inline void early_console_register(struct console *con, int keep_early)
+{
+        early_console = con;
+        if (keep_early)
+                early_console->flags &= ~CON_BOOT;
+        else
+                early_console->flags |= CON_BOOT;
+        register_console(early_console);
+}
 
 static int __init setup_early_printk(char *buf)
 {
-        int keep_early;
+        int keep;
 
         if (!buf)
                 return 0;
@@ -903,42 +197,34 @@ static int __init setup_early_printk(char *buf)
                 return 0;
         early_console_initialized = 1;
 
-        keep_early = (strstr(buf, "keep") != NULL);
-
-        if (!strncmp(buf, "serial", 6)) {
-                early_serial_init(buf + 6);
-                early_console = &early_serial_console;
-        } else if (!strncmp(buf, "ttyS", 4)) {
-                early_serial_init(buf);
-                early_console = &early_serial_console;
-        } else if (!strncmp(buf, "vga", 3)
-                && boot_params.screen_info.orig_video_isVGA == 1) {
-                max_xpos = boot_params.screen_info.orig_video_cols;
-                max_ypos = boot_params.screen_info.orig_video_lines;
-                current_ypos = boot_params.screen_info.orig_y;
-                early_console = &early_vga_console;
+        keep = (strstr(buf, "keep") != NULL);
+
+        while (*buf != '\0') {
+                if (!strncmp(buf, "serial", 6)) {
+                        early_serial_init(buf + 6);
+                        early_console_register(&early_serial_console, keep);
+                }
+                if (!strncmp(buf, "ttyS", 4)) {
+                        early_serial_init(buf + 4);
+                        early_console_register(&early_serial_console, keep);
+                }
+                if (!strncmp(buf, "vga", 3) &&
+                    boot_params.screen_info.orig_video_isVGA == 1) {
+                        max_xpos = boot_params.screen_info.orig_video_cols;
+                        max_ypos = boot_params.screen_info.orig_video_lines;
+                        current_ypos = boot_params.screen_info.orig_y;
+                        early_console_register(&early_vga_console, keep);
+                }
 #ifdef CONFIG_EARLY_PRINTK_DBGP
-        } else if (!strncmp(buf, "dbgp", 4)) {
-                if (early_dbgp_init(buf+4) < 0)
-                        return 0;
-                early_console = &early_dbgp_console;
-                /*
-                 * usb subsys will reset ehci controller, so don't keep
-                 * that early console
-                 */
-                keep_early = 0;
+                if (!strncmp(buf, "dbgp", 4) && !early_dbgp_init(buf + 4))
+                        early_console_register(&early_dbgp_console, keep);
 #endif
 #ifdef CONFIG_HVC_XEN
-        } else if (!strncmp(buf, "xen", 3)) {
-                early_console = &xenboot_console;
+                if (!strncmp(buf, "xen", 3))
+                        early_console_register(&xenboot_console, keep);
 #endif
+                buf++;
         }
-
-        if (keep_early)
-                early_console->flags &= ~CON_BOOT;
-        else
-                early_console->flags |= CON_BOOT;
-        register_console(early_console);
         return 0;
 }
 
diff --git a/arch/x86/kernel/efi.c b/arch/x86/kernel/efi.c
index fe26ba3e3451..ad5bd988fb79 100644
--- a/arch/x86/kernel/efi.c
+++ b/arch/x86/kernel/efi.c
@@ -42,6 +42,7 @@
 #include <asm/time.h>
 #include <asm/cacheflush.h>
 #include <asm/tlbflush.h>
+#include <asm/x86_init.h>
 
 #define EFI_DEBUG       1
 #define PFX             "EFI: "
@@ -453,6 +454,9 @@ void __init efi_init(void)
         if (add_efi_memmap)
                 do_add_efi_memmap();
 
+        x86_platform.get_wallclock = efi_get_time;
+        x86_platform.set_wallclock = efi_set_rtc_mmss;
+
         /* Setup for EFI runtime service */
         reboot_type = BOOT_EFI;
 
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index c251be745107..b5c061f8f358 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -146,7 +146,7 @@ ENTRY(ftrace_graph_caller)
 END(ftrace_graph_caller)
 
 GLOBAL(return_to_handler)
-        subq  $80, %rsp
+        subq  $24, %rsp
 
         /* Save the return values */
         movq %rax, (%rsp)
@@ -155,10 +155,10 @@ GLOBAL(return_to_handler)
 
         call ftrace_return_to_handler
 
-        movq %rax, 72(%rsp)
+        movq %rax, 16(%rsp)
         movq 8(%rsp), %rdx
         movq (%rsp), %rax
-        addq $72, %rsp
+        addq $16, %rsp
         retq
 #endif
 
@@ -536,20 +536,13 @@ sysret_signal:
         bt $TIF_SYSCALL_AUDIT,%edx
         jc sysret_audit
 #endif
-        /* edx: work flags (arg3) */
-        leaq -ARGOFFSET(%rsp),%rdi # &pt_regs -> arg1
-        xorl %esi,%esi # oldset -> arg2
-        SAVE_REST
-        FIXUP_TOP_OF_STACK %r11
-        call do_notify_resume
-        RESTORE_TOP_OF_STACK %r11
-        RESTORE_REST
-        movl $_TIF_WORK_MASK,%edi
-        /* Use IRET because user could have changed frame. This
-           works because ptregscall_common has called FIXUP_TOP_OF_STACK. */
-        DISABLE_INTERRUPTS(CLBR_NONE)
-        TRACE_IRQS_OFF
-        jmp int_with_check
+        /*
+         * We have a signal, or exit tracing or single-step.
+         * These all wind up with the iret return path anyway,
+         * so just join that path right now.
+         */
+        FIXUP_TOP_OF_STACK %r11, -ARGOFFSET
+        jmp int_check_syscall_exit_work
 
 badsys:
         movq $-ENOSYS,RAX-ARGOFFSET(%rsp)
@@ -654,6 +647,7 @@ int_careful:
 int_very_careful:
         TRACE_IRQS_ON
         ENABLE_INTERRUPTS(CLBR_NONE)
+int_check_syscall_exit_work:
         SAVE_REST
         /* Check for syscall exit trace */
         testl $_TIF_WORK_SYSCALL_EXIT,%edx
@@ -1021,7 +1015,7 @@ apicinterrupt ERROR_APIC_VECTOR \
 apicinterrupt SPURIOUS_APIC_VECTOR \
         spurious_interrupt smp_spurious_interrupt
 
-#ifdef CONFIG_PERF_COUNTERS
+#ifdef CONFIG_PERF_EVENTS
 apicinterrupt LOCAL_PENDING_VECTOR \
         perf_pending_interrupt smp_perf_pending_interrupt
 #endif
diff --git a/arch/x86/kernel/ftrace.c b/arch/x86/kernel/ftrace.c
index d94e1ea3b9fe..9dbb527e1652 100644
--- a/arch/x86/kernel/ftrace.c
+++ b/arch/x86/kernel/ftrace.c
@@ -417,10 +417,6 @@ void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr,
         unsigned long return_hooker = (unsigned long)
                                 &return_to_handler;
 
-        /* Nmi's are currently unsupported */
-        if (unlikely(in_nmi()))
-                return;
-
         if (unlikely(atomic_read(&current->tracing_graph_pause)))
                 return;
 
@@ -498,37 +494,56 @@ static struct syscall_metadata *find_syscall_meta(unsigned long *syscall)
 
 struct syscall_metadata *syscall_nr_to_meta(int nr)
 {
-        if (!syscalls_metadata || nr >= FTRACE_SYSCALL_MAX || nr < 0)
+        if (!syscalls_metadata || nr >= NR_syscalls || nr < 0)
                 return NULL;
 
         return syscalls_metadata[nr];
 }
 
-void arch_init_ftrace_syscalls(void)
+int syscall_name_to_nr(char *name)
+{
+        int i;
+
+        if (!syscalls_metadata)
+                return -1;
+
+        for (i = 0; i < NR_syscalls; i++) {
+                if (syscalls_metadata[i]) {
+                        if (!strcmp(syscalls_metadata[i]->name, name))
+                                return i;
+                }
+        }
+        return -1;
+}
+
+void set_syscall_enter_id(int num, int id)
+{
+        syscalls_metadata[num]->enter_id = id;
+}
+
+void set_syscall_exit_id(int num, int id)
+{
+        syscalls_metadata[num]->exit_id = id;
+}
+
+static int __init arch_init_ftrace_syscalls(void)
 {
         int i;
         struct syscall_metadata *meta;
         unsigned long **psys_syscall_table = &sys_call_table;
-        static atomic_t refs;
-
-        if (atomic_inc_return(&refs) != 1)
-                goto end;
 
         syscalls_metadata = kzalloc(sizeof(*syscalls_metadata) *
-                                        FTRACE_SYSCALL_MAX, GFP_KERNEL);
+                                        NR_syscalls, GFP_KERNEL);
         if (!syscalls_metadata) {
                 WARN_ON(1);
-                return;
+                return -ENOMEM;
         }
 
-        for (i = 0; i < FTRACE_SYSCALL_MAX; i++) {
+        for (i = 0; i < NR_syscalls; i++) {
                 meta = find_syscall_meta(psys_syscall_table[i]);
                 syscalls_metadata[i] = meta;
         }
-        return;
-
-        /* Paranoid: avoid overflow */
-end:
-        atomic_dec(&refs);
+        return 0;
 }
+arch_initcall(arch_init_ftrace_syscalls);
 #endif
diff --git a/arch/x86/kernel/head32.c b/arch/x86/kernel/head32.c
index 3f8579f8d42c..4f8e2507e8f3 100644
--- a/arch/x86/kernel/head32.c
+++ b/arch/x86/kernel/head32.c
@@ -11,8 +11,21 @@
 #include <asm/setup.h>
 #include <asm/sections.h>
 #include <asm/e820.h>
-#include <asm/bios_ebda.h>
+#include <asm/page.h>
 #include <asm/trampoline.h>
+#include <asm/apic.h>
+#include <asm/io_apic.h>
+#include <asm/bios_ebda.h>
+
+static void __init i386_default_early_setup(void)
+{
+        /* Initilize 32bit specific setup functions */
+        x86_init.resources.probe_roms = probe_roms;
+        x86_init.resources.reserve_resources = i386_reserve_resources;
+        x86_init.mpparse.setup_ioapic_ids = setup_ioapic_ids_from_mpc;
+
+        reserve_ebda_region();
+}
 
 void __init i386_start_kernel(void)
 {
@@ -29,7 +42,16 @@ void __init i386_start_kernel(void)
                 reserve_early(ramdisk_image, ramdisk_end, "RAMDISK");
         }
 #endif
-        reserve_ebda_region();
+
+        /* Call the subarch specific early setup function */
+        switch (boot_params.hdr.hardware_subarch) {
+        case X86_SUBARCH_MRST:
+                x86_mrst_early_setup();
+                break;
+        default:
+                i386_default_early_setup();
+                break;
+        }
 
         /*
          * At this point everything still needed from the boot loader
diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c
index 70eaa852c732..0b06cd778fd9 100644
--- a/arch/x86/kernel/head64.c
+++ b/arch/x86/kernel/head64.c
@@ -23,8 +23,8 @@
 #include <asm/sections.h>
 #include <asm/kdebug.h>
 #include <asm/e820.h>
-#include <asm/bios_ebda.h>
 #include <asm/trampoline.h>
+#include <asm/bios_ebda.h>
 
 static void __init zap_identity_mappings(void)
 {
diff --git a/arch/x86/kernel/head_32.S b/arch/x86/kernel/head_32.S
index cc827ac9e8d3..218aad7ee76e 100644
--- a/arch/x86/kernel/head_32.S
+++ b/arch/x86/kernel/head_32.S
@@ -157,6 +157,7 @@ subarch_entries:
         .long default_entry             /* normal x86/PC */
         .long lguest_entry              /* lguest hypervisor */
         .long xen_entry                 /* Xen hypervisor */
+        .long default_entry             /* Moorestown MID */
 num_subarch_entries = (. - subarch_entries) / 4
 .previous
 #endif /* CONFIG_PARAVIRT */
@@ -439,7 +440,6 @@ is386:	movl $2,%ecx		# set MP
         jne 1f
         movl $per_cpu__gdt_page,%eax
         movl $per_cpu__stack_canary,%ecx
-        subl $20, %ecx
         movw %cx, 8 * GDT_ENTRY_STACK_CANARY + 2(%eax)
         shrl $16, %ecx
         movb %cl, 8 * GDT_ENTRY_STACK_CANARY + 4(%eax)
@@ -608,7 +608,7 @@ ENTRY(initial_code)
 /*
  * BSS section
  */
-.section ".bss.page_aligned","wa"
+__PAGE_ALIGNED_BSS
         .align PAGE_SIZE_asm
 #ifdef CONFIG_X86_PAE
 swapper_pg_pmd:
@@ -626,7 +626,7 @@ ENTRY(empty_zero_page)
  * This starts the data section.
  */
 #ifdef CONFIG_X86_PAE
-.section ".data.page_aligned","wa"
+__PAGE_ALIGNED_DATA
         /* Page-aligned for the benefit of paravirt? */
         .align PAGE_SIZE_asm
 ENTRY(swapper_pg_dir)
diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S
index fa54f78e2a05..d0bc0a13a437 100644
--- a/arch/x86/kernel/head_64.S
+++ b/arch/x86/kernel/head_64.S
@@ -418,7 +418,7 @@ ENTRY(phys_base)
 ENTRY(idt_table)
         .skip IDT_ENTRIES * 16
 
-        .section .bss.page_aligned, "aw", @nobits
+        __PAGE_ALIGNED_BSS
         .align PAGE_SIZE
 ENTRY(empty_zero_page)
         .skip PAGE_SIZE
diff --git a/arch/x86/kernel/i8253.c b/arch/x86/kernel/i8253.c
index 5cf36c053ac4..23c167925a5c 100644
--- a/arch/x86/kernel/i8253.c
+++ b/arch/x86/kernel/i8253.c
@@ -19,12 +19,6 @@
 DEFINE_SPINLOCK(i8253_lock);
 EXPORT_SYMBOL(i8253_lock);
 
-#ifdef CONFIG_X86_32
-static void pit_disable_clocksource(void);
-#else
-static inline void pit_disable_clocksource(void) { }
-#endif
-
 /*
  * HPET replaces the PIT, when enabled. So we need to know, which of
  * the two timers is used
@@ -57,12 +51,10 @@ static void init_pit_timer(enum clock_event_mode mode,
                         outb_pit(0, PIT_CH0);
                         outb_pit(0, PIT_CH0);
                 }
-                pit_disable_clocksource();
                 break;
 
         case CLOCK_EVT_MODE_ONESHOT:
                 /* One shot setup */
-                pit_disable_clocksource();
                 outb_pit(0x38, PIT_MODE);
                 break;
 
@@ -200,17 +192,6 @@ static struct clocksource pit_cs = {
         .shift          = 20,
 };
 
-static void pit_disable_clocksource(void)
-{
-        /*
-         * Use mult to check whether it is registered or not
-         */
-        if (pit_cs.mult) {
-                clocksource_unregister(&pit_cs);
-                pit_cs.mult = 0;
-        }
-}
-
 static int __init init_pit_clocksource(void)
 {
          /*
diff --git a/arch/x86/kernel/init_task.c b/arch/x86/kernel/init_task.c
index 270ff83efc11..3a54dcb9cd0e 100644
--- a/arch/x86/kernel/init_task.c
+++ b/arch/x86/kernel/init_task.c
@@ -20,9 +20,8 @@ static struct sighand_struct init_sighand = INIT_SIGHAND(init_sighand);
  * way process stacks are handled. This is done by having a special
  * "init_task" linker map entry..
  */
-union thread_union init_thread_union
-        __attribute__((__section__(".data.init_task"))) =
-                { INIT_THREAD_INFO(init_task) };
+union thread_union init_thread_union __init_task_data =
+        { INIT_THREAD_INFO(init_task) };
 
 /*
  * Initial task structure.
diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c
index b0cdde6932f5..74656d1d4e30 100644
--- a/arch/x86/kernel/irq.c
+++ b/arch/x86/kernel/irq.c
@@ -104,7 +104,7 @@ static int show_other_interrupts(struct seq_file *p, int prec)
         seq_printf(p, "  Threshold APIC interrupts\n");
 # endif
 #endif
-#ifdef CONFIG_X86_NEW_MCE
+#ifdef CONFIG_X86_MCE
         seq_printf(p, "%*s: ", prec, "MCE");
         for_each_online_cpu(j)
                 seq_printf(p, "%10u ", per_cpu(mce_exception_count, j));
@@ -200,7 +200,7 @@ u64 arch_irq_stat_cpu(unsigned int cpu)
         sum += irq_stats(cpu)->irq_threshold_count;
 # endif
 #endif
-#ifdef CONFIG_X86_NEW_MCE
+#ifdef CONFIG_X86_MCE
         sum += per_cpu(mce_exception_count, cpu);
         sum += per_cpu(mce_poll_count, cpu);
 #endif
diff --git a/arch/x86/kernel/irq_32.c b/arch/x86/kernel/irq_32.c
index 3b09634a5153..7d35d0fe2329 100644
--- a/arch/x86/kernel/irq_32.c
+++ b/arch/x86/kernel/irq_32.c
@@ -218,7 +218,6 @@ bool handle_irq(unsigned irq, struct pt_regs *regs)
 void fixup_irqs(void)
 {
         unsigned int irq;
-        static int warned;
         struct irq_desc *desc;
 
         for_each_irq_desc(irq, desc) {
@@ -236,8 +235,8 @@ void fixup_irqs(void)
                 }
                 if (desc->chip->set_affinity)
                         desc->chip->set_affinity(irq, affinity);
-                else if (desc->action && !(warned++))
-                        printk("Cannot set affinity for irq %i\n", irq);
+                else if (desc->action)
+                        printk_once("Cannot set affinity for irq %i\n", irq);
         }
 
 #if 0
diff --git a/arch/x86/kernel/irqinit.c b/arch/x86/kernel/irqinit.c
index 92b7703d3d58..40f30773fb29 100644
--- a/arch/x86/kernel/irqinit.c
+++ b/arch/x86/kernel/irqinit.c
@@ -116,7 +116,7 @@ int vector_used_by_percpu_irq(unsigned int vector)
         return 0;
 }
 
-static void __init init_ISA_irqs(void)
+void __init init_ISA_irqs(void)
 {
         int i;
 
@@ -140,8 +140,10 @@ static void __init init_ISA_irqs(void)
         }
 }
 
-/* Overridden in paravirt.c */
-void init_IRQ(void) __attribute__((weak, alias("native_init_IRQ")));
+void __init init_IRQ(void)
+{
+        x86_init.irqs.intr_init();
+}
 
 static void __init smp_intr_init(void)
 {
@@ -190,7 +192,7 @@ static void __init apic_intr_init(void)
 #ifdef CONFIG_X86_MCE_THRESHOLD
         alloc_intr_gate(THRESHOLD_APIC_VECTOR, threshold_interrupt);
 #endif
-#if defined(CONFIG_X86_NEW_MCE) && defined(CONFIG_X86_LOCAL_APIC)
+#if defined(CONFIG_X86_MCE) && defined(CONFIG_X86_LOCAL_APIC)
         alloc_intr_gate(MCE_SELF_VECTOR, mce_self_interrupt);
 #endif
 
@@ -206,39 +208,19 @@ static void __init apic_intr_init(void)
         alloc_intr_gate(ERROR_APIC_VECTOR, error_interrupt);
 
         /* Performance monitoring interrupts: */
-# ifdef CONFIG_PERF_COUNTERS
+# ifdef CONFIG_PERF_EVENTS
         alloc_intr_gate(LOCAL_PENDING_VECTOR, perf_pending_interrupt);
 # endif
 
 #endif
 }
 
-/**
- * x86_quirk_pre_intr_init - initialisation prior to setting up interrupt vectors
- *
- * Description:
- *      Perform any necessary interrupt initialisation prior to setting up
- *      the "ordinary" interrupt call gates.  For legacy reasons, the ISA
- *      interrupts should be initialised here if the machine emulates a PC
- *      in any way.
- **/
-static void __init x86_quirk_pre_intr_init(void)
-{
-#ifdef CONFIG_X86_32
-        if (x86_quirks->arch_pre_intr_init) {
-                if (x86_quirks->arch_pre_intr_init())
-                        return;
-        }
-#endif
-        init_ISA_irqs();
-}
-
 void __init native_init_IRQ(void)
 {
         int i;
 
         /* Execute any quirks before the call gates are initialised: */
-        x86_quirk_pre_intr_init();
+        x86_init.irqs.pre_vector_init();
 
         apic_intr_init();
 
@@ -258,12 +240,6 @@ void __init native_init_IRQ(void)
 
 #ifdef CONFIG_X86_32
         /*
-         * Call quirks after call gates are initialised (usually add in
-         * the architecture specific gates):
-         */
-        x86_quirk_intr_init();
-
-        /*
          * External FPU? Set up irq13 if so, for
          * original braindamaged IBM FERR coupling.
          */
diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c
index c664d515f613..63b0ec8d3d4a 100644
--- a/arch/x86/kernel/kvm.c
+++ b/arch/x86/kernel/kvm.c
@@ -34,7 +34,6 @@
 struct kvm_para_state {
         u8 mmu_queue[MMU_QUEUE_SIZE];
         int mmu_queue_len;
-        enum paravirt_lazy_mode mode;
 };
 
 static DEFINE_PER_CPU(struct kvm_para_state, para_state);
@@ -77,7 +76,7 @@ static void kvm_deferred_mmu_op(void *buffer, int len)
 {
         struct kvm_para_state *state = kvm_para_state();
 
-        if (state->mode != PARAVIRT_LAZY_MMU) {
+        if (paravirt_get_lazy_mode() != PARAVIRT_LAZY_MMU) {
                 kvm_mmu_op(buffer, len);
                 return;
         }
@@ -185,10 +184,7 @@ static void kvm_release_pt(unsigned long pfn)
 
 static void kvm_enter_lazy_mmu(void)
 {
-        struct kvm_para_state *state = kvm_para_state();
-
         paravirt_enter_lazy_mmu();
-        state->mode = paravirt_get_lazy_mode();
 }
 
 static void kvm_leave_lazy_mmu(void)
@@ -197,7 +193,6 @@ static void kvm_leave_lazy_mmu(void)
 
         mmu_queue_flush(state);
         paravirt_leave_lazy_mmu();
-        state->mode = paravirt_get_lazy_mode();
 }
 
 static void __init paravirt_ops_setup(void)
diff --git a/arch/x86/kernel/kvmclock.c b/arch/x86/kernel/kvmclock.c
index 223af43f1526..feaeb0d3aa4f 100644
--- a/arch/x86/kernel/kvmclock.c
+++ b/arch/x86/kernel/kvmclock.c
@@ -22,6 +22,8 @@
 #include <asm/msr.h>
 #include <asm/apic.h>
 #include <linux/percpu.h>
+
+#include <asm/x86_init.h>
 #include <asm/reboot.h>
 
 #define KVM_SCALE 22
@@ -50,8 +52,8 @@ static unsigned long kvm_get_wallclock(void)
         struct timespec ts;
         int low, high;
 
-        low = (int)__pa(&wall_clock);
-        high = ((u64)__pa(&wall_clock) >> 32);
+        low = (int)__pa_symbol(&wall_clock);
+        high = ((u64)__pa_symbol(&wall_clock) >> 32);
         native_write_msr(MSR_KVM_WALL_CLOCK, low, high);
 
         vcpu_time = &get_cpu_var(hv_clock);
@@ -182,12 +184,13 @@ void __init kvmclock_init(void)
         if (kvmclock && kvm_para_has_feature(KVM_FEATURE_CLOCKSOURCE)) {
                 if (kvm_register_clock("boot clock"))
                         return;
-                pv_time_ops.get_wallclock = kvm_get_wallclock;
-                pv_time_ops.set_wallclock = kvm_set_wallclock;
                 pv_time_ops.sched_clock = kvm_clock_read;
-                pv_time_ops.get_tsc_khz = kvm_get_tsc_khz;
+                x86_platform.calibrate_tsc = kvm_get_tsc_khz;
+                x86_platform.get_wallclock = kvm_get_wallclock;
+                x86_platform.set_wallclock = kvm_set_wallclock;
 #ifdef CONFIG_X86_LOCAL_APIC
-                pv_apic_ops.setup_secondary_clock = kvm_setup_secondary_clock;
+                x86_cpuinit.setup_percpu_clockev =
+                        kvm_setup_secondary_clock;
 #endif
 #ifdef CONFIG_SMP
                 smp_ops.smp_prepare_boot_cpu = kvm_smp_prepare_boot_cpu;
diff --git a/arch/x86/kernel/ldt.c b/arch/x86/kernel/ldt.c
index 71f1d99a635d..ec6ef60cbd17 100644
--- a/arch/x86/kernel/ldt.c
+++ b/arch/x86/kernel/ldt.c
@@ -67,8 +67,8 @@ static int alloc_ldt(mm_context_t *pc, int mincount, int reload)
 #ifdef CONFIG_SMP
                 preempt_disable();
                 load_LDT(pc);
-                if (!cpus_equal(current->mm->cpu_vm_mask,
-                                cpumask_of_cpu(smp_processor_id())))
+                if (!cpumask_equal(mm_cpumask(current->mm),
+                                   cpumask_of(smp_processor_id())))
                         smp_call_function(flush_ldt, current->mm, 1);
                 preempt_enable();
 #else
diff --git a/arch/x86/kernel/microcode_core.c b/arch/x86/kernel/microcode_core.c
index 9371448290ac..378e9a8f1bf8 100644
--- a/arch/x86/kernel/microcode_core.c
+++ b/arch/x86/kernel/microcode_core.c
@@ -210,8 +210,8 @@ static ssize_t microcode_write(struct file *file, const char __user *buf,
 {
         ssize_t ret = -EINVAL;
 
-        if ((len >> PAGE_SHIFT) > num_physpages) {
-                pr_err("microcode: too much data (max %ld pages)\n", num_physpages);
+        if ((len >> PAGE_SHIFT) > totalram_pages) {
+                pr_err("microcode: too much data (max %ld pages)\n", totalram_pages);
                 return ret;
         }
 
@@ -236,7 +236,7 @@ static const struct file_operations microcode_fops = {
 static struct miscdevice microcode_dev = {
         .minor                  = MICROCODE_MINOR,
         .name                   = "microcode",
-        .devnode                = "cpu/microcode",
+        .nodename               = "cpu/microcode",
         .fops                   = &microcode_fops,
 };
 
diff --git a/arch/x86/kernel/mpparse.c b/arch/x86/kernel/mpparse.c
index 651c93b28862..5be95ef4ffec 100644
--- a/arch/x86/kernel/mpparse.c
+++ b/arch/x86/kernel/mpparse.c
@@ -45,6 +45,11 @@ static int __init mpf_checksum(unsigned char *mp, int len)
         return sum & 0xFF;
 }
 
+int __init default_mpc_apic_id(struct mpc_cpu *m)
+{
+        return m->apicid;
+}
+
 static void __init MP_processor_info(struct mpc_cpu *m)
 {
         int apicid;
@@ -55,10 +60,7 @@ static void __init MP_processor_info(struct mpc_cpu *m)
                 return;
         }
 
-        if (x86_quirks->mpc_apic_id)
-                apicid = x86_quirks->mpc_apic_id(m);
-        else
-                apicid = m->apicid;
+        apicid = x86_init.mpparse.mpc_apic_id(m);
 
         if (m->cpuflag & CPU_BOOTPROCESSOR) {
                 bootup_cpu = " (Bootup-CPU)";
@@ -70,16 +72,18 @@ static void __init MP_processor_info(struct mpc_cpu *m)
 }
 
 #ifdef CONFIG_X86_IO_APIC
-static void __init MP_bus_info(struct mpc_bus *m)
+void __init default_mpc_oem_bus_info(struct mpc_bus *m, char *str)
 {
-        char str[7];
         memcpy(str, m->bustype, 6);
         str[6] = 0;
+        apic_printk(APIC_VERBOSE, "Bus #%d is %s\n", m->busid, str);
+}
 
-        if (x86_quirks->mpc_oem_bus_info)
-                x86_quirks->mpc_oem_bus_info(m, str);
-        else
-                apic_printk(APIC_VERBOSE, "Bus #%d is %s\n", m->busid, str);
+static void __init MP_bus_info(struct mpc_bus *m)
+{
+        char str[7];
+
+        x86_init.mpparse.mpc_oem_bus_info(m, str);
 
 #if MAX_MP_BUSSES < 256
         if (m->busid >= MAX_MP_BUSSES) {
@@ -96,8 +100,8 @@ static void __init MP_bus_info(struct mpc_bus *m)
                 mp_bus_id_to_type[m->busid] = MP_BUS_ISA;
 #endif
         } else if (strncmp(str, BUSTYPE_PCI, sizeof(BUSTYPE_PCI) - 1) == 0) {
-                if (x86_quirks->mpc_oem_pci_bus)
-                        x86_quirks->mpc_oem_pci_bus(m);
+                if (x86_init.mpparse.mpc_oem_pci_bus)
+                        x86_init.mpparse.mpc_oem_pci_bus(m);
 
                 clear_bit(m->busid, mp_bus_not_pci);
 #if defined(CONFIG_EISA) || defined(CONFIG_MCA)
@@ -291,6 +295,8 @@ static void __init smp_dump_mptable(struct mpc_table *mpc, unsigned char *mpt)
                         1, mpc, mpc->length, 1);
 }
 
+void __init default_smp_read_mpc_oem(struct mpc_table *mpc) { }
+
 static int __init smp_read_mpc(struct mpc_table *mpc, unsigned early)
 {
         char str[16];
@@ -312,16 +318,13 @@ static int __init smp_read_mpc(struct mpc_table *mpc, unsigned early)
         if (early)
                 return 1;
 
-        if (mpc->oemptr && x86_quirks->smp_read_mpc_oem) {
-                struct mpc_oemtable *oem_table = (void *)(long)mpc->oemptr;
-                x86_quirks->smp_read_mpc_oem(oem_table, mpc->oemsize);
-        }
+        if (mpc->oemptr)
+                x86_init.mpparse.smp_read_mpc_oem(mpc);
 
         /*
          *      Now process the configuration blocks.
          */
-        if (x86_quirks->mpc_record)
-                *x86_quirks->mpc_record = 0;
+        x86_init.mpparse.mpc_record(0);
 
         while (count < mpc->length) {
                 switch (*mpt) {
@@ -353,8 +356,7 @@ static int __init smp_read_mpc(struct mpc_table *mpc, unsigned early)
                         count = mpc->length;
                         break;
                 }
-                if (x86_quirks->mpc_record)
-                        (*x86_quirks->mpc_record)++;
+                x86_init.mpparse.mpc_record(1);
         }
 
 #ifdef CONFIG_X86_BIGSMP
@@ -482,11 +484,11 @@ static void __init construct_ioapic_table(int mpc_default_type)
                 MP_bus_info(&bus);
         }
 
-        ioapic.type = MP_IOAPIC;
-        ioapic.apicid = 2;
-        ioapic.apicver = mpc_default_type > 4 ? 0x10 : 0x01;
-        ioapic.flags = MPC_APIC_USABLE;
-        ioapic.apicaddr = 0xFEC00000;
+        ioapic.type     = MP_IOAPIC;
+        ioapic.apicid   = 2;
+        ioapic.apicver  = mpc_default_type > 4 ? 0x10 : 0x01;
+        ioapic.flags    = MPC_APIC_USABLE;
+        ioapic.apicaddr = IO_APIC_DEFAULT_PHYS_BASE;
         MP_ioapic_info(&ioapic);
 
         /*
@@ -608,7 +610,7 @@ static int __init check_physptr(struct mpf_intel *mpf, unsigned int early)
 /*
  * Scan the memory blocks for an SMP configuration block.
  */
-static void __init __get_smp_config(unsigned int early)
+void __init default_get_smp_config(unsigned int early)
 {
         struct mpf_intel *mpf = mpf_found;
 
@@ -625,11 +627,6 @@ static void __init __get_smp_config(unsigned int early)
         if (acpi_lapic && acpi_ioapic)
                 return;
 
-        if (x86_quirks->mach_get_smp_config) {
-                if (x86_quirks->mach_get_smp_config(early))
-                        return;
-        }
-
         printk(KERN_INFO "Intel MultiProcessor Specification v1.%d\n",
                mpf->specification);
 #if defined(CONFIG_X86_LOCAL_APIC) && defined(CONFIG_X86_32)
@@ -670,16 +667,6 @@ static void __init __get_smp_config(unsigned int early)
          */
 }
 
-void __init early_get_smp_config(void)
-{
-        __get_smp_config(1);
-}
-
-void __init get_smp_config(void)
-{
-        __get_smp_config(0);
-}
-
 static void __init smp_reserve_bootmem(struct mpf_intel *mpf)
 {
         unsigned long size = get_mpc_size(mpf->physptr);
@@ -745,14 +732,10 @@ static int __init smp_scan_config(unsigned long base, unsigned long length,
         return 0;
 }
 
-static void __init __find_smp_config(unsigned int reserve)
+void __init default_find_smp_config(unsigned int reserve)
 {
         unsigned int address;
 
-        if (x86_quirks->mach_find_smp_config) {
-                if (x86_quirks->mach_find_smp_config(reserve))
-                        return;
-        }
         /*
          * FIXME: Linux assumes you have 640K of base ram..
          * this continues the error...
@@ -787,16 +770,6 @@ static void __init __find_smp_config(unsigned int reserve)
                 smp_scan_config(address, 0x400, reserve);
 }
 
-void __init early_find_smp_config(void)
-{
-        __find_smp_config(0);
-}
-
-void __init find_smp_config(void)
-{
-        __find_smp_config(1);
-}
-
 #ifdef CONFIG_X86_IO_APIC
 static u8 __initdata irq_used[MAX_IRQ_SOURCES];
 
diff --git a/arch/x86/kernel/mrst.c b/arch/x86/kernel/mrst.c
new file mode 100644
index 000000000000..3b7078abc871
--- /dev/null
+++ b/arch/x86/kernel/mrst.c
@@ -0,0 +1,24 @@
+/*
+ * mrst.c: Intel Moorestown platform specific setup code
+ *
+ * (C) Copyright 2008 Intel Corporation
+ * Author: Jacob Pan (jacob.jun.pan@intel.com)
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; version 2
+ * of the License.
+ */
+#include <linux/init.h>
+
+#include <asm/setup.h>
+
+/*
+ * Moorestown specific x86_init function overrides and early setup
+ * calls.
+ */
+void __init x86_mrst_early_setup(void)
+{
+        x86_init.resources.probe_roms = x86_init_noop;
+        x86_init.resources.reserve_resources = x86_init_noop;
+}
diff --git a/arch/x86/kernel/msr.c b/arch/x86/kernel/msr.c
index 98fd6cd4e3a4..6a3cefc7dda1 100644
--- a/arch/x86/kernel/msr.c
+++ b/arch/x86/kernel/msr.c
@@ -1,6 +1,7 @@
 /* ----------------------------------------------------------------------- *
  *
  *   Copyright 2000-2008 H. Peter Anvin - All Rights Reserved
+ *   Copyright 2009 Intel Corporation; author: H. Peter Anvin
  *
  *   This program is free software; you can redistribute it and/or modify
  *   it under the terms of the GNU General Public License as published by
@@ -80,11 +81,8 @@ static ssize_t msr_read(struct file *file, char __user *buf,
 
         for (; count; count -= 8) {
                 err = rdmsr_safe_on_cpu(cpu, reg, &data[0], &data[1]);
-                if (err) {
-                        if (err == -EFAULT) /* Fix idiotic error code */
-                                err = -EIO;
+                if (err)
                         break;
-                }
                 if (copy_to_user(tmp, &data, 8)) {
                         err = -EFAULT;
                         break;
@@ -115,11 +113,8 @@ static ssize_t msr_write(struct file *file, const char __user *buf,
                         break;
                 }
                 err = wrmsr_safe_on_cpu(cpu, reg, data[0], data[1]);
-                if (err) {
-                        if (err == -EFAULT) /* Fix idiotic error code */
-                                err = -EIO;
+                if (err)
                         break;
-                }
                 tmp += 2;
                 bytes += 8;
         }
@@ -127,6 +122,54 @@ static ssize_t msr_write(struct file *file, const char __user *buf,
         return bytes ? bytes : err;
 }
 
+static long msr_ioctl(struct file *file, unsigned int ioc, unsigned long arg)
+{
+        u32 __user *uregs = (u32 __user *)arg;
+        u32 regs[8];
+        int cpu = iminor(file->f_path.dentry->d_inode);
+        int err;
+
+        switch (ioc) {
+        case X86_IOC_RDMSR_REGS:
+                if (!(file->f_mode & FMODE_READ)) {
+                        err = -EBADF;
+                        break;
+                }
+                if (copy_from_user(&regs, uregs, sizeof regs)) {
+                        err = -EFAULT;
+                        break;
+                }
+                err = rdmsr_safe_regs_on_cpu(cpu, regs);
+                if (err)
+                        break;
+                if (copy_to_user(uregs, &regs, sizeof regs))
+                        err = -EFAULT;
+                break;
+
+        case X86_IOC_WRMSR_REGS:
+                if (!(file->f_mode & FMODE_WRITE)) {
+                        err = -EBADF;
+                        break;
+                }
+                if (copy_from_user(&regs, uregs, sizeof regs)) {
+                        err = -EFAULT;
+                        break;
+                }
+                err = wrmsr_safe_regs_on_cpu(cpu, regs);
+                if (err)
+                        break;
+                if (copy_to_user(uregs, &regs, sizeof regs))
+                        err = -EFAULT;
+                break;
+
+        default:
+                err = -ENOTTY;
+                break;
+        }
+
+        return err;
+}
+
 static int msr_open(struct inode *inode, struct file *file)
 {
         unsigned int cpu = iminor(file->f_path.dentry->d_inode);
@@ -157,6 +200,8 @@ static const struct file_operations msr_fops = {
         .read = msr_read,
         .write = msr_write,
         .open = msr_open,
+        .unlocked_ioctl = msr_ioctl,
+        .compat_ioctl = msr_ioctl,
 };
 
 static int __cpuinit msr_device_create(int cpu)
@@ -196,7 +241,7 @@ static struct notifier_block __refdata msr_class_cpu_notifier = {
         .notifier_call = msr_class_cpu_callback,
 };
 
-static char *msr_nodename(struct device *dev)
+static char *msr_devnode(struct device *dev, mode_t *mode)
 {
         return kasprintf(GFP_KERNEL, "cpu/%u/msr", MINOR(dev->devt));
 }
@@ -217,7 +262,7 @@ static int __init msr_init(void)
                 err = PTR_ERR(msr_class);
                 goto out_chrdev;
         }
-        msr_class->nodename = msr_nodename;
+        msr_class->devnode = msr_devnode;
         for_each_online_cpu(i) {
                 err = msr_device_create(i);
                 if (err != 0)
diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c
index 70ec9b951d76..1b1739d16310 100644
--- a/arch/x86/kernel/paravirt.c
+++ b/arch/x86/kernel/paravirt.c
@@ -54,17 +54,12 @@ u64 _paravirt_ident_64(u64 x)
         return x;
 }
 
-static void __init default_banner(void)
+void __init default_banner(void)
 {
         printk(KERN_INFO "Booting paravirtualized kernel on %s\n",
                pv_info.name);
 }
 
-char *memory_setup(void)
-{
-        return pv_init_ops.memory_setup();
-}
-
 /* Simple instruction patching code. */
 #define DEF_NATIVE(ops, name, code)                                     \
         extern const char start_##ops##_##name[], end_##ops##_##name[]; \
@@ -188,11 +183,6 @@ unsigned paravirt_patch_insns(void *insnbuf, unsigned len,
         return insn_len;
 }
 
-void init_IRQ(void)
-{
-        pv_irq_ops.init_IRQ();
-}
-
 static void native_flush_tlb(void)
 {
         __native_flush_tlb();
@@ -218,13 +208,6 @@ extern void native_irq_enable_sysexit(void);
 extern void native_usergs_sysret32(void);
 extern void native_usergs_sysret64(void);
 
-static int __init print_banner(void)
-{
-        pv_init_ops.banner();
-        return 0;
-}
-core_initcall(print_banner);
-
 static struct resource reserve_ioports = {
         .start = 0,
         .end = IO_SPACE_LIMIT,
@@ -320,21 +303,13 @@ struct pv_info pv_info = {
 
 struct pv_init_ops pv_init_ops = {
         .patch = native_patch,
-        .banner = default_banner,
-        .arch_setup = paravirt_nop,
-        .memory_setup = machine_specific_memory_setup,
 };
 
 struct pv_time_ops pv_time_ops = {
-        .time_init = hpet_time_init,
-        .get_wallclock = native_get_wallclock,
-        .set_wallclock = native_set_wallclock,
         .sched_clock = native_sched_clock,
-        .get_tsc_khz = native_calibrate_tsc,
 };
 
 struct pv_irq_ops pv_irq_ops = {
-        .init_IRQ = native_init_IRQ,
         .save_fl = __PV_IS_CALLEE_SAVE(native_save_fl),
         .restore_fl = __PV_IS_CALLEE_SAVE(native_restore_fl),
         .irq_disable = __PV_IS_CALLEE_SAVE(native_irq_disable),
@@ -362,8 +337,9 @@ struct pv_cpu_ops pv_cpu_ops = {
 #endif
         .wbinvd = native_wbinvd,
         .read_msr = native_read_msr_safe,
-        .read_msr_amd = native_read_msr_amd_safe,
+        .rdmsr_regs = native_rdmsr_safe_regs,
         .write_msr = native_write_msr_safe,
+        .wrmsr_regs = native_wrmsr_safe_regs,
         .read_tsc = native_read_tsc,
         .read_pmc = native_read_pmc,
         .read_tscp = native_read_tscp,
@@ -408,8 +384,6 @@ struct pv_cpu_ops pv_cpu_ops = {
 
 struct pv_apic_ops pv_apic_ops = {
 #ifdef CONFIG_X86_LOCAL_APIC
-        .setup_boot_clock = setup_boot_APIC_clock,
-        .setup_secondary_clock = setup_secondary_APIC_clock,
         .startup_ipi_hook = paravirt_nop,
 #endif
 };
@@ -423,13 +397,6 @@ struct pv_apic_ops pv_apic_ops = {
 #endif
 
 struct pv_mmu_ops pv_mmu_ops = {
-#ifndef CONFIG_X86_64
-        .pagetable_setup_start = native_pagetable_setup_start,
-        .pagetable_setup_done = native_pagetable_setup_done,
-#else
-        .pagetable_setup_start = paravirt_nop,
-        .pagetable_setup_done = paravirt_nop,
-#endif
 
         .read_cr2 = native_read_cr2,
         .write_cr2 = native_write_cr2,
diff --git a/arch/x86/kernel/pci-dma.c b/arch/x86/kernel/pci-dma.c
index 1a041bcf506b..64b838eac18c 100644
--- a/arch/x86/kernel/pci-dma.c
+++ b/arch/x86/kernel/pci-dma.c
@@ -3,6 +3,7 @@
 #include <linux/dmar.h>
 #include <linux/bootmem.h>
 #include <linux/pci.h>
+#include <linux/kmemleak.h>
 
 #include <asm/proto.h>
 #include <asm/dma.h>
@@ -32,7 +33,14 @@ int no_iommu __read_mostly;
 /* Set this to 1 if there is a HW IOMMU in the system */
 int iommu_detected __read_mostly = 0;
 
-int iommu_pass_through;
+/*
+ * This variable becomes 1 if iommu=pt is passed on the kernel command line.
+ * If this variable is 1, IOMMU implementations do no DMA ranslation for
+ * devices and allow every device to access to whole physical memory. This is
+ * useful if a user want to use an IOMMU only for KVM device assignment to
+ * guests and not for driver dma translation.
+ */
+int iommu_pass_through __read_mostly;
 
 dma_addr_t bad_dma_address __read_mostly = 0;
 EXPORT_SYMBOL(bad_dma_address);
@@ -88,6 +96,11 @@ void __init dma32_reserve_bootmem(void)
         size = roundup(dma32_bootmem_size, align);
         dma32_bootmem_ptr = __alloc_bootmem_nopanic(size, align,
                                  512ULL<<20);
+        /*
+         * Kmemleak should not scan this block as it may not be mapped via the
+         * kernel direct mapping.
+         */
+        kmemleak_ignore(dma32_bootmem_ptr);
         if (dma32_bootmem_ptr)
                 dma32_bootmem_size = size;
         else
@@ -147,7 +160,7 @@ again:
                 return NULL;
 
         addr = page_to_phys(page);
-        if (!is_buffer_dma_capable(dma_mask, addr, size)) {
+        if (addr + size > dma_mask) {
                 __free_pages(page, get_order(size));
 
                 if (dma_mask < DMA_BIT_MASK(32) && !(flag & GFP_DMA)) {
@@ -212,10 +225,8 @@ static __init int iommu_setup(char *p)
                 if (!strncmp(p, "soft", 4))
                         swiotlb = 1;
 #endif
-                if (!strncmp(p, "pt", 2)) {
+                if (!strncmp(p, "pt", 2))
                         iommu_pass_through = 1;
-                        return 1;
-                }
 
                 gart_parse_options(p);
 
diff --git a/arch/x86/kernel/pci-gart_64.c b/arch/x86/kernel/pci-gart_64.c
index d2e56b8f48e7..98a827ee9ed7 100644
--- a/arch/x86/kernel/pci-gart_64.c
+++ b/arch/x86/kernel/pci-gart_64.c
@@ -190,14 +190,13 @@ static void iommu_full(struct device *dev, size_t size, int dir)
 static inline int
 need_iommu(struct device *dev, unsigned long addr, size_t size)
 {
-        return force_iommu ||
-                !is_buffer_dma_capable(*dev->dma_mask, addr, size);
+        return force_iommu || !dma_capable(dev, addr, size);
 }
 
 static inline int
 nonforced_iommu(struct device *dev, unsigned long addr, size_t size)
 {
-        return !is_buffer_dma_capable(*dev->dma_mask, addr, size);
+        return !dma_capable(dev, addr, size);
 }
 
 /* Map a single continuous physical area into the IOMMU.
diff --git a/arch/x86/kernel/pci-nommu.c b/arch/x86/kernel/pci-nommu.c
index 71d412a09f30..a3933d4330cd 100644
--- a/arch/x86/kernel/pci-nommu.c
+++ b/arch/x86/kernel/pci-nommu.c
@@ -14,7 +14,7 @@
 static int
 check_addr(char *name, struct device *hwdev, dma_addr_t bus, size_t size)
 {
-        if (hwdev && !is_buffer_dma_capable(*hwdev->dma_mask, bus, size)) {
+        if (hwdev && !dma_capable(hwdev, bus, size)) {
                 if (*hwdev->dma_mask >= DMA_BIT_MASK(32))
                         printk(KERN_ERR
                             "nommu_%s: overflow %Lx+%zu of device mask %Lx\n",
@@ -79,12 +79,29 @@ static void nommu_free_coherent(struct device *dev, size_t size, void *vaddr,
         free_pages((unsigned long)vaddr, get_order(size));
 }
 
+static void nommu_sync_single_for_device(struct device *dev,
+                        dma_addr_t addr, size_t size,
+                        enum dma_data_direction dir)
+{
+        flush_write_buffers();
+}
+
+
+static void nommu_sync_sg_for_device(struct device *dev,
+                        struct scatterlist *sg, int nelems,
+                        enum dma_data_direction dir)
+{
+        flush_write_buffers();
+}
+
 struct dma_map_ops nommu_dma_ops = {
-        .alloc_coherent = dma_generic_alloc_coherent,
-        .free_coherent  = nommu_free_coherent,
-        .map_sg         = nommu_map_sg,
-        .map_page       = nommu_map_page,
-        .is_phys        = 1,
+        .alloc_coherent         = dma_generic_alloc_coherent,
+        .free_coherent          = nommu_free_coherent,
+        .map_sg                 = nommu_map_sg,
+        .map_page               = nommu_map_page,
+        .sync_single_for_device = nommu_sync_single_for_device,
+        .sync_sg_for_device     = nommu_sync_sg_for_device,
+        .is_phys                = 1,
 };
 
 void __init no_iommu_init(void)
diff --git a/arch/x86/kernel/pci-swiotlb.c b/arch/x86/kernel/pci-swiotlb.c
index 6af96ee44200..aaa6b7839f1e 100644
--- a/arch/x86/kernel/pci-swiotlb.c
+++ b/arch/x86/kernel/pci-swiotlb.c
@@ -13,31 +13,6 @@
 
 int swiotlb __read_mostly;
 
-void * __init swiotlb_alloc_boot(size_t size, unsigned long nslabs)
-{
-        return alloc_bootmem_low_pages(size);
-}
-
-void *swiotlb_alloc(unsigned order, unsigned long nslabs)
-{
-        return (void *)__get_free_pages(GFP_DMA | __GFP_NOWARN, order);
-}
-
-dma_addr_t swiotlb_phys_to_bus(struct device *hwdev, phys_addr_t paddr)
-{
-        return paddr;
-}
-
-phys_addr_t swiotlb_bus_to_phys(struct device *hwdev, dma_addr_t baddr)
-{
-        return baddr;
-}
-
-int __weak swiotlb_arch_range_needs_mapping(phys_addr_t paddr, size_t size)
-{
-        return 0;
-}
-
 static void *x86_swiotlb_alloc_coherent(struct device *hwdev, size_t size,
                                         dma_addr_t *dma_handle, gfp_t flags)
 {
@@ -71,9 +46,8 @@ void __init pci_swiotlb_init(void)
 {
         /* don't initialize swiotlb if iommu=off (no_iommu=1) */
 #ifdef CONFIG_X86_64
-        if ((!iommu_detected && !no_iommu && max_pfn > MAX_DMA32_PFN) ||
-                iommu_pass_through)
-               swiotlb = 1;
+        if ((!iommu_detected && !no_iommu && max_pfn > MAX_DMA32_PFN))
+                swiotlb = 1;
 #endif
         if (swiotlb_force)
                 swiotlb = 1;
diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c
index 071166a4ba83..5284cd2b5776 100644
--- a/arch/x86/kernel/process.c
+++ b/arch/x86/kernel/process.c
@@ -9,7 +9,7 @@
 #include <linux/pm.h>
 #include <linux/clockchips.h>
 #include <linux/random.h>
-#include <trace/power.h>
+#include <trace/events/power.h>
 #include <asm/system.h>
 #include <asm/apic.h>
 #include <asm/syscalls.h>
@@ -25,9 +25,6 @@ EXPORT_SYMBOL(idle_nomwait);
 
 struct kmem_cache *task_xstate_cachep;
 
-DEFINE_TRACE(power_start);
-DEFINE_TRACE(power_end);
-
 int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src)
 {
         *dst = *src;
@@ -299,9 +296,7 @@ static inline int hlt_use_halt(void)
 void default_idle(void)
 {
         if (hlt_use_halt()) {
-                struct power_trace it;
-
-                trace_power_start(&it, POWER_CSTATE, 1);
+                trace_power_start(POWER_CSTATE, 1);
                 current_thread_info()->status &= ~TS_POLLING;
                 /*
                  * TS_POLLING-cleared state must be visible before we
@@ -314,7 +309,6 @@ void default_idle(void)
                 else
                         local_irq_enable();
                 current_thread_info()->status |= TS_POLLING;
-                trace_power_end(&it);
         } else {
                 local_irq_enable();
                 /* loop is done by the caller */
@@ -372,9 +366,7 @@ EXPORT_SYMBOL_GPL(cpu_idle_wait);
  */
 void mwait_idle_with_hints(unsigned long ax, unsigned long cx)
 {
-        struct power_trace it;
-
-        trace_power_start(&it, POWER_CSTATE, (ax>>4)+1);
+        trace_power_start(POWER_CSTATE, (ax>>4)+1);
         if (!need_resched()) {
                 if (cpu_has(&current_cpu_data, X86_FEATURE_CLFLUSH_MONITOR))
                         clflush((void *)&current_thread_info()->flags);
@@ -384,15 +376,13 @@ void mwait_idle_with_hints(unsigned long ax, unsigned long cx)
                 if (!need_resched())
                         __mwait(ax, cx);
         }
-        trace_power_end(&it);
 }
 
 /* Default MONITOR/MWAIT with no hints, used for default C1 state */
 static void mwait_idle(void)
 {
-        struct power_trace it;
         if (!need_resched()) {
-                trace_power_start(&it, POWER_CSTATE, 1);
+                trace_power_start(POWER_CSTATE, 1);
                 if (cpu_has(&current_cpu_data, X86_FEATURE_CLFLUSH_MONITOR))
                         clflush((void *)&current_thread_info()->flags);
 
@@ -402,7 +392,6 @@ static void mwait_idle(void)
                         __sti_mwait(0, 0);
                 else
                         local_irq_enable();
-                trace_power_end(&it);
         } else
                 local_irq_enable();
 }
@@ -414,13 +403,11 @@ static void mwait_idle(void)
  */
 static void poll_idle(void)
 {
-        struct power_trace it;
-
-        trace_power_start(&it, POWER_CSTATE, 0);
+        trace_power_start(POWER_CSTATE, 0);
         local_irq_enable();
         while (!need_resched())
                 cpu_relax();
-        trace_power_end(&it);
+        trace_power_end(0);
 }
 
 /*
@@ -568,10 +555,8 @@ void __cpuinit select_idle_routine(const struct cpuinfo_x86 *c)
 void __init init_c1e_mask(void)
 {
         /* If we're using c1e_idle, we need to allocate c1e_mask. */
-        if (pm_idle == c1e_idle) {
-                alloc_cpumask_var(&c1e_mask, GFP_KERNEL);
-                cpumask_clear(c1e_mask);
-        }
+        if (pm_idle == c1e_idle)
+                zalloc_cpumask_var(&c1e_mask, GFP_KERNEL);
 }
 
 static int __init idle_setup(char *str)
diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c
index 59f4524984af..4cf79567cdab 100644
--- a/arch/x86/kernel/process_32.c
+++ b/arch/x86/kernel/process_32.c
@@ -61,9 +61,6 @@
 
 asmlinkage void ret_from_fork(void) __asm__("ret_from_fork");
 
-DEFINE_PER_CPU(struct task_struct *, current_task) = &init_task;
-EXPORT_PER_CPU_SYMBOL(current_task);
-
 /*
  * Return saved PC of a blocked thread.
  */
@@ -350,14 +347,21 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
                                  *next = &next_p->thread;
         int cpu = smp_processor_id();
         struct tss_struct *tss = &per_cpu(init_tss, cpu);
+        bool preload_fpu;
 
         /* never put a printk in __switch_to... printk() calls wake_up*() indirectly */
 
-        __unlazy_fpu(prev_p);
+        /*
+         * If the task has used fpu the last 5 timeslices, just do a full
+         * restore of the math state immediately to avoid the trap; the
+         * chances of needing FPU soon are obviously high now
+         */
+        preload_fpu = tsk_used_math(next_p) && next_p->fpu_counter > 5;
 
+        __unlazy_fpu(prev_p);
 
         /* we're going to use this soon, after a few expensive things */
-        if (next_p->fpu_counter > 5)
+        if (preload_fpu)
                 prefetch(next->xstate);
 
         /*
@@ -398,6 +402,11 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
                      task_thread_info(next_p)->flags & _TIF_WORK_CTXSW_NEXT))
                 __switch_to_xtra(prev_p, next_p, tss);
 
+        /* If we're going to preload the fpu context, make sure clts
+           is run while we're batching the cpu state updates. */
+        if (preload_fpu)
+                clts();
+
         /*
          * Leave lazy mode, flushing any hypercalls made here.
          * This must be done before restoring TLS segments so
@@ -407,15 +416,8 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
          */
         arch_end_context_switch(next_p);
 
-        /* If the task has used fpu the last 5 timeslices, just do a full
-         * restore of the math state immediately to avoid the trap; the
-         * chances of needing FPU soon are obviously high now
-         *
-         * tsk_used_math() checks prevent calling math_state_restore(),
-         * which can sleep in the case of !tsk_used_math()
-         */
-        if (tsk_used_math(next_p) && next_p->fpu_counter > 5)
-                math_state_restore();
+        if (preload_fpu)
+                __math_state_restore();
 
         /*
          * Restore %gs if needed (which is common)
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index ebefb5407b9d..ad535b683170 100644
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -55,9 +55,6 @@
 
 asmlinkage extern void ret_from_fork(void);
 
-DEFINE_PER_CPU(struct task_struct *, current_task) = &init_task;
-EXPORT_PER_CPU_SYMBOL(current_task);
-
 DEFINE_PER_CPU(unsigned long, old_rsp);
 static DEFINE_PER_CPU(unsigned char, is_idle);
 
@@ -386,9 +383,17 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
         int cpu = smp_processor_id();
         struct tss_struct *tss = &per_cpu(init_tss, cpu);
         unsigned fsindex, gsindex;
+        bool preload_fpu;
+
+        /*
+         * If the task has used fpu the last 5 timeslices, just do a full
+         * restore of the math state immediately to avoid the trap; the
+         * chances of needing FPU soon are obviously high now
+         */
+        preload_fpu = tsk_used_math(next_p) && next_p->fpu_counter > 5;
 
         /* we're going to use this soon, after a few expensive things */
-        if (next_p->fpu_counter > 5)
+        if (preload_fpu)
                 prefetch(next->xstate);
 
         /*
@@ -419,6 +424,13 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
 
         load_TLS(next, cpu);
 
+        /* Must be after DS reload */
+        unlazy_fpu(prev_p);
+
+        /* Make sure cpu is ready for new context */
+        if (preload_fpu)
+                clts();
+
         /*
          * Leave lazy mode, flushing any hypercalls made here.
          * This must be done before restoring TLS segments so
@@ -459,9 +471,6 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
                 wrmsrl(MSR_KERNEL_GS_BASE, next->gs);
         prev->gsindex = gsindex;
 
-        /* Must be after DS reload */
-        unlazy_fpu(prev_p);
-
         /*
          * Switch the PDA and FPU contexts.
          */
@@ -480,15 +489,12 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
                      task_thread_info(prev_p)->flags & _TIF_WORK_CTXSW_PREV))
                 __switch_to_xtra(prev_p, next_p, tss);
 
-        /* If the task has used fpu the last 5 timeslices, just do a full
-         * restore of the math state immediately to avoid the trap; the
-         * chances of needing FPU soon are obviously high now
-         *
-         * tsk_used_math() checks prevent calling math_state_restore(),
-         * which can sleep in the case of !tsk_used_math()
+        /*
+         * Preload the FPU context, now that we've determined that the
+         * task is likely to be using it. 
          */
-        if (tsk_used_math(next_p) && next_p->fpu_counter > 5)
-                math_state_restore();
+        if (preload_fpu)
+                __math_state_restore();
         return prev_p;
 }
 
diff --git a/arch/x86/kernel/ptrace.c b/arch/x86/kernel/ptrace.c
index 09ecbde91c13..7b058a2dc66a 100644
--- a/arch/x86/kernel/ptrace.c
+++ b/arch/x86/kernel/ptrace.c
@@ -35,10 +35,11 @@
 #include <asm/proto.h>
 #include <asm/ds.h>
 
-#include <trace/syscall.h>
-
 #include "tls.h"
 
+#define CREATE_TRACE_POINTS
+#include <trace/events/syscalls.h>
+
 enum x86_regset {
         REGSET_GENERAL,
         REGSET_FP,
@@ -324,16 +325,6 @@ static int putreg(struct task_struct *child,
                 return set_flags(child, value);
 
 #ifdef CONFIG_X86_64
-        /*
-         * Orig_ax is really just a flag with small positive and
-         * negative values, so make sure to always sign-extend it
-         * from 32 bits so that it works correctly regardless of
-         * whether we come from a 32-bit environment or not.
-         */
-        case offsetof(struct user_regs_struct, orig_ax):
-                value = (long) (s32) value;
-                break;
-
         case offsetof(struct user_regs_struct,fs_base):
                 if (value >= TASK_SIZE_OF(child))
                         return -EIO;
@@ -1125,10 +1116,15 @@ static int putreg32(struct task_struct *child, unsigned regno, u32 value)
 
         case offsetof(struct user32, regs.orig_eax):
                 /*
-                 * Sign-extend the value so that orig_eax = -1
-                 * causes (long)orig_ax < 0 tests to fire correctly.
+                 * A 32-bit debugger setting orig_eax means to restore
+                 * the state of the task restarting a 32-bit syscall.
+                 * Make sure we interpret the -ERESTART* codes correctly
+                 * in case the task is not actually still sitting at the
+                 * exit from a 32-bit syscall with TS_COMPAT still set.
                  */
-                regs->orig_ax = (long) (s32) value;
+                regs->orig_ax = value;
+                if (syscall_get_nr(child, regs) >= 0)
+                        task_thread_info(child)->status |= TS_COMPAT;
                 break;
 
         case offsetof(struct user32, regs.eflags):
@@ -1497,8 +1493,8 @@ asmregparm long syscall_trace_enter(struct pt_regs *regs)
             tracehook_report_syscall_entry(regs))
                 ret = -1L;
 
-        if (unlikely(test_thread_flag(TIF_SYSCALL_FTRACE)))
-                ftrace_syscall_enter(regs);
+        if (unlikely(test_thread_flag(TIF_SYSCALL_TRACEPOINT)))
+                trace_sys_enter(regs, regs->orig_ax);
 
         if (unlikely(current->audit_context)) {
                 if (IS_IA32)
@@ -1523,8 +1519,8 @@ asmregparm void syscall_trace_leave(struct pt_regs *regs)
         if (unlikely(current->audit_context))
                 audit_syscall_exit(AUDITSC_RESULT(regs->ax), regs->ax);
 
-        if (unlikely(test_thread_flag(TIF_SYSCALL_FTRACE)))
-                ftrace_syscall_exit(regs);
+        if (unlikely(test_thread_flag(TIF_SYSCALL_TRACEPOINT)))
+                trace_sys_exit(regs, regs->ax);
 
         if (test_thread_flag(TIF_SYSCALL_TRACE))
                 tracehook_report_syscall_exit(regs, 0);
diff --git a/arch/x86/kernel/quirks.c b/arch/x86/kernel/quirks.c
index af71d06624bf..6c3b2c6fd772 100644
--- a/arch/x86/kernel/quirks.c
+++ b/arch/x86/kernel/quirks.c
@@ -508,7 +508,7 @@ static void __init quirk_amd_nb_node(struct pci_dev *dev)
 
         pci_read_config_dword(nb_ht, 0x60, &val);
         set_dev_node(&dev->dev, val & 7);
-        pci_dev_put(dev);
+        pci_dev_put(nb_ht);
 }
 
 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_K8_NB,
diff --git a/arch/x86/kernel/reboot.c b/arch/x86/kernel/reboot.c
index a06e8d101844..27349f92a6d7 100644
--- a/arch/x86/kernel/reboot.c
+++ b/arch/x86/kernel/reboot.c
@@ -4,6 +4,7 @@
 #include <linux/pm.h>
 #include <linux/efi.h>
 #include <linux/dmi.h>
+#include <linux/tboot.h>
 #include <acpi/reboot.h>
 #include <asm/io.h>
 #include <asm/apic.h>
@@ -508,6 +509,8 @@ static void native_machine_emergency_restart(void)
         if (reboot_emergency)
                 emergency_vmx_disable_all();
 
+        tboot_shutdown(TB_SHUTDOWN_REBOOT);
+
         /* Tell the BIOS if we want cold or warm reboot */
         *((unsigned short *)__va(0x472)) = reboot_mode;
 
@@ -634,6 +637,8 @@ static void native_machine_halt(void)
         /* stop other cpus and apics */
         machine_shutdown();
 
+        tboot_shutdown(TB_SHUTDOWN_HALT);
+
         /* stop this cpu */
         stop_this_cpu(NULL);
 }
@@ -645,6 +650,8 @@ static void native_machine_power_off(void)
                         machine_shutdown();
                 pm_power_off();
         }
+        /* a fallback in case there is no PM info available */
+        tboot_shutdown(TB_SHUTDOWN_HALT);
 }
 
 struct machine_ops machine_ops = {
diff --git a/arch/x86/kernel/rtc.c b/arch/x86/kernel/rtc.c
index 5d465b207e72..1cfbbfc3ae26 100644
--- a/arch/x86/kernel/rtc.c
+++ b/arch/x86/kernel/rtc.c
@@ -8,6 +8,7 @@
 #include <linux/pnp.h>
 
 #include <asm/vsyscall.h>
+#include <asm/x86_init.h>
 #include <asm/time.h>
 
 #ifdef CONFIG_X86_32
@@ -165,33 +166,29 @@ void rtc_cmos_write(unsigned char val, unsigned char addr)
 }
 EXPORT_SYMBOL(rtc_cmos_write);
 
-static int set_rtc_mmss(unsigned long nowtime)
+int update_persistent_clock(struct timespec now)
 {
         unsigned long flags;
         int retval;
 
         spin_lock_irqsave(&rtc_lock, flags);
-        retval = set_wallclock(nowtime);
+        retval = x86_platform.set_wallclock(now.tv_sec);
         spin_unlock_irqrestore(&rtc_lock, flags);
 
         return retval;
 }
 
 /* not static: needed by APM */
-unsigned long read_persistent_clock(void)
+void read_persistent_clock(struct timespec *ts)
 {
         unsigned long retval, flags;
 
         spin_lock_irqsave(&rtc_lock, flags);
-        retval = get_wallclock();
+        retval = x86_platform.get_wallclock();
         spin_unlock_irqrestore(&rtc_lock, flags);
 
-        return retval;
-}
-
-int update_persistent_clock(struct timespec now)
-{
-        return set_rtc_mmss(now.tv_sec);
+        ts->tv_sec = retval;
+        ts->tv_nsec = 0;
 }
 
 unsigned long long native_read_tsc(void)
diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c
index 63f32d220ef2..e09f0e2c14b5 100644
--- a/arch/x86/kernel/setup.c
+++ b/arch/x86/kernel/setup.c
@@ -27,6 +27,7 @@
 #include <linux/screen_info.h>
 #include <linux/ioport.h>
 #include <linux/acpi.h>
+#include <linux/sfi.h>
 #include <linux/apm_bios.h>
 #include <linux/initrd.h>
 #include <linux/bootmem.h>
@@ -66,6 +67,7 @@
 
 #include <linux/percpu.h>
 #include <linux/crash_dump.h>
+#include <linux/tboot.h>
 
 #include <video/edid.h>
 
@@ -108,10 +110,6 @@
 #include <asm/numa_64.h>
 #endif
 
-#ifndef ARCH_SETUP
-#define ARCH_SETUP
-#endif
-
 /*
  * end_pfn only includes RAM, while max_pfn_mapped includes all e820 entries.
  * The direct mapping extends to max_pfn_mapped, so that we can directly access
@@ -133,9 +131,9 @@ int default_cpu_present_to_apicid(int mps_cpu)
         return __default_cpu_present_to_apicid(mps_cpu);
 }
 
-int default_check_phys_apicid_present(int boot_cpu_physical_apicid)
+int default_check_phys_apicid_present(int phys_apicid)
 {
-        return __default_check_phys_apicid_present(boot_cpu_physical_apicid);
+        return __default_check_phys_apicid_present(phys_apicid);
 }
 #endif
 
@@ -171,13 +169,6 @@ static struct resource bss_resource = {
 
 
 #ifdef CONFIG_X86_32
-static struct resource video_ram_resource = {
-        .name   = "Video RAM area",
-        .start  = 0xa0000,
-        .end    = 0xbffff,
-        .flags  = IORESOURCE_BUSY | IORESOURCE_MEM
-};
-
 /* cpu data as detected by the assembly code in head.S */
 struct cpuinfo_x86 new_cpu_data __cpuinitdata = {0, 0, 0, 0, -1, 1, 0, 0, -1};
 /* common cpu data for all cpus */
@@ -605,7 +596,7 @@ static struct resource standard_io_resources[] = {
                 .flags = IORESOURCE_BUSY | IORESOURCE_IO }
 };
 
-static void __init reserve_standard_io_resources(void)
+void __init reserve_standard_io_resources(void)
 {
         int i;
 
@@ -637,10 +628,6 @@ static int __init setup_elfcorehdr(char *arg)
 early_param("elfcorehdr", setup_elfcorehdr);
 #endif
 
-static struct x86_quirks default_x86_quirks __initdata;
-
-struct x86_quirks *x86_quirks __initdata = &default_x86_quirks;
-
 #ifdef CONFIG_X86_RESERVE_LOW_64K
 static int __init dmi_low_memory_corruption(const struct dmi_system_id *d)
 {
@@ -757,7 +744,7 @@ void __init setup_arch(char **cmdline_p)
         }
 #endif
 
-        ARCH_SETUP
+        x86_init.oem.arch_setup();
 
         setup_memory_map();
         parse_setup_data();
@@ -796,6 +783,16 @@ void __init setup_arch(char **cmdline_p)
         strlcpy(command_line, boot_command_line, COMMAND_LINE_SIZE);
         *cmdline_p = command_line;
 
+#ifdef CONFIG_X86_64
+        /*
+         * Must call this twice: Once just to detect whether hardware doesn't
+         * support NX (so that the early EHCI debug console setup can safely
+         * call set_fixmap(), and then again after parsing early parameters to
+         * honor the respective command line option.
+         */
+        check_efer();
+#endif
+
         parse_early_param();
 
 #ifdef CONFIG_X86_64
@@ -833,11 +830,9 @@ void __init setup_arch(char **cmdline_p)
          * VMware detection requires dmi to be available, so this
          * needs to be done after dmi_scan_machine, for the BP.
          */
-        init_hypervisor(&boot_cpu_data);
+        init_hypervisor_platform();
 
-#ifdef CONFIG_X86_32
-        probe_roms();
-#endif
+        x86_init.resources.probe_roms();
 
         /* after parse_early_param, so could debug it */
         insert_resource(&iomem_resource, &code_resource);
@@ -972,10 +967,11 @@ void __init setup_arch(char **cmdline_p)
         kvmclock_init();
 #endif
 
-        paravirt_pagetable_setup_start(swapper_pg_dir);
+        x86_init.paging.pagetable_setup_start(swapper_pg_dir);
         paging_init();
-        paravirt_pagetable_setup_done(swapper_pg_dir);
-        paravirt_post_allocator_init();
+        x86_init.paging.pagetable_setup_done(swapper_pg_dir);
+
+        tboot_probe();
 
 #ifdef CONFIG_X86_64
         map_vsyscall();
@@ -990,13 +986,13 @@ void __init setup_arch(char **cmdline_p)
          */
         acpi_boot_init();
 
-#if defined(CONFIG_X86_MPPARSE) || defined(CONFIG_X86_VISWS)
+        sfi_init();
+
         /*
          * get boot-time SMP configuration:
          */
         if (smp_found_config)
                 get_smp_config();
-#endif
 
         prefill_possible_map();
 
@@ -1015,10 +1011,7 @@ void __init setup_arch(char **cmdline_p)
         e820_reserve_resources();
         e820_mark_nosave_regions(max_low_pfn);
 
-#ifdef CONFIG_X86_32
-        request_resource(&iomem_resource, &video_ram_resource);
-#endif
-        reserve_standard_io_resources();
+        x86_init.resources.reserve_resources();
 
         e820_setup_gap();
 
@@ -1030,78 +1023,22 @@ void __init setup_arch(char **cmdline_p)
         conswitchp = &dummy_con;
 #endif
 #endif
+        x86_init.oem.banner();
 }
 
 #ifdef CONFIG_X86_32
 
-/**
- * x86_quirk_intr_init - post gate setup interrupt initialisation
- *
- * Description:
- *      Fill in any interrupts that may have been left out by the general
- *      init_IRQ() routine.  interrupts having to do with the machine rather
- *      than the devices on the I/O bus (like APIC interrupts in intel MP
- *      systems) are started here.
- **/
-void __init x86_quirk_intr_init(void)
-{
-        if (x86_quirks->arch_intr_init) {
-                if (x86_quirks->arch_intr_init())
-                        return;
-        }
-}
-
-/**
- * x86_quirk_trap_init - initialise system specific traps
- *
- * Description:
- *      Called as the final act of trap_init().  Used in VISWS to initialise
- *      the various board specific APIC traps.
- **/
-void __init x86_quirk_trap_init(void)
-{
-        if (x86_quirks->arch_trap_init) {
-                if (x86_quirks->arch_trap_init())
-                        return;
-        }
-}
-
-static struct irqaction irq0  = {
-        .handler = timer_interrupt,
-        .flags = IRQF_DISABLED | IRQF_NOBALANCING | IRQF_IRQPOLL | IRQF_TIMER,
-        .name = "timer"
+static struct resource video_ram_resource = {
+        .name   = "Video RAM area",
+        .start  = 0xa0000,
+        .end    = 0xbffff,
+        .flags  = IORESOURCE_BUSY | IORESOURCE_MEM
 };
 
-/**
- * x86_quirk_pre_time_init - do any specific initialisations before.
- *
- **/
-void __init x86_quirk_pre_time_init(void)
+void __init i386_reserve_resources(void)
 {
-        if (x86_quirks->arch_pre_time_init)
-                x86_quirks->arch_pre_time_init();
+        request_resource(&iomem_resource, &video_ram_resource);
+        reserve_standard_io_resources();
 }
 
-/**
- * x86_quirk_time_init - do any specific initialisations for the system timer.
- *
- * Description:
- *      Must plug the system timer interrupt source at HZ into the IRQ listed
- *      in irq_vectors.h:TIMER_IRQ
- **/
-void __init x86_quirk_time_init(void)
-{
-        if (x86_quirks->arch_time_init) {
-                /*
-                 * A nonzero return code does not mean failure, it means
-                 * that the architecture quirk does not want any
-                 * generic (timer) setup to be performed after this:
-                 */
-                if (x86_quirks->arch_time_init())
-                        return;
-        }
-
-        irq0.mask = cpumask_of_cpu(0);
-        setup_irq(0, &irq0);
-}
 #endif /* CONFIG_X86_32 */
diff --git a/arch/x86/kernel/setup_percpu.c b/arch/x86/kernel/setup_percpu.c
index 07d81916f212..d559af913e1f 100644
--- a/arch/x86/kernel/setup_percpu.c
+++ b/arch/x86/kernel/setup_percpu.c
@@ -55,6 +55,7 @@ EXPORT_SYMBOL(__per_cpu_offset);
 #define PERCPU_FIRST_CHUNK_RESERVE      0
 #endif
 
+#ifdef CONFIG_X86_32
 /**
  * pcpu_need_numa - determine percpu allocation needs to consider NUMA
  *
@@ -83,6 +84,7 @@ static bool __init pcpu_need_numa(void)
 #endif
         return false;
 }
+#endif
 
 /**
  * pcpu_alloc_bootmem - NUMA friendly alloc_bootmem wrapper for percpu
@@ -124,308 +126,35 @@ static void * __init pcpu_alloc_bootmem(unsigned int cpu, unsigned long size,
 }
 
 /*
- * Large page remap allocator
- *
- * This allocator uses PMD page as unit.  A PMD page is allocated for
- * each cpu and each is remapped into vmalloc area using PMD mapping.
- * As PMD page is quite large, only part of it is used for the first
- * chunk.  Unused part is returned to the bootmem allocator.
- *
- * So, the PMD pages are mapped twice - once to the physical mapping
- * and to the vmalloc area for the first percpu chunk.  The double
- * mapping does add one more PMD TLB entry pressure but still is much
- * better than only using 4k mappings while still being NUMA friendly.
+ * Helpers for first chunk memory allocation
  */
-#ifdef CONFIG_NEED_MULTIPLE_NODES
-struct pcpul_ent {
-        unsigned int    cpu;
-        void            *ptr;
-};
-
-static size_t pcpul_size;
-static struct pcpul_ent *pcpul_map;
-static struct vm_struct pcpul_vm;
-
-static struct page * __init pcpul_get_page(unsigned int cpu, int pageno)
+static void * __init pcpu_fc_alloc(unsigned int cpu, size_t size, size_t align)
 {
-        size_t off = (size_t)pageno << PAGE_SHIFT;
-
-        if (off >= pcpul_size)
-                return NULL;
-
-        return virt_to_page(pcpul_map[cpu].ptr + off);
+        return pcpu_alloc_bootmem(cpu, size, align);
 }
 
-static ssize_t __init setup_pcpu_lpage(size_t static_size, bool chosen)
+static void __init pcpu_fc_free(void *ptr, size_t size)
 {
-        size_t map_size, dyn_size;
-        unsigned int cpu;
-        int i, j;
-        ssize_t ret;
-
-        if (!chosen) {
-                size_t vm_size = VMALLOC_END - VMALLOC_START;
-                size_t tot_size = nr_cpu_ids * PMD_SIZE;
-
-                /* on non-NUMA, embedding is better */
-                if (!pcpu_need_numa())
-                        return -EINVAL;
-
-                /* don't consume more than 20% of vmalloc area */
-                if (tot_size > vm_size / 5) {
-                        pr_info("PERCPU: too large chunk size %zuMB for "
-                                "large page remap\n", tot_size >> 20);
-                        return -EINVAL;
-                }
-        }
-
-        /* need PSE */
-        if (!cpu_has_pse) {
-                pr_warning("PERCPU: lpage allocator requires PSE\n");
-                return -EINVAL;
-        }
-
-        /*
-         * Currently supports only single page.  Supporting multiple
-         * pages won't be too difficult if it ever becomes necessary.
-         */
-        pcpul_size = PFN_ALIGN(static_size + PERCPU_MODULE_RESERVE +
-                               PERCPU_DYNAMIC_RESERVE);
-        if (pcpul_size > PMD_SIZE) {
-                pr_warning("PERCPU: static data is larger than large page, "
-                           "can't use large page\n");
-                return -EINVAL;
-        }
-        dyn_size = pcpul_size - static_size - PERCPU_FIRST_CHUNK_RESERVE;
-
-        /* allocate pointer array and alloc large pages */
-        map_size = PFN_ALIGN(nr_cpu_ids * sizeof(pcpul_map[0]));
-        pcpul_map = alloc_bootmem(map_size);
-
-        for_each_possible_cpu(cpu) {
-                pcpul_map[cpu].cpu = cpu;
-                pcpul_map[cpu].ptr = pcpu_alloc_bootmem(cpu, PMD_SIZE,
-                                                        PMD_SIZE);
-                if (!pcpul_map[cpu].ptr) {
-                        pr_warning("PERCPU: failed to allocate large page "
-                                   "for cpu%u\n", cpu);
-                        goto enomem;
-                }
-
-                /*
-                 * Only use pcpul_size bytes and give back the rest.
-                 *
-                 * Ingo: The 2MB up-rounding bootmem is needed to make
-                 * sure the partial 2MB page is still fully RAM - it's
-                 * not well-specified to have a PAT-incompatible area
-                 * (unmapped RAM, device memory, etc.) in that hole.
-                 */
-                free_bootmem(__pa(pcpul_map[cpu].ptr + pcpul_size),
-                             PMD_SIZE - pcpul_size);
-
-                memcpy(pcpul_map[cpu].ptr, __per_cpu_load, static_size);
-        }
-
-        /* allocate address and map */
-        pcpul_vm.flags = VM_ALLOC;
-        pcpul_vm.size = nr_cpu_ids * PMD_SIZE;
-        vm_area_register_early(&pcpul_vm, PMD_SIZE);
-
-        for_each_possible_cpu(cpu) {
-                pmd_t *pmd, pmd_v;
-
-                pmd = populate_extra_pmd((unsigned long)pcpul_vm.addr +
-                                         cpu * PMD_SIZE);
-                pmd_v = pfn_pmd(page_to_pfn(virt_to_page(pcpul_map[cpu].ptr)),
-                                PAGE_KERNEL_LARGE);
-                set_pmd(pmd, pmd_v);
-        }
-
-        /* we're ready, commit */
-        pr_info("PERCPU: Remapped at %p with large pages, static data "
-                "%zu bytes\n", pcpul_vm.addr, static_size);
-
-        ret = pcpu_setup_first_chunk(pcpul_get_page, static_size,
-                                     PERCPU_FIRST_CHUNK_RESERVE, dyn_size,
-                                     PMD_SIZE, pcpul_vm.addr, NULL);
-
-        /* sort pcpul_map array for pcpu_lpage_remapped() */
-        for (i = 0; i < nr_cpu_ids - 1; i++)
-                for (j = i + 1; j < nr_cpu_ids; j++)
-                        if (pcpul_map[i].ptr > pcpul_map[j].ptr) {
-                                struct pcpul_ent tmp = pcpul_map[i];
-                                pcpul_map[i] = pcpul_map[j];
-                                pcpul_map[j] = tmp;
-                        }
-
-        return ret;
-
-enomem:
-        for_each_possible_cpu(cpu)
-                if (pcpul_map[cpu].ptr)
-                        free_bootmem(__pa(pcpul_map[cpu].ptr), pcpul_size);
-        free_bootmem(__pa(pcpul_map), map_size);
-        return -ENOMEM;
+        free_bootmem(__pa(ptr), size);
 }
 
-/**
- * pcpu_lpage_remapped - determine whether a kaddr is in pcpul recycled area
- * @kaddr: the kernel address in question
- *
- * Determine whether @kaddr falls in the pcpul recycled area.  This is
- * used by pageattr to detect VM aliases and break up the pcpu PMD
- * mapping such that the same physical page is not mapped under
- * different attributes.
- *
- * The recycled area is always at the tail of a partially used PMD
- * page.
- *
- * RETURNS:
- * Address of corresponding remapped pcpu address if match is found;
- * otherwise, NULL.
- */
-void *pcpu_lpage_remapped(void *kaddr)
+static int __init pcpu_cpu_distance(unsigned int from, unsigned int to)
 {
-        void *pmd_addr = (void *)((unsigned long)kaddr & PMD_MASK);
-        unsigned long offset = (unsigned long)kaddr & ~PMD_MASK;
-        int left = 0, right = nr_cpu_ids - 1;
-        int pos;
-
-        /* pcpul in use at all? */
-        if (!pcpul_map)
-                return NULL;
-
-        /* okay, perform binary search */
-        while (left <= right) {
-                pos = (left + right) / 2;
-
-                if (pcpul_map[pos].ptr < pmd_addr)
-                        left = pos + 1;
-                else if (pcpul_map[pos].ptr > pmd_addr)
-                        right = pos - 1;
-                else {
-                        /* it shouldn't be in the area for the first chunk */
-                        WARN_ON(offset < pcpul_size);
-
-                        return pcpul_vm.addr +
-                                pcpul_map[pos].cpu * PMD_SIZE + offset;
-                }
-        }
-
-        return NULL;
-}
+#ifdef CONFIG_NEED_MULTIPLE_NODES
+        if (early_cpu_to_node(from) == early_cpu_to_node(to))
+                return LOCAL_DISTANCE;
+        else
+                return REMOTE_DISTANCE;
 #else
-static ssize_t __init setup_pcpu_lpage(size_t static_size, bool chosen)
-{
-        return -EINVAL;
-}
+        return LOCAL_DISTANCE;
 #endif
-
-/*
- * Embedding allocator
- *
- * The first chunk is sized to just contain the static area plus
- * module and dynamic reserves and embedded into linear physical
- * mapping so that it can use PMD mapping without additional TLB
- * pressure.
- */
-static ssize_t __init setup_pcpu_embed(size_t static_size, bool chosen)
-{
-        size_t reserve = PERCPU_MODULE_RESERVE + PERCPU_DYNAMIC_RESERVE;
-
-        /*
-         * If large page isn't supported, there's no benefit in doing
-         * this.  Also, embedding allocation doesn't play well with
-         * NUMA.
-         */
-        if (!chosen && (!cpu_has_pse || pcpu_need_numa()))
-                return -EINVAL;
-
-        return pcpu_embed_first_chunk(static_size, PERCPU_FIRST_CHUNK_RESERVE,
-                                      reserve - PERCPU_FIRST_CHUNK_RESERVE, -1);
 }
 
-/*
- * 4k page allocator
- *
- * This is the basic allocator.  Static percpu area is allocated
- * page-by-page and most of initialization is done by the generic
- * setup function.
- */
-static struct page **pcpu4k_pages __initdata;
-static int pcpu4k_nr_static_pages __initdata;
-
-static struct page * __init pcpu4k_get_page(unsigned int cpu, int pageno)
-{
-        if (pageno < pcpu4k_nr_static_pages)
-                return pcpu4k_pages[cpu * pcpu4k_nr_static_pages + pageno];
-        return NULL;
-}
-
-static void __init pcpu4k_populate_pte(unsigned long addr)
+static void __init pcpup_populate_pte(unsigned long addr)
 {
         populate_extra_pte(addr);
 }
 
-static ssize_t __init setup_pcpu_4k(size_t static_size)
-{
-        size_t pages_size;
-        unsigned int cpu;
-        int i, j;
-        ssize_t ret;
-
-        pcpu4k_nr_static_pages = PFN_UP(static_size);
-
-        /* unaligned allocations can't be freed, round up to page size */
-        pages_size = PFN_ALIGN(pcpu4k_nr_static_pages * nr_cpu_ids
-                               * sizeof(pcpu4k_pages[0]));
-        pcpu4k_pages = alloc_bootmem(pages_size);
-
-        /* allocate and copy */
-        j = 0;
-        for_each_possible_cpu(cpu)
-                for (i = 0; i < pcpu4k_nr_static_pages; i++) {
-                        void *ptr;
-
-                        ptr = pcpu_alloc_bootmem(cpu, PAGE_SIZE, PAGE_SIZE);
-                        if (!ptr) {
-                                pr_warning("PERCPU: failed to allocate "
-                                           "4k page for cpu%u\n", cpu);
-                                goto enomem;
-                        }
-
-                        memcpy(ptr, __per_cpu_load + i * PAGE_SIZE, PAGE_SIZE);
-                        pcpu4k_pages[j++] = virt_to_page(ptr);
-                }
-
-        /* we're ready, commit */
-        pr_info("PERCPU: Allocated %d 4k pages, static data %zu bytes\n",
-                pcpu4k_nr_static_pages, static_size);
-
-        ret = pcpu_setup_first_chunk(pcpu4k_get_page, static_size,
-                                     PERCPU_FIRST_CHUNK_RESERVE, -1,
-                                     -1, NULL, pcpu4k_populate_pte);
-        goto out_free_ar;
-
-enomem:
-        while (--j >= 0)
-                free_bootmem(__pa(page_address(pcpu4k_pages[j])), PAGE_SIZE);
-        ret = -ENOMEM;
-out_free_ar:
-        free_bootmem(__pa(pcpu4k_pages), pages_size);
-        return ret;
-}
-
-/* for explicit first chunk allocator selection */
-static char pcpu_chosen_alloc[16] __initdata;
-
-static int __init percpu_alloc_setup(char *str)
-{
-        strncpy(pcpu_chosen_alloc, str, sizeof(pcpu_chosen_alloc) - 1);
-        return 0;
-}
-early_param("percpu_alloc", percpu_alloc_setup);
-
 static inline void setup_percpu_segment(int cpu)
 {
 #ifdef CONFIG_X86_32
@@ -441,52 +170,49 @@ static inline void setup_percpu_segment(int cpu)
 
 void __init setup_per_cpu_areas(void)
 {
-        size_t static_size = __per_cpu_end - __per_cpu_start;
         unsigned int cpu;
         unsigned long delta;
-        size_t pcpu_unit_size;
-        ssize_t ret;
+        int rc;
 
         pr_info("NR_CPUS:%d nr_cpumask_bits:%d nr_cpu_ids:%d nr_node_ids:%d\n",
                 NR_CPUS, nr_cpumask_bits, nr_cpu_ids, nr_node_ids);
 
         /*
-         * Allocate percpu area.  If PSE is supported, try to make use
-         * of large page mappings.  Please read comments on top of
-         * each allocator for details.
+         * Allocate percpu area.  Embedding allocator is our favorite;
+         * however, on NUMA configurations, it can result in very
+         * sparse unit mapping and vmalloc area isn't spacious enough
+         * on 32bit.  Use page in that case.
          */
-        ret = -EINVAL;
-        if (strlen(pcpu_chosen_alloc)) {
-                if (strcmp(pcpu_chosen_alloc, "4k")) {
-                        if (!strcmp(pcpu_chosen_alloc, "lpage"))
-                                ret = setup_pcpu_lpage(static_size, true);
-                        else if (!strcmp(pcpu_chosen_alloc, "embed"))
-                                ret = setup_pcpu_embed(static_size, true);
-                        else
-                                pr_warning("PERCPU: unknown allocator %s "
-                                           "specified\n", pcpu_chosen_alloc);
-                        if (ret < 0)
-                                pr_warning("PERCPU: %s allocator failed (%zd), "
-                                           "falling back to 4k\n",
-                                           pcpu_chosen_alloc, ret);
-                }
-        } else {
-                ret = setup_pcpu_lpage(static_size, false);
-                if (ret < 0)
-                        ret = setup_pcpu_embed(static_size, false);
+#ifdef CONFIG_X86_32
+        if (pcpu_chosen_fc == PCPU_FC_AUTO && pcpu_need_numa())
+                pcpu_chosen_fc = PCPU_FC_PAGE;
+#endif
+        rc = -EINVAL;
+        if (pcpu_chosen_fc != PCPU_FC_PAGE) {
+                const size_t atom_size = cpu_has_pse ? PMD_SIZE : PAGE_SIZE;
+                const size_t dyn_size = PERCPU_MODULE_RESERVE +
+                        PERCPU_DYNAMIC_RESERVE - PERCPU_FIRST_CHUNK_RESERVE;
+
+                rc = pcpu_embed_first_chunk(PERCPU_FIRST_CHUNK_RESERVE,
+                                            dyn_size, atom_size,
+                                            pcpu_cpu_distance,
+                                            pcpu_fc_alloc, pcpu_fc_free);
+                if (rc < 0)
+                        pr_warning("PERCPU: %s allocator failed (%d), "
+                                   "falling back to page size\n",
+                                   pcpu_fc_names[pcpu_chosen_fc], rc);
         }
-        if (ret < 0)
-                ret = setup_pcpu_4k(static_size);
-        if (ret < 0)
-                panic("cannot allocate static percpu area (%zu bytes, err=%zd)",
-                      static_size, ret);
-
-        pcpu_unit_size = ret;
+        if (rc < 0)
+                rc = pcpu_page_first_chunk(PERCPU_FIRST_CHUNK_RESERVE,
+                                           pcpu_fc_alloc, pcpu_fc_free,
+                                           pcpup_populate_pte);
+        if (rc < 0)
+                panic("cannot initialize percpu area (err=%d)", rc);
 
         /* alrighty, percpu areas up and running */
         delta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start;
         for_each_possible_cpu(cpu) {
-                per_cpu_offset(cpu) = delta + cpu * pcpu_unit_size;
+                per_cpu_offset(cpu) = delta + pcpu_unit_offsets[cpu];
                 per_cpu(this_cpu_off, cpu) = per_cpu_offset(cpu);
                 per_cpu(cpu_number, cpu) = cpu;
                 setup_percpu_segment(cpu);
diff --git a/arch/x86/kernel/sfi.c b/arch/x86/kernel/sfi.c
new file mode 100644
index 000000000000..34e099382651
--- /dev/null
+++ b/arch/x86/kernel/sfi.c
@@ -0,0 +1,122 @@
+/*
+ * sfi.c - x86 architecture SFI support.
+ *
+ * Copyright (c) 2009, Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ */
+
+#define KMSG_COMPONENT "SFI"
+#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
+
+#include <linux/acpi.h>
+#include <linux/init.h>
+#include <linux/sfi.h>
+#include <linux/io.h>
+
+#include <asm/io_apic.h>
+#include <asm/mpspec.h>
+#include <asm/setup.h>
+#include <asm/apic.h>
+
+#ifdef CONFIG_X86_LOCAL_APIC
+static unsigned long sfi_lapic_addr __initdata = APIC_DEFAULT_PHYS_BASE;
+
+void __init mp_sfi_register_lapic_address(unsigned long address)
+{
+        mp_lapic_addr = address;
+
+        set_fixmap_nocache(FIX_APIC_BASE, mp_lapic_addr);
+        if (boot_cpu_physical_apicid == -1U)
+                boot_cpu_physical_apicid = read_apic_id();
+
+        pr_info("Boot CPU = %d\n", boot_cpu_physical_apicid);
+}
+
+/* All CPUs enumerated by SFI must be present and enabled */
+void __cpuinit mp_sfi_register_lapic(u8 id)
+{
+        if (MAX_APICS - id <= 0) {
+                pr_warning("Processor #%d invalid (max %d)\n",
+                        id, MAX_APICS);
+                return;
+        }
+
+        pr_info("registering lapic[%d]\n", id);
+
+        generic_processor_info(id, GET_APIC_VERSION(apic_read(APIC_LVR)));
+}
+
+static int __init sfi_parse_cpus(struct sfi_table_header *table)
+{
+        struct sfi_table_simple *sb;
+        struct sfi_cpu_table_entry *pentry;
+        int i;
+        int cpu_num;
+
+        sb = (struct sfi_table_simple *)table;
+        cpu_num = SFI_GET_NUM_ENTRIES(sb, struct sfi_cpu_table_entry);
+        pentry = (struct sfi_cpu_table_entry *)sb->pentry;
+
+        for (i = 0; i < cpu_num; i++) {
+                mp_sfi_register_lapic(pentry->apic_id);
+                pentry++;
+        }
+
+        smp_found_config = 1;
+        return 0;
+}
+#endif /* CONFIG_X86_LOCAL_APIC */
+
+#ifdef CONFIG_X86_IO_APIC
+static u32 gsi_base;
+
+static int __init sfi_parse_ioapic(struct sfi_table_header *table)
+{
+        struct sfi_table_simple *sb;
+        struct sfi_apic_table_entry *pentry;
+        int i, num;
+
+        sb = (struct sfi_table_simple *)table;
+        num = SFI_GET_NUM_ENTRIES(sb, struct sfi_apic_table_entry);
+        pentry = (struct sfi_apic_table_entry *)sb->pentry;
+
+        for (i = 0; i < num; i++) {
+                mp_register_ioapic(i, pentry->phys_addr, gsi_base);
+                gsi_base += io_apic_get_redir_entries(i);
+                pentry++;
+        }
+
+        WARN(pic_mode, KERN_WARNING
+                "SFI: pic_mod shouldn't be 1 when IOAPIC table is present\n");
+        pic_mode = 0;
+        return 0;
+}
+#endif /* CONFIG_X86_IO_APIC */
+
+/*
+ * sfi_platform_init(): register lapics & io-apics
+ */
+int __init sfi_platform_init(void)
+{
+#ifdef CONFIG_X86_LOCAL_APIC
+        mp_sfi_register_lapic_address(sfi_lapic_addr);
+        sfi_table_parse(SFI_SIG_CPUS, NULL, NULL, sfi_parse_cpus);
+#endif
+#ifdef CONFIG_X86_IO_APIC
+        sfi_table_parse(SFI_SIG_APIC, NULL, NULL, sfi_parse_ioapic);
+#endif
+        return 0;
+}
diff --git a/arch/x86/kernel/signal.c b/arch/x86/kernel/signal.c
index 4c578751e94e..6a44a76055ad 100644
--- a/arch/x86/kernel/signal.c
+++ b/arch/x86/kernel/signal.c
@@ -856,7 +856,7 @@ static void do_signal(struct pt_regs *regs)
 void
 do_notify_resume(struct pt_regs *regs, void *unused, __u32 thread_info_flags)
 {
-#ifdef CONFIG_X86_NEW_MCE
+#ifdef CONFIG_X86_MCE
         /* notify userspace of pending MCEs */
         if (thread_info_flags & _TIF_MCE_NOTIFY)
                 mce_notify_process();
@@ -869,6 +869,8 @@ do_notify_resume(struct pt_regs *regs, void *unused, __u32 thread_info_flags)
         if (thread_info_flags & _TIF_NOTIFY_RESUME) {
                 clear_thread_flag(TIF_NOTIFY_RESUME);
                 tracehook_notify_resume(regs);
+                if (current->replacement_session_keyring)
+                        key_replace_session_keyring();
         }
 
 #ifdef CONFIG_X86_32
diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index 2fecda69ee64..565ebc65920e 100644
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -47,6 +47,7 @@
 #include <linux/bootmem.h>
 #include <linux/err.h>
 #include <linux/nmi.h>
+#include <linux/tboot.h>
 
 #include <asm/acpi.h>
 #include <asm/desc.h>
@@ -323,7 +324,7 @@ notrace static void __cpuinit start_secondary(void *unused)
         /* enable local interrupts */
         local_irq_enable();
 
-        setup_secondary_clock();
+        x86_cpuinit.setup_percpu_clockev();
 
         wmb();
         cpu_idle();
@@ -434,7 +435,8 @@ const struct cpumask *cpu_coregroup_mask(int cpu)
          * For perf, we return last level cache shared map.
          * And for power savings, we return cpu_core_map
          */
-        if (sched_mc_power_savings || sched_smt_power_savings)
+        if ((sched_mc_power_savings || sched_smt_power_savings) &&
+            !(cpu_has(c, X86_FEATURE_AMD_DCM)))
                 return cpu_core_mask(cpu);
         else
                 return c->llc_shared_map;
@@ -1057,12 +1059,9 @@ void __init native_smp_prepare_cpus(unsigned int max_cpus)
 #endif
         current_thread_info()->cpu = 0;  /* needed? */
         for_each_possible_cpu(i) {
-                alloc_cpumask_var(&per_cpu(cpu_sibling_map, i), GFP_KERNEL);
-                alloc_cpumask_var(&per_cpu(cpu_core_map, i), GFP_KERNEL);
-                alloc_cpumask_var(&cpu_data(i).llc_shared_map, GFP_KERNEL);
-                cpumask_clear(per_cpu(cpu_core_map, i));
-                cpumask_clear(per_cpu(cpu_sibling_map, i));
-                cpumask_clear(cpu_data(i).llc_shared_map);
+                zalloc_cpumask_var(&per_cpu(cpu_sibling_map, i), GFP_KERNEL);
+                zalloc_cpumask_var(&per_cpu(cpu_core_map, i), GFP_KERNEL);
+                zalloc_cpumask_var(&cpu_data(i).llc_shared_map, GFP_KERNEL);
         }
         set_cpu_sibling_map(0);
 
@@ -1112,13 +1111,26 @@ void __init native_smp_prepare_cpus(unsigned int max_cpus)
 
         printk(KERN_INFO "CPU%d: ", 0);
         print_cpu_info(&cpu_data(0));
-        setup_boot_clock();
+        x86_init.timers.setup_percpu_clockev();
 
         if (is_uv_system())
                 uv_system_init();
+
+        set_mtrr_aps_delayed_init();
 out:
         preempt_enable();
 }
+
+void arch_enable_nonboot_cpus_begin(void)
+{
+        set_mtrr_aps_delayed_init();
+}
+
+void arch_enable_nonboot_cpus_end(void)
+{
+        mtrr_aps_init();
+}
+
 /*
  * Early setup to make printk work.
  */
@@ -1140,6 +1152,7 @@ void __init native_smp_cpus_done(unsigned int max_cpus)
         setup_ioapic_dest();
 #endif
         check_nmi_watchdog();
+        mtrr_aps_init();
 }
 
 static int __initdata setup_possible_cpus = -1;
@@ -1317,6 +1330,7 @@ void play_dead_common(void)
 void native_play_dead(void)
 {
         play_dead_common();
+        tboot_shutdown(TB_SHUTDOWN_WFS);
         wbinvd_halt();
 }
 
diff --git a/arch/x86/kernel/step.c b/arch/x86/kernel/step.c
index e8b9863ef8c4..3149032ff107 100644
--- a/arch/x86/kernel/step.c
+++ b/arch/x86/kernel/step.c
@@ -4,6 +4,7 @@
 #include <linux/sched.h>
 #include <linux/mm.h>
 #include <linux/ptrace.h>
+#include <asm/desc.h>
 
 unsigned long convert_ip_to_linear(struct task_struct *child, struct pt_regs *regs)
 {
@@ -23,7 +24,7 @@ unsigned long convert_ip_to_linear(struct task_struct *child, struct pt_regs *re
          * and APM bios ones we just ignore here.
          */
         if ((seg & SEGMENT_TI_MASK) == SEGMENT_LDT) {
-                u32 *desc;
+                struct desc_struct *desc;
                 unsigned long base;
 
                 seg &= ~7UL;
@@ -33,12 +34,10 @@ unsigned long convert_ip_to_linear(struct task_struct *child, struct pt_regs *re
                         addr = -1L; /* bogus selector, access would fault */
                 else {
                         desc = child->mm->context.ldt + seg;
-                        base = ((desc[0] >> 16) |
-                                ((desc[1] & 0xff) << 16) |
-                                (desc[1] & 0xff000000));
+                        base = get_desc_base(desc);
 
                         /* 16-bit code segment? */
-                        if (!((desc[1] >> 22) & 1))
+                        if (!desc->d)
                                 addr &= 0xffff;
                         addr += base;
                 }
diff --git a/arch/x86/kernel/sys_x86_64.c b/arch/x86/kernel/sys_x86_64.c
index 6bc211accf08..45e00eb09c3a 100644
--- a/arch/x86/kernel/sys_x86_64.c
+++ b/arch/x86/kernel/sys_x86_64.c
@@ -18,9 +18,9 @@
 #include <asm/ia32.h>
 #include <asm/syscalls.h>
 
-asmlinkage long sys_mmap(unsigned long addr, unsigned long len,
-                unsigned long prot, unsigned long flags,
-                unsigned long fd, unsigned long off)
+SYSCALL_DEFINE6(mmap, unsigned long, addr, unsigned long, len,
+                unsigned long, prot, unsigned long, flags,
+                unsigned long, fd, unsigned long, off)
 {
         long error;
         struct file *file;
@@ -226,7 +226,7 @@ bottomup:
 }
 
 
-asmlinkage long sys_uname(struct new_utsname __user *name)
+SYSCALL_DEFINE1(uname, struct new_utsname __user *, name)
 {
         int err;
         down_read(&uts_sem);
diff --git a/arch/x86/kernel/syscall_table_32.S b/arch/x86/kernel/syscall_table_32.S
index d51321ddafda..0157cd26d7cc 100644
--- a/arch/x86/kernel/syscall_table_32.S
+++ b/arch/x86/kernel/syscall_table_32.S
@@ -335,4 +335,4 @@ ENTRY(sys_call_table)
         .long sys_preadv
         .long sys_pwritev
         .long sys_rt_tgsigqueueinfo     /* 335 */
-        .long sys_perf_counter_open
+        .long sys_perf_event_open
diff --git a/arch/x86/kernel/tboot.c b/arch/x86/kernel/tboot.c
new file mode 100644
index 000000000000..86c9f91b48ae
--- /dev/null
+++ b/arch/x86/kernel/tboot.c
@@ -0,0 +1,447 @@
+/*
+ * tboot.c: main implementation of helper functions used by kernel for
+ *          runtime support of Intel(R) Trusted Execution Technology
+ *
+ * Copyright (c) 2006-2009, Intel Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ */
+
+#include <linux/dma_remapping.h>
+#include <linux/init_task.h>
+#include <linux/spinlock.h>
+#include <linux/delay.h>
+#include <linux/sched.h>
+#include <linux/init.h>
+#include <linux/dmar.h>
+#include <linux/cpu.h>
+#include <linux/pfn.h>
+#include <linux/mm.h>
+#include <linux/tboot.h>
+
+#include <asm/trampoline.h>
+#include <asm/processor.h>
+#include <asm/bootparam.h>
+#include <asm/pgtable.h>
+#include <asm/pgalloc.h>
+#include <asm/fixmap.h>
+#include <asm/proto.h>
+#include <asm/setup.h>
+#include <asm/e820.h>
+#include <asm/io.h>
+
+#include "acpi/realmode/wakeup.h"
+
+/* Global pointer to shared data; NULL means no measured launch. */
+struct tboot *tboot __read_mostly;
+
+/* timeout for APs (in secs) to enter wait-for-SIPI state during shutdown */
+#define AP_WAIT_TIMEOUT         1
+
+#undef pr_fmt
+#define pr_fmt(fmt)     "tboot: " fmt
+
+static u8 tboot_uuid[16] __initdata = TBOOT_UUID;
+
+void __init tboot_probe(void)
+{
+        /* Look for valid page-aligned address for shared page. */
+        if (!boot_params.tboot_addr)
+                return;
+        /*
+         * also verify that it is mapped as we expect it before calling
+         * set_fixmap(), to reduce chance of garbage value causing crash
+         */
+        if (!e820_any_mapped(boot_params.tboot_addr,
+                             boot_params.tboot_addr, E820_RESERVED)) {
+                pr_warning("non-0 tboot_addr but it is not of type E820_RESERVED\n");
+                return;
+        }
+
+        /* only a natively booted kernel should be using TXT */
+        if (paravirt_enabled()) {
+                pr_warning("non-0 tboot_addr but pv_ops is enabled\n");
+                return;
+        }
+
+        /* Map and check for tboot UUID. */
+        set_fixmap(FIX_TBOOT_BASE, boot_params.tboot_addr);
+        tboot = (struct tboot *)fix_to_virt(FIX_TBOOT_BASE);
+        if (memcmp(&tboot_uuid, &tboot->uuid, sizeof(tboot->uuid))) {
+                pr_warning("tboot at 0x%llx is invalid\n",
+                           boot_params.tboot_addr);
+                tboot = NULL;
+                return;
+        }
+        if (tboot->version < 5) {
+                pr_warning("tboot version is invalid: %u\n", tboot->version);
+                tboot = NULL;
+                return;
+        }
+
+        pr_info("found shared page at phys addr 0x%llx:\n",
+                boot_params.tboot_addr);
+        pr_debug("version: %d\n", tboot->version);
+        pr_debug("log_addr: 0x%08x\n", tboot->log_addr);
+        pr_debug("shutdown_entry: 0x%x\n", tboot->shutdown_entry);
+        pr_debug("tboot_base: 0x%08x\n", tboot->tboot_base);
+        pr_debug("tboot_size: 0x%x\n", tboot->tboot_size);
+}
+
+static pgd_t *tboot_pg_dir;
+static struct mm_struct tboot_mm = {
+        .mm_rb          = RB_ROOT,
+        .pgd            = swapper_pg_dir,
+        .mm_users       = ATOMIC_INIT(2),
+        .mm_count       = ATOMIC_INIT(1),
+        .mmap_sem       = __RWSEM_INITIALIZER(init_mm.mmap_sem),
+        .page_table_lock =  __SPIN_LOCK_UNLOCKED(init_mm.page_table_lock),
+        .mmlist         = LIST_HEAD_INIT(init_mm.mmlist),
+        .cpu_vm_mask    = CPU_MASK_ALL,
+};
+
+static inline void switch_to_tboot_pt(void)
+{
+        write_cr3(virt_to_phys(tboot_pg_dir));
+}
+
+static int map_tboot_page(unsigned long vaddr, unsigned long pfn,
+                          pgprot_t prot)
+{
+        pgd_t *pgd;
+        pud_t *pud;
+        pmd_t *pmd;
+        pte_t *pte;
+
+        pgd = pgd_offset(&tboot_mm, vaddr);
+        pud = pud_alloc(&tboot_mm, pgd, vaddr);
+        if (!pud)
+                return -1;
+        pmd = pmd_alloc(&tboot_mm, pud, vaddr);
+        if (!pmd)
+                return -1;
+        pte = pte_alloc_map(&tboot_mm, pmd, vaddr);
+        if (!pte)
+                return -1;
+        set_pte_at(&tboot_mm, vaddr, pte, pfn_pte(pfn, prot));
+        pte_unmap(pte);
+        return 0;
+}
+
+static int map_tboot_pages(unsigned long vaddr, unsigned long start_pfn,
+                           unsigned long nr)
+{
+        /* Reuse the original kernel mapping */
+        tboot_pg_dir = pgd_alloc(&tboot_mm);
+        if (!tboot_pg_dir)
+                return -1;
+
+        for (; nr > 0; nr--, vaddr += PAGE_SIZE, start_pfn++) {
+                if (map_tboot_page(vaddr, start_pfn, PAGE_KERNEL_EXEC))
+                        return -1;
+        }
+
+        return 0;
+}
+
+static void tboot_create_trampoline(void)
+{
+        u32 map_base, map_size;
+
+        /* Create identity map for tboot shutdown code. */
+        map_base = PFN_DOWN(tboot->tboot_base);
+        map_size = PFN_UP(tboot->tboot_size);
+        if (map_tboot_pages(map_base << PAGE_SHIFT, map_base, map_size))
+                panic("tboot: Error mapping tboot pages (mfns) @ 0x%x, 0x%x\n",
+                      map_base, map_size);
+}
+
+#ifdef CONFIG_ACPI_SLEEP
+
+static void add_mac_region(phys_addr_t start, unsigned long size)
+{
+        struct tboot_mac_region *mr;
+        phys_addr_t end = start + size;
+
+        if (start && size) {
+                mr = &tboot->mac_regions[tboot->num_mac_regions++];
+                mr->start = round_down(start, PAGE_SIZE);
+                mr->size  = round_up(end, PAGE_SIZE) - mr->start;
+        }
+}
+
+static int tboot_setup_sleep(void)
+{
+        tboot->num_mac_regions = 0;
+
+        /* S3 resume code */
+        add_mac_region(acpi_wakeup_address, WAKEUP_SIZE);
+
+#ifdef CONFIG_X86_TRAMPOLINE
+        /* AP trampoline code */
+        add_mac_region(virt_to_phys(trampoline_base), TRAMPOLINE_SIZE);
+#endif
+
+        /* kernel code + data + bss */
+        add_mac_region(virt_to_phys(_text), _end - _text);
+
+        tboot->acpi_sinfo.kernel_s3_resume_vector = acpi_wakeup_address;
+
+        return 0;
+}
+
+#else /* no CONFIG_ACPI_SLEEP */
+
+static int tboot_setup_sleep(void)
+{
+        /* S3 shutdown requested, but S3 not supported by the kernel... */
+        BUG();
+        return -1;
+}
+
+#endif
+
+void tboot_shutdown(u32 shutdown_type)
+{
+        void (*shutdown)(void);
+
+        if (!tboot_enabled())
+                return;
+
+        /*
+         * if we're being called before the 1:1 mapping is set up then just
+         * return and let the normal shutdown happen; this should only be
+         * due to very early panic()
+         */
+        if (!tboot_pg_dir)
+                return;
+
+        /* if this is S3 then set regions to MAC */
+        if (shutdown_type == TB_SHUTDOWN_S3)
+                if (tboot_setup_sleep())
+                        return;
+
+        tboot->shutdown_type = shutdown_type;
+
+        switch_to_tboot_pt();
+
+        shutdown = (void(*)(void))(unsigned long)tboot->shutdown_entry;
+        shutdown();
+
+        /* should not reach here */
+        while (1)
+                halt();
+}
+
+static void tboot_copy_fadt(const struct acpi_table_fadt *fadt)
+{
+#define TB_COPY_GAS(tbg, g)                     \
+        tbg.space_id     = g.space_id;          \
+        tbg.bit_width    = g.bit_width;         \
+        tbg.bit_offset   = g.bit_offset;        \
+        tbg.access_width = g.access_width;      \
+        tbg.address      = g.address;
+
+        TB_COPY_GAS(tboot->acpi_sinfo.pm1a_cnt_blk, fadt->xpm1a_control_block);
+        TB_COPY_GAS(tboot->acpi_sinfo.pm1b_cnt_blk, fadt->xpm1b_control_block);
+        TB_COPY_GAS(tboot->acpi_sinfo.pm1a_evt_blk, fadt->xpm1a_event_block);
+        TB_COPY_GAS(tboot->acpi_sinfo.pm1b_evt_blk, fadt->xpm1b_event_block);
+
+        /*
+         * We need phys addr of waking vector, but can't use virt_to_phys() on
+         * &acpi_gbl_FACS because it is ioremap'ed, so calc from FACS phys
+         * addr.
+         */
+        tboot->acpi_sinfo.wakeup_vector = fadt->facs +
+                offsetof(struct acpi_table_facs, firmware_waking_vector);
+}
+
+void tboot_sleep(u8 sleep_state, u32 pm1a_control, u32 pm1b_control)
+{
+        static u32 acpi_shutdown_map[ACPI_S_STATE_COUNT] = {
+                /* S0,1,2: */ -1, -1, -1,
+                /* S3: */ TB_SHUTDOWN_S3,
+                /* S4: */ TB_SHUTDOWN_S4,
+                /* S5: */ TB_SHUTDOWN_S5 };
+
+        if (!tboot_enabled())
+                return;
+
+        tboot_copy_fadt(&acpi_gbl_FADT);
+        tboot->acpi_sinfo.pm1a_cnt_val = pm1a_control;
+        tboot->acpi_sinfo.pm1b_cnt_val = pm1b_control;
+        /* we always use the 32b wakeup vector */
+        tboot->acpi_sinfo.vector_width = 32;
+
+        if (sleep_state >= ACPI_S_STATE_COUNT ||
+            acpi_shutdown_map[sleep_state] == -1) {
+                pr_warning("unsupported sleep state 0x%x\n", sleep_state);
+                return;
+        }
+
+        tboot_shutdown(acpi_shutdown_map[sleep_state]);
+}
+
+static atomic_t ap_wfs_count;
+
+static int tboot_wait_for_aps(int num_aps)
+{
+        unsigned long timeout;
+
+        timeout = AP_WAIT_TIMEOUT*HZ;
+        while (atomic_read((atomic_t *)&tboot->num_in_wfs) != num_aps &&
+               timeout) {
+                mdelay(1);
+                timeout--;
+        }
+
+        if (timeout)
+                pr_warning("tboot wait for APs timeout\n");
+
+        return !(atomic_read((atomic_t *)&tboot->num_in_wfs) == num_aps);
+}
+
+static int __cpuinit tboot_cpu_callback(struct notifier_block *nfb,
+                        unsigned long action, void *hcpu)
+{
+        switch (action) {
+        case CPU_DYING:
+                atomic_inc(&ap_wfs_count);
+                if (num_online_cpus() == 1)
+                        if (tboot_wait_for_aps(atomic_read(&ap_wfs_count)))
+                                return NOTIFY_BAD;
+                break;
+        }
+        return NOTIFY_OK;
+}
+
+static struct notifier_block tboot_cpu_notifier __cpuinitdata =
+{
+        .notifier_call = tboot_cpu_callback,
+};
+
+static __init int tboot_late_init(void)
+{
+        if (!tboot_enabled())
+                return 0;
+
+        tboot_create_trampoline();
+
+        atomic_set(&ap_wfs_count, 0);
+        register_hotcpu_notifier(&tboot_cpu_notifier);
+        return 0;
+}
+
+late_initcall(tboot_late_init);
+
+/*
+ * TXT configuration registers (offsets from TXT_{PUB, PRIV}_CONFIG_REGS_BASE)
+ */
+
+#define TXT_PUB_CONFIG_REGS_BASE       0xfed30000
+#define TXT_PRIV_CONFIG_REGS_BASE      0xfed20000
+
+/* # pages for each config regs space - used by fixmap */
+#define NR_TXT_CONFIG_PAGES     ((TXT_PUB_CONFIG_REGS_BASE -                \
+                                  TXT_PRIV_CONFIG_REGS_BASE) >> PAGE_SHIFT)
+
+/* offsets from pub/priv config space */
+#define TXTCR_HEAP_BASE             0x0300
+#define TXTCR_HEAP_SIZE             0x0308
+
+#define SHA1_SIZE      20
+
+struct sha1_hash {
+        u8 hash[SHA1_SIZE];
+};
+
+struct sinit_mle_data {
+        u32               version;             /* currently 6 */
+        struct sha1_hash  bios_acm_id;
+        u32               edx_senter_flags;
+        u64               mseg_valid;
+        struct sha1_hash  sinit_hash;
+        struct sha1_hash  mle_hash;
+        struct sha1_hash  stm_hash;
+        struct sha1_hash  lcp_policy_hash;
+        u32               lcp_policy_control;
+        u32               rlp_wakeup_addr;
+        u32               reserved;
+        u32               num_mdrs;
+        u32               mdrs_off;
+        u32               num_vtd_dmars;
+        u32               vtd_dmars_off;
+} __packed;
+
+struct acpi_table_header *tboot_get_dmar_table(struct acpi_table_header *dmar_tbl)
+{
+        void *heap_base, *heap_ptr, *config;
+
+        if (!tboot_enabled())
+                return dmar_tbl;
+
+        /*
+         * ACPI tables may not be DMA protected by tboot, so use DMAR copy
+         * SINIT saved in SinitMleData in TXT heap (which is DMA protected)
+         */
+
+        /* map config space in order to get heap addr */
+        config = ioremap(TXT_PUB_CONFIG_REGS_BASE, NR_TXT_CONFIG_PAGES *
+                         PAGE_SIZE);
+        if (!config)
+                return NULL;
+
+        /* now map TXT heap */
+        heap_base = ioremap(*(u64 *)(config + TXTCR_HEAP_BASE),
+                            *(u64 *)(config + TXTCR_HEAP_SIZE));
+        iounmap(config);
+        if (!heap_base)
+                return NULL;
+
+        /* walk heap to SinitMleData */
+        /* skip BiosData */
+        heap_ptr = heap_base + *(u64 *)heap_base;
+        /* skip OsMleData */
+        heap_ptr += *(u64 *)heap_ptr;
+        /* skip OsSinitData */
+        heap_ptr += *(u64 *)heap_ptr;
+        /* now points to SinitMleDataSize; set to SinitMleData */
+        heap_ptr += sizeof(u64);
+        /* get addr of DMAR table */
+        dmar_tbl = (struct acpi_table_header *)(heap_ptr +
+                   ((struct sinit_mle_data *)heap_ptr)->vtd_dmars_off -
+                   sizeof(u64));
+
+        /* don't unmap heap because dmar.c needs access to this */
+
+        return dmar_tbl;
+}
+
+int tboot_force_iommu(void)
+{
+        if (!tboot_enabled())
+                return 0;
+
+        if (no_iommu || swiotlb || dmar_disabled)
+                pr_warning("Forcing Intel-IOMMU to enabled\n");
+
+        dmar_disabled = 0;
+#ifdef CONFIG_SWIOTLB
+        swiotlb = 0;
+#endif
+        no_iommu = 0;
+
+        return 1;
+}
diff --git a/arch/x86/kernel/time.c b/arch/x86/kernel/time.c
new file mode 100644
index 000000000000..dcb00d278512
--- /dev/null
+++ b/arch/x86/kernel/time.c
@@ -0,0 +1,120 @@
+/*
+ *  Copyright (c) 1991,1992,1995  Linus Torvalds
+ *  Copyright (c) 1994  Alan Modra
+ *  Copyright (c) 1995  Markus Kuhn
+ *  Copyright (c) 1996  Ingo Molnar
+ *  Copyright (c) 1998  Andrea Arcangeli
+ *  Copyright (c) 2002,2006  Vojtech Pavlik
+ *  Copyright (c) 2003  Andi Kleen
+ *
+ */
+
+#include <linux/clockchips.h>
+#include <linux/interrupt.h>
+#include <linux/time.h>
+#include <linux/mca.h>
+
+#include <asm/vsyscall.h>
+#include <asm/x86_init.h>
+#include <asm/i8259.h>
+#include <asm/i8253.h>
+#include <asm/timer.h>
+#include <asm/hpet.h>
+#include <asm/time.h>
+
+#if defined(CONFIG_X86_32) && defined(CONFIG_X86_IO_APIC)
+int timer_ack;
+#endif
+
+#ifdef CONFIG_X86_64
+volatile unsigned long __jiffies __section_jiffies = INITIAL_JIFFIES;
+#endif
+
+unsigned long profile_pc(struct pt_regs *regs)
+{
+        unsigned long pc = instruction_pointer(regs);
+
+        if (!user_mode_vm(regs) && in_lock_functions(pc)) {
+#ifdef CONFIG_FRAME_POINTER
+                return *(unsigned long *)(regs->bp + sizeof(long));
+#else
+                unsigned long *sp = (unsigned long *)regs->sp;
+                /*
+                 * Return address is either directly at stack pointer
+                 * or above a saved flags. Eflags has bits 22-31 zero,
+                 * kernel addresses don't.
+                 */
+                if (sp[0] >> 22)
+                        return sp[0];
+                if (sp[1] >> 22)
+                        return sp[1];
+#endif
+        }
+        return pc;
+}
+EXPORT_SYMBOL(profile_pc);
+
+/*
+ * Default timer interrupt handler for PIT/HPET
+ */
+static irqreturn_t timer_interrupt(int irq, void *dev_id)
+{
+        /* Keep nmi watchdog up to date */
+        inc_irq_stat(irq0_irqs);
+
+        /* Optimized out for !IO_APIC and x86_64 */
+        if (timer_ack) {
+                /*
+                 * Subtle, when I/O APICs are used we have to ack timer IRQ
+                 * manually to deassert NMI lines for the watchdog if run
+                 * on an 82489DX-based system.
+                 */
+                spin_lock(&i8259A_lock);
+                outb(0x0c, PIC_MASTER_OCW3);
+                /* Ack the IRQ; AEOI will end it automatically. */
+                inb(PIC_MASTER_POLL);
+                spin_unlock(&i8259A_lock);
+        }
+
+        global_clock_event->event_handler(global_clock_event);
+
+        /* MCA bus quirk: Acknowledge irq0 by setting bit 7 in port 0x61 */
+        if (MCA_bus)
+                outb_p(inb_p(0x61)| 0x80, 0x61);
+
+        return IRQ_HANDLED;
+}
+
+static struct irqaction irq0  = {
+        .handler = timer_interrupt,
+        .flags = IRQF_DISABLED | IRQF_NOBALANCING | IRQF_IRQPOLL | IRQF_TIMER,
+        .name = "timer"
+};
+
+void __init setup_default_timer_irq(void)
+{
+        setup_irq(0, &irq0);
+}
+
+/* Default timer init function */
+void __init hpet_time_init(void)
+{
+        if (!hpet_enable())
+                setup_pit_timer();
+        setup_default_timer_irq();
+}
+
+static __init void x86_late_time_init(void)
+{
+        x86_init.timers.timer_init();
+        tsc_init();
+}
+
+/*
+ * Initialize TSC and delay the periodic timer init to
+ * late x86_late_time_init() so ioremap works.
+ */
+void __init time_init(void)
+{
+        late_time_init = x86_late_time_init;
+}
diff --git a/arch/x86/kernel/time_32.c b/arch/x86/kernel/time_32.c
deleted file mode 100644
index 5c5d87f0b2e1..000000000000
--- a/arch/x86/kernel/time_32.c
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- *  Copyright (C) 1991, 1992, 1995  Linus Torvalds
- *
- * This file contains the PC-specific time handling details:
- * reading the RTC at bootup, etc..
- * 1994-07-02    Alan Modra
- *      fixed set_rtc_mmss, fixed time.year for >= 2000, new mktime
- * 1995-03-26    Markus Kuhn
- *      fixed 500 ms bug at call to set_rtc_mmss, fixed DS12887
- *      precision CMOS clock update
- * 1996-05-03    Ingo Molnar
- *      fixed time warps in do_[slow|fast]_gettimeoffset()
- * 1997-09-10   Updated NTP code according to technical memorandum Jan '96
- *              "A Kernel Model for Precision Timekeeping" by Dave Mills
- * 1998-09-05    (Various)
- *      More robust do_fast_gettimeoffset() algorithm implemented
- *      (works with APM, Cyrix 6x86MX and Centaur C6),
- *      monotonic gettimeofday() with fast_get_timeoffset(),
- *      drift-proof precision TSC calibration on boot
- *      (C. Scott Ananian <cananian@alumni.princeton.edu>, Andrew D.
- *      Balsa <andrebalsa@altern.org>, Philip Gladstone <philip@raptor.com>;
- *      ported from 2.0.35 Jumbo-9 by Michael Krause <m.krause@tu-harburg.de>).
- * 1998-12-16    Andrea Arcangeli
- *      Fixed Jumbo-9 code in 2.1.131: do_gettimeofday was missing 1 jiffy
- *      because was not accounting lost_ticks.
- * 1998-12-24 Copyright (C) 1998  Andrea Arcangeli
- *      Fixed a xtime SMP race (we need the xtime_lock rw spinlock to
- *      serialize accesses to xtime/lost_ticks).
- */
-
-#include <linux/init.h>
-#include <linux/interrupt.h>
-#include <linux/time.h>
-#include <linux/mca.h>
-
-#include <asm/setup.h>
-#include <asm/hpet.h>
-#include <asm/time.h>
-#include <asm/timer.h>
-
-#include <asm/do_timer.h>
-
-int timer_ack;
-
-unsigned long profile_pc(struct pt_regs *regs)
-{
-        unsigned long pc = instruction_pointer(regs);
-
-#ifdef CONFIG_SMP
-        if (!user_mode_vm(regs) && in_lock_functions(pc)) {
-#ifdef CONFIG_FRAME_POINTER
-                return *(unsigned long *)(regs->bp + sizeof(long));
-#else
-                unsigned long *sp = (unsigned long *)&regs->sp;
-
-                /* Return address is either directly at stack pointer
-                   or above a saved flags. Eflags has bits 22-31 zero,
-                   kernel addresses don't. */
-                if (sp[0] >> 22)
-                        return sp[0];
-                if (sp[1] >> 22)
-                        return sp[1];
-#endif
-        }
-#endif
-        return pc;
-}
-EXPORT_SYMBOL(profile_pc);
-
-/*
- * This is the same as the above, except we _also_ save the current
- * Time Stamp Counter value at the time of the timer interrupt, so that
- * we later on can estimate the time of day more exactly.
- */
-irqreturn_t timer_interrupt(int irq, void *dev_id)
-{
-        /* Keep nmi watchdog up to date */
-        inc_irq_stat(irq0_irqs);
-
-#ifdef CONFIG_X86_IO_APIC
-        if (timer_ack) {
-                /*
-                 * Subtle, when I/O APICs are used we have to ack timer IRQ
-                 * manually to deassert NMI lines for the watchdog if run
-                 * on an 82489DX-based system.
-                 */
-                spin_lock(&i8259A_lock);
-                outb(0x0c, PIC_MASTER_OCW3);
-                /* Ack the IRQ; AEOI will end it automatically. */
-                inb(PIC_MASTER_POLL);
-                spin_unlock(&i8259A_lock);
-        }
-#endif
-
-        do_timer_interrupt_hook();
-
-#ifdef CONFIG_MCA
-        if (MCA_bus) {
-                /* The PS/2 uses level-triggered interrupts.  You can't
-                turn them off, nor would you want to (any attempt to
-                enable edge-triggered interrupts usually gets intercepted by a
-                special hardware circuit).  Hence we have to acknowledge
-                the timer interrupt.  Through some incredibly stupid
-                design idea, the reset for IRQ 0 is done by setting the
-                high bit of the PPI port B (0x61).  Note that some PS/2s,
-                notably the 55SX, work fine if this is removed.  */
-
-                u8 irq_v = inb_p(0x61);         /* read the current state */
-                outb_p(irq_v | 0x80, 0x61);     /* reset the IRQ */
-        }
-#endif
-
-        return IRQ_HANDLED;
-}
-
-/* Duplicate of time_init() below, with hpet_enable part added */
-void __init hpet_time_init(void)
-{
-        if (!hpet_enable())
-                setup_pit_timer();
-        x86_quirk_time_init();
-}
-
-/*
- * This is called directly from init code; we must delay timer setup in the
- * HPET case as we can't make the decision to turn on HPET this early in the
- * boot process.
- *
- * The chosen time_init function will usually be hpet_time_init, above, but
- * in the case of virtual hardware, an alternative function may be substituted.
- */
-void __init time_init(void)
-{
-        x86_quirk_pre_time_init();
-        tsc_init();
-        late_time_init = choose_time_init();
-}
diff --git a/arch/x86/kernel/time_64.c b/arch/x86/kernel/time_64.c
deleted file mode 100644
index 5ba343e61844..000000000000
--- a/arch/x86/kernel/time_64.c
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- *  "High Precision Event Timer" based timekeeping.
- *
- *  Copyright (c) 1991,1992,1995  Linus Torvalds
- *  Copyright (c) 1994  Alan Modra
- *  Copyright (c) 1995  Markus Kuhn
- *  Copyright (c) 1996  Ingo Molnar
- *  Copyright (c) 1998  Andrea Arcangeli
- *  Copyright (c) 2002,2006  Vojtech Pavlik
- *  Copyright (c) 2003  Andi Kleen
- *  RTC support code taken from arch/i386/kernel/timers/time_hpet.c
- */
-
-#include <linux/clockchips.h>
-#include <linux/init.h>
-#include <linux/interrupt.h>
-#include <linux/module.h>
-#include <linux/time.h>
-#include <linux/mca.h>
-#include <linux/nmi.h>
-
-#include <asm/i8253.h>
-#include <asm/hpet.h>
-#include <asm/vgtod.h>
-#include <asm/time.h>
-#include <asm/timer.h>
-
-volatile unsigned long __jiffies __section_jiffies = INITIAL_JIFFIES;
-
-unsigned long profile_pc(struct pt_regs *regs)
-{
-        unsigned long pc = instruction_pointer(regs);
-
-        /* Assume the lock function has either no stack frame or a copy
-           of flags from PUSHF
-           Eflags always has bits 22 and up cleared unlike kernel addresses. */
-        if (!user_mode_vm(regs) && in_lock_functions(pc)) {
-#ifdef CONFIG_FRAME_POINTER
-                return *(unsigned long *)(regs->bp + sizeof(long));
-#else
-                unsigned long *sp = (unsigned long *)regs->sp;
-                if (sp[0] >> 22)
-                        return sp[0];
-                if (sp[1] >> 22)
-                        return sp[1];
-#endif
-        }
-        return pc;
-}
-EXPORT_SYMBOL(profile_pc);
-
-static irqreturn_t timer_interrupt(int irq, void *dev_id)
-{
-        inc_irq_stat(irq0_irqs);
-
-        global_clock_event->event_handler(global_clock_event);
-
-#ifdef CONFIG_MCA
-        if (MCA_bus) {
-                u8 irq_v = inb_p(0x61);       /* read the current state */
-                outb_p(irq_v|0x80, 0x61);     /* reset the IRQ */
-        }
-#endif
-
-        return IRQ_HANDLED;
-}
-
-/* calibrate_cpu is used on systems with fixed rate TSCs to determine
- * processor frequency */
-#define TICK_COUNT 100000000
-unsigned long __init calibrate_cpu(void)
-{
-        int tsc_start, tsc_now;
-        int i, no_ctr_free;
-        unsigned long evntsel3 = 0, pmc3 = 0, pmc_now = 0;
-        unsigned long flags;
-
-        for (i = 0; i < 4; i++)
-                if (avail_to_resrv_perfctr_nmi_bit(i))
-                        break;
-        no_ctr_free = (i == 4);
-        if (no_ctr_free) {
-                WARN(1, KERN_WARNING "Warning: AMD perfctrs busy ... "
-                     "cpu_khz value may be incorrect.\n");
-                i = 3;
-                rdmsrl(MSR_K7_EVNTSEL3, evntsel3);
-                wrmsrl(MSR_K7_EVNTSEL3, 0);
-                rdmsrl(MSR_K7_PERFCTR3, pmc3);
-        } else {
-                reserve_perfctr_nmi(MSR_K7_PERFCTR0 + i);
-                reserve_evntsel_nmi(MSR_K7_EVNTSEL0 + i);
-        }
-        local_irq_save(flags);
-        /* start measuring cycles, incrementing from 0 */
-        wrmsrl(MSR_K7_PERFCTR0 + i, 0);
-        wrmsrl(MSR_K7_EVNTSEL0 + i, 1 << 22 | 3 << 16 | 0x76);
-        rdtscl(tsc_start);
-        do {
-                rdmsrl(MSR_K7_PERFCTR0 + i, pmc_now);
-                tsc_now = get_cycles();
-        } while ((tsc_now - tsc_start) < TICK_COUNT);
-
-        local_irq_restore(flags);
-        if (no_ctr_free) {
-                wrmsrl(MSR_K7_EVNTSEL3, 0);
-                wrmsrl(MSR_K7_PERFCTR3, pmc3);
-                wrmsrl(MSR_K7_EVNTSEL3, evntsel3);
-        } else {
-                release_perfctr_nmi(MSR_K7_PERFCTR0 + i);
-                release_evntsel_nmi(MSR_K7_EVNTSEL0 + i);
-        }
-
-        return pmc_now * tsc_khz / (tsc_now - tsc_start);
-}
-
-static struct irqaction irq0 = {
-        .handler        = timer_interrupt,
-        .flags          = IRQF_DISABLED | IRQF_IRQPOLL | IRQF_NOBALANCING | IRQF_TIMER,
-        .name           = "timer"
-};
-
-void __init hpet_time_init(void)
-{
-        if (!hpet_enable())
-                setup_pit_timer();
-
-        setup_irq(0, &irq0);
-}
-
-void __init time_init(void)
-{
-        tsc_init();
-
-        late_time_init = choose_time_init();
-}
diff --git a/arch/x86/kernel/tlb_uv.c b/arch/x86/kernel/tlb_uv.c
index 77b9689f8edb..503c1f2e8835 100644
--- a/arch/x86/kernel/tlb_uv.c
+++ b/arch/x86/kernel/tlb_uv.c
@@ -640,13 +640,13 @@ static int __init uv_ptc_init(void)
         if (!is_uv_system())
                 return 0;
 
-        proc_uv_ptc = create_proc_entry(UV_PTC_BASENAME, 0444, NULL);
+        proc_uv_ptc = proc_create(UV_PTC_BASENAME, 0444, NULL,
+                                  &proc_uv_ptc_operations);
         if (!proc_uv_ptc) {
                 printk(KERN_ERR "unable to create %s proc entry\n",
                        UV_PTC_BASENAME);
                 return -EINVAL;
         }
-        proc_uv_ptc->proc_fops = &proc_uv_ptc_operations;
         return 0;
 }
 
diff --git a/arch/x86/kernel/trampoline.c b/arch/x86/kernel/trampoline.c
index 808031a5ba19..699f7eeb896a 100644
--- a/arch/x86/kernel/trampoline.c
+++ b/arch/x86/kernel/trampoline.c
@@ -4,7 +4,7 @@
 #include <asm/e820.h>
 
 /* ready for x86_64 and x86 */
-unsigned char *trampoline_base = __va(TRAMPOLINE_BASE);
+unsigned char *__cpuinitdata trampoline_base = __va(TRAMPOLINE_BASE);
 
 void __init reserve_trampoline_memory(void)
 {
@@ -26,7 +26,7 @@ void __init reserve_trampoline_memory(void)
  * bootstrap into the page concerned. The caller
  * has made sure it's suitably aligned.
  */
-unsigned long setup_trampoline(void)
+unsigned long __cpuinit setup_trampoline(void)
 {
         memcpy(trampoline_base, trampoline_data, TRAMPOLINE_SIZE);
         return virt_to_phys(trampoline_base);
diff --git a/arch/x86/kernel/trampoline_32.S b/arch/x86/kernel/trampoline_32.S
index 66d874e5404c..8508237e8e43 100644
--- a/arch/x86/kernel/trampoline_32.S
+++ b/arch/x86/kernel/trampoline_32.S
@@ -28,16 +28,12 @@
  */
 
 #include <linux/linkage.h>
+#include <linux/init.h>
 #include <asm/segment.h>
 #include <asm/page_types.h>
 
 /* We can free up trampoline after bootup if cpu hotplug is not supported. */
-#ifndef CONFIG_HOTPLUG_CPU
-.section ".cpuinit.data","aw",@progbits
-#else
-.section .rodata,"a",@progbits
-#endif
-
+__CPUINITRODATA
 .code16
 
 ENTRY(trampoline_data)
diff --git a/arch/x86/kernel/trampoline_64.S b/arch/x86/kernel/trampoline_64.S
index cddfb8d386b9..596d54c660a5 100644
--- a/arch/x86/kernel/trampoline_64.S
+++ b/arch/x86/kernel/trampoline_64.S
@@ -25,14 +25,15 @@
  */
 
 #include <linux/linkage.h>
+#include <linux/init.h>
 #include <asm/pgtable_types.h>
 #include <asm/page_types.h>
 #include <asm/msr.h>
 #include <asm/segment.h>
 #include <asm/processor-flags.h>
 
-.section .rodata, "a", @progbits
-
+/* We can free up the trampoline after bootup if cpu hotplug is not supported. */
+__CPUINITRODATA
 .code16
 
 ENTRY(trampoline_data)
diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c
index 5204332f475d..a665c71352b8 100644
--- a/arch/x86/kernel/traps.c
+++ b/arch/x86/kernel/traps.c
@@ -14,7 +14,6 @@
 #include <linux/spinlock.h>
 #include <linux/kprobes.h>
 #include <linux/uaccess.h>
-#include <linux/utsname.h>
 #include <linux/kdebug.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
@@ -59,12 +58,12 @@
 #include <asm/mach_traps.h>
 
 #ifdef CONFIG_X86_64
+#include <asm/x86_init.h>
 #include <asm/pgalloc.h>
 #include <asm/proto.h>
 #else
 #include <asm/processor-flags.h>
 #include <asm/setup.h>
-#include <asm/traps.h>
 
 asmlinkage int system_call(void);
 
@@ -76,7 +75,7 @@ char ignore_fpu_irq;
  * F0 0F bug workaround.. We have a special link segment
  * for this.
  */
-gate_desc idt_table[256]
+gate_desc idt_table[NR_VECTORS]
         __attribute__((__section__(".data.idt"))) = { { { { 0, 0 } } }, };
 #endif
 
@@ -786,33 +785,34 @@ do_spurious_interrupt_bug(struct pt_regs *regs, long error_code)
 #endif
 }
 
-#ifdef CONFIG_X86_32
-unsigned long patch_espfix_desc(unsigned long uesp, unsigned long kesp)
+asmlinkage void __attribute__((weak)) smp_thermal_interrupt(void)
 {
-        struct desc_struct *gdt = get_cpu_gdt_table(smp_processor_id());
-        unsigned long base = (kesp - uesp) & -THREAD_SIZE;
-        unsigned long new_kesp = kesp - base;
-        unsigned long lim_pages = (new_kesp | (THREAD_SIZE - 1)) >> PAGE_SHIFT;
-        __u64 desc = *(__u64 *)&gdt[GDT_ENTRY_ESPFIX_SS];
-
-        /* Set up base for espfix segment */
-        desc &= 0x00f0ff0000000000ULL;
-        desc |= ((((__u64)base) << 16) & 0x000000ffffff0000ULL) |
-                ((((__u64)base) << 32) & 0xff00000000000000ULL) |
-                ((((__u64)lim_pages) << 32) & 0x000f000000000000ULL) |
-                (lim_pages & 0xffff);
-        *(__u64 *)&gdt[GDT_ENTRY_ESPFIX_SS] = desc;
-
-        return new_kesp;
 }
-#endif
 
-asmlinkage void __attribute__((weak)) smp_thermal_interrupt(void)
+asmlinkage void __attribute__((weak)) smp_threshold_interrupt(void)
 {
 }
 
-asmlinkage void __attribute__((weak)) smp_threshold_interrupt(void)
+/*
+ * __math_state_restore assumes that cr0.TS is already clear and the
+ * fpu state is all ready for use.  Used during context switch.
+ */
+void __math_state_restore(void)
 {
+        struct thread_info *thread = current_thread_info();
+        struct task_struct *tsk = thread->task;
+
+        /*
+         * Paranoid restore. send a SIGSEGV if we fail to restore the state.
+         */
+        if (unlikely(restore_fpu_checking(tsk))) {
+                stts();
+                force_sig(SIGSEGV, tsk);
+                return;
+        }
+
+        thread->status |= TS_USEDFPU;   /* So we fnsave on switch_to() */
+        tsk->fpu_counter++;
 }
 
 /*
@@ -846,17 +846,8 @@ asmlinkage void math_state_restore(void)
         }
 
         clts();                         /* Allow maths ops (or we recurse) */
-        /*
-         * Paranoid restore. send a SIGSEGV if we fail to restore the state.
-         */
-        if (unlikely(restore_fpu_checking(tsk))) {
-                stts();
-                force_sig(SIGSEGV, tsk);
-                return;
-        }
 
-        thread->status |= TS_USEDFPU;   /* So we fnsave on switch_to() */
-        tsk->fpu_counter++;
+        __math_state_restore();
 }
 EXPORT_SYMBOL_GPL(math_state_restore);
 
@@ -980,7 +971,5 @@ void __init trap_init(void)
          */
         cpu_init();
 
-#ifdef CONFIG_X86_32
-        x86_quirk_trap_init();
-#endif
+        x86_init.irqs.trap_init();
 }
diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c
index 71f4368b357e..cd982f48e23e 100644
--- a/arch/x86/kernel/tsc.c
+++ b/arch/x86/kernel/tsc.c
@@ -17,6 +17,8 @@
 #include <asm/time.h>
 #include <asm/delay.h>
 #include <asm/hypervisor.h>
+#include <asm/nmi.h>
+#include <asm/x86_init.h>
 
 unsigned int __read_mostly cpu_khz;     /* TSC clocks / usec, not used here */
 EXPORT_SYMBOL(cpu_khz);
@@ -400,15 +402,9 @@ unsigned long native_calibrate_tsc(void)
 {
         u64 tsc1, tsc2, delta, ref1, ref2;
         unsigned long tsc_pit_min = ULONG_MAX, tsc_ref_min = ULONG_MAX;
-        unsigned long flags, latch, ms, fast_calibrate, hv_tsc_khz;
+        unsigned long flags, latch, ms, fast_calibrate;
         int hpet = is_hpet_enabled(), i, loopmin;
 
-        hv_tsc_khz = get_hypervisor_tsc_freq();
-        if (hv_tsc_khz) {
-                printk(KERN_INFO "TSC: Frequency read from the hypervisor\n");
-                return hv_tsc_khz;
-        }
-
         local_irq_save(flags);
         fast_calibrate = quick_pit_calibrate();
         local_irq_restore(flags);
@@ -566,7 +562,7 @@ int recalibrate_cpu_khz(void)
         unsigned long cpu_khz_old = cpu_khz;
 
         if (cpu_has_tsc) {
-                tsc_khz = calibrate_tsc();
+                tsc_khz = x86_platform.calibrate_tsc();
                 cpu_khz = tsc_khz;
                 cpu_data(0).loops_per_jiffy =
                         cpufreq_scale(cpu_data(0).loops_per_jiffy,
@@ -670,7 +666,7 @@ static int time_cpufreq_notifier(struct notifier_block *nb, unsigned long val,
         if ((val == CPUFREQ_PRECHANGE  && freq->old < freq->new) ||
                         (val == CPUFREQ_POSTCHANGE && freq->old > freq->new) ||
                         (val == CPUFREQ_RESUMECHANGE)) {
-                *lpj =  cpufreq_scale(loops_per_jiffy_ref, ref_freq, freq->new);
+                *lpj = cpufreq_scale(loops_per_jiffy_ref, ref_freq, freq->new);
 
                 tsc_khz = cpufreq_scale(tsc_khz_ref, ref_freq, freq->new);
                 if (!(freq->flags & CPUFREQ_CONST_LOOPS))
@@ -744,10 +740,16 @@ static cycle_t __vsyscall_fn vread_tsc(void)
 }
 #endif
 
+static void resume_tsc(void)
+{
+        clocksource_tsc.cycle_last = 0;
+}
+
 static struct clocksource clocksource_tsc = {
         .name                   = "tsc",
         .rating                 = 300,
         .read                   = read_tsc,
+        .resume                 = resume_tsc,
         .mask                   = CLOCKSOURCE_MASK(64),
         .shift                  = 22,
         .flags                  = CLOCK_SOURCE_IS_CONTINUOUS |
@@ -761,12 +763,14 @@ void mark_tsc_unstable(char *reason)
 {
         if (!tsc_unstable) {
                 tsc_unstable = 1;
-                printk("Marking TSC unstable due to %s\n", reason);
+                printk(KERN_INFO "Marking TSC unstable due to %s\n", reason);
                 /* Change only the rating, when not registered */
                 if (clocksource_tsc.mult)
-                        clocksource_change_rating(&clocksource_tsc, 0);
-                else
+                        clocksource_mark_unstable(&clocksource_tsc);
+                else {
+                        clocksource_tsc.flags |= CLOCK_SOURCE_UNSTABLE;
                         clocksource_tsc.rating = 0;
+                }
         }
 }
 
@@ -852,15 +856,71 @@ static void __init init_tsc_clocksource(void)
         clocksource_register(&clocksource_tsc);
 }
 
+#ifdef CONFIG_X86_64
+/*
+ * calibrate_cpu is used on systems with fixed rate TSCs to determine
+ * processor frequency
+ */
+#define TICK_COUNT 100000000
+static unsigned long __init calibrate_cpu(void)
+{
+        int tsc_start, tsc_now;
+        int i, no_ctr_free;
+        unsigned long evntsel3 = 0, pmc3 = 0, pmc_now = 0;
+        unsigned long flags;
+
+        for (i = 0; i < 4; i++)
+                if (avail_to_resrv_perfctr_nmi_bit(i))
+                        break;
+        no_ctr_free = (i == 4);
+        if (no_ctr_free) {
+                WARN(1, KERN_WARNING "Warning: AMD perfctrs busy ... "
+                     "cpu_khz value may be incorrect.\n");
+                i = 3;
+                rdmsrl(MSR_K7_EVNTSEL3, evntsel3);
+                wrmsrl(MSR_K7_EVNTSEL3, 0);
+                rdmsrl(MSR_K7_PERFCTR3, pmc3);
+        } else {
+                reserve_perfctr_nmi(MSR_K7_PERFCTR0 + i);
+                reserve_evntsel_nmi(MSR_K7_EVNTSEL0 + i);
+        }
+        local_irq_save(flags);
+        /* start measuring cycles, incrementing from 0 */
+        wrmsrl(MSR_K7_PERFCTR0 + i, 0);
+        wrmsrl(MSR_K7_EVNTSEL0 + i, 1 << 22 | 3 << 16 | 0x76);
+        rdtscl(tsc_start);
+        do {
+                rdmsrl(MSR_K7_PERFCTR0 + i, pmc_now);
+                tsc_now = get_cycles();
+        } while ((tsc_now - tsc_start) < TICK_COUNT);
+
+        local_irq_restore(flags);
+        if (no_ctr_free) {
+                wrmsrl(MSR_K7_EVNTSEL3, 0);
+                wrmsrl(MSR_K7_PERFCTR3, pmc3);
+                wrmsrl(MSR_K7_EVNTSEL3, evntsel3);
+        } else {
+                release_perfctr_nmi(MSR_K7_PERFCTR0 + i);
+                release_evntsel_nmi(MSR_K7_EVNTSEL0 + i);
+        }
+
+        return pmc_now * tsc_khz / (tsc_now - tsc_start);
+}
+#else
+static inline unsigned long calibrate_cpu(void) { return cpu_khz; }
+#endif
+
 void __init tsc_init(void)
 {
         u64 lpj;
         int cpu;
 
+        x86_init.timers.tsc_pre_init();
+
         if (!cpu_has_tsc)
                 return;
 
-        tsc_khz = calibrate_tsc();
+        tsc_khz = x86_platform.calibrate_tsc();
         cpu_khz = tsc_khz;
 
         if (!tsc_khz) {
@@ -868,11 +928,9 @@ void __init tsc_init(void)
                 return;
         }
 
-#ifdef CONFIG_X86_64
         if (cpu_has(&boot_cpu_data, X86_FEATURE_CONSTANT_TSC) &&
                         (boot_cpu_data.x86_vendor == X86_VENDOR_AMD))
                 cpu_khz = calibrate_cpu();
-#endif
 
         printk("Detected %lu.%03lu MHz processor.\n",
                         (unsigned long)cpu_khz / 1000,
diff --git a/arch/x86/kernel/visws_quirks.c b/arch/x86/kernel/visws_quirks.c
index 31ffc24eec4d..f068553a1b17 100644
--- a/arch/x86/kernel/visws_quirks.c
+++ b/arch/x86/kernel/visws_quirks.c
@@ -30,6 +30,7 @@
 #include <asm/setup.h>
 #include <asm/apic.h>
 #include <asm/e820.h>
+#include <asm/time.h>
 #include <asm/io.h>
 
 #include <linux/kernel_stat.h>
@@ -53,7 +54,7 @@ int is_visws_box(void)
         return visws_board_type >= 0;
 }
 
-static int __init visws_time_init(void)
+static void __init visws_time_init(void)
 {
         printk(KERN_INFO "Starting Cobalt Timer system clock\n");
 
@@ -66,21 +67,13 @@ static int __init visws_time_init(void)
         /* Enable (unmask) the timer interrupt */
         co_cpu_write(CO_CPU_CTRL, co_cpu_read(CO_CPU_CTRL) & ~CO_CTRL_TIMEMASK);
 
-        /*
-         * Zero return means the generic timer setup code will set up
-         * the standard vector:
-         */
-        return 0;
+        setup_default_timer_irq();
 }
 
-static int __init visws_pre_intr_init(void)
+/* Replaces the default init_ISA_irqs in the generic setup */
+static void __init visws_pre_intr_init(void)
 {
         init_VISWS_APIC_irqs();
-
-        /*
-         * We dont want ISA irqs to be set up by the generic code:
-         */
-        return 1;
 }
 
 /* Quirk for machine specific memory setup. */
@@ -156,12 +149,8 @@ static void visws_machine_power_off(void)
         outl(PIIX_SPECIAL_STOP, 0xCFC);
 }
 
-static int __init visws_get_smp_config(unsigned int early)
+static void __init visws_get_smp_config(unsigned int early)
 {
-        /*
-         * Prevent MP-table parsing by the generic code:
-         */
-        return 1;
 }
 
 /*
@@ -208,7 +197,7 @@ static void __init MP_processor_info(struct mpc_cpu *m)
         apic_version[m->apicid] = ver;
 }
 
-static int __init visws_find_smp_config(unsigned int reserve)
+static void __init visws_find_smp_config(unsigned int reserve)
 {
         struct mpc_cpu *mp = phys_to_virt(CO_CPU_TAB_PHYS);
         unsigned short ncpus = readw(phys_to_virt(CO_CPU_NUM_PHYS));
@@ -230,21 +219,9 @@ static int __init visws_find_smp_config(unsigned int reserve)
                 MP_processor_info(mp++);
 
         mp_lapic_addr = APIC_DEFAULT_PHYS_BASE;
-
-        return 1;
 }
 
-static int visws_trap_init(void);
-
-static struct x86_quirks visws_x86_quirks __initdata = {
-        .arch_time_init         = visws_time_init,
-        .arch_pre_intr_init     = visws_pre_intr_init,
-        .arch_memory_setup      = visws_memory_setup,
-        .arch_intr_init         = NULL,
-        .arch_trap_init         = visws_trap_init,
-        .mach_get_smp_config    = visws_get_smp_config,
-        .mach_find_smp_config   = visws_find_smp_config,
-};
+static void visws_trap_init(void);
 
 void __init visws_early_detect(void)
 {
@@ -257,11 +234,14 @@ void __init visws_early_detect(void)
                 return;
 
         /*
-         * Install special quirks for timer, interrupt and memory setup:
-         * Fall back to generic behavior for traps:
-         * Override generic MP-table parsing:
+         * Override the default platform setup functions
          */
-        x86_quirks = &visws_x86_quirks;
+        x86_init.resources.memory_setup = visws_memory_setup;
+        x86_init.mpparse.get_smp_config = visws_get_smp_config;
+        x86_init.mpparse.find_smp_config = visws_find_smp_config;
+        x86_init.irqs.pre_vector_init = visws_pre_intr_init;
+        x86_init.irqs.trap_init = visws_trap_init;
+        x86_init.timers.timer_init = visws_time_init;
 
         /*
          * Install reboot quirks:
@@ -400,12 +380,10 @@ static __init void cobalt_init(void)
                 co_apic_read(CO_APIC_ID));
 }
 
-static int __init visws_trap_init(void)
+static void __init visws_trap_init(void)
 {
         lithium_init();
         cobalt_init();
-
-        return 1;
 }
 
 /*
diff --git a/arch/x86/kernel/vmi_32.c b/arch/x86/kernel/vmi_32.c
index 95a7289e4b0c..31e6f6cfe53e 100644
--- a/arch/x86/kernel/vmi_32.c
+++ b/arch/x86/kernel/vmi_32.c
@@ -817,15 +817,15 @@ static inline int __init activate_vmi(void)
                 vmi_timer_ops.set_alarm = vmi_get_function(VMI_CALL_SetAlarm);
                 vmi_timer_ops.cancel_alarm =
                          vmi_get_function(VMI_CALL_CancelAlarm);
-                pv_time_ops.time_init = vmi_time_init;
-                pv_time_ops.get_wallclock = vmi_get_wallclock;
-                pv_time_ops.set_wallclock = vmi_set_wallclock;
+                x86_init.timers.timer_init = vmi_time_init;
 #ifdef CONFIG_X86_LOCAL_APIC
-                pv_apic_ops.setup_boot_clock = vmi_time_bsp_init;
-                pv_apic_ops.setup_secondary_clock = vmi_time_ap_init;
+                x86_init.timers.setup_percpu_clockev = vmi_time_bsp_init;
+                x86_cpuinit.setup_percpu_clockev = vmi_time_ap_init;
 #endif
                 pv_time_ops.sched_clock = vmi_sched_clock;
-                pv_time_ops.get_tsc_khz = vmi_tsc_khz;
+                x86_platform.calibrate_tsc = vmi_tsc_khz;
+                x86_platform.get_wallclock = vmi_get_wallclock;
+                x86_platform.set_wallclock = vmi_set_wallclock;
 
                 /* We have true wallclock functions; disable CMOS clock sync */
                 no_sync_cmos_clock = 1;
diff --git a/arch/x86/kernel/vmiclock_32.c b/arch/x86/kernel/vmiclock_32.c
index 2b3eb82efeeb..611b9e2360d3 100644
--- a/arch/x86/kernel/vmiclock_32.c
+++ b/arch/x86/kernel/vmiclock_32.c
@@ -68,7 +68,7 @@ unsigned long long vmi_sched_clock(void)
         return cycles_2_ns(vmi_timer_ops.get_cycle_counter(VMI_CYCLES_AVAILABLE));
 }
 
-/* paravirt_ops.get_tsc_khz = vmi_tsc_khz */
+/* x86_platform.calibrate_tsc = vmi_tsc_khz */
 unsigned long vmi_tsc_khz(void)
 {
         unsigned long long khz;
diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S
index 9fc178255c04..a46acccec38a 100644
--- a/arch/x86/kernel/vmlinux.lds.S
+++ b/arch/x86/kernel/vmlinux.lds.S
@@ -45,9 +45,9 @@ PHDRS {
         text PT_LOAD FLAGS(5);          /* R_E */
         data PT_LOAD FLAGS(7);          /* RWE */
 #ifdef CONFIG_X86_64
-        user PT_LOAD FLAGS(7);          /* RWE */
+        user PT_LOAD FLAGS(5);          /* R_E */
 #ifdef CONFIG_SMP
-        percpu PT_LOAD FLAGS(7);        /* RWE */
+        percpu PT_LOAD FLAGS(6);        /* RW_ */
 #endif
         init PT_LOAD FLAGS(7);          /* RWE */
 #endif
@@ -348,15 +348,12 @@ SECTIONS
                 _end = .;
         }
 
-        /* Sections to be discarded */
-        /DISCARD/ : {
-                *(.exitcall.exit)
-                *(.eh_frame)
-                *(.discard)
-        }
-
         STABS_DEBUG
         DWARF_DEBUG
+
+        /* Sections to be discarded */
+        DISCARDS
+        /DISCARD/ : { *(.eh_frame) }
 }
 
 
diff --git a/arch/x86/kernel/vsyscall_64.c b/arch/x86/kernel/vsyscall_64.c
index 25ee06a80aad..8cb4974ff599 100644
--- a/arch/x86/kernel/vsyscall_64.c
+++ b/arch/x86/kernel/vsyscall_64.c
@@ -87,6 +87,7 @@ void update_vsyscall(struct timespec *wall_time, struct clocksource *clock)
         vsyscall_gtod_data.wall_time_sec = wall_time->tv_sec;
         vsyscall_gtod_data.wall_time_nsec = wall_time->tv_nsec;
         vsyscall_gtod_data.wall_to_monotonic = wall_to_monotonic;
+        vsyscall_gtod_data.wall_time_coarse = __current_kernel_time();
         write_sequnlock_irqrestore(&vsyscall_gtod_data.lock, flags);
 }
 
@@ -227,19 +228,11 @@ static long __vsyscall(3) venosys_1(void)
 }
 
 #ifdef CONFIG_SYSCTL
-
-static int
-vsyscall_sysctl_change(ctl_table *ctl, int write, struct file * filp,
-                       void __user *buffer, size_t *lenp, loff_t *ppos)
-{
-        return proc_dointvec(ctl, write, filp, buffer, lenp, ppos);
-}
-
 static ctl_table kernel_table2[] = {
         { .procname = "vsyscall64",
           .data = &vsyscall_gtod_data.sysctl_enabled, .maxlen = sizeof(int),
           .mode = 0644,
-          .proc_handler = vsyscall_sysctl_change },
+          .proc_handler = proc_dointvec },
         {}
 };
 
diff --git a/arch/x86/kernel/x86_init.c b/arch/x86/kernel/x86_init.c
new file mode 100644
index 000000000000..4449a4a2c2ed
--- /dev/null
+++ b/arch/x86/kernel/x86_init.c
@@ -0,0 +1,75 @@
+/*
+ * Copyright (C) 2009 Thomas Gleixner <tglx@linutronix.de>
+ *
+ *  For licencing details see kernel-base/COPYING
+ */
+#include <linux/init.h>
+
+#include <asm/bios_ebda.h>
+#include <asm/paravirt.h>
+#include <asm/mpspec.h>
+#include <asm/setup.h>
+#include <asm/apic.h>
+#include <asm/e820.h>
+#include <asm/time.h>
+#include <asm/irq.h>
+#include <asm/tsc.h>
+
+void __cpuinit x86_init_noop(void) { }
+void __init x86_init_uint_noop(unsigned int unused) { }
+void __init x86_init_pgd_noop(pgd_t *unused) { }
+
+/*
+ * The platform setup functions are preset with the default functions
+ * for standard PC hardware.
+ */
+struct x86_init_ops x86_init __initdata = {
+
+        .resources = {
+                .probe_roms             = x86_init_noop,
+                .reserve_resources      = reserve_standard_io_resources,
+                .memory_setup           = default_machine_specific_memory_setup,
+        },
+
+        .mpparse = {
+                .mpc_record             = x86_init_uint_noop,
+                .setup_ioapic_ids       = x86_init_noop,
+                .mpc_apic_id            = default_mpc_apic_id,
+                .smp_read_mpc_oem       = default_smp_read_mpc_oem,
+                .mpc_oem_bus_info       = default_mpc_oem_bus_info,
+                .find_smp_config        = default_find_smp_config,
+                .get_smp_config         = default_get_smp_config,
+        },
+
+        .irqs = {
+                .pre_vector_init        = init_ISA_irqs,
+                .intr_init              = native_init_IRQ,
+                .trap_init              = x86_init_noop,
+        },
+
+        .oem = {
+                .arch_setup             = x86_init_noop,
+                .banner                 = default_banner,
+        },
+
+        .paging = {
+                .pagetable_setup_start  = native_pagetable_setup_start,
+                .pagetable_setup_done   = native_pagetable_setup_done,
+        },
+
+        .timers = {
+                .setup_percpu_clockev   = setup_boot_APIC_clock,
+                .tsc_pre_init           = x86_init_noop,
+                .timer_init             = hpet_time_init,
+        },
+};
+
+struct x86_cpuinit_ops x86_cpuinit __cpuinitdata = {
+        .setup_percpu_clockev           = setup_secondary_APIC_clock,
+};
+
+struct x86_platform_ops x86_platform = {
+        .calibrate_tsc                  = native_calibrate_tsc,
+        .get_wallclock                  = mach_get_cmos_time,
+        .set_wallclock                  = mach_set_rtc_mmss,
+};
diff --git a/arch/x86/kvm/Kconfig b/arch/x86/kvm/Kconfig
index 8600a09e0c6c..b84e571f4175 100644
--- a/arch/x86/kvm/Kconfig
+++ b/arch/x86/kvm/Kconfig
@@ -1,12 +1,8 @@
 #
 # KVM configuration
 #
-config HAVE_KVM
-       bool
 
-config HAVE_KVM_IRQCHIP
-       bool
-       default y
+source "virt/kvm/Kconfig"
 
 menuconfig VIRTUALIZATION
         bool "Virtualization"
@@ -29,6 +25,9 @@ config KVM
         select PREEMPT_NOTIFIERS
         select MMU_NOTIFIER
         select ANON_INODES
+        select HAVE_KVM_IRQCHIP
+        select HAVE_KVM_EVENTFD
+        select KVM_APIC_ARCHITECTURE
         ---help---
           Support hosting fully virtualized guest machines using hardware
           virtualization extensions.  You will need a fairly recent
@@ -63,18 +62,6 @@ config KVM_AMD
           To compile this as a module, choose M here: the module
           will be called kvm-amd.
 
-config KVM_TRACE
-        bool "KVM trace support"
-        depends on KVM && SYSFS
-        select MARKERS
-        select RELAY
-        select DEBUG_FS
-        default n
-        ---help---
-          This option allows reading a trace of kvm-related events through
-          relayfs.  Note the ABI is not considered stable and will be
-          modified in future updates.
-
 # OK, it's a little counter-intuitive to do this, but it puts it neatly under
 # the virtualization menu.
 source drivers/lguest/Kconfig
diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile
index b43c4efafe80..0e7fe78d0f74 100644
--- a/arch/x86/kvm/Makefile
+++ b/arch/x86/kvm/Makefile
@@ -1,22 +1,19 @@
-#
-# Makefile for Kernel-based Virtual Machine module
-#
-
-common-objs = $(addprefix ../../../virt/kvm/, kvm_main.o ioapic.o \
-                coalesced_mmio.o irq_comm.o)
-ifeq ($(CONFIG_KVM_TRACE),y)
-common-objs += $(addprefix ../../../virt/kvm/, kvm_trace.o)
-endif
-ifeq ($(CONFIG_IOMMU_API),y)
-common-objs += $(addprefix ../../../virt/kvm/, iommu.o)
-endif
 
 EXTRA_CFLAGS += -Ivirt/kvm -Iarch/x86/kvm
 
-kvm-objs := $(common-objs) x86.o mmu.o x86_emulate.o i8259.o irq.o lapic.o \
-        i8254.o timer.o
-obj-$(CONFIG_KVM) += kvm.o
-kvm-intel-objs = vmx.o
-obj-$(CONFIG_KVM_INTEL) += kvm-intel.o
-kvm-amd-objs = svm.o
-obj-$(CONFIG_KVM_AMD) += kvm-amd.o
+CFLAGS_x86.o := -I.
+CFLAGS_svm.o := -I.
+CFLAGS_vmx.o := -I.
+
+kvm-y                   += $(addprefix ../../../virt/kvm/, kvm_main.o ioapic.o \
+                                coalesced_mmio.o irq_comm.o eventfd.o)
+kvm-$(CONFIG_IOMMU_API) += $(addprefix ../../../virt/kvm/, iommu.o)
+
+kvm-y                   += x86.o mmu.o emulate.o i8259.o irq.o lapic.o \
+                           i8254.o timer.o
+kvm-intel-y             += vmx.o
+kvm-amd-y               += svm.o
+
+obj-$(CONFIG_KVM)       += kvm.o
+obj-$(CONFIG_KVM_INTEL) += kvm-intel.o
+obj-$(CONFIG_KVM_AMD)   += kvm-amd.o
diff --git a/arch/x86/kvm/x86_emulate.c b/arch/x86/kvm/emulate.c
index 616de4628d60..1be5cd640e93 100644
--- a/arch/x86/kvm/x86_emulate.c
+++ b/arch/x86/kvm/emulate.c
@@ -1,5 +1,5 @@
 /******************************************************************************
- * x86_emulate.c
+ * emulate.c
  *
  * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
  *
@@ -30,7 +30,9 @@
 #define DPRINTF(x...) do {} while (0)
 #endif
 #include <linux/module.h>
-#include <asm/kvm_x86_emulate.h>
+#include <asm/kvm_emulate.h>
+
+#include "mmu.h"                /* for is_long_mode() */
 
 /*
  * Opcode effective-address decode tables.
@@ -60,6 +62,7 @@
 #define SrcImmByte  (6<<4)      /* 8-bit sign-extended immediate operand. */
 #define SrcOne      (7<<4)      /* Implied '1' */
 #define SrcImmUByte (8<<4)      /* 8-bit unsigned immediate operand. */
+#define SrcImmU     (9<<4)      /* Immediate operand, unsigned */
 #define SrcMask     (0xf<<4)
 /* Generic ModRM decode. */
 #define ModRM       (1<<8)
@@ -97,11 +100,11 @@ static u32 opcode_table[256] = {
         /* 0x10 - 0x17 */
         ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
         ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
-        0, 0, 0, 0,
+        ByteOp | DstAcc | SrcImm, DstAcc | SrcImm, 0, 0,
         /* 0x18 - 0x1F */
         ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
         ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
-        0, 0, 0, 0,
+        ByteOp | DstAcc | SrcImm, DstAcc | SrcImm, 0, 0,
         /* 0x20 - 0x27 */
         ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
         ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
@@ -195,7 +198,7 @@ static u32 opcode_table[256] = {
         ByteOp | SrcImmUByte, SrcImmUByte,
         /* 0xE8 - 0xEF */
         SrcImm | Stack, SrcImm | ImplicitOps,
-        SrcImm | Src2Imm16, SrcImmByte | ImplicitOps,
+        SrcImmU | Src2Imm16, SrcImmByte | ImplicitOps,
         SrcNone | ByteOp | ImplicitOps, SrcNone | ImplicitOps,
         SrcNone | ByteOp | ImplicitOps, SrcNone | ImplicitOps,
         /* 0xF0 - 0xF7 */
@@ -208,7 +211,7 @@ static u32 opcode_table[256] = {
 
 static u32 twobyte_table[256] = {
         /* 0x00 - 0x0F */
-        0, Group | GroupDual | Group7, 0, 0, 0, 0, ImplicitOps, 0,
+        0, Group | GroupDual | Group7, 0, 0, 0, ImplicitOps, ImplicitOps, 0,
         ImplicitOps, ImplicitOps, 0, 0, 0, ImplicitOps | ModRM, 0, 0,
         /* 0x10 - 0x1F */
         0, 0, 0, 0, 0, 0, 0, 0, ImplicitOps | ModRM, 0, 0, 0, 0, 0, 0, 0,
@@ -216,7 +219,9 @@ static u32 twobyte_table[256] = {
         ModRM | ImplicitOps, ModRM, ModRM | ImplicitOps, ModRM, 0, 0, 0, 0,
         0, 0, 0, 0, 0, 0, 0, 0,
         /* 0x30 - 0x3F */
-        ImplicitOps, 0, ImplicitOps, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        ImplicitOps, 0, ImplicitOps, 0,
+        ImplicitOps, ImplicitOps, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0,
         /* 0x40 - 0x47 */
         DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
         DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
@@ -319,8 +324,11 @@ static u32 group2_table[] = {
 };
 
 /* EFLAGS bit definitions. */
+#define EFLG_VM (1<<17)
+#define EFLG_RF (1<<16)
 #define EFLG_OF (1<<11)
 #define EFLG_DF (1<<10)
+#define EFLG_IF (1<<9)
 #define EFLG_SF (1<<7)
 #define EFLG_ZF (1<<6)
 #define EFLG_AF (1<<4)
@@ -1027,6 +1035,7 @@ done_prefixes:
                 c->src.type = OP_MEM;
                 break;
         case SrcImm:
+        case SrcImmU:
                 c->src.type = OP_IMM;
                 c->src.ptr = (unsigned long *)c->eip;
                 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
@@ -1044,6 +1053,19 @@ done_prefixes:
                         c->src.val = insn_fetch(s32, 4, c->eip);
                         break;
                 }
+                if ((c->d & SrcMask) == SrcImmU) {
+                        switch (c->src.bytes) {
+                        case 1:
+                                c->src.val &= 0xff;
+                                break;
+                        case 2:
+                                c->src.val &= 0xffff;
+                                break;
+                        case 4:
+                                c->src.val &= 0xffffffff;
+                                break;
+                        }
+                }
                 break;
         case SrcImmByte:
         case SrcImmUByte:
@@ -1375,6 +1397,217 @@ static void toggle_interruptibility(struct x86_emulate_ctxt *ctxt, u32 mask)
                 ctxt->interruptibility = mask;
 }
 
+static inline void
+setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
+        struct kvm_segment *cs, struct kvm_segment *ss)
+{
+        memset(cs, 0, sizeof(struct kvm_segment));
+        kvm_x86_ops->get_segment(ctxt->vcpu, cs, VCPU_SREG_CS);
+        memset(ss, 0, sizeof(struct kvm_segment));
+
+        cs->l = 0;              /* will be adjusted later */
+        cs->base = 0;           /* flat segment */
+        cs->g = 1;              /* 4kb granularity */
+        cs->limit = 0xffffffff; /* 4GB limit */
+        cs->type = 0x0b;        /* Read, Execute, Accessed */
+        cs->s = 1;
+        cs->dpl = 0;            /* will be adjusted later */
+        cs->present = 1;
+        cs->db = 1;
+
+        ss->unusable = 0;
+        ss->base = 0;           /* flat segment */
+        ss->limit = 0xffffffff; /* 4GB limit */
+        ss->g = 1;              /* 4kb granularity */
+        ss->s = 1;
+        ss->type = 0x03;        /* Read/Write, Accessed */
+        ss->db = 1;             /* 32bit stack segment */
+        ss->dpl = 0;
+        ss->present = 1;
+}
+
+static int
+emulate_syscall(struct x86_emulate_ctxt *ctxt)
+{
+        struct decode_cache *c = &ctxt->decode;
+        struct kvm_segment cs, ss;
+        u64 msr_data;
+
+        /* syscall is not available in real mode */
+        if (c->lock_prefix || ctxt->mode == X86EMUL_MODE_REAL
+                || !(ctxt->vcpu->arch.cr0 & X86_CR0_PE))
+                return -1;
+
+        setup_syscalls_segments(ctxt, &cs, &ss);
+
+        kvm_x86_ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
+        msr_data >>= 32;
+        cs.selector = (u16)(msr_data & 0xfffc);
+        ss.selector = (u16)(msr_data + 8);
+
+        if (is_long_mode(ctxt->vcpu)) {
+                cs.db = 0;
+                cs.l = 1;
+        }
+        kvm_x86_ops->set_segment(ctxt->vcpu, &cs, VCPU_SREG_CS);
+        kvm_x86_ops->set_segment(ctxt->vcpu, &ss, VCPU_SREG_SS);
+
+        c->regs[VCPU_REGS_RCX] = c->eip;
+        if (is_long_mode(ctxt->vcpu)) {
+#ifdef CONFIG_X86_64
+                c->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
+
+                kvm_x86_ops->get_msr(ctxt->vcpu,
+                        ctxt->mode == X86EMUL_MODE_PROT64 ?
+                        MSR_LSTAR : MSR_CSTAR, &msr_data);
+                c->eip = msr_data;
+
+                kvm_x86_ops->get_msr(ctxt->vcpu, MSR_SYSCALL_MASK, &msr_data);
+                ctxt->eflags &= ~(msr_data | EFLG_RF);
+#endif
+        } else {
+                /* legacy mode */
+                kvm_x86_ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
+                c->eip = (u32)msr_data;
+
+                ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
+        }
+
+        return 0;
+}
+
+static int
+emulate_sysenter(struct x86_emulate_ctxt *ctxt)
+{
+        struct decode_cache *c = &ctxt->decode;
+        struct kvm_segment cs, ss;
+        u64 msr_data;
+
+        /* inject #UD if LOCK prefix is used */
+        if (c->lock_prefix)
+                return -1;
+
+        /* inject #GP if in real mode or paging is disabled */
+        if (ctxt->mode == X86EMUL_MODE_REAL ||
+                !(ctxt->vcpu->arch.cr0 & X86_CR0_PE)) {
+                kvm_inject_gp(ctxt->vcpu, 0);
+                return -1;
+        }
+
+        /* XXX sysenter/sysexit have not been tested in 64bit mode.
+        * Therefore, we inject an #UD.
+        */
+        if (ctxt->mode == X86EMUL_MODE_PROT64)
+                return -1;
+
+        setup_syscalls_segments(ctxt, &cs, &ss);
+
+        kvm_x86_ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
+        switch (ctxt->mode) {
+        case X86EMUL_MODE_PROT32:
+                if ((msr_data & 0xfffc) == 0x0) {
+                        kvm_inject_gp(ctxt->vcpu, 0);
+                        return -1;
+                }
+                break;
+        case X86EMUL_MODE_PROT64:
+                if (msr_data == 0x0) {
+                        kvm_inject_gp(ctxt->vcpu, 0);
+                        return -1;
+                }
+                break;
+        }
+
+        ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
+        cs.selector = (u16)msr_data;
+        cs.selector &= ~SELECTOR_RPL_MASK;
+        ss.selector = cs.selector + 8;
+        ss.selector &= ~SELECTOR_RPL_MASK;
+        if (ctxt->mode == X86EMUL_MODE_PROT64
+                || is_long_mode(ctxt->vcpu)) {
+                cs.db = 0;
+                cs.l = 1;
+        }
+
+        kvm_x86_ops->set_segment(ctxt->vcpu, &cs, VCPU_SREG_CS);
+        kvm_x86_ops->set_segment(ctxt->vcpu, &ss, VCPU_SREG_SS);
+
+        kvm_x86_ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_EIP, &msr_data);
+        c->eip = msr_data;
+
+        kvm_x86_ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_ESP, &msr_data);
+        c->regs[VCPU_REGS_RSP] = msr_data;
+
+        return 0;
+}
+
+static int
+emulate_sysexit(struct x86_emulate_ctxt *ctxt)
+{
+        struct decode_cache *c = &ctxt->decode;
+        struct kvm_segment cs, ss;
+        u64 msr_data;
+        int usermode;
+
+        /* inject #UD if LOCK prefix is used */
+        if (c->lock_prefix)
+                return -1;
+
+        /* inject #GP if in real mode or paging is disabled */
+        if (ctxt->mode == X86EMUL_MODE_REAL
+                || !(ctxt->vcpu->arch.cr0 & X86_CR0_PE)) {
+                kvm_inject_gp(ctxt->vcpu, 0);
+                return -1;
+        }
+
+        /* sysexit must be called from CPL 0 */
+        if (kvm_x86_ops->get_cpl(ctxt->vcpu) != 0) {
+                kvm_inject_gp(ctxt->vcpu, 0);
+                return -1;
+        }
+
+        setup_syscalls_segments(ctxt, &cs, &ss);
+
+        if ((c->rex_prefix & 0x8) != 0x0)
+                usermode = X86EMUL_MODE_PROT64;
+        else
+                usermode = X86EMUL_MODE_PROT32;
+
+        cs.dpl = 3;
+        ss.dpl = 3;
+        kvm_x86_ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
+        switch (usermode) {
+        case X86EMUL_MODE_PROT32:
+                cs.selector = (u16)(msr_data + 16);
+                if ((msr_data & 0xfffc) == 0x0) {
+                        kvm_inject_gp(ctxt->vcpu, 0);
+                        return -1;
+                }
+                ss.selector = (u16)(msr_data + 24);
+                break;
+        case X86EMUL_MODE_PROT64:
+                cs.selector = (u16)(msr_data + 32);
+                if (msr_data == 0x0) {
+                        kvm_inject_gp(ctxt->vcpu, 0);
+                        return -1;
+                }
+                ss.selector = cs.selector + 8;
+                cs.db = 0;
+                cs.l = 1;
+                break;
+        }
+        cs.selector |= SELECTOR_RPL_MASK;
+        ss.selector |= SELECTOR_RPL_MASK;
+
+        kvm_x86_ops->set_segment(ctxt->vcpu, &cs, VCPU_SREG_CS);
+        kvm_x86_ops->set_segment(ctxt->vcpu, &ss, VCPU_SREG_SS);
+
+        c->eip = ctxt->vcpu->arch.regs[VCPU_REGS_RDX];
+        c->regs[VCPU_REGS_RSP] = ctxt->vcpu->arch.regs[VCPU_REGS_RCX];
+
+        return 0;
+}
+
 int
 x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
 {
@@ -1970,6 +2203,12 @@ twobyte_insn:
                         goto cannot_emulate;
                 }
                 break;
+        case 0x05:              /* syscall */
+                if (emulate_syscall(ctxt) == -1)
+                        goto cannot_emulate;
+                else
+                        goto writeback;
+                break;
         case 0x06:
                 emulate_clts(ctxt->vcpu);
                 c->dst.type = OP_NONE;
@@ -2036,6 +2275,18 @@ twobyte_insn:
                 rc = X86EMUL_CONTINUE;
                 c->dst.type = OP_NONE;
                 break;
+        case 0x34:              /* sysenter */
+                if (emulate_sysenter(ctxt) == -1)
+                        goto cannot_emulate;
+                else
+                        goto writeback;
+                break;
+        case 0x35:              /* sysexit */
+                if (emulate_sysexit(ctxt) == -1)
+                        goto cannot_emulate;
+                else
+                        goto writeback;
+                break;
         case 0x40 ... 0x4f:     /* cmov */
                 c->dst.val = c->dst.orig_val = c->src.val;
                 if (!test_cc(c->b, ctxt->eflags))
diff --git a/arch/x86/kvm/i8254.c b/arch/x86/kvm/i8254.c
index 21f68e00524f..82ad523b4901 100644
--- a/arch/x86/kvm/i8254.c
+++ b/arch/x86/kvm/i8254.c
@@ -231,7 +231,7 @@ int pit_has_pending_timer(struct kvm_vcpu *vcpu)
 {
         struct kvm_pit *pit = vcpu->kvm->arch.vpit;
 
-        if (pit && vcpu->vcpu_id == 0 && pit->pit_state.irq_ack)
+        if (pit && kvm_vcpu_is_bsp(vcpu) && pit->pit_state.irq_ack)
                 return atomic_read(&pit->pit_state.pit_timer.pending);
         return 0;
 }
@@ -252,7 +252,7 @@ void __kvm_migrate_pit_timer(struct kvm_vcpu *vcpu)
         struct kvm_pit *pit = vcpu->kvm->arch.vpit;
         struct hrtimer *timer;
 
-        if (vcpu->vcpu_id != 0 || !pit)
+        if (!kvm_vcpu_is_bsp(vcpu) || !pit)
                 return;
 
         timer = &pit->pit_state.pit_timer.timer;
@@ -294,7 +294,7 @@ static void create_pit_timer(struct kvm_kpit_state *ps, u32 val, int is_period)
         pt->timer.function = kvm_timer_fn;
         pt->t_ops = &kpit_ops;
         pt->kvm = ps->pit->kvm;
-        pt->vcpu_id = 0;
+        pt->vcpu = pt->kvm->bsp_vcpu;
 
         atomic_set(&pt->pending, 0);
         ps->irq_ack = 1;
@@ -332,33 +332,62 @@ static void pit_load_count(struct kvm *kvm, int channel, u32 val)
         case 1:
         /* FIXME: enhance mode 4 precision */
         case 4:
-                create_pit_timer(ps, val, 0);
+                if (!(ps->flags & KVM_PIT_FLAGS_HPET_LEGACY)) {
+                        create_pit_timer(ps, val, 0);
+                }
                 break;
         case 2:
         case 3:
-                create_pit_timer(ps, val, 1);
+                if (!(ps->flags & KVM_PIT_FLAGS_HPET_LEGACY)){
+                        create_pit_timer(ps, val, 1);
+                }
                 break;
         default:
                 destroy_pit_timer(&ps->pit_timer);
         }
 }
 
-void kvm_pit_load_count(struct kvm *kvm, int channel, u32 val)
+void kvm_pit_load_count(struct kvm *kvm, int channel, u32 val, int hpet_legacy_start)
+{
+        u8 saved_mode;
+        if (hpet_legacy_start) {
+                /* save existing mode for later reenablement */
+                saved_mode = kvm->arch.vpit->pit_state.channels[0].mode;
+                kvm->arch.vpit->pit_state.channels[0].mode = 0xff; /* disable timer */
+                pit_load_count(kvm, channel, val);
+                kvm->arch.vpit->pit_state.channels[0].mode = saved_mode;
+        } else {
+                pit_load_count(kvm, channel, val);
+        }
+}
+
+static inline struct kvm_pit *dev_to_pit(struct kvm_io_device *dev)
+{
+        return container_of(dev, struct kvm_pit, dev);
+}
+
+static inline struct kvm_pit *speaker_to_pit(struct kvm_io_device *dev)
 {
-        mutex_lock(&kvm->arch.vpit->pit_state.lock);
-        pit_load_count(kvm, channel, val);
-        mutex_unlock(&kvm->arch.vpit->pit_state.lock);
+        return container_of(dev, struct kvm_pit, speaker_dev);
 }
 
-static void pit_ioport_write(struct kvm_io_device *this,
-                             gpa_t addr, int len, const void *data)
+static inline int pit_in_range(gpa_t addr)
 {
-        struct kvm_pit *pit = (struct kvm_pit *)this->private;
+        return ((addr >= KVM_PIT_BASE_ADDRESS) &&
+                (addr < KVM_PIT_BASE_ADDRESS + KVM_PIT_MEM_LENGTH));
+}
+
+static int pit_ioport_write(struct kvm_io_device *this,
+                            gpa_t addr, int len, const void *data)
+{
+        struct kvm_pit *pit = dev_to_pit(this);
         struct kvm_kpit_state *pit_state = &pit->pit_state;
         struct kvm *kvm = pit->kvm;
         int channel, access;
         struct kvm_kpit_channel_state *s;
         u32 val = *(u32 *) data;
+        if (!pit_in_range(addr))
+                return -EOPNOTSUPP;
 
         val  &= 0xff;
         addr &= KVM_PIT_CHANNEL_MASK;
@@ -421,16 +450,19 @@ static void pit_ioport_write(struct kvm_io_device *this,
         }
 
         mutex_unlock(&pit_state->lock);
+        return 0;
 }
 
-static void pit_ioport_read(struct kvm_io_device *this,
-                            gpa_t addr, int len, void *data)
+static int pit_ioport_read(struct kvm_io_device *this,
+                           gpa_t addr, int len, void *data)
 {
-        struct kvm_pit *pit = (struct kvm_pit *)this->private;
+        struct kvm_pit *pit = dev_to_pit(this);
         struct kvm_kpit_state *pit_state = &pit->pit_state;
         struct kvm *kvm = pit->kvm;
         int ret, count;
         struct kvm_kpit_channel_state *s;
+        if (!pit_in_range(addr))
+                return -EOPNOTSUPP;
 
         addr &= KVM_PIT_CHANNEL_MASK;
         s = &pit_state->channels[addr];
@@ -485,37 +517,36 @@ static void pit_ioport_read(struct kvm_io_device *this,
         memcpy(data, (char *)&ret, len);
 
         mutex_unlock(&pit_state->lock);
+        return 0;
 }
 
-static int pit_in_range(struct kvm_io_device *this, gpa_t addr,
-                        int len, int is_write)
-{
-        return ((addr >= KVM_PIT_BASE_ADDRESS) &&
-                (addr < KVM_PIT_BASE_ADDRESS + KVM_PIT_MEM_LENGTH));
-}
-
-static void speaker_ioport_write(struct kvm_io_device *this,
-                                 gpa_t addr, int len, const void *data)
+static int speaker_ioport_write(struct kvm_io_device *this,
+                                gpa_t addr, int len, const void *data)
 {
-        struct kvm_pit *pit = (struct kvm_pit *)this->private;
+        struct kvm_pit *pit = speaker_to_pit(this);
         struct kvm_kpit_state *pit_state = &pit->pit_state;
         struct kvm *kvm = pit->kvm;
         u32 val = *(u32 *) data;
+        if (addr != KVM_SPEAKER_BASE_ADDRESS)
+                return -EOPNOTSUPP;
 
         mutex_lock(&pit_state->lock);
         pit_state->speaker_data_on = (val >> 1) & 1;
         pit_set_gate(kvm, 2, val & 1);
         mutex_unlock(&pit_state->lock);
+        return 0;
 }
 
-static void speaker_ioport_read(struct kvm_io_device *this,
-                                gpa_t addr, int len, void *data)
+static int speaker_ioport_read(struct kvm_io_device *this,
+                               gpa_t addr, int len, void *data)
 {
-        struct kvm_pit *pit = (struct kvm_pit *)this->private;
+        struct kvm_pit *pit = speaker_to_pit(this);
         struct kvm_kpit_state *pit_state = &pit->pit_state;
         struct kvm *kvm = pit->kvm;
         unsigned int refresh_clock;
         int ret;
+        if (addr != KVM_SPEAKER_BASE_ADDRESS)
+                return -EOPNOTSUPP;
 
         /* Refresh clock toggles at about 15us. We approximate as 2^14ns. */
         refresh_clock = ((unsigned int)ktime_to_ns(ktime_get()) >> 14) & 1;
@@ -527,12 +558,7 @@ static void speaker_ioport_read(struct kvm_io_device *this,
                 len = sizeof(ret);
         memcpy(data, (char *)&ret, len);
         mutex_unlock(&pit_state->lock);
-}
-
-static int speaker_in_range(struct kvm_io_device *this, gpa_t addr,
-                            int len, int is_write)
-{
-        return (addr == KVM_SPEAKER_BASE_ADDRESS);
+        return 0;
 }
 
 void kvm_pit_reset(struct kvm_pit *pit)
@@ -541,6 +567,7 @@ void kvm_pit_reset(struct kvm_pit *pit)
         struct kvm_kpit_channel_state *c;
 
         mutex_lock(&pit->pit_state.lock);
+        pit->pit_state.flags = 0;
         for (i = 0; i < 3; i++) {
                 c = &pit->pit_state.channels[i];
                 c->mode = 0xff;
@@ -563,10 +590,22 @@ static void pit_mask_notifer(struct kvm_irq_mask_notifier *kimn, bool mask)
         }
 }
 
-struct kvm_pit *kvm_create_pit(struct kvm *kvm)
+static const struct kvm_io_device_ops pit_dev_ops = {
+        .read     = pit_ioport_read,
+        .write    = pit_ioport_write,
+};
+
+static const struct kvm_io_device_ops speaker_dev_ops = {
+        .read     = speaker_ioport_read,
+        .write    = speaker_ioport_write,
+};
+
+/* Caller must have writers lock on slots_lock */
+struct kvm_pit *kvm_create_pit(struct kvm *kvm, u32 flags)
 {
         struct kvm_pit *pit;
         struct kvm_kpit_state *pit_state;
+        int ret;
 
         pit = kzalloc(sizeof(struct kvm_pit), GFP_KERNEL);
         if (!pit)
@@ -582,19 +621,6 @@ struct kvm_pit *kvm_create_pit(struct kvm *kvm)
         mutex_lock(&pit->pit_state.lock);
         spin_lock_init(&pit->pit_state.inject_lock);
 
-        /* Initialize PIO device */
-        pit->dev.read = pit_ioport_read;
-        pit->dev.write = pit_ioport_write;
-        pit->dev.in_range = pit_in_range;
-        pit->dev.private = pit;
-        kvm_io_bus_register_dev(&kvm->pio_bus, &pit->dev);
-
-        pit->speaker_dev.read = speaker_ioport_read;
-        pit->speaker_dev.write = speaker_ioport_write;
-        pit->speaker_dev.in_range = speaker_in_range;
-        pit->speaker_dev.private = pit;
-        kvm_io_bus_register_dev(&kvm->pio_bus, &pit->speaker_dev);
-
         kvm->arch.vpit = pit;
         pit->kvm = kvm;
 
@@ -613,7 +639,30 @@ struct kvm_pit *kvm_create_pit(struct kvm *kvm)
         pit->mask_notifier.func = pit_mask_notifer;
         kvm_register_irq_mask_notifier(kvm, 0, &pit->mask_notifier);
 
+        kvm_iodevice_init(&pit->dev, &pit_dev_ops);
+        ret = __kvm_io_bus_register_dev(&kvm->pio_bus, &pit->dev);
+        if (ret < 0)
+                goto fail;
+
+        if (flags & KVM_PIT_SPEAKER_DUMMY) {
+                kvm_iodevice_init(&pit->speaker_dev, &speaker_dev_ops);
+                ret = __kvm_io_bus_register_dev(&kvm->pio_bus,
+                                                &pit->speaker_dev);
+                if (ret < 0)
+                        goto fail_unregister;
+        }
+
         return pit;
+
+fail_unregister:
+        __kvm_io_bus_unregister_dev(&kvm->pio_bus, &pit->dev);
+
+fail:
+        if (pit->irq_source_id >= 0)
+                kvm_free_irq_source_id(kvm, pit->irq_source_id);
+
+        kfree(pit);
+        return NULL;
 }
 
 void kvm_free_pit(struct kvm *kvm)
@@ -623,6 +672,8 @@ void kvm_free_pit(struct kvm *kvm)
         if (kvm->arch.vpit) {
                 kvm_unregister_irq_mask_notifier(kvm, 0,
                                                &kvm->arch.vpit->mask_notifier);
+                kvm_unregister_irq_ack_notifier(kvm,
+                                &kvm->arch.vpit->pit_state.irq_ack_notifier);
                 mutex_lock(&kvm->arch.vpit->pit_state.lock);
                 timer = &kvm->arch.vpit->pit_state.pit_timer.timer;
                 hrtimer_cancel(timer);
@@ -637,10 +688,10 @@ static void __inject_pit_timer_intr(struct kvm *kvm)
         struct kvm_vcpu *vcpu;
         int i;
 
-        mutex_lock(&kvm->lock);
+        mutex_lock(&kvm->irq_lock);
         kvm_set_irq(kvm, kvm->arch.vpit->irq_source_id, 0, 1);
         kvm_set_irq(kvm, kvm->arch.vpit->irq_source_id, 0, 0);
-        mutex_unlock(&kvm->lock);
+        mutex_unlock(&kvm->irq_lock);
 
         /*
          * Provides NMI watchdog support via Virtual Wire mode.
@@ -652,11 +703,8 @@ static void __inject_pit_timer_intr(struct kvm *kvm)
          * VCPU0, and only if its LVT0 is in EXTINT mode.
          */
         if (kvm->arch.vapics_in_nmi_mode > 0)
-                for (i = 0; i < KVM_MAX_VCPUS; ++i) {
-                        vcpu = kvm->vcpus[i];
-                        if (vcpu)
-                                kvm_apic_nmi_wd_deliver(vcpu);
-                }
+                kvm_for_each_vcpu(i, vcpu, kvm)
+                        kvm_apic_nmi_wd_deliver(vcpu);
 }
 
 void kvm_inject_pit_timer_irqs(struct kvm_vcpu *vcpu)
@@ -665,7 +713,7 @@ void kvm_inject_pit_timer_irqs(struct kvm_vcpu *vcpu)
         struct kvm *kvm = vcpu->kvm;
         struct kvm_kpit_state *ps;
 
-        if (vcpu && pit) {
+        if (pit) {
                 int inject = 0;
                 ps = &pit->pit_state;
 
diff --git a/arch/x86/kvm/i8254.h b/arch/x86/kvm/i8254.h
index bbd863ff60b7..d4c1c7ffdc09 100644
--- a/arch/x86/kvm/i8254.h
+++ b/arch/x86/kvm/i8254.h
@@ -21,6 +21,7 @@ struct kvm_kpit_channel_state {
 
 struct kvm_kpit_state {
         struct kvm_kpit_channel_state channels[3];
+        u32 flags;
         struct kvm_timer pit_timer;
         bool is_periodic;
         u32    speaker_data_on;
@@ -49,8 +50,8 @@ struct kvm_pit {
 #define KVM_PIT_CHANNEL_MASK        0x3
 
 void kvm_inject_pit_timer_irqs(struct kvm_vcpu *vcpu);
-void kvm_pit_load_count(struct kvm *kvm, int channel, u32 val);
-struct kvm_pit *kvm_create_pit(struct kvm *kvm);
+void kvm_pit_load_count(struct kvm *kvm, int channel, u32 val, int hpet_legacy_start);
+struct kvm_pit *kvm_create_pit(struct kvm *kvm, u32 flags);
 void kvm_free_pit(struct kvm *kvm);
 void kvm_pit_reset(struct kvm_pit *pit);
 
diff --git a/arch/x86/kvm/i8259.c b/arch/x86/kvm/i8259.c
index 1ccb50c74f18..01f151682802 100644
--- a/arch/x86/kvm/i8259.c
+++ b/arch/x86/kvm/i8259.c
@@ -30,50 +30,24 @@
 #include "irq.h"
 
 #include <linux/kvm_host.h>
-
-static void pic_lock(struct kvm_pic *s)
-        __acquires(&s->lock)
-{
-        spin_lock(&s->lock);
-}
-
-static void pic_unlock(struct kvm_pic *s)
-        __releases(&s->lock)
-{
-        struct kvm *kvm = s->kvm;
-        unsigned acks = s->pending_acks;
-        bool wakeup = s->wakeup_needed;
-        struct kvm_vcpu *vcpu;
-
-        s->pending_acks = 0;
-        s->wakeup_needed = false;
-
-        spin_unlock(&s->lock);
-
-        while (acks) {
-                kvm_notify_acked_irq(kvm, SELECT_PIC(__ffs(acks)),
-                                     __ffs(acks));
-                acks &= acks - 1;
-        }
-
-        if (wakeup) {
-                vcpu = s->kvm->vcpus[0];
-                if (vcpu)
-                        kvm_vcpu_kick(vcpu);
-        }
-}
+#include "trace.h"
 
 static void pic_clear_isr(struct kvm_kpic_state *s, int irq)
 {
         s->isr &= ~(1 << irq);
         s->isr_ack |= (1 << irq);
+        if (s != &s->pics_state->pics[0])
+                irq += 8;
+        kvm_notify_acked_irq(s->pics_state->kvm, SELECT_PIC(irq), irq);
 }
 
 void kvm_pic_clear_isr_ack(struct kvm *kvm)
 {
         struct kvm_pic *s = pic_irqchip(kvm);
+        spin_lock(&s->lock);
         s->pics[0].isr_ack = 0xff;
         s->pics[1].isr_ack = 0xff;
+        spin_unlock(&s->lock);
 }
 
 /*
@@ -174,9 +148,9 @@ static void pic_update_irq(struct kvm_pic *s)
 
 void kvm_pic_update_irq(struct kvm_pic *s)
 {
-        pic_lock(s);
+        spin_lock(&s->lock);
         pic_update_irq(s);
-        pic_unlock(s);
+        spin_unlock(&s->lock);
 }
 
 int kvm_pic_set_irq(void *opaque, int irq, int level)
@@ -184,12 +158,14 @@ int kvm_pic_set_irq(void *opaque, int irq, int level)
         struct kvm_pic *s = opaque;
         int ret = -1;
 
-        pic_lock(s);
+        spin_lock(&s->lock);
         if (irq >= 0 && irq < PIC_NUM_PINS) {
                 ret = pic_set_irq1(&s->pics[irq >> 3], irq & 7, level);
                 pic_update_irq(s);
+                trace_kvm_pic_set_irq(irq >> 3, irq & 7, s->pics[irq >> 3].elcr,
+                                      s->pics[irq >> 3].imr, ret == 0);
         }
-        pic_unlock(s);
+        spin_unlock(&s->lock);
 
         return ret;
 }
@@ -217,7 +193,7 @@ int kvm_pic_read_irq(struct kvm *kvm)
         int irq, irq2, intno;
         struct kvm_pic *s = pic_irqchip(kvm);
 
-        pic_lock(s);
+        spin_lock(&s->lock);
         irq = pic_get_irq(&s->pics[0]);
         if (irq >= 0) {
                 pic_intack(&s->pics[0], irq);
@@ -242,8 +218,7 @@ int kvm_pic_read_irq(struct kvm *kvm)
                 intno = s->pics[0].irq_base + irq;
         }
         pic_update_irq(s);
-        pic_unlock(s);
-        kvm_notify_acked_irq(kvm, SELECT_PIC(irq), irq);
+        spin_unlock(&s->lock);
 
         return intno;
 }
@@ -252,7 +227,7 @@ void kvm_pic_reset(struct kvm_kpic_state *s)
 {
         int irq, irqbase, n;
         struct kvm *kvm = s->pics_state->irq_request_opaque;
-        struct kvm_vcpu *vcpu0 = kvm->vcpus[0];
+        struct kvm_vcpu *vcpu0 = kvm->bsp_vcpu;
 
         if (s == &s->pics_state->pics[0])
                 irqbase = 0;
@@ -263,7 +238,7 @@ void kvm_pic_reset(struct kvm_kpic_state *s)
                 if (vcpu0 && kvm_apic_accept_pic_intr(vcpu0))
                         if (s->irr & (1 << irq) || s->isr & (1 << irq)) {
                                 n = irq + irqbase;
-                                s->pics_state->pending_acks |= 1 << n;
+                                kvm_notify_acked_irq(kvm, SELECT_PIC(n), n);
                         }
         }
         s->last_irr = 0;
@@ -428,8 +403,7 @@ static u32 elcr_ioport_read(void *opaque, u32 addr1)
         return s->elcr;
 }
 
-static int picdev_in_range(struct kvm_io_device *this, gpa_t addr,
-                           int len, int is_write)
+static int picdev_in_range(gpa_t addr)
 {
         switch (addr) {
         case 0x20:
@@ -444,18 +418,25 @@ static int picdev_in_range(struct kvm_io_device *this, gpa_t addr,
         }
 }
 
-static void picdev_write(struct kvm_io_device *this,
+static inline struct kvm_pic *to_pic(struct kvm_io_device *dev)
+{
+        return container_of(dev, struct kvm_pic, dev);
+}
+
+static int picdev_write(struct kvm_io_device *this,
                          gpa_t addr, int len, const void *val)
 {
-        struct kvm_pic *s = this->private;
+        struct kvm_pic *s = to_pic(this);
         unsigned char data = *(unsigned char *)val;
+        if (!picdev_in_range(addr))
+                return -EOPNOTSUPP;
 
         if (len != 1) {
                 if (printk_ratelimit())
                         printk(KERN_ERR "PIC: non byte write\n");
-                return;
+                return 0;
         }
-        pic_lock(s);
+        spin_lock(&s->lock);
         switch (addr) {
         case 0x20:
         case 0x21:
@@ -468,21 +449,24 @@ static void picdev_write(struct kvm_io_device *this,
                 elcr_ioport_write(&s->pics[addr & 1], addr, data);
                 break;
         }
-        pic_unlock(s);
+        spin_unlock(&s->lock);
+        return 0;
 }
 
-static void picdev_read(struct kvm_io_device *this,
-                        gpa_t addr, int len, void *val)
+static int picdev_read(struct kvm_io_device *this,
+                       gpa_t addr, int len, void *val)
 {
-        struct kvm_pic *s = this->private;
+        struct kvm_pic *s = to_pic(this);
         unsigned char data = 0;
+        if (!picdev_in_range(addr))
+                return -EOPNOTSUPP;
 
         if (len != 1) {
                 if (printk_ratelimit())
                         printk(KERN_ERR "PIC: non byte read\n");
-                return;
+                return 0;
         }
-        pic_lock(s);
+        spin_lock(&s->lock);
         switch (addr) {
         case 0x20:
         case 0x21:
@@ -496,7 +480,8 @@ static void picdev_read(struct kvm_io_device *this,
                 break;
         }
         *(unsigned char *)val = data;
-        pic_unlock(s);
+        spin_unlock(&s->lock);
+        return 0;
 }
 
 /*
@@ -505,20 +490,27 @@ static void picdev_read(struct kvm_io_device *this,
 static void pic_irq_request(void *opaque, int level)
 {
         struct kvm *kvm = opaque;
-        struct kvm_vcpu *vcpu = kvm->vcpus[0];
+        struct kvm_vcpu *vcpu = kvm->bsp_vcpu;
         struct kvm_pic *s = pic_irqchip(kvm);
         int irq = pic_get_irq(&s->pics[0]);
 
         s->output = level;
         if (vcpu && level && (s->pics[0].isr_ack & (1 << irq))) {
                 s->pics[0].isr_ack &= ~(1 << irq);
-                s->wakeup_needed = true;
+                kvm_vcpu_kick(vcpu);
         }
 }
 
+static const struct kvm_io_device_ops picdev_ops = {
+        .read     = picdev_read,
+        .write    = picdev_write,
+};
+
 struct kvm_pic *kvm_create_pic(struct kvm *kvm)
 {
         struct kvm_pic *s;
+        int ret;
+
         s = kzalloc(sizeof(struct kvm_pic), GFP_KERNEL);
         if (!s)
                 return NULL;
@@ -534,10 +526,12 @@ struct kvm_pic *kvm_create_pic(struct kvm *kvm)
         /*
          * Initialize PIO device
          */
-        s->dev.read = picdev_read;
-        s->dev.write = picdev_write;
-        s->dev.in_range = picdev_in_range;
-        s->dev.private = s;
-        kvm_io_bus_register_dev(&kvm->pio_bus, &s->dev);
+        kvm_iodevice_init(&s->dev, &picdev_ops);
+        ret = kvm_io_bus_register_dev(kvm, &kvm->pio_bus, &s->dev);
+        if (ret < 0) {
+                kfree(s);
+                return NULL;
+        }
+
         return s;
 }
diff --git a/arch/x86/kvm/irq.h b/arch/x86/kvm/irq.h
index 9f593188129e..7d6058a2fd38 100644
--- a/arch/x86/kvm/irq.h
+++ b/arch/x86/kvm/irq.h
@@ -63,7 +63,6 @@ struct kvm_kpic_state {
 
 struct kvm_pic {
         spinlock_t lock;
-        bool wakeup_needed;
         unsigned pending_acks;
         struct kvm *kvm;
         struct kvm_kpic_state pics[2]; /* 0 is master pic, 1 is slave pic */
diff --git a/arch/x86/kvm/kvm_cache_regs.h b/arch/x86/kvm/kvm_cache_regs.h
index 1ff819dce7d3..7bcc5b6a4403 100644
--- a/arch/x86/kvm/kvm_cache_regs.h
+++ b/arch/x86/kvm/kvm_cache_regs.h
@@ -29,4 +29,13 @@ static inline void kvm_rip_write(struct kvm_vcpu *vcpu, unsigned long val)
         kvm_register_write(vcpu, VCPU_REGS_RIP, val);
 }
 
+static inline u64 kvm_pdptr_read(struct kvm_vcpu *vcpu, int index)
+{
+        if (!test_bit(VCPU_EXREG_PDPTR,
+                      (unsigned long *)&vcpu->arch.regs_avail))
+                kvm_x86_ops->cache_reg(vcpu, VCPU_EXREG_PDPTR);
+
+        return vcpu->arch.pdptrs[index];
+}
+
 #endif
diff --git a/arch/x86/kvm/kvm_svm.h b/arch/x86/kvm/kvm_svm.h
deleted file mode 100644
index ed66e4c078dc..000000000000
--- a/arch/x86/kvm/kvm_svm.h
+++ /dev/null
@@ -1,51 +0,0 @@
-#ifndef __KVM_SVM_H
-#define __KVM_SVM_H
-
-#include <linux/kernel.h>
-#include <linux/types.h>
-#include <linux/list.h>
-#include <linux/kvm_host.h>
-#include <asm/msr.h>
-
-#include <asm/svm.h>
-
-static const u32 host_save_user_msrs[] = {
-#ifdef CONFIG_X86_64
-        MSR_STAR, MSR_LSTAR, MSR_CSTAR, MSR_SYSCALL_MASK, MSR_KERNEL_GS_BASE,
-        MSR_FS_BASE,
-#endif
-        MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
-};
-
-#define NR_HOST_SAVE_USER_MSRS ARRAY_SIZE(host_save_user_msrs)
-
-struct kvm_vcpu;
-
-struct vcpu_svm {
-        struct kvm_vcpu vcpu;
-        struct vmcb *vmcb;
-        unsigned long vmcb_pa;
-        struct svm_cpu_data *svm_data;
-        uint64_t asid_generation;
-
-        u64 next_rip;
-
-        u64 host_user_msrs[NR_HOST_SAVE_USER_MSRS];
-        u64 host_gs_base;
-        unsigned long host_cr2;
-
-        u32 *msrpm;
-        struct vmcb *hsave;
-        u64 hsave_msr;
-
-        u64 nested_vmcb;
-
-        /* These are the merged vectors */
-        u32 *nested_msrpm;
-
-        /* gpa pointers to the real vectors */
-        u64 nested_vmcb_msrpm;
-};
-
-#endif
-
diff --git a/arch/x86/kvm/kvm_timer.h b/arch/x86/kvm/kvm_timer.h
index 26bd6ba74e1c..55c7524dda54 100644
--- a/arch/x86/kvm/kvm_timer.h
+++ b/arch/x86/kvm/kvm_timer.h
@@ -6,7 +6,7 @@ struct kvm_timer {
         bool reinject;
         struct kvm_timer_ops *t_ops;
         struct kvm *kvm;
-        int vcpu_id;
+        struct kvm_vcpu *vcpu;
 };
 
 struct kvm_timer_ops {
diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c
index ae99d83f81a3..1ae5ceba7eb2 100644
--- a/arch/x86/kvm/lapic.c
+++ b/arch/x86/kvm/lapic.c
@@ -32,8 +32,11 @@
 #include <asm/current.h>
 #include <asm/apicdef.h>
 #include <asm/atomic.h>
+#include <asm/apicdef.h>
 #include "kvm_cache_regs.h"
 #include "irq.h"
+#include "trace.h"
+#include "x86.h"
 
 #ifndef CONFIG_X86_64
 #define mod_64(x, y) ((x) - (y) * div64_u64(x, y))
@@ -141,6 +144,26 @@ static inline int apic_lvt_nmi_mode(u32 lvt_val)
         return (lvt_val & (APIC_MODE_MASK | APIC_LVT_MASKED)) == APIC_DM_NMI;
 }
 
+void kvm_apic_set_version(struct kvm_vcpu *vcpu)
+{
+        struct kvm_lapic *apic = vcpu->arch.apic;
+        struct kvm_cpuid_entry2 *feat;
+        u32 v = APIC_VERSION;
+
+        if (!irqchip_in_kernel(vcpu->kvm))
+                return;
+
+        feat = kvm_find_cpuid_entry(apic->vcpu, 0x1, 0);
+        if (feat && (feat->ecx & (1 << (X86_FEATURE_X2APIC & 31))))
+                v |= APIC_LVR_DIRECTED_EOI;
+        apic_set_reg(apic, APIC_LVR, v);
+}
+
+static inline int apic_x2apic_mode(struct kvm_lapic *apic)
+{
+        return apic->vcpu->arch.apic_base & X2APIC_ENABLE;
+}
+
 static unsigned int apic_lvt_mask[APIC_LVT_NUM] = {
         LVT_MASK | APIC_LVT_TIMER_PERIODIC,     /* LVTT */
         LVT_MASK | APIC_MODE_MASK,      /* LVTTHMR */
@@ -165,36 +188,52 @@ static int find_highest_vector(void *bitmap)
 
 static inline int apic_test_and_set_irr(int vec, struct kvm_lapic *apic)
 {
+        apic->irr_pending = true;
         return apic_test_and_set_vector(vec, apic->regs + APIC_IRR);
 }
 
-static inline void apic_clear_irr(int vec, struct kvm_lapic *apic)
+static inline int apic_search_irr(struct kvm_lapic *apic)
 {
-        apic_clear_vector(vec, apic->regs + APIC_IRR);
+        return find_highest_vector(apic->regs + APIC_IRR);
 }
 
 static inline int apic_find_highest_irr(struct kvm_lapic *apic)
 {
         int result;
 
-        result = find_highest_vector(apic->regs + APIC_IRR);
+        if (!apic->irr_pending)
+                return -1;
+
+        result = apic_search_irr(apic);
         ASSERT(result == -1 || result >= 16);
 
         return result;
 }
 
+static inline void apic_clear_irr(int vec, struct kvm_lapic *apic)
+{
+        apic->irr_pending = false;
+        apic_clear_vector(vec, apic->regs + APIC_IRR);
+        if (apic_search_irr(apic) != -1)
+                apic->irr_pending = true;
+}
+
 int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu)
 {
         struct kvm_lapic *apic = vcpu->arch.apic;
         int highest_irr;
 
+        /* This may race with setting of irr in __apic_accept_irq() and
+         * value returned may be wrong, but kvm_vcpu_kick() in __apic_accept_irq
+         * will cause vmexit immediately and the value will be recalculated
+         * on the next vmentry.
+         */
         if (!apic)
                 return 0;
         highest_irr = apic_find_highest_irr(apic);
 
         return highest_irr;
 }
-EXPORT_SYMBOL_GPL(kvm_lapic_find_highest_irr);
 
 static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode,
                              int vector, int level, int trig_mode);
@@ -251,7 +290,12 @@ int kvm_apic_match_physical_addr(struct kvm_lapic *apic, u16 dest)
 int kvm_apic_match_logical_addr(struct kvm_lapic *apic, u8 mda)
 {
         int result = 0;
-        u8 logical_id;
+        u32 logical_id;
+
+        if (apic_x2apic_mode(apic)) {
+                logical_id = apic_get_reg(apic, APIC_LDR);
+                return logical_id & mda;
+        }
 
         logical_id = GET_APIC_LOGICAL_ID(apic_get_reg(apic, APIC_LDR));
 
@@ -331,6 +375,8 @@ static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode,
                         break;
 
                 result = !apic_test_and_set_irr(vector, apic);
+                trace_kvm_apic_accept_irq(vcpu->vcpu_id, delivery_mode,
+                                          trig_mode, vector, !result);
                 if (!result) {
                         if (trig_mode)
                                 apic_debug("level trig mode repeatedly for "
@@ -425,7 +471,11 @@ static void apic_set_eoi(struct kvm_lapic *apic)
                 trigger_mode = IOAPIC_LEVEL_TRIG;
         else
                 trigger_mode = IOAPIC_EDGE_TRIG;
-        kvm_ioapic_update_eoi(apic->vcpu->kvm, vector, trigger_mode);
+        if (!(apic_get_reg(apic, APIC_SPIV) & APIC_SPIV_DIRECTED_EOI)) {
+                mutex_lock(&apic->vcpu->kvm->irq_lock);
+                kvm_ioapic_update_eoi(apic->vcpu->kvm, vector, trigger_mode);
+                mutex_unlock(&apic->vcpu->kvm->irq_lock);
+        }
 }
 
 static void apic_send_ipi(struct kvm_lapic *apic)
@@ -440,7 +490,12 @@ static void apic_send_ipi(struct kvm_lapic *apic)
         irq.level = icr_low & APIC_INT_ASSERT;
         irq.trig_mode = icr_low & APIC_INT_LEVELTRIG;
         irq.shorthand = icr_low & APIC_SHORT_MASK;
-        irq.dest_id = GET_APIC_DEST_FIELD(icr_high);
+        if (apic_x2apic_mode(apic))
+                irq.dest_id = icr_high;
+        else
+                irq.dest_id = GET_APIC_DEST_FIELD(icr_high);
+
+        trace_kvm_apic_ipi(icr_low, irq.dest_id);
 
         apic_debug("icr_high 0x%x, icr_low 0x%x, "
                    "short_hand 0x%x, dest 0x%x, trig_mode 0x%x, level 0x%x, "
@@ -449,7 +504,9 @@ static void apic_send_ipi(struct kvm_lapic *apic)
                    irq.trig_mode, irq.level, irq.dest_mode, irq.delivery_mode,
                    irq.vector);
 
+        mutex_lock(&apic->vcpu->kvm->irq_lock);
         kvm_irq_delivery_to_apic(apic->vcpu->kvm, apic, &irq);
+        mutex_unlock(&apic->vcpu->kvm->irq_lock);
 }
 
 static u32 apic_get_tmcct(struct kvm_lapic *apic)
@@ -495,12 +552,16 @@ static u32 __apic_read(struct kvm_lapic *apic, unsigned int offset)
 {
         u32 val = 0;
 
-        KVMTRACE_1D(APIC_ACCESS, apic->vcpu, (u32)offset, handler);
-
         if (offset >= LAPIC_MMIO_LENGTH)
                 return 0;
 
         switch (offset) {
+        case APIC_ID:
+                if (apic_x2apic_mode(apic))
+                        val = kvm_apic_id(apic);
+                else
+                        val = kvm_apic_id(apic) << 24;
+                break;
         case APIC_ARBPRI:
                 printk(KERN_WARNING "Access APIC ARBPRI register "
                        "which is for P6\n");
@@ -522,21 +583,35 @@ static u32 __apic_read(struct kvm_lapic *apic, unsigned int offset)
         return val;
 }
 
-static void apic_mmio_read(struct kvm_io_device *this,
-                           gpa_t address, int len, void *data)
+static inline struct kvm_lapic *to_lapic(struct kvm_io_device *dev)
+{
+        return container_of(dev, struct kvm_lapic, dev);
+}
+
+static int apic_reg_read(struct kvm_lapic *apic, u32 offset, int len,
+                void *data)
 {
-        struct kvm_lapic *apic = (struct kvm_lapic *)this->private;
-        unsigned int offset = address - apic->base_address;
         unsigned char alignment = offset & 0xf;
         u32 result;
+        /* this bitmask has a bit cleared for each reserver register */
+        static const u64 rmask = 0x43ff01ffffffe70cULL;
 
         if ((alignment + len) > 4) {
-                printk(KERN_ERR "KVM_APIC_READ: alignment error %lx %d",
-                       (unsigned long)address, len);
-                return;
+                apic_debug("KVM_APIC_READ: alignment error %x %d\n",
+                           offset, len);
+                return 1;
         }
+
+        if (offset > 0x3f0 || !(rmask & (1ULL << (offset >> 4)))) {
+                apic_debug("KVM_APIC_READ: read reserved register %x\n",
+                           offset);
+                return 1;
+        }
+
         result = __apic_read(apic, offset & ~0xf);
 
+        trace_kvm_apic_read(offset, result);
+
         switch (len) {
         case 1:
         case 2:
@@ -548,6 +623,28 @@ static void apic_mmio_read(struct kvm_io_device *this,
                        "should be 1,2, or 4 instead\n", len);
                 break;
         }
+        return 0;
+}
+
+static int apic_mmio_in_range(struct kvm_lapic *apic, gpa_t addr)
+{
+        return apic_hw_enabled(apic) &&
+            addr >= apic->base_address &&
+            addr < apic->base_address + LAPIC_MMIO_LENGTH;
+}
+
+static int apic_mmio_read(struct kvm_io_device *this,
+                           gpa_t address, int len, void *data)
+{
+        struct kvm_lapic *apic = to_lapic(this);
+        u32 offset = address - apic->base_address;
+
+        if (!apic_mmio_in_range(apic, address))
+                return -EOPNOTSUPP;
+
+        apic_reg_read(apic, offset, len, data);
+
+        return 0;
 }
 
 static void update_divide_count(struct kvm_lapic *apic)
@@ -573,6 +670,15 @@ static void start_apic_timer(struct kvm_lapic *apic)
 
         if (!apic->lapic_timer.period)
                 return;
+        /*
+         * Do not allow the guest to program periodic timers with small
+         * interval, since the hrtimers are not throttled by the host
+         * scheduler.
+         */
+        if (apic_lvtt_period(apic)) {
+                if (apic->lapic_timer.period < NSEC_PER_MSEC/2)
+                        apic->lapic_timer.period = NSEC_PER_MSEC/2;
+        }
 
         hrtimer_start(&apic->lapic_timer.timer,
                       ktime_add_ns(now, apic->lapic_timer.period),
@@ -603,40 +709,18 @@ static void apic_manage_nmi_watchdog(struct kvm_lapic *apic, u32 lvt0_val)
                 apic->vcpu->kvm->arch.vapics_in_nmi_mode--;
 }
 
-static void apic_mmio_write(struct kvm_io_device *this,
-                            gpa_t address, int len, const void *data)
+static int apic_reg_write(struct kvm_lapic *apic, u32 reg, u32 val)
 {
-        struct kvm_lapic *apic = (struct kvm_lapic *)this->private;
-        unsigned int offset = address - apic->base_address;
-        unsigned char alignment = offset & 0xf;
-        u32 val;
-
-        /*
-         * APIC register must be aligned on 128-bits boundary.
-         * 32/64/128 bits registers must be accessed thru 32 bits.
-         * Refer SDM 8.4.1
-         */
-        if (len != 4 || alignment) {
-                /* Don't shout loud, $infamous_os would cause only noise. */
-                apic_debug("apic write: bad size=%d %lx\n",
-                           len, (long)address);
-                return;
-        }
-
-        val = *(u32 *) data;
-
-        /* too common printing */
-        if (offset != APIC_EOI)
-                apic_debug("%s: offset 0x%x with length 0x%x, and value is "
-                           "0x%x\n", __func__, offset, len, val);
-
-        offset &= 0xff0;
+        int ret = 0;
 
-        KVMTRACE_1D(APIC_ACCESS, apic->vcpu, (u32)offset, handler);
+        trace_kvm_apic_write(reg, val);
 
-        switch (offset) {
+        switch (reg) {
         case APIC_ID:           /* Local APIC ID */
-                apic_set_reg(apic, APIC_ID, val);
+                if (!apic_x2apic_mode(apic))
+                        apic_set_reg(apic, APIC_ID, val);
+                else
+                        ret = 1;
                 break;
 
         case APIC_TASKPRI:
@@ -649,15 +733,24 @@ static void apic_mmio_write(struct kvm_io_device *this,
                 break;
 
         case APIC_LDR:
-                apic_set_reg(apic, APIC_LDR, val & APIC_LDR_MASK);
+                if (!apic_x2apic_mode(apic))
+                        apic_set_reg(apic, APIC_LDR, val & APIC_LDR_MASK);
+                else
+                        ret = 1;
                 break;
 
         case APIC_DFR:
-                apic_set_reg(apic, APIC_DFR, val | 0x0FFFFFFF);
+                if (!apic_x2apic_mode(apic))
+                        apic_set_reg(apic, APIC_DFR, val | 0x0FFFFFFF);
+                else
+                        ret = 1;
                 break;
 
-        case APIC_SPIV:
-                apic_set_reg(apic, APIC_SPIV, val & 0x3ff);
+        case APIC_SPIV: {
+                u32 mask = 0x3ff;
+                if (apic_get_reg(apic, APIC_LVR) & APIC_LVR_DIRECTED_EOI)
+                        mask |= APIC_SPIV_DIRECTED_EOI;
+                apic_set_reg(apic, APIC_SPIV, val & mask);
                 if (!(val & APIC_SPIV_APIC_ENABLED)) {
                         int i;
                         u32 lvt_val;
@@ -672,7 +765,7 @@ static void apic_mmio_write(struct kvm_io_device *this,
 
                 }
                 break;
-
+        }
         case APIC_ICR:
                 /* No delay here, so we always clear the pending bit */
                 apic_set_reg(apic, APIC_ICR, val & ~(1 << 12));
@@ -680,7 +773,9 @@ static void apic_mmio_write(struct kvm_io_device *this,
                 break;
 
         case APIC_ICR2:
-                apic_set_reg(apic, APIC_ICR2, val & 0xff000000);
+                if (!apic_x2apic_mode(apic))
+                        val &= 0xff000000;
+                apic_set_reg(apic, APIC_ICR2, val);
                 break;
 
         case APIC_LVT0:
@@ -694,8 +789,8 @@ static void apic_mmio_write(struct kvm_io_device *this,
                 if (!apic_sw_enabled(apic))
                         val |= APIC_LVT_MASKED;
 
-                val &= apic_lvt_mask[(offset - APIC_LVTT) >> 4];
-                apic_set_reg(apic, offset, val);
+                val &= apic_lvt_mask[(reg - APIC_LVTT) >> 4];
+                apic_set_reg(apic, reg, val);
 
                 break;
 
@@ -703,7 +798,7 @@ static void apic_mmio_write(struct kvm_io_device *this,
                 hrtimer_cancel(&apic->lapic_timer.timer);
                 apic_set_reg(apic, APIC_TMICT, val);
                 start_apic_timer(apic);
-                return;
+                break;
 
         case APIC_TDCR:
                 if (val & 4)
@@ -712,27 +807,59 @@ static void apic_mmio_write(struct kvm_io_device *this,
                 update_divide_count(apic);
                 break;
 
+        case APIC_ESR:
+                if (apic_x2apic_mode(apic) && val != 0) {
+                        printk(KERN_ERR "KVM_WRITE:ESR not zero %x\n", val);
+                        ret = 1;
+                }
+                break;
+
+        case APIC_SELF_IPI:
+                if (apic_x2apic_mode(apic)) {
+                        apic_reg_write(apic, APIC_ICR, 0x40000 | (val & 0xff));
+                } else
+                        ret = 1;
+                break;
         default:
-                apic_debug("Local APIC Write to read-only register %x\n",
-                           offset);
+                ret = 1;
                 break;
         }
-
+        if (ret)
+                apic_debug("Local APIC Write to read-only register %x\n", reg);
+        return ret;
 }
 
-static int apic_mmio_range(struct kvm_io_device *this, gpa_t addr,
-                           int len, int size)
+static int apic_mmio_write(struct kvm_io_device *this,
+                            gpa_t address, int len, const void *data)
 {
-        struct kvm_lapic *apic = (struct kvm_lapic *)this->private;
-        int ret = 0;
+        struct kvm_lapic *apic = to_lapic(this);
+        unsigned int offset = address - apic->base_address;
+        u32 val;
 
+        if (!apic_mmio_in_range(apic, address))
+                return -EOPNOTSUPP;
 
-        if (apic_hw_enabled(apic) &&
-            (addr >= apic->base_address) &&
-            (addr < (apic->base_address + LAPIC_MMIO_LENGTH)))
-                ret = 1;
+        /*
+         * APIC register must be aligned on 128-bits boundary.
+         * 32/64/128 bits registers must be accessed thru 32 bits.
+         * Refer SDM 8.4.1
+         */
+        if (len != 4 || (offset & 0xf)) {
+                /* Don't shout loud, $infamous_os would cause only noise. */
+                apic_debug("apic write: bad size=%d %lx\n", len, (long)address);
+                return 0;
+        }
 
-        return ret;
+        val = *(u32*)data;
+
+        /* too common printing */
+        if (offset != APIC_EOI)
+                apic_debug("%s: offset 0x%x with length 0x%x, and value is "
+                           "0x%x\n", __func__, offset, len, val);
+
+        apic_reg_write(apic, offset & 0xff0, val);
+
+        return 0;
 }
 
 void kvm_free_lapic(struct kvm_vcpu *vcpu)
@@ -763,7 +890,6 @@ void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8)
         apic_set_tpr(apic, ((cr8 & 0x0f) << 4)
                      | (apic_get_reg(apic, APIC_TASKPRI) & 4));
 }
-EXPORT_SYMBOL_GPL(kvm_lapic_set_tpr);
 
 u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu)
 {
@@ -776,7 +902,6 @@ u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu)
 
         return (tpr & 0xf0) >> 4;
 }
-EXPORT_SYMBOL_GPL(kvm_lapic_get_cr8);
 
 void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value)
 {
@@ -787,10 +912,16 @@ void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value)
                 vcpu->arch.apic_base = value;
                 return;
         }
-        if (apic->vcpu->vcpu_id)
+
+        if (!kvm_vcpu_is_bsp(apic->vcpu))
                 value &= ~MSR_IA32_APICBASE_BSP;
 
         vcpu->arch.apic_base = value;
+        if (apic_x2apic_mode(apic)) {
+                u32 id = kvm_apic_id(apic);
+                u32 ldr = ((id & ~0xf) << 16) | (1 << (id & 0xf));
+                apic_set_reg(apic, APIC_LDR, ldr);
+        }
         apic->base_address = apic->vcpu->arch.apic_base &
                              MSR_IA32_APICBASE_BASE;
 
@@ -800,12 +931,6 @@ void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value)
 
 }
 
-u64 kvm_lapic_get_base(struct kvm_vcpu *vcpu)
-{
-        return vcpu->arch.apic_base;
-}
-EXPORT_SYMBOL_GPL(kvm_lapic_get_base);
-
 void kvm_lapic_reset(struct kvm_vcpu *vcpu)
 {
         struct kvm_lapic *apic;
@@ -821,7 +946,7 @@ void kvm_lapic_reset(struct kvm_vcpu *vcpu)
         hrtimer_cancel(&apic->lapic_timer.timer);
 
         apic_set_reg(apic, APIC_ID, vcpu->vcpu_id << 24);
-        apic_set_reg(apic, APIC_LVR, APIC_VERSION);
+        kvm_apic_set_version(apic->vcpu);
 
         for (i = 0; i < APIC_LVT_NUM; i++)
                 apic_set_reg(apic, APIC_LVTT + 0x10 * i, APIC_LVT_MASKED);
@@ -842,9 +967,10 @@ void kvm_lapic_reset(struct kvm_vcpu *vcpu)
                 apic_set_reg(apic, APIC_ISR + 0x10 * i, 0);
                 apic_set_reg(apic, APIC_TMR + 0x10 * i, 0);
         }
+        apic->irr_pending = false;
         update_divide_count(apic);
         atomic_set(&apic->lapic_timer.pending, 0);
-        if (vcpu->vcpu_id == 0)
+        if (kvm_vcpu_is_bsp(vcpu))
                 vcpu->arch.apic_base |= MSR_IA32_APICBASE_BSP;
         apic_update_ppr(apic);
 
@@ -855,7 +981,6 @@ void kvm_lapic_reset(struct kvm_vcpu *vcpu)
                    vcpu, kvm_apic_id(apic),
                    vcpu->arch.apic_base, apic->base_address);
 }
-EXPORT_SYMBOL_GPL(kvm_lapic_reset);
 
 bool kvm_apic_present(struct kvm_vcpu *vcpu)
 {
@@ -866,7 +991,6 @@ int kvm_lapic_enabled(struct kvm_vcpu *vcpu)
 {
         return kvm_apic_present(vcpu) && apic_sw_enabled(vcpu->arch.apic);
 }
-EXPORT_SYMBOL_GPL(kvm_lapic_enabled);
 
 /*
  *----------------------------------------------------------------------
@@ -917,6 +1041,11 @@ static struct kvm_timer_ops lapic_timer_ops = {
         .is_periodic = lapic_is_periodic,
 };
 
+static const struct kvm_io_device_ops apic_mmio_ops = {
+        .read     = apic_mmio_read,
+        .write    = apic_mmio_write,
+};
+
 int kvm_create_lapic(struct kvm_vcpu *vcpu)
 {
         struct kvm_lapic *apic;
@@ -945,16 +1074,13 @@ int kvm_create_lapic(struct kvm_vcpu *vcpu)
         apic->lapic_timer.timer.function = kvm_timer_fn;
         apic->lapic_timer.t_ops = &lapic_timer_ops;
         apic->lapic_timer.kvm = vcpu->kvm;
-        apic->lapic_timer.vcpu_id = vcpu->vcpu_id;
+        apic->lapic_timer.vcpu = vcpu;
 
         apic->base_address = APIC_DEFAULT_PHYS_BASE;
         vcpu->arch.apic_base = APIC_DEFAULT_PHYS_BASE;
 
         kvm_lapic_reset(vcpu);
-        apic->dev.read = apic_mmio_read;
-        apic->dev.write = apic_mmio_write;
-        apic->dev.in_range = apic_mmio_range;
-        apic->dev.private = apic;
+        kvm_iodevice_init(&apic->dev, &apic_mmio_ops);
 
         return 0;
 nomem_free_apic:
@@ -962,7 +1088,6 @@ nomem_free_apic:
 nomem:
         return -ENOMEM;
 }
-EXPORT_SYMBOL_GPL(kvm_create_lapic);
 
 int kvm_apic_has_interrupt(struct kvm_vcpu *vcpu)
 {
@@ -985,7 +1110,7 @@ int kvm_apic_accept_pic_intr(struct kvm_vcpu *vcpu)
         u32 lvt0 = apic_get_reg(vcpu->arch.apic, APIC_LVT0);
         int r = 0;
 
-        if (vcpu->vcpu_id == 0) {
+        if (kvm_vcpu_is_bsp(vcpu)) {
                 if (!apic_hw_enabled(vcpu->arch.apic))
                         r = 1;
                 if ((lvt0 & APIC_LVT_MASKED) == 0 &&
@@ -1025,7 +1150,8 @@ void kvm_apic_post_state_restore(struct kvm_vcpu *vcpu)
 
         apic->base_address = vcpu->arch.apic_base &
                              MSR_IA32_APICBASE_BASE;
-        apic_set_reg(apic, APIC_LVR, APIC_VERSION);
+        kvm_apic_set_version(vcpu);
+
         apic_update_ppr(apic);
         hrtimer_cancel(&apic->lapic_timer.timer);
         update_divide_count(apic);
@@ -1092,3 +1218,35 @@ void kvm_lapic_set_vapic_addr(struct kvm_vcpu *vcpu, gpa_t vapic_addr)
 
         vcpu->arch.apic->vapic_addr = vapic_addr;
 }
+
+int kvm_x2apic_msr_write(struct kvm_vcpu *vcpu, u32 msr, u64 data)
+{
+        struct kvm_lapic *apic = vcpu->arch.apic;
+        u32 reg = (msr - APIC_BASE_MSR) << 4;
+
+        if (!irqchip_in_kernel(vcpu->kvm) || !apic_x2apic_mode(apic))
+                return 1;
+
+        /* if this is ICR write vector before command */
+        if (msr == 0x830)
+                apic_reg_write(apic, APIC_ICR2, (u32)(data >> 32));
+        return apic_reg_write(apic, reg, (u32)data);
+}
+
+int kvm_x2apic_msr_read(struct kvm_vcpu *vcpu, u32 msr, u64 *data)
+{
+        struct kvm_lapic *apic = vcpu->arch.apic;
+        u32 reg = (msr - APIC_BASE_MSR) << 4, low, high = 0;
+
+        if (!irqchip_in_kernel(vcpu->kvm) || !apic_x2apic_mode(apic))
+                return 1;
+
+        if (apic_reg_read(apic, reg, 4, &low))
+                return 1;
+        if (msr == 0x830)
+                apic_reg_read(apic, APIC_ICR2, 4, &high);
+
+        *data = (((u64)high) << 32) | low;
+
+        return 0;
+}
diff --git a/arch/x86/kvm/lapic.h b/arch/x86/kvm/lapic.h
index a587f8349c46..40010b09c4aa 100644
--- a/arch/x86/kvm/lapic.h
+++ b/arch/x86/kvm/lapic.h
@@ -12,6 +12,7 @@ struct kvm_lapic {
         struct kvm_timer lapic_timer;
         u32 divide_count;
         struct kvm_vcpu *vcpu;
+        bool irr_pending;
         struct page *regs_page;
         void *regs;
         gpa_t vapic_addr;
@@ -28,6 +29,7 @@ u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu);
 void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8);
 void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value);
 u64 kvm_lapic_get_base(struct kvm_vcpu *vcpu);
+void kvm_apic_set_version(struct kvm_vcpu *vcpu);
 
 int kvm_apic_match_physical_addr(struct kvm_lapic *apic, u16 dest);
 int kvm_apic_match_logical_addr(struct kvm_lapic *apic, u8 mda);
@@ -44,4 +46,6 @@ void kvm_lapic_set_vapic_addr(struct kvm_vcpu *vcpu, gpa_t vapic_addr);
 void kvm_lapic_sync_from_vapic(struct kvm_vcpu *vcpu);
 void kvm_lapic_sync_to_vapic(struct kvm_vcpu *vcpu);
 
+int kvm_x2apic_msr_write(struct kvm_vcpu *vcpu, u32 msr, u64 data);
+int kvm_x2apic_msr_read(struct kvm_vcpu *vcpu, u32 msr, u64 *data);
 #endif
diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c
index 0ef5bb2b4043..eca41ae9f453 100644
--- a/arch/x86/kvm/mmu.c
+++ b/arch/x86/kvm/mmu.c
@@ -18,6 +18,7 @@
  */
 
 #include "mmu.h"
+#include "kvm_cache_regs.h"
 
 #include <linux/kvm_host.h>
 #include <linux/types.h>
@@ -107,6 +108,9 @@ module_param(oos_shadow, bool, 0644);
 
 #define PT32_LEVEL_MASK(level) \
                 (((1ULL << PT32_LEVEL_BITS) - 1) << PT32_LEVEL_SHIFT(level))
+#define PT32_LVL_OFFSET_MASK(level) \
+        (PT32_BASE_ADDR_MASK & ((1ULL << (PAGE_SHIFT + (((level) - 1) \
+                                                * PT32_LEVEL_BITS))) - 1))
 
 #define PT32_INDEX(address, level)\
         (((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1))
@@ -115,10 +119,19 @@ module_param(oos_shadow, bool, 0644);
 #define PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1))
 #define PT64_DIR_BASE_ADDR_MASK \
         (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + PT64_LEVEL_BITS)) - 1))
+#define PT64_LVL_ADDR_MASK(level) \
+        (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \
+                                                * PT64_LEVEL_BITS))) - 1))
+#define PT64_LVL_OFFSET_MASK(level) \
+        (PT64_BASE_ADDR_MASK & ((1ULL << (PAGE_SHIFT + (((level) - 1) \
+                                                * PT64_LEVEL_BITS))) - 1))
 
 #define PT32_BASE_ADDR_MASK PAGE_MASK
 #define PT32_DIR_BASE_ADDR_MASK \
         (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + PT32_LEVEL_BITS)) - 1))
+#define PT32_LVL_ADDR_MASK(level) \
+        (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \
+                                            * PT32_LEVEL_BITS))) - 1))
 
 #define PT64_PERM_MASK (PT_PRESENT_MASK | PT_WRITABLE_MASK | PT_USER_MASK \
                         | PT64_NX_MASK)
@@ -129,6 +142,7 @@ module_param(oos_shadow, bool, 0644);
 #define PFERR_RSVD_MASK (1U << 3)
 #define PFERR_FETCH_MASK (1U << 4)
 
+#define PT_PDPE_LEVEL 3
 #define PT_DIRECTORY_LEVEL 2
 #define PT_PAGE_TABLE_LEVEL 1
 
@@ -139,10 +153,13 @@ module_param(oos_shadow, bool, 0644);
 #define ACC_USER_MASK    PT_USER_MASK
 #define ACC_ALL          (ACC_EXEC_MASK | ACC_WRITE_MASK | ACC_USER_MASK)
 
+#define CREATE_TRACE_POINTS
+#include "mmutrace.h"
+
 #define SHADOW_PT_INDEX(addr, level) PT64_INDEX(addr, level)
 
 struct kvm_rmap_desc {
-        u64 *shadow_ptes[RMAP_EXT];
+        u64 *sptes[RMAP_EXT];
         struct kvm_rmap_desc *more;
 };
 
@@ -239,16 +256,25 @@ static int is_writeble_pte(unsigned long pte)
         return pte & PT_WRITABLE_MASK;
 }
 
-static int is_dirty_pte(unsigned long pte)
+static int is_dirty_gpte(unsigned long pte)
 {
-        return pte & shadow_dirty_mask;
+        return pte & PT_DIRTY_MASK;
 }
 
-static int is_rmap_pte(u64 pte)
+static int is_rmap_spte(u64 pte)
 {
         return is_shadow_present_pte(pte);
 }
 
+static int is_last_spte(u64 pte, int level)
+{
+        if (level == PT_PAGE_TABLE_LEVEL)
+                return 1;
+        if (is_large_pte(pte))
+                return 1;
+        return 0;
+}
+
 static pfn_t spte_to_pfn(u64 pte)
 {
         return (pte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT;
@@ -261,7 +287,7 @@ static gfn_t pse36_gfn_delta(u32 gpte)
         return (gpte & PT32_DIR_PSE36_MASK) << shift;
 }
 
-static void set_shadow_pte(u64 *sptep, u64 spte)
+static void __set_spte(u64 *sptep, u64 spte)
 {
 #ifdef CONFIG_X86_64
         set_64bit((unsigned long *)sptep, spte);
@@ -380,37 +406,52 @@ static void mmu_free_rmap_desc(struct kvm_rmap_desc *rd)
  * Return the pointer to the largepage write count for a given
  * gfn, handling slots that are not large page aligned.
  */
-static int *slot_largepage_idx(gfn_t gfn, struct kvm_memory_slot *slot)
+static int *slot_largepage_idx(gfn_t gfn,
+                               struct kvm_memory_slot *slot,
+                               int level)
 {
         unsigned long idx;
 
-        idx = (gfn / KVM_PAGES_PER_HPAGE) -
-              (slot->base_gfn / KVM_PAGES_PER_HPAGE);
-        return &slot->lpage_info[idx].write_count;
+        idx = (gfn / KVM_PAGES_PER_HPAGE(level)) -
+              (slot->base_gfn / KVM_PAGES_PER_HPAGE(level));
+        return &slot->lpage_info[level - 2][idx].write_count;
 }
 
 static void account_shadowed(struct kvm *kvm, gfn_t gfn)
 {
+        struct kvm_memory_slot *slot;
         int *write_count;
+        int i;
 
         gfn = unalias_gfn(kvm, gfn);
-        write_count = slot_largepage_idx(gfn,
-                                         gfn_to_memslot_unaliased(kvm, gfn));
-        *write_count += 1;
+
+        slot = gfn_to_memslot_unaliased(kvm, gfn);
+        for (i = PT_DIRECTORY_LEVEL;
+             i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
+                write_count   = slot_largepage_idx(gfn, slot, i);
+                *write_count += 1;
+        }
 }
 
 static void unaccount_shadowed(struct kvm *kvm, gfn_t gfn)
 {
+        struct kvm_memory_slot *slot;
         int *write_count;
+        int i;
 
         gfn = unalias_gfn(kvm, gfn);
-        write_count = slot_largepage_idx(gfn,
-                                         gfn_to_memslot_unaliased(kvm, gfn));
-        *write_count -= 1;
-        WARN_ON(*write_count < 0);
+        for (i = PT_DIRECTORY_LEVEL;
+             i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
+                slot          = gfn_to_memslot_unaliased(kvm, gfn);
+                write_count   = slot_largepage_idx(gfn, slot, i);
+                *write_count -= 1;
+                WARN_ON(*write_count < 0);
+        }
 }
 
-static int has_wrprotected_page(struct kvm *kvm, gfn_t gfn)
+static int has_wrprotected_page(struct kvm *kvm,
+                                gfn_t gfn,
+                                int level)
 {
         struct kvm_memory_slot *slot;
         int *largepage_idx;
@@ -418,47 +459,67 @@ static int has_wrprotected_page(struct kvm *kvm, gfn_t gfn)
         gfn = unalias_gfn(kvm, gfn);
         slot = gfn_to_memslot_unaliased(kvm, gfn);
         if (slot) {
-                largepage_idx = slot_largepage_idx(gfn, slot);
+                largepage_idx = slot_largepage_idx(gfn, slot, level);
                 return *largepage_idx;
         }
 
         return 1;
 }
 
-static int host_largepage_backed(struct kvm *kvm, gfn_t gfn)
+static int host_mapping_level(struct kvm *kvm, gfn_t gfn)
 {
+        unsigned long page_size = PAGE_SIZE;
         struct vm_area_struct *vma;
         unsigned long addr;
-        int ret = 0;
+        int i, ret = 0;
 
         addr = gfn_to_hva(kvm, gfn);
         if (kvm_is_error_hva(addr))
-                return ret;
+                return page_size;
 
         down_read(&current->mm->mmap_sem);
         vma = find_vma(current->mm, addr);
-        if (vma && is_vm_hugetlb_page(vma))
-                ret = 1;
+        if (!vma)
+                goto out;
+
+        page_size = vma_kernel_pagesize(vma);
+
+out:
         up_read(&current->mm->mmap_sem);
 
+        for (i = PT_PAGE_TABLE_LEVEL;
+             i < (PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES); ++i) {
+                if (page_size >= KVM_HPAGE_SIZE(i))
+                        ret = i;
+                else
+                        break;
+        }
+
         return ret;
 }
 
-static int is_largepage_backed(struct kvm_vcpu *vcpu, gfn_t large_gfn)
+static int mapping_level(struct kvm_vcpu *vcpu, gfn_t large_gfn)
 {
         struct kvm_memory_slot *slot;
-
-        if (has_wrprotected_page(vcpu->kvm, large_gfn))
-                return 0;
-
-        if (!host_largepage_backed(vcpu->kvm, large_gfn))
-                return 0;
+        int host_level;
+        int level = PT_PAGE_TABLE_LEVEL;
 
         slot = gfn_to_memslot(vcpu->kvm, large_gfn);
         if (slot && slot->dirty_bitmap)
-                return 0;
+                return PT_PAGE_TABLE_LEVEL;
 
-        return 1;
+        host_level = host_mapping_level(vcpu->kvm, large_gfn);
+
+        if (host_level == PT_PAGE_TABLE_LEVEL)
+                return host_level;
+
+        for (level = PT_DIRECTORY_LEVEL; level <= host_level; ++level) {
+
+                if (has_wrprotected_page(vcpu->kvm, large_gfn, level))
+                        break;
+        }
+
+        return level - 1;
 }
 
 /*
@@ -466,19 +527,19 @@ static int is_largepage_backed(struct kvm_vcpu *vcpu, gfn_t large_gfn)
  * Note: gfn must be unaliased before this function get called
  */
 
-static unsigned long *gfn_to_rmap(struct kvm *kvm, gfn_t gfn, int lpage)
+static unsigned long *gfn_to_rmap(struct kvm *kvm, gfn_t gfn, int level)
 {
         struct kvm_memory_slot *slot;
         unsigned long idx;
 
         slot = gfn_to_memslot(kvm, gfn);
-        if (!lpage)
+        if (likely(level == PT_PAGE_TABLE_LEVEL))
                 return &slot->rmap[gfn - slot->base_gfn];
 
-        idx = (gfn / KVM_PAGES_PER_HPAGE) -
-              (slot->base_gfn / KVM_PAGES_PER_HPAGE);
+        idx = (gfn / KVM_PAGES_PER_HPAGE(level)) -
+                (slot->base_gfn / KVM_PAGES_PER_HPAGE(level));
 
-        return &slot->lpage_info[idx].rmap_pde;
+        return &slot->lpage_info[level - 2][idx].rmap_pde;
 }
 
 /*
@@ -494,42 +555,42 @@ static unsigned long *gfn_to_rmap(struct kvm *kvm, gfn_t gfn, int lpage)
  * the spte was not added.
  *
  */
-static int rmap_add(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn, int lpage)
+static int rmap_add(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn)
 {
         struct kvm_mmu_page *sp;
         struct kvm_rmap_desc *desc;
         unsigned long *rmapp;
         int i, count = 0;
 
-        if (!is_rmap_pte(*spte))
+        if (!is_rmap_spte(*spte))
                 return count;
         gfn = unalias_gfn(vcpu->kvm, gfn);
         sp = page_header(__pa(spte));
         sp->gfns[spte - sp->spt] = gfn;
-        rmapp = gfn_to_rmap(vcpu->kvm, gfn, lpage);
+        rmapp = gfn_to_rmap(vcpu->kvm, gfn, sp->role.level);
         if (!*rmapp) {
                 rmap_printk("rmap_add: %p %llx 0->1\n", spte, *spte);
                 *rmapp = (unsigned long)spte;
         } else if (!(*rmapp & 1)) {
                 rmap_printk("rmap_add: %p %llx 1->many\n", spte, *spte);
                 desc = mmu_alloc_rmap_desc(vcpu);
-                desc->shadow_ptes[0] = (u64 *)*rmapp;
-                desc->shadow_ptes[1] = spte;
+                desc->sptes[0] = (u64 *)*rmapp;
+                desc->sptes[1] = spte;
                 *rmapp = (unsigned long)desc | 1;
         } else {
                 rmap_printk("rmap_add: %p %llx many->many\n", spte, *spte);
                 desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
-                while (desc->shadow_ptes[RMAP_EXT-1] && desc->more) {
+                while (desc->sptes[RMAP_EXT-1] && desc->more) {
                         desc = desc->more;
                         count += RMAP_EXT;
                 }
-                if (desc->shadow_ptes[RMAP_EXT-1]) {
+                if (desc->sptes[RMAP_EXT-1]) {
                         desc->more = mmu_alloc_rmap_desc(vcpu);
                         desc = desc->more;
                 }
-                for (i = 0; desc->shadow_ptes[i]; ++i)
+                for (i = 0; desc->sptes[i]; ++i)
                         ;
-                desc->shadow_ptes[i] = spte;
+                desc->sptes[i] = spte;
         }
         return count;
 }
@@ -541,14 +602,14 @@ static void rmap_desc_remove_entry(unsigned long *rmapp,
 {
         int j;
 
-        for (j = RMAP_EXT - 1; !desc->shadow_ptes[j] && j > i; --j)
+        for (j = RMAP_EXT - 1; !desc->sptes[j] && j > i; --j)
                 ;
-        desc->shadow_ptes[i] = desc->shadow_ptes[j];
-        desc->shadow_ptes[j] = NULL;
+        desc->sptes[i] = desc->sptes[j];
+        desc->sptes[j] = NULL;
         if (j != 0)
                 return;
         if (!prev_desc && !desc->more)
-                *rmapp = (unsigned long)desc->shadow_ptes[0];
+                *rmapp = (unsigned long)desc->sptes[0];
         else
                 if (prev_desc)
                         prev_desc->more = desc->more;
@@ -566,7 +627,7 @@ static void rmap_remove(struct kvm *kvm, u64 *spte)
         unsigned long *rmapp;
         int i;
 
-        if (!is_rmap_pte(*spte))
+        if (!is_rmap_spte(*spte))
                 return;
         sp = page_header(__pa(spte));
         pfn = spte_to_pfn(*spte);
@@ -576,7 +637,7 @@ static void rmap_remove(struct kvm *kvm, u64 *spte)
                 kvm_release_pfn_dirty(pfn);
         else
                 kvm_release_pfn_clean(pfn);
-        rmapp = gfn_to_rmap(kvm, sp->gfns[spte - sp->spt], is_large_pte(*spte));
+        rmapp = gfn_to_rmap(kvm, sp->gfns[spte - sp->spt], sp->role.level);
         if (!*rmapp) {
                 printk(KERN_ERR "rmap_remove: %p %llx 0->BUG\n", spte, *spte);
                 BUG();
@@ -593,8 +654,8 @@ static void rmap_remove(struct kvm *kvm, u64 *spte)
                 desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
                 prev_desc = NULL;
                 while (desc) {
-                        for (i = 0; i < RMAP_EXT && desc->shadow_ptes[i]; ++i)
-                                if (desc->shadow_ptes[i] == spte) {
+                        for (i = 0; i < RMAP_EXT && desc->sptes[i]; ++i)
+                                if (desc->sptes[i] == spte) {
                                         rmap_desc_remove_entry(rmapp,
                                                                desc, i,
                                                                prev_desc);
@@ -625,10 +686,10 @@ static u64 *rmap_next(struct kvm *kvm, unsigned long *rmapp, u64 *spte)
         prev_desc = NULL;
         prev_spte = NULL;
         while (desc) {
-                for (i = 0; i < RMAP_EXT && desc->shadow_ptes[i]; ++i) {
+                for (i = 0; i < RMAP_EXT && desc->sptes[i]; ++i) {
                         if (prev_spte == spte)
-                                return desc->shadow_ptes[i];
-                        prev_spte = desc->shadow_ptes[i];
+                                return desc->sptes[i];
+                        prev_spte = desc->sptes[i];
                 }
                 desc = desc->more;
         }
@@ -639,10 +700,10 @@ static int rmap_write_protect(struct kvm *kvm, u64 gfn)
 {
         unsigned long *rmapp;
         u64 *spte;
-        int write_protected = 0;
+        int i, write_protected = 0;
 
         gfn = unalias_gfn(kvm, gfn);
-        rmapp = gfn_to_rmap(kvm, gfn, 0);
+        rmapp = gfn_to_rmap(kvm, gfn, PT_PAGE_TABLE_LEVEL);
 
         spte = rmap_next(kvm, rmapp, NULL);
         while (spte) {
@@ -650,7 +711,7 @@ static int rmap_write_protect(struct kvm *kvm, u64 gfn)
                 BUG_ON(!(*spte & PT_PRESENT_MASK));
                 rmap_printk("rmap_write_protect: spte %p %llx\n", spte, *spte);
                 if (is_writeble_pte(*spte)) {
-                        set_shadow_pte(spte, *spte & ~PT_WRITABLE_MASK);
+                        __set_spte(spte, *spte & ~PT_WRITABLE_MASK);
                         write_protected = 1;
                 }
                 spte = rmap_next(kvm, rmapp, spte);
@@ -664,21 +725,24 @@ static int rmap_write_protect(struct kvm *kvm, u64 gfn)
         }
 
         /* check for huge page mappings */
-        rmapp = gfn_to_rmap(kvm, gfn, 1);
-        spte = rmap_next(kvm, rmapp, NULL);
-        while (spte) {
-                BUG_ON(!spte);
-                BUG_ON(!(*spte & PT_PRESENT_MASK));
-                BUG_ON((*spte & (PT_PAGE_SIZE_MASK|PT_PRESENT_MASK)) != (PT_PAGE_SIZE_MASK|PT_PRESENT_MASK));
-                pgprintk("rmap_write_protect(large): spte %p %llx %lld\n", spte, *spte, gfn);
-                if (is_writeble_pte(*spte)) {
-                        rmap_remove(kvm, spte);
-                        --kvm->stat.lpages;
-                        set_shadow_pte(spte, shadow_trap_nonpresent_pte);
-                        spte = NULL;
-                        write_protected = 1;
+        for (i = PT_DIRECTORY_LEVEL;
+             i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
+                rmapp = gfn_to_rmap(kvm, gfn, i);
+                spte = rmap_next(kvm, rmapp, NULL);
+                while (spte) {
+                        BUG_ON(!spte);
+                        BUG_ON(!(*spte & PT_PRESENT_MASK));
+                        BUG_ON((*spte & (PT_PAGE_SIZE_MASK|PT_PRESENT_MASK)) != (PT_PAGE_SIZE_MASK|PT_PRESENT_MASK));
+                        pgprintk("rmap_write_protect(large): spte %p %llx %lld\n", spte, *spte, gfn);
+                        if (is_writeble_pte(*spte)) {
+                                rmap_remove(kvm, spte);
+                                --kvm->stat.lpages;
+                                __set_spte(spte, shadow_trap_nonpresent_pte);
+                                spte = NULL;
+                                write_protected = 1;
+                        }
+                        spte = rmap_next(kvm, rmapp, spte);
                 }
-                spte = rmap_next(kvm, rmapp, spte);
         }
 
         return write_protected;
@@ -693,7 +757,7 @@ static int kvm_unmap_rmapp(struct kvm *kvm, unsigned long *rmapp)
                 BUG_ON(!(*spte & PT_PRESENT_MASK));
                 rmap_printk("kvm_rmap_unmap_hva: spte %p %llx\n", spte, *spte);
                 rmap_remove(kvm, spte);
-                set_shadow_pte(spte, shadow_trap_nonpresent_pte);
+                __set_spte(spte, shadow_trap_nonpresent_pte);
                 need_tlb_flush = 1;
         }
         return need_tlb_flush;
@@ -702,7 +766,7 @@ static int kvm_unmap_rmapp(struct kvm *kvm, unsigned long *rmapp)
 static int kvm_handle_hva(struct kvm *kvm, unsigned long hva,
                           int (*handler)(struct kvm *kvm, unsigned long *rmapp))
 {
-        int i;
+        int i, j;
         int retval = 0;
 
         /*
@@ -721,11 +785,15 @@ static int kvm_handle_hva(struct kvm *kvm, unsigned long hva,
                 end = start + (memslot->npages << PAGE_SHIFT);
                 if (hva >= start && hva < end) {
                         gfn_t gfn_offset = (hva - start) >> PAGE_SHIFT;
+
                         retval |= handler(kvm, &memslot->rmap[gfn_offset]);
-                        retval |= handler(kvm,
-                                          &memslot->lpage_info[
-                                                  gfn_offset /
-                                                  KVM_PAGES_PER_HPAGE].rmap_pde);
+
+                        for (j = 0; j < KVM_NR_PAGE_SIZES - 1; ++j) {
+                                int idx = gfn_offset;
+                                idx /= KVM_PAGES_PER_HPAGE(PT_DIRECTORY_LEVEL + j);
+                                retval |= handler(kvm,
+                                        &memslot->lpage_info[j][idx].rmap_pde);
+                        }
                 }
         }
 
@@ -763,12 +831,15 @@ static int kvm_age_rmapp(struct kvm *kvm, unsigned long *rmapp)
 
 #define RMAP_RECYCLE_THRESHOLD 1000
 
-static void rmap_recycle(struct kvm_vcpu *vcpu, gfn_t gfn, int lpage)
+static void rmap_recycle(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn)
 {
         unsigned long *rmapp;
+        struct kvm_mmu_page *sp;
+
+        sp = page_header(__pa(spte));
 
         gfn = unalias_gfn(vcpu->kvm, gfn);
-        rmapp = gfn_to_rmap(vcpu->kvm, gfn, lpage);
+        rmapp = gfn_to_rmap(vcpu->kvm, gfn, sp->role.level);
 
         kvm_unmap_rmapp(vcpu->kvm, rmapp);
         kvm_flush_remote_tlbs(vcpu->kvm);
@@ -1109,6 +1180,7 @@ static int kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
                 return 1;
         }
 
+        trace_kvm_mmu_sync_page(sp);
         if (rmap_write_protect(vcpu->kvm, sp->gfn))
                 kvm_flush_remote_tlbs(vcpu->kvm);
         kvm_unlink_unsync_page(vcpu->kvm, sp);
@@ -1231,8 +1303,6 @@ static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu,
                 quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1;
                 role.quadrant = quadrant;
         }
-        pgprintk("%s: looking gfn %lx role %x\n", __func__,
-                 gfn, role.word);
         index = kvm_page_table_hashfn(gfn);
         bucket = &vcpu->kvm->arch.mmu_page_hash[index];
         hlist_for_each_entry_safe(sp, node, tmp, bucket, hash_link)
@@ -1249,14 +1319,13 @@ static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu,
                                 set_bit(KVM_REQ_MMU_SYNC, &vcpu->requests);
                                 kvm_mmu_mark_parents_unsync(vcpu, sp);
                         }
-                        pgprintk("%s: found\n", __func__);
+                        trace_kvm_mmu_get_page(sp, false);
                         return sp;
                 }
         ++vcpu->kvm->stat.mmu_cache_miss;
         sp = kvm_mmu_alloc_page(vcpu, parent_pte);
         if (!sp)
                 return sp;
-        pgprintk("%s: adding gfn %lx role %x\n", __func__, gfn, role.word);
         sp->gfn = gfn;
         sp->role = role;
         hlist_add_head(&sp->hash_link, bucket);
@@ -1269,6 +1338,7 @@ static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu,
                 vcpu->arch.mmu.prefetch_page(vcpu, sp);
         else
                 nonpaging_prefetch_page(vcpu, sp);
+        trace_kvm_mmu_get_page(sp, true);
         return sp;
 }
 
@@ -1292,6 +1362,11 @@ static bool shadow_walk_okay(struct kvm_shadow_walk_iterator *iterator)
 {
         if (iterator->level < PT_PAGE_TABLE_LEVEL)
                 return false;
+
+        if (iterator->level == PT_PAGE_TABLE_LEVEL)
+                if (is_large_pte(*iterator->sptep))
+                        return false;
+
         iterator->index = SHADOW_PT_INDEX(iterator->addr, iterator->level);
         iterator->sptep = ((u64 *)__va(iterator->shadow_addr)) + iterator->index;
         return true;
@@ -1312,25 +1387,17 @@ static void kvm_mmu_page_unlink_children(struct kvm *kvm,
 
         pt = sp->spt;
 
-        if (sp->role.level == PT_PAGE_TABLE_LEVEL) {
-                for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
-                        if (is_shadow_present_pte(pt[i]))
-                                rmap_remove(kvm, &pt[i]);
-                        pt[i] = shadow_trap_nonpresent_pte;
-                }
-                return;
-        }
-
         for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
                 ent = pt[i];
 
                 if (is_shadow_present_pte(ent)) {
-                        if (!is_large_pte(ent)) {
+                        if (!is_last_spte(ent, sp->role.level)) {
                                 ent &= PT64_BASE_ADDR_MASK;
                                 mmu_page_remove_parent_pte(page_header(ent),
                                                            &pt[i]);
                         } else {
-                                --kvm->stat.lpages;
+                                if (is_large_pte(ent))
+                                        --kvm->stat.lpages;
                                 rmap_remove(kvm, &pt[i]);
                         }
                 }
@@ -1346,10 +1413,10 @@ static void kvm_mmu_put_page(struct kvm_mmu_page *sp, u64 *parent_pte)
 static void kvm_mmu_reset_last_pte_updated(struct kvm *kvm)
 {
         int i;
+        struct kvm_vcpu *vcpu;
 
-        for (i = 0; i < KVM_MAX_VCPUS; ++i)
-                if (kvm->vcpus[i])
-                        kvm->vcpus[i]->arch.last_pte_updated = NULL;
+        kvm_for_each_vcpu(i, vcpu, kvm)
+                vcpu->arch.last_pte_updated = NULL;
 }
 
 static void kvm_mmu_unlink_parents(struct kvm *kvm, struct kvm_mmu_page *sp)
@@ -1368,7 +1435,7 @@ static void kvm_mmu_unlink_parents(struct kvm *kvm, struct kvm_mmu_page *sp)
                 }
                 BUG_ON(!parent_pte);
                 kvm_mmu_put_page(sp, parent_pte);
-                set_shadow_pte(parent_pte, shadow_trap_nonpresent_pte);
+                __set_spte(parent_pte, shadow_trap_nonpresent_pte);
         }
 }
 
@@ -1400,6 +1467,8 @@ static int mmu_zap_unsync_children(struct kvm *kvm,
 static int kvm_mmu_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp)
 {
         int ret;
+
+        trace_kvm_mmu_zap_page(sp);
         ++kvm->stat.mmu_shadow_zapped;
         ret = mmu_zap_unsync_children(kvm, sp);
         kvm_mmu_page_unlink_children(kvm, sp);
@@ -1516,7 +1585,7 @@ static void mmu_convert_notrap(struct kvm_mmu_page *sp)
 
         for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
                 if (pt[i] == shadow_notrap_nonpresent_pte)
-                        set_shadow_pte(&pt[i], shadow_trap_nonpresent_pte);
+                        __set_spte(&pt[i], shadow_trap_nonpresent_pte);
         }
 }
 
@@ -1646,6 +1715,7 @@ static int kvm_unsync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
         struct kvm_mmu_page *s;
         struct hlist_node *node, *n;
 
+        trace_kvm_mmu_unsync_page(sp);
         index = kvm_page_table_hashfn(sp->gfn);
         bucket = &vcpu->kvm->arch.mmu_page_hash[index];
         /* don't unsync if pagetable is shadowed with multiple roles */
@@ -1682,9 +1752,9 @@ static int mmu_need_write_protect(struct kvm_vcpu *vcpu, gfn_t gfn,
         return 0;
 }
 
-static int set_spte(struct kvm_vcpu *vcpu, u64 *shadow_pte,
+static int set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
                     unsigned pte_access, int user_fault,
-                    int write_fault, int dirty, int largepage,
+                    int write_fault, int dirty, int level,
                     gfn_t gfn, pfn_t pfn, bool speculative,
                     bool can_unsync)
 {
@@ -1707,7 +1777,7 @@ static int set_spte(struct kvm_vcpu *vcpu, u64 *shadow_pte,
                 spte |= shadow_nx_mask;
         if (pte_access & ACC_USER_MASK)
                 spte |= shadow_user_mask;
-        if (largepage)
+        if (level > PT_PAGE_TABLE_LEVEL)
                 spte |= PT_PAGE_SIZE_MASK;
         if (tdp_enabled)
                 spte |= kvm_x86_ops->get_mt_mask(vcpu, gfn,
@@ -1718,7 +1788,8 @@ static int set_spte(struct kvm_vcpu *vcpu, u64 *shadow_pte,
         if ((pte_access & ACC_WRITE_MASK)
             || (write_fault && !is_write_protection(vcpu) && !user_fault)) {
 
-                if (largepage && has_wrprotected_page(vcpu->kvm, gfn)) {
+                if (level > PT_PAGE_TABLE_LEVEL &&
+                    has_wrprotected_page(vcpu->kvm, gfn, level)) {
                         ret = 1;
                         spte = shadow_trap_nonpresent_pte;
                         goto set_pte;
@@ -1732,7 +1803,7 @@ static int set_spte(struct kvm_vcpu *vcpu, u64 *shadow_pte,
                  * is responsibility of mmu_get_page / kvm_sync_page.
                  * Same reasoning can be applied to dirty page accounting.
                  */
-                if (!can_unsync && is_writeble_pte(*shadow_pte))
+                if (!can_unsync && is_writeble_pte(*sptep))
                         goto set_pte;
 
                 if (mmu_need_write_protect(vcpu, gfn, can_unsync)) {
@@ -1749,65 +1820,67 @@ static int set_spte(struct kvm_vcpu *vcpu, u64 *shadow_pte,
                 mark_page_dirty(vcpu->kvm, gfn);
 
 set_pte:
-        set_shadow_pte(shadow_pte, spte);
+        __set_spte(sptep, spte);
         return ret;
 }
 
-static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *shadow_pte,
+static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
                          unsigned pt_access, unsigned pte_access,
                          int user_fault, int write_fault, int dirty,
-                         int *ptwrite, int largepage, gfn_t gfn,
+                         int *ptwrite, int level, gfn_t gfn,
                          pfn_t pfn, bool speculative)
 {
         int was_rmapped = 0;
-        int was_writeble = is_writeble_pte(*shadow_pte);
+        int was_writeble = is_writeble_pte(*sptep);
         int rmap_count;
 
         pgprintk("%s: spte %llx access %x write_fault %d"
                  " user_fault %d gfn %lx\n",
-                 __func__, *shadow_pte, pt_access,
+                 __func__, *sptep, pt_access,
                  write_fault, user_fault, gfn);
 
-        if (is_rmap_pte(*shadow_pte)) {
+        if (is_rmap_spte(*sptep)) {
                 /*
                  * If we overwrite a PTE page pointer with a 2MB PMD, unlink
                  * the parent of the now unreachable PTE.
                  */
-                if (largepage && !is_large_pte(*shadow_pte)) {
+                if (level > PT_PAGE_TABLE_LEVEL &&
+                    !is_large_pte(*sptep)) {
                         struct kvm_mmu_page *child;
-                        u64 pte = *shadow_pte;
+                        u64 pte = *sptep;
 
                         child = page_header(pte & PT64_BASE_ADDR_MASK);
-                        mmu_page_remove_parent_pte(child, shadow_pte);
-                } else if (pfn != spte_to_pfn(*shadow_pte)) {
+                        mmu_page_remove_parent_pte(child, sptep);
+                } else if (pfn != spte_to_pfn(*sptep)) {
                         pgprintk("hfn old %lx new %lx\n",
-                                 spte_to_pfn(*shadow_pte), pfn);
-                        rmap_remove(vcpu->kvm, shadow_pte);
+                                 spte_to_pfn(*sptep), pfn);
+                        rmap_remove(vcpu->kvm, sptep);
                 } else
                         was_rmapped = 1;
         }
-        if (set_spte(vcpu, shadow_pte, pte_access, user_fault, write_fault,
-                      dirty, largepage, gfn, pfn, speculative, true)) {
+
+        if (set_spte(vcpu, sptep, pte_access, user_fault, write_fault,
+                      dirty, level, gfn, pfn, speculative, true)) {
                 if (write_fault)
                         *ptwrite = 1;
                 kvm_x86_ops->tlb_flush(vcpu);
         }
 
-        pgprintk("%s: setting spte %llx\n", __func__, *shadow_pte);
+        pgprintk("%s: setting spte %llx\n", __func__, *sptep);
         pgprintk("instantiating %s PTE (%s) at %ld (%llx) addr %p\n",
-                 is_large_pte(*shadow_pte)? "2MB" : "4kB",
-                 is_present_pte(*shadow_pte)?"RW":"R", gfn,
-                 *shadow_pte, shadow_pte);
-        if (!was_rmapped && is_large_pte(*shadow_pte))
+                 is_large_pte(*sptep)? "2MB" : "4kB",
+                 *sptep & PT_PRESENT_MASK ?"RW":"R", gfn,
+                 *sptep, sptep);
+        if (!was_rmapped && is_large_pte(*sptep))
                 ++vcpu->kvm->stat.lpages;
 
-        page_header_update_slot(vcpu->kvm, shadow_pte, gfn);
+        page_header_update_slot(vcpu->kvm, sptep, gfn);
         if (!was_rmapped) {
-                rmap_count = rmap_add(vcpu, shadow_pte, gfn, largepage);
-                if (!is_rmap_pte(*shadow_pte))
+                rmap_count = rmap_add(vcpu, sptep, gfn);
+                if (!is_rmap_spte(*sptep))
                         kvm_release_pfn_clean(pfn);
                 if (rmap_count > RMAP_RECYCLE_THRESHOLD)
-                        rmap_recycle(vcpu, gfn, largepage);
+                        rmap_recycle(vcpu, sptep, gfn);
         } else {
                 if (was_writeble)
                         kvm_release_pfn_dirty(pfn);
@@ -1815,7 +1888,7 @@ static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *shadow_pte,
                         kvm_release_pfn_clean(pfn);
         }
         if (speculative) {
-                vcpu->arch.last_pte_updated = shadow_pte;
+                vcpu->arch.last_pte_updated = sptep;
                 vcpu->arch.last_pte_gfn = gfn;
         }
 }
@@ -1825,7 +1898,7 @@ static void nonpaging_new_cr3(struct kvm_vcpu *vcpu)
 }
 
 static int __direct_map(struct kvm_vcpu *vcpu, gpa_t v, int write,
-                        int largepage, gfn_t gfn, pfn_t pfn)
+                        int level, gfn_t gfn, pfn_t pfn)
 {
         struct kvm_shadow_walk_iterator iterator;
         struct kvm_mmu_page *sp;
@@ -1833,11 +1906,10 @@ static int __direct_map(struct kvm_vcpu *vcpu, gpa_t v, int write,
         gfn_t pseudo_gfn;
 
         for_each_shadow_entry(vcpu, (u64)gfn << PAGE_SHIFT, iterator) {
-                if (iterator.level == PT_PAGE_TABLE_LEVEL
-                    || (largepage && iterator.level == PT_DIRECTORY_LEVEL)) {
+                if (iterator.level == level) {
                         mmu_set_spte(vcpu, iterator.sptep, ACC_ALL, ACC_ALL,
                                      0, write, 1, &pt_write,
-                                     largepage, gfn, pfn, false);
+                                     level, gfn, pfn, false);
                         ++vcpu->stat.pf_fixed;
                         break;
                 }
@@ -1853,10 +1925,10 @@ static int __direct_map(struct kvm_vcpu *vcpu, gpa_t v, int write,
                                 return -ENOMEM;
                         }
 
-                        set_shadow_pte(iterator.sptep,
-                                       __pa(sp->spt)
-                                       | PT_PRESENT_MASK | PT_WRITABLE_MASK
-                                       | shadow_user_mask | shadow_x_mask);
+                        __set_spte(iterator.sptep,
+                                   __pa(sp->spt)
+                                   | PT_PRESENT_MASK | PT_WRITABLE_MASK
+                                   | shadow_user_mask | shadow_x_mask);
                 }
         }
         return pt_write;
@@ -1865,14 +1937,20 @@ static int __direct_map(struct kvm_vcpu *vcpu, gpa_t v, int write,
 static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, int write, gfn_t gfn)
 {
         int r;
-        int largepage = 0;
+        int level;
         pfn_t pfn;
         unsigned long mmu_seq;
 
-        if (is_largepage_backed(vcpu, gfn & ~(KVM_PAGES_PER_HPAGE-1))) {
-                gfn &= ~(KVM_PAGES_PER_HPAGE-1);
-                largepage = 1;
-        }
+        level = mapping_level(vcpu, gfn);
+
+        /*
+         * This path builds a PAE pagetable - so we can map 2mb pages at
+         * maximum. Therefore check if the level is larger than that.
+         */
+        if (level > PT_DIRECTORY_LEVEL)
+                level = PT_DIRECTORY_LEVEL;
+
+        gfn &= ~(KVM_PAGES_PER_HPAGE(level) - 1);
 
         mmu_seq = vcpu->kvm->mmu_notifier_seq;
         smp_rmb();
@@ -1888,7 +1966,7 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, int write, gfn_t gfn)
         if (mmu_notifier_retry(vcpu, mmu_seq))
                 goto out_unlock;
         kvm_mmu_free_some_pages(vcpu);
-        r = __direct_map(vcpu, v, write, largepage, gfn, pfn);
+        r = __direct_map(vcpu, v, write, level, gfn, pfn);
         spin_unlock(&vcpu->kvm->mmu_lock);
 
 
@@ -1954,6 +2032,7 @@ static int mmu_alloc_roots(struct kvm_vcpu *vcpu)
         gfn_t root_gfn;
         struct kvm_mmu_page *sp;
         int direct = 0;
+        u64 pdptr;
 
         root_gfn = vcpu->arch.cr3 >> PAGE_SHIFT;
 
@@ -1981,11 +2060,12 @@ static int mmu_alloc_roots(struct kvm_vcpu *vcpu)
 
                 ASSERT(!VALID_PAGE(root));
                 if (vcpu->arch.mmu.root_level == PT32E_ROOT_LEVEL) {
-                        if (!is_present_pte(vcpu->arch.pdptrs[i])) {
+                        pdptr = kvm_pdptr_read(vcpu, i);
+                        if (!is_present_gpte(pdptr)) {
                                 vcpu->arch.mmu.pae_root[i] = 0;
                                 continue;
                         }
-                        root_gfn = vcpu->arch.pdptrs[i] >> PAGE_SHIFT;
+                        root_gfn = pdptr >> PAGE_SHIFT;
                 } else if (vcpu->arch.mmu.root_level == 0)
                         root_gfn = 0;
                 if (mmu_check_root(vcpu, root_gfn))
@@ -2062,7 +2142,7 @@ static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa,
 {
         pfn_t pfn;
         int r;
-        int largepage = 0;
+        int level;
         gfn_t gfn = gpa >> PAGE_SHIFT;
         unsigned long mmu_seq;
 
@@ -2073,10 +2153,10 @@ static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa,
         if (r)
                 return r;
 
-        if (is_largepage_backed(vcpu, gfn & ~(KVM_PAGES_PER_HPAGE-1))) {
-                gfn &= ~(KVM_PAGES_PER_HPAGE-1);
-                largepage = 1;
-        }
+        level = mapping_level(vcpu, gfn);
+
+        gfn &= ~(KVM_PAGES_PER_HPAGE(level) - 1);
+
         mmu_seq = vcpu->kvm->mmu_notifier_seq;
         smp_rmb();
         pfn = gfn_to_pfn(vcpu->kvm, gfn);
@@ -2089,7 +2169,7 @@ static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa,
                 goto out_unlock;
         kvm_mmu_free_some_pages(vcpu);
         r = __direct_map(vcpu, gpa, error_code & PFERR_WRITE_MASK,
-                         largepage, gfn, pfn);
+                         level, gfn, pfn);
         spin_unlock(&vcpu->kvm->mmu_lock);
 
         return r;
@@ -2206,7 +2286,9 @@ static void reset_rsvds_bits_mask(struct kvm_vcpu *vcpu, int level)
                 context->rsvd_bits_mask[0][0] = exb_bit_rsvd |
                         rsvd_bits(maxphyaddr, 51);
                 context->rsvd_bits_mask[1][3] = context->rsvd_bits_mask[0][3];
-                context->rsvd_bits_mask[1][2] = context->rsvd_bits_mask[0][2];
+                context->rsvd_bits_mask[1][2] = exb_bit_rsvd |
+                        rsvd_bits(maxphyaddr, 51) |
+                        rsvd_bits(13, 29);
                 context->rsvd_bits_mask[1][1] = exb_bit_rsvd |
                         rsvd_bits(maxphyaddr, 51) |
                         rsvd_bits(13, 20);              /* large page */
@@ -2357,8 +2439,8 @@ int kvm_mmu_load(struct kvm_vcpu *vcpu)
         spin_unlock(&vcpu->kvm->mmu_lock);
         if (r)
                 goto out;
+        /* set_cr3() should ensure TLB has been flushed */
         kvm_x86_ops->set_cr3(vcpu, vcpu->arch.mmu.root_hpa);
-        kvm_mmu_flush_tlb(vcpu);
 out:
         return r;
 }
@@ -2378,15 +2460,14 @@ static void mmu_pte_write_zap_pte(struct kvm_vcpu *vcpu,
 
         pte = *spte;
         if (is_shadow_present_pte(pte)) {
-                if (sp->role.level == PT_PAGE_TABLE_LEVEL ||
-                    is_large_pte(pte))
+                if (is_last_spte(pte, sp->role.level))
                         rmap_remove(vcpu->kvm, spte);
                 else {
                         child = page_header(pte & PT64_BASE_ADDR_MASK);
                         mmu_page_remove_parent_pte(child, spte);
                 }
         }
-        set_shadow_pte(spte, shadow_trap_nonpresent_pte);
+        __set_spte(spte, shadow_trap_nonpresent_pte);
         if (is_large_pte(pte))
                 --vcpu->kvm->stat.lpages;
 }
@@ -2397,11 +2478,8 @@ static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu,
                                   const void *new)
 {
         if (sp->role.level != PT_PAGE_TABLE_LEVEL) {
-                if (!vcpu->arch.update_pte.largepage ||
-                    sp->role.glevels == PT32_ROOT_LEVEL) {
-                        ++vcpu->kvm->stat.mmu_pde_zapped;
-                        return;
-                }
+                ++vcpu->kvm->stat.mmu_pde_zapped;
+                return;
         }
 
         ++vcpu->kvm->stat.mmu_pte_updated;
@@ -2447,8 +2525,6 @@ static void mmu_guess_page_from_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
         u64 gpte = 0;
         pfn_t pfn;
 
-        vcpu->arch.update_pte.largepage = 0;
-
         if (bytes != 4 && bytes != 8)
                 return;
 
@@ -2472,14 +2548,10 @@ static void mmu_guess_page_from_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
                 if ((bytes == 4) && (gpa % 4 == 0))
                         memcpy((void *)&gpte, new, 4);
         }
-        if (!is_present_pte(gpte))
+        if (!is_present_gpte(gpte))
                 return;
         gfn = (gpte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT;
 
-        if (is_large_pte(gpte) && is_largepage_backed(vcpu, gfn)) {
-                gfn &= ~(KVM_PAGES_PER_HPAGE-1);
-                vcpu->arch.update_pte.largepage = 1;
-        }
         vcpu->arch.update_pte.mmu_seq = vcpu->kvm->mmu_notifier_seq;
         smp_rmb();
         pfn = gfn_to_pfn(vcpu->kvm, gfn);
@@ -2622,6 +2694,9 @@ int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva)
         gpa_t gpa;
         int r;
 
+        if (tdp_enabled)
+                return 0;
+
         gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, gva);
 
         spin_lock(&vcpu->kvm->mmu_lock);
@@ -2633,7 +2708,8 @@ EXPORT_SYMBOL_GPL(kvm_mmu_unprotect_page_virt);
 
 void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu)
 {
-        while (vcpu->kvm->arch.n_free_mmu_pages < KVM_REFILL_PAGES) {
+        while (vcpu->kvm->arch.n_free_mmu_pages < KVM_REFILL_PAGES &&
+               !list_empty(&vcpu->kvm->arch.active_mmu_pages)) {
                 struct kvm_mmu_page *sp;
 
                 sp = container_of(vcpu->kvm->arch.active_mmu_pages.prev,
@@ -2670,8 +2746,9 @@ int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code)
                 ++vcpu->stat.mmio_exits;
                 return 0;
         case EMULATE_FAIL:
-                kvm_report_emulation_failure(vcpu, "pagetable");
-                return 1;
+                vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
+                vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
+                return 0;
         default:
                 BUG();
         }
@@ -2712,12 +2789,6 @@ static int alloc_mmu_pages(struct kvm_vcpu *vcpu)
 
         ASSERT(vcpu);
 
-        if (vcpu->kvm->arch.n_requested_mmu_pages)
-                vcpu->kvm->arch.n_free_mmu_pages =
-                                        vcpu->kvm->arch.n_requested_mmu_pages;
-        else
-                vcpu->kvm->arch.n_free_mmu_pages =
-                                        vcpu->kvm->arch.n_alloc_mmu_pages;
         /*
          * When emulating 32-bit mode, cr3 is only 32 bits even on x86_64.
          * Therefore we need to allocate shadow page tables in the first
@@ -3029,6 +3100,24 @@ out:
         return r;
 }
 
+int kvm_mmu_get_spte_hierarchy(struct kvm_vcpu *vcpu, u64 addr, u64 sptes[4])
+{
+        struct kvm_shadow_walk_iterator iterator;
+        int nr_sptes = 0;
+
+        spin_lock(&vcpu->kvm->mmu_lock);
+        for_each_shadow_entry(vcpu, addr, iterator) {
+                sptes[iterator.level-1] = *iterator.sptep;
+                nr_sptes++;
+                if (!is_shadow_present_pte(*iterator.sptep))
+                        break;
+        }
+        spin_unlock(&vcpu->kvm->mmu_lock);
+
+        return nr_sptes;
+}
+EXPORT_SYMBOL_GPL(kvm_mmu_get_spte_hierarchy);
+
 #ifdef AUDIT
 
 static const char *audit_msg;
@@ -3041,6 +3130,54 @@ static gva_t canonicalize(gva_t gva)
         return gva;
 }
 
+
+typedef void (*inspect_spte_fn) (struct kvm *kvm, struct kvm_mmu_page *sp,
+                                 u64 *sptep);
+
+static void __mmu_spte_walk(struct kvm *kvm, struct kvm_mmu_page *sp,
+                            inspect_spte_fn fn)
+{
+        int i;
+
+        for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
+                u64 ent = sp->spt[i];
+
+                if (is_shadow_present_pte(ent)) {
+                        if (!is_last_spte(ent, sp->role.level)) {
+                                struct kvm_mmu_page *child;
+                                child = page_header(ent & PT64_BASE_ADDR_MASK);
+                                __mmu_spte_walk(kvm, child, fn);
+                        } else
+                                fn(kvm, sp, &sp->spt[i]);
+                }
+        }
+}
+
+static void mmu_spte_walk(struct kvm_vcpu *vcpu, inspect_spte_fn fn)
+{
+        int i;
+        struct kvm_mmu_page *sp;
+
+        if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
+                return;
+        if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
+                hpa_t root = vcpu->arch.mmu.root_hpa;
+                sp = page_header(root);
+                __mmu_spte_walk(vcpu->kvm, sp, fn);
+                return;
+        }
+        for (i = 0; i < 4; ++i) {
+                hpa_t root = vcpu->arch.mmu.pae_root[i];
+
+                if (root && VALID_PAGE(root)) {
+                        root &= PT64_BASE_ADDR_MASK;
+                        sp = page_header(root);
+                        __mmu_spte_walk(vcpu->kvm, sp, fn);
+                }
+        }
+        return;
+}
+
 static void audit_mappings_page(struct kvm_vcpu *vcpu, u64 page_pte,
                                 gva_t va, int level)
 {
@@ -3055,20 +3192,19 @@ static void audit_mappings_page(struct kvm_vcpu *vcpu, u64 page_pte,
                         continue;
 
                 va = canonicalize(va);
-                if (level > 1) {
-                        if (ent == shadow_notrap_nonpresent_pte)
-                                printk(KERN_ERR "audit: (%s) nontrapping pte"
-                                       " in nonleaf level: levels %d gva %lx"
-                                       " level %d pte %llx\n", audit_msg,
-                                       vcpu->arch.mmu.root_level, va, level, ent);
-                        else
-                                audit_mappings_page(vcpu, ent, va, level - 1);
-                } else {
+                if (is_shadow_present_pte(ent) && !is_last_spte(ent, level))
+                        audit_mappings_page(vcpu, ent, va, level - 1);
+                else {
                         gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, va);
                         gfn_t gfn = gpa >> PAGE_SHIFT;
                         pfn_t pfn = gfn_to_pfn(vcpu->kvm, gfn);
                         hpa_t hpa = (hpa_t)pfn << PAGE_SHIFT;
 
+                        if (is_error_pfn(pfn)) {
+                                kvm_release_pfn_clean(pfn);
+                                continue;
+                        }
+
                         if (is_shadow_present_pte(ent)
                             && (ent & PT64_BASE_ADDR_MASK) != hpa)
                                 printk(KERN_ERR "xx audit error: (%s) levels %d"
@@ -3122,7 +3258,7 @@ static int count_rmaps(struct kvm_vcpu *vcpu)
                         d = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
                         while (d) {
                                 for (k = 0; k < RMAP_EXT; ++k)
-                                        if (d->shadow_ptes[k])
+                                        if (d->sptes[k])
                                                 ++nmaps;
                                         else
                                                 break;
@@ -3133,9 +3269,48 @@ static int count_rmaps(struct kvm_vcpu *vcpu)
         return nmaps;
 }
 
-static int count_writable_mappings(struct kvm_vcpu *vcpu)
+void inspect_spte_has_rmap(struct kvm *kvm, struct kvm_mmu_page *sp, u64 *sptep)
+{
+        unsigned long *rmapp;
+        struct kvm_mmu_page *rev_sp;
+        gfn_t gfn;
+
+        if (*sptep & PT_WRITABLE_MASK) {
+                rev_sp = page_header(__pa(sptep));
+                gfn = rev_sp->gfns[sptep - rev_sp->spt];
+
+                if (!gfn_to_memslot(kvm, gfn)) {
+                        if (!printk_ratelimit())
+                                return;
+                        printk(KERN_ERR "%s: no memslot for gfn %ld\n",
+                                         audit_msg, gfn);
+                        printk(KERN_ERR "%s: index %ld of sp (gfn=%lx)\n",
+                                        audit_msg, sptep - rev_sp->spt,
+                                        rev_sp->gfn);
+                        dump_stack();
+                        return;
+                }
+
+                rmapp = gfn_to_rmap(kvm, rev_sp->gfns[sptep - rev_sp->spt],
+                                    is_large_pte(*sptep));
+                if (!*rmapp) {
+                        if (!printk_ratelimit())
+                                return;
+                        printk(KERN_ERR "%s: no rmap for writable spte %llx\n",
+                                         audit_msg, *sptep);
+                        dump_stack();
+                }
+        }
+
+}
+
+void audit_writable_sptes_have_rmaps(struct kvm_vcpu *vcpu)
+{
+        mmu_spte_walk(vcpu, inspect_spte_has_rmap);
+}
+
+static void check_writable_mappings_rmap(struct kvm_vcpu *vcpu)
 {
-        int nmaps = 0;
         struct kvm_mmu_page *sp;
         int i;
 
@@ -3152,20 +3327,16 @@ static int count_writable_mappings(struct kvm_vcpu *vcpu)
                                 continue;
                         if (!(ent & PT_WRITABLE_MASK))
                                 continue;
-                        ++nmaps;
+                        inspect_spte_has_rmap(vcpu->kvm, sp, &pt[i]);
                 }
         }
-        return nmaps;
+        return;
 }
 
 static void audit_rmap(struct kvm_vcpu *vcpu)
 {
-        int n_rmap = count_rmaps(vcpu);
-        int n_actual = count_writable_mappings(vcpu);
-
-        if (n_rmap != n_actual)
-                printk(KERN_ERR "%s: (%s) rmap %d actual %d\n",
-                       __func__, audit_msg, n_rmap, n_actual);
+        check_writable_mappings_rmap(vcpu);
+        count_rmaps(vcpu);
 }
 
 static void audit_write_protection(struct kvm_vcpu *vcpu)
@@ -3173,20 +3344,28 @@ static void audit_write_protection(struct kvm_vcpu *vcpu)
         struct kvm_mmu_page *sp;
         struct kvm_memory_slot *slot;
         unsigned long *rmapp;
+        u64 *spte;
         gfn_t gfn;
 
         list_for_each_entry(sp, &vcpu->kvm->arch.active_mmu_pages, link) {
                 if (sp->role.direct)
                         continue;
+                if (sp->unsync)
+                        continue;
 
                 gfn = unalias_gfn(vcpu->kvm, sp->gfn);
                 slot = gfn_to_memslot_unaliased(vcpu->kvm, sp->gfn);
                 rmapp = &slot->rmap[gfn - slot->base_gfn];
-                if (*rmapp)
-                        printk(KERN_ERR "%s: (%s) shadow page has writable"
-                               " mappings: gfn %lx role %x\n",
+
+                spte = rmap_next(vcpu->kvm, rmapp, NULL);
+                while (spte) {
+                        if (*spte & PT_WRITABLE_MASK)
+                                printk(KERN_ERR "%s: (%s) shadow page has "
+                                "writable mappings: gfn %lx role %x\n",
                                __func__, audit_msg, sp->gfn,
                                sp->role.word);
+                        spte = rmap_next(vcpu->kvm, rmapp, spte);
+                }
         }
 }
 
@@ -3198,7 +3377,9 @@ static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg)
         audit_msg = msg;
         audit_rmap(vcpu);
         audit_write_protection(vcpu);
-        audit_mappings(vcpu);
+        if (strcmp("pre pte write", audit_msg) != 0)
+                audit_mappings(vcpu);
+        audit_writable_sptes_have_rmaps(vcpu);
         dbg = olddbg;
 }
 
diff --git a/arch/x86/kvm/mmu.h b/arch/x86/kvm/mmu.h
index 3494a2fb136e..61a1b3884b49 100644
--- a/arch/x86/kvm/mmu.h
+++ b/arch/x86/kvm/mmu.h
@@ -37,6 +37,8 @@
 #define PT32_ROOT_LEVEL 2
 #define PT32E_ROOT_LEVEL 3
 
+int kvm_mmu_get_spte_hierarchy(struct kvm_vcpu *vcpu, u64 addr, u64 sptes[4]);
+
 static inline void kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu)
 {
         if (unlikely(vcpu->kvm->arch.n_free_mmu_pages < KVM_MIN_FREE_MMU_PAGES))
@@ -75,7 +77,7 @@ static inline int is_paging(struct kvm_vcpu *vcpu)
         return vcpu->arch.cr0 & X86_CR0_PG;
 }
 
-static inline int is_present_pte(unsigned long pte)
+static inline int is_present_gpte(unsigned long pte)
 {
         return pte & PT_PRESENT_MASK;
 }
diff --git a/arch/x86/kvm/mmutrace.h b/arch/x86/kvm/mmutrace.h
new file mode 100644
index 000000000000..3e4a5c6ca2a9
--- /dev/null
+++ b/arch/x86/kvm/mmutrace.h
@@ -0,0 +1,220 @@
+#if !defined(_TRACE_KVMMMU_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_KVMMMU_H
+
+#include <linux/tracepoint.h>
+#include <linux/ftrace_event.h>
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM kvmmmu
+#define TRACE_INCLUDE_PATH .
+#define TRACE_INCLUDE_FILE mmutrace
+
+#define KVM_MMU_PAGE_FIELDS \
+        __field(__u64, gfn) \
+        __field(__u32, role) \
+        __field(__u32, root_count) \
+        __field(__u32, unsync)
+
+#define KVM_MMU_PAGE_ASSIGN(sp)                      \
+        __entry->gfn = sp->gfn;                      \
+        __entry->role = sp->role.word;               \
+        __entry->root_count = sp->root_count;        \
+        __entry->unsync = sp->unsync;
+
+#define KVM_MMU_PAGE_PRINTK() ({                                        \
+        const char *ret = p->buffer + p->len;                           \
+        static const char *access_str[] = {                             \
+                "---", "--x", "w--", "w-x", "-u-", "-ux", "wu-", "wux"  \
+        };                                                              \
+        union kvm_mmu_page_role role;                                   \
+                                                                        \
+        role.word = __entry->role;                                      \
+                                                                        \
+        trace_seq_printf(p, "sp gfn %llx %u/%u q%u%s %s%s %spge"        \
+                         " %snxe root %u %s%c",                         \
+                         __entry->gfn, role.level, role.glevels,        \
+                         role.quadrant,                                 \
+                         role.direct ? " direct" : "",                  \
+                         access_str[role.access],                       \
+                         role.invalid ? " invalid" : "",                \
+                         role.cr4_pge ? "" : "!",                       \
+                         role.nxe ? "" : "!",                           \
+                         __entry->root_count,                           \
+                         __entry->unsync ? "unsync" : "sync", 0);       \
+        ret;                                                            \
+                })
+
+#define kvm_mmu_trace_pferr_flags       \
+        { PFERR_PRESENT_MASK, "P" },    \
+        { PFERR_WRITE_MASK, "W" },      \
+        { PFERR_USER_MASK, "U" },       \
+        { PFERR_RSVD_MASK, "RSVD" },    \
+        { PFERR_FETCH_MASK, "F" }
+
+/*
+ * A pagetable walk has started
+ */
+TRACE_EVENT(
+        kvm_mmu_pagetable_walk,
+        TP_PROTO(u64 addr, int write_fault, int user_fault, int fetch_fault),
+        TP_ARGS(addr, write_fault, user_fault, fetch_fault),
+
+        TP_STRUCT__entry(
+                __field(__u64, addr)
+                __field(__u32, pferr)
+        ),
+
+        TP_fast_assign(
+                __entry->addr = addr;
+                __entry->pferr = (!!write_fault << 1) | (!!user_fault << 2)
+                                 | (!!fetch_fault << 4);
+        ),
+
+        TP_printk("addr %llx pferr %x %s", __entry->addr, __entry->pferr,
+                  __print_flags(__entry->pferr, "|", kvm_mmu_trace_pferr_flags))
+);
+
+
+/* We just walked a paging element */
+TRACE_EVENT(
+        kvm_mmu_paging_element,
+        TP_PROTO(u64 pte, int level),
+        TP_ARGS(pte, level),
+
+        TP_STRUCT__entry(
+                __field(__u64, pte)
+                __field(__u32, level)
+                ),
+
+        TP_fast_assign(
+                __entry->pte = pte;
+                __entry->level = level;
+                ),
+
+        TP_printk("pte %llx level %u", __entry->pte, __entry->level)
+);
+
+/* We set a pte accessed bit */
+TRACE_EVENT(
+        kvm_mmu_set_accessed_bit,
+        TP_PROTO(unsigned long table_gfn, unsigned index, unsigned size),
+        TP_ARGS(table_gfn, index, size),
+
+        TP_STRUCT__entry(
+                __field(__u64, gpa)
+                ),
+
+        TP_fast_assign(
+                __entry->gpa = ((u64)table_gfn << PAGE_SHIFT)
+                                + index * size;
+                ),
+
+        TP_printk("gpa %llx", __entry->gpa)
+);
+
+/* We set a pte dirty bit */
+TRACE_EVENT(
+        kvm_mmu_set_dirty_bit,
+        TP_PROTO(unsigned long table_gfn, unsigned index, unsigned size),
+        TP_ARGS(table_gfn, index, size),
+
+        TP_STRUCT__entry(
+                __field(__u64, gpa)
+                ),
+
+        TP_fast_assign(
+                __entry->gpa = ((u64)table_gfn << PAGE_SHIFT)
+                                + index * size;
+                ),
+
+        TP_printk("gpa %llx", __entry->gpa)
+);
+
+TRACE_EVENT(
+        kvm_mmu_walker_error,
+        TP_PROTO(u32 pferr),
+        TP_ARGS(pferr),
+
+        TP_STRUCT__entry(
+                __field(__u32, pferr)
+                ),
+
+        TP_fast_assign(
+                __entry->pferr = pferr;
+                ),
+
+        TP_printk("pferr %x %s", __entry->pferr,
+                  __print_flags(__entry->pferr, "|", kvm_mmu_trace_pferr_flags))
+);
+
+TRACE_EVENT(
+        kvm_mmu_get_page,
+        TP_PROTO(struct kvm_mmu_page *sp, bool created),
+        TP_ARGS(sp, created),
+
+        TP_STRUCT__entry(
+                KVM_MMU_PAGE_FIELDS
+                __field(bool, created)
+                ),
+
+        TP_fast_assign(
+                KVM_MMU_PAGE_ASSIGN(sp)
+                __entry->created = created;
+                ),
+
+        TP_printk("%s %s", KVM_MMU_PAGE_PRINTK(),
+                  __entry->created ? "new" : "existing")
+);
+
+TRACE_EVENT(
+        kvm_mmu_sync_page,
+        TP_PROTO(struct kvm_mmu_page *sp),
+        TP_ARGS(sp),
+
+        TP_STRUCT__entry(
+                KVM_MMU_PAGE_FIELDS
+                ),
+
+        TP_fast_assign(
+                KVM_MMU_PAGE_ASSIGN(sp)
+                ),
+
+        TP_printk("%s", KVM_MMU_PAGE_PRINTK())
+);
+
+TRACE_EVENT(
+        kvm_mmu_unsync_page,
+        TP_PROTO(struct kvm_mmu_page *sp),
+        TP_ARGS(sp),
+
+        TP_STRUCT__entry(
+                KVM_MMU_PAGE_FIELDS
+                ),
+
+        TP_fast_assign(
+                KVM_MMU_PAGE_ASSIGN(sp)
+                ),
+
+        TP_printk("%s", KVM_MMU_PAGE_PRINTK())
+);
+
+TRACE_EVENT(
+        kvm_mmu_zap_page,
+        TP_PROTO(struct kvm_mmu_page *sp),
+        TP_ARGS(sp),
+
+        TP_STRUCT__entry(
+                KVM_MMU_PAGE_FIELDS
+                ),
+
+        TP_fast_assign(
+                KVM_MMU_PAGE_ASSIGN(sp)
+                ),
+
+        TP_printk("%s", KVM_MMU_PAGE_PRINTK())
+);
+
+#endif /* _TRACE_KVMMMU_H */
+
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/arch/x86/kvm/paging_tmpl.h b/arch/x86/kvm/paging_tmpl.h
index 67785f635399..d2fec9c12d22 100644
--- a/arch/x86/kvm/paging_tmpl.h
+++ b/arch/x86/kvm/paging_tmpl.h
@@ -27,7 +27,8 @@
         #define guest_walker guest_walker64
         #define FNAME(name) paging##64_##name
         #define PT_BASE_ADDR_MASK PT64_BASE_ADDR_MASK
-        #define PT_DIR_BASE_ADDR_MASK PT64_DIR_BASE_ADDR_MASK
+        #define PT_LVL_ADDR_MASK(lvl) PT64_LVL_ADDR_MASK(lvl)
+        #define PT_LVL_OFFSET_MASK(lvl) PT64_LVL_OFFSET_MASK(lvl)
         #define PT_INDEX(addr, level) PT64_INDEX(addr, level)
         #define PT_LEVEL_MASK(level) PT64_LEVEL_MASK(level)
         #define PT_LEVEL_BITS PT64_LEVEL_BITS
@@ -43,7 +44,8 @@
         #define guest_walker guest_walker32
         #define FNAME(name) paging##32_##name
         #define PT_BASE_ADDR_MASK PT32_BASE_ADDR_MASK
-        #define PT_DIR_BASE_ADDR_MASK PT32_DIR_BASE_ADDR_MASK
+        #define PT_LVL_ADDR_MASK(lvl) PT32_LVL_ADDR_MASK(lvl)
+        #define PT_LVL_OFFSET_MASK(lvl) PT32_LVL_OFFSET_MASK(lvl)
         #define PT_INDEX(addr, level) PT32_INDEX(addr, level)
         #define PT_LEVEL_MASK(level) PT32_LEVEL_MASK(level)
         #define PT_LEVEL_BITS PT32_LEVEL_BITS
@@ -53,8 +55,8 @@
         #error Invalid PTTYPE value
 #endif
 
-#define gpte_to_gfn FNAME(gpte_to_gfn)
-#define gpte_to_gfn_pde FNAME(gpte_to_gfn_pde)
+#define gpte_to_gfn_lvl FNAME(gpte_to_gfn_lvl)
+#define gpte_to_gfn(pte) gpte_to_gfn_lvl((pte), PT_PAGE_TABLE_LEVEL)
 
 /*
  * The guest_walker structure emulates the behavior of the hardware page
@@ -71,14 +73,9 @@ struct guest_walker {
         u32 error_code;
 };
 
-static gfn_t gpte_to_gfn(pt_element_t gpte)
+static gfn_t gpte_to_gfn_lvl(pt_element_t gpte, int lvl)
 {
-        return (gpte & PT_BASE_ADDR_MASK) >> PAGE_SHIFT;
-}
-
-static gfn_t gpte_to_gfn_pde(pt_element_t gpte)
-{
-        return (gpte & PT_DIR_BASE_ADDR_MASK) >> PAGE_SHIFT;
+        return (gpte & PT_LVL_ADDR_MASK(lvl)) >> PAGE_SHIFT;
 }
 
 static bool FNAME(cmpxchg_gpte)(struct kvm *kvm,
@@ -125,14 +122,16 @@ static int FNAME(walk_addr)(struct guest_walker *walker,
         gpa_t pte_gpa;
         int rsvd_fault = 0;
 
-        pgprintk("%s: addr %lx\n", __func__, addr);
+        trace_kvm_mmu_pagetable_walk(addr, write_fault, user_fault,
+                                     fetch_fault);
 walk:
         walker->level = vcpu->arch.mmu.root_level;
         pte = vcpu->arch.cr3;
 #if PTTYPE == 64
         if (!is_long_mode(vcpu)) {
-                pte = vcpu->arch.pdptrs[(addr >> 30) & 3];
-                if (!is_present_pte(pte))
+                pte = kvm_pdptr_read(vcpu, (addr >> 30) & 3);
+                trace_kvm_mmu_paging_element(pte, walker->level);
+                if (!is_present_gpte(pte))
                         goto not_present;
                 --walker->level;
         }
@@ -150,12 +149,11 @@ walk:
                 pte_gpa += index * sizeof(pt_element_t);
                 walker->table_gfn[walker->level - 1] = table_gfn;
                 walker->pte_gpa[walker->level - 1] = pte_gpa;
-                pgprintk("%s: table_gfn[%d] %lx\n", __func__,
-                         walker->level - 1, table_gfn);
 
                 kvm_read_guest(vcpu->kvm, pte_gpa, &pte, sizeof(pte));
+                trace_kvm_mmu_paging_element(pte, walker->level);
 
-                if (!is_present_pte(pte))
+                if (!is_present_gpte(pte))
                         goto not_present;
 
                 rsvd_fault = is_rsvd_bits_set(vcpu, pte, walker->level);
@@ -175,6 +173,8 @@ walk:
 #endif
 
                 if (!(pte & PT_ACCESSED_MASK)) {
+                        trace_kvm_mmu_set_accessed_bit(table_gfn, index,
+                                                       sizeof(pte));
                         mark_page_dirty(vcpu->kvm, table_gfn);
                         if (FNAME(cmpxchg_gpte)(vcpu->kvm, table_gfn,
                             index, pte, pte|PT_ACCESSED_MASK))
@@ -186,18 +186,24 @@ walk:
 
                 walker->ptes[walker->level - 1] = pte;
 
-                if (walker->level == PT_PAGE_TABLE_LEVEL) {
-                        walker->gfn = gpte_to_gfn(pte);
-                        break;
-                }
-
-                if (walker->level == PT_DIRECTORY_LEVEL
-                    && (pte & PT_PAGE_SIZE_MASK)
-                    && (PTTYPE == 64 || is_pse(vcpu))) {
-                        walker->gfn = gpte_to_gfn_pde(pte);
-                        walker->gfn += PT_INDEX(addr, PT_PAGE_TABLE_LEVEL);
-                        if (PTTYPE == 32 && is_cpuid_PSE36())
+                if ((walker->level == PT_PAGE_TABLE_LEVEL) ||
+                    ((walker->level == PT_DIRECTORY_LEVEL) &&
+                                (pte & PT_PAGE_SIZE_MASK)  &&
+                                (PTTYPE == 64 || is_pse(vcpu))) ||
+                    ((walker->level == PT_PDPE_LEVEL) &&
+                                (pte & PT_PAGE_SIZE_MASK)  &&
+                                is_long_mode(vcpu))) {
+                        int lvl = walker->level;
+
+                        walker->gfn = gpte_to_gfn_lvl(pte, lvl);
+                        walker->gfn += (addr & PT_LVL_OFFSET_MASK(lvl))
+                                        >> PAGE_SHIFT;
+
+                        if (PTTYPE == 32 &&
+                            walker->level == PT_DIRECTORY_LEVEL &&
+                            is_cpuid_PSE36())
                                 walker->gfn += pse36_gfn_delta(pte);
+
                         break;
                 }
 
@@ -205,9 +211,10 @@ walk:
                 --walker->level;
         }
 
-        if (write_fault && !is_dirty_pte(pte)) {
+        if (write_fault && !is_dirty_gpte(pte)) {
                 bool ret;
 
+                trace_kvm_mmu_set_dirty_bit(table_gfn, index, sizeof(pte));
                 mark_page_dirty(vcpu->kvm, table_gfn);
                 ret = FNAME(cmpxchg_gpte)(vcpu->kvm, table_gfn, index, pte,
                             pte|PT_DIRTY_MASK);
@@ -239,6 +246,7 @@ err:
                 walker->error_code |= PFERR_FETCH_MASK;
         if (rsvd_fault)
                 walker->error_code |= PFERR_RSVD_MASK;
+        trace_kvm_mmu_walker_error(walker->error_code);
         return 0;
 }
 
@@ -248,12 +256,11 @@ static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *page,
         pt_element_t gpte;
         unsigned pte_access;
         pfn_t pfn;
-        int largepage = vcpu->arch.update_pte.largepage;
 
         gpte = *(const pt_element_t *)pte;
         if (~gpte & (PT_PRESENT_MASK | PT_ACCESSED_MASK)) {
-                if (!is_present_pte(gpte))
-                        set_shadow_pte(spte, shadow_notrap_nonpresent_pte);
+                if (!is_present_gpte(gpte))
+                        __set_spte(spte, shadow_notrap_nonpresent_pte);
                 return;
         }
         pgprintk("%s: gpte %llx spte %p\n", __func__, (u64)gpte, spte);
@@ -267,7 +274,7 @@ static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *page,
                 return;
         kvm_get_pfn(pfn);
         mmu_set_spte(vcpu, spte, page->role.access, pte_access, 0, 0,
-                     gpte & PT_DIRTY_MASK, NULL, largepage,
+                     gpte & PT_DIRTY_MASK, NULL, PT_PAGE_TABLE_LEVEL,
                      gpte_to_gfn(gpte), pfn, true);
 }
 
@@ -276,7 +283,7 @@ static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *page,
  */
 static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr,
                          struct guest_walker *gw,
-                         int user_fault, int write_fault, int largepage,
+                         int user_fault, int write_fault, int hlevel,
                          int *ptwrite, pfn_t pfn)
 {
         unsigned access = gw->pt_access;
@@ -289,19 +296,18 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr,
         pt_element_t curr_pte;
         struct kvm_shadow_walk_iterator iterator;
 
-        if (!is_present_pte(gw->ptes[gw->level - 1]))
+        if (!is_present_gpte(gw->ptes[gw->level - 1]))
                 return NULL;
 
         for_each_shadow_entry(vcpu, addr, iterator) {
                 level = iterator.level;
                 sptep = iterator.sptep;
-                if (level == PT_PAGE_TABLE_LEVEL
-                    || (largepage && level == PT_DIRECTORY_LEVEL)) {
+                if (iterator.level == hlevel) {
                         mmu_set_spte(vcpu, sptep, access,
                                      gw->pte_access & access,
                                      user_fault, write_fault,
                                      gw->ptes[gw->level-1] & PT_DIRTY_MASK,
-                                     ptwrite, largepage,
+                                     ptwrite, level,
                                      gw->gfn, pfn, false);
                         break;
                 }
@@ -311,16 +317,19 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr,
 
                 if (is_large_pte(*sptep)) {
                         rmap_remove(vcpu->kvm, sptep);
-                        set_shadow_pte(sptep, shadow_trap_nonpresent_pte);
+                        __set_spte(sptep, shadow_trap_nonpresent_pte);
                         kvm_flush_remote_tlbs(vcpu->kvm);
                 }
 
-                if (level == PT_DIRECTORY_LEVEL
-                    && gw->level == PT_DIRECTORY_LEVEL) {
+                if (level <= gw->level) {
+                        int delta = level - gw->level + 1;
                         direct = 1;
-                        if (!is_dirty_pte(gw->ptes[level - 1]))
+                        if (!is_dirty_gpte(gw->ptes[level - delta]))
                                 access &= ~ACC_WRITE_MASK;
-                        table_gfn = gpte_to_gfn(gw->ptes[level - 1]);
+                        table_gfn = gpte_to_gfn(gw->ptes[level - delta]);
+                        /* advance table_gfn when emulating 1gb pages with 4k */
+                        if (delta == 0)
+                                table_gfn += PT_INDEX(addr, level);
                 } else {
                         direct = 0;
                         table_gfn = gw->table_gfn[level - 2];
@@ -369,11 +378,11 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr,
         int user_fault = error_code & PFERR_USER_MASK;
         int fetch_fault = error_code & PFERR_FETCH_MASK;
         struct guest_walker walker;
-        u64 *shadow_pte;
+        u64 *sptep;
         int write_pt = 0;
         int r;
         pfn_t pfn;
-        int largepage = 0;
+        int level = PT_PAGE_TABLE_LEVEL;
         unsigned long mmu_seq;
 
         pgprintk("%s: addr %lx err %x\n", __func__, addr, error_code);
@@ -399,14 +408,11 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr,
                 return 0;
         }
 
-        if (walker.level == PT_DIRECTORY_LEVEL) {
-                gfn_t large_gfn;
-                large_gfn = walker.gfn & ~(KVM_PAGES_PER_HPAGE-1);
-                if (is_largepage_backed(vcpu, large_gfn)) {
-                        walker.gfn = large_gfn;
-                        largepage = 1;
-                }
+        if (walker.level >= PT_DIRECTORY_LEVEL) {
+                level = min(walker.level, mapping_level(vcpu, walker.gfn));
+                walker.gfn = walker.gfn & ~(KVM_PAGES_PER_HPAGE(level) - 1);
         }
+
         mmu_seq = vcpu->kvm->mmu_notifier_seq;
         smp_rmb();
         pfn = gfn_to_pfn(vcpu->kvm, walker.gfn);
@@ -422,11 +428,10 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr,
         if (mmu_notifier_retry(vcpu, mmu_seq))
                 goto out_unlock;
         kvm_mmu_free_some_pages(vcpu);
-        shadow_pte = FNAME(fetch)(vcpu, addr, &walker, user_fault, write_fault,
-                                  largepage, &write_pt, pfn);
-
+        sptep = FNAME(fetch)(vcpu, addr, &walker, user_fault, write_fault,
+                             level, &write_pt, pfn);
         pgprintk("%s: shadow pte %p %llx ptwrite %d\n", __func__,
-                 shadow_pte, *shadow_pte, write_pt);
+                 sptep, *sptep, write_pt);
 
         if (!write_pt)
                 vcpu->arch.last_pt_write_count = 0; /* reset fork detector */
@@ -459,8 +464,9 @@ static void FNAME(invlpg)(struct kvm_vcpu *vcpu, gva_t gva)
                 sptep = iterator.sptep;
 
                 /* FIXME: properly handle invlpg on large guest pages */
-                if (level == PT_PAGE_TABLE_LEVEL ||
-                    ((level == PT_DIRECTORY_LEVEL) && is_large_pte(*sptep))) {
+                if (level == PT_PAGE_TABLE_LEVEL  ||
+                    ((level == PT_DIRECTORY_LEVEL && is_large_pte(*sptep))) ||
+                    ((level == PT_PDPE_LEVEL && is_large_pte(*sptep)))) {
                         struct kvm_mmu_page *sp = page_header(__pa(sptep));
 
                         pte_gpa = (sp->gfn << PAGE_SHIFT);
@@ -472,7 +478,7 @@ static void FNAME(invlpg)(struct kvm_vcpu *vcpu, gva_t gva)
                                         --vcpu->kvm->stat.lpages;
                                 need_flush = 1;
                         }
-                        set_shadow_pte(sptep, shadow_trap_nonpresent_pte);
+                        __set_spte(sptep, shadow_trap_nonpresent_pte);
                         break;
                 }
 
@@ -489,7 +495,7 @@ static void FNAME(invlpg)(struct kvm_vcpu *vcpu, gva_t gva)
         if (kvm_read_guest_atomic(vcpu->kvm, pte_gpa, &gpte,
                                   sizeof(pt_element_t)))
                 return;
-        if (is_present_pte(gpte) && (gpte & PT_ACCESSED_MASK)) {
+        if (is_present_gpte(gpte) && (gpte & PT_ACCESSED_MASK)) {
                 if (mmu_topup_memory_caches(vcpu))
                         return;
                 kvm_mmu_pte_write(vcpu, pte_gpa, (const u8 *)&gpte,
@@ -536,7 +542,7 @@ static void FNAME(prefetch_page)(struct kvm_vcpu *vcpu,
                 r = kvm_read_guest_atomic(vcpu->kvm, pte_gpa, pt, sizeof pt);
                 pte_gpa += ARRAY_SIZE(pt) * sizeof(pt_element_t);
                 for (j = 0; j < ARRAY_SIZE(pt); ++j)
-                        if (r || is_present_pte(pt[j]))
+                        if (r || is_present_gpte(pt[j]))
                                 sp->spt[i+j] = shadow_trap_nonpresent_pte;
                         else
                                 sp->spt[i+j] = shadow_notrap_nonpresent_pte;
@@ -574,23 +580,23 @@ static int FNAME(sync_page)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
                                           sizeof(pt_element_t)))
                         return -EINVAL;
 
-                if (gpte_to_gfn(gpte) != gfn || !is_present_pte(gpte) ||
+                if (gpte_to_gfn(gpte) != gfn || !is_present_gpte(gpte) ||
                     !(gpte & PT_ACCESSED_MASK)) {
                         u64 nonpresent;
 
                         rmap_remove(vcpu->kvm, &sp->spt[i]);
-                        if (is_present_pte(gpte))
+                        if (is_present_gpte(gpte))
                                 nonpresent = shadow_trap_nonpresent_pte;
                         else
                                 nonpresent = shadow_notrap_nonpresent_pte;
-                        set_shadow_pte(&sp->spt[i], nonpresent);
+                        __set_spte(&sp->spt[i], nonpresent);
                         continue;
                 }
 
                 nr_present++;
                 pte_access = sp->role.access & FNAME(gpte_access)(vcpu, gpte);
                 set_spte(vcpu, &sp->spt[i], pte_access, 0, 0,
-                         is_dirty_pte(gpte), 0, gfn,
+                         is_dirty_gpte(gpte), PT_PAGE_TABLE_LEVEL, gfn,
                          spte_to_pfn(sp->spt[i]), true, false);
         }
 
@@ -603,9 +609,10 @@ static int FNAME(sync_page)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
 #undef PT_BASE_ADDR_MASK
 #undef PT_INDEX
 #undef PT_LEVEL_MASK
-#undef PT_DIR_BASE_ADDR_MASK
+#undef PT_LVL_ADDR_MASK
+#undef PT_LVL_OFFSET_MASK
 #undef PT_LEVEL_BITS
 #undef PT_MAX_FULL_LEVELS
 #undef gpte_to_gfn
-#undef gpte_to_gfn_pde
+#undef gpte_to_gfn_lvl
 #undef CMPXCHG
diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c
index b1f658ad2f06..944cc9c04b3c 100644
--- a/arch/x86/kvm/svm.c
+++ b/arch/x86/kvm/svm.c
@@ -15,7 +15,6 @@
  */
 #include <linux/kvm_host.h>
 
-#include "kvm_svm.h"
 #include "irq.h"
 #include "mmu.h"
 #include "kvm_cache_regs.h"
@@ -26,10 +25,12 @@
 #include <linux/vmalloc.h>
 #include <linux/highmem.h>
 #include <linux/sched.h>
+#include <linux/ftrace_event.h>
 
 #include <asm/desc.h>
 
 #include <asm/virtext.h>
+#include "trace.h"
 
 #define __ex(x) __kvm_handle_fault_on_reboot(x)
 
@@ -46,6 +47,10 @@ MODULE_LICENSE("GPL");
 #define SVM_FEATURE_LBRV (1 << 1)
 #define SVM_FEATURE_SVML (1 << 2)
 
+#define NESTED_EXIT_HOST        0       /* Exit handled on host level */
+#define NESTED_EXIT_DONE        1       /* Exit caused nested vmexit  */
+#define NESTED_EXIT_CONTINUE    2       /* Further checks needed      */
+
 #define DEBUGCTL_RESERVED_BITS (~(0x3fULL))
 
 /* Turn on to get debugging output*/
@@ -57,6 +62,58 @@ MODULE_LICENSE("GPL");
 #define nsvm_printk(fmt, args...) do {} while(0)
 #endif
 
+static const u32 host_save_user_msrs[] = {
+#ifdef CONFIG_X86_64
+        MSR_STAR, MSR_LSTAR, MSR_CSTAR, MSR_SYSCALL_MASK, MSR_KERNEL_GS_BASE,
+        MSR_FS_BASE,
+#endif
+        MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
+};
+
+#define NR_HOST_SAVE_USER_MSRS ARRAY_SIZE(host_save_user_msrs)
+
+struct kvm_vcpu;
+
+struct nested_state {
+        struct vmcb *hsave;
+        u64 hsave_msr;
+        u64 vmcb;
+
+        /* These are the merged vectors */
+        u32 *msrpm;
+
+        /* gpa pointers to the real vectors */
+        u64 vmcb_msrpm;
+
+        /* cache for intercepts of the guest */
+        u16 intercept_cr_read;
+        u16 intercept_cr_write;
+        u16 intercept_dr_read;
+        u16 intercept_dr_write;
+        u32 intercept_exceptions;
+        u64 intercept;
+
+};
+
+struct vcpu_svm {
+        struct kvm_vcpu vcpu;
+        struct vmcb *vmcb;
+        unsigned long vmcb_pa;
+        struct svm_cpu_data *svm_data;
+        uint64_t asid_generation;
+        uint64_t sysenter_esp;
+        uint64_t sysenter_eip;
+
+        u64 next_rip;
+
+        u64 host_user_msrs[NR_HOST_SAVE_USER_MSRS];
+        u64 host_gs_base;
+
+        u32 *msrpm;
+
+        struct nested_state nested;
+};
+
 /* enable NPT for AMD64 and X86 with PAE */
 #if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
 static bool npt_enabled = true;
@@ -67,15 +124,14 @@ static int npt = 1;
 
 module_param(npt, int, S_IRUGO);
 
-static int nested = 0;
+static int nested = 1;
 module_param(nested, int, S_IRUGO);
 
 static void svm_flush_tlb(struct kvm_vcpu *vcpu);
+static void svm_complete_interrupts(struct vcpu_svm *svm);
 
-static int nested_svm_exit_handled(struct vcpu_svm *svm, bool kvm_override);
+static int nested_svm_exit_handled(struct vcpu_svm *svm);
 static int nested_svm_vmexit(struct vcpu_svm *svm);
-static int nested_svm_vmsave(struct vcpu_svm *svm, void *nested_vmcb,
-                             void *arg2, void *opaque);
 static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr,
                                       bool has_error_code, u32 error_code);
 
@@ -86,7 +142,22 @@ static inline struct vcpu_svm *to_svm(struct kvm_vcpu *vcpu)
 
 static inline bool is_nested(struct vcpu_svm *svm)
 {
-        return svm->nested_vmcb;
+        return svm->nested.vmcb;
+}
+
+static inline void enable_gif(struct vcpu_svm *svm)
+{
+        svm->vcpu.arch.hflags |= HF_GIF_MASK;
+}
+
+static inline void disable_gif(struct vcpu_svm *svm)
+{
+        svm->vcpu.arch.hflags &= ~HF_GIF_MASK;
+}
+
+static inline bool gif_set(struct vcpu_svm *svm)
+{
+        return !!(svm->vcpu.arch.hflags & HF_GIF_MASK);
 }
 
 static unsigned long iopm_base;
@@ -147,19 +218,6 @@ static inline void invlpga(unsigned long addr, u32 asid)
         asm volatile (__ex(SVM_INVLPGA) :: "a"(addr), "c"(asid));
 }
 
-static inline unsigned long kvm_read_cr2(void)
-{
-        unsigned long cr2;
-
-        asm volatile ("mov %%cr2, %0" : "=r" (cr2));
-        return cr2;
-}
-
-static inline void kvm_write_cr2(unsigned long val)
-{
-        asm volatile ("mov %0, %%cr2" :: "r" (val));
-}
-
 static inline void force_new_asid(struct kvm_vcpu *vcpu)
 {
         to_svm(vcpu)->asid_generation--;
@@ -263,7 +321,7 @@ static void svm_hardware_enable(void *garbage)
 
         struct svm_cpu_data *svm_data;
         uint64_t efer;
-        struct desc_ptr gdt_descr;
+        struct descriptor_table gdt_descr;
         struct desc_struct *gdt;
         int me = raw_smp_processor_id();
 
@@ -283,8 +341,8 @@ static void svm_hardware_enable(void *garbage)
         svm_data->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1;
         svm_data->next_asid = svm_data->max_asid + 1;
 
-        asm volatile ("sgdt %0" : "=m"(gdt_descr));
-        gdt = (struct desc_struct *)gdt_descr.address;
+        kvm_get_gdt(&gdt_descr);
+        gdt = (struct desc_struct *)gdt_descr.base;
         svm_data->tss_desc = (struct kvm_ldttss_desc *)(gdt + GDT_ENTRY_TSS);
 
         rdmsrl(MSR_EFER, efer);
@@ -367,8 +425,6 @@ static void svm_vcpu_init_msrpm(u32 *msrpm)
 #endif
         set_msr_interception(msrpm, MSR_K6_STAR, 1, 1);
         set_msr_interception(msrpm, MSR_IA32_SYSENTER_CS, 1, 1);
-        set_msr_interception(msrpm, MSR_IA32_SYSENTER_ESP, 1, 1);
-        set_msr_interception(msrpm, MSR_IA32_SYSENTER_EIP, 1, 1);
 }
 
 static void svm_enable_lbrv(struct vcpu_svm *svm)
@@ -595,8 +651,10 @@ static void init_vmcb(struct vcpu_svm *svm)
         }
         force_new_asid(&svm->vcpu);
 
-        svm->nested_vmcb = 0;
-        svm->vcpu.arch.hflags = HF_GIF_MASK;
+        svm->nested.vmcb = 0;
+        svm->vcpu.arch.hflags = 0;
+
+        enable_gif(svm);
 }
 
 static int svm_vcpu_reset(struct kvm_vcpu *vcpu)
@@ -605,7 +663,7 @@ static int svm_vcpu_reset(struct kvm_vcpu *vcpu)
 
         init_vmcb(svm);
 
-        if (vcpu->vcpu_id != 0) {
+        if (!kvm_vcpu_is_bsp(vcpu)) {
                 kvm_rip_write(vcpu, 0);
                 svm->vmcb->save.cs.base = svm->vcpu.arch.sipi_vector << 12;
                 svm->vmcb->save.cs.selector = svm->vcpu.arch.sipi_vector << 8;
@@ -656,9 +714,9 @@ static struct kvm_vcpu *svm_create_vcpu(struct kvm *kvm, unsigned int id)
         hsave_page = alloc_page(GFP_KERNEL);
         if (!hsave_page)
                 goto uninit;
-        svm->hsave = page_address(hsave_page);
+        svm->nested.hsave = page_address(hsave_page);
 
-        svm->nested_msrpm = page_address(nested_msrpm_pages);
+        svm->nested.msrpm = page_address(nested_msrpm_pages);
 
         svm->vmcb = page_address(page);
         clear_page(svm->vmcb);
@@ -669,7 +727,7 @@ static struct kvm_vcpu *svm_create_vcpu(struct kvm *kvm, unsigned int id)
         fx_init(&svm->vcpu);
         svm->vcpu.fpu_active = 1;
         svm->vcpu.arch.apic_base = 0xfee00000 | MSR_IA32_APICBASE_ENABLE;
-        if (svm->vcpu.vcpu_id == 0)
+        if (kvm_vcpu_is_bsp(&svm->vcpu))
                 svm->vcpu.arch.apic_base |= MSR_IA32_APICBASE_BSP;
 
         return &svm->vcpu;
@@ -688,8 +746,8 @@ static void svm_free_vcpu(struct kvm_vcpu *vcpu)
 
         __free_page(pfn_to_page(svm->vmcb_pa >> PAGE_SHIFT));
         __free_pages(virt_to_page(svm->msrpm), MSRPM_ALLOC_ORDER);
-        __free_page(virt_to_page(svm->hsave));
-        __free_pages(virt_to_page(svm->nested_msrpm), MSRPM_ALLOC_ORDER);
+        __free_page(virt_to_page(svm->nested.hsave));
+        __free_pages(virt_to_page(svm->nested.msrpm), MSRPM_ALLOC_ORDER);
         kvm_vcpu_uninit(vcpu);
         kmem_cache_free(kvm_vcpu_cache, svm);
 }
@@ -740,6 +798,18 @@ static void svm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
         to_svm(vcpu)->vmcb->save.rflags = rflags;
 }
 
+static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
+{
+        switch (reg) {
+        case VCPU_EXREG_PDPTR:
+                BUG_ON(!npt_enabled);
+                load_pdptrs(vcpu, vcpu->arch.cr3);
+                break;
+        default:
+                BUG();
+        }
+}
+
 static void svm_set_vintr(struct vcpu_svm *svm)
 {
         svm->vmcb->control.intercept |= 1ULL << INTERCEPT_VINTR;
@@ -1061,7 +1131,6 @@ static unsigned long svm_get_dr(struct kvm_vcpu *vcpu, int dr)
                 val = 0;
         }
 
-        KVMTRACE_2D(DR_READ, vcpu, (u32)dr, (u32)val, handler);
         return val;
 }
 
@@ -1070,8 +1139,6 @@ static void svm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long value,
 {
         struct vcpu_svm *svm = to_svm(vcpu);
 
-        KVMTRACE_2D(DR_WRITE, vcpu, (u32)dr, (u32)value, handler);
-
         *exception = 0;
 
         switch (dr) {
@@ -1119,25 +1186,9 @@ static int pf_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
         fault_address  = svm->vmcb->control.exit_info_2;
         error_code = svm->vmcb->control.exit_info_1;
 
-        if (!npt_enabled)
-                KVMTRACE_3D(PAGE_FAULT, &svm->vcpu, error_code,
-                            (u32)fault_address, (u32)(fault_address >> 32),
-                            handler);
-        else
-                KVMTRACE_3D(TDP_FAULT, &svm->vcpu, error_code,
-                            (u32)fault_address, (u32)(fault_address >> 32),
-                            handler);
-        /*
-         * FIXME: Tis shouldn't be necessary here, but there is a flush
-         * missing in the MMU code. Until we find this bug, flush the
-         * complete TLB here on an NPF
-         */
-        if (npt_enabled)
-                svm_flush_tlb(&svm->vcpu);
-        else {
-                if (kvm_event_needs_reinjection(&svm->vcpu))
-                        kvm_mmu_unprotect_page_virt(&svm->vcpu, fault_address);
-        }
+        trace_kvm_page_fault(fault_address, error_code);
+        if (!npt_enabled && kvm_event_needs_reinjection(&svm->vcpu))
+                kvm_mmu_unprotect_page_virt(&svm->vcpu, fault_address);
         return kvm_mmu_page_fault(&svm->vcpu, fault_address, error_code);
 }
 
@@ -1253,14 +1304,12 @@ static int io_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
 
 static int nmi_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
 {
-        KVMTRACE_0D(NMI, &svm->vcpu, handler);
         return 1;
 }
 
 static int intr_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
 {
         ++svm->vcpu.stat.irq_exits;
-        KVMTRACE_0D(INTR, &svm->vcpu, handler);
         return 1;
 }
 
@@ -1303,44 +1352,39 @@ static int nested_svm_check_permissions(struct vcpu_svm *svm)
 static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr,
                                       bool has_error_code, u32 error_code)
 {
-        if (is_nested(svm)) {
-                svm->vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + nr;
-                svm->vmcb->control.exit_code_hi = 0;
-                svm->vmcb->control.exit_info_1 = error_code;
-                svm->vmcb->control.exit_info_2 = svm->vcpu.arch.cr2;
-                if (nested_svm_exit_handled(svm, false)) {
-                        nsvm_printk("VMexit -> EXCP 0x%x\n", nr);
-
-                        nested_svm_vmexit(svm);
-                        return 1;
-                }
-        }
+        if (!is_nested(svm))
+                return 0;
 
-        return 0;
+        svm->vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + nr;
+        svm->vmcb->control.exit_code_hi = 0;
+        svm->vmcb->control.exit_info_1 = error_code;
+        svm->vmcb->control.exit_info_2 = svm->vcpu.arch.cr2;
+
+        return nested_svm_exit_handled(svm);
 }
 
 static inline int nested_svm_intr(struct vcpu_svm *svm)
 {
-        if (is_nested(svm)) {
-                if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK))
-                        return 0;
+        if (!is_nested(svm))
+                return 0;
 
-                if (!(svm->vcpu.arch.hflags & HF_HIF_MASK))
-                        return 0;
+        if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK))
+                return 0;
 
-                svm->vmcb->control.exit_code = SVM_EXIT_INTR;
+        if (!(svm->vcpu.arch.hflags & HF_HIF_MASK))
+                return 0;
 
-                if (nested_svm_exit_handled(svm, false)) {
-                        nsvm_printk("VMexit -> INTR\n");
-                        nested_svm_vmexit(svm);
-                        return 1;
-                }
+        svm->vmcb->control.exit_code = SVM_EXIT_INTR;
+
+        if (nested_svm_exit_handled(svm)) {
+                nsvm_printk("VMexit -> INTR\n");
+                return 1;
         }
 
         return 0;
 }
 
-static struct page *nested_svm_get_page(struct vcpu_svm *svm, u64 gpa)
+static void *nested_svm_map(struct vcpu_svm *svm, u64 gpa, enum km_type idx)
 {
         struct page *page;
 
@@ -1348,236 +1392,246 @@ static struct page *nested_svm_get_page(struct vcpu_svm *svm, u64 gpa)
         page = gfn_to_page(svm->vcpu.kvm, gpa >> PAGE_SHIFT);
         up_read(&current->mm->mmap_sem);
 
-        if (is_error_page(page)) {
-                printk(KERN_INFO "%s: could not find page at 0x%llx\n",
-                       __func__, gpa);
-                kvm_release_page_clean(page);
-                kvm_inject_gp(&svm->vcpu, 0);
-                return NULL;
-        }
-        return page;
+        if (is_error_page(page))
+                goto error;
+
+        return kmap_atomic(page, idx);
+
+error:
+        kvm_release_page_clean(page);
+        kvm_inject_gp(&svm->vcpu, 0);
+
+        return NULL;
 }
 
-static int nested_svm_do(struct vcpu_svm *svm,
-                         u64 arg1_gpa, u64 arg2_gpa, void *opaque,
-                         int (*handler)(struct vcpu_svm *svm,
-                                        void *arg1,
-                                        void *arg2,
-                                        void *opaque))
+static void nested_svm_unmap(void *addr, enum km_type idx)
 {
-        struct page *arg1_page;
-        struct page *arg2_page = NULL;
-        void *arg1;
-        void *arg2 = NULL;
-        int retval;
+        struct page *page;
 
-        arg1_page = nested_svm_get_page(svm, arg1_gpa);
-        if(arg1_page == NULL)
-                return 1;
+        if (!addr)
+                return;
 
-        if (arg2_gpa) {
-                arg2_page = nested_svm_get_page(svm, arg2_gpa);
-                if(arg2_page == NULL) {
-                        kvm_release_page_clean(arg1_page);
-                        return 1;
-                }
-        }
+        page = kmap_atomic_to_page(addr);
+
+        kunmap_atomic(addr, idx);
+        kvm_release_page_dirty(page);
+}
+
+static bool nested_svm_exit_handled_msr(struct vcpu_svm *svm)
+{
+        u32 param = svm->vmcb->control.exit_info_1 & 1;
+        u32 msr = svm->vcpu.arch.regs[VCPU_REGS_RCX];
+        bool ret = false;
+        u32 t0, t1;
+        u8 *msrpm;
 
-        arg1 = kmap_atomic(arg1_page, KM_USER0);
-        if (arg2_gpa)
-                arg2 = kmap_atomic(arg2_page, KM_USER1);
+        if (!(svm->nested.intercept & (1ULL << INTERCEPT_MSR_PROT)))
+                return false;
 
-        retval = handler(svm, arg1, arg2, opaque);
+        msrpm = nested_svm_map(svm, svm->nested.vmcb_msrpm, KM_USER0);
+
+        if (!msrpm)
+                goto out;
+
+        switch (msr) {
+        case 0 ... 0x1fff:
+                t0 = (msr * 2) % 8;
+                t1 = msr / 8;
+                break;
+        case 0xc0000000 ... 0xc0001fff:
+                t0 = (8192 + msr - 0xc0000000) * 2;
+                t1 = (t0 / 8);
+                t0 %= 8;
+                break;
+        case 0xc0010000 ... 0xc0011fff:
+                t0 = (16384 + msr - 0xc0010000) * 2;
+                t1 = (t0 / 8);
+                t0 %= 8;
+                break;
+        default:
+                ret = true;
+                goto out;
+        }
 
-        kunmap_atomic(arg1, KM_USER0);
-        if (arg2_gpa)
-                kunmap_atomic(arg2, KM_USER1);
+        ret = msrpm[t1] & ((1 << param) << t0);
 
-        kvm_release_page_dirty(arg1_page);
-        if (arg2_gpa)
-                kvm_release_page_dirty(arg2_page);
+out:
+        nested_svm_unmap(msrpm, KM_USER0);
 
-        return retval;
+        return ret;
 }
 
-static int nested_svm_exit_handled_real(struct vcpu_svm *svm,
-                                        void *arg1,
-                                        void *arg2,
-                                        void *opaque)
+static int nested_svm_exit_special(struct vcpu_svm *svm)
 {
-        struct vmcb *nested_vmcb = (struct vmcb *)arg1;
-        bool kvm_overrides = *(bool *)opaque;
         u32 exit_code = svm->vmcb->control.exit_code;
 
-        if (kvm_overrides) {
-                switch (exit_code) {
-                case SVM_EXIT_INTR:
-                case SVM_EXIT_NMI:
-                        return 0;
+        switch (exit_code) {
+        case SVM_EXIT_INTR:
+        case SVM_EXIT_NMI:
+                return NESTED_EXIT_HOST;
                 /* For now we are always handling NPFs when using them */
-                case SVM_EXIT_NPF:
-                        if (npt_enabled)
-                                return 0;
-                        break;
-                /* When we're shadowing, trap PFs */
-                case SVM_EXIT_EXCP_BASE + PF_VECTOR:
-                        if (!npt_enabled)
-                                return 0;
-                        break;
-                default:
-                        break;
-                }
+        case SVM_EXIT_NPF:
+                if (npt_enabled)
+                        return NESTED_EXIT_HOST;
+                break;
+        /* When we're shadowing, trap PFs */
+        case SVM_EXIT_EXCP_BASE + PF_VECTOR:
+                if (!npt_enabled)
+                        return NESTED_EXIT_HOST;
+                break;
+        default:
+                break;
         }
 
+        return NESTED_EXIT_CONTINUE;
+}
+
+/*
+ * If this function returns true, this #vmexit was already handled
+ */
+static int nested_svm_exit_handled(struct vcpu_svm *svm)
+{
+        u32 exit_code = svm->vmcb->control.exit_code;
+        int vmexit = NESTED_EXIT_HOST;
+
         switch (exit_code) {
+        case SVM_EXIT_MSR:
+                vmexit = nested_svm_exit_handled_msr(svm);
+                break;
         case SVM_EXIT_READ_CR0 ... SVM_EXIT_READ_CR8: {
                 u32 cr_bits = 1 << (exit_code - SVM_EXIT_READ_CR0);
-                if (nested_vmcb->control.intercept_cr_read & cr_bits)
-                        return 1;
+                if (svm->nested.intercept_cr_read & cr_bits)
+                        vmexit = NESTED_EXIT_DONE;
                 break;
         }
         case SVM_EXIT_WRITE_CR0 ... SVM_EXIT_WRITE_CR8: {
                 u32 cr_bits = 1 << (exit_code - SVM_EXIT_WRITE_CR0);
-                if (nested_vmcb->control.intercept_cr_write & cr_bits)
-                        return 1;
+                if (svm->nested.intercept_cr_write & cr_bits)
+                        vmexit = NESTED_EXIT_DONE;
                 break;
         }
         case SVM_EXIT_READ_DR0 ... SVM_EXIT_READ_DR7: {
                 u32 dr_bits = 1 << (exit_code - SVM_EXIT_READ_DR0);
-                if (nested_vmcb->control.intercept_dr_read & dr_bits)
-                        return 1;
+                if (svm->nested.intercept_dr_read & dr_bits)
+                        vmexit = NESTED_EXIT_DONE;
                 break;
         }
         case SVM_EXIT_WRITE_DR0 ... SVM_EXIT_WRITE_DR7: {
                 u32 dr_bits = 1 << (exit_code - SVM_EXIT_WRITE_DR0);
-                if (nested_vmcb->control.intercept_dr_write & dr_bits)
-                        return 1;
+                if (svm->nested.intercept_dr_write & dr_bits)
+                        vmexit = NESTED_EXIT_DONE;
                 break;
         }
         case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: {
                 u32 excp_bits = 1 << (exit_code - SVM_EXIT_EXCP_BASE);
-                if (nested_vmcb->control.intercept_exceptions & excp_bits)
-                        return 1;
+                if (svm->nested.intercept_exceptions & excp_bits)
+                        vmexit = NESTED_EXIT_DONE;
                 break;
         }
         default: {
                 u64 exit_bits = 1ULL << (exit_code - SVM_EXIT_INTR);
                 nsvm_printk("exit code: 0x%x\n", exit_code);
-                if (nested_vmcb->control.intercept & exit_bits)
-                        return 1;
+                if (svm->nested.intercept & exit_bits)
+                        vmexit = NESTED_EXIT_DONE;
         }
         }
 
-        return 0;
-}
-
-static int nested_svm_exit_handled_msr(struct vcpu_svm *svm,
-                                       void *arg1, void *arg2,
-                                       void *opaque)
-{
-        struct vmcb *nested_vmcb = (struct vmcb *)arg1;
-        u8 *msrpm = (u8 *)arg2;
-        u32 t0, t1;
-        u32 msr = svm->vcpu.arch.regs[VCPU_REGS_RCX];
-        u32 param = svm->vmcb->control.exit_info_1 & 1;
-
-        if (!(nested_vmcb->control.intercept & (1ULL << INTERCEPT_MSR_PROT)))
-                return 0;
-
-        switch(msr) {
-        case 0 ... 0x1fff:
-                t0 = (msr * 2) % 8;
-                t1 = msr / 8;
-                break;
-        case 0xc0000000 ... 0xc0001fff:
-                t0 = (8192 + msr - 0xc0000000) * 2;
-                t1 = (t0 / 8);
-                t0 %= 8;
-                break;
-        case 0xc0010000 ... 0xc0011fff:
-                t0 = (16384 + msr - 0xc0010000) * 2;
-                t1 = (t0 / 8);
-                t0 %= 8;
-                break;
-        default:
-                return 1;
-                break;
+        if (vmexit == NESTED_EXIT_DONE) {
+                nsvm_printk("#VMEXIT reason=%04x\n", exit_code);
+                nested_svm_vmexit(svm);
         }
-        if (msrpm[t1] & ((1 << param) << t0))
-                return 1;
 
-        return 0;
+        return vmexit;
+}
+
+static inline void copy_vmcb_control_area(struct vmcb *dst_vmcb, struct vmcb *from_vmcb)
+{
+        struct vmcb_control_area *dst  = &dst_vmcb->control;
+        struct vmcb_control_area *from = &from_vmcb->control;
+
+        dst->intercept_cr_read    = from->intercept_cr_read;
+        dst->intercept_cr_write   = from->intercept_cr_write;
+        dst->intercept_dr_read    = from->intercept_dr_read;
+        dst->intercept_dr_write   = from->intercept_dr_write;
+        dst->intercept_exceptions = from->intercept_exceptions;
+        dst->intercept            = from->intercept;
+        dst->iopm_base_pa         = from->iopm_base_pa;
+        dst->msrpm_base_pa        = from->msrpm_base_pa;
+        dst->tsc_offset           = from->tsc_offset;
+        dst->asid                 = from->asid;
+        dst->tlb_ctl              = from->tlb_ctl;
+        dst->int_ctl              = from->int_ctl;
+        dst->int_vector           = from->int_vector;
+        dst->int_state            = from->int_state;
+        dst->exit_code            = from->exit_code;
+        dst->exit_code_hi         = from->exit_code_hi;
+        dst->exit_info_1          = from->exit_info_1;
+        dst->exit_info_2          = from->exit_info_2;
+        dst->exit_int_info        = from->exit_int_info;
+        dst->exit_int_info_err    = from->exit_int_info_err;
+        dst->nested_ctl           = from->nested_ctl;
+        dst->event_inj            = from->event_inj;
+        dst->event_inj_err        = from->event_inj_err;
+        dst->nested_cr3           = from->nested_cr3;
+        dst->lbr_ctl              = from->lbr_ctl;
 }
 
-static int nested_svm_exit_handled(struct vcpu_svm *svm, bool kvm_override)
+static int nested_svm_vmexit(struct vcpu_svm *svm)
 {
-        bool k = kvm_override;
-
-        switch (svm->vmcb->control.exit_code) {
-        case SVM_EXIT_MSR:
-                return nested_svm_do(svm, svm->nested_vmcb,
-                                     svm->nested_vmcb_msrpm, NULL,
-                                     nested_svm_exit_handled_msr);
-        default: break;
-        }
+        struct vmcb *nested_vmcb;
+        struct vmcb *hsave = svm->nested.hsave;
+        struct vmcb *vmcb = svm->vmcb;
 
-        return nested_svm_do(svm, svm->nested_vmcb, 0, &k,
-                             nested_svm_exit_handled_real);
-}
-
-static int nested_svm_vmexit_real(struct vcpu_svm *svm, void *arg1,
-                                  void *arg2, void *opaque)
-{
-        struct vmcb *nested_vmcb = (struct vmcb *)arg1;
-        struct vmcb *hsave = svm->hsave;
-        u64 nested_save[] = { nested_vmcb->save.cr0,
-                              nested_vmcb->save.cr3,
-                              nested_vmcb->save.cr4,
-                              nested_vmcb->save.efer,
-                              nested_vmcb->control.intercept_cr_read,
-                              nested_vmcb->control.intercept_cr_write,
-                              nested_vmcb->control.intercept_dr_read,
-                              nested_vmcb->control.intercept_dr_write,
-                              nested_vmcb->control.intercept_exceptions,
-                              nested_vmcb->control.intercept,
-                              nested_vmcb->control.msrpm_base_pa,
-                              nested_vmcb->control.iopm_base_pa,
-                              nested_vmcb->control.tsc_offset };
+        nested_vmcb = nested_svm_map(svm, svm->nested.vmcb, KM_USER0);
+        if (!nested_vmcb)
+                return 1;
 
         /* Give the current vmcb to the guest */
-        memcpy(nested_vmcb, svm->vmcb, sizeof(struct vmcb));
-        nested_vmcb->save.cr0 = nested_save[0];
-        if (!npt_enabled)
-                nested_vmcb->save.cr3 = nested_save[1];
-        nested_vmcb->save.cr4 = nested_save[2];
-        nested_vmcb->save.efer = nested_save[3];
-        nested_vmcb->control.intercept_cr_read = nested_save[4];
-        nested_vmcb->control.intercept_cr_write = nested_save[5];
-        nested_vmcb->control.intercept_dr_read = nested_save[6];
-        nested_vmcb->control.intercept_dr_write = nested_save[7];
-        nested_vmcb->control.intercept_exceptions = nested_save[8];
-        nested_vmcb->control.intercept = nested_save[9];
-        nested_vmcb->control.msrpm_base_pa = nested_save[10];
-        nested_vmcb->control.iopm_base_pa = nested_save[11];
-        nested_vmcb->control.tsc_offset = nested_save[12];
+        disable_gif(svm);
+
+        nested_vmcb->save.es     = vmcb->save.es;
+        nested_vmcb->save.cs     = vmcb->save.cs;
+        nested_vmcb->save.ss     = vmcb->save.ss;
+        nested_vmcb->save.ds     = vmcb->save.ds;
+        nested_vmcb->save.gdtr   = vmcb->save.gdtr;
+        nested_vmcb->save.idtr   = vmcb->save.idtr;
+        if (npt_enabled)
+                nested_vmcb->save.cr3    = vmcb->save.cr3;
+        nested_vmcb->save.cr2    = vmcb->save.cr2;
+        nested_vmcb->save.rflags = vmcb->save.rflags;
+        nested_vmcb->save.rip    = vmcb->save.rip;
+        nested_vmcb->save.rsp    = vmcb->save.rsp;
+        nested_vmcb->save.rax    = vmcb->save.rax;
+        nested_vmcb->save.dr7    = vmcb->save.dr7;
+        nested_vmcb->save.dr6    = vmcb->save.dr6;
+        nested_vmcb->save.cpl    = vmcb->save.cpl;
+
+        nested_vmcb->control.int_ctl           = vmcb->control.int_ctl;
+        nested_vmcb->control.int_vector        = vmcb->control.int_vector;
+        nested_vmcb->control.int_state         = vmcb->control.int_state;
+        nested_vmcb->control.exit_code         = vmcb->control.exit_code;
+        nested_vmcb->control.exit_code_hi      = vmcb->control.exit_code_hi;
+        nested_vmcb->control.exit_info_1       = vmcb->control.exit_info_1;
+        nested_vmcb->control.exit_info_2       = vmcb->control.exit_info_2;
+        nested_vmcb->control.exit_int_info     = vmcb->control.exit_int_info;
+        nested_vmcb->control.exit_int_info_err = vmcb->control.exit_int_info_err;
+        nested_vmcb->control.tlb_ctl           = 0;
+        nested_vmcb->control.event_inj         = 0;
+        nested_vmcb->control.event_inj_err     = 0;
 
         /* We always set V_INTR_MASKING and remember the old value in hflags */
         if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK))
                 nested_vmcb->control.int_ctl &= ~V_INTR_MASKING_MASK;
 
-        if ((nested_vmcb->control.int_ctl & V_IRQ_MASK) &&
-            (nested_vmcb->control.int_vector)) {
-                nsvm_printk("WARNING: IRQ 0x%x still enabled on #VMEXIT\n",
-                                nested_vmcb->control.int_vector);
-        }
-
         /* Restore the original control entries */
-        svm->vmcb->control = hsave->control;
+        copy_vmcb_control_area(vmcb, hsave);
 
         /* Kill any pending exceptions */
         if (svm->vcpu.arch.exception.pending == true)
                 nsvm_printk("WARNING: Pending Exception\n");
-        svm->vcpu.arch.exception.pending = false;
+
+        kvm_clear_exception_queue(&svm->vcpu);
+        kvm_clear_interrupt_queue(&svm->vcpu);
 
         /* Restore selected save entries */
         svm->vmcb->save.es = hsave->save.es;
@@ -1603,19 +1657,10 @@ static int nested_svm_vmexit_real(struct vcpu_svm *svm, void *arg1,
         svm->vmcb->save.cpl = 0;
         svm->vmcb->control.exit_int_info = 0;
 
-        svm->vcpu.arch.hflags &= ~HF_GIF_MASK;
         /* Exit nested SVM mode */
-        svm->nested_vmcb = 0;
+        svm->nested.vmcb = 0;
 
-        return 0;
-}
-
-static int nested_svm_vmexit(struct vcpu_svm *svm)
-{
-        nsvm_printk("VMexit\n");
-        if (nested_svm_do(svm, svm->nested_vmcb, 0,
-                          NULL, nested_svm_vmexit_real))
-                return 1;
+        nested_svm_unmap(nested_vmcb, KM_USER0);
 
         kvm_mmu_reset_context(&svm->vcpu);
         kvm_mmu_load(&svm->vcpu);
@@ -1623,38 +1668,63 @@ static int nested_svm_vmexit(struct vcpu_svm *svm)
         return 0;
 }
 
-static int nested_svm_vmrun_msrpm(struct vcpu_svm *svm, void *arg1,
-                                  void *arg2, void *opaque)
+static bool nested_svm_vmrun_msrpm(struct vcpu_svm *svm)
 {
+        u32 *nested_msrpm;
         int i;
-        u32 *nested_msrpm = (u32*)arg1;
+
+        nested_msrpm = nested_svm_map(svm, svm->nested.vmcb_msrpm, KM_USER0);
+        if (!nested_msrpm)
+                return false;
+
         for (i=0; i< PAGE_SIZE * (1 << MSRPM_ALLOC_ORDER) / 4; i++)
-                svm->nested_msrpm[i] = svm->msrpm[i] | nested_msrpm[i];
-        svm->vmcb->control.msrpm_base_pa = __pa(svm->nested_msrpm);
+                svm->nested.msrpm[i] = svm->msrpm[i] | nested_msrpm[i];
 
-        return 0;
+        svm->vmcb->control.msrpm_base_pa = __pa(svm->nested.msrpm);
+
+        nested_svm_unmap(nested_msrpm, KM_USER0);
+
+        return true;
 }
 
-static int nested_svm_vmrun(struct vcpu_svm *svm, void *arg1,
-                            void *arg2, void *opaque)
+static bool nested_svm_vmrun(struct vcpu_svm *svm)
 {
-        struct vmcb *nested_vmcb = (struct vmcb *)arg1;
-        struct vmcb *hsave = svm->hsave;
+        struct vmcb *nested_vmcb;
+        struct vmcb *hsave = svm->nested.hsave;
+        struct vmcb *vmcb = svm->vmcb;
+
+        nested_vmcb = nested_svm_map(svm, svm->vmcb->save.rax, KM_USER0);
+        if (!nested_vmcb)
+                return false;
 
         /* nested_vmcb is our indicator if nested SVM is activated */
-        svm->nested_vmcb = svm->vmcb->save.rax;
+        svm->nested.vmcb = svm->vmcb->save.rax;
 
         /* Clear internal status */
-        svm->vcpu.arch.exception.pending = false;
+        kvm_clear_exception_queue(&svm->vcpu);
+        kvm_clear_interrupt_queue(&svm->vcpu);
 
         /* Save the old vmcb, so we don't need to pick what we save, but
            can restore everything when a VMEXIT occurs */
-        memcpy(hsave, svm->vmcb, sizeof(struct vmcb));
-        /* We need to remember the original CR3 in the SPT case */
-        if (!npt_enabled)
-                hsave->save.cr3 = svm->vcpu.arch.cr3;
-        hsave->save.cr4 = svm->vcpu.arch.cr4;
-        hsave->save.rip = svm->next_rip;
+        hsave->save.es     = vmcb->save.es;
+        hsave->save.cs     = vmcb->save.cs;
+        hsave->save.ss     = vmcb->save.ss;
+        hsave->save.ds     = vmcb->save.ds;
+        hsave->save.gdtr   = vmcb->save.gdtr;
+        hsave->save.idtr   = vmcb->save.idtr;
+        hsave->save.efer   = svm->vcpu.arch.shadow_efer;
+        hsave->save.cr0    = svm->vcpu.arch.cr0;
+        hsave->save.cr4    = svm->vcpu.arch.cr4;
+        hsave->save.rflags = vmcb->save.rflags;
+        hsave->save.rip    = svm->next_rip;
+        hsave->save.rsp    = vmcb->save.rsp;
+        hsave->save.rax    = vmcb->save.rax;
+        if (npt_enabled)
+                hsave->save.cr3    = vmcb->save.cr3;
+        else
+                hsave->save.cr3    = svm->vcpu.arch.cr3;
+
+        copy_vmcb_control_area(hsave, vmcb);
 
         if (svm->vmcb->save.rflags & X86_EFLAGS_IF)
                 svm->vcpu.arch.hflags |= HF_HIF_MASK;
@@ -1679,7 +1749,7 @@ static int nested_svm_vmrun(struct vcpu_svm *svm, void *arg1,
                 kvm_set_cr3(&svm->vcpu, nested_vmcb->save.cr3);
                 kvm_mmu_reset_context(&svm->vcpu);
         }
-        svm->vmcb->save.cr2 = nested_vmcb->save.cr2;
+        svm->vmcb->save.cr2 = svm->vcpu.arch.cr2 = nested_vmcb->save.cr2;
         kvm_register_write(&svm->vcpu, VCPU_REGS_RAX, nested_vmcb->save.rax);
         kvm_register_write(&svm->vcpu, VCPU_REGS_RSP, nested_vmcb->save.rsp);
         kvm_register_write(&svm->vcpu, VCPU_REGS_RIP, nested_vmcb->save.rip);
@@ -1706,7 +1776,15 @@ static int nested_svm_vmrun(struct vcpu_svm *svm, void *arg1,
 
         svm->vmcb->control.intercept |= nested_vmcb->control.intercept;
 
-        svm->nested_vmcb_msrpm = nested_vmcb->control.msrpm_base_pa;
+        svm->nested.vmcb_msrpm = nested_vmcb->control.msrpm_base_pa;
+
+        /* cache intercepts */
+        svm->nested.intercept_cr_read    = nested_vmcb->control.intercept_cr_read;
+        svm->nested.intercept_cr_write   = nested_vmcb->control.intercept_cr_write;
+        svm->nested.intercept_dr_read    = nested_vmcb->control.intercept_dr_read;
+        svm->nested.intercept_dr_write   = nested_vmcb->control.intercept_dr_write;
+        svm->nested.intercept_exceptions = nested_vmcb->control.intercept_exceptions;
+        svm->nested.intercept            = nested_vmcb->control.intercept;
 
         force_new_asid(&svm->vcpu);
         svm->vmcb->control.exit_int_info = nested_vmcb->control.exit_int_info;
@@ -1734,12 +1812,14 @@ static int nested_svm_vmrun(struct vcpu_svm *svm, void *arg1,
         svm->vmcb->control.event_inj = nested_vmcb->control.event_inj;
         svm->vmcb->control.event_inj_err = nested_vmcb->control.event_inj_err;
 
-        svm->vcpu.arch.hflags |= HF_GIF_MASK;
+        nested_svm_unmap(nested_vmcb, KM_USER0);
 
-        return 0;
+        enable_gif(svm);
+
+        return true;
 }
 
-static int nested_svm_vmloadsave(struct vmcb *from_vmcb, struct vmcb *to_vmcb)
+static void nested_svm_vmloadsave(struct vmcb *from_vmcb, struct vmcb *to_vmcb)
 {
         to_vmcb->save.fs = from_vmcb->save.fs;
         to_vmcb->save.gs = from_vmcb->save.gs;
@@ -1753,44 +1833,44 @@ static int nested_svm_vmloadsave(struct vmcb *from_vmcb, struct vmcb *to_vmcb)
         to_vmcb->save.sysenter_cs = from_vmcb->save.sysenter_cs;
         to_vmcb->save.sysenter_esp = from_vmcb->save.sysenter_esp;
         to_vmcb->save.sysenter_eip = from_vmcb->save.sysenter_eip;
-
-        return 1;
-}
-
-static int nested_svm_vmload(struct vcpu_svm *svm, void *nested_vmcb,
-                             void *arg2, void *opaque)
-{
-        return nested_svm_vmloadsave((struct vmcb *)nested_vmcb, svm->vmcb);
-}
-
-static int nested_svm_vmsave(struct vcpu_svm *svm, void *nested_vmcb,
-                             void *arg2, void *opaque)
-{
-        return nested_svm_vmloadsave(svm->vmcb, (struct vmcb *)nested_vmcb);
 }
 
 static int vmload_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
 {
+        struct vmcb *nested_vmcb;
+
         if (nested_svm_check_permissions(svm))
                 return 1;
 
         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
         skip_emulated_instruction(&svm->vcpu);
 
-        nested_svm_do(svm, svm->vmcb->save.rax, 0, NULL, nested_svm_vmload);
+        nested_vmcb = nested_svm_map(svm, svm->vmcb->save.rax, KM_USER0);
+        if (!nested_vmcb)
+                return 1;
+
+        nested_svm_vmloadsave(nested_vmcb, svm->vmcb);
+        nested_svm_unmap(nested_vmcb, KM_USER0);
 
         return 1;
 }
 
 static int vmsave_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
 {
+        struct vmcb *nested_vmcb;
+
         if (nested_svm_check_permissions(svm))
                 return 1;
 
         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
         skip_emulated_instruction(&svm->vcpu);
 
-        nested_svm_do(svm, svm->vmcb->save.rax, 0, NULL, nested_svm_vmsave);
+        nested_vmcb = nested_svm_map(svm, svm->vmcb->save.rax, KM_USER0);
+        if (!nested_vmcb)
+                return 1;
+
+        nested_svm_vmloadsave(svm->vmcb, nested_vmcb);
+        nested_svm_unmap(nested_vmcb, KM_USER0);
 
         return 1;
 }
@@ -1798,19 +1878,29 @@ static int vmsave_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
 static int vmrun_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
 {
         nsvm_printk("VMrun\n");
+
         if (nested_svm_check_permissions(svm))
                 return 1;
 
         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
         skip_emulated_instruction(&svm->vcpu);
 
-        if (nested_svm_do(svm, svm->vmcb->save.rax, 0,
-                          NULL, nested_svm_vmrun))
+        if (!nested_svm_vmrun(svm))
                 return 1;
 
-        if (nested_svm_do(svm, svm->nested_vmcb_msrpm, 0,
-                      NULL, nested_svm_vmrun_msrpm))
-                return 1;
+        if (!nested_svm_vmrun_msrpm(svm))
+                goto failed;
+
+        return 1;
+
+failed:
+
+        svm->vmcb->control.exit_code    = SVM_EXIT_ERR;
+        svm->vmcb->control.exit_code_hi = 0;
+        svm->vmcb->control.exit_info_1  = 0;
+        svm->vmcb->control.exit_info_2  = 0;
+
+        nested_svm_vmexit(svm);
 
         return 1;
 }
@@ -1823,7 +1913,7 @@ static int stgi_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
         skip_emulated_instruction(&svm->vcpu);
 
-        svm->vcpu.arch.hflags |= HF_GIF_MASK;
+        enable_gif(svm);
 
         return 1;
 }
@@ -1836,7 +1926,7 @@ static int clgi_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
         skip_emulated_instruction(&svm->vcpu);
 
-        svm->vcpu.arch.hflags &= ~HF_GIF_MASK;
+        disable_gif(svm);
 
         /* After a CLGI no interrupts should come */
         svm_clear_vintr(svm);
@@ -1845,6 +1935,19 @@ static int clgi_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
         return 1;
 }
 
+static int invlpga_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
+{
+        struct kvm_vcpu *vcpu = &svm->vcpu;
+        nsvm_printk("INVLPGA\n");
+
+        /* Let's treat INVLPGA the same as INVLPG (can be optimized!) */
+        kvm_mmu_invlpg(vcpu, vcpu->arch.regs[VCPU_REGS_RAX]);
+
+        svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
+        skip_emulated_instruction(&svm->vcpu);
+        return 1;
+}
+
 static int invalid_op_interception(struct vcpu_svm *svm,
                                    struct kvm_run *kvm_run)
 {
@@ -1953,7 +2056,7 @@ static int svm_get_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 *data)
         struct vcpu_svm *svm = to_svm(vcpu);
 
         switch (ecx) {
-        case MSR_IA32_TIME_STAMP_COUNTER: {
+        case MSR_IA32_TSC: {
                 u64 tsc;
 
                 rdtscll(tsc);
@@ -1981,10 +2084,10 @@ static int svm_get_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 *data)
                 *data = svm->vmcb->save.sysenter_cs;
                 break;
         case MSR_IA32_SYSENTER_EIP:
-                *data = svm->vmcb->save.sysenter_eip;
+                *data = svm->sysenter_eip;
                 break;
         case MSR_IA32_SYSENTER_ESP:
-                *data = svm->vmcb->save.sysenter_esp;
+                *data = svm->sysenter_esp;
                 break;
         /* Nobody will change the following 5 values in the VMCB so
            we can safely return them on rdmsr. They will always be 0
@@ -2005,7 +2108,7 @@ static int svm_get_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 *data)
                 *data = svm->vmcb->save.last_excp_to;
                 break;
         case MSR_VM_HSAVE_PA:
-                *data = svm->hsave_msr;
+                *data = svm->nested.hsave_msr;
                 break;
         case MSR_VM_CR:
                 *data = 0;
@@ -2027,8 +2130,7 @@ static int rdmsr_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
         if (svm_get_msr(&svm->vcpu, ecx, &data))
                 kvm_inject_gp(&svm->vcpu, 0);
         else {
-                KVMTRACE_3D(MSR_READ, &svm->vcpu, ecx, (u32)data,
-                            (u32)(data >> 32), handler);
+                trace_kvm_msr_read(ecx, data);
 
                 svm->vcpu.arch.regs[VCPU_REGS_RAX] = data & 0xffffffff;
                 svm->vcpu.arch.regs[VCPU_REGS_RDX] = data >> 32;
@@ -2043,7 +2145,7 @@ static int svm_set_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 data)
         struct vcpu_svm *svm = to_svm(vcpu);
 
         switch (ecx) {
-        case MSR_IA32_TIME_STAMP_COUNTER: {
+        case MSR_IA32_TSC: {
                 u64 tsc;
 
                 rdtscll(tsc);
@@ -2071,9 +2173,11 @@ static int svm_set_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 data)
                 svm->vmcb->save.sysenter_cs = data;
                 break;
         case MSR_IA32_SYSENTER_EIP:
+                svm->sysenter_eip = data;
                 svm->vmcb->save.sysenter_eip = data;
                 break;
         case MSR_IA32_SYSENTER_ESP:
+                svm->sysenter_esp = data;
                 svm->vmcb->save.sysenter_esp = data;
                 break;
         case MSR_IA32_DEBUGCTLMSR:
@@ -2091,24 +2195,12 @@ static int svm_set_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 data)
                 else
                         svm_disable_lbrv(svm);
                 break;
-        case MSR_K7_EVNTSEL0:
-        case MSR_K7_EVNTSEL1:
-        case MSR_K7_EVNTSEL2:
-        case MSR_K7_EVNTSEL3:
-        case MSR_K7_PERFCTR0:
-        case MSR_K7_PERFCTR1:
-        case MSR_K7_PERFCTR2:
-        case MSR_K7_PERFCTR3:
-                /*
-                 * Just discard all writes to the performance counters; this
-                 * should keep both older linux and windows 64-bit guests
-                 * happy
-                 */
-                pr_unimpl(vcpu, "unimplemented perfctr wrmsr: 0x%x data 0x%llx\n", ecx, data);
-
-                break;
         case MSR_VM_HSAVE_PA:
-                svm->hsave_msr = data;
+                svm->nested.hsave_msr = data;
+                break;
+        case MSR_VM_CR:
+        case MSR_VM_IGNNE:
+                pr_unimpl(vcpu, "unimplemented wrmsr: 0x%x data 0x%llx\n", ecx, data);
                 break;
         default:
                 return kvm_set_msr_common(vcpu, ecx, data);
@@ -2122,8 +2214,7 @@ static int wrmsr_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
         u64 data = (svm->vcpu.arch.regs[VCPU_REGS_RAX] & -1u)
                 | ((u64)(svm->vcpu.arch.regs[VCPU_REGS_RDX] & -1u) << 32);
 
-        KVMTRACE_3D(MSR_WRITE, &svm->vcpu, ecx, (u32)data, (u32)(data >> 32),
-                    handler);
+        trace_kvm_msr_write(ecx, data);
 
         svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
         if (svm_set_msr(&svm->vcpu, ecx, data))
@@ -2144,8 +2235,6 @@ static int msr_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
 static int interrupt_window_interception(struct vcpu_svm *svm,
                                    struct kvm_run *kvm_run)
 {
-        KVMTRACE_0D(PEND_INTR, &svm->vcpu, handler);
-
         svm_clear_vintr(svm);
         svm->vmcb->control.int_ctl &= ~V_IRQ_MASK;
         /*
@@ -2201,7 +2290,7 @@ static int (*svm_exit_handlers[])(struct vcpu_svm *svm,
         [SVM_EXIT_INVD]                         = emulate_on_interception,
         [SVM_EXIT_HLT]                          = halt_interception,
         [SVM_EXIT_INVLPG]                       = invlpg_interception,
-        [SVM_EXIT_INVLPGA]                      = invalid_op_interception,
+        [SVM_EXIT_INVLPGA]                      = invlpga_interception,
         [SVM_EXIT_IOIO]                         = io_interception,
         [SVM_EXIT_MSR]                          = msr_interception,
         [SVM_EXIT_TASK_SWITCH]                  = task_switch_interception,
@@ -2224,20 +2313,26 @@ static int handle_exit(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
         struct vcpu_svm *svm = to_svm(vcpu);
         u32 exit_code = svm->vmcb->control.exit_code;
 
-        KVMTRACE_3D(VMEXIT, vcpu, exit_code, (u32)svm->vmcb->save.rip,
-                    (u32)((u64)svm->vmcb->save.rip >> 32), entryexit);
+        trace_kvm_exit(exit_code, svm->vmcb->save.rip);
 
         if (is_nested(svm)) {
+                int vmexit;
+
                 nsvm_printk("nested handle_exit: 0x%x | 0x%lx | 0x%lx | 0x%lx\n",
                             exit_code, svm->vmcb->control.exit_info_1,
                             svm->vmcb->control.exit_info_2, svm->vmcb->save.rip);
-                if (nested_svm_exit_handled(svm, true)) {
-                        nested_svm_vmexit(svm);
-                        nsvm_printk("-> #VMEXIT\n");
+
+                vmexit = nested_svm_exit_special(svm);
+
+                if (vmexit == NESTED_EXIT_CONTINUE)
+                        vmexit = nested_svm_exit_handled(svm);
+
+                if (vmexit == NESTED_EXIT_DONE)
                         return 1;
-                }
         }
 
+        svm_complete_interrupts(svm);
+
         if (npt_enabled) {
                 int mmu_reload = 0;
                 if ((vcpu->arch.cr0 ^ svm->vmcb->save.cr0) & X86_CR0_PG) {
@@ -2246,12 +2341,6 @@ static int handle_exit(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
                 }
                 vcpu->arch.cr0 = svm->vmcb->save.cr0;
                 vcpu->arch.cr3 = svm->vmcb->save.cr3;
-                if (is_paging(vcpu) && is_pae(vcpu) && !is_long_mode(vcpu)) {
-                        if (!load_pdptrs(vcpu, vcpu->arch.cr3)) {
-                                kvm_inject_gp(vcpu, 0);
-                                return 1;
-                        }
-                }
                 if (mmu_reload) {
                         kvm_mmu_reset_context(vcpu);
                         kvm_mmu_load(vcpu);
@@ -2319,7 +2408,7 @@ static inline void svm_inject_irq(struct vcpu_svm *svm, int irq)
 {
         struct vmcb_control_area *control;
 
-        KVMTRACE_1D(INJ_VIRQ, &svm->vcpu, (u32)irq, handler);
+        trace_kvm_inj_virq(irq);
 
         ++svm->vcpu.stat.irq_injections;
         control = &svm->vmcb->control;
@@ -2329,21 +2418,14 @@ static inline void svm_inject_irq(struct vcpu_svm *svm, int irq)
                 ((/*control->int_vector >> 4*/ 0xf) << V_INTR_PRIO_SHIFT);
 }
 
-static void svm_queue_irq(struct kvm_vcpu *vcpu, unsigned nr)
-{
-        struct vcpu_svm *svm = to_svm(vcpu);
-
-        svm->vmcb->control.event_inj = nr |
-                SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR;
-}
-
 static void svm_set_irq(struct kvm_vcpu *vcpu)
 {
         struct vcpu_svm *svm = to_svm(vcpu);
 
-        nested_svm_intr(svm);
+        BUG_ON(!(gif_set(svm)));
 
-        svm_queue_irq(vcpu, vcpu->arch.interrupt.nr);
+        svm->vmcb->control.event_inj = vcpu->arch.interrupt.nr |
+                SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR;
 }
 
 static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
@@ -2371,13 +2453,25 @@ static int svm_interrupt_allowed(struct kvm_vcpu *vcpu)
         struct vmcb *vmcb = svm->vmcb;
         return (vmcb->save.rflags & X86_EFLAGS_IF) &&
                 !(vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) &&
-                (svm->vcpu.arch.hflags & HF_GIF_MASK);
+                gif_set(svm) &&
+                !(is_nested(svm) && (svm->vcpu.arch.hflags & HF_VINTR_MASK));
 }
 
 static void enable_irq_window(struct kvm_vcpu *vcpu)
 {
-        svm_set_vintr(to_svm(vcpu));
-        svm_inject_irq(to_svm(vcpu), 0x0);
+        struct vcpu_svm *svm = to_svm(vcpu);
+        nsvm_printk("Trying to open IRQ window\n");
+
+        nested_svm_intr(svm);
+
+        /* In case GIF=0 we can't rely on the CPU to tell us when
+         * GIF becomes 1, because that's a separate STGI/VMRUN intercept.
+         * The next time we get that intercept, this function will be
+         * called again though and we'll get the vintr intercept. */
+        if (gif_set(svm)) {
+                svm_set_vintr(svm);
+                svm_inject_irq(svm, 0x0);
+        }
 }
 
 static void enable_nmi_window(struct kvm_vcpu *vcpu)
@@ -2456,6 +2550,8 @@ static void svm_complete_interrupts(struct vcpu_svm *svm)
         case SVM_EXITINTINFO_TYPE_EXEPT:
                 /* In case of software exception do not reinject an exception
                    vector, but re-execute and instruction instead */
+                if (is_nested(svm))
+                        break;
                 if (kvm_exception_is_soft(vector))
                         break;
                 if (exitintinfo & SVM_EXITINTINFO_VALID_ERR) {
@@ -2498,9 +2594,7 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
         fs_selector = kvm_read_fs();
         gs_selector = kvm_read_gs();
         ldt_selector = kvm_read_ldt();
-        svm->host_cr2 = kvm_read_cr2();
-        if (!is_nested(svm))
-                svm->vmcb->save.cr2 = vcpu->arch.cr2;
+        svm->vmcb->save.cr2 = vcpu->arch.cr2;
         /* required for live migration with NPT */
         if (npt_enabled)
                 svm->vmcb->save.cr3 = vcpu->arch.cr3;
@@ -2585,8 +2679,6 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
         vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp;
         vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip;
 
-        kvm_write_cr2(svm->host_cr2);
-
         kvm_load_fs(fs_selector);
         kvm_load_gs(gs_selector);
         kvm_load_ldt(ldt_selector);
@@ -2602,7 +2694,10 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 
         svm->next_rip = 0;
 
-        svm_complete_interrupts(svm);
+        if (npt_enabled) {
+                vcpu->arch.regs_avail &= ~(1 << VCPU_EXREG_PDPTR);
+                vcpu->arch.regs_dirty &= ~(1 << VCPU_EXREG_PDPTR);
+        }
 }
 
 #undef R
@@ -2673,6 +2768,64 @@ static u64 svm_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
         return 0;
 }
 
+static const struct trace_print_flags svm_exit_reasons_str[] = {
+        { SVM_EXIT_READ_CR0,                    "read_cr0" },
+        { SVM_EXIT_READ_CR3,                    "read_cr3" },
+        { SVM_EXIT_READ_CR4,                    "read_cr4" },
+        { SVM_EXIT_READ_CR8,                    "read_cr8" },
+        { SVM_EXIT_WRITE_CR0,                   "write_cr0" },
+        { SVM_EXIT_WRITE_CR3,                   "write_cr3" },
+        { SVM_EXIT_WRITE_CR4,                   "write_cr4" },
+        { SVM_EXIT_WRITE_CR8,                   "write_cr8" },
+        { SVM_EXIT_READ_DR0,                    "read_dr0" },
+        { SVM_EXIT_READ_DR1,                    "read_dr1" },
+        { SVM_EXIT_READ_DR2,                    "read_dr2" },
+        { SVM_EXIT_READ_DR3,                    "read_dr3" },
+        { SVM_EXIT_WRITE_DR0,                   "write_dr0" },
+        { SVM_EXIT_WRITE_DR1,                   "write_dr1" },
+        { SVM_EXIT_WRITE_DR2,                   "write_dr2" },
+        { SVM_EXIT_WRITE_DR3,                   "write_dr3" },
+        { SVM_EXIT_WRITE_DR5,                   "write_dr5" },
+        { SVM_EXIT_WRITE_DR7,                   "write_dr7" },
+        { SVM_EXIT_EXCP_BASE + DB_VECTOR,       "DB excp" },
+        { SVM_EXIT_EXCP_BASE + BP_VECTOR,       "BP excp" },
+        { SVM_EXIT_EXCP_BASE + UD_VECTOR,       "UD excp" },
+        { SVM_EXIT_EXCP_BASE + PF_VECTOR,       "PF excp" },
+        { SVM_EXIT_EXCP_BASE + NM_VECTOR,       "NM excp" },
+        { SVM_EXIT_EXCP_BASE + MC_VECTOR,       "MC excp" },
+        { SVM_EXIT_INTR,                        "interrupt" },
+        { SVM_EXIT_NMI,                         "nmi" },
+        { SVM_EXIT_SMI,                         "smi" },
+        { SVM_EXIT_INIT,                        "init" },
+        { SVM_EXIT_VINTR,                       "vintr" },
+        { SVM_EXIT_CPUID,                       "cpuid" },
+        { SVM_EXIT_INVD,                        "invd" },
+        { SVM_EXIT_HLT,                         "hlt" },
+        { SVM_EXIT_INVLPG,                      "invlpg" },
+        { SVM_EXIT_INVLPGA,                     "invlpga" },
+        { SVM_EXIT_IOIO,                        "io" },
+        { SVM_EXIT_MSR,                         "msr" },
+        { SVM_EXIT_TASK_SWITCH,                 "task_switch" },
+        { SVM_EXIT_SHUTDOWN,                    "shutdown" },
+        { SVM_EXIT_VMRUN,                       "vmrun" },
+        { SVM_EXIT_VMMCALL,                     "hypercall" },
+        { SVM_EXIT_VMLOAD,                      "vmload" },
+        { SVM_EXIT_VMSAVE,                      "vmsave" },
+        { SVM_EXIT_STGI,                        "stgi" },
+        { SVM_EXIT_CLGI,                        "clgi" },
+        { SVM_EXIT_SKINIT,                      "skinit" },
+        { SVM_EXIT_WBINVD,                      "wbinvd" },
+        { SVM_EXIT_MONITOR,                     "monitor" },
+        { SVM_EXIT_MWAIT,                       "mwait" },
+        { SVM_EXIT_NPF,                         "npf" },
+        { -1, NULL }
+};
+
+static bool svm_gb_page_enable(void)
+{
+        return true;
+}
+
 static struct kvm_x86_ops svm_x86_ops = {
         .cpu_has_kvm_support = has_svm,
         .disabled_by_bios = is_disabled,
@@ -2710,6 +2863,7 @@ static struct kvm_x86_ops svm_x86_ops = {
         .set_gdt = svm_set_gdt,
         .get_dr = svm_get_dr,
         .set_dr = svm_set_dr,
+        .cache_reg = svm_cache_reg,
         .get_rflags = svm_get_rflags,
         .set_rflags = svm_set_rflags,
 
@@ -2733,6 +2887,9 @@ static struct kvm_x86_ops svm_x86_ops = {
         .set_tss_addr = svm_set_tss_addr,
         .get_tdp_level = get_npt_level,
         .get_mt_mask = svm_get_mt_mask,
+
+        .exit_reasons_str = svm_exit_reasons_str,
+        .gb_page_enable = svm_gb_page_enable,
 };
 
 static int __init svm_init(void)
diff --git a/arch/x86/kvm/timer.c b/arch/x86/kvm/timer.c
index 86dbac072d0c..eea40439066c 100644
--- a/arch/x86/kvm/timer.c
+++ b/arch/x86/kvm/timer.c
@@ -9,12 +9,16 @@ static int __kvm_timer_fn(struct kvm_vcpu *vcpu, struct kvm_timer *ktimer)
         int restart_timer = 0;
         wait_queue_head_t *q = &vcpu->wq;
 
-        /* FIXME: this code should not know anything about vcpus */
-        if (!atomic_inc_and_test(&ktimer->pending))
+        /*
+         * There is a race window between reading and incrementing, but we do
+         * not care about potentially loosing timer events in the !reinject
+         * case anyway.
+         */
+        if (ktimer->reinject || !atomic_read(&ktimer->pending)) {
+                atomic_inc(&ktimer->pending);
+                /* FIXME: this code should not know anything about vcpus */
                 set_bit(KVM_REQ_PENDING_TIMER, &vcpu->requests);
-
-        if (!ktimer->reinject)
-                atomic_set(&ktimer->pending, 1);
+        }
 
         if (waitqueue_active(q))
                 wake_up_interruptible(q);
@@ -33,7 +37,7 @@ enum hrtimer_restart kvm_timer_fn(struct hrtimer *data)
         struct kvm_vcpu *vcpu;
         struct kvm_timer *ktimer = container_of(data, struct kvm_timer, timer);
 
-        vcpu = ktimer->kvm->vcpus[ktimer->vcpu_id];
+        vcpu = ktimer->vcpu;
         if (!vcpu)
                 return HRTIMER_NORESTART;
 
diff --git a/arch/x86/kvm/trace.h b/arch/x86/kvm/trace.h
new file mode 100644
index 000000000000..0d480e77eacf
--- /dev/null
+++ b/arch/x86/kvm/trace.h
@@ -0,0 +1,355 @@
+#if !defined(_TRACE_KVM_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_KVM_H
+
+#include <linux/tracepoint.h>
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM kvm
+#define TRACE_INCLUDE_PATH arch/x86/kvm
+#define TRACE_INCLUDE_FILE trace
+
+/*
+ * Tracepoint for guest mode entry.
+ */
+TRACE_EVENT(kvm_entry,
+        TP_PROTO(unsigned int vcpu_id),
+        TP_ARGS(vcpu_id),
+
+        TP_STRUCT__entry(
+                __field(        unsigned int,   vcpu_id         )
+        ),
+
+        TP_fast_assign(
+                __entry->vcpu_id        = vcpu_id;
+        ),
+
+        TP_printk("vcpu %u", __entry->vcpu_id)
+);
+
+/*
+ * Tracepoint for hypercall.
+ */
+TRACE_EVENT(kvm_hypercall,
+        TP_PROTO(unsigned long nr, unsigned long a0, unsigned long a1,
+                 unsigned long a2, unsigned long a3),
+        TP_ARGS(nr, a0, a1, a2, a3),
+
+        TP_STRUCT__entry(
+                __field(        unsigned long,  nr              )
+                __field(        unsigned long,  a0              )
+                __field(        unsigned long,  a1              )
+                __field(        unsigned long,  a2              )
+                __field(        unsigned long,  a3              )
+        ),
+
+        TP_fast_assign(
+                __entry->nr             = nr;
+                __entry->a0             = a0;
+                __entry->a1             = a1;
+                __entry->a2             = a2;
+                __entry->a3             = a3;
+        ),
+
+        TP_printk("nr 0x%lx a0 0x%lx a1 0x%lx a2 0x%lx a3 0x%lx",
+                 __entry->nr, __entry->a0, __entry->a1,  __entry->a2,
+                 __entry->a3)
+);
+
+/*
+ * Tracepoint for PIO.
+ */
+TRACE_EVENT(kvm_pio,
+        TP_PROTO(unsigned int rw, unsigned int port, unsigned int size,
+                 unsigned int count),
+        TP_ARGS(rw, port, size, count),
+
+        TP_STRUCT__entry(
+                __field(        unsigned int,   rw              )
+                __field(        unsigned int,   port            )
+                __field(        unsigned int,   size            )
+                __field(        unsigned int,   count           )
+        ),
+
+        TP_fast_assign(
+                __entry->rw             = rw;
+                __entry->port           = port;
+                __entry->size           = size;
+                __entry->count          = count;
+        ),
+
+        TP_printk("pio_%s at 0x%x size %d count %d",
+                  __entry->rw ? "write" : "read",
+                  __entry->port, __entry->size, __entry->count)
+);
+
+/*
+ * Tracepoint for cpuid.
+ */
+TRACE_EVENT(kvm_cpuid,
+        TP_PROTO(unsigned int function, unsigned long rax, unsigned long rbx,
+                 unsigned long rcx, unsigned long rdx),
+        TP_ARGS(function, rax, rbx, rcx, rdx),
+
+        TP_STRUCT__entry(
+                __field(        unsigned int,   function        )
+                __field(        unsigned long,  rax             )
+                __field(        unsigned long,  rbx             )
+                __field(        unsigned long,  rcx             )
+                __field(        unsigned long,  rdx             )
+        ),
+
+        TP_fast_assign(
+                __entry->function       = function;
+                __entry->rax            = rax;
+                __entry->rbx            = rbx;
+                __entry->rcx            = rcx;
+                __entry->rdx            = rdx;
+        ),
+
+        TP_printk("func %x rax %lx rbx %lx rcx %lx rdx %lx",
+                  __entry->function, __entry->rax,
+                  __entry->rbx, __entry->rcx, __entry->rdx)
+);
+
+#define AREG(x) { APIC_##x, "APIC_" #x }
+
+#define kvm_trace_symbol_apic                                               \
+        AREG(ID), AREG(LVR), AREG(TASKPRI), AREG(ARBPRI), AREG(PROCPRI),    \
+        AREG(EOI), AREG(RRR), AREG(LDR), AREG(DFR), AREG(SPIV), AREG(ISR),  \
+        AREG(TMR), AREG(IRR), AREG(ESR), AREG(ICR), AREG(ICR2), AREG(LVTT), \
+        AREG(LVTTHMR), AREG(LVTPC), AREG(LVT0), AREG(LVT1), AREG(LVTERR),   \
+        AREG(TMICT), AREG(TMCCT), AREG(TDCR), AREG(SELF_IPI), AREG(EFEAT),  \
+        AREG(ECTRL)
+/*
+ * Tracepoint for apic access.
+ */
+TRACE_EVENT(kvm_apic,
+        TP_PROTO(unsigned int rw, unsigned int reg, unsigned int val),
+        TP_ARGS(rw, reg, val),
+
+        TP_STRUCT__entry(
+                __field(        unsigned int,   rw              )
+                __field(        unsigned int,   reg             )
+                __field(        unsigned int,   val             )
+        ),
+
+        TP_fast_assign(
+                __entry->rw             = rw;
+                __entry->reg            = reg;
+                __entry->val            = val;
+        ),
+
+        TP_printk("apic_%s %s = 0x%x",
+                  __entry->rw ? "write" : "read",
+                  __print_symbolic(__entry->reg, kvm_trace_symbol_apic),
+                  __entry->val)
+);
+
+#define trace_kvm_apic_read(reg, val)           trace_kvm_apic(0, reg, val)
+#define trace_kvm_apic_write(reg, val)          trace_kvm_apic(1, reg, val)
+
+/*
+ * Tracepoint for kvm guest exit:
+ */
+TRACE_EVENT(kvm_exit,
+        TP_PROTO(unsigned int exit_reason, unsigned long guest_rip),
+        TP_ARGS(exit_reason, guest_rip),
+
+        TP_STRUCT__entry(
+                __field(        unsigned int,   exit_reason     )
+                __field(        unsigned long,  guest_rip       )
+        ),
+
+        TP_fast_assign(
+                __entry->exit_reason    = exit_reason;
+                __entry->guest_rip      = guest_rip;
+        ),
+
+        TP_printk("reason %s rip 0x%lx",
+                 ftrace_print_symbols_seq(p, __entry->exit_reason,
+                                          kvm_x86_ops->exit_reasons_str),
+                 __entry->guest_rip)
+);
+
+/*
+ * Tracepoint for kvm interrupt injection:
+ */
+TRACE_EVENT(kvm_inj_virq,
+        TP_PROTO(unsigned int irq),
+        TP_ARGS(irq),
+
+        TP_STRUCT__entry(
+                __field(        unsigned int,   irq             )
+        ),
+
+        TP_fast_assign(
+                __entry->irq            = irq;
+        ),
+
+        TP_printk("irq %u", __entry->irq)
+);
+
+/*
+ * Tracepoint for page fault.
+ */
+TRACE_EVENT(kvm_page_fault,
+        TP_PROTO(unsigned long fault_address, unsigned int error_code),
+        TP_ARGS(fault_address, error_code),
+
+        TP_STRUCT__entry(
+                __field(        unsigned long,  fault_address   )
+                __field(        unsigned int,   error_code      )
+        ),
+
+        TP_fast_assign(
+                __entry->fault_address  = fault_address;
+                __entry->error_code     = error_code;
+        ),
+
+        TP_printk("address %lx error_code %x",
+                  __entry->fault_address, __entry->error_code)
+);
+
+/*
+ * Tracepoint for guest MSR access.
+ */
+TRACE_EVENT(kvm_msr,
+        TP_PROTO(unsigned int rw, unsigned int ecx, unsigned long data),
+        TP_ARGS(rw, ecx, data),
+
+        TP_STRUCT__entry(
+                __field(        unsigned int,   rw              )
+                __field(        unsigned int,   ecx             )
+                __field(        unsigned long,  data            )
+        ),
+
+        TP_fast_assign(
+                __entry->rw             = rw;
+                __entry->ecx            = ecx;
+                __entry->data           = data;
+        ),
+
+        TP_printk("msr_%s %x = 0x%lx",
+                  __entry->rw ? "write" : "read",
+                  __entry->ecx, __entry->data)
+);
+
+#define trace_kvm_msr_read(ecx, data)           trace_kvm_msr(0, ecx, data)
+#define trace_kvm_msr_write(ecx, data)          trace_kvm_msr(1, ecx, data)
+
+/*
+ * Tracepoint for guest CR access.
+ */
+TRACE_EVENT(kvm_cr,
+        TP_PROTO(unsigned int rw, unsigned int cr, unsigned long val),
+        TP_ARGS(rw, cr, val),
+
+        TP_STRUCT__entry(
+                __field(        unsigned int,   rw              )
+                __field(        unsigned int,   cr              )
+                __field(        unsigned long,  val             )
+        ),
+
+        TP_fast_assign(
+                __entry->rw             = rw;
+                __entry->cr             = cr;
+                __entry->val            = val;
+        ),
+
+        TP_printk("cr_%s %x = 0x%lx",
+                  __entry->rw ? "write" : "read",
+                  __entry->cr, __entry->val)
+);
+
+#define trace_kvm_cr_read(cr, val)              trace_kvm_cr(0, cr, val)
+#define trace_kvm_cr_write(cr, val)             trace_kvm_cr(1, cr, val)
+
+TRACE_EVENT(kvm_pic_set_irq,
+            TP_PROTO(__u8 chip, __u8 pin, __u8 elcr, __u8 imr, bool coalesced),
+            TP_ARGS(chip, pin, elcr, imr, coalesced),
+
+        TP_STRUCT__entry(
+                __field(        __u8,           chip            )
+                __field(        __u8,           pin             )
+                __field(        __u8,           elcr            )
+                __field(        __u8,           imr             )
+                __field(        bool,           coalesced       )
+        ),
+
+        TP_fast_assign(
+                __entry->chip           = chip;
+                __entry->pin            = pin;
+                __entry->elcr           = elcr;
+                __entry->imr            = imr;
+                __entry->coalesced      = coalesced;
+        ),
+
+        TP_printk("chip %u pin %u (%s%s)%s",
+                  __entry->chip, __entry->pin,
+                  (__entry->elcr & (1 << __entry->pin)) ? "level":"edge",
+                  (__entry->imr & (1 << __entry->pin)) ? "|masked":"",
+                  __entry->coalesced ? " (coalesced)" : "")
+);
+
+#define kvm_apic_dst_shorthand          \
+        {0x0, "dst"},                   \
+        {0x1, "self"},                  \
+        {0x2, "all"},                   \
+        {0x3, "all-but-self"}
+
+TRACE_EVENT(kvm_apic_ipi,
+            TP_PROTO(__u32 icr_low, __u32 dest_id),
+            TP_ARGS(icr_low, dest_id),
+
+        TP_STRUCT__entry(
+                __field(        __u32,          icr_low         )
+                __field(        __u32,          dest_id         )
+        ),
+
+        TP_fast_assign(
+                __entry->icr_low        = icr_low;
+                __entry->dest_id        = dest_id;
+        ),
+
+        TP_printk("dst %x vec %u (%s|%s|%s|%s|%s)",
+                  __entry->dest_id, (u8)__entry->icr_low,
+                  __print_symbolic((__entry->icr_low >> 8 & 0x7),
+                                   kvm_deliver_mode),
+                  (__entry->icr_low & (1<<11)) ? "logical" : "physical",
+                  (__entry->icr_low & (1<<14)) ? "assert" : "de-assert",
+                  (__entry->icr_low & (1<<15)) ? "level" : "edge",
+                  __print_symbolic((__entry->icr_low >> 18 & 0x3),
+                                   kvm_apic_dst_shorthand))
+);
+
+TRACE_EVENT(kvm_apic_accept_irq,
+            TP_PROTO(__u32 apicid, __u16 dm, __u8 tm, __u8 vec, bool coalesced),
+            TP_ARGS(apicid, dm, tm, vec, coalesced),
+
+        TP_STRUCT__entry(
+                __field(        __u32,          apicid          )
+                __field(        __u16,          dm              )
+                __field(        __u8,           tm              )
+                __field(        __u8,           vec             )
+                __field(        bool,           coalesced       )
+        ),
+
+        TP_fast_assign(
+                __entry->apicid         = apicid;
+                __entry->dm             = dm;
+                __entry->tm             = tm;
+                __entry->vec            = vec;
+                __entry->coalesced      = coalesced;
+        ),
+
+        TP_printk("apicid %x vec %u (%s|%s)%s",
+                  __entry->apicid, __entry->vec,
+                  __print_symbolic((__entry->dm >> 8 & 0x7), kvm_deliver_mode),
+                  __entry->tm ? "level" : "edge",
+                  __entry->coalesced ? " (coalesced)" : "")
+);
+
+#endif /* _TRACE_KVM_H */
+
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index 29f912927a58..f3812014bd0b 100644
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -25,6 +25,7 @@
 #include <linux/highmem.h>
 #include <linux/sched.h>
 #include <linux/moduleparam.h>
+#include <linux/ftrace_event.h>
 #include "kvm_cache_regs.h"
 #include "x86.h"
 
@@ -34,6 +35,8 @@
 #include <asm/virtext.h>
 #include <asm/mce.h>
 
+#include "trace.h"
+
 #define __ex(x) __kvm_handle_fault_on_reboot(x)
 
 MODULE_AUTHOR("Qumranet");
@@ -51,6 +54,10 @@ module_param_named(flexpriority, flexpriority_enabled, bool, S_IRUGO);
 static int __read_mostly enable_ept = 1;
 module_param_named(ept, enable_ept, bool, S_IRUGO);
 
+static int __read_mostly enable_unrestricted_guest = 1;
+module_param_named(unrestricted_guest,
+                        enable_unrestricted_guest, bool, S_IRUGO);
+
 static int __read_mostly emulate_invalid_guest_state = 0;
 module_param(emulate_invalid_guest_state, bool, S_IRUGO);
 
@@ -84,6 +91,14 @@ struct vcpu_vmx {
                 int           guest_efer_loaded;
         } host_state;
         struct {
+                int vm86_active;
+                u8 save_iopl;
+                struct kvm_save_segment {
+                        u16 selector;
+                        unsigned long base;
+                        u32 limit;
+                        u32 ar;
+                } tr, es, ds, fs, gs;
                 struct {
                         bool pending;
                         u8 vector;
@@ -161,6 +176,8 @@ static struct kvm_vmx_segment_field {
         VMX_SEGMENT_FIELD(LDTR),
 };
 
+static void ept_save_pdptrs(struct kvm_vcpu *vcpu);
+
 /*
  * Keep MSR_K6_STAR at the end, as setup_msrs() will try to optimize it
  * away by decrementing the array size.
@@ -256,6 +273,26 @@ static inline bool cpu_has_vmx_flexpriority(void)
                 cpu_has_vmx_virtualize_apic_accesses();
 }
 
+static inline bool cpu_has_vmx_ept_execute_only(void)
+{
+        return !!(vmx_capability.ept & VMX_EPT_EXECUTE_ONLY_BIT);
+}
+
+static inline bool cpu_has_vmx_eptp_uncacheable(void)
+{
+        return !!(vmx_capability.ept & VMX_EPTP_UC_BIT);
+}
+
+static inline bool cpu_has_vmx_eptp_writeback(void)
+{
+        return !!(vmx_capability.ept & VMX_EPTP_WB_BIT);
+}
+
+static inline bool cpu_has_vmx_ept_2m_page(void)
+{
+        return !!(vmx_capability.ept & VMX_EPT_2MB_PAGE_BIT);
+}
+
 static inline int cpu_has_vmx_invept_individual_addr(void)
 {
         return !!(vmx_capability.ept & VMX_EPT_EXTENT_INDIVIDUAL_BIT);
@@ -277,6 +314,12 @@ static inline int cpu_has_vmx_ept(void)
                 SECONDARY_EXEC_ENABLE_EPT;
 }
 
+static inline int cpu_has_vmx_unrestricted_guest(void)
+{
+        return vmcs_config.cpu_based_2nd_exec_ctrl &
+                SECONDARY_EXEC_UNRESTRICTED_GUEST;
+}
+
 static inline int vm_need_virtualize_apic_accesses(struct kvm *kvm)
 {
         return flexpriority_enabled &&
@@ -497,14 +540,16 @@ static void update_exception_bitmap(struct kvm_vcpu *vcpu)
         eb = (1u << PF_VECTOR) | (1u << UD_VECTOR) | (1u << MC_VECTOR);
         if (!vcpu->fpu_active)
                 eb |= 1u << NM_VECTOR;
+        /*
+         * Unconditionally intercept #DB so we can maintain dr6 without
+         * reading it every exit.
+         */
+        eb |= 1u << DB_VECTOR;
         if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) {
-                if (vcpu->guest_debug &
-                    (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))
-                        eb |= 1u << DB_VECTOR;
                 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
                         eb |= 1u << BP_VECTOR;
         }
-        if (vcpu->arch.rmode.vm86_active)
+        if (to_vmx(vcpu)->rmode.vm86_active)
                 eb = ~0;
         if (enable_ept)
                 eb &= ~(1u << PF_VECTOR); /* bypass_guest_pf = 0 */
@@ -528,12 +573,15 @@ static void reload_tss(void)
 static void load_transition_efer(struct vcpu_vmx *vmx)
 {
         int efer_offset = vmx->msr_offset_efer;
-        u64 host_efer = vmx->host_msrs[efer_offset].data;
-        u64 guest_efer = vmx->guest_msrs[efer_offset].data;
+        u64 host_efer;
+        u64 guest_efer;
         u64 ignore_bits;
 
         if (efer_offset < 0)
                 return;
+        host_efer = vmx->host_msrs[efer_offset].data;
+        guest_efer = vmx->guest_msrs[efer_offset].data;
+
         /*
          * NX is emulated; LMA and LME handled by hardware; SCE meaninless
          * outside long mode
@@ -735,12 +783,17 @@ static void vmx_fpu_deactivate(struct kvm_vcpu *vcpu)
 
 static unsigned long vmx_get_rflags(struct kvm_vcpu *vcpu)
 {
-        return vmcs_readl(GUEST_RFLAGS);
+        unsigned long rflags;
+
+        rflags = vmcs_readl(GUEST_RFLAGS);
+        if (to_vmx(vcpu)->rmode.vm86_active)
+                rflags &= ~(unsigned long)(X86_EFLAGS_IOPL | X86_EFLAGS_VM);
+        return rflags;
 }
 
 static void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
 {
-        if (vcpu->arch.rmode.vm86_active)
+        if (to_vmx(vcpu)->rmode.vm86_active)
                 rflags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM;
         vmcs_writel(GUEST_RFLAGS, rflags);
 }
@@ -797,12 +850,13 @@ static void vmx_queue_exception(struct kvm_vcpu *vcpu, unsigned nr,
                 intr_info |= INTR_INFO_DELIVER_CODE_MASK;
         }
 
-        if (vcpu->arch.rmode.vm86_active) {
+        if (vmx->rmode.vm86_active) {
                 vmx->rmode.irq.pending = true;
                 vmx->rmode.irq.vector = nr;
                 vmx->rmode.irq.rip = kvm_rip_read(vcpu);
-                if (nr == BP_VECTOR || nr == OF_VECTOR)
-                        vmx->rmode.irq.rip++;
+                if (kvm_exception_is_soft(nr))
+                        vmx->rmode.irq.rip +=
+                                vmx->vcpu.arch.event_exit_inst_len;
                 intr_info |= INTR_TYPE_SOFT_INTR;
                 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr_info);
                 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, 1);
@@ -940,7 +994,7 @@ static int vmx_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
         case MSR_EFER:
                 return kvm_get_msr_common(vcpu, msr_index, pdata);
 #endif
-        case MSR_IA32_TIME_STAMP_COUNTER:
+        case MSR_IA32_TSC:
                 data = guest_read_tsc();
                 break;
         case MSR_IA32_SYSENTER_CS:
@@ -953,9 +1007,9 @@ static int vmx_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
                 data = vmcs_readl(GUEST_SYSENTER_ESP);
                 break;
         default:
-                vmx_load_host_state(to_vmx(vcpu));
                 msr = find_msr_entry(to_vmx(vcpu), msr_index);
                 if (msr) {
+                        vmx_load_host_state(to_vmx(vcpu));
                         data = msr->data;
                         break;
                 }
@@ -1000,22 +1054,10 @@ static int vmx_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data)
         case MSR_IA32_SYSENTER_ESP:
                 vmcs_writel(GUEST_SYSENTER_ESP, data);
                 break;
-        case MSR_IA32_TIME_STAMP_COUNTER:
+        case MSR_IA32_TSC:
                 rdtscll(host_tsc);
                 guest_write_tsc(data, host_tsc);
                 break;
-        case MSR_P6_PERFCTR0:
-        case MSR_P6_PERFCTR1:
-        case MSR_P6_EVNTSEL0:
-        case MSR_P6_EVNTSEL1:
-                /*
-                 * Just discard all writes to the performance counters; this
-                 * should keep both older linux and windows 64-bit guests
-                 * happy
-                 */
-                pr_unimpl(vcpu, "unimplemented perfctr wrmsr: 0x%x data 0x%llx\n", msr_index, data);
-
-                break;
         case MSR_IA32_CR_PAT:
                 if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) {
                         vmcs_write64(GUEST_IA32_PAT, data);
@@ -1024,9 +1066,9 @@ static int vmx_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data)
                 }
                 /* Otherwise falls through to kvm_set_msr_common */
         default:
-                vmx_load_host_state(vmx);
                 msr = find_msr_entry(vmx, msr_index);
                 if (msr) {
+                        vmx_load_host_state(vmx);
                         msr->data = data;
                         break;
                 }
@@ -1046,6 +1088,10 @@ static void vmx_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
         case VCPU_REGS_RIP:
                 vcpu->arch.regs[VCPU_REGS_RIP] = vmcs_readl(GUEST_RIP);
                 break;
+        case VCPU_EXREG_PDPTR:
+                if (enable_ept)
+                        ept_save_pdptrs(vcpu);
+                break;
         default:
                 break;
         }
@@ -1203,7 +1249,8 @@ static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf)
                 opt2 = SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
                         SECONDARY_EXEC_WBINVD_EXITING |
                         SECONDARY_EXEC_ENABLE_VPID |
-                        SECONDARY_EXEC_ENABLE_EPT;
+                        SECONDARY_EXEC_ENABLE_EPT |
+                        SECONDARY_EXEC_UNRESTRICTED_GUEST;
                 if (adjust_vmx_controls(min2, opt2,
                                         MSR_IA32_VMX_PROCBASED_CTLS2,
                                         &_cpu_based_2nd_exec_control) < 0)
@@ -1217,12 +1264,9 @@ static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf)
         if (_cpu_based_2nd_exec_control & SECONDARY_EXEC_ENABLE_EPT) {
                 /* CR3 accesses and invlpg don't need to cause VM Exits when EPT
                    enabled */
-                min &= ~(CPU_BASED_CR3_LOAD_EXITING |
-                         CPU_BASED_CR3_STORE_EXITING |
-                         CPU_BASED_INVLPG_EXITING);
-                if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PROCBASED_CTLS,
-                                        &_cpu_based_exec_control) < 0)
-                        return -EIO;
+                _cpu_based_exec_control &= ~(CPU_BASED_CR3_LOAD_EXITING |
+                                             CPU_BASED_CR3_STORE_EXITING |
+                                             CPU_BASED_INVLPG_EXITING);
                 rdmsr(MSR_IA32_VMX_EPT_VPID_CAP,
                       vmx_capability.ept, vmx_capability.vpid);
         }
@@ -1333,8 +1377,13 @@ static __init int hardware_setup(void)
         if (!cpu_has_vmx_vpid())
                 enable_vpid = 0;
 
-        if (!cpu_has_vmx_ept())
+        if (!cpu_has_vmx_ept()) {
                 enable_ept = 0;
+                enable_unrestricted_guest = 0;
+        }
+
+        if (!cpu_has_vmx_unrestricted_guest())
+                enable_unrestricted_guest = 0;
 
         if (!cpu_has_vmx_flexpriority())
                 flexpriority_enabled = 0;
@@ -1342,6 +1391,9 @@ static __init int hardware_setup(void)
         if (!cpu_has_vmx_tpr_shadow())
                 kvm_x86_ops->update_cr8_intercept = NULL;
 
+        if (enable_ept && !cpu_has_vmx_ept_2m_page())
+                kvm_disable_largepages();
+
         return alloc_kvm_area();
 }
 
@@ -1372,15 +1424,15 @@ static void enter_pmode(struct kvm_vcpu *vcpu)
         struct vcpu_vmx *vmx = to_vmx(vcpu);
 
         vmx->emulation_required = 1;
-        vcpu->arch.rmode.vm86_active = 0;
+        vmx->rmode.vm86_active = 0;
 
-        vmcs_writel(GUEST_TR_BASE, vcpu->arch.rmode.tr.base);
-        vmcs_write32(GUEST_TR_LIMIT, vcpu->arch.rmode.tr.limit);
-        vmcs_write32(GUEST_TR_AR_BYTES, vcpu->arch.rmode.tr.ar);
+        vmcs_writel(GUEST_TR_BASE, vmx->rmode.tr.base);
+        vmcs_write32(GUEST_TR_LIMIT, vmx->rmode.tr.limit);
+        vmcs_write32(GUEST_TR_AR_BYTES, vmx->rmode.tr.ar);
 
         flags = vmcs_readl(GUEST_RFLAGS);
         flags &= ~(X86_EFLAGS_IOPL | X86_EFLAGS_VM);
-        flags |= (vcpu->arch.rmode.save_iopl << IOPL_SHIFT);
+        flags |= (vmx->rmode.save_iopl << IOPL_SHIFT);
         vmcs_writel(GUEST_RFLAGS, flags);
 
         vmcs_writel(GUEST_CR4, (vmcs_readl(GUEST_CR4) & ~X86_CR4_VME) |
@@ -1391,10 +1443,10 @@ static void enter_pmode(struct kvm_vcpu *vcpu)
         if (emulate_invalid_guest_state)
                 return;
 
-        fix_pmode_dataseg(VCPU_SREG_ES, &vcpu->arch.rmode.es);
-        fix_pmode_dataseg(VCPU_SREG_DS, &vcpu->arch.rmode.ds);
-        fix_pmode_dataseg(VCPU_SREG_GS, &vcpu->arch.rmode.gs);
-        fix_pmode_dataseg(VCPU_SREG_FS, &vcpu->arch.rmode.fs);
+        fix_pmode_dataseg(VCPU_SREG_ES, &vmx->rmode.es);
+        fix_pmode_dataseg(VCPU_SREG_DS, &vmx->rmode.ds);
+        fix_pmode_dataseg(VCPU_SREG_GS, &vmx->rmode.gs);
+        fix_pmode_dataseg(VCPU_SREG_FS, &vmx->rmode.fs);
 
         vmcs_write16(GUEST_SS_SELECTOR, 0);
         vmcs_write32(GUEST_SS_AR_BYTES, 0x93);
@@ -1433,20 +1485,23 @@ static void enter_rmode(struct kvm_vcpu *vcpu)
         unsigned long flags;
         struct vcpu_vmx *vmx = to_vmx(vcpu);
 
+        if (enable_unrestricted_guest)
+                return;
+
         vmx->emulation_required = 1;
-        vcpu->arch.rmode.vm86_active = 1;
+        vmx->rmode.vm86_active = 1;
 
-        vcpu->arch.rmode.tr.base = vmcs_readl(GUEST_TR_BASE);
+        vmx->rmode.tr.base = vmcs_readl(GUEST_TR_BASE);
         vmcs_writel(GUEST_TR_BASE, rmode_tss_base(vcpu->kvm));
 
-        vcpu->arch.rmode.tr.limit = vmcs_read32(GUEST_TR_LIMIT);
+        vmx->rmode.tr.limit = vmcs_read32(GUEST_TR_LIMIT);
         vmcs_write32(GUEST_TR_LIMIT, RMODE_TSS_SIZE - 1);
 
-        vcpu->arch.rmode.tr.ar = vmcs_read32(GUEST_TR_AR_BYTES);
+        vmx->rmode.tr.ar = vmcs_read32(GUEST_TR_AR_BYTES);
         vmcs_write32(GUEST_TR_AR_BYTES, 0x008b);
 
         flags = vmcs_readl(GUEST_RFLAGS);
-        vcpu->arch.rmode.save_iopl
+        vmx->rmode.save_iopl
                 = (flags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
 
         flags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM;
@@ -1468,10 +1523,10 @@ static void enter_rmode(struct kvm_vcpu *vcpu)
                 vmcs_writel(GUEST_CS_BASE, 0xf0000);
         vmcs_write16(GUEST_CS_SELECTOR, vmcs_readl(GUEST_CS_BASE) >> 4);
 
-        fix_rmode_seg(VCPU_SREG_ES, &vcpu->arch.rmode.es);
-        fix_rmode_seg(VCPU_SREG_DS, &vcpu->arch.rmode.ds);
-        fix_rmode_seg(VCPU_SREG_GS, &vcpu->arch.rmode.gs);
-        fix_rmode_seg(VCPU_SREG_FS, &vcpu->arch.rmode.fs);
+        fix_rmode_seg(VCPU_SREG_ES, &vmx->rmode.es);
+        fix_rmode_seg(VCPU_SREG_DS, &vmx->rmode.ds);
+        fix_rmode_seg(VCPU_SREG_GS, &vmx->rmode.gs);
+        fix_rmode_seg(VCPU_SREG_FS, &vmx->rmode.fs);
 
 continue_rmode:
         kvm_mmu_reset_context(vcpu);
@@ -1545,11 +1600,11 @@ static void vmx_decache_cr4_guest_bits(struct kvm_vcpu *vcpu)
 
 static void ept_load_pdptrs(struct kvm_vcpu *vcpu)
 {
+        if (!test_bit(VCPU_EXREG_PDPTR,
+                      (unsigned long *)&vcpu->arch.regs_dirty))
+                return;
+
         if (is_paging(vcpu) && is_pae(vcpu) && !is_long_mode(vcpu)) {
-                if (!load_pdptrs(vcpu, vcpu->arch.cr3)) {
-                        printk(KERN_ERR "EPT: Fail to load pdptrs!\n");
-                        return;
-                }
                 vmcs_write64(GUEST_PDPTR0, vcpu->arch.pdptrs[0]);
                 vmcs_write64(GUEST_PDPTR1, vcpu->arch.pdptrs[1]);
                 vmcs_write64(GUEST_PDPTR2, vcpu->arch.pdptrs[2]);
@@ -1557,6 +1612,21 @@ static void ept_load_pdptrs(struct kvm_vcpu *vcpu)
         }
 }
 
+static void ept_save_pdptrs(struct kvm_vcpu *vcpu)
+{
+        if (is_paging(vcpu) && is_pae(vcpu) && !is_long_mode(vcpu)) {
+                vcpu->arch.pdptrs[0] = vmcs_read64(GUEST_PDPTR0);
+                vcpu->arch.pdptrs[1] = vmcs_read64(GUEST_PDPTR1);
+                vcpu->arch.pdptrs[2] = vmcs_read64(GUEST_PDPTR2);
+                vcpu->arch.pdptrs[3] = vmcs_read64(GUEST_PDPTR3);
+        }
+
+        __set_bit(VCPU_EXREG_PDPTR,
+                  (unsigned long *)&vcpu->arch.regs_avail);
+        __set_bit(VCPU_EXREG_PDPTR,
+                  (unsigned long *)&vcpu->arch.regs_dirty);
+}
+
 static void vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4);
 
 static void ept_update_paging_mode_cr0(unsigned long *hw_cr0,
@@ -1571,8 +1641,6 @@ static void ept_update_paging_mode_cr0(unsigned long *hw_cr0,
                               CPU_BASED_CR3_STORE_EXITING));
                 vcpu->arch.cr0 = cr0;
                 vmx_set_cr4(vcpu, vcpu->arch.cr4);
-                *hw_cr0 |= X86_CR0_PE | X86_CR0_PG;
-                *hw_cr0 &= ~X86_CR0_WP;
         } else if (!is_paging(vcpu)) {
                 /* From nonpaging to paging */
                 vmcs_write32(CPU_BASED_VM_EXEC_CONTROL,
@@ -1581,9 +1649,10 @@ static void ept_update_paging_mode_cr0(unsigned long *hw_cr0,
                                CPU_BASED_CR3_STORE_EXITING));
                 vcpu->arch.cr0 = cr0;
                 vmx_set_cr4(vcpu, vcpu->arch.cr4);
-                if (!(vcpu->arch.cr0 & X86_CR0_WP))
-                        *hw_cr0 &= ~X86_CR0_WP;
         }
+
+        if (!(cr0 & X86_CR0_WP))
+                *hw_cr0 &= ~X86_CR0_WP;
 }
 
 static void ept_update_paging_mode_cr4(unsigned long *hw_cr4,
@@ -1598,15 +1667,21 @@ static void ept_update_paging_mode_cr4(unsigned long *hw_cr4,
 
 static void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
 {
-        unsigned long hw_cr0 = (cr0 & ~KVM_GUEST_CR0_MASK) |
-                                KVM_VM_CR0_ALWAYS_ON;
+        struct vcpu_vmx *vmx = to_vmx(vcpu);
+        unsigned long hw_cr0;
+
+        if (enable_unrestricted_guest)
+                hw_cr0 = (cr0 & ~KVM_GUEST_CR0_MASK_UNRESTRICTED_GUEST)
+                        | KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST;
+        else
+                hw_cr0 = (cr0 & ~KVM_GUEST_CR0_MASK) | KVM_VM_CR0_ALWAYS_ON;
 
         vmx_fpu_deactivate(vcpu);
 
-        if (vcpu->arch.rmode.vm86_active && (cr0 & X86_CR0_PE))
+        if (vmx->rmode.vm86_active && (cr0 & X86_CR0_PE))
                 enter_pmode(vcpu);
 
-        if (!vcpu->arch.rmode.vm86_active && !(cr0 & X86_CR0_PE))
+        if (!vmx->rmode.vm86_active && !(cr0 & X86_CR0_PE))
                 enter_rmode(vcpu);
 
 #ifdef CONFIG_X86_64
@@ -1650,10 +1725,8 @@ static void vmx_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
         if (enable_ept) {
                 eptp = construct_eptp(cr3);
                 vmcs_write64(EPT_POINTER, eptp);
-                ept_sync_context(eptp);
-                ept_load_pdptrs(vcpu);
                 guest_cr3 = is_paging(vcpu) ? vcpu->arch.cr3 :
-                        VMX_EPT_IDENTITY_PAGETABLE_ADDR;
+                        vcpu->kvm->arch.ept_identity_map_addr;
         }
 
         vmx_flush_tlb(vcpu);
@@ -1664,7 +1737,7 @@ static void vmx_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
 
 static void vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
 {
-        unsigned long hw_cr4 = cr4 | (vcpu->arch.rmode.vm86_active ?
+        unsigned long hw_cr4 = cr4 | (to_vmx(vcpu)->rmode.vm86_active ?
                     KVM_RMODE_VM_CR4_ALWAYS_ON : KVM_PMODE_VM_CR4_ALWAYS_ON);
 
         vcpu->arch.cr4 = cr4;
@@ -1707,16 +1780,13 @@ static void vmx_get_segment(struct kvm_vcpu *vcpu,
 
 static int vmx_get_cpl(struct kvm_vcpu *vcpu)
 {
-        struct kvm_segment kvm_seg;
-
         if (!(vcpu->arch.cr0 & X86_CR0_PE)) /* if real mode */
                 return 0;
 
         if (vmx_get_rflags(vcpu) & X86_EFLAGS_VM) /* if virtual 8086 */
                 return 3;
 
-        vmx_get_segment(vcpu, &kvm_seg, VCPU_SREG_CS);
-        return kvm_seg.selector & 3;
+        return vmcs_read16(GUEST_CS_SELECTOR) & 3;
 }
 
 static u32 vmx_segment_access_rights(struct kvm_segment *var)
@@ -1744,20 +1814,21 @@ static u32 vmx_segment_access_rights(struct kvm_segment *var)
 static void vmx_set_segment(struct kvm_vcpu *vcpu,
                             struct kvm_segment *var, int seg)
 {
+        struct vcpu_vmx *vmx = to_vmx(vcpu);
         struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
         u32 ar;
 
-        if (vcpu->arch.rmode.vm86_active && seg == VCPU_SREG_TR) {
-                vcpu->arch.rmode.tr.selector = var->selector;
-                vcpu->arch.rmode.tr.base = var->base;
-                vcpu->arch.rmode.tr.limit = var->limit;
-                vcpu->arch.rmode.tr.ar = vmx_segment_access_rights(var);
+        if (vmx->rmode.vm86_active && seg == VCPU_SREG_TR) {
+                vmx->rmode.tr.selector = var->selector;
+                vmx->rmode.tr.base = var->base;
+                vmx->rmode.tr.limit = var->limit;
+                vmx->rmode.tr.ar = vmx_segment_access_rights(var);
                 return;
         }
         vmcs_writel(sf->base, var->base);
         vmcs_write32(sf->limit, var->limit);
         vmcs_write16(sf->selector, var->selector);
-        if (vcpu->arch.rmode.vm86_active && var->s) {
+        if (vmx->rmode.vm86_active && var->s) {
                 /*
                  * Hack real-mode segments into vm86 compatibility.
                  */
@@ -1766,6 +1837,21 @@ static void vmx_set_segment(struct kvm_vcpu *vcpu,
                 ar = 0xf3;
         } else
                 ar = vmx_segment_access_rights(var);
+
+        /*
+         *   Fix the "Accessed" bit in AR field of segment registers for older
+         * qemu binaries.
+         *   IA32 arch specifies that at the time of processor reset the
+         * "Accessed" bit in the AR field of segment registers is 1. And qemu
+         * is setting it to 0 in the usedland code. This causes invalid guest
+         * state vmexit when "unrestricted guest" mode is turned on.
+         *    Fix for this setup issue in cpu_reset is being pushed in the qemu
+         * tree. Newer qemu binaries with that qemu fix would not need this
+         * kvm hack.
+         */
+        if (enable_unrestricted_guest && (seg != VCPU_SREG_LDTR))
+                ar |= 0x1; /* Accessed */
+
         vmcs_write32(sf->ar_bytes, ar);
 }
 
@@ -2040,7 +2126,7 @@ static int init_rmode_identity_map(struct kvm *kvm)
         if (likely(kvm->arch.ept_identity_pagetable_done))
                 return 1;
         ret = 0;
-        identity_map_pfn = VMX_EPT_IDENTITY_PAGETABLE_ADDR >> PAGE_SHIFT;
+        identity_map_pfn = kvm->arch.ept_identity_map_addr >> PAGE_SHIFT;
         r = kvm_clear_guest_page(kvm, identity_map_pfn, 0, PAGE_SIZE);
         if (r < 0)
                 goto out;
@@ -2062,11 +2148,19 @@ out:
 static void seg_setup(int seg)
 {
         struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
+        unsigned int ar;
 
         vmcs_write16(sf->selector, 0);
         vmcs_writel(sf->base, 0);
         vmcs_write32(sf->limit, 0xffff);
-        vmcs_write32(sf->ar_bytes, 0xf3);
+        if (enable_unrestricted_guest) {
+                ar = 0x93;
+                if (seg == VCPU_SREG_CS)
+                        ar |= 0x08; /* code segment */
+        } else
+                ar = 0xf3;
+
+        vmcs_write32(sf->ar_bytes, ar);
 }
 
 static int alloc_apic_access_page(struct kvm *kvm)
@@ -2101,14 +2195,15 @@ static int alloc_identity_pagetable(struct kvm *kvm)
                 goto out;
         kvm_userspace_mem.slot = IDENTITY_PAGETABLE_PRIVATE_MEMSLOT;
         kvm_userspace_mem.flags = 0;
-        kvm_userspace_mem.guest_phys_addr = VMX_EPT_IDENTITY_PAGETABLE_ADDR;
+        kvm_userspace_mem.guest_phys_addr =
+                kvm->arch.ept_identity_map_addr;
         kvm_userspace_mem.memory_size = PAGE_SIZE;
         r = __kvm_set_memory_region(kvm, &kvm_userspace_mem, 0);
         if (r)
                 goto out;
 
         kvm->arch.ept_identity_pagetable = gfn_to_page(kvm,
-                        VMX_EPT_IDENTITY_PAGETABLE_ADDR >> PAGE_SHIFT);
+                        kvm->arch.ept_identity_map_addr >> PAGE_SHIFT);
 out:
         up_write(&kvm->slots_lock);
         return r;
@@ -2209,6 +2304,8 @@ static int vmx_vcpu_setup(struct vcpu_vmx *vmx)
                         exec_control &= ~SECONDARY_EXEC_ENABLE_VPID;
                 if (!enable_ept)
                         exec_control &= ~SECONDARY_EXEC_ENABLE_EPT;
+                if (!enable_unrestricted_guest)
+                        exec_control &= ~SECONDARY_EXEC_UNRESTRICTED_GUEST;
                 vmcs_write32(SECONDARY_VM_EXEC_CONTROL, exec_control);
         }
 
@@ -2326,14 +2423,14 @@ static int vmx_vcpu_reset(struct kvm_vcpu *vcpu)
                 goto out;
         }
 
-        vmx->vcpu.arch.rmode.vm86_active = 0;
+        vmx->rmode.vm86_active = 0;
 
         vmx->soft_vnmi_blocked = 0;
 
         vmx->vcpu.arch.regs[VCPU_REGS_RDX] = get_rdx_init_val();
         kvm_set_cr8(&vmx->vcpu, 0);
         msr = 0xfee00000 | MSR_IA32_APICBASE_ENABLE;
-        if (vmx->vcpu.vcpu_id == 0)
+        if (kvm_vcpu_is_bsp(&vmx->vcpu))
                 msr |= MSR_IA32_APICBASE_BSP;
         kvm_set_apic_base(&vmx->vcpu, msr);
 
@@ -2344,7 +2441,7 @@ static int vmx_vcpu_reset(struct kvm_vcpu *vcpu)
          * GUEST_CS_BASE should really be 0xffff0000, but VT vm86 mode
          * insists on having GUEST_CS_BASE == GUEST_CS_SELECTOR << 4.  Sigh.
          */
-        if (vmx->vcpu.vcpu_id == 0) {
+        if (kvm_vcpu_is_bsp(&vmx->vcpu)) {
                 vmcs_write16(GUEST_CS_SELECTOR, 0xf000);
                 vmcs_writel(GUEST_CS_BASE, 0x000f0000);
         } else {
@@ -2373,7 +2470,7 @@ static int vmx_vcpu_reset(struct kvm_vcpu *vcpu)
         vmcs_writel(GUEST_SYSENTER_EIP, 0);
 
         vmcs_writel(GUEST_RFLAGS, 0x02);
-        if (vmx->vcpu.vcpu_id == 0)
+        if (kvm_vcpu_is_bsp(&vmx->vcpu))
                 kvm_rip_write(vcpu, 0xfff0);
         else
                 kvm_rip_write(vcpu, 0);
@@ -2461,13 +2558,16 @@ static void vmx_inject_irq(struct kvm_vcpu *vcpu)
         uint32_t intr;
         int irq = vcpu->arch.interrupt.nr;
 
-        KVMTRACE_1D(INJ_VIRQ, vcpu, (u32)irq, handler);
+        trace_kvm_inj_virq(irq);
 
         ++vcpu->stat.irq_injections;
-        if (vcpu->arch.rmode.vm86_active) {
+        if (vmx->rmode.vm86_active) {
                 vmx->rmode.irq.pending = true;
                 vmx->rmode.irq.vector = irq;
                 vmx->rmode.irq.rip = kvm_rip_read(vcpu);
+                if (vcpu->arch.interrupt.soft)
+                        vmx->rmode.irq.rip +=
+                                vmx->vcpu.arch.event_exit_inst_len;
                 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
                              irq | INTR_TYPE_SOFT_INTR | INTR_INFO_VALID_MASK);
                 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, 1);
@@ -2502,7 +2602,7 @@ static void vmx_inject_nmi(struct kvm_vcpu *vcpu)
         }
 
         ++vcpu->stat.nmi_injections;
-        if (vcpu->arch.rmode.vm86_active) {
+        if (vmx->rmode.vm86_active) {
                 vmx->rmode.irq.pending = true;
                 vmx->rmode.irq.vector = NMI_VECTOR;
                 vmx->rmode.irq.rip = kvm_rip_read(vcpu);
@@ -2659,14 +2759,14 @@ static int handle_exception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
                 if (enable_ept)
                         BUG();
                 cr2 = vmcs_readl(EXIT_QUALIFICATION);
-                KVMTRACE_3D(PAGE_FAULT, vcpu, error_code, (u32)cr2,
-                            (u32)((u64)cr2 >> 32), handler);
+                trace_kvm_page_fault(cr2, error_code);
+
                 if (kvm_event_needs_reinjection(vcpu))
                         kvm_mmu_unprotect_page_virt(vcpu, cr2);
                 return kvm_mmu_page_fault(vcpu, cr2, error_code);
         }
 
-        if (vcpu->arch.rmode.vm86_active &&
+        if (vmx->rmode.vm86_active &&
             handle_rmode_exception(vcpu, intr_info & INTR_INFO_VECTOR_MASK,
                                                                 error_code)) {
                 if (vcpu->arch.halt_request) {
@@ -2707,7 +2807,6 @@ static int handle_external_interrupt(struct kvm_vcpu *vcpu,
                                      struct kvm_run *kvm_run)
 {
         ++vcpu->stat.irq_exits;
-        KVMTRACE_1D(INTR, vcpu, vmcs_read32(VM_EXIT_INTR_INFO), handler);
         return 1;
 }
 
@@ -2755,7 +2854,7 @@ vmx_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
 
 static int handle_cr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 {
-        unsigned long exit_qualification;
+        unsigned long exit_qualification, val;
         int cr;
         int reg;
 
@@ -2764,21 +2863,19 @@ static int handle_cr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
         reg = (exit_qualification >> 8) & 15;
         switch ((exit_qualification >> 4) & 3) {
         case 0: /* mov to cr */
-                KVMTRACE_3D(CR_WRITE, vcpu, (u32)cr,
-                            (u32)kvm_register_read(vcpu, reg),
-                            (u32)((u64)kvm_register_read(vcpu, reg) >> 32),
-                            handler);
+                val = kvm_register_read(vcpu, reg);
+                trace_kvm_cr_write(cr, val);
                 switch (cr) {
                 case 0:
-                        kvm_set_cr0(vcpu, kvm_register_read(vcpu, reg));
+                        kvm_set_cr0(vcpu, val);
                         skip_emulated_instruction(vcpu);
                         return 1;
                 case 3:
-                        kvm_set_cr3(vcpu, kvm_register_read(vcpu, reg));
+                        kvm_set_cr3(vcpu, val);
                         skip_emulated_instruction(vcpu);
                         return 1;
                 case 4:
-                        kvm_set_cr4(vcpu, kvm_register_read(vcpu, reg));
+                        kvm_set_cr4(vcpu, val);
                         skip_emulated_instruction(vcpu);
                         return 1;
                 case 8: {
@@ -2800,23 +2897,19 @@ static int handle_cr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
                 vcpu->arch.cr0 &= ~X86_CR0_TS;
                 vmcs_writel(CR0_READ_SHADOW, vcpu->arch.cr0);
                 vmx_fpu_activate(vcpu);
-                KVMTRACE_0D(CLTS, vcpu, handler);
                 skip_emulated_instruction(vcpu);
                 return 1;
         case 1: /*mov from cr*/
                 switch (cr) {
                 case 3:
                         kvm_register_write(vcpu, reg, vcpu->arch.cr3);
-                        KVMTRACE_3D(CR_READ, vcpu, (u32)cr,
-                                    (u32)kvm_register_read(vcpu, reg),
-                                    (u32)((u64)kvm_register_read(vcpu, reg) >> 32),
-                                    handler);
+                        trace_kvm_cr_read(cr, vcpu->arch.cr3);
                         skip_emulated_instruction(vcpu);
                         return 1;
                 case 8:
-                        kvm_register_write(vcpu, reg, kvm_get_cr8(vcpu));
-                        KVMTRACE_2D(CR_READ, vcpu, (u32)cr,
-                                    (u32)kvm_register_read(vcpu, reg), handler);
+                        val = kvm_get_cr8(vcpu);
+                        kvm_register_write(vcpu, reg, val);
+                        trace_kvm_cr_read(cr, val);
                         skip_emulated_instruction(vcpu);
                         return 1;
                 }
@@ -2841,6 +2934,8 @@ static int handle_dr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
         unsigned long val;
         int dr, reg;
 
+        if (!kvm_require_cpl(vcpu, 0))
+                return 1;
         dr = vmcs_readl(GUEST_DR7);
         if (dr & DR7_GD) {
                 /*
@@ -2884,7 +2979,6 @@ static int handle_dr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
                         val = 0;
                 }
                 kvm_register_write(vcpu, reg, val);
-                KVMTRACE_2D(DR_READ, vcpu, (u32)dr, (u32)val, handler);
         } else {
                 val = vcpu->arch.regs[reg];
                 switch (dr) {
@@ -2917,7 +3011,6 @@ static int handle_dr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
                         }
                         break;
                 }
-                KVMTRACE_2D(DR_WRITE, vcpu, (u32)dr, (u32)val, handler);
         }
         skip_emulated_instruction(vcpu);
         return 1;
@@ -2939,8 +3032,7 @@ static int handle_rdmsr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
                 return 1;
         }
 
-        KVMTRACE_3D(MSR_READ, vcpu, ecx, (u32)data, (u32)(data >> 32),
-                    handler);
+        trace_kvm_msr_read(ecx, data);
 
         /* FIXME: handling of bits 32:63 of rax, rdx */
         vcpu->arch.regs[VCPU_REGS_RAX] = data & -1u;
@@ -2955,8 +3047,7 @@ static int handle_wrmsr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
         u64 data = (vcpu->arch.regs[VCPU_REGS_RAX] & -1u)
                 | ((u64)(vcpu->arch.regs[VCPU_REGS_RDX] & -1u) << 32);
 
-        KVMTRACE_3D(MSR_WRITE, vcpu, ecx, (u32)data, (u32)(data >> 32),
-                    handler);
+        trace_kvm_msr_write(ecx, data);
 
         if (vmx_set_msr(vcpu, ecx, data) != 0) {
                 kvm_inject_gp(vcpu, 0);
@@ -2983,7 +3074,6 @@ static int handle_interrupt_window(struct kvm_vcpu *vcpu,
         cpu_based_vm_exec_control &= ~CPU_BASED_VIRTUAL_INTR_PENDING;
         vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
 
-        KVMTRACE_0D(PEND_INTR, vcpu, handler);
         ++vcpu->stat.irq_window_exits;
 
         /*
@@ -3049,7 +3139,7 @@ static int handle_apic_access(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
                 printk(KERN_ERR
                        "Fail to handle apic access vmexit! Offset is 0x%lx\n",
                        offset);
-                return -ENOTSUPP;
+                return -ENOEXEC;
         }
         return 1;
 }
@@ -3118,7 +3208,7 @@ static int handle_ept_violation(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 
         if (exit_qualification & (1 << 6)) {
                 printk(KERN_ERR "EPT: GPA exceeds GAW!\n");
-                return -ENOTSUPP;
+                return -EINVAL;
         }
 
         gla_validity = (exit_qualification >> 7) & 0x3;
@@ -3130,14 +3220,98 @@ static int handle_ept_violation(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
                 printk(KERN_ERR "EPT: Exit qualification is 0x%lx\n",
                         (long unsigned int)exit_qualification);
                 kvm_run->exit_reason = KVM_EXIT_UNKNOWN;
-                kvm_run->hw.hardware_exit_reason = 0;
-                return -ENOTSUPP;
+                kvm_run->hw.hardware_exit_reason = EXIT_REASON_EPT_VIOLATION;
+                return 0;
         }
 
         gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS);
+        trace_kvm_page_fault(gpa, exit_qualification);
         return kvm_mmu_page_fault(vcpu, gpa & PAGE_MASK, 0);
 }
 
+static u64 ept_rsvd_mask(u64 spte, int level)
+{
+        int i;
+        u64 mask = 0;
+
+        for (i = 51; i > boot_cpu_data.x86_phys_bits; i--)
+                mask |= (1ULL << i);
+
+        if (level > 2)
+                /* bits 7:3 reserved */
+                mask |= 0xf8;
+        else if (level == 2) {
+                if (spte & (1ULL << 7))
+                        /* 2MB ref, bits 20:12 reserved */
+                        mask |= 0x1ff000;
+                else
+                        /* bits 6:3 reserved */
+                        mask |= 0x78;
+        }
+
+        return mask;
+}
+
+static void ept_misconfig_inspect_spte(struct kvm_vcpu *vcpu, u64 spte,
+                                       int level)
+{
+        printk(KERN_ERR "%s: spte 0x%llx level %d\n", __func__, spte, level);
+
+        /* 010b (write-only) */
+        WARN_ON((spte & 0x7) == 0x2);
+
+        /* 110b (write/execute) */
+        WARN_ON((spte & 0x7) == 0x6);
+
+        /* 100b (execute-only) and value not supported by logical processor */
+        if (!cpu_has_vmx_ept_execute_only())
+                WARN_ON((spte & 0x7) == 0x4);
+
+        /* not 000b */
+        if ((spte & 0x7)) {
+                u64 rsvd_bits = spte & ept_rsvd_mask(spte, level);
+
+                if (rsvd_bits != 0) {
+                        printk(KERN_ERR "%s: rsvd_bits = 0x%llx\n",
+                                         __func__, rsvd_bits);
+                        WARN_ON(1);
+                }
+
+                if (level == 1 || (level == 2 && (spte & (1ULL << 7)))) {
+                        u64 ept_mem_type = (spte & 0x38) >> 3;
+
+                        if (ept_mem_type == 2 || ept_mem_type == 3 ||
+                            ept_mem_type == 7) {
+                                printk(KERN_ERR "%s: ept_mem_type=0x%llx\n",
+                                                __func__, ept_mem_type);
+                                WARN_ON(1);
+                        }
+                }
+        }
+}
+
+static int handle_ept_misconfig(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
+{
+        u64 sptes[4];
+        int nr_sptes, i;
+        gpa_t gpa;
+
+        gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS);
+
+        printk(KERN_ERR "EPT: Misconfiguration.\n");
+        printk(KERN_ERR "EPT: GPA: 0x%llx\n", gpa);
+
+        nr_sptes = kvm_mmu_get_spte_hierarchy(vcpu, gpa, sptes);
+
+        for (i = PT64_ROOT_LEVEL; i > PT64_ROOT_LEVEL - nr_sptes; --i)
+                ept_misconfig_inspect_spte(vcpu, sptes[i-1], i);
+
+        kvm_run->exit_reason = KVM_EXIT_UNKNOWN;
+        kvm_run->hw.hardware_exit_reason = EXIT_REASON_EPT_MISCONFIG;
+
+        return 0;
+}
+
 static int handle_nmi_window(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 {
         u32 cpu_based_vm_exec_control;
@@ -3217,8 +3391,9 @@ static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu,
         [EXIT_REASON_APIC_ACCESS]             = handle_apic_access,
         [EXIT_REASON_WBINVD]                  = handle_wbinvd,
         [EXIT_REASON_TASK_SWITCH]             = handle_task_switch,
-        [EXIT_REASON_EPT_VIOLATION]           = handle_ept_violation,
         [EXIT_REASON_MCE_DURING_VMENTRY]      = handle_machine_check,
+        [EXIT_REASON_EPT_VIOLATION]           = handle_ept_violation,
+        [EXIT_REASON_EPT_MISCONFIG]           = handle_ept_misconfig,
 };
 
 static const int kvm_vmx_max_exit_handlers =
@@ -3234,8 +3409,7 @@ static int vmx_handle_exit(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
         u32 exit_reason = vmx->exit_reason;
         u32 vectoring_info = vmx->idt_vectoring_info;
 
-        KVMTRACE_3D(VMEXIT, vcpu, exit_reason, (u32)kvm_rip_read(vcpu),
-                    (u32)((u64)kvm_rip_read(vcpu) >> 32), entryexit);
+        trace_kvm_exit(exit_reason, kvm_rip_read(vcpu));
 
         /* If we need to emulate an MMIO from handle_invalid_guest_state
          * we just return 0 */
@@ -3247,10 +3421,8 @@ static int vmx_handle_exit(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
 
         /* Access CR3 don't cause VMExit in paging mode, so we need
          * to sync with guest real CR3. */
-        if (enable_ept && is_paging(vcpu)) {
+        if (enable_ept && is_paging(vcpu))
                 vcpu->arch.cr3 = vmcs_readl(GUEST_CR3);
-                ept_load_pdptrs(vcpu);
-        }
 
         if (unlikely(vmx->fail)) {
                 kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY;
@@ -3326,10 +3498,8 @@ static void vmx_complete_interrupts(struct vcpu_vmx *vmx)
 
         /* We need to handle NMIs before interrupts are enabled */
         if ((exit_intr_info & INTR_INFO_INTR_TYPE_MASK) == INTR_TYPE_NMI_INTR &&
-            (exit_intr_info & INTR_INFO_VALID_MASK)) {
-                KVMTRACE_0D(NMI, &vmx->vcpu, handler);
+            (exit_intr_info & INTR_INFO_VALID_MASK))
                 asm("int $2");
-        }
 
         idtv_info_valid = idt_vectoring_info & VECTORING_INFO_VALID_MASK;
 
@@ -3434,6 +3604,10 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 {
         struct vcpu_vmx *vmx = to_vmx(vcpu);
 
+        if (enable_ept && is_paging(vcpu)) {
+                vmcs_writel(GUEST_CR3, vcpu->arch.cr3);
+                ept_load_pdptrs(vcpu);
+        }
         /* Record the guest's net vcpu time for enforced NMI injections. */
         if (unlikely(!cpu_has_virtual_nmis() && vmx->soft_vnmi_blocked))
                 vmx->entry_time = ktime_get();
@@ -3449,12 +3623,21 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
         if (test_bit(VCPU_REGS_RIP, (unsigned long *)&vcpu->arch.regs_dirty))
                 vmcs_writel(GUEST_RIP, vcpu->arch.regs[VCPU_REGS_RIP]);
 
+        /* When single-stepping over STI and MOV SS, we must clear the
+         * corresponding interruptibility bits in the guest state. Otherwise
+         * vmentry fails as it then expects bit 14 (BS) in pending debug
+         * exceptions being set, but that's not correct for the guest debugging
+         * case. */
+        if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
+                vmx_set_interrupt_shadow(vcpu, 0);
+
         /*
          * Loading guest fpu may have cleared host cr0.ts
          */
         vmcs_writel(HOST_CR0, read_cr0());
 
-        set_debugreg(vcpu->arch.dr6, 6);
+        if (vcpu->arch.switch_db_regs)
+                set_debugreg(vcpu->arch.dr6, 6);
 
         asm(
                 /* Store host registers */
@@ -3465,11 +3648,16 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
                 "mov %%"R"sp, %c[host_rsp](%0) \n\t"
                 __ex(ASM_VMX_VMWRITE_RSP_RDX) "\n\t"
                 "1: \n\t"
+                /* Reload cr2 if changed */
+                "mov %c[cr2](%0), %%"R"ax \n\t"
+                "mov %%cr2, %%"R"dx \n\t"
+                "cmp %%"R"ax, %%"R"dx \n\t"
+                "je 2f \n\t"
+                "mov %%"R"ax, %%cr2 \n\t"
+                "2: \n\t"
                 /* Check if vmlaunch of vmresume is needed */
                 "cmpl $0, %c[launched](%0) \n\t"
                 /* Load guest registers.  Don't clobber flags. */
-                "mov %c[cr2](%0), %%"R"ax \n\t"
-                "mov %%"R"ax, %%cr2 \n\t"
                 "mov %c[rax](%0), %%"R"ax \n\t"
                 "mov %c[rbx](%0), %%"R"bx \n\t"
                 "mov %c[rdx](%0), %%"R"dx \n\t"
@@ -3547,10 +3735,12 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 #endif
               );
 
-        vcpu->arch.regs_avail = ~((1 << VCPU_REGS_RIP) | (1 << VCPU_REGS_RSP));
+        vcpu->arch.regs_avail = ~((1 << VCPU_REGS_RIP) | (1 << VCPU_REGS_RSP)
+                                  | (1 << VCPU_EXREG_PDPTR));
         vcpu->arch.regs_dirty = 0;
 
-        get_debugreg(vcpu->arch.dr6, 6);
+        if (vcpu->arch.switch_db_regs)
+                get_debugreg(vcpu->arch.dr6, 6);
 
         vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD);
         if (vmx->rmode.irq.pending)
@@ -3633,9 +3823,13 @@ static struct kvm_vcpu *vmx_create_vcpu(struct kvm *kvm, unsigned int id)
                 if (alloc_apic_access_page(kvm) != 0)
                         goto free_vmcs;
 
-        if (enable_ept)
+        if (enable_ept) {
+                if (!kvm->arch.ept_identity_map_addr)
+                        kvm->arch.ept_identity_map_addr =
+                                VMX_EPT_IDENTITY_PAGETABLE_ADDR;
                 if (alloc_identity_pagetable(kvm) != 0)
                         goto free_vmcs;
+        }
 
         return &vmx->vcpu;
 
@@ -3699,6 +3893,34 @@ static u64 vmx_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
         return ret;
 }
 
+static const struct trace_print_flags vmx_exit_reasons_str[] = {
+        { EXIT_REASON_EXCEPTION_NMI,           "exception" },
+        { EXIT_REASON_EXTERNAL_INTERRUPT,      "ext_irq" },
+        { EXIT_REASON_TRIPLE_FAULT,            "triple_fault" },
+        { EXIT_REASON_NMI_WINDOW,              "nmi_window" },
+        { EXIT_REASON_IO_INSTRUCTION,          "io_instruction" },
+        { EXIT_REASON_CR_ACCESS,               "cr_access" },
+        { EXIT_REASON_DR_ACCESS,               "dr_access" },
+        { EXIT_REASON_CPUID,                   "cpuid" },
+        { EXIT_REASON_MSR_READ,                "rdmsr" },
+        { EXIT_REASON_MSR_WRITE,               "wrmsr" },
+        { EXIT_REASON_PENDING_INTERRUPT,       "interrupt_window" },
+        { EXIT_REASON_HLT,                     "halt" },
+        { EXIT_REASON_INVLPG,                  "invlpg" },
+        { EXIT_REASON_VMCALL,                  "hypercall" },
+        { EXIT_REASON_TPR_BELOW_THRESHOLD,     "tpr_below_thres" },
+        { EXIT_REASON_APIC_ACCESS,             "apic_access" },
+        { EXIT_REASON_WBINVD,                  "wbinvd" },
+        { EXIT_REASON_TASK_SWITCH,             "task_switch" },
+        { EXIT_REASON_EPT_VIOLATION,           "ept_violation" },
+        { -1, NULL }
+};
+
+static bool vmx_gb_page_enable(void)
+{
+        return false;
+}
+
 static struct kvm_x86_ops vmx_x86_ops = {
         .cpu_has_kvm_support = cpu_has_kvm_support,
         .disabled_by_bios = vmx_disabled_by_bios,
@@ -3758,6 +3980,9 @@ static struct kvm_x86_ops vmx_x86_ops = {
         .set_tss_addr = vmx_set_tss_addr,
         .get_tdp_level = get_ept_level,
         .get_mt_mask = vmx_get_mt_mask,
+
+        .exit_reasons_str = vmx_exit_reasons_str,
+        .gb_page_enable = vmx_gb_page_enable,
 };
 
 static int __init vmx_init(void)
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 3d4529011828..be451ee44249 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -37,11 +37,16 @@
 #include <linux/iommu.h>
 #include <linux/intel-iommu.h>
 #include <linux/cpufreq.h>
+#include <trace/events/kvm.h>
+#undef TRACE_INCLUDE_FILE
+#define CREATE_TRACE_POINTS
+#include "trace.h"
 
 #include <asm/uaccess.h>
 #include <asm/msr.h>
 #include <asm/desc.h>
 #include <asm/mtrr.h>
+#include <asm/mce.h>
 
 #define MAX_IO_MSRS 256
 #define CR0_RESERVED_BITS                                               \
@@ -55,6 +60,10 @@
                           | X86_CR4_OSXMMEXCPT | X86_CR4_VMXE))
 
 #define CR8_RESERVED_BITS (~(unsigned long)X86_CR8_TPR)
+
+#define KVM_MAX_MCE_BANKS 32
+#define KVM_MCE_CAP_SUPPORTED MCG_CTL_P
+
 /* EFER defaults:
  * - enable syscall per default because its emulated by KVM
  * - enable LME and LMA per default on 64 bit KVM
@@ -68,14 +77,16 @@ static u64 __read_mostly efer_reserved_bits = 0xfffffffffffffffeULL;
 #define VM_STAT(x) offsetof(struct kvm, stat.x), KVM_STAT_VM
 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
 
+static void update_cr8_intercept(struct kvm_vcpu *vcpu);
 static int kvm_dev_ioctl_get_supported_cpuid(struct kvm_cpuid2 *cpuid,
                                     struct kvm_cpuid_entry2 __user *entries);
-struct kvm_cpuid_entry2 *kvm_find_cpuid_entry(struct kvm_vcpu *vcpu,
-                                              u32 function, u32 index);
 
 struct kvm_x86_ops *kvm_x86_ops;
 EXPORT_SYMBOL_GPL(kvm_x86_ops);
 
+int ignore_msrs = 0;
+module_param_named(ignore_msrs, ignore_msrs, bool, S_IRUGO | S_IWUSR);
+
 struct kvm_stats_debugfs_item debugfs_entries[] = {
         { "pf_fixed", VCPU_STAT(pf_fixed) },
         { "pf_guest", VCPU_STAT(pf_guest) },
@@ -122,18 +133,16 @@ unsigned long segment_base(u16 selector)
         if (selector == 0)
                 return 0;
 
-        asm("sgdt %0" : "=m"(gdt));
+        kvm_get_gdt(&gdt);
         table_base = gdt.base;
 
         if (selector & 4) {           /* from ldt */
-                u16 ldt_selector;
+                u16 ldt_selector = kvm_read_ldt();
 
-                asm("sldt %0" : "=g"(ldt_selector));
                 table_base = segment_base(ldt_selector);
         }
         d = (struct desc_struct *)(table_base + (selector & ~7));
-        v = d->base0 | ((unsigned long)d->base1 << 16) |
-                ((unsigned long)d->base2 << 24);
+        v = get_desc_base(d);
 #ifdef CONFIG_X86_64
         if (d->s == 0 && (d->type == 2 || d->type == 9 || d->type == 11))
                 v |= ((unsigned long)((struct ldttss_desc64 *)d)->base3) << 32;
@@ -176,16 +185,22 @@ void kvm_inject_page_fault(struct kvm_vcpu *vcpu, unsigned long addr,
         ++vcpu->stat.pf_guest;
 
         if (vcpu->arch.exception.pending) {
-                if (vcpu->arch.exception.nr == PF_VECTOR) {
-                        printk(KERN_DEBUG "kvm: inject_page_fault:"
-                                        " double fault 0x%lx\n", addr);
-                        vcpu->arch.exception.nr = DF_VECTOR;
-                        vcpu->arch.exception.error_code = 0;
-                } else if (vcpu->arch.exception.nr == DF_VECTOR) {
+                switch(vcpu->arch.exception.nr) {
+                case DF_VECTOR:
                         /* triple fault -> shutdown */
                         set_bit(KVM_REQ_TRIPLE_FAULT, &vcpu->requests);
+                        return;
+                case PF_VECTOR:
+                        vcpu->arch.exception.nr = DF_VECTOR;
+                        vcpu->arch.exception.error_code = 0;
+                        return;
+                default:
+                        /* replace previous exception with a new one in a hope
+                           that instruction re-execution will regenerate lost
+                           exception */
+                        vcpu->arch.exception.pending = false;
+                        break;
                 }
-                return;
         }
         vcpu->arch.cr2 = addr;
         kvm_queue_exception_e(vcpu, PF_VECTOR, error_code);
@@ -207,12 +222,18 @@ void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)
 }
 EXPORT_SYMBOL_GPL(kvm_queue_exception_e);
 
-static void __queue_exception(struct kvm_vcpu *vcpu)
+/*
+ * Checks if cpl <= required_cpl; if true, return true.  Otherwise queue
+ * a #GP and return false.
+ */
+bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl)
 {
-        kvm_x86_ops->queue_exception(vcpu, vcpu->arch.exception.nr,
-                                     vcpu->arch.exception.has_error_code,
-                                     vcpu->arch.exception.error_code);
+        if (kvm_x86_ops->get_cpl(vcpu) <= required_cpl)
+                return true;
+        kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
+        return false;
 }
+EXPORT_SYMBOL_GPL(kvm_require_cpl);
 
 /*
  * Load the pae pdptrs.  Return true is they are all valid.
@@ -232,7 +253,7 @@ int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3)
                 goto out;
         }
         for (i = 0; i < ARRAY_SIZE(pdpte); ++i) {
-                if (is_present_pte(pdpte[i]) &&
+                if (is_present_gpte(pdpte[i]) &&
                     (pdpte[i] & vcpu->arch.mmu.rsvd_bits_mask[0][2])) {
                         ret = 0;
                         goto out;
@@ -241,6 +262,10 @@ int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3)
         ret = 1;
 
         memcpy(vcpu->arch.pdptrs, pdpte, sizeof(vcpu->arch.pdptrs));
+        __set_bit(VCPU_EXREG_PDPTR,
+                  (unsigned long *)&vcpu->arch.regs_avail);
+        __set_bit(VCPU_EXREG_PDPTR,
+                  (unsigned long *)&vcpu->arch.regs_dirty);
 out:
 
         return ret;
@@ -256,6 +281,10 @@ static bool pdptrs_changed(struct kvm_vcpu *vcpu)
         if (is_long_mode(vcpu) || !is_pae(vcpu))
                 return false;
 
+        if (!test_bit(VCPU_EXREG_PDPTR,
+                      (unsigned long *)&vcpu->arch.regs_avail))
+                return true;
+
         r = kvm_read_guest(vcpu->kvm, vcpu->arch.cr3 & ~31u, pdpte, sizeof(pdpte));
         if (r < 0)
                 goto out;
@@ -328,9 +357,6 @@ EXPORT_SYMBOL_GPL(kvm_set_cr0);
 void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw)
 {
         kvm_set_cr0(vcpu, (vcpu->arch.cr0 & ~0x0ful) | (msw & 0x0f));
-        KVMTRACE_1D(LMSW, vcpu,
-                    (u32)((vcpu->arch.cr0 & ~0x0ful) | (msw & 0x0f)),
-                    handler);
 }
 EXPORT_SYMBOL_GPL(kvm_lmsw);
 
@@ -466,7 +492,7 @@ static u32 msrs_to_save[] = {
 #ifdef CONFIG_X86_64
         MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR,
 #endif
-        MSR_IA32_TIME_STAMP_COUNTER, MSR_KVM_SYSTEM_TIME, MSR_KVM_WALL_CLOCK,
+        MSR_IA32_TSC, MSR_KVM_SYSTEM_TIME, MSR_KVM_WALL_CLOCK,
         MSR_IA32_PERF_STATUS, MSR_IA32_CR_PAT, MSR_VM_HSAVE_PA
 };
 
@@ -644,8 +670,7 @@ static void kvm_write_guest_time(struct kvm_vcpu *v)
 
         /* Keep irq disabled to prevent changes to the clock */
         local_irq_save(flags);
-        kvm_get_msr(v, MSR_IA32_TIME_STAMP_COUNTER,
-                          &vcpu->hv_clock.tsc_timestamp);
+        kvm_get_msr(v, MSR_IA32_TSC, &vcpu->hv_clock.tsc_timestamp);
         ktime_get_ts(&ts);
         local_irq_restore(flags);
 
@@ -778,23 +803,60 @@ static int set_msr_mtrr(struct kvm_vcpu *vcpu, u32 msr, u64 data)
         return 0;
 }
 
+static int set_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 data)
+{
+        u64 mcg_cap = vcpu->arch.mcg_cap;
+        unsigned bank_num = mcg_cap & 0xff;
+
+        switch (msr) {
+        case MSR_IA32_MCG_STATUS:
+                vcpu->arch.mcg_status = data;
+                break;
+        case MSR_IA32_MCG_CTL:
+                if (!(mcg_cap & MCG_CTL_P))
+                        return 1;
+                if (data != 0 && data != ~(u64)0)
+                        return -1;
+                vcpu->arch.mcg_ctl = data;
+                break;
+        default:
+                if (msr >= MSR_IA32_MC0_CTL &&
+                    msr < MSR_IA32_MC0_CTL + 4 * bank_num) {
+                        u32 offset = msr - MSR_IA32_MC0_CTL;
+                        /* only 0 or all 1s can be written to IA32_MCi_CTL */
+                        if ((offset & 0x3) == 0 &&
+                            data != 0 && data != ~(u64)0)
+                                return -1;
+                        vcpu->arch.mce_banks[offset] = data;
+                        break;
+                }
+                return 1;
+        }
+        return 0;
+}
+
 int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data)
 {
         switch (msr) {
         case MSR_EFER:
                 set_efer(vcpu, data);
                 break;
-        case MSR_IA32_MC0_STATUS:
-                pr_unimpl(vcpu, "%s: MSR_IA32_MC0_STATUS 0x%llx, nop\n",
-                       __func__, data);
+        case MSR_K7_HWCR:
+                data &= ~(u64)0x40;     /* ignore flush filter disable */
+                if (data != 0) {
+                        pr_unimpl(vcpu, "unimplemented HWCR wrmsr: 0x%llx\n",
+                                data);
+                        return 1;
+                }
                 break;
-        case MSR_IA32_MCG_STATUS:
-                pr_unimpl(vcpu, "%s: MSR_IA32_MCG_STATUS 0x%llx, nop\n",
-                        __func__, data);
+        case MSR_FAM10H_MMIO_CONF_BASE:
+                if (data != 0) {
+                        pr_unimpl(vcpu, "unimplemented MMIO_CONF_BASE wrmsr: "
+                                "0x%llx\n", data);
+                        return 1;
+                }
                 break;
-        case MSR_IA32_MCG_CTL:
-                pr_unimpl(vcpu, "%s: MSR_IA32_MCG_CTL 0x%llx, nop\n",
-                        __func__, data);
+        case MSR_AMD64_NB_CFG:
                 break;
         case MSR_IA32_DEBUGCTLMSR:
                 if (!data) {
@@ -811,12 +873,15 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data)
         case MSR_IA32_UCODE_REV:
         case MSR_IA32_UCODE_WRITE:
         case MSR_VM_HSAVE_PA:
+        case MSR_AMD64_PATCH_LOADER:
                 break;
         case 0x200 ... 0x2ff:
                 return set_msr_mtrr(vcpu, msr, data);
         case MSR_IA32_APICBASE:
                 kvm_set_apic_base(vcpu, data);
                 break;
+        case APIC_BASE_MSR ... APIC_BASE_MSR + 0x3ff:
+                return kvm_x2apic_msr_write(vcpu, msr, data);
         case MSR_IA32_MISC_ENABLE:
                 vcpu->arch.ia32_misc_enable_msr = data;
                 break;
@@ -850,9 +915,50 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data)
                 kvm_request_guest_time_update(vcpu);
                 break;
         }
+        case MSR_IA32_MCG_CTL:
+        case MSR_IA32_MCG_STATUS:
+        case MSR_IA32_MC0_CTL ... MSR_IA32_MC0_CTL + 4 * KVM_MAX_MCE_BANKS - 1:
+                return set_msr_mce(vcpu, msr, data);
+
+        /* Performance counters are not protected by a CPUID bit,
+         * so we should check all of them in the generic path for the sake of
+         * cross vendor migration.
+         * Writing a zero into the event select MSRs disables them,
+         * which we perfectly emulate ;-). Any other value should be at least
+         * reported, some guests depend on them.
+         */
+        case MSR_P6_EVNTSEL0:
+        case MSR_P6_EVNTSEL1:
+        case MSR_K7_EVNTSEL0:
+        case MSR_K7_EVNTSEL1:
+        case MSR_K7_EVNTSEL2:
+        case MSR_K7_EVNTSEL3:
+                if (data != 0)
+                        pr_unimpl(vcpu, "unimplemented perfctr wrmsr: "
+                                "0x%x data 0x%llx\n", msr, data);
+                break;
+        /* at least RHEL 4 unconditionally writes to the perfctr registers,
+         * so we ignore writes to make it happy.
+         */
+        case MSR_P6_PERFCTR0:
+        case MSR_P6_PERFCTR1:
+        case MSR_K7_PERFCTR0:
+        case MSR_K7_PERFCTR1:
+        case MSR_K7_PERFCTR2:
+        case MSR_K7_PERFCTR3:
+                pr_unimpl(vcpu, "unimplemented perfctr wrmsr: "
+                        "0x%x data 0x%llx\n", msr, data);
+                break;
         default:
-                pr_unimpl(vcpu, "unhandled wrmsr: 0x%x data %llx\n", msr, data);
-                return 1;
+                if (!ignore_msrs) {
+                        pr_unimpl(vcpu, "unhandled wrmsr: 0x%x data %llx\n",
+                                msr, data);
+                        return 1;
+                } else {
+                        pr_unimpl(vcpu, "ignored wrmsr: 0x%x data %llx\n",
+                                msr, data);
+                        break;
+                }
         }
         return 0;
 }
@@ -905,26 +1011,47 @@ static int get_msr_mtrr(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
         return 0;
 }
 
-int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
+static int get_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
 {
         u64 data;
+        u64 mcg_cap = vcpu->arch.mcg_cap;
+        unsigned bank_num = mcg_cap & 0xff;
 
         switch (msr) {
-        case 0xc0010010: /* SYSCFG */
-        case 0xc0010015: /* HWCR */
-        case MSR_IA32_PLATFORM_ID:
         case MSR_IA32_P5_MC_ADDR:
         case MSR_IA32_P5_MC_TYPE:
-        case MSR_IA32_MC0_CTL:
-        case MSR_IA32_MCG_STATUS:
+                data = 0;
+                break;
         case MSR_IA32_MCG_CAP:
+                data = vcpu->arch.mcg_cap;
+                break;
         case MSR_IA32_MCG_CTL:
-        case MSR_IA32_MC0_MISC:
-        case MSR_IA32_MC0_MISC+4:
-        case MSR_IA32_MC0_MISC+8:
-        case MSR_IA32_MC0_MISC+12:
-        case MSR_IA32_MC0_MISC+16:
-        case MSR_IA32_MC0_MISC+20:
+                if (!(mcg_cap & MCG_CTL_P))
+                        return 1;
+                data = vcpu->arch.mcg_ctl;
+                break;
+        case MSR_IA32_MCG_STATUS:
+                data = vcpu->arch.mcg_status;
+                break;
+        default:
+                if (msr >= MSR_IA32_MC0_CTL &&
+                    msr < MSR_IA32_MC0_CTL + 4 * bank_num) {
+                        u32 offset = msr - MSR_IA32_MC0_CTL;
+                        data = vcpu->arch.mce_banks[offset];
+                        break;
+                }
+                return 1;
+        }
+        *pdata = data;
+        return 0;
+}
+
+int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
+{
+        u64 data;
+
+        switch (msr) {
+        case MSR_IA32_PLATFORM_ID:
         case MSR_IA32_UCODE_REV:
         case MSR_IA32_EBL_CR_POWERON:
         case MSR_IA32_DEBUGCTLMSR:
@@ -932,10 +1059,18 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
         case MSR_IA32_LASTBRANCHTOIP:
         case MSR_IA32_LASTINTFROMIP:
         case MSR_IA32_LASTINTTOIP:
+        case MSR_K8_SYSCFG:
+        case MSR_K7_HWCR:
         case MSR_VM_HSAVE_PA:
+        case MSR_P6_PERFCTR0:
+        case MSR_P6_PERFCTR1:
         case MSR_P6_EVNTSEL0:
         case MSR_P6_EVNTSEL1:
         case MSR_K7_EVNTSEL0:
+        case MSR_K7_PERFCTR0:
+        case MSR_K8_INT_PENDING_MSG:
+        case MSR_AMD64_NB_CFG:
+        case MSR_FAM10H_MMIO_CONF_BASE:
                 data = 0;
                 break;
         case MSR_MTRRcap:
@@ -949,6 +1084,9 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
         case MSR_IA32_APICBASE:
                 data = kvm_get_apic_base(vcpu);
                 break;
+        case APIC_BASE_MSR ... APIC_BASE_MSR + 0x3ff:
+                return kvm_x2apic_msr_read(vcpu, msr, pdata);
+                break;
         case MSR_IA32_MISC_ENABLE:
                 data = vcpu->arch.ia32_misc_enable_msr;
                 break;
@@ -967,9 +1105,22 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
         case MSR_KVM_SYSTEM_TIME:
                 data = vcpu->arch.time;
                 break;
+        case MSR_IA32_P5_MC_ADDR:
+        case MSR_IA32_P5_MC_TYPE:
+        case MSR_IA32_MCG_CAP:
+        case MSR_IA32_MCG_CTL:
+        case MSR_IA32_MCG_STATUS:
+        case MSR_IA32_MC0_CTL ... MSR_IA32_MC0_CTL + 4 * KVM_MAX_MCE_BANKS - 1:
+                return get_msr_mce(vcpu, msr, pdata);
         default:
-                pr_unimpl(vcpu, "unhandled rdmsr: 0x%x\n", msr);
-                return 1;
+                if (!ignore_msrs) {
+                        pr_unimpl(vcpu, "unhandled rdmsr: 0x%x\n", msr);
+                        return 1;
+                } else {
+                        pr_unimpl(vcpu, "ignored rdmsr: 0x%x\n", msr);
+                        data = 0;
+                }
+                break;
         }
         *pdata = data;
         return 0;
@@ -1068,6 +1219,11 @@ int kvm_dev_ioctl_check_extension(long ext)
         case KVM_CAP_REINJECT_CONTROL:
         case KVM_CAP_IRQ_INJECT_STATUS:
         case KVM_CAP_ASSIGN_DEV_IRQ:
+        case KVM_CAP_IRQFD:
+        case KVM_CAP_IOEVENTFD:
+        case KVM_CAP_PIT2:
+        case KVM_CAP_PIT_STATE2:
+        case KVM_CAP_SET_IDENTITY_MAP_ADDR:
                 r = 1;
                 break;
         case KVM_CAP_COALESCED_MMIO:
@@ -1088,6 +1244,9 @@ int kvm_dev_ioctl_check_extension(long ext)
         case KVM_CAP_IOMMU:
                 r = iommu_found();
                 break;
+        case KVM_CAP_MCE:
+                r = KVM_MAX_MCE_BANKS;
+                break;
         default:
                 r = 0;
                 break;
@@ -1147,6 +1306,16 @@ long kvm_arch_dev_ioctl(struct file *filp,
                 r = 0;
                 break;
         }
+        case KVM_X86_GET_MCE_CAP_SUPPORTED: {
+                u64 mce_cap;
+
+                mce_cap = KVM_MCE_CAP_SUPPORTED;
+                r = -EFAULT;
+                if (copy_to_user(argp, &mce_cap, sizeof mce_cap))
+                        goto out;
+                r = 0;
+                break;
+        }
         default:
                 r = -EINVAL;
         }
@@ -1227,6 +1396,7 @@ static int kvm_vcpu_ioctl_set_cpuid(struct kvm_vcpu *vcpu,
         vcpu->arch.cpuid_nent = cpuid->nent;
         cpuid_fix_nx_cap(vcpu);
         r = 0;
+        kvm_apic_set_version(vcpu);
 
 out_free:
         vfree(cpuid_entries);
@@ -1248,6 +1418,7 @@ static int kvm_vcpu_ioctl_set_cpuid2(struct kvm_vcpu *vcpu,
                            cpuid->nent * sizeof(struct kvm_cpuid_entry2)))
                 goto out;
         vcpu->arch.cpuid_nent = cpuid->nent;
+        kvm_apic_set_version(vcpu);
         return 0;
 
 out:
@@ -1290,6 +1461,7 @@ static void do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function,
                          u32 index, int *nent, int maxnent)
 {
         unsigned f_nx = is_efer_nx() ? F(NX) : 0;
+        unsigned f_gbpages = kvm_x86_ops->gb_page_enable() ? F(GBPAGES) : 0;
 #ifdef CONFIG_X86_64
         unsigned f_lm = F(LM);
 #else
@@ -1314,7 +1486,7 @@ static void do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function,
                 F(MTRR) | F(PGE) | F(MCA) | F(CMOV) |
                 F(PAT) | F(PSE36) | 0 /* Reserved */ |
                 f_nx | 0 /* Reserved */ | F(MMXEXT) | F(MMX) |
-                F(FXSR) | F(FXSR_OPT) | 0 /* GBPAGES */ | 0 /* RDTSCP */ |
+                F(FXSR) | F(FXSR_OPT) | f_gbpages | 0 /* RDTSCP */ |
                 0 /* Reserved */ | f_lm | F(3DNOWEXT) | F(3DNOW);
         /* cpuid 1.ecx */
         const u32 kvm_supported_word4_x86_features =
@@ -1323,7 +1495,7 @@ static void do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function,
                 0 /* TM2 */ | F(SSSE3) | 0 /* CNXT-ID */ | 0 /* Reserved */ |
                 0 /* Reserved */ | F(CX16) | 0 /* xTPR Update, PDCM */ |
                 0 /* Reserved, DCA */ | F(XMM4_1) |
-                F(XMM4_2) | 0 /* x2APIC */ | F(MOVBE) | F(POPCNT) |
+                F(XMM4_2) | F(X2APIC) | F(MOVBE) | F(POPCNT) |
                 0 /* Reserved, XSAVE, OSXSAVE */;
         /* cpuid 0x80000001.ecx */
         const u32 kvm_supported_word6_x86_features =
@@ -1344,6 +1516,9 @@ static void do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function,
         case 1:
                 entry->edx &= kvm_supported_word0_x86_features;
                 entry->ecx &= kvm_supported_word4_x86_features;
+                /* we support x2apic emulation even if host does not support
+                 * it since we emulate x2apic in software */
+                entry->ecx |= F(X2APIC);
                 break;
         /* function 2 entries are STATEFUL. That is, repeated cpuid commands
          * may return different values. This forces us to get_cpu() before
@@ -1435,6 +1610,10 @@ static int kvm_dev_ioctl_get_supported_cpuid(struct kvm_cpuid2 *cpuid,
         for (func = 0x80000001; func <= limit && nent < cpuid->nent; ++func)
                 do_cpuid_ent(&cpuid_entries[nent], func, 0,
                              &nent, cpuid->nent);
+        r = -E2BIG;
+        if (nent >= cpuid->nent)
+                goto out_free;
+
         r = -EFAULT;
         if (copy_to_user(entries, cpuid_entries,
                          nent * sizeof(struct kvm_cpuid_entry2)))
@@ -1464,6 +1643,7 @@ static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu,
         vcpu_load(vcpu);
         memcpy(vcpu->arch.apic->regs, s->regs, sizeof *s);
         kvm_apic_post_state_restore(vcpu);
+        update_cr8_intercept(vcpu);
         vcpu_put(vcpu);
 
         return 0;
@@ -1503,6 +1683,80 @@ static int vcpu_ioctl_tpr_access_reporting(struct kvm_vcpu *vcpu,
         return 0;
 }
 
+static int kvm_vcpu_ioctl_x86_setup_mce(struct kvm_vcpu *vcpu,
+                                        u64 mcg_cap)
+{
+        int r;
+        unsigned bank_num = mcg_cap & 0xff, bank;
+
+        r = -EINVAL;
+        if (!bank_num)
+                goto out;
+        if (mcg_cap & ~(KVM_MCE_CAP_SUPPORTED | 0xff | 0xff0000))
+                goto out;
+        r = 0;
+        vcpu->arch.mcg_cap = mcg_cap;
+        /* Init IA32_MCG_CTL to all 1s */
+        if (mcg_cap & MCG_CTL_P)
+                vcpu->arch.mcg_ctl = ~(u64)0;
+        /* Init IA32_MCi_CTL to all 1s */
+        for (bank = 0; bank < bank_num; bank++)
+                vcpu->arch.mce_banks[bank*4] = ~(u64)0;
+out:
+        return r;
+}
+
+static int kvm_vcpu_ioctl_x86_set_mce(struct kvm_vcpu *vcpu,
+                                      struct kvm_x86_mce *mce)
+{
+        u64 mcg_cap = vcpu->arch.mcg_cap;
+        unsigned bank_num = mcg_cap & 0xff;
+        u64 *banks = vcpu->arch.mce_banks;
+
+        if (mce->bank >= bank_num || !(mce->status & MCI_STATUS_VAL))
+                return -EINVAL;
+        /*
+         * if IA32_MCG_CTL is not all 1s, the uncorrected error
+         * reporting is disabled
+         */
+        if ((mce->status & MCI_STATUS_UC) && (mcg_cap & MCG_CTL_P) &&
+            vcpu->arch.mcg_ctl != ~(u64)0)
+                return 0;
+        banks += 4 * mce->bank;
+        /*
+         * if IA32_MCi_CTL is not all 1s, the uncorrected error
+         * reporting is disabled for the bank
+         */
+        if ((mce->status & MCI_STATUS_UC) && banks[0] != ~(u64)0)
+                return 0;
+        if (mce->status & MCI_STATUS_UC) {
+                if ((vcpu->arch.mcg_status & MCG_STATUS_MCIP) ||
+                    !(vcpu->arch.cr4 & X86_CR4_MCE)) {
+                        printk(KERN_DEBUG "kvm: set_mce: "
+                               "injects mce exception while "
+                               "previous one is in progress!\n");
+                        set_bit(KVM_REQ_TRIPLE_FAULT, &vcpu->requests);
+                        return 0;
+                }
+                if (banks[1] & MCI_STATUS_VAL)
+                        mce->status |= MCI_STATUS_OVER;
+                banks[2] = mce->addr;
+                banks[3] = mce->misc;
+                vcpu->arch.mcg_status = mce->mcg_status;
+                banks[1] = mce->status;
+                kvm_queue_exception(vcpu, MC_VECTOR);
+        } else if (!(banks[1] & MCI_STATUS_VAL)
+                   || !(banks[1] & MCI_STATUS_UC)) {
+                if (banks[1] & MCI_STATUS_VAL)
+                        mce->status |= MCI_STATUS_OVER;
+                banks[2] = mce->addr;
+                banks[3] = mce->misc;
+                banks[1] = mce->status;
+        } else
+                banks[1] |= MCI_STATUS_OVER;
+        return 0;
+}
+
 long kvm_arch_vcpu_ioctl(struct file *filp,
                          unsigned int ioctl, unsigned long arg)
 {
@@ -1636,6 +1890,24 @@ long kvm_arch_vcpu_ioctl(struct file *filp,
                 kvm_lapic_set_vapic_addr(vcpu, va.vapic_addr);
                 break;
         }
+        case KVM_X86_SETUP_MCE: {
+                u64 mcg_cap;
+
+                r = -EFAULT;
+                if (copy_from_user(&mcg_cap, argp, sizeof mcg_cap))
+                        goto out;
+                r = kvm_vcpu_ioctl_x86_setup_mce(vcpu, mcg_cap);
+                break;
+        }
+        case KVM_X86_SET_MCE: {
+                struct kvm_x86_mce mce;
+
+                r = -EFAULT;
+                if (copy_from_user(&mce, argp, sizeof mce))
+                        goto out;
+                r = kvm_vcpu_ioctl_x86_set_mce(vcpu, &mce);
+                break;
+        }
         default:
                 r = -EINVAL;
         }
@@ -1654,6 +1926,13 @@ static int kvm_vm_ioctl_set_tss_addr(struct kvm *kvm, unsigned long addr)
         return ret;
 }
 
+static int kvm_vm_ioctl_set_identity_map_addr(struct kvm *kvm,
+                                              u64 ident_addr)
+{
+        kvm->arch.ept_identity_map_addr = ident_addr;
+        return 0;
+}
+
 static int kvm_vm_ioctl_set_nr_mmu_pages(struct kvm *kvm,
                                           u32 kvm_nr_mmu_pages)
 {
@@ -1775,19 +2054,25 @@ static int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
         r = 0;
         switch (chip->chip_id) {
         case KVM_IRQCHIP_PIC_MASTER:
+                spin_lock(&pic_irqchip(kvm)->lock);
                 memcpy(&pic_irqchip(kvm)->pics[0],
                         &chip->chip.pic,
                         sizeof(struct kvm_pic_state));
+                spin_unlock(&pic_irqchip(kvm)->lock);
                 break;
         case KVM_IRQCHIP_PIC_SLAVE:
+                spin_lock(&pic_irqchip(kvm)->lock);
                 memcpy(&pic_irqchip(kvm)->pics[1],
                         &chip->chip.pic,
                         sizeof(struct kvm_pic_state));
+                spin_unlock(&pic_irqchip(kvm)->lock);
                 break;
         case KVM_IRQCHIP_IOAPIC:
+                mutex_lock(&kvm->irq_lock);
                 memcpy(ioapic_irqchip(kvm),
                         &chip->chip.ioapic,
                         sizeof(struct kvm_ioapic_state));
+                mutex_unlock(&kvm->irq_lock);
                 break;
         default:
                 r = -EINVAL;
@@ -1801,7 +2086,9 @@ static int kvm_vm_ioctl_get_pit(struct kvm *kvm, struct kvm_pit_state *ps)
 {
         int r = 0;
 
+        mutex_lock(&kvm->arch.vpit->pit_state.lock);
         memcpy(ps, &kvm->arch.vpit->pit_state, sizeof(struct kvm_pit_state));
+        mutex_unlock(&kvm->arch.vpit->pit_state.lock);
         return r;
 }
 
@@ -1809,8 +2096,39 @@ static int kvm_vm_ioctl_set_pit(struct kvm *kvm, struct kvm_pit_state *ps)
 {
         int r = 0;
 
+        mutex_lock(&kvm->arch.vpit->pit_state.lock);
         memcpy(&kvm->arch.vpit->pit_state, ps, sizeof(struct kvm_pit_state));
-        kvm_pit_load_count(kvm, 0, ps->channels[0].count);
+        kvm_pit_load_count(kvm, 0, ps->channels[0].count, 0);
+        mutex_unlock(&kvm->arch.vpit->pit_state.lock);
+        return r;
+}
+
+static int kvm_vm_ioctl_get_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps)
+{
+        int r = 0;
+
+        mutex_lock(&kvm->arch.vpit->pit_state.lock);
+        memcpy(ps->channels, &kvm->arch.vpit->pit_state.channels,
+                sizeof(ps->channels));
+        ps->flags = kvm->arch.vpit->pit_state.flags;
+        mutex_unlock(&kvm->arch.vpit->pit_state.lock);
+        return r;
+}
+
+static int kvm_vm_ioctl_set_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps)
+{
+        int r = 0, start = 0;
+        u32 prev_legacy, cur_legacy;
+        mutex_lock(&kvm->arch.vpit->pit_state.lock);
+        prev_legacy = kvm->arch.vpit->pit_state.flags & KVM_PIT_FLAGS_HPET_LEGACY;
+        cur_legacy = ps->flags & KVM_PIT_FLAGS_HPET_LEGACY;
+        if (!prev_legacy && cur_legacy)
+                start = 1;
+        memcpy(&kvm->arch.vpit->pit_state.channels, &ps->channels,
+               sizeof(kvm->arch.vpit->pit_state.channels));
+        kvm->arch.vpit->pit_state.flags = ps->flags;
+        kvm_pit_load_count(kvm, 0, kvm->arch.vpit->pit_state.channels[0].count, start);
+        mutex_unlock(&kvm->arch.vpit->pit_state.lock);
         return r;
 }
 
@@ -1819,7 +2137,9 @@ static int kvm_vm_ioctl_reinject(struct kvm *kvm,
 {
         if (!kvm->arch.vpit)
                 return -ENXIO;
+        mutex_lock(&kvm->arch.vpit->pit_state.lock);
         kvm->arch.vpit->pit_state.pit_timer.reinject = control->pit_reinject;
+        mutex_unlock(&kvm->arch.vpit->pit_state.lock);
         return 0;
 }
 
@@ -1845,7 +2165,6 @@ int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
                 spin_lock(&kvm->mmu_lock);
                 kvm_mmu_slot_remove_write_access(kvm, log->slot);
                 spin_unlock(&kvm->mmu_lock);
-                kvm_flush_remote_tlbs(kvm);
                 memslot = &kvm->memslots[log->slot];
                 n = ALIGN(memslot->npages, BITS_PER_LONG) / 8;
                 memset(memslot->dirty_bitmap, 0, n);
@@ -1869,7 +2188,9 @@ long kvm_arch_vm_ioctl(struct file *filp,
          */
         union {
                 struct kvm_pit_state ps;
+                struct kvm_pit_state2 ps2;
                 struct kvm_memory_alias alias;
+                struct kvm_pit_config pit_config;
         } u;
 
         switch (ioctl) {
@@ -1878,6 +2199,17 @@ long kvm_arch_vm_ioctl(struct file *filp,
                 if (r < 0)
                         goto out;
                 break;
+        case KVM_SET_IDENTITY_MAP_ADDR: {
+                u64 ident_addr;
+
+                r = -EFAULT;
+                if (copy_from_user(&ident_addr, argp, sizeof ident_addr))
+                        goto out;
+                r = kvm_vm_ioctl_set_identity_map_addr(kvm, ident_addr);
+                if (r < 0)
+                        goto out;
+                break;
+        }
         case KVM_SET_MEMORY_REGION: {
                 struct kvm_memory_region kvm_mem;
                 struct kvm_userspace_memory_region kvm_userspace_mem;
@@ -1930,16 +2262,24 @@ long kvm_arch_vm_ioctl(struct file *filp,
                 }
                 break;
         case KVM_CREATE_PIT:
-                mutex_lock(&kvm->lock);
+                u.pit_config.flags = KVM_PIT_SPEAKER_DUMMY;
+                goto create_pit;
+        case KVM_CREATE_PIT2:
+                r = -EFAULT;
+                if (copy_from_user(&u.pit_config, argp,
+                                   sizeof(struct kvm_pit_config)))
+                        goto out;
+        create_pit:
+                down_write(&kvm->slots_lock);
                 r = -EEXIST;
                 if (kvm->arch.vpit)
                         goto create_pit_unlock;
                 r = -ENOMEM;
-                kvm->arch.vpit = kvm_create_pit(kvm);
+                kvm->arch.vpit = kvm_create_pit(kvm, u.pit_config.flags);
                 if (kvm->arch.vpit)
                         r = 0;
         create_pit_unlock:
-                mutex_unlock(&kvm->lock);
+                up_write(&kvm->slots_lock);
                 break;
         case KVM_IRQ_LINE_STATUS:
         case KVM_IRQ_LINE: {
@@ -1950,10 +2290,10 @@ long kvm_arch_vm_ioctl(struct file *filp,
                         goto out;
                 if (irqchip_in_kernel(kvm)) {
                         __s32 status;
-                        mutex_lock(&kvm->lock);
+                        mutex_lock(&kvm->irq_lock);
                         status = kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID,
                                         irq_event.irq, irq_event.level);
-                        mutex_unlock(&kvm->lock);
+                        mutex_unlock(&kvm->irq_lock);
                         if (ioctl == KVM_IRQ_LINE_STATUS) {
                                 irq_event.status = status;
                                 if (copy_to_user(argp, &irq_event,
@@ -2042,6 +2382,32 @@ long kvm_arch_vm_ioctl(struct file *filp,
                 r = 0;
                 break;
         }
+        case KVM_GET_PIT2: {
+                r = -ENXIO;
+                if (!kvm->arch.vpit)
+                        goto out;
+                r = kvm_vm_ioctl_get_pit2(kvm, &u.ps2);
+                if (r)
+                        goto out;
+                r = -EFAULT;
+                if (copy_to_user(argp, &u.ps2, sizeof(u.ps2)))
+                        goto out;
+                r = 0;
+                break;
+        }
+        case KVM_SET_PIT2: {
+                r = -EFAULT;
+                if (copy_from_user(&u.ps2, argp, sizeof(u.ps2)))
+                        goto out;
+                r = -ENXIO;
+                if (!kvm->arch.vpit)
+                        goto out;
+                r = kvm_vm_ioctl_set_pit2(kvm, &u.ps2);
+                if (r)
+                        goto out;
+                r = 0;
+                break;
+        }
         case KVM_REINJECT_CONTROL: {
                 struct kvm_reinject_control control;
                 r =  -EFAULT;
@@ -2075,35 +2441,23 @@ static void kvm_init_msr_list(void)
         num_msrs_to_save = j;
 }
 
-/*
- * Only apic need an MMIO device hook, so shortcut now..
- */
-static struct kvm_io_device *vcpu_find_pervcpu_dev(struct kvm_vcpu *vcpu,
-                                                gpa_t addr, int len,
-                                                int is_write)
+static int vcpu_mmio_write(struct kvm_vcpu *vcpu, gpa_t addr, int len,
+                           const void *v)
 {
-        struct kvm_io_device *dev;
+        if (vcpu->arch.apic &&
+            !kvm_iodevice_write(&vcpu->arch.apic->dev, addr, len, v))
+                return 0;
 
-        if (vcpu->arch.apic) {
-                dev = &vcpu->arch.apic->dev;
-                if (dev->in_range(dev, addr, len, is_write))
-                        return dev;
-        }
-        return NULL;
+        return kvm_io_bus_write(&vcpu->kvm->mmio_bus, addr, len, v);
 }
 
-
-static struct kvm_io_device *vcpu_find_mmio_dev(struct kvm_vcpu *vcpu,
-                                                gpa_t addr, int len,
-                                                int is_write)
+static int vcpu_mmio_read(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *v)
 {
-        struct kvm_io_device *dev;
+        if (vcpu->arch.apic &&
+            !kvm_iodevice_read(&vcpu->arch.apic->dev, addr, len, v))
+                return 0;
 
-        dev = vcpu_find_pervcpu_dev(vcpu, addr, len, is_write);
-        if (dev == NULL)
-                dev = kvm_io_bus_find_dev(&vcpu->kvm->mmio_bus, addr, len,
-                                          is_write);
-        return dev;
+        return kvm_io_bus_read(&vcpu->kvm->mmio_bus, addr, len, v);
 }
 
 static int kvm_read_guest_virt(gva_t addr, void *val, unsigned int bytes,
@@ -2172,11 +2526,12 @@ static int emulator_read_emulated(unsigned long addr,
                                   unsigned int bytes,
                                   struct kvm_vcpu *vcpu)
 {
-        struct kvm_io_device *mmio_dev;
         gpa_t                 gpa;
 
         if (vcpu->mmio_read_completed) {
                 memcpy(val, vcpu->mmio_data, bytes);
+                trace_kvm_mmio(KVM_TRACE_MMIO_READ, bytes,
+                               vcpu->mmio_phys_addr, *(u64 *)val);
                 vcpu->mmio_read_completed = 0;
                 return X86EMUL_CONTINUE;
         }
@@ -2197,14 +2552,12 @@ mmio:
         /*
          * Is this MMIO handled locally?
          */
-        mutex_lock(&vcpu->kvm->lock);
-        mmio_dev = vcpu_find_mmio_dev(vcpu, gpa, bytes, 0);
-        if (mmio_dev) {
-                kvm_iodevice_read(mmio_dev, gpa, bytes, val);
-                mutex_unlock(&vcpu->kvm->lock);
+        if (!vcpu_mmio_read(vcpu, gpa, bytes, val)) {
+                trace_kvm_mmio(KVM_TRACE_MMIO_READ, bytes, gpa, *(u64 *)val);
                 return X86EMUL_CONTINUE;
         }
-        mutex_unlock(&vcpu->kvm->lock);
+
+        trace_kvm_mmio(KVM_TRACE_MMIO_READ_UNSATISFIED, bytes, gpa, 0);
 
         vcpu->mmio_needed = 1;
         vcpu->mmio_phys_addr = gpa;
@@ -2231,7 +2584,6 @@ static int emulator_write_emulated_onepage(unsigned long addr,
                                            unsigned int bytes,
                                            struct kvm_vcpu *vcpu)
 {
-        struct kvm_io_device *mmio_dev;
         gpa_t                 gpa;
 
         gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, addr);
@@ -2249,17 +2601,12 @@ static int emulator_write_emulated_onepage(unsigned long addr,
                 return X86EMUL_CONTINUE;
 
 mmio:
+        trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, bytes, gpa, *(u64 *)val);
         /*
          * Is this MMIO handled locally?
          */
-        mutex_lock(&vcpu->kvm->lock);
-        mmio_dev = vcpu_find_mmio_dev(vcpu, gpa, bytes, 1);
-        if (mmio_dev) {
-                kvm_iodevice_write(mmio_dev, gpa, bytes, val);
-                mutex_unlock(&vcpu->kvm->lock);
+        if (!vcpu_mmio_write(vcpu, gpa, bytes, val))
                 return X86EMUL_CONTINUE;
-        }
-        mutex_unlock(&vcpu->kvm->lock);
 
         vcpu->mmio_needed = 1;
         vcpu->mmio_phys_addr = gpa;
@@ -2297,12 +2644,7 @@ static int emulator_cmpxchg_emulated(unsigned long addr,
                                      unsigned int bytes,
                                      struct kvm_vcpu *vcpu)
 {
-        static int reported;
-
-        if (!reported) {
-                reported = 1;
-                printk(KERN_WARNING "kvm: emulating exchange as write\n");
-        }
+        printk_once(KERN_WARNING "kvm: emulating exchange as write\n");
 #ifndef CONFIG_X86_64
         /* guests cmpxchg8b have to be emulated atomically */
         if (bytes == 8) {
@@ -2348,7 +2690,6 @@ int emulate_invlpg(struct kvm_vcpu *vcpu, gva_t address)
 
 int emulate_clts(struct kvm_vcpu *vcpu)
 {
-        KVMTRACE_0D(CLTS, vcpu, handler);
         kvm_x86_ops->set_cr0(vcpu, vcpu->arch.cr0 & ~X86_CR0_TS);
         return X86EMUL_CONTINUE;
 }
@@ -2425,7 +2766,7 @@ int emulate_instruction(struct kvm_vcpu *vcpu,
         kvm_clear_exception_queue(vcpu);
         vcpu->arch.mmio_fault_cr2 = cr2;
         /*
-         * TODO: fix x86_emulate.c to use guest_read/write_register
+         * TODO: fix emulate.c to use guest_read/write_register
          * instead of direct ->regs accesses, can save hundred cycles
          * on Intel for instructions that don't read/change RSP, for
          * for example.
@@ -2449,14 +2790,33 @@ int emulate_instruction(struct kvm_vcpu *vcpu,
 
                 r = x86_decode_insn(&vcpu->arch.emulate_ctxt, &emulate_ops);
 
-                /* Reject the instructions other than VMCALL/VMMCALL when
-                 * try to emulate invalid opcode */
+                /* Only allow emulation of specific instructions on #UD
+                 * (namely VMMCALL, sysenter, sysexit, syscall)*/
                 c = &vcpu->arch.emulate_ctxt.decode;
-                if ((emulation_type & EMULTYPE_TRAP_UD) &&
-                    (!(c->twobyte && c->b == 0x01 &&
-                      (c->modrm_reg == 0 || c->modrm_reg == 3) &&
-                       c->modrm_mod == 3 && c->modrm_rm == 1)))
-                        return EMULATE_FAIL;
+                if (emulation_type & EMULTYPE_TRAP_UD) {
+                        if (!c->twobyte)
+                                return EMULATE_FAIL;
+                        switch (c->b) {
+                        case 0x01: /* VMMCALL */
+                                if (c->modrm_mod != 3 || c->modrm_rm != 1)
+                                        return EMULATE_FAIL;
+                                break;
+                        case 0x34: /* sysenter */
+                        case 0x35: /* sysexit */
+                                if (c->modrm_mod != 0 || c->modrm_rm != 0)
+                                        return EMULATE_FAIL;
+                                break;
+                        case 0x05: /* syscall */
+                                if (c->modrm_mod != 0 || c->modrm_rm != 0)
+                                        return EMULATE_FAIL;
+                                break;
+                        default:
+                                return EMULATE_FAIL;
+                        }
+
+                        if (!(c->modrm_reg == 0 || c->modrm_reg == 3))
+                                return EMULATE_FAIL;
+                }
 
                 ++vcpu->stat.insn_emulation;
                 if (r)  {
@@ -2576,52 +2936,40 @@ int complete_pio(struct kvm_vcpu *vcpu)
         return 0;
 }
 
-static void kernel_pio(struct kvm_io_device *pio_dev,
-                       struct kvm_vcpu *vcpu,
-                       void *pd)
+static int kernel_pio(struct kvm_vcpu *vcpu, void *pd)
 {
         /* TODO: String I/O for in kernel device */
+        int r;
 
-        mutex_lock(&vcpu->kvm->lock);
         if (vcpu->arch.pio.in)
-                kvm_iodevice_read(pio_dev, vcpu->arch.pio.port,
-                                  vcpu->arch.pio.size,
-                                  pd);
+                r = kvm_io_bus_read(&vcpu->kvm->pio_bus, vcpu->arch.pio.port,
+                                    vcpu->arch.pio.size, pd);
         else
-                kvm_iodevice_write(pio_dev, vcpu->arch.pio.port,
-                                   vcpu->arch.pio.size,
-                                   pd);
-        mutex_unlock(&vcpu->kvm->lock);
+                r = kvm_io_bus_write(&vcpu->kvm->pio_bus, vcpu->arch.pio.port,
+                                     vcpu->arch.pio.size, pd);
+        return r;
 }
 
-static void pio_string_write(struct kvm_io_device *pio_dev,
-                             struct kvm_vcpu *vcpu)
+static int pio_string_write(struct kvm_vcpu *vcpu)
 {
         struct kvm_pio_request *io = &vcpu->arch.pio;
         void *pd = vcpu->arch.pio_data;
-        int i;
+        int i, r = 0;
 
-        mutex_lock(&vcpu->kvm->lock);
         for (i = 0; i < io->cur_count; i++) {
-                kvm_iodevice_write(pio_dev, io->port,
-                                   io->size,
-                                   pd);
+                if (kvm_io_bus_write(&vcpu->kvm->pio_bus,
+                                     io->port, io->size, pd)) {
+                        r = -EOPNOTSUPP;
+                        break;
+                }
                 pd += io->size;
         }
-        mutex_unlock(&vcpu->kvm->lock);
-}
-
-static struct kvm_io_device *vcpu_find_pio_dev(struct kvm_vcpu *vcpu,
-                                               gpa_t addr, int len,
-                                               int is_write)
-{
-        return kvm_io_bus_find_dev(&vcpu->kvm->pio_bus, addr, len, is_write);
+        return r;
 }
 
 int kvm_emulate_pio(struct kvm_vcpu *vcpu, struct kvm_run *run, int in,
                   int size, unsigned port)
 {
-        struct kvm_io_device *pio_dev;
         unsigned long val;
 
         vcpu->run->exit_reason = KVM_EXIT_IO;
@@ -2635,19 +2983,13 @@ int kvm_emulate_pio(struct kvm_vcpu *vcpu, struct kvm_run *run, int in,
         vcpu->arch.pio.down = 0;
         vcpu->arch.pio.rep = 0;
 
-        if (vcpu->run->io.direction == KVM_EXIT_IO_IN)
-                KVMTRACE_2D(IO_READ, vcpu, vcpu->run->io.port, (u32)size,
-                            handler);
-        else
-                KVMTRACE_2D(IO_WRITE, vcpu, vcpu->run->io.port, (u32)size,
-                            handler);
+        trace_kvm_pio(vcpu->run->io.direction == KVM_EXIT_IO_OUT, port,
+                      size, 1);
 
         val = kvm_register_read(vcpu, VCPU_REGS_RAX);
         memcpy(vcpu->arch.pio_data, &val, 4);
 
-        pio_dev = vcpu_find_pio_dev(vcpu, port, size, !in);
-        if (pio_dev) {
-                kernel_pio(pio_dev, vcpu, vcpu->arch.pio_data);
+        if (!kernel_pio(vcpu, vcpu->arch.pio_data)) {
                 complete_pio(vcpu);
                 return 1;
         }
@@ -2661,7 +3003,6 @@ int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, struct kvm_run *run, int in,
 {
         unsigned now, in_page;
         int ret = 0;
-        struct kvm_io_device *pio_dev;
 
         vcpu->run->exit_reason = KVM_EXIT_IO;
         vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;
@@ -2674,12 +3015,8 @@ int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, struct kvm_run *run, int in,
         vcpu->arch.pio.down = down;
         vcpu->arch.pio.rep = rep;
 
-        if (vcpu->run->io.direction == KVM_EXIT_IO_IN)
-                KVMTRACE_2D(IO_READ, vcpu, vcpu->run->io.port, (u32)size,
-                            handler);
-        else
-                KVMTRACE_2D(IO_WRITE, vcpu, vcpu->run->io.port, (u32)size,
-                            handler);
+        trace_kvm_pio(vcpu->run->io.direction == KVM_EXIT_IO_OUT, port,
+                      size, count);
 
         if (!count) {
                 kvm_x86_ops->skip_emulated_instruction(vcpu);
@@ -2709,9 +3046,6 @@ int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, struct kvm_run *run, int in,
 
         vcpu->arch.pio.guest_gva = address;
 
-        pio_dev = vcpu_find_pio_dev(vcpu, port,
-                                    vcpu->arch.pio.cur_count,
-                                    !vcpu->arch.pio.in);
         if (!vcpu->arch.pio.in) {
                 /* string PIO write */
                 ret = pio_copy_data(vcpu);
@@ -2719,16 +3053,13 @@ int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, struct kvm_run *run, int in,
                         kvm_inject_gp(vcpu, 0);
                         return 1;
                 }
-                if (ret == 0 && pio_dev) {
-                        pio_string_write(pio_dev, vcpu);
+                if (ret == 0 && !pio_string_write(vcpu)) {
                         complete_pio(vcpu);
                         if (vcpu->arch.pio.count == 0)
                                 ret = 1;
                 }
-        } else if (pio_dev)
-                pr_unimpl(vcpu, "no string pio read support yet, "
-                       "port %x size %d count %ld\n",
-                        port, size, count);
+        }
+        /* no string PIO read support yet */
 
         return ret;
 }
@@ -2761,10 +3092,7 @@ static int kvmclock_cpufreq_notifier(struct notifier_block *nb, unsigned long va
 
         spin_lock(&kvm_lock);
         list_for_each_entry(kvm, &vm_list, vm_list) {
-                for (i = 0; i < KVM_MAX_VCPUS; ++i) {
-                        vcpu = kvm->vcpus[i];
-                        if (!vcpu)
-                                continue;
+                kvm_for_each_vcpu(i, vcpu, kvm) {
                         if (vcpu->cpu != freq->cpu)
                                 continue;
                         if (!kvm_request_guest_time_update(vcpu))
@@ -2857,7 +3185,6 @@ void kvm_arch_exit(void)
 int kvm_emulate_halt(struct kvm_vcpu *vcpu)
 {
         ++vcpu->stat.halt_exits;
-        KVMTRACE_0D(HLT, vcpu, handler);
         if (irqchip_in_kernel(vcpu->kvm)) {
                 vcpu->arch.mp_state = KVM_MP_STATE_HALTED;
                 return 1;
@@ -2888,7 +3215,7 @@ int kvm_emulate_hypercall(struct kvm_vcpu *vcpu)
         a2 = kvm_register_read(vcpu, VCPU_REGS_RDX);
         a3 = kvm_register_read(vcpu, VCPU_REGS_RSI);
 
-        KVMTRACE_1D(VMMCALL, vcpu, (u32)nr, handler);
+        trace_kvm_hypercall(nr, a0, a1, a2, a3);
 
         if (!is_long_mode(vcpu)) {
                 nr &= 0xFFFFFFFF;
@@ -2898,6 +3225,11 @@ int kvm_emulate_hypercall(struct kvm_vcpu *vcpu)
                 a3 &= 0xFFFFFFFF;
         }
 
+        if (kvm_x86_ops->get_cpl(vcpu) != 0) {
+                ret = -KVM_EPERM;
+                goto out;
+        }
+
         switch (nr) {
         case KVM_HC_VAPIC_POLL_IRQ:
                 ret = 0;
@@ -2909,6 +3241,7 @@ int kvm_emulate_hypercall(struct kvm_vcpu *vcpu)
                 ret = -KVM_ENOSYS;
                 break;
         }
+out:
         kvm_register_write(vcpu, VCPU_REGS_RAX, ret);
         ++vcpu->stat.hypercalls;
         return r;
@@ -2988,8 +3321,6 @@ unsigned long realmode_get_cr(struct kvm_vcpu *vcpu, int cr)
                 vcpu_printf(vcpu, "%s: unexpected cr %u\n", __func__, cr);
                 return 0;
         }
-        KVMTRACE_3D(CR_READ, vcpu, (u32)cr, (u32)value,
-                    (u32)((u64)value >> 32), handler);
 
         return value;
 }
@@ -2997,9 +3328,6 @@ unsigned long realmode_get_cr(struct kvm_vcpu *vcpu, int cr)
 void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long val,
                      unsigned long *rflags)
 {
-        KVMTRACE_3D(CR_WRITE, vcpu, (u32)cr, (u32)val,
-                    (u32)((u64)val >> 32), handler);
-
         switch (cr) {
         case 0:
                 kvm_set_cr0(vcpu, mk_cr_64(vcpu->arch.cr0, val));
@@ -3109,11 +3437,11 @@ void kvm_emulate_cpuid(struct kvm_vcpu *vcpu)
                 kvm_register_write(vcpu, VCPU_REGS_RDX, best->edx);
         }
         kvm_x86_ops->skip_emulated_instruction(vcpu);
-        KVMTRACE_5D(CPUID, vcpu, function,
-                    (u32)kvm_register_read(vcpu, VCPU_REGS_RAX),
-                    (u32)kvm_register_read(vcpu, VCPU_REGS_RBX),
-                    (u32)kvm_register_read(vcpu, VCPU_REGS_RCX),
-                    (u32)kvm_register_read(vcpu, VCPU_REGS_RDX), handler);
+        trace_kvm_cpuid(function,
+                        kvm_register_read(vcpu, VCPU_REGS_RAX),
+                        kvm_register_read(vcpu, VCPU_REGS_RBX),
+                        kvm_register_read(vcpu, VCPU_REGS_RCX),
+                        kvm_register_read(vcpu, VCPU_REGS_RDX));
 }
 EXPORT_SYMBOL_GPL(kvm_emulate_cpuid);
 
@@ -3179,6 +3507,9 @@ static void update_cr8_intercept(struct kvm_vcpu *vcpu)
         if (!kvm_x86_ops->update_cr8_intercept)
                 return;
 
+        if (!vcpu->arch.apic)
+                return;
+
         if (!vcpu->arch.apic->vapic_addr)
                 max_irr = kvm_lapic_find_highest_irr(vcpu);
         else
@@ -3192,12 +3523,16 @@ static void update_cr8_intercept(struct kvm_vcpu *vcpu)
         kvm_x86_ops->update_cr8_intercept(vcpu, tpr, max_irr);
 }
 
-static void inject_pending_irq(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
+static void inject_pending_event(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 {
-        if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
-                kvm_x86_ops->set_interrupt_shadow(vcpu, 0);
-
         /* try to reinject previous events if any */
+        if (vcpu->arch.exception.pending) {
+                kvm_x86_ops->queue_exception(vcpu, vcpu->arch.exception.nr,
+                                          vcpu->arch.exception.has_error_code,
+                                          vcpu->arch.exception.error_code);
+                return;
+        }
+
         if (vcpu->arch.nmi_injected) {
                 kvm_x86_ops->set_nmi(vcpu);
                 return;
@@ -3271,16 +3606,14 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
         smp_mb__after_clear_bit();
 
         if (vcpu->requests || need_resched() || signal_pending(current)) {
+                set_bit(KVM_REQ_KICK, &vcpu->requests);
                 local_irq_enable();
                 preempt_enable();
                 r = 1;
                 goto out;
         }
 
-        if (vcpu->arch.exception.pending)
-                __queue_exception(vcpu);
-        else
-                inject_pending_irq(vcpu, kvm_run);
+        inject_pending_event(vcpu, kvm_run);
 
         /* enable NMI/IRQ window open exits if needed */
         if (vcpu->arch.nmi_pending)
@@ -3297,14 +3630,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 
         kvm_guest_enter();
 
-        get_debugreg(vcpu->arch.host_dr6, 6);
-        get_debugreg(vcpu->arch.host_dr7, 7);
         if (unlikely(vcpu->arch.switch_db_regs)) {
-                get_debugreg(vcpu->arch.host_db[0], 0);
-                get_debugreg(vcpu->arch.host_db[1], 1);
-                get_debugreg(vcpu->arch.host_db[2], 2);
-                get_debugreg(vcpu->arch.host_db[3], 3);
-
                 set_debugreg(0, 7);
                 set_debugreg(vcpu->arch.eff_db[0], 0);
                 set_debugreg(vcpu->arch.eff_db[1], 1);
@@ -3312,18 +3638,17 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
                 set_debugreg(vcpu->arch.eff_db[3], 3);
         }
 
-        KVMTRACE_0D(VMENTRY, vcpu, entryexit);
+        trace_kvm_entry(vcpu->vcpu_id);
         kvm_x86_ops->run(vcpu, kvm_run);
 
-        if (unlikely(vcpu->arch.switch_db_regs)) {
-                set_debugreg(0, 7);
-                set_debugreg(vcpu->arch.host_db[0], 0);
-                set_debugreg(vcpu->arch.host_db[1], 1);
-                set_debugreg(vcpu->arch.host_db[2], 2);
-                set_debugreg(vcpu->arch.host_db[3], 3);
+        if (unlikely(vcpu->arch.switch_db_regs || test_thread_flag(TIF_DEBUG))) {
+                set_debugreg(current->thread.debugreg0, 0);
+                set_debugreg(current->thread.debugreg1, 1);
+                set_debugreg(current->thread.debugreg2, 2);
+                set_debugreg(current->thread.debugreg3, 3);
+                set_debugreg(current->thread.debugreg6, 6);
+                set_debugreg(current->thread.debugreg7, 7);
         }
-        set_debugreg(vcpu->arch.host_dr6, 6);
-        set_debugreg(vcpu->arch.host_dr7, 7);
 
         set_bit(KVM_REQ_KICK, &vcpu->requests);
         local_irq_enable();
@@ -3653,11 +3978,8 @@ static void kvm_set_segment(struct kvm_vcpu *vcpu,
 static void seg_desct_to_kvm_desct(struct desc_struct *seg_desc, u16 selector,
                                    struct kvm_segment *kvm_desct)
 {
-        kvm_desct->base = seg_desc->base0;
-        kvm_desct->base |= seg_desc->base1 << 16;
-        kvm_desct->base |= seg_desc->base2 << 24;
-        kvm_desct->limit = seg_desc->limit0;
-        kvm_desct->limit |= seg_desc->limit << 16;
+        kvm_desct->base = get_desc_base(seg_desc);
+        kvm_desct->limit = get_desc_limit(seg_desc);
         if (seg_desc->g) {
                 kvm_desct->limit <<= 12;
                 kvm_desct->limit |= 0xfff;
@@ -3701,7 +4023,6 @@ static void get_segment_descriptor_dtable(struct kvm_vcpu *vcpu,
 static int load_guest_segment_descriptor(struct kvm_vcpu *vcpu, u16 selector,
                                          struct desc_struct *seg_desc)
 {
-        gpa_t gpa;
         struct descriptor_table dtable;
         u16 index = selector >> 3;
 
@@ -3711,16 +4032,13 @@ static int load_guest_segment_descriptor(struct kvm_vcpu *vcpu, u16 selector,
                 kvm_queue_exception_e(vcpu, GP_VECTOR, selector & 0xfffc);
                 return 1;
         }
-        gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, dtable.base);
-        gpa += index * 8;
-        return kvm_read_guest(vcpu->kvm, gpa, seg_desc, 8);
+        return kvm_read_guest_virt(dtable.base + index*8, seg_desc, sizeof(*seg_desc), vcpu);
 }
 
 /* allowed just for 8 bytes segments */
 static int save_guest_segment_descriptor(struct kvm_vcpu *vcpu, u16 selector,
                                          struct desc_struct *seg_desc)
 {
-        gpa_t gpa;
         struct descriptor_table dtable;
         u16 index = selector >> 3;
 
@@ -3728,19 +4046,13 @@ static int save_guest_segment_descriptor(struct kvm_vcpu *vcpu, u16 selector,
 
         if (dtable.limit < index * 8 + 7)
                 return 1;
-        gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, dtable.base);
-        gpa += index * 8;
-        return kvm_write_guest(vcpu->kvm, gpa, seg_desc, 8);
+        return kvm_write_guest_virt(dtable.base + index*8, seg_desc, sizeof(*seg_desc), vcpu);
 }
 
 static u32 get_tss_base_addr(struct kvm_vcpu *vcpu,
                              struct desc_struct *seg_desc)
 {
-        u32 base_addr;
-
-        base_addr = seg_desc->base0;
-        base_addr |= (seg_desc->base1 << 16);
-        base_addr |= (seg_desc->base2 << 24);
+        u32 base_addr = get_desc_base(seg_desc);
 
         return vcpu->arch.mmu.gva_to_gpa(vcpu, base_addr);
 }
@@ -3785,12 +4097,19 @@ static int kvm_load_realmode_segment(struct kvm_vcpu *vcpu, u16 selector, int se
         return 0;
 }
 
+static int is_vm86_segment(struct kvm_vcpu *vcpu, int seg)
+{
+        return (seg != VCPU_SREG_LDTR) &&
+                (seg != VCPU_SREG_TR) &&
+                (kvm_x86_ops->get_rflags(vcpu) & X86_EFLAGS_VM);
+}
+
 int kvm_load_segment_descriptor(struct kvm_vcpu *vcpu, u16 selector,
                                 int type_bits, int seg)
 {
         struct kvm_segment kvm_seg;
 
-        if (!(vcpu->arch.cr0 & X86_CR0_PE))
+        if (is_vm86_segment(vcpu, seg) || !(vcpu->arch.cr0 & X86_CR0_PE))
                 return kvm_load_realmode_segment(vcpu, selector, seg);
         if (load_segment_descriptor_to_kvm_desct(vcpu, selector, &kvm_seg))
                 return 1;
@@ -4029,7 +4348,7 @@ int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int reason)
                 }
         }
 
-        if (!nseg_desc.p || (nseg_desc.limit0 | nseg_desc.limit << 16) < 0x67) {
+        if (!nseg_desc.p || get_desc_limit(&nseg_desc) < 0x67) {
                 kvm_queue_exception_e(vcpu, TS_VECTOR, tss_selector & 0xfffc);
                 return 1;
         }
@@ -4099,13 +4418,7 @@ int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
 
         vcpu->arch.cr2 = sregs->cr2;
         mmu_reset_needed |= vcpu->arch.cr3 != sregs->cr3;
-
-        down_read(&vcpu->kvm->slots_lock);
-        if (gfn_to_memslot(vcpu->kvm, sregs->cr3 >> PAGE_SHIFT))
-                vcpu->arch.cr3 = sregs->cr3;
-        else
-                set_bit(KVM_REQ_TRIPLE_FAULT, &vcpu->requests);
-        up_read(&vcpu->kvm->slots_lock);
+        vcpu->arch.cr3 = sregs->cr3;
 
         kvm_set_cr8(vcpu, sregs->cr8);
 
@@ -4147,8 +4460,10 @@ int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
         kvm_set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
         kvm_set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
 
+        update_cr8_intercept(vcpu);
+
         /* Older userspace won't unhalt the vcpu on reset. */
-        if (vcpu->vcpu_id == 0 && kvm_rip_read(vcpu) == 0xfff0 &&
+        if (kvm_vcpu_is_bsp(vcpu) && kvm_rip_read(vcpu) == 0xfff0 &&
             sregs->cs.selector == 0xf000 && sregs->cs.base == 0xffff0000 &&
             !(vcpu->arch.cr0 & X86_CR0_PE))
                 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
@@ -4419,7 +4734,7 @@ int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
         kvm = vcpu->kvm;
 
         vcpu->arch.mmu.root_hpa = INVALID_PAGE;
-        if (!irqchip_in_kernel(kvm) || vcpu->vcpu_id == 0)
+        if (!irqchip_in_kernel(kvm) || kvm_vcpu_is_bsp(vcpu))
                 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
         else
                 vcpu->arch.mp_state = KVM_MP_STATE_UNINITIALIZED;
@@ -4441,6 +4756,14 @@ int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
                         goto fail_mmu_destroy;
         }
 
+        vcpu->arch.mce_banks = kzalloc(KVM_MAX_MCE_BANKS * sizeof(u64) * 4,
+                                       GFP_KERNEL);
+        if (!vcpu->arch.mce_banks) {
+                r = -ENOMEM;
+                goto fail_mmu_destroy;
+        }
+        vcpu->arch.mcg_cap = KVM_MAX_MCE_BANKS;
+
         return 0;
 
 fail_mmu_destroy:
@@ -4488,20 +4811,22 @@ static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu)
 static void kvm_free_vcpus(struct kvm *kvm)
 {
         unsigned int i;
+        struct kvm_vcpu *vcpu;
 
         /*
          * Unpin any mmu pages first.
          */
-        for (i = 0; i < KVM_MAX_VCPUS; ++i)
-                if (kvm->vcpus[i])
-                        kvm_unload_vcpu_mmu(kvm->vcpus[i]);
-        for (i = 0; i < KVM_MAX_VCPUS; ++i) {
-                if (kvm->vcpus[i]) {
-                        kvm_arch_vcpu_free(kvm->vcpus[i]);
-                        kvm->vcpus[i] = NULL;
-                }
-        }
+        kvm_for_each_vcpu(i, vcpu, kvm)
+                kvm_unload_vcpu_mmu(vcpu);
+        kvm_for_each_vcpu(i, vcpu, kvm)
+                kvm_arch_vcpu_free(vcpu);
+
+        mutex_lock(&kvm->lock);
+        for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
+                kvm->vcpus[i] = NULL;
 
+        atomic_set(&kvm->online_vcpus, 0);
+        mutex_unlock(&kvm->lock);
 }
 
 void kvm_arch_sync_events(struct kvm *kvm)
@@ -4578,7 +4903,6 @@ int kvm_arch_set_memory_region(struct kvm *kvm,
 
         kvm_mmu_slot_remove_write_access(kvm, mem->slot);
         spin_unlock(&kvm->mmu_lock);
-        kvm_flush_remote_tlbs(kvm);
 
         return 0;
 }
@@ -4592,8 +4916,10 @@ void kvm_arch_flush_shadow(struct kvm *kvm)
 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
 {
         return vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE
-               || vcpu->arch.mp_state == KVM_MP_STATE_SIPI_RECEIVED
-               || vcpu->arch.nmi_pending;
+                || vcpu->arch.mp_state == KVM_MP_STATE_SIPI_RECEIVED
+                || vcpu->arch.nmi_pending ||
+                (kvm_arch_interrupt_allowed(vcpu) &&
+                 kvm_cpu_has_interrupt(vcpu));
 }
 
 void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
@@ -4617,3 +4943,9 @@ int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu)
 {
         return kvm_x86_ops->interrupt_allowed(vcpu);
 }
+
+EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_exit);
+EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_inj_virq);
+EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_page_fault);
+EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_msr);
+EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_cr);
diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h
index 4c8e10af78e8..5eadea585d2a 100644
--- a/arch/x86/kvm/x86.h
+++ b/arch/x86/kvm/x86.h
@@ -31,4 +31,8 @@ static inline bool kvm_exception_is_soft(unsigned int nr)
 {
         return (nr == BP_VECTOR) || (nr == OF_VECTOR);
 }
+
+struct kvm_cpuid_entry2 *kvm_find_cpuid_entry(struct kvm_vcpu *vcpu,
+                                             u32 function, u32 index);
+
 #endif
diff --git a/arch/x86/lguest/boot.c b/arch/x86/lguest/boot.c
index d677fa9ca650..7e59dc1d3fc2 100644
--- a/arch/x86/lguest/boot.c
+++ b/arch/x86/lguest/boot.c
@@ -1135,11 +1135,6 @@ static struct notifier_block paniced = {
 /* Setting up memory is fairly easy. */
 static __init char *lguest_memory_setup(void)
 {
-        /* We do this here and not earlier because lockcheck used to barf if we
-         * did it before start_kernel().  I think we fixed that, so it'd be
-         * nice to move it back to lguest_init.  Patch welcome... */
-        atomic_notifier_chain_register(&panic_notifier_list, &paniced);
-
         /*
          *The Linux bootloader header contains an "e820" memory map: the
          * Launcher populated the first entry with our memory limit.
@@ -1262,7 +1257,6 @@ __init void lguest_init(void)
          */
 
         /* Interrupt-related operations */
-        pv_irq_ops.init_IRQ = lguest_init_IRQ;
         pv_irq_ops.save_fl = PV_CALLEE_SAVE(save_fl);
         pv_irq_ops.restore_fl = __PV_IS_CALLEE_SAVE(lg_restore_fl);
         pv_irq_ops.irq_disable = PV_CALLEE_SAVE(irq_disable);
@@ -1270,7 +1264,6 @@ __init void lguest_init(void)
         pv_irq_ops.safe_halt = lguest_safe_halt;
 
         /* Setup operations */
-        pv_init_ops.memory_setup = lguest_memory_setup;
         pv_init_ops.patch = lguest_patch;
 
         /* Intercepts of various CPU instructions */
@@ -1320,10 +1313,11 @@ __init void lguest_init(void)
         set_lguest_basic_apic_ops();
 #endif
 
-        /* Time operations */
-        pv_time_ops.get_wallclock = lguest_get_wallclock;
-        pv_time_ops.time_init = lguest_time_init;
-        pv_time_ops.get_tsc_khz = lguest_tsc_khz;
+        x86_init.resources.memory_setup = lguest_memory_setup;
+        x86_init.irqs.intr_init = lguest_init_IRQ;
+        x86_init.timers.timer_init = lguest_time_init;
+        x86_platform.calibrate_tsc = lguest_tsc_khz;
+        x86_platform.get_wallclock =  lguest_get_wallclock;
 
         /*
          * Now is a good time to look at the implementations of these functions
@@ -1365,10 +1359,13 @@ __init void lguest_init(void)
 
         /*
          * If we don't initialize the lock dependency checker now, it crashes
-         * paravirt_disable_iospace.
+         * atomic_notifier_chain_register, then paravirt_disable_iospace.
          */
         lockdep_init();
 
+        /* Hook in our special panic hypercall code. */
+        atomic_notifier_chain_register(&panic_notifier_list, &paniced);
+
         /*
          * The IDE code spends about 3 seconds probing for disks: if we reserve
          * all the I/O ports up front it can't get them and so doesn't probe.
diff --git a/arch/x86/lib/Makefile b/arch/x86/lib/Makefile
index 07c31899c9c2..9e609206fac9 100644
--- a/arch/x86/lib/Makefile
+++ b/arch/x86/lib/Makefile
@@ -9,6 +9,8 @@ lib-y += thunk_$(BITS).o
 lib-y += usercopy_$(BITS).o getuser.o putuser.o
 lib-y += memcpy_$(BITS).o
 
+obj-y += msr-reg.o msr-reg-export.o
+
 ifeq ($(CONFIG_X86_32),y)
         obj-y += atomic64_32.o
         lib-y += checksum_32.o
diff --git a/arch/x86/lib/msr-reg-export.c b/arch/x86/lib/msr-reg-export.c
new file mode 100644
index 000000000000..a311cc59b65d
--- /dev/null
+++ b/arch/x86/lib/msr-reg-export.c
@@ -0,0 +1,5 @@
+#include <linux/module.h>
+#include <asm/msr.h>
+
+EXPORT_SYMBOL(native_rdmsr_safe_regs);
+EXPORT_SYMBOL(native_wrmsr_safe_regs);
diff --git a/arch/x86/lib/msr-reg.S b/arch/x86/lib/msr-reg.S
new file mode 100644
index 000000000000..69fa10623f21
--- /dev/null
+++ b/arch/x86/lib/msr-reg.S
@@ -0,0 +1,102 @@
+#include <linux/linkage.h>
+#include <linux/errno.h>
+#include <asm/dwarf2.h>
+#include <asm/asm.h>
+#include <asm/msr.h>
+
+#ifdef CONFIG_X86_64
+/*
+ * int native_{rdmsr,wrmsr}_safe_regs(u32 gprs[8]);
+ *
+ * reg layout: u32 gprs[eax, ecx, edx, ebx, esp, ebp, esi, edi]
+ *
+ */
+.macro op_safe_regs op
+ENTRY(native_\op\()_safe_regs)
+        CFI_STARTPROC
+        pushq_cfi %rbx
+        pushq_cfi %rbp
+        movq    %rdi, %r10      /* Save pointer */
+        xorl    %r11d, %r11d    /* Return value */
+        movl    (%rdi), %eax
+        movl    4(%rdi), %ecx
+        movl    8(%rdi), %edx
+        movl    12(%rdi), %ebx
+        movl    20(%rdi), %ebp
+        movl    24(%rdi), %esi
+        movl    28(%rdi), %edi
+        CFI_REMEMBER_STATE
+1:      \op
+2:      movl    %eax, (%r10)
+        movl    %r11d, %eax     /* Return value */
+        movl    %ecx, 4(%r10)
+        movl    %edx, 8(%r10)
+        movl    %ebx, 12(%r10)
+        movl    %ebp, 20(%r10)
+        movl    %esi, 24(%r10)
+        movl    %edi, 28(%r10)
+        popq_cfi %rbp
+        popq_cfi %rbx
+        ret
+3:
+        CFI_RESTORE_STATE
+        movl    $-EIO, %r11d
+        jmp     2b
+
+        _ASM_EXTABLE(1b, 3b)
+        CFI_ENDPROC
+ENDPROC(native_\op\()_safe_regs)
+.endm
+
+#else /* X86_32 */
+
+.macro op_safe_regs op
+ENTRY(native_\op\()_safe_regs)
+        CFI_STARTPROC
+        pushl_cfi %ebx
+        pushl_cfi %ebp
+        pushl_cfi %esi
+        pushl_cfi %edi
+        pushl_cfi $0              /* Return value */
+        pushl_cfi %eax
+        movl    4(%eax), %ecx
+        movl    8(%eax), %edx
+        movl    12(%eax), %ebx
+        movl    20(%eax), %ebp
+        movl    24(%eax), %esi
+        movl    28(%eax), %edi
+        movl    (%eax), %eax
+        CFI_REMEMBER_STATE
+1:      \op
+2:      pushl_cfi %eax
+        movl    4(%esp), %eax
+        popl_cfi (%eax)
+        addl    $4, %esp
+        CFI_ADJUST_CFA_OFFSET -4
+        movl    %ecx, 4(%eax)
+        movl    %edx, 8(%eax)
+        movl    %ebx, 12(%eax)
+        movl    %ebp, 20(%eax)
+        movl    %esi, 24(%eax)
+        movl    %edi, 28(%eax)
+        popl_cfi %eax
+        popl_cfi %edi
+        popl_cfi %esi
+        popl_cfi %ebp
+        popl_cfi %ebx
+        ret
+3:
+        CFI_RESTORE_STATE
+        movl    $-EIO, 4(%esp)
+        jmp     2b
+
+        _ASM_EXTABLE(1b, 3b)
+        CFI_ENDPROC
+ENDPROC(native_\op\()_safe_regs)
+.endm
+
+#endif
+
+op_safe_regs rdmsr
+op_safe_regs wrmsr
+
diff --git a/arch/x86/lib/msr.c b/arch/x86/lib/msr.c
index caa24aca8115..33a1e3ca22d8 100644
--- a/arch/x86/lib/msr.c
+++ b/arch/x86/lib/msr.c
@@ -175,3 +175,52 @@ int wrmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h)
         return err ? err : rv.err;
 }
 EXPORT_SYMBOL(wrmsr_safe_on_cpu);
+
+/*
+ * These variants are significantly slower, but allows control over
+ * the entire 32-bit GPR set.
+ */
+struct msr_regs_info {
+        u32 *regs;
+        int err;
+};
+
+static void __rdmsr_safe_regs_on_cpu(void *info)
+{
+        struct msr_regs_info *rv = info;
+
+        rv->err = rdmsr_safe_regs(rv->regs);
+}
+
+static void __wrmsr_safe_regs_on_cpu(void *info)
+{
+        struct msr_regs_info *rv = info;
+
+        rv->err = wrmsr_safe_regs(rv->regs);
+}
+
+int rdmsr_safe_regs_on_cpu(unsigned int cpu, u32 *regs)
+{
+        int err;
+        struct msr_regs_info rv;
+
+        rv.regs   = regs;
+        rv.err    = -EIO;
+        err = smp_call_function_single(cpu, __rdmsr_safe_regs_on_cpu, &rv, 1);
+
+        return err ? err : rv.err;
+}
+EXPORT_SYMBOL(rdmsr_safe_regs_on_cpu);
+
+int wrmsr_safe_regs_on_cpu(unsigned int cpu, u32 *regs)
+{
+        int err;
+        struct msr_regs_info rv;
+
+        rv.regs = regs;
+        rv.err  = -EIO;
+        err = smp_call_function_single(cpu, __wrmsr_safe_regs_on_cpu, &rv, 1);
+
+        return err ? err : rv.err;
+}
+EXPORT_SYMBOL(wrmsr_safe_regs_on_cpu);
diff --git a/arch/x86/mm/Makefile b/arch/x86/mm/Makefile
index eefdeee8a871..9b5a9f59a478 100644
--- a/arch/x86/mm/Makefile
+++ b/arch/x86/mm/Makefile
@@ -1,5 +1,9 @@
 obj-y   :=  init.o init_$(BITS).o fault.o ioremap.o extable.o pageattr.o mmap.o \
-            pat.o pgtable.o gup.o
+            pat.o pgtable.o physaddr.o gup.o
+
+# Make sure __phys_addr has no stackprotector
+nostackp := $(call cc-option, -fno-stack-protector)
+CFLAGS_physaddr.o               := $(nostackp)
 
 obj-$(CONFIG_SMP)               += tlb.o
 
diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c
index bfae139182ff..f4cee9028cf0 100644
--- a/arch/x86/mm/fault.c
+++ b/arch/x86/mm/fault.c
@@ -10,7 +10,7 @@
 #include <linux/bootmem.h>              /* max_low_pfn                  */
 #include <linux/kprobes.h>              /* __kprobes, ...               */
 #include <linux/mmiotrace.h>            /* kmmio_handler, ...           */
-#include <linux/perf_counter.h>         /* perf_swcounter_event         */
+#include <linux/perf_event.h>           /* perf_sw_event                */
 
 #include <asm/traps.h>                  /* dotraplinkage, ...           */
 #include <asm/pgalloc.h>                /* pgd_*(), ...                 */
@@ -167,6 +167,7 @@ force_sig_info_fault(int si_signo, int si_code, unsigned long address,
         info.si_errno   = 0;
         info.si_code    = si_code;
         info.si_addr    = (void __user *)address;
+        info.si_addr_lsb = si_code == BUS_MCEERR_AR ? PAGE_SHIFT : 0;
 
         force_sig_info(si_signo, &info, tsk);
 }
@@ -285,26 +286,25 @@ check_v8086_mode(struct pt_regs *regs, unsigned long address,
                 tsk->thread.screen_bitmap |= 1 << bit;
 }
 
-static void dump_pagetable(unsigned long address)
+static bool low_pfn(unsigned long pfn)
 {
-        __typeof__(pte_val(__pte(0))) page;
+        return pfn < max_low_pfn;
+}
 
-        page = read_cr3();
-        page = ((__typeof__(page) *) __va(page))[address >> PGDIR_SHIFT];
+static void dump_pagetable(unsigned long address)
+{
+        pgd_t *base = __va(read_cr3());
+        pgd_t *pgd = &base[pgd_index(address)];
+        pmd_t *pmd;
+        pte_t *pte;
 
 #ifdef CONFIG_X86_PAE
-        printk("*pdpt = %016Lx ", page);
-        if ((page >> PAGE_SHIFT) < max_low_pfn
-            && page & _PAGE_PRESENT) {
-                page &= PAGE_MASK;
-                page = ((__typeof__(page) *) __va(page))[(address >> PMD_SHIFT)
-                                                        & (PTRS_PER_PMD - 1)];
-                printk(KERN_CONT "*pde = %016Lx ", page);
-                page &= ~_PAGE_NX;
-        }
-#else
-        printk("*pde = %08lx ", page);
+        printk("*pdpt = %016Lx ", pgd_val(*pgd));
+        if (!low_pfn(pgd_val(*pgd) >> PAGE_SHIFT) || !pgd_present(*pgd))
+                goto out;
 #endif
+        pmd = pmd_offset(pud_offset(pgd, address), address);
+        printk(KERN_CONT "*pde = %0*Lx ", sizeof(*pmd) * 2, (u64)pmd_val(*pmd));
 
         /*
          * We must not directly access the pte in the highpte
@@ -312,16 +312,12 @@ static void dump_pagetable(unsigned long address)
          * And let's rather not kmap-atomic the pte, just in case
          * it's allocated already:
          */
-        if ((page >> PAGE_SHIFT) < max_low_pfn
-            && (page & _PAGE_PRESENT)
-            && !(page & _PAGE_PSE)) {
-
-                page &= PAGE_MASK;
-                page = ((__typeof__(page) *) __va(page))[(address >> PAGE_SHIFT)
-                                                        & (PTRS_PER_PTE - 1)];
-                printk("*pte = %0*Lx ", sizeof(page)*2, (u64)page);
-        }
+        if (!low_pfn(pmd_pfn(*pmd)) || !pmd_present(*pmd) || pmd_large(*pmd))
+                goto out;
 
+        pte = pte_offset_kernel(pmd, address);
+        printk("*pte = %0*Lx ", sizeof(*pte) * 2, (u64)pte_val(*pte));
+out:
         printk("\n");
 }
 
@@ -450,16 +446,12 @@ static int bad_address(void *p)
 
 static void dump_pagetable(unsigned long address)
 {
-        pgd_t *pgd;
+        pgd_t *base = __va(read_cr3() & PHYSICAL_PAGE_MASK);
+        pgd_t *pgd = base + pgd_index(address);
         pud_t *pud;
         pmd_t *pmd;
         pte_t *pte;
 
-        pgd = (pgd_t *)read_cr3();
-
-        pgd = __va((unsigned long)pgd & PHYSICAL_PAGE_MASK);
-
-        pgd += pgd_index(address);
         if (bad_address(pgd))
                 goto bad;
 
@@ -799,10 +791,12 @@ out_of_memory(struct pt_regs *regs, unsigned long error_code,
 }
 
 static void
-do_sigbus(struct pt_regs *regs, unsigned long error_code, unsigned long address)
+do_sigbus(struct pt_regs *regs, unsigned long error_code, unsigned long address,
+          unsigned int fault)
 {
         struct task_struct *tsk = current;
         struct mm_struct *mm = tsk->mm;
+        int code = BUS_ADRERR;
 
         up_read(&mm->mmap_sem);
 
@@ -818,7 +812,15 @@ do_sigbus(struct pt_regs *regs, unsigned long error_code, unsigned long address)
         tsk->thread.error_code  = error_code;
         tsk->thread.trap_no     = 14;
 
-        force_sig_info_fault(SIGBUS, BUS_ADRERR, address, tsk);
+#ifdef CONFIG_MEMORY_FAILURE
+        if (fault & VM_FAULT_HWPOISON) {
+                printk(KERN_ERR
+        "MCE: Killing %s:%d due to hardware memory corruption fault at %lx\n",
+                        tsk->comm, tsk->pid, address);
+                code = BUS_MCEERR_AR;
+        }
+#endif
+        force_sig_info_fault(SIGBUS, code, address, tsk);
 }
 
 static noinline void
@@ -828,8 +830,8 @@ mm_fault_error(struct pt_regs *regs, unsigned long error_code,
         if (fault & VM_FAULT_OOM) {
                 out_of_memory(regs, error_code, address);
         } else {
-                if (fault & VM_FAULT_SIGBUS)
-                        do_sigbus(regs, error_code, address);
+                if (fault & (VM_FAULT_SIGBUS|VM_FAULT_HWPOISON))
+                        do_sigbus(regs, error_code, address, fault);
                 else
                         BUG();
         }
@@ -1026,7 +1028,7 @@ do_page_fault(struct pt_regs *regs, unsigned long error_code)
         if (unlikely(error_code & PF_RSVD))
                 pgtable_bad(regs, error_code, address);
 
-        perf_swcounter_event(PERF_COUNT_SW_PAGE_FAULTS, 1, 0, regs, address);
+        perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, 0, regs, address);
 
         /*
          * If we're in an interrupt, have no user context or are running
@@ -1123,11 +1125,11 @@ good_area:
 
         if (fault & VM_FAULT_MAJOR) {
                 tsk->maj_flt++;
-                perf_swcounter_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, 0,
+                perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, 0,
                                      regs, address);
         } else {
                 tsk->min_flt++;
-                perf_swcounter_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, 0,
+                perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, 0,
                                      regs, address);
         }
 
diff --git a/arch/x86/mm/highmem_32.c b/arch/x86/mm/highmem_32.c
index 2112ed55e7ea..63a6ba66cbe0 100644
--- a/arch/x86/mm/highmem_32.c
+++ b/arch/x86/mm/highmem_32.c
@@ -24,7 +24,7 @@ void kunmap(struct page *page)
  * no global lock is needed and because the kmap code must perform a global TLB
  * invalidation when the kmap pool wraps.
  *
- * However when holding an atomic kmap is is not legal to sleep, so atomic
+ * However when holding an atomic kmap it is not legal to sleep, so atomic
  * kmaps are appropriate for short, tight code paths only.
  */
 void *kmap_atomic_prot(struct page *page, enum km_type type, pgprot_t prot)
@@ -104,6 +104,7 @@ EXPORT_SYMBOL(kunmap);
 EXPORT_SYMBOL(kmap_atomic);
 EXPORT_SYMBOL(kunmap_atomic);
 EXPORT_SYMBOL(kmap_atomic_prot);
+EXPORT_SYMBOL(kmap_atomic_to_page);
 
 void __init set_highmem_pages_init(void)
 {
diff --git a/arch/x86/mm/init_32.c b/arch/x86/mm/init_32.c
index 3cd7711bb949..30938c1d8d5d 100644
--- a/arch/x86/mm/init_32.c
+++ b/arch/x86/mm/init_32.c
@@ -84,7 +84,7 @@ static pmd_t * __init one_md_table_init(pgd_t *pgd)
 #ifdef CONFIG_X86_PAE
         if (!(pgd_val(*pgd) & _PAGE_PRESENT)) {
                 if (after_bootmem)
-                        pmd_table = (pmd_t *)alloc_bootmem_low_pages(PAGE_SIZE);
+                        pmd_table = (pmd_t *)alloc_bootmem_pages(PAGE_SIZE);
                 else
                         pmd_table = (pmd_t *)alloc_low_page();
                 paravirt_alloc_pmd(&init_mm, __pa(pmd_table) >> PAGE_SHIFT);
@@ -116,7 +116,7 @@ static pte_t * __init one_page_table_init(pmd_t *pmd)
 #endif
                         if (!page_table)
                                 page_table =
-                                (pte_t *)alloc_bootmem_low_pages(PAGE_SIZE);
+                                (pte_t *)alloc_bootmem_pages(PAGE_SIZE);
                 } else
                         page_table = (pte_t *)alloc_low_page();
 
@@ -857,8 +857,6 @@ static void __init test_wp_bit(void)
         }
 }
 
-static struct kcore_list kcore_mem, kcore_vmalloc;
-
 void __init mem_init(void)
 {
         int codesize, reservedpages, datasize, initsize;
@@ -886,13 +884,9 @@ void __init mem_init(void)
         datasize =  (unsigned long) &_edata - (unsigned long) &_etext;
         initsize =  (unsigned long) &__init_end - (unsigned long) &__init_begin;
 
-        kclist_add(&kcore_mem, __va(0), max_low_pfn << PAGE_SHIFT);
-        kclist_add(&kcore_vmalloc, (void *)VMALLOC_START,
-                   VMALLOC_END-VMALLOC_START);
-
         printk(KERN_INFO "Memory: %luk/%luk available (%dk kernel code, "
                         "%dk reserved, %dk data, %dk init, %ldk highmem)\n",
-                (unsigned long) nr_free_pages() << (PAGE_SHIFT-10),
+                nr_free_pages() << (PAGE_SHIFT-10),
                 num_physpages << (PAGE_SHIFT-10),
                 codesize >> 10,
                 reservedpages << (PAGE_SHIFT-10),
diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
index ea56b8cbb6a6..5a4398a6006b 100644
--- a/arch/x86/mm/init_64.c
+++ b/arch/x86/mm/init_64.c
@@ -647,8 +647,7 @@ EXPORT_SYMBOL_GPL(memory_add_physaddr_to_nid);
 
 #endif /* CONFIG_MEMORY_HOTPLUG */
 
-static struct kcore_list kcore_mem, kcore_vmalloc, kcore_kernel,
-                         kcore_modules, kcore_vsyscall;
+static struct kcore_list kcore_vsyscall;
 
 void __init mem_init(void)
 {
@@ -677,17 +676,12 @@ void __init mem_init(void)
         initsize =  (unsigned long) &__init_end - (unsigned long) &__init_begin;
 
         /* Register memory areas for /proc/kcore */
-        kclist_add(&kcore_mem, __va(0), max_low_pfn << PAGE_SHIFT);
-        kclist_add(&kcore_vmalloc, (void *)VMALLOC_START,
-                   VMALLOC_END-VMALLOC_START);
-        kclist_add(&kcore_kernel, &_stext, _end - _stext);
-        kclist_add(&kcore_modules, (void *)MODULES_VADDR, MODULES_LEN);
         kclist_add(&kcore_vsyscall, (void *)VSYSCALL_START,
-                                 VSYSCALL_END - VSYSCALL_START);
+                         VSYSCALL_END - VSYSCALL_START, KCORE_OTHER);
 
         printk(KERN_INFO "Memory: %luk/%luk available (%ldk kernel code, "
                          "%ldk absent, %ldk reserved, %ldk data, %ldk init)\n",
-                (unsigned long) nr_free_pages() << (PAGE_SHIFT-10),
+                nr_free_pages() << (PAGE_SHIFT-10),
                 max_pfn << (PAGE_SHIFT-10),
                 codesize >> 10,
                 absent_pages << (PAGE_SHIFT-10),
diff --git a/arch/x86/mm/iomap_32.c b/arch/x86/mm/iomap_32.c
index fe6f84ca121e..84e236ce76ba 100644
--- a/arch/x86/mm/iomap_32.c
+++ b/arch/x86/mm/iomap_32.c
@@ -21,7 +21,7 @@
 #include <linux/module.h>
 #include <linux/highmem.h>
 
-int is_io_mapping_possible(resource_size_t base, unsigned long size)
+static int is_io_mapping_possible(resource_size_t base, unsigned long size)
 {
 #if !defined(CONFIG_X86_PAE) && defined(CONFIG_PHYS_ADDR_T_64BIT)
         /* There is no way to map greater than 1 << 32 address without PAE */
@@ -30,7 +30,30 @@ int is_io_mapping_possible(resource_size_t base, unsigned long size)
 #endif
         return 1;
 }
-EXPORT_SYMBOL_GPL(is_io_mapping_possible);
+
+int iomap_create_wc(resource_size_t base, unsigned long size, pgprot_t *prot)
+{
+        unsigned long flag = _PAGE_CACHE_WC;
+        int ret;
+
+        if (!is_io_mapping_possible(base, size))
+                return -EINVAL;
+
+        ret = io_reserve_memtype(base, base + size, &flag);
+        if (ret)
+                return ret;
+
+        *prot = __pgprot(__PAGE_KERNEL | flag);
+        return 0;
+}
+EXPORT_SYMBOL_GPL(iomap_create_wc);
+
+void
+iomap_free(resource_size_t base, unsigned long size)
+{
+        io_free_memtype(base, base + size);
+}
+EXPORT_SYMBOL_GPL(iomap_free);
 
 void *kmap_atomic_prot_pfn(unsigned long pfn, enum km_type type, pgprot_t prot)
 {
diff --git a/arch/x86/mm/ioremap.c b/arch/x86/mm/ioremap.c
index 8a450930834f..334e63ca7b2b 100644
--- a/arch/x86/mm/ioremap.c
+++ b/arch/x86/mm/ioremap.c
@@ -22,77 +22,7 @@
 #include <asm/pgalloc.h>
 #include <asm/pat.h>
 
-static inline int phys_addr_valid(resource_size_t addr)
-{
-#ifdef CONFIG_PHYS_ADDR_T_64BIT
-        return !(addr >> boot_cpu_data.x86_phys_bits);
-#else
-        return 1;
-#endif
-}
-
-#ifdef CONFIG_X86_64
-
-unsigned long __phys_addr(unsigned long x)
-{
-        if (x >= __START_KERNEL_map) {
-                x -= __START_KERNEL_map;
-                VIRTUAL_BUG_ON(x >= KERNEL_IMAGE_SIZE);
-                x += phys_base;
-        } else {
-                VIRTUAL_BUG_ON(x < PAGE_OFFSET);
-                x -= PAGE_OFFSET;
-                VIRTUAL_BUG_ON(!phys_addr_valid(x));
-        }
-        return x;
-}
-EXPORT_SYMBOL(__phys_addr);
-
-bool __virt_addr_valid(unsigned long x)
-{
-        if (x >= __START_KERNEL_map) {
-                x -= __START_KERNEL_map;
-                if (x >= KERNEL_IMAGE_SIZE)
-                        return false;
-                x += phys_base;
-        } else {
-                if (x < PAGE_OFFSET)
-                        return false;
-                x -= PAGE_OFFSET;
-                if (!phys_addr_valid(x))
-                        return false;
-        }
-
-        return pfn_valid(x >> PAGE_SHIFT);
-}
-EXPORT_SYMBOL(__virt_addr_valid);
-
-#else
-
-#ifdef CONFIG_DEBUG_VIRTUAL
-unsigned long __phys_addr(unsigned long x)
-{
-        /* VMALLOC_* aren't constants  */
-        VIRTUAL_BUG_ON(x < PAGE_OFFSET);
-        VIRTUAL_BUG_ON(__vmalloc_start_set && is_vmalloc_addr((void *) x));
-        return x - PAGE_OFFSET;
-}
-EXPORT_SYMBOL(__phys_addr);
-#endif
-
-bool __virt_addr_valid(unsigned long x)
-{
-        if (x < PAGE_OFFSET)
-                return false;
-        if (__vmalloc_start_set && is_vmalloc_addr((void *) x))
-                return false;
-        if (x >= FIXADDR_START)
-                return false;
-        return pfn_valid((x - PAGE_OFFSET) >> PAGE_SHIFT);
-}
-EXPORT_SYMBOL(__virt_addr_valid);
-
-#endif
+#include "physaddr.h"
 
 int page_is_ram(unsigned long pagenr)
 {
@@ -228,24 +158,14 @@ static void __iomem *__ioremap_caller(resource_size_t phys_addr,
         retval = reserve_memtype(phys_addr, (u64)phys_addr + size,
                                                 prot_val, &new_prot_val);
         if (retval) {
-                pr_debug("Warning: reserve_memtype returned %d\n", retval);
+                printk(KERN_ERR "ioremap reserve_memtype failed %d\n", retval);
                 return NULL;
         }
 
         if (prot_val != new_prot_val) {
-                /*
-                 * Do not fallback to certain memory types with certain
-                 * requested type:
-                 * - request is uc-, return cannot be write-back
-                 * - request is uc-, return cannot be write-combine
-                 * - request is write-combine, return cannot be write-back
-                 */
-                if ((prot_val == _PAGE_CACHE_UC_MINUS &&
-                     (new_prot_val == _PAGE_CACHE_WB ||
-                      new_prot_val == _PAGE_CACHE_WC)) ||
-                    (prot_val == _PAGE_CACHE_WC &&
-                     new_prot_val == _PAGE_CACHE_WB)) {
-                        pr_debug(
+                if (!is_new_memtype_allowed(phys_addr, size,
+                                            prot_val, new_prot_val)) {
+                        printk(KERN_ERR
                 "ioremap error for 0x%llx-0x%llx, requested 0x%lx, got 0x%lx\n",
                                 (unsigned long long)phys_addr,
                                 (unsigned long long)(phys_addr + size),
diff --git a/arch/x86/mm/kmemcheck/kmemcheck.c b/arch/x86/mm/kmemcheck/kmemcheck.c
index 2c55ed098654..8cc183344140 100644
--- a/arch/x86/mm/kmemcheck/kmemcheck.c
+++ b/arch/x86/mm/kmemcheck/kmemcheck.c
@@ -225,9 +225,6 @@ void kmemcheck_hide(struct pt_regs *regs)
 
         BUG_ON(!irqs_disabled());
 
-        if (data->balance == 0)
-                return;
-
         if (unlikely(data->balance != 1)) {
                 kmemcheck_show_all();
                 kmemcheck_error_save_bug(regs);
@@ -331,6 +328,20 @@ static void kmemcheck_read_strict(struct pt_regs *regs,
         kmemcheck_shadow_set(shadow, size);
 }
 
+bool kmemcheck_is_obj_initialized(unsigned long addr, size_t size)
+{
+        enum kmemcheck_shadow status;
+        void *shadow;
+
+        shadow = kmemcheck_shadow_lookup(addr);
+        if (!shadow)
+                return true;
+
+        status = kmemcheck_shadow_test(shadow, size);
+
+        return status == KMEMCHECK_SHADOW_INITIALIZED;
+}
+
 /* Access may cross page boundary */
 static void kmemcheck_read(struct pt_regs *regs,
         unsigned long addr, unsigned int size)
diff --git a/arch/x86/mm/kmemcheck/shadow.c b/arch/x86/mm/kmemcheck/shadow.c
index e773b6bd0079..3f66b82076a3 100644
--- a/arch/x86/mm/kmemcheck/shadow.c
+++ b/arch/x86/mm/kmemcheck/shadow.c
@@ -1,7 +1,6 @@
 #include <linux/kmemcheck.h>
 #include <linux/module.h>
 #include <linux/mm.h>
-#include <linux/module.h>
 
 #include <asm/page.h>
 #include <asm/pgtable.h>
diff --git a/arch/x86/mm/mmap.c b/arch/x86/mm/mmap.c
index 165829600566..c8191defc38a 100644
--- a/arch/x86/mm/mmap.c
+++ b/arch/x86/mm/mmap.c
@@ -29,13 +29,26 @@
 #include <linux/random.h>
 #include <linux/limits.h>
 #include <linux/sched.h>
+#include <asm/elf.h>
+
+static unsigned int stack_maxrandom_size(void)
+{
+        unsigned int max = 0;
+        if ((current->flags & PF_RANDOMIZE) &&
+                !(current->personality & ADDR_NO_RANDOMIZE)) {
+                max = ((-1U) & STACK_RND_MASK) << PAGE_SHIFT;
+        }
+
+        return max;
+}
+
 
 /*
  * Top of mmap area (just below the process stack).
  *
- * Leave an at least ~128 MB hole.
+ * Leave an at least ~128 MB hole with possible stack randomization.
  */
-#define MIN_GAP (128*1024*1024)
+#define MIN_GAP (128*1024*1024UL + stack_maxrandom_size())
 #define MAX_GAP (TASK_SIZE/6*5)
 
 /*
diff --git a/arch/x86/mm/pageattr.c b/arch/x86/mm/pageattr.c
index 5866b28eede1..dd38bfbefd1f 100644
--- a/arch/x86/mm/pageattr.c
+++ b/arch/x86/mm/pageattr.c
@@ -12,6 +12,7 @@
 #include <linux/seq_file.h>
 #include <linux/debugfs.h>
 #include <linux/pfn.h>
+#include <linux/percpu.h>
 
 #include <asm/e820.h>
 #include <asm/processor.h>
@@ -687,7 +688,7 @@ static int cpa_process_alias(struct cpa_data *cpa)
 {
         struct cpa_data alias_cpa;
         unsigned long laddr = (unsigned long)__va(cpa->pfn << PAGE_SHIFT);
-        unsigned long vaddr, remapped;
+        unsigned long vaddr;
         int ret;
 
         if (cpa->pfn >= max_pfn_mapped)
@@ -745,24 +746,6 @@ static int cpa_process_alias(struct cpa_data *cpa)
         }
 #endif
 
-        /*
-         * If the PMD page was partially used for per-cpu remapping,
-         * the recycled area needs to be split and modified.  Because
-         * the area is always proper subset of a PMD page
-         * cpa->numpages is guaranteed to be 1 for these areas, so
-         * there's no need to loop over and check for further remaps.
-         */
-        remapped = (unsigned long)pcpu_lpage_remapped((void *)laddr);
-        if (remapped) {
-                WARN_ON(cpa->numpages > 1);
-                alias_cpa = *cpa;
-                alias_cpa.vaddr = &remapped;
-                alias_cpa.flags &= ~(CPA_PAGES_ARRAY | CPA_ARRAY);
-                ret = __change_page_attr_set_clr(&alias_cpa, 0);
-                if (ret)
-                        return ret;
-        }
-
         return 0;
 }
 
@@ -823,6 +806,7 @@ static int change_page_attr_set_clr(unsigned long *addr, int numpages,
 {
         struct cpa_data cpa;
         int ret, cache, checkalias;
+        unsigned long baddr = 0;
 
         /*
          * Check, if we are requested to change a not supported
@@ -854,6 +838,11 @@ static int change_page_attr_set_clr(unsigned long *addr, int numpages,
                          */
                         WARN_ON_ONCE(1);
                 }
+                /*
+                 * Save address for cache flush. *addr is modified in the call
+                 * to __change_page_attr_set_clr() below.
+                 */
+                baddr = *addr;
         }
 
         /* Must avoid aliasing mappings in the highmem code */
@@ -901,7 +890,7 @@ static int change_page_attr_set_clr(unsigned long *addr, int numpages,
                         cpa_flush_array(addr, numpages, cache,
                                         cpa.flags, pages);
                 } else
-                        cpa_flush_range(*addr, numpages, cache);
+                        cpa_flush_range(baddr, numpages, cache);
         } else
                 cpa_flush_all(cache);
 
diff --git a/arch/x86/mm/pat.c b/arch/x86/mm/pat.c
index 352aa9e927e2..7257cf3decf9 100644
--- a/arch/x86/mm/pat.c
+++ b/arch/x86/mm/pat.c
@@ -15,6 +15,7 @@
 #include <linux/gfp.h>
 #include <linux/mm.h>
 #include <linux/fs.h>
+#include <linux/rbtree.h>
 
 #include <asm/cacheflush.h>
 #include <asm/processor.h>
@@ -148,11 +149,10 @@ static char *cattr_name(unsigned long flags)
  * areas). All the aliases have the same cache attributes of course.
  * Zero attributes are represented as holes.
  *
- * Currently the data structure is a list because the number of mappings
- * are expected to be relatively small. If this should be a problem
- * it could be changed to a rbtree or similar.
+ * The data structure is a list that is also organized as an rbtree
+ * sorted on the start address of memtype range.
  *
- * memtype_lock protects the whole list.
+ * memtype_lock protects both the linear list and rbtree.
  */
 
 struct memtype {
@@ -160,11 +160,53 @@ struct memtype {
         u64                     end;
         unsigned long           type;
         struct list_head        nd;
+        struct rb_node          rb;
 };
 
+static struct rb_root memtype_rbroot = RB_ROOT;
 static LIST_HEAD(memtype_list);
 static DEFINE_SPINLOCK(memtype_lock);   /* protects memtype list */
 
+static struct memtype *memtype_rb_search(struct rb_root *root, u64 start)
+{
+        struct rb_node *node = root->rb_node;
+        struct memtype *last_lower = NULL;
+
+        while (node) {
+                struct memtype *data = container_of(node, struct memtype, rb);
+
+                if (data->start < start) {
+                        last_lower = data;
+                        node = node->rb_right;
+                } else if (data->start > start) {
+                        node = node->rb_left;
+                } else
+                        return data;
+        }
+
+        /* Will return NULL if there is no entry with its start <= start */
+        return last_lower;
+}
+
+static void memtype_rb_insert(struct rb_root *root, struct memtype *data)
+{
+        struct rb_node **new = &(root->rb_node);
+        struct rb_node *parent = NULL;
+
+        while (*new) {
+                struct memtype *this = container_of(*new, struct memtype, rb);
+
+                parent = *new;
+                if (data->start <= this->start)
+                        new = &((*new)->rb_left);
+                else if (data->start > this->start)
+                        new = &((*new)->rb_right);
+        }
+
+        rb_link_node(&data->rb, parent, new);
+        rb_insert_color(&data->rb, root);
+}
+
 /*
  * Does intersection of PAT memory type and MTRR memory type and returns
  * the resulting memory type as PAT understands it.
@@ -218,9 +260,6 @@ chk_conflict(struct memtype *new, struct memtype *entry, unsigned long *type)
         return -EBUSY;
 }
 
-static struct memtype *cached_entry;
-static u64 cached_start;
-
 static int pat_pagerange_is_ram(unsigned long start, unsigned long end)
 {
         int ram_page = 0, not_rampage = 0;
@@ -249,63 +288,61 @@ static int pat_pagerange_is_ram(unsigned long start, unsigned long end)
 }
 
 /*
- * For RAM pages, mark the pages as non WB memory type using
- * PageNonWB (PG_arch_1). We allow only one set_memory_uc() or
- * set_memory_wc() on a RAM page at a time before marking it as WB again.
- * This is ok, because only one driver will be owning the page and
- * doing set_memory_*() calls.
+ * For RAM pages, we use page flags to mark the pages with appropriate type.
+ * Here we do two pass:
+ * - Find the memtype of all the pages in the range, look for any conflicts
+ * - In case of no conflicts, set the new memtype for pages in the range
  *
- * For now, we use PageNonWB to track that the RAM page is being mapped
- * as non WB. In future, we will have to use one more flag
- * (or some other mechanism in page_struct) to distinguish between
- * UC and WC mapping.
+ * Caller must hold memtype_lock for atomicity.
  */
 static int reserve_ram_pages_type(u64 start, u64 end, unsigned long req_type,
                                   unsigned long *new_type)
 {
         struct page *page;
-        u64 pfn, end_pfn;
+        u64 pfn;
+
+        if (req_type == _PAGE_CACHE_UC) {
+                /* We do not support strong UC */
+                WARN_ON_ONCE(1);
+                req_type = _PAGE_CACHE_UC_MINUS;
+        }
 
         for (pfn = (start >> PAGE_SHIFT); pfn < (end >> PAGE_SHIFT); ++pfn) {
-                page = pfn_to_page(pfn);
-                if (page_mapped(page) || PageNonWB(page))
-                        goto out;
+                unsigned long type;
 
-                SetPageNonWB(page);
+                page = pfn_to_page(pfn);
+                type = get_page_memtype(page);
+                if (type != -1) {
+                        printk(KERN_INFO "reserve_ram_pages_type failed "
+                                "0x%Lx-0x%Lx, track 0x%lx, req 0x%lx\n",
+                                start, end, type, req_type);
+                        if (new_type)
+                                *new_type = type;
+
+                        return -EBUSY;
+                }
         }
-        return 0;
 
-out:
-        end_pfn = pfn;
-        for (pfn = (start >> PAGE_SHIFT); pfn < end_pfn; ++pfn) {
+        if (new_type)
+                *new_type = req_type;
+
+        for (pfn = (start >> PAGE_SHIFT); pfn < (end >> PAGE_SHIFT); ++pfn) {
                 page = pfn_to_page(pfn);
-                ClearPageNonWB(page);
+                set_page_memtype(page, req_type);
         }
-
-        return -EINVAL;
+        return 0;
 }
 
 static int free_ram_pages_type(u64 start, u64 end)
 {
         struct page *page;
-        u64 pfn, end_pfn;
+        u64 pfn;
 
         for (pfn = (start >> PAGE_SHIFT); pfn < (end >> PAGE_SHIFT); ++pfn) {
                 page = pfn_to_page(pfn);
-                if (page_mapped(page) || !PageNonWB(page))
-                        goto out;
-
-                ClearPageNonWB(page);
+                set_page_memtype(page, -1);
         }
         return 0;
-
-out:
-        end_pfn = pfn;
-        for (pfn = (start >> PAGE_SHIFT); pfn < end_pfn; ++pfn) {
-                page = pfn_to_page(pfn);
-                SetPageNonWB(page);
-        }
-        return -EINVAL;
 }
 
 /*
@@ -339,6 +376,8 @@ int reserve_memtype(u64 start, u64 end, unsigned long req_type,
                 if (new_type) {
                         if (req_type == -1)
                                 *new_type = _PAGE_CACHE_WB;
+                        else if (req_type == _PAGE_CACHE_WC)
+                                *new_type = _PAGE_CACHE_UC_MINUS;
                         else
                                 *new_type = req_type & _PAGE_CACHE_MASK;
                 }
@@ -364,11 +403,16 @@ int reserve_memtype(u64 start, u64 end, unsigned long req_type,
                 *new_type = actual_type;
 
         is_range_ram = pat_pagerange_is_ram(start, end);
-        if (is_range_ram == 1)
-                return reserve_ram_pages_type(start, end, req_type,
-                                              new_type);
-        else if (is_range_ram < 0)
+        if (is_range_ram == 1) {
+
+                spin_lock(&memtype_lock);
+                err = reserve_ram_pages_type(start, end, req_type, new_type);
+                spin_unlock(&memtype_lock);
+
+                return err;
+        } else if (is_range_ram < 0) {
                 return -EINVAL;
+        }
 
         new  = kmalloc(sizeof(struct memtype), GFP_KERNEL);
         if (!new)
@@ -380,17 +424,11 @@ int reserve_memtype(u64 start, u64 end, unsigned long req_type,
 
         spin_lock(&memtype_lock);
 
-        if (cached_entry && start >= cached_start)
-                entry = cached_entry;
-        else
-                entry = list_entry(&memtype_list, struct memtype, nd);
-
         /* Search for existing mapping that overlaps the current range */
         where = NULL;
-        list_for_each_entry_continue(entry, &memtype_list, nd) {
+        list_for_each_entry(entry, &memtype_list, nd) {
                 if (end <= entry->start) {
                         where = entry->nd.prev;
-                        cached_entry = list_entry(where, struct memtype, nd);
                         break;
                 } else if (start <= entry->start) { /* end > entry->start */
                         err = chk_conflict(new, entry, new_type);
@@ -398,8 +436,6 @@ int reserve_memtype(u64 start, u64 end, unsigned long req_type,
                                 dprintk("Overlap at 0x%Lx-0x%Lx\n",
                                         entry->start, entry->end);
                                 where = entry->nd.prev;
-                                cached_entry = list_entry(where,
-                                                        struct memtype, nd);
                         }
                         break;
                 } else if (start < entry->end) { /* start > entry->start */
@@ -407,8 +443,6 @@ int reserve_memtype(u64 start, u64 end, unsigned long req_type,
                         if (!err) {
                                 dprintk("Overlap at 0x%Lx-0x%Lx\n",
                                         entry->start, entry->end);
-                                cached_entry = list_entry(entry->nd.prev,
-                                                        struct memtype, nd);
 
                                 /*
                                  * Move to right position in the linked
@@ -436,13 +470,13 @@ int reserve_memtype(u64 start, u64 end, unsigned long req_type,
                 return err;
         }
 
-        cached_start = start;
-
         if (where)
                 list_add(&new->nd, where);
         else
                 list_add_tail(&new->nd, &memtype_list);
 
+        memtype_rb_insert(&memtype_rbroot, new);
+
         spin_unlock(&memtype_lock);
 
         dprintk("reserve_memtype added 0x%Lx-0x%Lx, track %s, req %s, ret %s\n",
@@ -454,7 +488,7 @@ int reserve_memtype(u64 start, u64 end, unsigned long req_type,
 
 int free_memtype(u64 start, u64 end)
 {
-        struct memtype *entry;
+        struct memtype *entry, *saved_entry;
         int err = -EINVAL;
         int is_range_ram;
 
@@ -466,23 +500,58 @@ int free_memtype(u64 start, u64 end)
                 return 0;
 
         is_range_ram = pat_pagerange_is_ram(start, end);
-        if (is_range_ram == 1)
-                return free_ram_pages_type(start, end);
-        else if (is_range_ram < 0)
+        if (is_range_ram == 1) {
+
+                spin_lock(&memtype_lock);
+                err = free_ram_pages_type(start, end);
+                spin_unlock(&memtype_lock);
+
+                return err;
+        } else if (is_range_ram < 0) {
                 return -EINVAL;
+        }
 
         spin_lock(&memtype_lock);
-        list_for_each_entry(entry, &memtype_list, nd) {
+
+        entry = memtype_rb_search(&memtype_rbroot, start);
+        if (unlikely(entry == NULL))
+                goto unlock_ret;
+
+        /*
+         * Saved entry points to an entry with start same or less than what
+         * we searched for. Now go through the list in both directions to look
+         * for the entry that matches with both start and end, with list stored
+         * in sorted start address
+         */
+        saved_entry = entry;
+        list_for_each_entry_from(entry, &memtype_list, nd) {
                 if (entry->start == start && entry->end == end) {
-                        if (cached_entry == entry || cached_start == start)
-                                cached_entry = NULL;
+                        rb_erase(&entry->rb, &memtype_rbroot);
+                        list_del(&entry->nd);
+                        kfree(entry);
+                        err = 0;
+                        break;
+                } else if (entry->start > start) {
+                        break;
+                }
+        }
+
+        if (!err)
+                goto unlock_ret;
 
+        entry = saved_entry;
+        list_for_each_entry_reverse(entry, &memtype_list, nd) {
+                if (entry->start == start && entry->end == end) {
+                        rb_erase(&entry->rb, &memtype_rbroot);
                         list_del(&entry->nd);
                         kfree(entry);
                         err = 0;
                         break;
+                } else if (entry->start < start) {
+                        break;
                 }
         }
+unlock_ret:
         spin_unlock(&memtype_lock);
 
         if (err) {
@@ -496,6 +565,101 @@ int free_memtype(u64 start, u64 end)
 }
 
 
+/**
+ * lookup_memtype - Looksup the memory type for a physical address
+ * @paddr: physical address of which memory type needs to be looked up
+ *
+ * Only to be called when PAT is enabled
+ *
+ * Returns _PAGE_CACHE_WB, _PAGE_CACHE_WC, _PAGE_CACHE_UC_MINUS or
+ * _PAGE_CACHE_UC
+ */
+static unsigned long lookup_memtype(u64 paddr)
+{
+        int rettype = _PAGE_CACHE_WB;
+        struct memtype *entry;
+
+        if (is_ISA_range(paddr, paddr + PAGE_SIZE - 1))
+                return rettype;
+
+        if (pat_pagerange_is_ram(paddr, paddr + PAGE_SIZE)) {
+                struct page *page;
+                spin_lock(&memtype_lock);
+                page = pfn_to_page(paddr >> PAGE_SHIFT);
+                rettype = get_page_memtype(page);
+                spin_unlock(&memtype_lock);
+                /*
+                 * -1 from get_page_memtype() implies RAM page is in its
+                 * default state and not reserved, and hence of type WB
+                 */
+                if (rettype == -1)
+                        rettype = _PAGE_CACHE_WB;
+
+                return rettype;
+        }
+
+        spin_lock(&memtype_lock);
+
+        entry = memtype_rb_search(&memtype_rbroot, paddr);
+        if (entry != NULL)
+                rettype = entry->type;
+        else
+                rettype = _PAGE_CACHE_UC_MINUS;
+
+        spin_unlock(&memtype_lock);
+        return rettype;
+}
+
+/**
+ * io_reserve_memtype - Request a memory type mapping for a region of memory
+ * @start: start (physical address) of the region
+ * @end: end (physical address) of the region
+ * @type: A pointer to memtype, with requested type. On success, requested
+ * or any other compatible type that was available for the region is returned
+ *
+ * On success, returns 0
+ * On failure, returns non-zero
+ */
+int io_reserve_memtype(resource_size_t start, resource_size_t end,
+                        unsigned long *type)
+{
+        resource_size_t size = end - start;
+        unsigned long req_type = *type;
+        unsigned long new_type;
+        int ret;
+
+        WARN_ON_ONCE(iomem_map_sanity_check(start, size));
+
+        ret = reserve_memtype(start, end, req_type, &new_type);
+        if (ret)
+                goto out_err;
+
+        if (!is_new_memtype_allowed(start, size, req_type, new_type))
+                goto out_free;
+
+        if (kernel_map_sync_memtype(start, size, new_type) < 0)
+                goto out_free;
+
+        *type = new_type;
+        return 0;
+
+out_free:
+        free_memtype(start, end);
+        ret = -EBUSY;
+out_err:
+        return ret;
+}
+
+/**
+ * io_free_memtype - Release a memory type mapping for a region of memory
+ * @start: start (physical address) of the region
+ * @end: end (physical address) of the region
+ */
+void io_free_memtype(resource_size_t start, resource_size_t end)
+{
+        free_memtype(start, end);
+}
+
 pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
                                 unsigned long size, pgprot_t vma_prot)
 {
@@ -577,7 +741,7 @@ int kernel_map_sync_memtype(u64 base, unsigned long size, unsigned long flags)
 {
         unsigned long id_sz;
 
-        if (!pat_enabled || base >= __pa(high_memory))
+        if (base >= __pa(high_memory))
                 return 0;
 
         id_sz = (__pa(high_memory) < base + size) ?
@@ -612,11 +776,29 @@ static int reserve_pfn_range(u64 paddr, unsigned long size, pgprot_t *vma_prot,
         is_ram = pat_pagerange_is_ram(paddr, paddr + size);
 
         /*
-         * reserve_pfn_range() doesn't support RAM pages. Maintain the current
-         * behavior with RAM pages by returning success.
+         * reserve_pfn_range() for RAM pages. We do not refcount to keep
+         * track of number of mappings of RAM pages. We can assert that
+         * the type requested matches the type of first page in the range.
          */
-        if (is_ram != 0)
+        if (is_ram) {
+                if (!pat_enabled)
+                        return 0;
+
+                flags = lookup_memtype(paddr);
+                if (want_flags != flags) {
+                        printk(KERN_WARNING
+                        "%s:%d map pfn RAM range req %s for %Lx-%Lx, got %s\n",
+                                current->comm, current->pid,
+                                cattr_name(want_flags),
+                                (unsigned long long)paddr,
+                                (unsigned long long)(paddr + size),
+                                cattr_name(flags));
+                        *vma_prot = __pgprot((pgprot_val(*vma_prot) &
+                                              (~_PAGE_CACHE_MASK)) |
+                                             flags);
+                }
                 return 0;
+        }
 
         ret = reserve_memtype(paddr, paddr + size, want_flags, &flags);
         if (ret)
@@ -678,14 +860,6 @@ int track_pfn_vma_copy(struct vm_area_struct *vma)
         unsigned long vma_size = vma->vm_end - vma->vm_start;
         pgprot_t pgprot;
 
-        if (!pat_enabled)
-                return 0;
-
-        /*
-         * For now, only handle remap_pfn_range() vmas where
-         * is_linear_pfn_mapping() == TRUE. Handling of
-         * vm_insert_pfn() is TBD.
-         */
         if (is_linear_pfn_mapping(vma)) {
                 /*
                  * reserve the whole chunk covered by vma. We need the
@@ -713,23 +887,24 @@ int track_pfn_vma_copy(struct vm_area_struct *vma)
 int track_pfn_vma_new(struct vm_area_struct *vma, pgprot_t *prot,
                         unsigned long pfn, unsigned long size)
 {
+        unsigned long flags;
         resource_size_t paddr;
         unsigned long vma_size = vma->vm_end - vma->vm_start;
 
-        if (!pat_enabled)
-                return 0;
-
-        /*
-         * For now, only handle remap_pfn_range() vmas where
-         * is_linear_pfn_mapping() == TRUE. Handling of
-         * vm_insert_pfn() is TBD.
-         */
         if (is_linear_pfn_mapping(vma)) {
                 /* reserve the whole chunk starting from vm_pgoff */
                 paddr = (resource_size_t)vma->vm_pgoff << PAGE_SHIFT;
                 return reserve_pfn_range(paddr, vma_size, prot, 0);
         }
 
+        if (!pat_enabled)
+                return 0;
+
+        /* for vm_insert_pfn and friends, we set prot based on lookup */
+        flags = lookup_memtype(pfn << PAGE_SHIFT);
+        *prot = __pgprot((pgprot_val(vma->vm_page_prot) & (~_PAGE_CACHE_MASK)) |
+                         flags);
+
         return 0;
 }
 
@@ -744,14 +919,6 @@ void untrack_pfn_vma(struct vm_area_struct *vma, unsigned long pfn,
         resource_size_t paddr;
         unsigned long vma_size = vma->vm_end - vma->vm_start;
 
-        if (!pat_enabled)
-                return;
-
-        /*
-         * For now, only handle remap_pfn_range() vmas where
-         * is_linear_pfn_mapping() == TRUE. Handling of
-         * vm_insert_pfn() is TBD.
-         */
         if (is_linear_pfn_mapping(vma)) {
                 /* free the whole chunk starting from vm_pgoff */
                 paddr = (resource_size_t)vma->vm_pgoff << PAGE_SHIFT;
@@ -827,7 +994,7 @@ static int memtype_seq_show(struct seq_file *seq, void *v)
         return 0;
 }
 
-static struct seq_operations memtype_seq_ops = {
+static const struct seq_operations memtype_seq_ops = {
         .start = memtype_seq_start,
         .next  = memtype_seq_next,
         .stop  = memtype_seq_stop,
diff --git a/arch/x86/mm/physaddr.c b/arch/x86/mm/physaddr.c
new file mode 100644
index 000000000000..d2e2735327b4
--- /dev/null
+++ b/arch/x86/mm/physaddr.c
@@ -0,0 +1,70 @@
+#include <linux/mmdebug.h>
+#include <linux/module.h>
+#include <linux/mm.h>
+
+#include <asm/page.h>
+
+#include "physaddr.h"
+
+#ifdef CONFIG_X86_64
+
+unsigned long __phys_addr(unsigned long x)
+{
+        if (x >= __START_KERNEL_map) {
+                x -= __START_KERNEL_map;
+                VIRTUAL_BUG_ON(x >= KERNEL_IMAGE_SIZE);
+                x += phys_base;
+        } else {
+                VIRTUAL_BUG_ON(x < PAGE_OFFSET);
+                x -= PAGE_OFFSET;
+                VIRTUAL_BUG_ON(!phys_addr_valid(x));
+        }
+        return x;
+}
+EXPORT_SYMBOL(__phys_addr);
+
+bool __virt_addr_valid(unsigned long x)
+{
+        if (x >= __START_KERNEL_map) {
+                x -= __START_KERNEL_map;
+                if (x >= KERNEL_IMAGE_SIZE)
+                        return false;
+                x += phys_base;
+        } else {
+                if (x < PAGE_OFFSET)
+                        return false;
+                x -= PAGE_OFFSET;
+                if (!phys_addr_valid(x))
+                        return false;
+        }
+
+        return pfn_valid(x >> PAGE_SHIFT);
+}
+EXPORT_SYMBOL(__virt_addr_valid);
+
+#else
+
+#ifdef CONFIG_DEBUG_VIRTUAL
+unsigned long __phys_addr(unsigned long x)
+{
+        /* VMALLOC_* aren't constants  */
+        VIRTUAL_BUG_ON(x < PAGE_OFFSET);
+        VIRTUAL_BUG_ON(__vmalloc_start_set && is_vmalloc_addr((void *) x));
+        return x - PAGE_OFFSET;
+}
+EXPORT_SYMBOL(__phys_addr);
+#endif
+
+bool __virt_addr_valid(unsigned long x)
+{
+        if (x < PAGE_OFFSET)
+                return false;
+        if (__vmalloc_start_set && is_vmalloc_addr((void *) x))
+                return false;
+        if (x >= FIXADDR_START)
+                return false;
+        return pfn_valid((x - PAGE_OFFSET) >> PAGE_SHIFT);
+}
+EXPORT_SYMBOL(__virt_addr_valid);
+
+#endif  /* CONFIG_X86_64 */
diff --git a/arch/x86/mm/physaddr.h b/arch/x86/mm/physaddr.h
new file mode 100644
index 000000000000..a3cd5a0c97b3
--- /dev/null
+++ b/arch/x86/mm/physaddr.h
@@ -0,0 +1,10 @@
+#include <asm/processor.h>
+
+static inline int phys_addr_valid(resource_size_t addr)
+{
+#ifdef CONFIG_PHYS_ADDR_T_64BIT
+        return !(addr >> boot_cpu_data.x86_phys_bits);
+#else
+        return 1;
+#endif
+}
diff --git a/arch/x86/mm/srat_32.c b/arch/x86/mm/srat_32.c
index 29a0e37114f8..6f8aa33031c7 100644
--- a/arch/x86/mm/srat_32.c
+++ b/arch/x86/mm/srat_32.c
@@ -215,7 +215,7 @@ int __init get_memcfg_from_srat(void)
                 goto out_fail;
 
         if (num_memory_chunks == 0) {
-                printk(KERN_WARNING
+                printk(KERN_DEBUG
                          "could not find any ACPI SRAT memory areas.\n");
                 goto out_fail;
         }
@@ -277,7 +277,7 @@ int __init get_memcfg_from_srat(void)
         }
         return 1;
 out_fail:
-        printk(KERN_ERR "failed to get NUMA memory information from SRAT"
+        printk(KERN_DEBUG "failed to get NUMA memory information from SRAT"
                         " table\n");
         return 0;
 }
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index c814e144a3f0..36fe08eeb5c3 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -59,7 +59,8 @@ void leave_mm(int cpu)
 {
         if (percpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
                 BUG();
-        cpu_clear(cpu, percpu_read(cpu_tlbstate.active_mm)->cpu_vm_mask);
+        cpumask_clear_cpu(cpu,
+                          mm_cpumask(percpu_read(cpu_tlbstate.active_mm)));
         load_cr3(swapper_pg_dir);
 }
 EXPORT_SYMBOL_GPL(leave_mm);
@@ -234,8 +235,8 @@ void flush_tlb_current_task(void)
         preempt_disable();
 
         local_flush_tlb();
-        if (cpumask_any_but(&mm->cpu_vm_mask, smp_processor_id()) < nr_cpu_ids)
-                flush_tlb_others(&mm->cpu_vm_mask, mm, TLB_FLUSH_ALL);
+        if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
+                flush_tlb_others(mm_cpumask(mm), mm, TLB_FLUSH_ALL);
         preempt_enable();
 }
 
@@ -249,8 +250,8 @@ void flush_tlb_mm(struct mm_struct *mm)
                 else
                         leave_mm(smp_processor_id());
         }
-        if (cpumask_any_but(&mm->cpu_vm_mask, smp_processor_id()) < nr_cpu_ids)
-                flush_tlb_others(&mm->cpu_vm_mask, mm, TLB_FLUSH_ALL);
+        if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
+                flush_tlb_others(mm_cpumask(mm), mm, TLB_FLUSH_ALL);
 
         preempt_enable();
 }
@@ -268,8 +269,8 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long va)
                         leave_mm(smp_processor_id());
         }
 
-        if (cpumask_any_but(&mm->cpu_vm_mask, smp_processor_id()) < nr_cpu_ids)
-                flush_tlb_others(&mm->cpu_vm_mask, mm, va);
+        if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
+                flush_tlb_others(mm_cpumask(mm), mm, va);
 
         preempt_enable();
 }
diff --git a/arch/x86/oprofile/nmi_int.c b/arch/x86/oprofile/nmi_int.c
index 89b9a5cd63da..cb88b1a0bd5f 100644
--- a/arch/x86/oprofile/nmi_int.c
+++ b/arch/x86/oprofile/nmi_int.c
@@ -1,11 +1,14 @@
 /**
  * @file nmi_int.c
  *
- * @remark Copyright 2002-2008 OProfile authors
+ * @remark Copyright 2002-2009 OProfile authors
  * @remark Read the file COPYING
  *
  * @author John Levon <levon@movementarian.org>
  * @author Robert Richter <robert.richter@amd.com>
+ * @author Barry Kasindorf <barry.kasindorf@amd.com>
+ * @author Jason Yeh <jason.yeh@amd.com>
+ * @author Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
  */
 
 #include <linux/init.h>
@@ -24,13 +27,35 @@
 #include "op_counter.h"
 #include "op_x86_model.h"
 
-static struct op_x86_model_spec const *model;
+static struct op_x86_model_spec *model;
 static DEFINE_PER_CPU(struct op_msrs, cpu_msrs);
 static DEFINE_PER_CPU(unsigned long, saved_lvtpc);
 
 /* 0 == registered but off, 1 == registered and on */
 static int nmi_enabled = 0;
 
+struct op_counter_config counter_config[OP_MAX_COUNTER];
+
+/* common functions */
+
+u64 op_x86_get_ctrl(struct op_x86_model_spec const *model,
+                    struct op_counter_config *counter_config)
+{
+        u64 val = 0;
+        u16 event = (u16)counter_config->event;
+
+        val |= ARCH_PERFMON_EVENTSEL_INT;
+        val |= counter_config->user ? ARCH_PERFMON_EVENTSEL_USR : 0;
+        val |= counter_config->kernel ? ARCH_PERFMON_EVENTSEL_OS : 0;
+        val |= (counter_config->unit_mask & 0xFF) << 8;
+        event &= model->event_mask ? model->event_mask : 0xFF;
+        val |= event & 0xFF;
+        val |= (event & 0x0F00) << 24;
+
+        return val;
+}
+
+
 static int profile_exceptions_notify(struct notifier_block *self,
                                      unsigned long val, void *data)
 {
@@ -52,36 +77,214 @@ static int profile_exceptions_notify(struct notifier_block *self,
 
 static void nmi_cpu_save_registers(struct op_msrs *msrs)
 {
-        unsigned int const nr_ctrs = model->num_counters;
-        unsigned int const nr_ctrls = model->num_controls;
         struct op_msr *counters = msrs->counters;
         struct op_msr *controls = msrs->controls;
         unsigned int i;
 
-        for (i = 0; i < nr_ctrs; ++i) {
-                if (counters[i].addr) {
-                        rdmsr(counters[i].addr,
-                                counters[i].saved.low,
-                                counters[i].saved.high);
-                }
+        for (i = 0; i < model->num_counters; ++i) {
+                if (counters[i].addr)
+                        rdmsrl(counters[i].addr, counters[i].saved);
+        }
+
+        for (i = 0; i < model->num_controls; ++i) {
+                if (controls[i].addr)
+                        rdmsrl(controls[i].addr, controls[i].saved);
+        }
+}
+
+static void nmi_cpu_start(void *dummy)
+{
+        struct op_msrs const *msrs = &__get_cpu_var(cpu_msrs);
+        model->start(msrs);
+}
+
+static int nmi_start(void)
+{
+        on_each_cpu(nmi_cpu_start, NULL, 1);
+        return 0;
+}
+
+static void nmi_cpu_stop(void *dummy)
+{
+        struct op_msrs const *msrs = &__get_cpu_var(cpu_msrs);
+        model->stop(msrs);
+}
+
+static void nmi_stop(void)
+{
+        on_each_cpu(nmi_cpu_stop, NULL, 1);
+}
+
+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
+
+static DEFINE_PER_CPU(int, switch_index);
+
+static inline int has_mux(void)
+{
+        return !!model->switch_ctrl;
+}
+
+inline int op_x86_phys_to_virt(int phys)
+{
+        return __get_cpu_var(switch_index) + phys;
+}
+
+inline int op_x86_virt_to_phys(int virt)
+{
+        return virt % model->num_counters;
+}
+
+static void nmi_shutdown_mux(void)
+{
+        int i;
+
+        if (!has_mux())
+                return;
+
+        for_each_possible_cpu(i) {
+                kfree(per_cpu(cpu_msrs, i).multiplex);
+                per_cpu(cpu_msrs, i).multiplex = NULL;
+                per_cpu(switch_index, i) = 0;
         }
+}
+
+static int nmi_setup_mux(void)
+{
+        size_t multiplex_size =
+                sizeof(struct op_msr) * model->num_virt_counters;
+        int i;
+
+        if (!has_mux())
+                return 1;
+
+        for_each_possible_cpu(i) {
+                per_cpu(cpu_msrs, i).multiplex =
+                        kmalloc(multiplex_size, GFP_KERNEL);
+                if (!per_cpu(cpu_msrs, i).multiplex)
+                        return 0;
+        }
+
+        return 1;
+}
+
+static void nmi_cpu_setup_mux(int cpu, struct op_msrs const * const msrs)
+{
+        int i;
+        struct op_msr *multiplex = msrs->multiplex;
+
+        if (!has_mux())
+                return;
 
-        for (i = 0; i < nr_ctrls; ++i) {
-                if (controls[i].addr) {
-                        rdmsr(controls[i].addr,
-                                controls[i].saved.low,
-                                controls[i].saved.high);
+        for (i = 0; i < model->num_virt_counters; ++i) {
+                if (counter_config[i].enabled) {
+                        multiplex[i].saved = -(u64)counter_config[i].count;
+                } else {
+                        multiplex[i].addr  = 0;
+                        multiplex[i].saved = 0;
                 }
         }
+
+        per_cpu(switch_index, cpu) = 0;
+}
+
+static void nmi_cpu_save_mpx_registers(struct op_msrs *msrs)
+{
+        struct op_msr *multiplex = msrs->multiplex;
+        int i;
+
+        for (i = 0; i < model->num_counters; ++i) {
+                int virt = op_x86_phys_to_virt(i);
+                if (multiplex[virt].addr)
+                        rdmsrl(multiplex[virt].addr, multiplex[virt].saved);
+        }
+}
+
+static void nmi_cpu_restore_mpx_registers(struct op_msrs *msrs)
+{
+        struct op_msr *multiplex = msrs->multiplex;
+        int i;
+
+        for (i = 0; i < model->num_counters; ++i) {
+                int virt = op_x86_phys_to_virt(i);
+                if (multiplex[virt].addr)
+                        wrmsrl(multiplex[virt].addr, multiplex[virt].saved);
+        }
 }
 
-static void nmi_save_registers(void *dummy)
+static void nmi_cpu_switch(void *dummy)
 {
         int cpu = smp_processor_id();
+        int si = per_cpu(switch_index, cpu);
         struct op_msrs *msrs = &per_cpu(cpu_msrs, cpu);
-        nmi_cpu_save_registers(msrs);
+
+        nmi_cpu_stop(NULL);
+        nmi_cpu_save_mpx_registers(msrs);
+
+        /* move to next set */
+        si += model->num_counters;
+        if ((si > model->num_virt_counters) || (counter_config[si].count == 0))
+                per_cpu(switch_index, cpu) = 0;
+        else
+                per_cpu(switch_index, cpu) = si;
+
+        model->switch_ctrl(model, msrs);
+        nmi_cpu_restore_mpx_registers(msrs);
+
+        nmi_cpu_start(NULL);
+}
+
+
+/*
+ * Quick check to see if multiplexing is necessary.
+ * The check should be sufficient since counters are used
+ * in ordre.
+ */
+static int nmi_multiplex_on(void)
+{
+        return counter_config[model->num_counters].count ? 0 : -EINVAL;
+}
+
+static int nmi_switch_event(void)
+{
+        if (!has_mux())
+                return -ENOSYS;         /* not implemented */
+        if (nmi_multiplex_on() < 0)
+                return -EINVAL;         /* not necessary */
+
+        on_each_cpu(nmi_cpu_switch, NULL, 1);
+
+        return 0;
+}
+
+static inline void mux_init(struct oprofile_operations *ops)
+{
+        if (has_mux())
+                ops->switch_events = nmi_switch_event;
+}
+
+static void mux_clone(int cpu)
+{
+        if (!has_mux())
+                return;
+
+        memcpy(per_cpu(cpu_msrs, cpu).multiplex,
+               per_cpu(cpu_msrs, 0).multiplex,
+               sizeof(struct op_msr) * model->num_virt_counters);
 }
 
+#else
+
+inline int op_x86_phys_to_virt(int phys) { return phys; }
+inline int op_x86_virt_to_phys(int virt) { return virt; }
+static inline void nmi_shutdown_mux(void) { }
+static inline int nmi_setup_mux(void) { return 1; }
+static inline void
+nmi_cpu_setup_mux(int cpu, struct op_msrs const * const msrs) { }
+static inline void mux_init(struct oprofile_operations *ops) { }
+static void mux_clone(int cpu) { }
+
+#endif
+
 static void free_msrs(void)
 {
         int i;
@@ -95,38 +298,32 @@ static void free_msrs(void)
 
 static int allocate_msrs(void)
 {
-        int success = 1;
         size_t controls_size = sizeof(struct op_msr) * model->num_controls;
         size_t counters_size = sizeof(struct op_msr) * model->num_counters;
 
         int i;
         for_each_possible_cpu(i) {
                 per_cpu(cpu_msrs, i).counters = kmalloc(counters_size,
-                                                                GFP_KERNEL);
-                if (!per_cpu(cpu_msrs, i).counters) {
-                        success = 0;
-                        break;
-                }
+                                                        GFP_KERNEL);
+                if (!per_cpu(cpu_msrs, i).counters)
+                        return 0;
                 per_cpu(cpu_msrs, i).controls = kmalloc(controls_size,
-                                                                GFP_KERNEL);
-                if (!per_cpu(cpu_msrs, i).controls) {
-                        success = 0;
-                        break;
-                }
+                                                        GFP_KERNEL);
+                if (!per_cpu(cpu_msrs, i).controls)
+                        return 0;
         }
 
-        if (!success)
-                free_msrs();
-
-        return success;
+        return 1;
 }
 
 static void nmi_cpu_setup(void *dummy)
 {
         int cpu = smp_processor_id();
         struct op_msrs *msrs = &per_cpu(cpu_msrs, cpu);
+        nmi_cpu_save_registers(msrs);
         spin_lock(&oprofilefs_lock);
-        model->setup_ctrs(msrs);
+        model->setup_ctrs(model, msrs);
+        nmi_cpu_setup_mux(cpu, msrs);
         spin_unlock(&oprofilefs_lock);
         per_cpu(saved_lvtpc, cpu) = apic_read(APIC_LVTPC);
         apic_write(APIC_LVTPC, APIC_DM_NMI);
@@ -144,11 +341,15 @@ static int nmi_setup(void)
         int cpu;
 
         if (!allocate_msrs())
-                return -ENOMEM;
+                err = -ENOMEM;
+        else if (!nmi_setup_mux())
+                err = -ENOMEM;
+        else
+                err = register_die_notifier(&profile_exceptions_nb);
 
-        err = register_die_notifier(&profile_exceptions_nb);
         if (err) {
                 free_msrs();
+                nmi_shutdown_mux();
                 return err;
         }
 
@@ -159,45 +360,38 @@ static int nmi_setup(void)
         /* Assume saved/restored counters are the same on all CPUs */
         model->fill_in_addresses(&per_cpu(cpu_msrs, 0));
         for_each_possible_cpu(cpu) {
-                if (cpu != 0) {
-                        memcpy(per_cpu(cpu_msrs, cpu).counters,
-                                per_cpu(cpu_msrs, 0).counters,
-                                sizeof(struct op_msr) * model->num_counters);
-
-                        memcpy(per_cpu(cpu_msrs, cpu).controls,
-                                per_cpu(cpu_msrs, 0).controls,
-                                sizeof(struct op_msr) * model->num_controls);
-                }
+                if (!cpu)
+                        continue;
+
+                memcpy(per_cpu(cpu_msrs, cpu).counters,
+                       per_cpu(cpu_msrs, 0).counters,
+                       sizeof(struct op_msr) * model->num_counters);
+
+                memcpy(per_cpu(cpu_msrs, cpu).controls,
+                       per_cpu(cpu_msrs, 0).controls,
+                       sizeof(struct op_msr) * model->num_controls);
 
+                mux_clone(cpu);
         }
-        on_each_cpu(nmi_save_registers, NULL, 1);
         on_each_cpu(nmi_cpu_setup, NULL, 1);
         nmi_enabled = 1;
         return 0;
 }
 
-static void nmi_restore_registers(struct op_msrs *msrs)
+static void nmi_cpu_restore_registers(struct op_msrs *msrs)
 {
-        unsigned int const nr_ctrs = model->num_counters;
-        unsigned int const nr_ctrls = model->num_controls;
         struct op_msr *counters = msrs->counters;
         struct op_msr *controls = msrs->controls;
         unsigned int i;
 
-        for (i = 0; i < nr_ctrls; ++i) {
-                if (controls[i].addr) {
-                        wrmsr(controls[i].addr,
-                                controls[i].saved.low,
-                                controls[i].saved.high);
-                }
+        for (i = 0; i < model->num_controls; ++i) {
+                if (controls[i].addr)
+                        wrmsrl(controls[i].addr, controls[i].saved);
         }
 
-        for (i = 0; i < nr_ctrs; ++i) {
-                if (counters[i].addr) {
-                        wrmsr(counters[i].addr,
-                                counters[i].saved.low,
-                                counters[i].saved.high);
-                }
+        for (i = 0; i < model->num_counters; ++i) {
+                if (counters[i].addr)
+                        wrmsrl(counters[i].addr, counters[i].saved);
         }
 }
 
@@ -205,7 +399,7 @@ static void nmi_cpu_shutdown(void *dummy)
 {
         unsigned int v;
         int cpu = smp_processor_id();
-        struct op_msrs *msrs = &__get_cpu_var(cpu_msrs);
+        struct op_msrs *msrs = &per_cpu(cpu_msrs, cpu);
 
         /* restoring APIC_LVTPC can trigger an apic error because the delivery
          * mode and vector nr combination can be illegal. That's by design: on
@@ -216,7 +410,7 @@ static void nmi_cpu_shutdown(void *dummy)
         apic_write(APIC_LVTERR, v | APIC_LVT_MASKED);
         apic_write(APIC_LVTPC, per_cpu(saved_lvtpc, cpu));
         apic_write(APIC_LVTERR, v);
-        nmi_restore_registers(msrs);
+        nmi_cpu_restore_registers(msrs);
 }
 
 static void nmi_shutdown(void)
@@ -226,42 +420,18 @@ static void nmi_shutdown(void)
         nmi_enabled = 0;
         on_each_cpu(nmi_cpu_shutdown, NULL, 1);
         unregister_die_notifier(&profile_exceptions_nb);
+        nmi_shutdown_mux();
         msrs = &get_cpu_var(cpu_msrs);
         model->shutdown(msrs);
         free_msrs();
         put_cpu_var(cpu_msrs);
 }
 
-static void nmi_cpu_start(void *dummy)
-{
-        struct op_msrs const *msrs = &__get_cpu_var(cpu_msrs);
-        model->start(msrs);
-}
-
-static int nmi_start(void)
-{
-        on_each_cpu(nmi_cpu_start, NULL, 1);
-        return 0;
-}
-
-static void nmi_cpu_stop(void *dummy)
-{
-        struct op_msrs const *msrs = &__get_cpu_var(cpu_msrs);
-        model->stop(msrs);
-}
-
-static void nmi_stop(void)
-{
-        on_each_cpu(nmi_cpu_stop, NULL, 1);
-}
-
-struct op_counter_config counter_config[OP_MAX_COUNTER];
-
 static int nmi_create_files(struct super_block *sb, struct dentry *root)
 {
         unsigned int i;
 
-        for (i = 0; i < model->num_counters; ++i) {
+        for (i = 0; i < model->num_virt_counters; ++i) {
                 struct dentry *dir;
                 char buf[4];
 
@@ -270,7 +440,7 @@ static int nmi_create_files(struct super_block *sb, struct dentry *root)
                  * NOTE:  assumes 1:1 mapping here (that counters are organized
                  *        sequentially in their struct assignment).
                  */
-                if (unlikely(!avail_to_resrv_perfctr_nmi_bit(i)))
+                if (!avail_to_resrv_perfctr_nmi_bit(op_x86_virt_to_phys(i)))
                         continue;
 
                 snprintf(buf,  sizeof(buf), "%d", i);
@@ -402,6 +572,7 @@ module_param_call(cpu_type, force_cpu_type, NULL, NULL, 0);
 static int __init ppro_init(char **cpu_type)
 {
         __u8 cpu_model = boot_cpu_data.x86_model;
+        struct op_x86_model_spec *spec = &op_ppro_spec; /* default */
 
         if (force_arch_perfmon && cpu_has_arch_perfmon)
                 return 0;
@@ -428,7 +599,7 @@ static int __init ppro_init(char **cpu_type)
                 *cpu_type = "i386/core_2";
                 break;
         case 26:
-                arch_perfmon_setup_counters();
+                spec = &op_arch_perfmon_spec;
                 *cpu_type = "i386/core_i7";
                 break;
         case 28:
@@ -439,17 +610,7 @@ static int __init ppro_init(char **cpu_type)
                 return 0;
         }
 
-        model = &op_ppro_spec;
-        return 1;
-}
-
-static int __init arch_perfmon_init(char **cpu_type)
-{
-        if (!cpu_has_arch_perfmon)
-                return 0;
-        *cpu_type = "i386/arch_perfmon";
-        model = &op_arch_perfmon_spec;
-        arch_perfmon_setup_counters();
+        model = spec;
         return 1;
 }
 
@@ -471,27 +632,26 @@ int __init op_nmi_init(struct oprofile_operations *ops)
                 /* Needs to be at least an Athlon (or hammer in 32bit mode) */
 
                 switch (family) {
-                default:
-                        return -ENODEV;
                 case 6:
-                        model = &op_amd_spec;
                         cpu_type = "i386/athlon";
                         break;
                 case 0xf:
-                        model = &op_amd_spec;
-                        /* Actually it could be i386/hammer too, but give
-                         user space an consistent name. */
+                        /*
+                         * Actually it could be i386/hammer too, but
+                         * give user space an consistent name.
+                         */
                         cpu_type = "x86-64/hammer";
                         break;
                 case 0x10:
-                        model = &op_amd_spec;
                         cpu_type = "x86-64/family10";
                         break;
                 case 0x11:
-                        model = &op_amd_spec;
                         cpu_type = "x86-64/family11h";
                         break;
+                default:
+                        return -ENODEV;
                 }
+                model = &op_amd_spec;
                 break;
 
         case X86_VENDOR_INTEL:
@@ -510,8 +670,15 @@ int __init op_nmi_init(struct oprofile_operations *ops)
                         break;
                 }
 
-                if (!cpu_type && !arch_perfmon_init(&cpu_type))
+                if (cpu_type)
+                        break;
+
+                if (!cpu_has_arch_perfmon)
                         return -ENODEV;
+
+                /* use arch perfmon as fallback */
+                cpu_type = "i386/arch_perfmon";
+                model = &op_arch_perfmon_spec;
                 break;
 
         default:
@@ -522,18 +689,23 @@ int __init op_nmi_init(struct oprofile_operations *ops)
         register_cpu_notifier(&oprofile_cpu_nb);
 #endif
         /* default values, can be overwritten by model */
-        ops->create_files = nmi_create_files;
-        ops->setup = nmi_setup;
-        ops->shutdown = nmi_shutdown;
-        ops->start = nmi_start;
-        ops->stop = nmi_stop;
-        ops->cpu_type = cpu_type;
+        ops->create_files       = nmi_create_files;
+        ops->setup              = nmi_setup;
+        ops->shutdown           = nmi_shutdown;
+        ops->start              = nmi_start;
+        ops->stop               = nmi_stop;
+        ops->cpu_type           = cpu_type;
 
         if (model->init)
                 ret = model->init(ops);
         if (ret)
                 return ret;
 
+        if (!model->num_virt_counters)
+                model->num_virt_counters = model->num_counters;
+
+        mux_init(ops);
+
         init_sysfs();
         using_nmi = 1;
         printk(KERN_INFO "oprofile: using NMI interrupt.\n");
diff --git a/arch/x86/oprofile/op_counter.h b/arch/x86/oprofile/op_counter.h
index 91b6a116165e..e28398df0df2 100644
--- a/arch/x86/oprofile/op_counter.h
+++ b/arch/x86/oprofile/op_counter.h
@@ -10,7 +10,7 @@
 #ifndef OP_COUNTER_H
 #define OP_COUNTER_H
 
-#define OP_MAX_COUNTER 8
+#define OP_MAX_COUNTER 32
 
 /* Per-perfctr configuration as set via
  * oprofilefs.
diff --git a/arch/x86/oprofile/op_model_amd.c b/arch/x86/oprofile/op_model_amd.c
index 8fdf06e4edf9..39686c29f03a 100644
--- a/arch/x86/oprofile/op_model_amd.c
+++ b/arch/x86/oprofile/op_model_amd.c
@@ -9,12 +9,15 @@
  * @author Philippe Elie
  * @author Graydon Hoare
  * @author Robert Richter <robert.richter@amd.com>
- * @author Barry Kasindorf
+ * @author Barry Kasindorf <barry.kasindorf@amd.com>
+ * @author Jason Yeh <jason.yeh@amd.com>
+ * @author Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
  */
 
 #include <linux/oprofile.h>
 #include <linux/device.h>
 #include <linux/pci.h>
+#include <linux/percpu.h>
 
 #include <asm/ptrace.h>
 #include <asm/msr.h>
@@ -25,43 +28,36 @@
 
 #define NUM_COUNTERS 4
 #define NUM_CONTROLS 4
+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
+#define NUM_VIRT_COUNTERS 32
+#define NUM_VIRT_CONTROLS 32
+#else
+#define NUM_VIRT_COUNTERS NUM_COUNTERS
+#define NUM_VIRT_CONTROLS NUM_CONTROLS
+#endif
+
+#define OP_EVENT_MASK                   0x0FFF
+#define OP_CTR_OVERFLOW                 (1ULL<<31)
 
-#define CTR_IS_RESERVED(msrs, c) (msrs->counters[(c)].addr ? 1 : 0)
-#define CTR_READ(l, h, msrs, c) do {rdmsr(msrs->counters[(c)].addr, (l), (h)); } while (0)
-#define CTR_WRITE(l, msrs, c) do {wrmsr(msrs->counters[(c)].addr, -(unsigned int)(l), -1); } while (0)
-#define CTR_OVERFLOWED(n) (!((n) & (1U<<31)))
-
-#define CTRL_IS_RESERVED(msrs, c) (msrs->controls[(c)].addr ? 1 : 0)
-#define CTRL_READ(l, h, msrs, c) do {rdmsr(msrs->controls[(c)].addr, (l), (h)); } while (0)
-#define CTRL_WRITE(l, h, msrs, c) do {wrmsr(msrs->controls[(c)].addr, (l), (h)); } while (0)
-#define CTRL_SET_ACTIVE(n) (n |= (1<<22))
-#define CTRL_SET_INACTIVE(n) (n &= ~(1<<22))
-#define CTRL_CLEAR_LO(x) (x &= (1<<21))
-#define CTRL_CLEAR_HI(x) (x &= 0xfffffcf0)
-#define CTRL_SET_ENABLE(val) (val |= 1<<20)
-#define CTRL_SET_USR(val, u) (val |= ((u & 1) << 16))
-#define CTRL_SET_KERN(val, k) (val |= ((k & 1) << 17))
-#define CTRL_SET_UM(val, m) (val |= (m << 8))
-#define CTRL_SET_EVENT_LOW(val, e) (val |= (e & 0xff))
-#define CTRL_SET_EVENT_HIGH(val, e) (val |= ((e >> 8) & 0xf))
-#define CTRL_SET_HOST_ONLY(val, h) (val |= ((h & 1) << 9))
-#define CTRL_SET_GUEST_ONLY(val, h) (val |= ((h & 1) << 8))
-
-static unsigned long reset_value[NUM_COUNTERS];
+#define MSR_AMD_EVENTSEL_RESERVED       ((0xFFFFFCF0ULL<<32)|(1ULL<<21))
+
+static unsigned long reset_value[NUM_VIRT_COUNTERS];
 
 #ifdef CONFIG_OPROFILE_IBS
 
 /* IbsFetchCtl bits/masks */
-#define IBS_FETCH_HIGH_VALID_BIT        (1UL << 17)     /* bit 49 */
-#define IBS_FETCH_HIGH_ENABLE           (1UL << 16)     /* bit 48 */
-#define IBS_FETCH_LOW_MAX_CNT_MASK      0x0000FFFFUL    /* MaxCnt mask */
+#define IBS_FETCH_RAND_EN               (1ULL<<57)
+#define IBS_FETCH_VAL                   (1ULL<<49)
+#define IBS_FETCH_ENABLE                (1ULL<<48)
+#define IBS_FETCH_CNT_MASK              0xFFFF0000ULL
 
 /*IbsOpCtl bits */
-#define IBS_OP_LOW_VALID_BIT            (1ULL<<18)      /* bit 18 */
-#define IBS_OP_LOW_ENABLE               (1ULL<<17)      /* bit 17 */
+#define IBS_OP_CNT_CTL                  (1ULL<<19)
+#define IBS_OP_VAL                      (1ULL<<18)
+#define IBS_OP_ENABLE                   (1ULL<<17)
 
-#define IBS_FETCH_SIZE  6
-#define IBS_OP_SIZE     12
+#define IBS_FETCH_SIZE                  6
+#define IBS_OP_SIZE                     12
 
 static int has_ibs;     /* AMD Family10h and later */
 
@@ -78,6 +74,45 @@ static struct op_ibs_config ibs_config;
 
 #endif
 
+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
+
+static void op_mux_fill_in_addresses(struct op_msrs * const msrs)
+{
+        int i;
+
+        for (i = 0; i < NUM_VIRT_COUNTERS; i++) {
+                int hw_counter = op_x86_virt_to_phys(i);
+                if (reserve_perfctr_nmi(MSR_K7_PERFCTR0 + i))
+                        msrs->multiplex[i].addr = MSR_K7_PERFCTR0 + hw_counter;
+                else
+                        msrs->multiplex[i].addr = 0;
+        }
+}
+
+static void op_mux_switch_ctrl(struct op_x86_model_spec const *model,
+                               struct op_msrs const * const msrs)
+{
+        u64 val;
+        int i;
+
+        /* enable active counters */
+        for (i = 0; i < NUM_COUNTERS; ++i) {
+                int virt = op_x86_phys_to_virt(i);
+                if (!counter_config[virt].enabled)
+                        continue;
+                rdmsrl(msrs->controls[i].addr, val);
+                val &= model->reserved;
+                val |= op_x86_get_ctrl(model, &counter_config[virt]);
+                wrmsrl(msrs->controls[i].addr, val);
+        }
+}
+
+#else
+
+static inline void op_mux_fill_in_addresses(struct op_msrs * const msrs) { }
+
+#endif
+
 /* functions for op_amd_spec */
 
 static void op_amd_fill_in_addresses(struct op_msrs * const msrs)
@@ -97,150 +132,174 @@ static void op_amd_fill_in_addresses(struct op_msrs * const msrs)
                 else
                         msrs->controls[i].addr = 0;
         }
-}
 
+        op_mux_fill_in_addresses(msrs);
+}
 
-static void op_amd_setup_ctrs(struct op_msrs const * const msrs)
+static void op_amd_setup_ctrs(struct op_x86_model_spec const *model,
+                              struct op_msrs const * const msrs)
 {
-        unsigned int low, high;
+        u64 val;
         int i;
 
+        /* setup reset_value */
+        for (i = 0; i < NUM_VIRT_COUNTERS; ++i) {
+                if (counter_config[i].enabled)
+                        reset_value[i] = counter_config[i].count;
+                else
+                        reset_value[i] = 0;
+        }
+
         /* clear all counters */
-        for (i = 0 ; i < NUM_CONTROLS; ++i) {
-                if (unlikely(!CTRL_IS_RESERVED(msrs, i)))
+        for (i = 0; i < NUM_CONTROLS; ++i) {
+                if (unlikely(!msrs->controls[i].addr))
                         continue;
-                CTRL_READ(low, high, msrs, i);
-                CTRL_CLEAR_LO(low);
-                CTRL_CLEAR_HI(high);
-                CTRL_WRITE(low, high, msrs, i);
+                rdmsrl(msrs->controls[i].addr, val);
+                val &= model->reserved;
+                wrmsrl(msrs->controls[i].addr, val);
         }
 
         /* avoid a false detection of ctr overflows in NMI handler */
         for (i = 0; i < NUM_COUNTERS; ++i) {
-                if (unlikely(!CTR_IS_RESERVED(msrs, i)))
+                if (unlikely(!msrs->counters[i].addr))
                         continue;
-                CTR_WRITE(1, msrs, i);
+                wrmsrl(msrs->counters[i].addr, -1LL);
         }
 
         /* enable active counters */
         for (i = 0; i < NUM_COUNTERS; ++i) {
-                if ((counter_config[i].enabled) && (CTR_IS_RESERVED(msrs, i))) {
-                        reset_value[i] = counter_config[i].count;
+                int virt = op_x86_phys_to_virt(i);
+                if (!counter_config[virt].enabled)
+                        continue;
+                if (!msrs->counters[i].addr)
+                        continue;
 
-                        CTR_WRITE(counter_config[i].count, msrs, i);
-
-                        CTRL_READ(low, high, msrs, i);
-                        CTRL_CLEAR_LO(low);
-                        CTRL_CLEAR_HI(high);
-                        CTRL_SET_ENABLE(low);
-                        CTRL_SET_USR(low, counter_config[i].user);
-                        CTRL_SET_KERN(low, counter_config[i].kernel);
-                        CTRL_SET_UM(low, counter_config[i].unit_mask);
-                        CTRL_SET_EVENT_LOW(low, counter_config[i].event);
-                        CTRL_SET_EVENT_HIGH(high, counter_config[i].event);
-                        CTRL_SET_HOST_ONLY(high, 0);
-                        CTRL_SET_GUEST_ONLY(high, 0);
-
-                        CTRL_WRITE(low, high, msrs, i);
-                } else {
-                        reset_value[i] = 0;
-                }
+                /* setup counter registers */
+                wrmsrl(msrs->counters[i].addr, -(u64)reset_value[virt]);
+
+                /* setup control registers */
+                rdmsrl(msrs->controls[i].addr, val);
+                val &= model->reserved;
+                val |= op_x86_get_ctrl(model, &counter_config[virt]);
+                wrmsrl(msrs->controls[i].addr, val);
         }
 }
 
 #ifdef CONFIG_OPROFILE_IBS
 
-static inline int
+static inline void
 op_amd_handle_ibs(struct pt_regs * const regs,
                   struct op_msrs const * const msrs)
 {
-        u32 low, high;
-        u64 msr;
+        u64 val, ctl;
         struct op_entry entry;
 
         if (!has_ibs)
-                return 1;
+                return;
 
         if (ibs_config.fetch_enabled) {
-                rdmsr(MSR_AMD64_IBSFETCHCTL, low, high);
-                if (high & IBS_FETCH_HIGH_VALID_BIT) {
-                        rdmsrl(MSR_AMD64_IBSFETCHLINAD, msr);
-                        oprofile_write_reserve(&entry, regs, msr,
+                rdmsrl(MSR_AMD64_IBSFETCHCTL, ctl);
+                if (ctl & IBS_FETCH_VAL) {
+                        rdmsrl(MSR_AMD64_IBSFETCHLINAD, val);
+                        oprofile_write_reserve(&entry, regs, val,
                                                IBS_FETCH_CODE, IBS_FETCH_SIZE);
-                        oprofile_add_data(&entry, (u32)msr);
-                        oprofile_add_data(&entry, (u32)(msr >> 32));
-                        oprofile_add_data(&entry, low);
-                        oprofile_add_data(&entry, high);
-                        rdmsrl(MSR_AMD64_IBSFETCHPHYSAD, msr);
-                        oprofile_add_data(&entry, (u32)msr);
-                        oprofile_add_data(&entry, (u32)(msr >> 32));
+                        oprofile_add_data64(&entry, val);
+                        oprofile_add_data64(&entry, ctl);
+                        rdmsrl(MSR_AMD64_IBSFETCHPHYSAD, val);
+                        oprofile_add_data64(&entry, val);
                         oprofile_write_commit(&entry);
 
                         /* reenable the IRQ */
-                        high &= ~IBS_FETCH_HIGH_VALID_BIT;
-                        high |= IBS_FETCH_HIGH_ENABLE;
-                        low &= IBS_FETCH_LOW_MAX_CNT_MASK;
-                        wrmsr(MSR_AMD64_IBSFETCHCTL, low, high);
+                        ctl &= ~(IBS_FETCH_VAL | IBS_FETCH_CNT_MASK);
+                        ctl |= IBS_FETCH_ENABLE;
+                        wrmsrl(MSR_AMD64_IBSFETCHCTL, ctl);
                 }
         }
 
         if (ibs_config.op_enabled) {
-                rdmsr(MSR_AMD64_IBSOPCTL, low, high);
-                if (low & IBS_OP_LOW_VALID_BIT) {
-                        rdmsrl(MSR_AMD64_IBSOPRIP, msr);
-                        oprofile_write_reserve(&entry, regs, msr,
+                rdmsrl(MSR_AMD64_IBSOPCTL, ctl);
+                if (ctl & IBS_OP_VAL) {
+                        rdmsrl(MSR_AMD64_IBSOPRIP, val);
+                        oprofile_write_reserve(&entry, regs, val,
                                                IBS_OP_CODE, IBS_OP_SIZE);
-                        oprofile_add_data(&entry, (u32)msr);
-                        oprofile_add_data(&entry, (u32)(msr >> 32));
-                        rdmsrl(MSR_AMD64_IBSOPDATA, msr);
-                        oprofile_add_data(&entry, (u32)msr);
-                        oprofile_add_data(&entry, (u32)(msr >> 32));
-                        rdmsrl(MSR_AMD64_IBSOPDATA2, msr);
-                        oprofile_add_data(&entry, (u32)msr);
-                        oprofile_add_data(&entry, (u32)(msr >> 32));
-                        rdmsrl(MSR_AMD64_IBSOPDATA3, msr);
-                        oprofile_add_data(&entry, (u32)msr);
-                        oprofile_add_data(&entry, (u32)(msr >> 32));
-                        rdmsrl(MSR_AMD64_IBSDCLINAD, msr);
-                        oprofile_add_data(&entry, (u32)msr);
-                        oprofile_add_data(&entry, (u32)(msr >> 32));
-                        rdmsrl(MSR_AMD64_IBSDCPHYSAD, msr);
-                        oprofile_add_data(&entry, (u32)msr);
-                        oprofile_add_data(&entry, (u32)(msr >> 32));
+                        oprofile_add_data64(&entry, val);
+                        rdmsrl(MSR_AMD64_IBSOPDATA, val);
+                        oprofile_add_data64(&entry, val);
+                        rdmsrl(MSR_AMD64_IBSOPDATA2, val);
+                        oprofile_add_data64(&entry, val);
+                        rdmsrl(MSR_AMD64_IBSOPDATA3, val);
+                        oprofile_add_data64(&entry, val);
+                        rdmsrl(MSR_AMD64_IBSDCLINAD, val);
+                        oprofile_add_data64(&entry, val);
+                        rdmsrl(MSR_AMD64_IBSDCPHYSAD, val);
+                        oprofile_add_data64(&entry, val);
                         oprofile_write_commit(&entry);
 
                         /* reenable the IRQ */
-                        high = 0;
-                        low &= ~IBS_OP_LOW_VALID_BIT;
-                        low |= IBS_OP_LOW_ENABLE;
-                        wrmsr(MSR_AMD64_IBSOPCTL, low, high);
+                        ctl &= ~IBS_OP_VAL & 0xFFFFFFFF;
+                        ctl |= IBS_OP_ENABLE;
+                        wrmsrl(MSR_AMD64_IBSOPCTL, ctl);
                 }
         }
+}
 
-        return 1;
+static inline void op_amd_start_ibs(void)
+{
+        u64 val;
+        if (has_ibs && ibs_config.fetch_enabled) {
+                val = (ibs_config.max_cnt_fetch >> 4) & 0xFFFF;
+                val |= ibs_config.rand_en ? IBS_FETCH_RAND_EN : 0;
+                val |= IBS_FETCH_ENABLE;
+                wrmsrl(MSR_AMD64_IBSFETCHCTL, val);
+        }
+
+        if (has_ibs && ibs_config.op_enabled) {
+                val = (ibs_config.max_cnt_op >> 4) & 0xFFFF;
+                val |= ibs_config.dispatched_ops ? IBS_OP_CNT_CTL : 0;
+                val |= IBS_OP_ENABLE;
+                wrmsrl(MSR_AMD64_IBSOPCTL, val);
+        }
+}
+
+static void op_amd_stop_ibs(void)
+{
+        if (has_ibs && ibs_config.fetch_enabled)
+                /* clear max count and enable */
+                wrmsrl(MSR_AMD64_IBSFETCHCTL, 0);
+
+        if (has_ibs && ibs_config.op_enabled)
+                /* clear max count and enable */
+                wrmsrl(MSR_AMD64_IBSOPCTL, 0);
 }
 
+#else
+
+static inline void op_amd_handle_ibs(struct pt_regs * const regs,
+                                    struct op_msrs const * const msrs) { }
+static inline void op_amd_start_ibs(void) { }
+static inline void op_amd_stop_ibs(void) { }
+
 #endif
 
 static int op_amd_check_ctrs(struct pt_regs * const regs,
                              struct op_msrs const * const msrs)
 {
-        unsigned int low, high;
+        u64 val;
         int i;
 
-        for (i = 0 ; i < NUM_COUNTERS; ++i) {
-                if (!reset_value[i])
+        for (i = 0; i < NUM_COUNTERS; ++i) {
+                int virt = op_x86_phys_to_virt(i);
+                if (!reset_value[virt])
                         continue;
-                CTR_READ(low, high, msrs, i);
-                if (CTR_OVERFLOWED(low)) {
-                        oprofile_add_sample(regs, i);
-                        CTR_WRITE(reset_value[i], msrs, i);
-                }
+                rdmsrl(msrs->counters[i].addr, val);
+                /* bit is clear if overflowed: */
+                if (val & OP_CTR_OVERFLOW)
+                        continue;
+                oprofile_add_sample(regs, virt);
+                wrmsrl(msrs->counters[i].addr, -(u64)reset_value[virt]);
         }
 
-#ifdef CONFIG_OPROFILE_IBS
         op_amd_handle_ibs(regs, msrs);
-#endif
 
         /* See op_model_ppro.c */
         return 1;
@@ -248,79 +307,50 @@ static int op_amd_check_ctrs(struct pt_regs * const regs,
 
 static void op_amd_start(struct op_msrs const * const msrs)
 {
-        unsigned int low, high;
+        u64 val;
         int i;
-        for (i = 0 ; i < NUM_COUNTERS ; ++i) {
-                if (reset_value[i]) {
-                        CTRL_READ(low, high, msrs, i);
-                        CTRL_SET_ACTIVE(low);
-                        CTRL_WRITE(low, high, msrs, i);
-                }
-        }
 
-#ifdef CONFIG_OPROFILE_IBS
-        if (has_ibs && ibs_config.fetch_enabled) {
-                low = (ibs_config.max_cnt_fetch >> 4) & 0xFFFF;
-                high = ((ibs_config.rand_en & 0x1) << 25) /* bit 57 */
-                        + IBS_FETCH_HIGH_ENABLE;
-                wrmsr(MSR_AMD64_IBSFETCHCTL, low, high);
+        for (i = 0; i < NUM_COUNTERS; ++i) {
+                if (!reset_value[op_x86_phys_to_virt(i)])
+                        continue;
+                rdmsrl(msrs->controls[i].addr, val);
+                val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
+                wrmsrl(msrs->controls[i].addr, val);
         }
 
-        if (has_ibs && ibs_config.op_enabled) {
-                low = ((ibs_config.max_cnt_op >> 4) & 0xFFFF)
-                        + ((ibs_config.dispatched_ops & 0x1) << 19) /* bit 19 */
-                        + IBS_OP_LOW_ENABLE;
-                high = 0;
-                wrmsr(MSR_AMD64_IBSOPCTL, low, high);
-        }
-#endif
+        op_amd_start_ibs();
 }
 
-
 static void op_amd_stop(struct op_msrs const * const msrs)
 {
-        unsigned int low, high;
+        u64 val;
         int i;
 
         /*
          * Subtle: stop on all counters to avoid race with setting our
          * pm callback
          */
-        for (i = 0 ; i < NUM_COUNTERS ; ++i) {
-                if (!reset_value[i])
+        for (i = 0; i < NUM_COUNTERS; ++i) {
+                if (!reset_value[op_x86_phys_to_virt(i)])
                         continue;
-                CTRL_READ(low, high, msrs, i);
-                CTRL_SET_INACTIVE(low);
-                CTRL_WRITE(low, high, msrs, i);
-        }
-
-#ifdef CONFIG_OPROFILE_IBS
-        if (has_ibs && ibs_config.fetch_enabled) {
-                /* clear max count and enable */
-                low = 0;
-                high = 0;
-                wrmsr(MSR_AMD64_IBSFETCHCTL, low, high);
+                rdmsrl(msrs->controls[i].addr, val);
+                val &= ~ARCH_PERFMON_EVENTSEL0_ENABLE;
+                wrmsrl(msrs->controls[i].addr, val);
         }
 
-        if (has_ibs && ibs_config.op_enabled) {
-                /* clear max count and enable */
-                low = 0;
-                high = 0;
-                wrmsr(MSR_AMD64_IBSOPCTL, low, high);
-        }
-#endif
+        op_amd_stop_ibs();
 }
 
 static void op_amd_shutdown(struct op_msrs const * const msrs)
 {
         int i;
 
-        for (i = 0 ; i < NUM_COUNTERS ; ++i) {
-                if (CTR_IS_RESERVED(msrs, i))
+        for (i = 0; i < NUM_COUNTERS; ++i) {
+                if (msrs->counters[i].addr)
                         release_perfctr_nmi(MSR_K7_PERFCTR0 + i);
         }
-        for (i = 0 ; i < NUM_CONTROLS ; ++i) {
-                if (CTRL_IS_RESERVED(msrs, i))
+        for (i = 0; i < NUM_CONTROLS; ++i) {
+                if (msrs->controls[i].addr)
                         release_evntsel_nmi(MSR_K7_EVNTSEL0 + i);
         }
 }
@@ -490,15 +520,21 @@ static void op_amd_exit(void) {}
 
 #endif /* CONFIG_OPROFILE_IBS */
 
-struct op_x86_model_spec const op_amd_spec = {
-        .init                   = op_amd_init,
-        .exit                   = op_amd_exit,
+struct op_x86_model_spec op_amd_spec = {
         .num_counters           = NUM_COUNTERS,
         .num_controls           = NUM_CONTROLS,
+        .num_virt_counters      = NUM_VIRT_COUNTERS,
+        .reserved               = MSR_AMD_EVENTSEL_RESERVED,
+        .event_mask             = OP_EVENT_MASK,
+        .init                   = op_amd_init,
+        .exit                   = op_amd_exit,
         .fill_in_addresses      = &op_amd_fill_in_addresses,
         .setup_ctrs             = &op_amd_setup_ctrs,
         .check_ctrs             = &op_amd_check_ctrs,
         .start                  = &op_amd_start,
         .stop                   = &op_amd_stop,
-        .shutdown               = &op_amd_shutdown
+        .shutdown               = &op_amd_shutdown,
+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
+        .switch_ctrl            = &op_mux_switch_ctrl,
+#endif
 };
diff --git a/arch/x86/oprofile/op_model_p4.c b/arch/x86/oprofile/op_model_p4.c
index 819b131fd752..ac6b354becdf 100644
--- a/arch/x86/oprofile/op_model_p4.c
+++ b/arch/x86/oprofile/op_model_p4.c
@@ -32,6 +32,8 @@
 #define NUM_CCCRS_HT2 9
 #define NUM_CONTROLS_HT2 (NUM_ESCRS_HT2 + NUM_CCCRS_HT2)
 
+#define OP_CTR_OVERFLOW                 (1ULL<<31)
+
 static unsigned int num_counters = NUM_COUNTERS_NON_HT;
 static unsigned int num_controls = NUM_CONTROLS_NON_HT;
 
@@ -350,8 +352,6 @@ static struct p4_event_binding p4_events[NUM_EVENTS] = {
 #define ESCR_SET_OS_1(escr, os) ((escr) |= (((os) & 1) << 1))
 #define ESCR_SET_EVENT_SELECT(escr, sel) ((escr) |= (((sel) & 0x3f) << 25))
 #define ESCR_SET_EVENT_MASK(escr, mask) ((escr) |= (((mask) & 0xffff) << 9))
-#define ESCR_READ(escr, high, ev, i) do {rdmsr(ev->bindings[(i)].escr_address, (escr), (high)); } while (0)
-#define ESCR_WRITE(escr, high, ev, i) do {wrmsr(ev->bindings[(i)].escr_address, (escr), (high)); } while (0)
 
 #define CCCR_RESERVED_BITS 0x38030FFF
 #define CCCR_CLEAR(cccr) ((cccr) &= CCCR_RESERVED_BITS)
@@ -361,17 +361,9 @@ static struct p4_event_binding p4_events[NUM_EVENTS] = {
 #define CCCR_SET_PMI_OVF_1(cccr) ((cccr) |= (1<<27))
 #define CCCR_SET_ENABLE(cccr) ((cccr) |= (1<<12))
 #define CCCR_SET_DISABLE(cccr) ((cccr) &= ~(1<<12))
-#define CCCR_READ(low, high, i) do {rdmsr(p4_counters[(i)].cccr_address, (low), (high)); } while (0)
-#define CCCR_WRITE(low, high, i) do {wrmsr(p4_counters[(i)].cccr_address, (low), (high)); } while (0)
 #define CCCR_OVF_P(cccr) ((cccr) & (1U<<31))
 #define CCCR_CLEAR_OVF(cccr) ((cccr) &= (~(1U<<31)))
 
-#define CTRL_IS_RESERVED(msrs, c) (msrs->controls[(c)].addr ? 1 : 0)
-#define CTR_IS_RESERVED(msrs, c) (msrs->counters[(c)].addr ? 1 : 0)
-#define CTR_READ(l, h, i) do {rdmsr(p4_counters[(i)].counter_address, (l), (h)); } while (0)
-#define CTR_WRITE(l, i) do {wrmsr(p4_counters[(i)].counter_address, -(u32)(l), -1); } while (0)
-#define CTR_OVERFLOW_P(ctr) (!((ctr) & 0x80000000))
-
 
 /* this assigns a "stagger" to the current CPU, which is used throughout
    the code in this module as an extra array offset, to select the "even"
@@ -515,7 +507,7 @@ static void pmc_setup_one_p4_counter(unsigned int ctr)
                 if (ev->bindings[i].virt_counter & counter_bit) {
 
                         /* modify ESCR */
-                        ESCR_READ(escr, high, ev, i);
+                        rdmsr(ev->bindings[i].escr_address, escr, high);
                         ESCR_CLEAR(escr);
                         if (stag == 0) {
                                 ESCR_SET_USR_0(escr, counter_config[ctr].user);
@@ -526,10 +518,11 @@ static void pmc_setup_one_p4_counter(unsigned int ctr)
                         }
                         ESCR_SET_EVENT_SELECT(escr, ev->event_select);
                         ESCR_SET_EVENT_MASK(escr, counter_config[ctr].unit_mask);
-                        ESCR_WRITE(escr, high, ev, i);
+                        wrmsr(ev->bindings[i].escr_address, escr, high);
 
                         /* modify CCCR */
-                        CCCR_READ(cccr, high, VIRT_CTR(stag, ctr));
+                        rdmsr(p4_counters[VIRT_CTR(stag, ctr)].cccr_address,
+                              cccr, high);
                         CCCR_CLEAR(cccr);
                         CCCR_SET_REQUIRED_BITS(cccr);
                         CCCR_SET_ESCR_SELECT(cccr, ev->escr_select);
@@ -537,7 +530,8 @@ static void pmc_setup_one_p4_counter(unsigned int ctr)
                                 CCCR_SET_PMI_OVF_0(cccr);
                         else
                                 CCCR_SET_PMI_OVF_1(cccr);
-                        CCCR_WRITE(cccr, high, VIRT_CTR(stag, ctr));
+                        wrmsr(p4_counters[VIRT_CTR(stag, ctr)].cccr_address,
+                              cccr, high);
                         return;
                 }
         }
@@ -548,7 +542,8 @@ static void pmc_setup_one_p4_counter(unsigned int ctr)
 }
 
 
-static void p4_setup_ctrs(struct op_msrs const * const msrs)
+static void p4_setup_ctrs(struct op_x86_model_spec const *model,
+                          struct op_msrs const * const msrs)
 {
         unsigned int i;
         unsigned int low, high;
@@ -563,8 +558,8 @@ static void p4_setup_ctrs(struct op_msrs const * const msrs)
         }
 
         /* clear the cccrs we will use */
-        for (i = 0 ; i < num_counters ; i++) {
-                if (unlikely(!CTRL_IS_RESERVED(msrs, i)))
+        for (i = 0; i < num_counters; i++) {
+                if (unlikely(!msrs->controls[i].addr))
                         continue;
                 rdmsr(p4_counters[VIRT_CTR(stag, i)].cccr_address, low, high);
                 CCCR_CLEAR(low);
@@ -574,17 +569,18 @@ static void p4_setup_ctrs(struct op_msrs const * const msrs)
 
         /* clear all escrs (including those outside our concern) */
         for (i = num_counters; i < num_controls; i++) {
-                if (unlikely(!CTRL_IS_RESERVED(msrs, i)))
+                if (unlikely(!msrs->controls[i].addr))
                         continue;
                 wrmsr(msrs->controls[i].addr, 0, 0);
         }
 
         /* setup all counters */
-        for (i = 0 ; i < num_counters ; ++i) {
-                if ((counter_config[i].enabled) && (CTRL_IS_RESERVED(msrs, i))) {
+        for (i = 0; i < num_counters; ++i) {
+                if (counter_config[i].enabled && msrs->controls[i].addr) {
                         reset_value[i] = counter_config[i].count;
                         pmc_setup_one_p4_counter(i);
-                        CTR_WRITE(counter_config[i].count, VIRT_CTR(stag, i));
+                        wrmsrl(p4_counters[VIRT_CTR(stag, i)].counter_address,
+                               -(u64)counter_config[i].count);
                 } else {
                         reset_value[i] = 0;
                 }
@@ -624,14 +620,16 @@ static int p4_check_ctrs(struct pt_regs * const regs,
 
                 real = VIRT_CTR(stag, i);
 
-                CCCR_READ(low, high, real);
-                CTR_READ(ctr, high, real);
-                if (CCCR_OVF_P(low) || CTR_OVERFLOW_P(ctr)) {
+                rdmsr(p4_counters[real].cccr_address, low, high);
+                rdmsr(p4_counters[real].counter_address, ctr, high);
+                if (CCCR_OVF_P(low) || !(ctr & OP_CTR_OVERFLOW)) {
                         oprofile_add_sample(regs, i);
-                        CTR_WRITE(reset_value[i], real);
+                        wrmsrl(p4_counters[real].counter_address,
+                               -(u64)reset_value[i]);
                         CCCR_CLEAR_OVF(low);
-                        CCCR_WRITE(low, high, real);
-                        CTR_WRITE(reset_value[i], real);
+                        wrmsr(p4_counters[real].cccr_address, low, high);
+                        wrmsrl(p4_counters[real].counter_address,
+                               -(u64)reset_value[i]);
                 }
         }
 
@@ -653,9 +651,9 @@ static void p4_start(struct op_msrs const * const msrs)
         for (i = 0; i < num_counters; ++i) {
                 if (!reset_value[i])
                         continue;
-                CCCR_READ(low, high, VIRT_CTR(stag, i));
+                rdmsr(p4_counters[VIRT_CTR(stag, i)].cccr_address, low, high);
                 CCCR_SET_ENABLE(low);
-                CCCR_WRITE(low, high, VIRT_CTR(stag, i));
+                wrmsr(p4_counters[VIRT_CTR(stag, i)].cccr_address, low, high);
         }
 }
 
@@ -670,9 +668,9 @@ static void p4_stop(struct op_msrs const * const msrs)
         for (i = 0; i < num_counters; ++i) {
                 if (!reset_value[i])
                         continue;
-                CCCR_READ(low, high, VIRT_CTR(stag, i));
+                rdmsr(p4_counters[VIRT_CTR(stag, i)].cccr_address, low, high);
                 CCCR_SET_DISABLE(low);
-                CCCR_WRITE(low, high, VIRT_CTR(stag, i));
+                wrmsr(p4_counters[VIRT_CTR(stag, i)].cccr_address, low, high);
         }
 }
 
@@ -680,8 +678,8 @@ static void p4_shutdown(struct op_msrs const * const msrs)
 {
         int i;
 
-        for (i = 0 ; i < num_counters ; ++i) {
-                if (CTR_IS_RESERVED(msrs, i))
+        for (i = 0; i < num_counters; ++i) {
+                if (msrs->counters[i].addr)
                         release_perfctr_nmi(msrs->counters[i].addr);
         }
         /*
@@ -689,15 +687,15 @@ static void p4_shutdown(struct op_msrs const * const msrs)
          * conjunction with the counter registers (hence the starting offset).
          * This saves a few bits.
          */
-        for (i = num_counters ; i < num_controls ; ++i) {
-                if (CTRL_IS_RESERVED(msrs, i))
+        for (i = num_counters; i < num_controls; ++i) {
+                if (msrs->controls[i].addr)
                         release_evntsel_nmi(msrs->controls[i].addr);
         }
 }
 
 
 #ifdef CONFIG_SMP
-struct op_x86_model_spec const op_p4_ht2_spec = {
+struct op_x86_model_spec op_p4_ht2_spec = {
         .num_counters           = NUM_COUNTERS_HT2,
         .num_controls           = NUM_CONTROLS_HT2,
         .fill_in_addresses      = &p4_fill_in_addresses,
@@ -709,7 +707,7 @@ struct op_x86_model_spec const op_p4_ht2_spec = {
 };
 #endif
 
-struct op_x86_model_spec const op_p4_spec = {
+struct op_x86_model_spec op_p4_spec = {
         .num_counters           = NUM_COUNTERS_NON_HT,
         .num_controls           = NUM_CONTROLS_NON_HT,
         .fill_in_addresses      = &p4_fill_in_addresses,
diff --git a/arch/x86/oprofile/op_model_ppro.c b/arch/x86/oprofile/op_model_ppro.c
index 4da7230b3d17..8eb05878554c 100644
--- a/arch/x86/oprofile/op_model_ppro.c
+++ b/arch/x86/oprofile/op_model_ppro.c
@@ -10,6 +10,7 @@
  * @author Philippe Elie
  * @author Graydon Hoare
  * @author Andi Kleen
+ * @author Robert Richter <robert.richter@amd.com>
  */
 
 #include <linux/oprofile.h>
@@ -18,7 +19,6 @@
 #include <asm/msr.h>
 #include <asm/apic.h>
 #include <asm/nmi.h>
-#include <asm/perf_counter.h>
 
 #include "op_x86_model.h"
 #include "op_counter.h"
@@ -26,20 +26,7 @@
 static int num_counters = 2;
 static int counter_width = 32;
 
-#define CTR_IS_RESERVED(msrs, c) (msrs->counters[(c)].addr ? 1 : 0)
-#define CTR_OVERFLOWED(n) (!((n) & (1ULL<<(counter_width-1))))
-
-#define CTRL_IS_RESERVED(msrs, c) (msrs->controls[(c)].addr ? 1 : 0)
-#define CTRL_READ(l, h, msrs, c) do {rdmsr((msrs->controls[(c)].addr), (l), (h)); } while (0)
-#define CTRL_WRITE(l, h, msrs, c) do {wrmsr((msrs->controls[(c)].addr), (l), (h)); } while (0)
-#define CTRL_SET_ACTIVE(n) (n |= (1<<22))
-#define CTRL_SET_INACTIVE(n) (n &= ~(1<<22))
-#define CTRL_CLEAR(x) (x &= (1<<21))
-#define CTRL_SET_ENABLE(val) (val |= 1<<20)
-#define CTRL_SET_USR(val, u) (val |= ((u & 1) << 16))
-#define CTRL_SET_KERN(val, k) (val |= ((k & 1) << 17))
-#define CTRL_SET_UM(val, m) (val |= (m << 8))
-#define CTRL_SET_EVENT(val, e) (val |= e)
+#define MSR_PPRO_EVENTSEL_RESERVED      ((0xFFFFFFFFULL<<32)|(1ULL<<21))
 
 static u64 *reset_value;
 
@@ -63,9 +50,10 @@ static void ppro_fill_in_addresses(struct op_msrs * const msrs)
 }
 
 
-static void ppro_setup_ctrs(struct op_msrs const * const msrs)
+static void ppro_setup_ctrs(struct op_x86_model_spec const *model,
+                            struct op_msrs const * const msrs)
 {
-        unsigned int low, high;
+        u64 val;
         int i;
 
         if (!reset_value) {
@@ -93,36 +81,30 @@ static void ppro_setup_ctrs(struct op_msrs const * const msrs)
         }
 
         /* clear all counters */
-        for (i = 0 ; i < num_counters; ++i) {
-                if (unlikely(!CTRL_IS_RESERVED(msrs, i)))
+        for (i = 0; i < num_counters; ++i) {
+                if (unlikely(!msrs->controls[i].addr))
                         continue;
-                CTRL_READ(low, high, msrs, i);
-                CTRL_CLEAR(low);
-                CTRL_WRITE(low, high, msrs, i);
+                rdmsrl(msrs->controls[i].addr, val);
+                val &= model->reserved;
+                wrmsrl(msrs->controls[i].addr, val);
         }
 
         /* avoid a false detection of ctr overflows in NMI handler */
         for (i = 0; i < num_counters; ++i) {
-                if (unlikely(!CTR_IS_RESERVED(msrs, i)))
+                if (unlikely(!msrs->counters[i].addr))
                         continue;
                 wrmsrl(msrs->counters[i].addr, -1LL);
         }
 
         /* enable active counters */
         for (i = 0; i < num_counters; ++i) {
-                if ((counter_config[i].enabled) && (CTR_IS_RESERVED(msrs, i))) {
+                if (counter_config[i].enabled && msrs->counters[i].addr) {
                         reset_value[i] = counter_config[i].count;
-
                         wrmsrl(msrs->counters[i].addr, -reset_value[i]);
-
-                        CTRL_READ(low, high, msrs, i);
-                        CTRL_CLEAR(low);
-                        CTRL_SET_ENABLE(low);
-                        CTRL_SET_USR(low, counter_config[i].user);
-                        CTRL_SET_KERN(low, counter_config[i].kernel);
-                        CTRL_SET_UM(low, counter_config[i].unit_mask);
-                        CTRL_SET_EVENT(low, counter_config[i].event);
-                        CTRL_WRITE(low, high, msrs, i);
+                        rdmsrl(msrs->controls[i].addr, val);
+                        val &= model->reserved;
+                        val |= op_x86_get_ctrl(model, &counter_config[i]);
+                        wrmsrl(msrs->controls[i].addr, val);
                 } else {
                         reset_value[i] = 0;
                 }
@@ -143,14 +125,14 @@ static int ppro_check_ctrs(struct pt_regs * const regs,
         if (unlikely(!reset_value))
                 goto out;
 
-        for (i = 0 ; i < num_counters; ++i) {
+        for (i = 0; i < num_counters; ++i) {
                 if (!reset_value[i])
                         continue;
                 rdmsrl(msrs->counters[i].addr, val);
-                if (CTR_OVERFLOWED(val)) {
-                        oprofile_add_sample(regs, i);
-                        wrmsrl(msrs->counters[i].addr, -reset_value[i]);
-                }
+                if (val & (1ULL << (counter_width - 1)))
+                        continue;
+                oprofile_add_sample(regs, i);
+                wrmsrl(msrs->counters[i].addr, -reset_value[i]);
         }
 
 out:
@@ -171,16 +153,16 @@ out:
 
 static void ppro_start(struct op_msrs const * const msrs)
 {
-        unsigned int low, high;
+        u64 val;
         int i;
 
         if (!reset_value)
                 return;
         for (i = 0; i < num_counters; ++i) {
                 if (reset_value[i]) {
-                        CTRL_READ(low, high, msrs, i);
-                        CTRL_SET_ACTIVE(low);
-                        CTRL_WRITE(low, high, msrs, i);
+                        rdmsrl(msrs->controls[i].addr, val);
+                        val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
+                        wrmsrl(msrs->controls[i].addr, val);
                 }
         }
 }
@@ -188,7 +170,7 @@ static void ppro_start(struct op_msrs const * const msrs)
 
 static void ppro_stop(struct op_msrs const * const msrs)
 {
-        unsigned int low, high;
+        u64 val;
         int i;
 
         if (!reset_value)
@@ -196,9 +178,9 @@ static void ppro_stop(struct op_msrs const * const msrs)
         for (i = 0; i < num_counters; ++i) {
                 if (!reset_value[i])
                         continue;
-                CTRL_READ(low, high, msrs, i);
-                CTRL_SET_INACTIVE(low);
-                CTRL_WRITE(low, high, msrs, i);
+                rdmsrl(msrs->controls[i].addr, val);
+                val &= ~ARCH_PERFMON_EVENTSEL0_ENABLE;
+                wrmsrl(msrs->controls[i].addr, val);
         }
 }
 
@@ -206,12 +188,12 @@ static void ppro_shutdown(struct op_msrs const * const msrs)
 {
         int i;
 
-        for (i = 0 ; i < num_counters ; ++i) {
-                if (CTR_IS_RESERVED(msrs, i))
+        for (i = 0; i < num_counters; ++i) {
+                if (msrs->counters[i].addr)
                         release_perfctr_nmi(MSR_P6_PERFCTR0 + i);
         }
-        for (i = 0 ; i < num_counters ; ++i) {
-                if (CTRL_IS_RESERVED(msrs, i))
+        for (i = 0; i < num_counters; ++i) {
+                if (msrs->controls[i].addr)
                         release_evntsel_nmi(MSR_P6_EVNTSEL0 + i);
         }
         if (reset_value) {
@@ -222,8 +204,9 @@ static void ppro_shutdown(struct op_msrs const * const msrs)
 
 
 struct op_x86_model_spec op_ppro_spec = {
-        .num_counters           = 2,    /* can be overriden */
-        .num_controls           = 2,    /* dito */
+        .num_counters           = 2,
+        .num_controls           = 2,
+        .reserved               = MSR_PPRO_EVENTSEL_RESERVED,
         .fill_in_addresses      = &ppro_fill_in_addresses,
         .setup_ctrs             = &ppro_setup_ctrs,
         .check_ctrs             = &ppro_check_ctrs,
@@ -241,7 +224,7 @@ struct op_x86_model_spec op_ppro_spec = {
  * the specific CPU.
  */
 
-void arch_perfmon_setup_counters(void)
+static void arch_perfmon_setup_counters(void)
 {
         union cpuid10_eax eax;
 
@@ -251,19 +234,25 @@ void arch_perfmon_setup_counters(void)
         if (eax.split.version_id == 0 && current_cpu_data.x86 == 6 &&
                 current_cpu_data.x86_model == 15) {
                 eax.split.version_id = 2;
-                eax.split.num_counters = 2;
+                eax.split.num_events = 2;
                 eax.split.bit_width = 40;
         }
 
-        num_counters = eax.split.num_counters;
+        num_counters = eax.split.num_events;
 
         op_arch_perfmon_spec.num_counters = num_counters;
         op_arch_perfmon_spec.num_controls = num_counters;
-        op_ppro_spec.num_counters = num_counters;
-        op_ppro_spec.num_controls = num_counters;
+}
+
+static int arch_perfmon_init(struct oprofile_operations *ignore)
+{
+        arch_perfmon_setup_counters();
+        return 0;
 }
 
 struct op_x86_model_spec op_arch_perfmon_spec = {
+        .reserved               = MSR_PPRO_EVENTSEL_RESERVED,
+        .init                   = &arch_perfmon_init,
         /* num_counters/num_controls filled in at runtime */
         .fill_in_addresses      = &ppro_fill_in_addresses,
         /* user space does the cpuid check for available events */
diff --git a/arch/x86/oprofile/op_x86_model.h b/arch/x86/oprofile/op_x86_model.h
index 825e79064d64..7b8e75d16081 100644
--- a/arch/x86/oprofile/op_x86_model.h
+++ b/arch/x86/oprofile/op_x86_model.h
@@ -6,51 +6,66 @@
  * @remark Read the file COPYING
  *
  * @author Graydon Hoare
+ * @author Robert Richter <robert.richter@amd.com>
  */
 
 #ifndef OP_X86_MODEL_H
 #define OP_X86_MODEL_H
 
-struct op_saved_msr {
-        unsigned int high;
-        unsigned int low;
-};
+#include <asm/types.h>
+#include <asm/perf_event.h>
 
 struct op_msr {
-        unsigned long addr;
-        struct op_saved_msr saved;
+        unsigned long   addr;
+        u64             saved;
 };
 
 struct op_msrs {
         struct op_msr *counters;
         struct op_msr *controls;
+        struct op_msr *multiplex;
 };
 
 struct pt_regs;
 
+struct oprofile_operations;
+
 /* The model vtable abstracts the differences between
  * various x86 CPU models' perfctr support.
  */
 struct op_x86_model_spec {
-        int (*init)(struct oprofile_operations *ops);
-        void (*exit)(void);
-        unsigned int num_counters;
-        unsigned int num_controls;
-        void (*fill_in_addresses)(struct op_msrs * const msrs);
-        void (*setup_ctrs)(struct op_msrs const * const msrs);
-        int (*check_ctrs)(struct pt_regs * const regs,
-                struct op_msrs const * const msrs);
-        void (*start)(struct op_msrs const * const msrs);
-        void (*stop)(struct op_msrs const * const msrs);
-        void (*shutdown)(struct op_msrs const * const msrs);
+        unsigned int    num_counters;
+        unsigned int    num_controls;
+        unsigned int    num_virt_counters;
+        u64             reserved;
+        u16             event_mask;
+        int             (*init)(struct oprofile_operations *ops);
+        void            (*exit)(void);
+        void            (*fill_in_addresses)(struct op_msrs * const msrs);
+        void            (*setup_ctrs)(struct op_x86_model_spec const *model,
+                                      struct op_msrs const * const msrs);
+        int             (*check_ctrs)(struct pt_regs * const regs,
+                                      struct op_msrs const * const msrs);
+        void            (*start)(struct op_msrs const * const msrs);
+        void            (*stop)(struct op_msrs const * const msrs);
+        void            (*shutdown)(struct op_msrs const * const msrs);
+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
+        void            (*switch_ctrl)(struct op_x86_model_spec const *model,
+                                       struct op_msrs const * const msrs);
+#endif
 };
 
+struct op_counter_config;
+
+extern u64 op_x86_get_ctrl(struct op_x86_model_spec const *model,
+                           struct op_counter_config *counter_config);
+extern int op_x86_phys_to_virt(int phys);
+extern int op_x86_virt_to_phys(int virt);
+
 extern struct op_x86_model_spec op_ppro_spec;
-extern struct op_x86_model_spec const op_p4_spec;
-extern struct op_x86_model_spec const op_p4_ht2_spec;
-extern struct op_x86_model_spec const op_amd_spec;
+extern struct op_x86_model_spec op_p4_spec;
+extern struct op_x86_model_spec op_p4_ht2_spec;
+extern struct op_x86_model_spec op_amd_spec;
 extern struct op_x86_model_spec op_arch_perfmon_spec;
 
-extern void arch_perfmon_setup_counters(void);
-
 #endif /* OP_X86_MODEL_H */
diff --git a/arch/x86/pci/amd_bus.c b/arch/x86/pci/amd_bus.c
index 3ffa10df20b9..572ee9782f2a 100644
--- a/arch/x86/pci/amd_bus.c
+++ b/arch/x86/pci/amd_bus.c
@@ -15,63 +15,6 @@
  * also get peer root bus resource for io,mmio
  */
 
-#ifdef CONFIG_NUMA
-
-#define BUS_NR 256
-
-#ifdef CONFIG_X86_64
-
-static int mp_bus_to_node[BUS_NR];
-
-void set_mp_bus_to_node(int busnum, int node)
-{
-        if (busnum >= 0 &&  busnum < BUS_NR)
-                mp_bus_to_node[busnum] = node;
-}
-
-int get_mp_bus_to_node(int busnum)
-{
-        int node = -1;
-
-        if (busnum < 0 || busnum > (BUS_NR - 1))
-                return node;
-
-        node = mp_bus_to_node[busnum];
-
-        /*
-         * let numa_node_id to decide it later in dma_alloc_pages
-         * if there is no ram on that node
-         */
-        if (node != -1 && !node_online(node))
-                node = -1;
-
-        return node;
-}
-
-#else /* CONFIG_X86_32 */
-
-static unsigned char mp_bus_to_node[BUS_NR];
-
-void set_mp_bus_to_node(int busnum, int node)
-{
-        if (busnum >= 0 &&  busnum < BUS_NR)
-        mp_bus_to_node[busnum] = (unsigned char) node;
-}
-
-int get_mp_bus_to_node(int busnum)
-{
-        int node;
-
-        if (busnum < 0 || busnum > (BUS_NR - 1))
-                return 0;
-        node = mp_bus_to_node[busnum];
-        return node;
-}
-
-#endif /* CONFIG_X86_32 */
-
-#endif /* CONFIG_NUMA */
-
 #ifdef CONFIG_X86_64
 
 /*
@@ -301,11 +244,6 @@ static int __init early_fill_mp_bus_info(void)
         u64 val;
         u32 address;
 
-#ifdef CONFIG_NUMA
-        for (i = 0; i < BUS_NR; i++)
-                mp_bus_to_node[i] = -1;
-#endif
-
         if (!early_pci_allowed())
                 return -1;
 
@@ -346,7 +284,7 @@ static int __init early_fill_mp_bus_info(void)
                 node = (reg >> 4) & 0x07;
 #ifdef CONFIG_NUMA
                 for (j = min_bus; j <= max_bus; j++)
-                        mp_bus_to_node[j] = (unsigned char) node;
+                        set_mp_bus_to_node(j, node);
 #endif
                 link = (reg >> 8) & 0x03;
 
diff --git a/arch/x86/pci/common.c b/arch/x86/pci/common.c
index 2202b6257b82..1331fcf26143 100644
--- a/arch/x86/pci/common.c
+++ b/arch/x86/pci/common.c
@@ -600,3 +600,72 @@ struct pci_bus * __devinit pci_scan_bus_with_sysdata(int busno)
 {
         return pci_scan_bus_on_node(busno, &pci_root_ops, -1);
 }
+
+/*
+ * NUMA info for PCI busses
+ *
+ * Early arch code is responsible for filling in reasonable values here.
+ * A node id of "-1" means "use current node".  In other words, if a bus
+ * has a -1 node id, it's not tightly coupled to any particular chunk
+ * of memory (as is the case on some Nehalem systems).
+ */
+#ifdef CONFIG_NUMA
+
+#define BUS_NR 256
+
+#ifdef CONFIG_X86_64
+
+static int mp_bus_to_node[BUS_NR] = {
+        [0 ... BUS_NR - 1] = -1
+};
+
+void set_mp_bus_to_node(int busnum, int node)
+{
+        if (busnum >= 0 &&  busnum < BUS_NR)
+                mp_bus_to_node[busnum] = node;
+}
+
+int get_mp_bus_to_node(int busnum)
+{
+        int node = -1;
+
+        if (busnum < 0 || busnum > (BUS_NR - 1))
+                return node;
+
+        node = mp_bus_to_node[busnum];
+
+        /*
+         * let numa_node_id to decide it later in dma_alloc_pages
+         * if there is no ram on that node
+         */
+        if (node != -1 && !node_online(node))
+                node = -1;
+
+        return node;
+}
+
+#else /* CONFIG_X86_32 */
+
+static int mp_bus_to_node[BUS_NR] = {
+        [0 ... BUS_NR - 1] = -1
+};
+
+void set_mp_bus_to_node(int busnum, int node)
+{
+        if (busnum >= 0 &&  busnum < BUS_NR)
+        mp_bus_to_node[busnum] = (unsigned char) node;
+}
+
+int get_mp_bus_to_node(int busnum)
+{
+        int node;
+
+        if (busnum < 0 || busnum > (BUS_NR - 1))
+                return 0;
+        node = mp_bus_to_node[busnum];
+        return node;
+}
+
+#endif /* CONFIG_X86_32 */
+
+#endif /* CONFIG_NUMA */
diff --git a/arch/x86/pci/direct.c b/arch/x86/pci/direct.c
index bd13c3e4c6db..347d882b3bb3 100644
--- a/arch/x86/pci/direct.c
+++ b/arch/x86/pci/direct.c
@@ -192,13 +192,14 @@ struct pci_raw_ops pci_direct_conf2 = {
 static int __init pci_sanity_check(struct pci_raw_ops *o)
 {
         u32 x = 0;
-        int devfn;
+        int year, devfn;
 
         if (pci_probe & PCI_NO_CHECKS)
                 return 1;
         /* Assume Type 1 works for newer systems.
            This handles machines that don't have anything on PCI Bus 0. */
-        if (dmi_get_year(DMI_BIOS_DATE) >= 2001)
+        dmi_get_date(DMI_BIOS_DATE, &year, NULL, NULL);
+        if (year >= 2001)
                 return 1;
 
         for (devfn = 0; devfn < 0x100; devfn++) {
diff --git a/arch/x86/pci/mmconfig-shared.c b/arch/x86/pci/mmconfig-shared.c
index 712443ec6d43..602c172d3bd5 100644
--- a/arch/x86/pci/mmconfig-shared.c
+++ b/arch/x86/pci/mmconfig-shared.c
@@ -13,10 +13,14 @@
 #include <linux/pci.h>
 #include <linux/init.h>
 #include <linux/acpi.h>
+#include <linux/sfi_acpi.h>
 #include <linux/bitmap.h>
 #include <linux/sort.h>
 #include <asm/e820.h>
 #include <asm/pci_x86.h>
+#include <asm/acpi.h>
+
+#define PREFIX "PCI: "
 
 /* aperture is up to 256MB but BIOS may reserve less */
 #define MMCONFIG_APER_MIN       (2 * 1024*1024)
@@ -491,7 +495,7 @@ static void __init pci_mmcfg_reject_broken(int early)
                        (unsigned int)cfg->start_bus_number,
                        (unsigned int)cfg->end_bus_number);
 
-                if (!early)
+                if (!early && !acpi_disabled)
                         valid = is_mmconf_reserved(is_acpi_reserved, addr, size, i, cfg, 0);
 
                 if (valid)
@@ -606,7 +610,7 @@ static void __init __pci_mmcfg_init(int early)
         }
 
         if (!known_bridge)
-                acpi_table_parse(ACPI_SIG_MCFG, pci_parse_mcfg);
+                acpi_sfi_table_parse(ACPI_SIG_MCFG, pci_parse_mcfg);
 
         pci_mmcfg_reject_broken(early);
 
diff --git a/arch/x86/pci/mmconfig_32.c b/arch/x86/pci/mmconfig_32.c
index 8b2d561046a3..f10a7e94a84c 100644
--- a/arch/x86/pci/mmconfig_32.c
+++ b/arch/x86/pci/mmconfig_32.c
@@ -11,9 +11,9 @@
 
 #include <linux/pci.h>
 #include <linux/init.h>
-#include <linux/acpi.h>
 #include <asm/e820.h>
 #include <asm/pci_x86.h>
+#include <acpi/acpi.h>
 
 /* Assume systems with more busses have correct MCFG */
 #define mmcfg_virt_addr ((void __iomem *) fix_to_virt(FIX_PCIE_MCFG))
diff --git a/arch/x86/power/cpu.c b/arch/x86/power/cpu.c
index b3d20b9cac63..8aa85f17667e 100644
--- a/arch/x86/power/cpu.c
+++ b/arch/x86/power/cpu.c
@@ -242,11 +242,7 @@ static void __restore_processor_state(struct saved_context *ctxt)
         fix_processor_context();
 
         do_fpu_end();
-        mtrr_ap_init();
-
-#ifdef CONFIG_X86_OLD_MCE
-        mcheck_init(&boot_cpu_data);
-#endif
+        mtrr_bp_restore();
 }
 
 /* Needed by apm.c */
diff --git a/arch/x86/vdso/Makefile b/arch/x86/vdso/Makefile
index 88112b49f02c..6b4ffedb93c9 100644
--- a/arch/x86/vdso/Makefile
+++ b/arch/x86/vdso/Makefile
@@ -122,7 +122,7 @@ quiet_cmd_vdso = VDSO    $@
                        $(VDSO_LDFLAGS) $(VDSO_LDFLAGS_$(filter %.lds,$(^F))) \
                        -Wl,-T,$(filter %.lds,$^) $(filter %.o,$^)
 
-VDSO_LDFLAGS = -fPIC -shared $(call ld-option, -Wl$(comma)--hash-style=sysv)
+VDSO_LDFLAGS = -fPIC -shared $(call cc-ldoption, -Wl$(comma)--hash-style=sysv)
 GCOV_PROFILE := n
 
 #
diff --git a/arch/x86/vdso/vclock_gettime.c b/arch/x86/vdso/vclock_gettime.c
index 6a40b78b46aa..ee55754cc3c5 100644
--- a/arch/x86/vdso/vclock_gettime.c
+++ b/arch/x86/vdso/vclock_gettime.c
@@ -86,14 +86,47 @@ notrace static noinline int do_monotonic(struct timespec *ts)
         return 0;
 }
 
+notrace static noinline int do_realtime_coarse(struct timespec *ts)
+{
+        unsigned long seq;
+        do {
+                seq = read_seqbegin(&gtod->lock);
+                ts->tv_sec = gtod->wall_time_coarse.tv_sec;
+                ts->tv_nsec = gtod->wall_time_coarse.tv_nsec;
+        } while (unlikely(read_seqretry(&gtod->lock, seq)));
+        return 0;
+}
+
+notrace static noinline int do_monotonic_coarse(struct timespec *ts)
+{
+        unsigned long seq, ns, secs;
+        do {
+                seq = read_seqbegin(&gtod->lock);
+                secs = gtod->wall_time_coarse.tv_sec;
+                ns = gtod->wall_time_coarse.tv_nsec;
+                secs += gtod->wall_to_monotonic.tv_sec;
+                ns += gtod->wall_to_monotonic.tv_nsec;
+        } while (unlikely(read_seqretry(&gtod->lock, seq)));
+        vset_normalized_timespec(ts, secs, ns);
+        return 0;
+}
+
 notrace int __vdso_clock_gettime(clockid_t clock, struct timespec *ts)
 {
-        if (likely(gtod->sysctl_enabled && gtod->clock.vread))
+        if (likely(gtod->sysctl_enabled))
                 switch (clock) {
                 case CLOCK_REALTIME:
-                        return do_realtime(ts);
+                        if (likely(gtod->clock.vread))
+                                return do_realtime(ts);
+                        break;
                 case CLOCK_MONOTONIC:
-                        return do_monotonic(ts);
+                        if (likely(gtod->clock.vread))
+                                return do_monotonic(ts);
+                        break;
+                case CLOCK_REALTIME_COARSE:
+                        return do_realtime_coarse(ts);
+                case CLOCK_MONOTONIC_COARSE:
+                        return do_monotonic_coarse(ts);
                 }
         return vdso_fallback_gettime(clock, ts);
 }
diff --git a/arch/x86/xen/Makefile b/arch/x86/xen/Makefile
index 7410640db173..3bb4fc21f4f2 100644
--- a/arch/x86/xen/Makefile
+++ b/arch/x86/xen/Makefile
@@ -8,6 +8,7 @@ endif
 # Make sure early boot has no stackprotector
 nostackp := $(call cc-option, -fno-stack-protector)
 CFLAGS_enlighten.o              := $(nostackp)
+CFLAGS_mmu.o                    := $(nostackp)
 
 obj-y           := enlighten.o setup.o multicalls.o mmu.o irq.o \
                         time.o xen-asm.o xen-asm_$(BITS).o \
@@ -16,3 +17,4 @@ obj-y		:= enlighten.o setup.o multicalls.o mmu.o irq.o \
 obj-$(CONFIG_SMP)               += smp.o
 obj-$(CONFIG_PARAVIRT_SPINLOCKS)+= spinlock.o
 obj-$(CONFIG_XEN_DEBUG_FS)      += debugfs.o
+
diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
index eb33aaa8415d..544eb7496531 100644
--- a/arch/x86/xen/enlighten.c
+++ b/arch/x86/xen/enlighten.c
@@ -51,6 +51,7 @@
 #include <asm/pgtable.h>
 #include <asm/tlbflush.h>
 #include <asm/reboot.h>
+#include <asm/stackprotector.h>
 
 #include "xen-ops.h"
 #include "mmu.h"
@@ -330,18 +331,28 @@ static void xen_load_gdt(const struct desc_ptr *dtr)
         unsigned long frames[pages];
         int f;
 
-        /* A GDT can be up to 64k in size, which corresponds to 8192
-           8-byte entries, or 16 4k pages.. */
+        /*
+         * A GDT can be up to 64k in size, which corresponds to 8192
+         * 8-byte entries, or 16 4k pages..
+         */
 
         BUG_ON(size > 65536);
         BUG_ON(va & ~PAGE_MASK);
 
         for (f = 0; va < dtr->address + size; va += PAGE_SIZE, f++) {
                 int level;
-                pte_t *ptep = lookup_address(va, &level);
+                pte_t *ptep;
                 unsigned long pfn, mfn;
                 void *virt;
 
+                /*
+                 * The GDT is per-cpu and is in the percpu data area.
+                 * That can be virtually mapped, so we need to do a
+                 * page-walk to get the underlying MFN for the
+                 * hypercall.  The page can also be in the kernel's
+                 * linear range, so we need to RO that mapping too.
+                 */
+                ptep = lookup_address(va, &level);
                 BUG_ON(ptep == NULL);
 
                 pfn = pte_pfn(*ptep);
@@ -358,6 +369,44 @@ static void xen_load_gdt(const struct desc_ptr *dtr)
                 BUG();
 }
 
+/*
+ * load_gdt for early boot, when the gdt is only mapped once
+ */
+static __init void xen_load_gdt_boot(const struct desc_ptr *dtr)
+{
+        unsigned long va = dtr->address;
+        unsigned int size = dtr->size + 1;
+        unsigned pages = (size + PAGE_SIZE - 1) / PAGE_SIZE;
+        unsigned long frames[pages];
+        int f;
+
+        /*
+         * A GDT can be up to 64k in size, which corresponds to 8192
+         * 8-byte entries, or 16 4k pages..
+         */
+
+        BUG_ON(size > 65536);
+        BUG_ON(va & ~PAGE_MASK);
+
+        for (f = 0; va < dtr->address + size; va += PAGE_SIZE, f++) {
+                pte_t pte;
+                unsigned long pfn, mfn;
+
+                pfn = virt_to_pfn(va);
+                mfn = pfn_to_mfn(pfn);
+
+                pte = pfn_pte(pfn, PAGE_KERNEL_RO);
+
+                if (HYPERVISOR_update_va_mapping((unsigned long)va, pte, 0))
+                        BUG();
+
+                frames[f] = mfn;
+        }
+
+        if (HYPERVISOR_set_gdt(frames, size / sizeof(struct desc_struct)))
+                BUG();
+}
+
 static void load_TLS_descriptor(struct thread_struct *t,
                                 unsigned int cpu, unsigned int i)
 {
@@ -581,6 +630,29 @@ static void xen_write_gdt_entry(struct desc_struct *dt, int entry,
         preempt_enable();
 }
 
+/*
+ * Version of write_gdt_entry for use at early boot-time needed to
+ * update an entry as simply as possible.
+ */
+static __init void xen_write_gdt_entry_boot(struct desc_struct *dt, int entry,
+                                            const void *desc, int type)
+{
+        switch (type) {
+        case DESC_LDT:
+        case DESC_TSS:
+                /* ignore */
+                break;
+
+        default: {
+                xmaddr_t maddr = virt_to_machine(&dt[entry]);
+
+                if (HYPERVISOR_update_descriptor(maddr.maddr, *(u64 *)desc))
+                        dt[entry] = *(struct desc_struct *)desc;
+        }
+
+        }
+}
+
 static void xen_load_sp0(struct tss_struct *tss,
                          struct thread_struct *thread)
 {
@@ -714,7 +786,7 @@ static int xen_write_msr_safe(unsigned int msr, unsigned low, unsigned high)
         set:
                 base = ((u64)high << 32) | low;
                 if (HYPERVISOR_set_segment_base(which, base) != 0)
-                        ret = -EFAULT;
+                        ret = -EIO;
                 break;
 #endif
 
@@ -840,19 +912,9 @@ static const struct pv_info xen_info __initdata = {
 
 static const struct pv_init_ops xen_init_ops __initdata = {
         .patch = xen_patch,
-
-        .banner = xen_banner,
-        .memory_setup = xen_memory_setup,
-        .arch_setup = xen_arch_setup,
-        .post_allocator_init = xen_post_allocator_init,
 };
 
 static const struct pv_time_ops xen_time_ops __initdata = {
-        .time_init = xen_time_init,
-
-        .set_wallclock = xen_set_wallclock,
-        .get_wallclock = xen_get_wallclock,
-        .get_tsc_khz = xen_tsc_khz,
         .sched_clock = xen_sched_clock,
 };
 
@@ -918,8 +980,6 @@ static const struct pv_cpu_ops xen_cpu_ops __initdata = {
 
 static const struct pv_apic_ops xen_apic_ops __initdata = {
 #ifdef CONFIG_X86_LOCAL_APIC
-        .setup_boot_clock = paravirt_nop,
-        .setup_secondary_clock = paravirt_nop,
         .startup_ipi_hook = paravirt_nop,
 #endif
 };
@@ -965,6 +1025,23 @@ static const struct machine_ops __initdata xen_machine_ops = {
         .emergency_restart = xen_emergency_restart,
 };
 
+/*
+ * Set up the GDT and segment registers for -fstack-protector.  Until
+ * we do this, we have to be careful not to call any stack-protected
+ * function, which is most of the kernel.
+ */
+static void __init xen_setup_stackprotector(void)
+{
+        pv_cpu_ops.write_gdt_entry = xen_write_gdt_entry_boot;
+        pv_cpu_ops.load_gdt = xen_load_gdt_boot;
+
+        setup_stack_canary_segment(0);
+        switch_to_new_gdt(0);
+
+        pv_cpu_ops.write_gdt_entry = xen_write_gdt_entry;
+        pv_cpu_ops.load_gdt = xen_load_gdt;
+}
+
 /* First C function to be called on Xen boot */
 asmlinkage void __init xen_start_kernel(void)
 {
@@ -981,16 +1058,43 @@ asmlinkage void __init xen_start_kernel(void)
         pv_time_ops = xen_time_ops;
         pv_cpu_ops = xen_cpu_ops;
         pv_apic_ops = xen_apic_ops;
-        pv_mmu_ops = xen_mmu_ops;
 
-#ifdef CONFIG_X86_64
+        x86_init.resources.memory_setup = xen_memory_setup;
+        x86_init.oem.arch_setup = xen_arch_setup;
+        x86_init.oem.banner = xen_banner;
+
+        x86_init.timers.timer_init = xen_time_init;
+        x86_init.timers.setup_percpu_clockev = x86_init_noop;
+        x86_cpuinit.setup_percpu_clockev = x86_init_noop;
+
+        x86_platform.calibrate_tsc = xen_tsc_khz;
+        x86_platform.get_wallclock = xen_get_wallclock;
+        x86_platform.set_wallclock = xen_set_wallclock;
+
         /*
-         * Setup percpu state.  We only need to do this for 64-bit
-         * because 32-bit already has %fs set properly.
+         * Set up some pagetable state before starting to set any ptes.
          */
-        load_percpu_segment(0);
-#endif
 
+        /* Prevent unwanted bits from being set in PTEs. */
+        __supported_pte_mask &= ~_PAGE_GLOBAL;
+        if (!xen_initial_domain())
+                __supported_pte_mask &= ~(_PAGE_PWT | _PAGE_PCD);
+
+        __supported_pte_mask |= _PAGE_IOMAP;
+
+        xen_setup_features();
+
+        /* Get mfn list */
+        if (!xen_feature(XENFEAT_auto_translated_physmap))
+                xen_build_dynamic_phys_to_machine();
+
+        /*
+         * Set up kernel GDT and segment registers, mainly so that
+         * -fstack-protector code can be executed.
+         */
+        xen_setup_stackprotector();
+
+        xen_init_mmu_ops();
         xen_init_irq_ops();
         xen_init_cpuid_mask();
 
@@ -1001,8 +1105,6 @@ asmlinkage void __init xen_start_kernel(void)
         set_xen_basic_apic_ops();
 #endif
 
-        xen_setup_features();
-
         if (xen_feature(XENFEAT_mmu_pt_update_preserve_ad)) {
                 pv_mmu_ops.ptep_modify_prot_start = xen_ptep_modify_prot_start;
                 pv_mmu_ops.ptep_modify_prot_commit = xen_ptep_modify_prot_commit;
@@ -1019,17 +1121,8 @@ asmlinkage void __init xen_start_kernel(void)
 
         xen_smp_init();
 
-        /* Get mfn list */
-        if (!xen_feature(XENFEAT_auto_translated_physmap))
-                xen_build_dynamic_phys_to_machine();
-
         pgd = (pgd_t *)xen_start_info->pt_base;
 
-        /* Prevent unwanted bits from being set in PTEs. */
-        __supported_pte_mask &= ~_PAGE_GLOBAL;
-        if (!xen_initial_domain())
-                __supported_pte_mask &= ~(_PAGE_PWT | _PAGE_PCD);
-
 #ifdef CONFIG_X86_64
         /* Work out if we support NX */
         check_efer();
diff --git a/arch/x86/xen/irq.c b/arch/x86/xen/irq.c
index cfd17799bd6d..9d30105a0c4a 100644
--- a/arch/x86/xen/irq.c
+++ b/arch/x86/xen/irq.c
@@ -1,5 +1,7 @@
 #include <linux/hardirq.h>
 
+#include <asm/x86_init.h>
+
 #include <xen/interface/xen.h>
 #include <xen/interface/sched.h>
 #include <xen/interface/vcpu.h>
@@ -112,8 +114,6 @@ static void xen_halt(void)
 }
 
 static const struct pv_irq_ops xen_irq_ops __initdata = {
-        .init_IRQ = xen_init_IRQ,
-
         .save_fl = PV_CALLEE_SAVE(xen_save_fl),
         .restore_fl = PV_CALLEE_SAVE(xen_restore_fl),
         .irq_disable = PV_CALLEE_SAVE(xen_irq_disable),
@@ -129,4 +129,5 @@ static const struct pv_irq_ops xen_irq_ops __initdata = {
 void __init xen_init_irq_ops()
 {
         pv_irq_ops = xen_irq_ops;
+        x86_init.irqs.intr_init = xen_init_IRQ;
 }
diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
index 4ceb28581652..3bf7b1d250ce 100644
--- a/arch/x86/xen/mmu.c
+++ b/arch/x86/xen/mmu.c
@@ -1165,14 +1165,14 @@ static void xen_drop_mm_ref(struct mm_struct *mm)
         /* Get the "official" set of cpus referring to our pagetable. */
         if (!alloc_cpumask_var(&mask, GFP_ATOMIC)) {
                 for_each_online_cpu(cpu) {
-                        if (!cpumask_test_cpu(cpu, &mm->cpu_vm_mask)
+                        if (!cpumask_test_cpu(cpu, mm_cpumask(mm))
                             && per_cpu(xen_current_cr3, cpu) != __pa(mm->pgd))
                                 continue;
                         smp_call_function_single(cpu, drop_other_mm_ref, mm, 1);
                 }
                 return;
         }
-        cpumask_copy(mask, &mm->cpu_vm_mask);
+        cpumask_copy(mask, mm_cpumask(mm));
 
         /* It's possible that a vcpu may have a stale reference to our
            cr3, because its in lazy mode, and it hasn't yet flushed
@@ -1229,9 +1229,12 @@ static __init void xen_pagetable_setup_start(pgd_t *base)
 {
 }
 
+static void xen_post_allocator_init(void);
+
 static __init void xen_pagetable_setup_done(pgd_t *base)
 {
         xen_setup_shared_info();
+        xen_post_allocator_init();
 }
 
 static void xen_write_cr2(unsigned long cr2)
@@ -1841,7 +1844,7 @@ static void xen_set_fixmap(unsigned idx, phys_addr_t phys, pgprot_t prot)
 #endif
 }
 
-__init void xen_post_allocator_init(void)
+static __init void xen_post_allocator_init(void)
 {
         pv_mmu_ops.set_pte = xen_set_pte;
         pv_mmu_ops.set_pmd = xen_set_pmd;
@@ -1875,10 +1878,7 @@ static void xen_leave_lazy_mmu(void)
         preempt_enable();
 }
 
-const struct pv_mmu_ops xen_mmu_ops __initdata = {
-        .pagetable_setup_start = xen_pagetable_setup_start,
-        .pagetable_setup_done = xen_pagetable_setup_done,
-
+static const struct pv_mmu_ops xen_mmu_ops __initdata = {
         .read_cr2 = xen_read_cr2,
         .write_cr2 = xen_write_cr2,
 
@@ -1954,6 +1954,12 @@ const struct pv_mmu_ops xen_mmu_ops __initdata = {
         .set_fixmap = xen_set_fixmap,
 };
 
+void __init xen_init_mmu_ops(void)
+{
+        x86_init.paging.pagetable_setup_start = xen_pagetable_setup_start;
+        x86_init.paging.pagetable_setup_done = xen_pagetable_setup_done;
+        pv_mmu_ops = xen_mmu_ops;
+}
 
 #ifdef CONFIG_XEN_DEBUG_FS
 
diff --git a/arch/x86/xen/mmu.h b/arch/x86/xen/mmu.h
index da7302624897..5fe6bc7f5ecf 100644
--- a/arch/x86/xen/mmu.h
+++ b/arch/x86/xen/mmu.h
@@ -59,5 +59,5 @@ void  xen_ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr,
 
 unsigned long xen_read_cr2_direct(void);
 
-extern const struct pv_mmu_ops xen_mmu_ops;
+extern void xen_init_mmu_ops(void);
 #endif  /* _XEN_MMU_H */
diff --git a/arch/x86/xen/smp.c b/arch/x86/xen/smp.c
index 429834ec1687..fe03eeed7b48 100644
--- a/arch/x86/xen/smp.c
+++ b/arch/x86/xen/smp.c
@@ -236,6 +236,7 @@ cpu_initialize_context(unsigned int cpu, struct task_struct *idle)
         ctxt->user_regs.ss = __KERNEL_DS;
 #ifdef CONFIG_X86_32
         ctxt->user_regs.fs = __KERNEL_PERCPU;
+        ctxt->user_regs.gs = __KERNEL_STACK_CANARY;
 #else
         ctxt->gs_base_kernel = per_cpu_offset(cpu);
 #endif
diff --git a/arch/x86/xen/spinlock.c b/arch/x86/xen/spinlock.c
index 5601506f2dd9..36a5141108df 100644
--- a/arch/x86/xen/spinlock.c
+++ b/arch/x86/xen/spinlock.c
@@ -187,7 +187,6 @@ static noinline int xen_spin_lock_slow(struct raw_spinlock *lock, bool irq_enabl
         struct xen_spinlock *prev;
         int irq = __get_cpu_var(lock_kicker_irq);
         int ret;
-        unsigned long flags;
         u64 start;
 
         /* If kicker interrupts not initialized yet, just spin */
@@ -199,16 +198,12 @@ static noinline int xen_spin_lock_slow(struct raw_spinlock *lock, bool irq_enabl
         /* announce we're spinning */
         prev = spinning_lock(xl);
 
-        flags = __raw_local_save_flags();
-        if (irq_enable) {
-                ADD_STATS(taken_slow_irqenable, 1);
-                raw_local_irq_enable();
-        }
-
         ADD_STATS(taken_slow, 1);
         ADD_STATS(taken_slow_nested, prev != NULL);
 
         do {
+                unsigned long flags;
+
                 /* clear pending */
                 xen_clear_irq_pending(irq);
 
@@ -228,6 +223,12 @@ static noinline int xen_spin_lock_slow(struct raw_spinlock *lock, bool irq_enabl
                         goto out;
                 }
 
+                flags = __raw_local_save_flags();
+                if (irq_enable) {
+                        ADD_STATS(taken_slow_irqenable, 1);
+                        raw_local_irq_enable();
+                }
+
                 /*
                  * Block until irq becomes pending.  If we're
                  * interrupted at this point (after the trylock but
@@ -238,13 +239,15 @@ static noinline int xen_spin_lock_slow(struct raw_spinlock *lock, bool irq_enabl
                  * pending.
                  */
                 xen_poll_irq(irq);
+
+                raw_local_irq_restore(flags);
+
                 ADD_STATS(taken_slow_spurious, !xen_test_irq_pending(irq));
         } while (!xen_test_irq_pending(irq)); /* check for spurious wakeups */
 
         kstat_incr_irqs_this_cpu(irq, irq_to_desc(irq));
 
 out:
-        raw_local_irq_restore(flags);
         unspinning_lock(xl, prev);
         spin_time_accum_blocked(start);
 
@@ -323,8 +326,13 @@ static void xen_spin_unlock(struct raw_spinlock *lock)
         smp_wmb();              /* make sure no writes get moved after unlock */
         xl->lock = 0;           /* release lock */
 
-        /* make sure unlock happens before kick */
-        barrier();
+        /*
+         * Make sure unlock happens before checking for waiting
+         * spinners.  We need a strong barrier to enforce the
+         * write-read ordering to different memory locations, as the
+         * CPU makes no implied guarantees about their ordering.
+         */
+        mb();
 
         if (unlikely(xl->spinners))
                 xen_spin_unlock_slow(xl);
diff --git a/arch/x86/xen/xen-ops.h b/arch/x86/xen/xen-ops.h
index 22494fd4c9b5..355fa6b99c9c 100644
--- a/arch/x86/xen/xen-ops.h
+++ b/arch/x86/xen/xen-ops.h
@@ -30,8 +30,6 @@ pgd_t *xen_setup_kernel_pagetable(pgd_t *pgd, unsigned long max_pfn);
 void xen_ident_map_ISA(void);
 void xen_reserve_top(void);
 
-void xen_post_allocator_init(void);
-
 char * __init xen_memory_setup(void);
 void __init xen_arch_setup(void);
 void __init xen_init_IRQ(void);