Merge branch 'linus' into x86/mce3

Conflicts:
	arch/x86/kernel/cpu/mcheck/mce_64.c
	arch/x86/kernel/irq.c

Merge reason: Resolve the conflicts above.

Signed-off-by: Ingo Molnar <mingo@elte.hu>

205 files changed, 8485 insertions, 4741 deletions

diff --git a/arch/x86/Kbuild b/arch/x86/Kbuild
new file mode 100644
index 000000000000..ad8ec356fb36
--- /dev/null
+++ b/arch/x86/Kbuild
@@ -0,0 +1,16 @@
+
+obj-$(CONFIG_KVM) += kvm/
+
+# Xen paravirtualization support
+obj-$(CONFIG_XEN) += xen/
+
+# lguest paravirtualization support
+obj-$(CONFIG_LGUEST_GUEST) += lguest/
+
+obj-y += kernel/
+obj-y += mm/
+
+obj-y += crypto/
+obj-y += vdso/
+obj-$(CONFIG_IA32_EMULATION) += ia32/
+
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index afd1168eeefb..356d2ec8e2fb 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -47,6 +47,11 @@ config X86
         select HAVE_KERNEL_BZIP2
         select HAVE_KERNEL_LZMA
 
+config OUTPUT_FORMAT
+        string
+        default "elf32-i386" if X86_32
+        default "elf64-x86-64" if X86_64
+
 config ARCH_DEFCONFIG
         string
         default "arch/x86/configs/i386_defconfig" if X86_32
@@ -734,6 +739,7 @@ config X86_UP_IOAPIC
 config X86_LOCAL_APIC
         def_bool y
         depends on X86_64 || SMP || X86_32_NON_STANDARD || X86_UP_APIC
+        select HAVE_PERF_COUNTERS if (!M386 && !M486)
 
 config X86_IO_APIC
         def_bool y
@@ -1497,9 +1503,7 @@ config KEXEC_JUMP
 
 config PHYSICAL_START
         hex "Physical address where the kernel is loaded" if (EMBEDDED || CRASH_DUMP)
-        default "0x1000000" if X86_NUMAQ
-        default "0x200000" if X86_64
-        default "0x100000"
+        default "0x1000000"
         ---help---
           This gives the physical address where the kernel is loaded.
 
@@ -1518,15 +1522,15 @@ config PHYSICAL_START
           to be specifically compiled to run from a specific memory area
           (normally a reserved region) and this option comes handy.
 
-          So if you are using bzImage for capturing the crash dump, leave
-          the value here unchanged to 0x100000 and set CONFIG_RELOCATABLE=y.
-          Otherwise if you plan to use vmlinux for capturing the crash dump
-          change this value to start of the reserved region (Typically 16MB
-          0x1000000). In other words, it can be set based on the "X" value as
-          specified in the "crashkernel=YM@XM" command line boot parameter
-          passed to the panic-ed kernel. Typically this parameter is set as
-          crashkernel=64M@16M. Please take a look at
-          Documentation/kdump/kdump.txt for more details about crash dumps.
+          So if you are using bzImage for capturing the crash dump,
+          leave the value here unchanged to 0x1000000 and set
+          CONFIG_RELOCATABLE=y.  Otherwise if you plan to use vmlinux
+          for capturing the crash dump change this value to start of
+          the reserved region.  In other words, it can be set based on
+          the "X" value as specified in the "crashkernel=YM@XM"
+          command line boot parameter passed to the panic-ed
+          kernel. Please take a look at Documentation/kdump/kdump.txt
+          for more details about crash dumps.
 
           Usage of bzImage for capturing the crash dump is recommended as
           one does not have to build two kernels. Same kernel can be used
@@ -1539,8 +1543,8 @@ config PHYSICAL_START
           Don't change this unless you know what you are doing.
 
 config RELOCATABLE
-        bool "Build a relocatable kernel (EXPERIMENTAL)"
-        depends on EXPERIMENTAL
+        bool "Build a relocatable kernel"
+        default y
         ---help---
           This builds a kernel image that retains relocation information
           so it can be loaded someplace besides the default 1MB.
@@ -1555,12 +1559,16 @@ config RELOCATABLE
           it has been loaded at and the compile time physical address
           (CONFIG_PHYSICAL_START) is ignored.
 
+# Relocation on x86-32 needs some additional build support
+config X86_NEED_RELOCS
+        def_bool y
+        depends on X86_32 && RELOCATABLE
+
 config PHYSICAL_ALIGN
         hex
         prompt "Alignment value to which kernel should be aligned" if X86_32
-        default "0x100000" if X86_32
-        default "0x200000" if X86_64
-        range 0x2000 0x400000
+        default "0x1000000"
+        range 0x2000 0x1000000
         ---help---
           This value puts the alignment restrictions on physical address
           where kernel is loaded and run from. Kernel is compiled for an
diff --git a/arch/x86/Kconfig.debug b/arch/x86/Kconfig.debug
index d8359e73317f..d105f29bb6bb 100644
--- a/arch/x86/Kconfig.debug
+++ b/arch/x86/Kconfig.debug
@@ -159,14 +159,30 @@ config IOMMU_DEBUG
           options. See Documentation/x86_64/boot-options.txt for more
           details.
 
+config IOMMU_STRESS
+        bool "Enable IOMMU stress-test mode"
+        ---help---
+          This option disables various optimizations in IOMMU related
+          code to do real stress testing of the IOMMU code. This option
+          will cause a performance drop and should only be enabled for
+          testing.
+
 config IOMMU_LEAK
         bool "IOMMU leak tracing"
-        depends on DEBUG_KERNEL
-        depends on IOMMU_DEBUG
+        depends on IOMMU_DEBUG && DMA_API_DEBUG
         ---help---
           Add a simple leak tracer to the IOMMU code. This is useful when you
           are debugging a buggy device driver that leaks IOMMU mappings.
 
+config X86_DS_SELFTEST
+    bool "DS selftest"
+    default y
+    depends on DEBUG_KERNEL
+    depends on X86_DS
+        ---help---
+          Perform Debug Store selftests at boot time.
+          If in doubt, say "N".
+
 config HAVE_MMIOTRACE_SUPPORT
         def_bool y
 
diff --git a/arch/x86/Makefile b/arch/x86/Makefile
index 8c86b72afdc2..edbd0ca62067 100644
--- a/arch/x86/Makefile
+++ b/arch/x86/Makefile
@@ -7,8 +7,6 @@ else
         KBUILD_DEFCONFIG := $(ARCH)_defconfig
 endif
 
-core-$(CONFIG_KVM) += arch/x86/kvm/
-
 # BITS is used as extension for files which are available in a 32 bit
 # and a 64 bit version to simplify shared Makefiles.
 # e.g.: obj-y += foo_$(BITS).o
@@ -118,21 +116,8 @@ head-y += arch/x86/kernel/init_task.o
 
 libs-y  += arch/x86/lib/
 
-# Sub architecture files that needs linking first
-core-y += $(fcore-y)
-
-# Xen paravirtualization support
-core-$(CONFIG_XEN) += arch/x86/xen/
-
-# lguest paravirtualization support
-core-$(CONFIG_LGUEST_GUEST) += arch/x86/lguest/
-
-core-y += arch/x86/kernel/
-core-y += arch/x86/mm/
-
-core-y += arch/x86/crypto/
-core-y += arch/x86/vdso/
-core-$(CONFIG_IA32_EMULATION) += arch/x86/ia32/
+# See arch/x86/Kbuild for content of core part of the kernel
+core-y += arch/x86/
 
 # drivers-y are linked after core-y
 drivers-$(CONFIG_MATH_EMULATION) += arch/x86/math-emu/
diff --git a/arch/x86/boot/.gitignore b/arch/x86/boot/.gitignore
index 172cf8a98bdd..851fe936d242 100644
--- a/arch/x86/boot/.gitignore
+++ b/arch/x86/boot/.gitignore
@@ -3,6 +3,8 @@ bzImage
 cpustr.h
 mkcpustr
 offsets.h
+voffset.h
+zoffset.h
 setup
 setup.bin
 setup.elf
diff --git a/arch/x86/boot/Makefile b/arch/x86/boot/Makefile
index 6633b6e7505a..8d16ada25048 100644
--- a/arch/x86/boot/Makefile
+++ b/arch/x86/boot/Makefile
@@ -26,9 +26,10 @@ targets		:= vmlinux.bin setup.bin setup.elf bzImage
 targets         += fdimage fdimage144 fdimage288 image.iso mtools.conf
 subdir-         := compressed
 
-setup-y         += a20.o cmdline.o copy.o cpu.o cpucheck.o edd.o
+setup-y         += a20.o bioscall.o cmdline.o copy.o cpu.o cpucheck.o edd.o
 setup-y         += header.o main.o mca.o memory.o pm.o pmjump.o
-setup-y         += printf.o string.o tty.o video.o video-mode.o version.o
+setup-y         += printf.o regs.o string.o tty.o video.o video-mode.o
+setup-y         += version.o
 setup-$(CONFIG_X86_APM_BOOT) += apm.o
 
 # The link order of the video-*.o modules can matter.  In particular,
@@ -86,19 +87,27 @@ $(obj)/vmlinux.bin: $(obj)/compressed/vmlinux FORCE
 
 SETUP_OBJS = $(addprefix $(obj)/,$(setup-y))
 
-sed-offsets := -e 's/^00*/0/' \
-        -e 's/^\([0-9a-fA-F]*\) . \(input_data\|input_data_end\)$$/\#define \2 0x\1/p'
+sed-voffset := -e 's/^\([0-9a-fA-F]*\) . \(_text\|_end\)$$/\#define VO_\2 0x\1/p'
 
-quiet_cmd_offsets = OFFSETS $@
-      cmd_offsets = $(NM) $< | sed -n $(sed-offsets) > $@
+quiet_cmd_voffset = VOFFSET $@
+      cmd_voffset = $(NM) $< | sed -n $(sed-voffset) > $@
 
-$(obj)/offsets.h: $(obj)/compressed/vmlinux FORCE
-        $(call if_changed,offsets)
+targets += voffset.h
+$(obj)/voffset.h: vmlinux FORCE
+        $(call if_changed,voffset)
+
+sed-zoffset := -e 's/^\([0-9a-fA-F]*\) . \(startup_32\|input_data\|_end\|z_.*\)$$/\#define ZO_\2 0x\1/p'
+
+quiet_cmd_zoffset = ZOFFSET $@
+      cmd_zoffset = $(NM) $< | sed -n $(sed-zoffset) > $@
+
+targets += zoffset.h
+$(obj)/zoffset.h: $(obj)/compressed/vmlinux FORCE
+        $(call if_changed,zoffset)
 
-targets += offsets.h
 
 AFLAGS_header.o += -I$(obj)
-$(obj)/header.o: $(obj)/offsets.h
+$(obj)/header.o: $(obj)/voffset.h $(obj)/zoffset.h
 
 LDFLAGS_setup.elf       := -T
 $(obj)/setup.elf: $(src)/setup.ld $(SETUP_OBJS) FORCE
diff --git a/arch/x86/boot/a20.c b/arch/x86/boot/a20.c
index 7c19ce8c2442..64a31a6d751a 100644
--- a/arch/x86/boot/a20.c
+++ b/arch/x86/boot/a20.c
@@ -2,7 +2,7 @@
  *
  *   Copyright (C) 1991, 1992 Linus Torvalds
  *   Copyright 2007-2008 rPath, Inc. - All Rights Reserved
- *   Copyright 2009 Intel Corporation
+ *   Copyright 2009 Intel Corporation; author H. Peter Anvin
  *
  *   This file is part of the Linux kernel, and is made available under
  *   the terms of the GNU General Public License version 2.
@@ -90,8 +90,11 @@ static int a20_test_long(void)
 
 static void enable_a20_bios(void)
 {
-        asm volatile("pushfl; int $0x15; popfl"
-                     : : "a" ((u16)0x2401));
+        struct biosregs ireg;
+
+        initregs(&ireg);
+        ireg.ax = 0x2401;
+        intcall(0x15, &ireg, NULL);
 }
 
 static void enable_a20_kbc(void)
diff --git a/arch/x86/boot/apm.c b/arch/x86/boot/apm.c
index 7aa6033001f9..ee274834ea8b 100644
--- a/arch/x86/boot/apm.c
+++ b/arch/x86/boot/apm.c
@@ -2,6 +2,7 @@
  *
  *   Copyright (C) 1991, 1992 Linus Torvalds
  *   Copyright 2007 rPath, Inc. - All Rights Reserved
+ *   Copyright 2009 Intel Corporation; author H. Peter Anvin
  *
  *   Original APM BIOS checking by Stephen Rothwell, May 1994
  *   (sfr@canb.auug.org.au)
@@ -19,75 +20,56 @@
 
 int query_apm_bios(void)
 {
-        u16 ax, bx, cx, dx, di;
-        u32 ebx, esi;
-        u8 err;
+        struct biosregs ireg, oreg;
 
         /* APM BIOS installation check */
-        ax = 0x5300;
-        bx = cx = 0;
-        asm volatile("pushl %%ebp ; int $0x15 ; popl %%ebp ; setc %0"
-                     : "=d" (err), "+a" (ax), "+b" (bx), "+c" (cx)
-                     : : "esi", "edi");
+        initregs(&ireg);
+        ireg.ah = 0x53;
+        intcall(0x15, &ireg, &oreg);
 
-        if (err)
+        if (oreg.flags & X86_EFLAGS_CF)
                 return -1;              /* No APM BIOS */
 
-        if (bx != 0x504d)       /* "PM" signature */
+        if (oreg.bx != 0x504d)          /* "PM" signature */
                 return -1;
 
-        if (!(cx & 0x02))               /* 32 bits supported? */
+        if (!(oreg.cx & 0x02))          /* 32 bits supported? */
                 return -1;
 
         /* Disconnect first, just in case */
-        ax = 0x5304;
-        bx = 0;
-        asm volatile("pushl %%ebp ; int $0x15 ; popl %%ebp"
-                     : "+a" (ax), "+b" (bx)
-                     : : "ecx", "edx", "esi", "edi");
-
-        /* Paranoia */
-        ebx = esi = 0;
-        cx = dx = di = 0;
+        ireg.al = 0x04;
+        intcall(0x15, &ireg, NULL);
 
         /* 32-bit connect */
-        asm volatile("pushl %%ebp ; int $0x15 ; popl %%ebp ; setc %6"
-                     : "=a" (ax), "+b" (ebx), "+c" (cx), "+d" (dx),
-                       "+S" (esi), "+D" (di), "=m" (err)
-                     : "a" (0x5303));
-
-        boot_params.apm_bios_info.cseg = ax;
-        boot_params.apm_bios_info.offset = ebx;
-        boot_params.apm_bios_info.cseg_16 = cx;
-        boot_params.apm_bios_info.dseg = dx;
-        boot_params.apm_bios_info.cseg_len = (u16)esi;
-        boot_params.apm_bios_info.cseg_16_len = esi >> 16;
-        boot_params.apm_bios_info.dseg_len = di;
-
-        if (err)
+        ireg.al = 0x03;
+        intcall(0x15, &ireg, &oreg);
+
+        boot_params.apm_bios_info.cseg        = oreg.ax;
+        boot_params.apm_bios_info.offset      = oreg.ebx;
+        boot_params.apm_bios_info.cseg_16     = oreg.cx;
+        boot_params.apm_bios_info.dseg        = oreg.dx;
+        boot_params.apm_bios_info.cseg_len    = oreg.si;
+        boot_params.apm_bios_info.cseg_16_len = oreg.hsi;
+        boot_params.apm_bios_info.dseg_len    = oreg.di;
+
+        if (oreg.flags & X86_EFLAGS_CF)
                 return -1;
 
         /* Redo the installation check as the 32-bit connect;
            some BIOSes return different flags this way... */
 
-        ax = 0x5300;
-        bx = cx = 0;
-        asm volatile("pushl %%ebp ; int $0x15 ; popl %%ebp ; setc %0"
-                     : "=d" (err), "+a" (ax), "+b" (bx), "+c" (cx)
-                     : : "esi", "edi");
+        ireg.al = 0x00;
+        intcall(0x15, &ireg, &oreg);
 
-        if (err || bx != 0x504d) {
+        if ((oreg.eflags & X86_EFLAGS_CF) || oreg.bx != 0x504d) {
                 /* Failure with 32-bit connect, try to disconect and ignore */
-                ax = 0x5304;
-                bx = 0;
-                asm volatile("pushl %%ebp ; int $0x15 ; popl %%ebp"
-                             : "+a" (ax), "+b" (bx)
-                             : : "ecx", "edx", "esi", "edi");
+                ireg.al = 0x04;
+                intcall(0x15, &ireg, NULL);
                 return -1;
         }
 
-        boot_params.apm_bios_info.version = ax;
-        boot_params.apm_bios_info.flags = cx;
+        boot_params.apm_bios_info.version = oreg.ax;
+        boot_params.apm_bios_info.flags   = oreg.cx;
         return 0;
 }
 
diff --git a/arch/x86/boot/bioscall.S b/arch/x86/boot/bioscall.S
new file mode 100644
index 000000000000..507793739ea5
--- /dev/null
+++ b/arch/x86/boot/bioscall.S
@@ -0,0 +1,82 @@
+/* -----------------------------------------------------------------------
+ *
+ *   Copyright 2009 Intel Corporation; author H. Peter Anvin
+ *
+ *   This file is part of the Linux kernel, and is made available under
+ *   the terms of the GNU General Public License version 2 or (at your
+ *   option) any later version; incorporated herein by reference.
+ *
+ * ----------------------------------------------------------------------- */
+
+/*
+ * "Glove box" for BIOS calls.  Avoids the constant problems with BIOSes
+ * touching registers they shouldn't be.
+ */
+
+        .code16
+        .text
+        .globl  intcall
+        .type   intcall, @function
+intcall:
+        /* Self-modify the INT instruction.  Ugly, but works. */
+        cmpb    %al, 3f
+        je      1f
+        movb    %al, 3f
+        jmp     1f              /* Synchronize pipeline */
+1:
+        /* Save state */
+        pushfl
+        pushw   %fs
+        pushw   %gs
+        pushal
+
+        /* Copy input state to stack frame */
+        subw    $44, %sp
+        movw    %dx, %si
+        movw    %sp, %di
+        movw    $11, %cx
+        rep; movsd
+
+        /* Pop full state from the stack */
+        popal
+        popw    %gs
+        popw    %fs
+        popw    %es
+        popw    %ds
+        popfl
+
+        /* Actual INT */
+        .byte   0xcd            /* INT opcode */
+3:      .byte   0
+
+        /* Push full state to the stack */
+        pushfl
+        pushw   %ds
+        pushw   %es
+        pushw   %fs
+        pushw   %gs
+        pushal
+
+        /* Re-establish C environment invariants */
+        cld
+        movzwl  %sp, %esp
+        movw    %cs, %ax
+        movw    %ax, %ds
+        movw    %ax, %es
+
+        /* Copy output state from stack frame */
+        movw    68(%esp), %di   /* Original %cx == 3rd argument */
+        andw    %di, %di
+        jz      4f
+        movw    %sp, %si
+        movw    $11, %cx
+        rep; movsd
+4:      addw    $44, %sp
+
+        /* Restore state and return */
+        popal
+        popw    %gs
+        popw    %fs
+        popfl
+        retl
+        .size   intcall, .-intcall
diff --git a/arch/x86/boot/boot.h b/arch/x86/boot/boot.h
index 7b2692e897e5..98239d2658f2 100644
--- a/arch/x86/boot/boot.h
+++ b/arch/x86/boot/boot.h
@@ -2,6 +2,7 @@
  *
  *   Copyright (C) 1991, 1992 Linus Torvalds
  *   Copyright 2007 rPath, Inc. - All Rights Reserved
+ *   Copyright 2009 Intel Corporation; author H. Peter Anvin
  *
  *   This file is part of the Linux kernel, and is made available under
  *   the terms of the GNU General Public License version 2.
@@ -26,6 +27,7 @@
 #include <asm/setup.h>
 #include "bitops.h"
 #include <asm/cpufeature.h>
+#include <asm/processor-flags.h>
 
 /* Useful macros */
 #define BUILD_BUG_ON(condition) ((void)sizeof(char[1 - 2*!!(condition)]))
@@ -241,6 +243,49 @@ int enable_a20(void);
 /* apm.c */
 int query_apm_bios(void);
 
+/* bioscall.c */
+struct biosregs {
+        union {
+                struct {
+                        u32 edi;
+                        u32 esi;
+                        u32 ebp;
+                        u32 _esp;
+                        u32 ebx;
+                        u32 edx;
+                        u32 ecx;
+                        u32 eax;
+                        u32 _fsgs;
+                        u32 _dses;
+                        u32 eflags;
+                };
+                struct {
+                        u16 di, hdi;
+                        u16 si, hsi;
+                        u16 bp, hbp;
+                        u16 _sp, _hsp;
+                        u16 bx, hbx;
+                        u16 dx, hdx;
+                        u16 cx, hcx;
+                        u16 ax, hax;
+                        u16 gs, fs;
+                        u16 es, ds;
+                        u16 flags, hflags;
+                };
+                struct {
+                        u8 dil, dih, edi2, edi3;
+                        u8 sil, sih, esi2, esi3;
+                        u8 bpl, bph, ebp2, ebp3;
+                        u8 _spl, _sph, _esp2, _esp3;
+                        u8 bl, bh, ebx2, ebx3;
+                        u8 dl, dh, edx2, edx3;
+                        u8 cl, ch, ecx2, ecx3;
+                        u8 al, ah, eax2, eax3;
+                };
+        };
+};
+void intcall(u8 int_no, const struct biosregs *ireg, struct biosregs *oreg);
+
 /* cmdline.c */
 int cmdline_find_option(const char *option, char *buffer, int bufsize);
 int cmdline_find_option_bool(const char *option);
@@ -279,6 +324,9 @@ int sprintf(char *buf, const char *fmt, ...);
 int vsprintf(char *buf, const char *fmt, va_list args);
 int printf(const char *fmt, ...);
 
+/* regs.c */
+void initregs(struct biosregs *regs);
+
 /* string.c */
 int strcmp(const char *str1, const char *str2);
 size_t strnlen(const char *s, size_t maxlen);
diff --git a/arch/x86/boot/compressed/.gitignore b/arch/x86/boot/compressed/.gitignore
index 63eff3b04d01..4a46fab7162e 100644
--- a/arch/x86/boot/compressed/.gitignore
+++ b/arch/x86/boot/compressed/.gitignore
@@ -1,3 +1,6 @@
 relocs
 vmlinux.bin.all
 vmlinux.relocs
+vmlinux.lds
+mkpiggy
+piggy.S
diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile
index 65551c9f8571..49c8a4c37d7c 100644
--- a/arch/x86/boot/compressed/Makefile
+++ b/arch/x86/boot/compressed/Makefile
@@ -19,7 +19,9 @@ KBUILD_AFLAGS  := $(KBUILD_CFLAGS) -D__ASSEMBLY__
 LDFLAGS := -m elf_$(UTS_MACHINE)
 LDFLAGS_vmlinux := -T
 
-$(obj)/vmlinux: $(src)/vmlinux_$(BITS).lds $(obj)/head_$(BITS).o $(obj)/misc.o $(obj)/piggy.o FORCE
+hostprogs-y     := mkpiggy
+
+$(obj)/vmlinux: $(obj)/vmlinux.lds $(obj)/head_$(BITS).o $(obj)/misc.o $(obj)/piggy.o FORCE
         $(call if_changed,ld)
         @:
 
@@ -29,7 +31,7 @@ $(obj)/vmlinux.bin: vmlinux FORCE
 
 
 targets += vmlinux.bin.all vmlinux.relocs relocs
-hostprogs-$(CONFIG_X86_32) += relocs
+hostprogs-$(CONFIG_X86_NEED_RELOCS) += relocs
 
 quiet_cmd_relocs = RELOCS  $@
       cmd_relocs = $(obj)/relocs $< > $@;$(obj)/relocs --abs-relocs $<
@@ -37,46 +39,22 @@ $(obj)/vmlinux.relocs: vmlinux $(obj)/relocs FORCE
         $(call if_changed,relocs)
 
 vmlinux.bin.all-y := $(obj)/vmlinux.bin
-vmlinux.bin.all-$(CONFIG_RELOCATABLE) += $(obj)/vmlinux.relocs
-quiet_cmd_relocbin = BUILD   $@
-      cmd_relocbin = cat $(filter-out FORCE,$^) > $@
-$(obj)/vmlinux.bin.all: $(vmlinux.bin.all-y) FORCE
-        $(call if_changed,relocbin)
-
-ifeq ($(CONFIG_X86_32),y)
+vmlinux.bin.all-$(CONFIG_X86_NEED_RELOCS) += $(obj)/vmlinux.relocs
 
-ifdef CONFIG_RELOCATABLE
-$(obj)/vmlinux.bin.gz: $(obj)/vmlinux.bin.all FORCE
-        $(call if_changed,gzip)
-$(obj)/vmlinux.bin.bz2: $(obj)/vmlinux.bin.all FORCE
-        $(call if_changed,bzip2)
-$(obj)/vmlinux.bin.lzma: $(obj)/vmlinux.bin.all FORCE
-        $(call if_changed,lzma)
-else
-$(obj)/vmlinux.bin.gz: $(obj)/vmlinux.bin FORCE
+$(obj)/vmlinux.bin.gz: $(vmlinux.bin.all-y) FORCE
         $(call if_changed,gzip)
-$(obj)/vmlinux.bin.bz2: $(obj)/vmlinux.bin FORCE
+$(obj)/vmlinux.bin.bz2: $(vmlinux.bin.all-y) FORCE
         $(call if_changed,bzip2)
-$(obj)/vmlinux.bin.lzma: $(obj)/vmlinux.bin FORCE
+$(obj)/vmlinux.bin.lzma: $(vmlinux.bin.all-y) FORCE
         $(call if_changed,lzma)
-endif
-LDFLAGS_piggy.o := -r --format binary --oformat elf32-i386 -T
 
-else
+suffix-$(CONFIG_KERNEL_GZIP)    := gz
+suffix-$(CONFIG_KERNEL_BZIP2)   := bz2
+suffix-$(CONFIG_KERNEL_LZMA)    := lzma
 
-$(obj)/vmlinux.bin.gz: $(obj)/vmlinux.bin FORCE
-        $(call if_changed,gzip)
-$(obj)/vmlinux.bin.bz2: $(obj)/vmlinux.bin FORCE
-        $(call if_changed,bzip2)
-$(obj)/vmlinux.bin.lzma: $(obj)/vmlinux.bin FORCE
-        $(call if_changed,lzma)
-
-LDFLAGS_piggy.o := -r --format binary --oformat elf64-x86-64 -T
-endif
+quiet_cmd_mkpiggy = MKPIGGY $@
+      cmd_mkpiggy = $(obj)/mkpiggy $< > $@ || ( rm -f $@ ; false )
 
-suffix_$(CONFIG_KERNEL_GZIP)  = gz
-suffix_$(CONFIG_KERNEL_BZIP2) = bz2
-suffix_$(CONFIG_KERNEL_LZMA)  = lzma
-
-$(obj)/piggy.o: $(obj)/vmlinux.scr $(obj)/vmlinux.bin.$(suffix_y) FORCE
-        $(call if_changed,ld)
+targets += piggy.S
+$(obj)/piggy.S: $(obj)/vmlinux.bin.$(suffix-y) $(obj)/mkpiggy FORCE
+        $(call if_changed,mkpiggy)
diff --git a/arch/x86/boot/compressed/head_32.S b/arch/x86/boot/compressed/head_32.S
index 3a8a866fb2e2..75e4f001e706 100644
--- a/arch/x86/boot/compressed/head_32.S
+++ b/arch/x86/boot/compressed/head_32.S
@@ -12,16 +12,16 @@
  * the page directory. [According to comments etc elsewhere on a compressed
  * kernel it will end up at 0x1000 + 1Mb I hope so as I assume this. - AC]
  *
- * Page 0 is deliberately kept safe, since System Management Mode code in 
+ * Page 0 is deliberately kept safe, since System Management Mode code in
  * laptops may need to access the BIOS data stored there.  This is also
- * useful for future device drivers that either access the BIOS via VM86 
+ * useful for future device drivers that either access the BIOS via VM86
  * mode.
  */
 
 /*
  * High loaded stuff by Hans Lermen & Werner Almesberger, Feb. 1996
  */
-.text
+        .text
 
 #include <linux/linkage.h>
 #include <asm/segment.h>
@@ -29,161 +29,151 @@
 #include <asm/boot.h>
 #include <asm/asm-offsets.h>
 
-.section ".text.head","ax",@progbits
+        .section ".text.head","ax",@progbits
 ENTRY(startup_32)
         cld
-        /* test KEEP_SEGMENTS flag to see if the bootloader is asking
-         * us to not reload segments */
-        testb $(1<<6), BP_loadflags(%esi)
-        jnz 1f
+        /*
+         * Test KEEP_SEGMENTS flag to see if the bootloader is asking
+         * us to not reload segments
+         */
+        testb   $(1<<6), BP_loadflags(%esi)
+        jnz     1f
 
         cli
-        movl $(__BOOT_DS),%eax
-        movl %eax,%ds
-        movl %eax,%es
-        movl %eax,%fs
-        movl %eax,%gs
-        movl %eax,%ss
+        movl    $__BOOT_DS, %eax
+        movl    %eax, %ds
+        movl    %eax, %es
+        movl    %eax, %fs
+        movl    %eax, %gs
+        movl    %eax, %ss
 1:
 
-/* Calculate the delta between where we were compiled to run
+/*
+ * Calculate the delta between where we were compiled to run
  * at and where we were actually loaded at.  This can only be done
  * with a short local call on x86.  Nothing  else will tell us what
  * address we are running at.  The reserved chunk of the real-mode
  * data at 0x1e4 (defined as a scratch field) are used as the stack
  * for this calculation. Only 4 bytes are needed.
  */
-        leal (0x1e4+4)(%esi), %esp
-        call 1f
-1:      popl %ebp
-        subl $1b, %ebp
+        leal    (BP_scratch+4)(%esi), %esp
+        call    1f
+1:      popl    %ebp
+        subl    $1b, %ebp
 
-/* %ebp contains the address we are loaded at by the boot loader and %ebx
+/*
+ * %ebp contains the address we are loaded at by the boot loader and %ebx
  * contains the address where we should move the kernel image temporarily
  * for safe in-place decompression.
  */
 
 #ifdef CONFIG_RELOCATABLE
-        movl    %ebp, %ebx
-        addl    $(CONFIG_PHYSICAL_ALIGN - 1), %ebx
-        andl    $(~(CONFIG_PHYSICAL_ALIGN - 1)), %ebx
+        movl    %ebp, %ebx
+        movl    BP_kernel_alignment(%esi), %eax
+        decl    %eax
+        addl    %eax, %ebx
+        notl    %eax
+        andl    %eax, %ebx
 #else
-        movl $LOAD_PHYSICAL_ADDR, %ebx
+        movl    $LOAD_PHYSICAL_ADDR, %ebx
 #endif
 
-        /* Replace the compressed data size with the uncompressed size */
-        subl input_len(%ebp), %ebx
-        movl output_len(%ebp), %eax
-        addl %eax, %ebx
-        /* Add 8 bytes for every 32K input block */
-        shrl $12, %eax
-        addl %eax, %ebx
-        /* Add 32K + 18 bytes of extra slack */
-        addl $(32768 + 18), %ebx
-        /* Align on a 4K boundary */
-        addl $4095, %ebx
-        andl $~4095, %ebx
-
-/* Copy the compressed kernel to the end of our buffer
+        /* Target address to relocate to for decompression */
+        addl    $z_extract_offset, %ebx
+
+        /* Set up the stack */
+        leal    boot_stack_end(%ebx), %esp
+
+        /* Zero EFLAGS */
+        pushl   $0
+        popfl
+
+/*
+ * Copy the compressed kernel to the end of our buffer
  * where decompression in place becomes safe.
  */
-        pushl %esi
-        leal _end(%ebp), %esi
-        leal _end(%ebx), %edi
-        movl $(_end - startup_32), %ecx
+        pushl   %esi
+        leal    (_bss-4)(%ebp), %esi
+        leal    (_bss-4)(%ebx), %edi
+        movl    $(_bss - startup_32), %ecx
+        shrl    $2, %ecx
         std
-        rep
-        movsb
+        rep     movsl
         cld
-        popl %esi
-
-/* Compute the kernel start address.
- */
-#ifdef CONFIG_RELOCATABLE
-        addl    $(CONFIG_PHYSICAL_ALIGN - 1), %ebp
-        andl    $(~(CONFIG_PHYSICAL_ALIGN - 1)), %ebp
-#else
-        movl    $LOAD_PHYSICAL_ADDR, %ebp
-#endif
+        popl    %esi
 
 /*
  * Jump to the relocated address.
  */
-        leal relocated(%ebx), %eax
-        jmp *%eax
+        leal    relocated(%ebx), %eax
+        jmp     *%eax
 ENDPROC(startup_32)
 
-.section ".text"
+        .text
 relocated:
 
 /*
- * Clear BSS
- */
-        xorl %eax,%eax
-        leal _edata(%ebx),%edi
-        leal _end(%ebx), %ecx
-        subl %edi,%ecx
-        cld
-        rep
-        stosb
-
-/*
- * Setup the stack for the decompressor
+ * Clear BSS (stack is currently empty)
  */
-        leal boot_stack_end(%ebx), %esp
+        xorl    %eax, %eax
+        leal    _bss(%ebx), %edi
+        leal    _ebss(%ebx), %ecx
+        subl    %edi, %ecx
+        shrl    $2, %ecx
+        rep     stosl
 
 /*
  * Do the decompression, and jump to the new kernel..
  */
-        movl output_len(%ebx), %eax
-        pushl %eax
-                        # push arguments for decompress_kernel:
-        pushl %ebp      # output address
-        movl input_len(%ebx), %eax
-        pushl %eax      # input_len
-        leal input_data(%ebx), %eax
-        pushl %eax      # input_data
-        leal boot_heap(%ebx), %eax
-        pushl %eax      # heap area
-        pushl %esi      # real mode pointer
-        call decompress_kernel
-        addl $20, %esp
-        popl %ecx
+        leal    z_extract_offset_negative(%ebx), %ebp
+                                /* push arguments for decompress_kernel: */
+        pushl   %ebp            /* output address */
+        pushl   $z_input_len    /* input_len */
+        leal    input_data(%ebx), %eax
+        pushl   %eax            /* input_data */
+        leal    boot_heap(%ebx), %eax
+        pushl   %eax            /* heap area */
+        pushl   %esi            /* real mode pointer */
+        call    decompress_kernel
+        addl    $20, %esp
 
 #if CONFIG_RELOCATABLE
-/* Find the address of the relocations.
+/*
+ * Find the address of the relocations.
  */
-        movl %ebp, %edi
-        addl %ecx, %edi
+        leal    z_output_len(%ebp), %edi
 
-/* Calculate the delta between where vmlinux was compiled to run
+/*
+ * Calculate the delta between where vmlinux was compiled to run
  * and where it was actually loaded.
  */
-        movl %ebp, %ebx
-        subl $LOAD_PHYSICAL_ADDR, %ebx
-        jz   2f         /* Nothing to be done if loaded at compiled addr. */
+        movl    %ebp, %ebx
+        subl    $LOAD_PHYSICAL_ADDR, %ebx
+        jz      2f      /* Nothing to be done if loaded at compiled addr. */
 /*
  * Process relocations.
  */
 
-1:      subl $4, %edi
-        movl 0(%edi), %ecx
-        testl %ecx, %ecx
-        jz 2f
-        addl %ebx, -__PAGE_OFFSET(%ebx, %ecx)
-        jmp 1b
+1:      subl    $4, %edi
+        movl    (%edi), %ecx
+        testl   %ecx, %ecx
+        jz      2f
+        addl    %ebx, -__PAGE_OFFSET(%ebx, %ecx)
+        jmp     1b
 2:
 #endif
 
 /*
  * Jump to the decompressed kernel.
  */
-        xorl %ebx,%ebx
-        jmp *%ebp
+        xorl    %ebx, %ebx
+        jmp     *%ebp
 
-.bss
-/* Stack and heap for uncompression */
-.balign 4
+/*
+ * Stack and heap for uncompression
+ */
+        .bss
+        .balign 4
 boot_heap:
         .fill BOOT_HEAP_SIZE, 1, 0
 boot_stack:
diff --git a/arch/x86/boot/compressed/head_64.S b/arch/x86/boot/compressed/head_64.S
index ed4a82948002..f62c284db9eb 100644
--- a/arch/x86/boot/compressed/head_64.S
+++ b/arch/x86/boot/compressed/head_64.S
@@ -21,8 +21,8 @@
 /*
  * High loaded stuff by Hans Lermen & Werner Almesberger, Feb. 1996
  */
-.code32
-.text
+        .code32
+        .text
 
 #include <linux/linkage.h>
 #include <asm/segment.h>
@@ -33,12 +33,14 @@
 #include <asm/processor-flags.h>
 #include <asm/asm-offsets.h>
 
-.section ".text.head"
+        .section ".text.head"
         .code32
 ENTRY(startup_32)
         cld
-        /* test KEEP_SEGMENTS flag to see if the bootloader is asking
-         * us to not reload segments */
+        /*
+         * Test KEEP_SEGMENTS flag to see if the bootloader is asking
+         * us to not reload segments
+         */
         testb $(1<<6), BP_loadflags(%esi)
         jnz 1f
 
@@ -49,14 +51,15 @@ ENTRY(startup_32)
         movl    %eax, %ss
 1:
 
-/* Calculate the delta between where we were compiled to run
+/*
+ * Calculate the delta between where we were compiled to run
  * at and where we were actually loaded at.  This can only be done
  * with a short local call on x86.  Nothing  else will tell us what
  * address we are running at.  The reserved chunk of the real-mode
  * data at 0x1e4 (defined as a scratch field) are used as the stack
  * for this calculation. Only 4 bytes are needed.
  */
-        leal    (0x1e4+4)(%esi), %esp
+        leal    (BP_scratch+4)(%esi), %esp
         call    1f
 1:      popl    %ebp
         subl    $1b, %ebp
@@ -70,32 +73,28 @@ ENTRY(startup_32)
         testl   %eax, %eax
         jnz     no_longmode
 
-/* Compute the delta between where we were compiled to run at
+/*
+ * Compute the delta between where we were compiled to run at
  * and where the code will actually run at.
- */
-/* %ebp contains the address we are loaded at by the boot loader and %ebx
+ *
+ * %ebp contains the address we are loaded at by the boot loader and %ebx
  * contains the address where we should move the kernel image temporarily
  * for safe in-place decompression.
  */
 
 #ifdef CONFIG_RELOCATABLE
         movl    %ebp, %ebx
-        addl    $(PMD_PAGE_SIZE -1), %ebx
-        andl    $PMD_PAGE_MASK, %ebx
+        movl    BP_kernel_alignment(%esi), %eax
+        decl    %eax
+        addl    %eax, %ebx
+        notl    %eax
+        andl    %eax, %ebx
 #else
-        movl    $CONFIG_PHYSICAL_START, %ebx
+        movl    $LOAD_PHYSICAL_ADDR, %ebx
 #endif
 
-        /* Replace the compressed data size with the uncompressed size */
-        subl    input_len(%ebp), %ebx
-        movl    output_len(%ebp), %eax
-        addl    %eax, %ebx
-        /* Add 8 bytes for every 32K input block */
-        shrl    $12, %eax
-        addl    %eax, %ebx
-        /* Add 32K + 18 bytes of extra slack and align on a 4K boundary */
-        addl    $(32768 + 18 + 4095), %ebx
-        andl    $~4095, %ebx
+        /* Target address to relocate to for decompression */
+        addl    $z_extract_offset, %ebx
 
 /*
  * Prepare for entering 64 bit mode
@@ -114,7 +113,7 @@ ENTRY(startup_32)
  /*
   * Build early 4G boot pagetable
   */
-        /* Initialize Page tables to 0*/
+        /* Initialize Page tables to 0 */
         leal    pgtable(%ebx), %edi
         xorl    %eax, %eax
         movl    $((4096*6)/4), %ecx
@@ -155,7 +154,8 @@ ENTRY(startup_32)
         btsl    $_EFER_LME, %eax
         wrmsr
 
-        /* Setup for the jump to 64bit mode
+        /*
+         * Setup for the jump to 64bit mode
          *
          * When the jump is performend we will be in long mode but
          * in 32bit compatibility mode with EFER.LME = 1, CS.L = 0, CS.D = 1
@@ -184,7 +184,8 @@ no_longmode:
 
 #include "../../kernel/verify_cpu_64.S"
 
-        /* Be careful here startup_64 needs to be at a predictable
+        /*
+         * Be careful here startup_64 needs to be at a predictable
          * address so I can export it in an ELF header.  Bootloaders
          * should look at the ELF header to find this address, as
          * it may change in the future.
@@ -192,7 +193,8 @@ no_longmode:
         .code64
         .org 0x200
 ENTRY(startup_64)
-        /* We come here either from startup_32 or directly from a
+        /*
+         * We come here either from startup_32 or directly from a
          * 64bit bootloader.  If we come here from a bootloader we depend on
          * an identity mapped page table being provied that maps our
          * entire text+data+bss and hopefully all of memory.
@@ -209,50 +211,54 @@ ENTRY(startup_64)
         movl    $0x20, %eax
         ltr     %ax
 
-        /* Compute the decompressed kernel start address.  It is where
+        /*
+         * Compute the decompressed kernel start address.  It is where
          * we were loaded at aligned to a 2M boundary. %rbp contains the
          * decompressed kernel start address.
          *
          * If it is a relocatable kernel then decompress and run the kernel
          * from load address aligned to 2MB addr, otherwise decompress and
-         * run the kernel from CONFIG_PHYSICAL_START
+         * run the kernel from LOAD_PHYSICAL_ADDR
+         *
+         * We cannot rely on the calculation done in 32-bit mode, since we
+         * may have been invoked via the 64-bit entry point.
          */
 
         /* Start with the delta to where the kernel will run at. */
 #ifdef CONFIG_RELOCATABLE
         leaq    startup_32(%rip) /* - $startup_32 */, %rbp
-        addq    $(PMD_PAGE_SIZE - 1), %rbp
-        andq    $PMD_PAGE_MASK, %rbp
-        movq    %rbp, %rbx
+        movl    BP_kernel_alignment(%rsi), %eax
+        decl    %eax
+        addq    %rax, %rbp
+        notq    %rax
+        andq    %rax, %rbp
 #else
-        movq    $CONFIG_PHYSICAL_START, %rbp
-        movq    %rbp, %rbx
+        movq    $LOAD_PHYSICAL_ADDR, %rbp
 #endif
 
-        /* Replace the compressed data size with the uncompressed size */
-        movl    input_len(%rip), %eax
-        subq    %rax, %rbx
-        movl    output_len(%rip), %eax
-        addq    %rax, %rbx
-        /* Add 8 bytes for every 32K input block */
-        shrq    $12, %rax
-        addq    %rax, %rbx
-        /* Add 32K + 18 bytes of extra slack and align on a 4K boundary */
-        addq    $(32768 + 18 + 4095), %rbx
-        andq    $~4095, %rbx
-
-/* Copy the compressed kernel to the end of our buffer
+        /* Target address to relocate to for decompression */
+        leaq    z_extract_offset(%rbp), %rbx
+
+        /* Set up the stack */
+        leaq    boot_stack_end(%rbx), %rsp
+
+        /* Zero EFLAGS */
+        pushq   $0
+        popfq
+
+/*
+ * Copy the compressed kernel to the end of our buffer
  * where decompression in place becomes safe.
  */
-        leaq    _end_before_pgt(%rip), %r8
-        leaq    _end_before_pgt(%rbx), %r9
-        movq    $_end_before_pgt /* - $startup_32 */, %rcx
-1:      subq    $8, %r8
-        subq    $8, %r9
-        movq    0(%r8), %rax
-        movq    %rax, 0(%r9)
-        subq    $8, %rcx
-        jnz     1b
+        pushq   %rsi
+        leaq    (_bss-8)(%rip), %rsi
+        leaq    (_bss-8)(%rbx), %rdi
+        movq    $_bss /* - $startup_32 */, %rcx
+        shrq    $3, %rcx
+        std
+        rep     movsq
+        cld
+        popq    %rsi
 
 /*
  * Jump to the relocated address.
@@ -260,37 +266,28 @@ ENTRY(startup_64)
         leaq    relocated(%rbx), %rax
         jmp     *%rax
 
-.section ".text"
+        .text
 relocated:
 
 /*
- * Clear BSS
+ * Clear BSS (stack is currently empty)
  */
-        xorq    %rax, %rax
-        leaq    _edata(%rbx), %rdi
-        leaq    _end_before_pgt(%rbx), %rcx
+        xorl    %eax, %eax
+        leaq    _bss(%rip), %rdi
+        leaq    _ebss(%rip), %rcx
         subq    %rdi, %rcx
-        cld
-        rep
-        stosb
-
-        /* Setup the stack */
-        leaq    boot_stack_end(%rip), %rsp
-
-        /* zero EFLAGS after setting rsp */
-        pushq   $0
-        popfq
+        shrq    $3, %rcx
+        rep     stosq
 
 /*
  * Do the decompression, and jump to the new kernel..
  */
-        pushq   %rsi                    # Save the real mode argument
-        movq    %rsi, %rdi              # real mode address
-        leaq    boot_heap(%rip), %rsi   # malloc area for uncompression
-        leaq    input_data(%rip), %rdx  # input_data
-        movl    input_len(%rip), %eax
-        movq    %rax, %rcx              # input_len
-        movq    %rbp, %r8               # output
+        pushq   %rsi                    /* Save the real mode argument */
+        movq    %rsi, %rdi              /* real mode address */
+        leaq    boot_heap(%rip), %rsi   /* malloc area for uncompression */
+        leaq    input_data(%rip), %rdx  /* input_data */
+        movl    $z_input_len, %ecx      /* input_len */
+        movq    %rbp, %r8               /* output target address */
         call    decompress_kernel
         popq    %rsi
 
@@ -311,11 +308,21 @@ gdt:
         .quad   0x0000000000000000      /* TS continued */
 gdt_end:
 
-.bss
-/* Stack and heap for uncompression */
-.balign 4
+/*
+ * Stack and heap for uncompression
+ */
+        .bss
+        .balign 4
 boot_heap:
         .fill BOOT_HEAP_SIZE, 1, 0
 boot_stack:
         .fill BOOT_STACK_SIZE, 1, 0
 boot_stack_end:
+
+/*
+ * Space for page tables (not in .bss so not zeroed)
+ */
+        .section ".pgtable","a",@nobits
+        .balign 4096
+pgtable:
+        .fill 6*4096, 1, 0
diff --git a/arch/x86/boot/compressed/misc.c b/arch/x86/boot/compressed/misc.c
index e45be73684ff..842b2a36174a 100644
--- a/arch/x86/boot/compressed/misc.c
+++ b/arch/x86/boot/compressed/misc.c
@@ -325,21 +325,19 @@ asmlinkage void decompress_kernel(void *rmode, memptr heap,
         free_mem_ptr     = heap;        /* Heap */
         free_mem_end_ptr = heap + BOOT_HEAP_SIZE;
 
+        if ((unsigned long)output & (MIN_KERNEL_ALIGN - 1))
+                error("Destination address inappropriately aligned");
 #ifdef CONFIG_X86_64
-        if ((unsigned long)output & (__KERNEL_ALIGN - 1))
-                error("Destination address not 2M aligned");
-        if ((unsigned long)output >= 0xffffffffffUL)
+        if (heap > 0x3fffffffffffUL)
                 error("Destination address too large");
 #else
-        if ((u32)output & (CONFIG_PHYSICAL_ALIGN - 1))
-                error("Destination address not CONFIG_PHYSICAL_ALIGN aligned");
         if (heap > ((-__PAGE_OFFSET-(512<<20)-1) & 0x7fffffff))
                 error("Destination address too large");
+#endif
 #ifndef CONFIG_RELOCATABLE
-        if ((u32)output != LOAD_PHYSICAL_ADDR)
+        if ((unsigned long)output != LOAD_PHYSICAL_ADDR)
                 error("Wrong destination address");
 #endif
-#endif
 
         if (!quiet)
                 putstr("\nDecompressing Linux... ");
diff --git a/arch/x86/boot/compressed/mkpiggy.c b/arch/x86/boot/compressed/mkpiggy.c
new file mode 100644
index 000000000000..bcbd36c41432
--- /dev/null
+++ b/arch/x86/boot/compressed/mkpiggy.c
@@ -0,0 +1,97 @@
+/* ----------------------------------------------------------------------- *
+ *
+ *  Copyright (C) 2009 Intel Corporation. All rights reserved.
+ *
+ *  This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version
+ *  2 as published by the Free Software Foundation.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ *  02110-1301, USA.
+ *
+ *  H. Peter Anvin <hpa@linux.intel.com>
+ *
+ * ----------------------------------------------------------------------- */
+
+/*
+ * Compute the desired load offset from a compressed program; outputs
+ * a small assembly wrapper with the appropriate symbols defined.
+ */
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <inttypes.h>
+
+static uint32_t getle32(const void *p)
+{
+        const uint8_t *cp = p;
+
+        return (uint32_t)cp[0] + ((uint32_t)cp[1] << 8) +
+                ((uint32_t)cp[2] << 16) + ((uint32_t)cp[3] << 24);
+}
+
+int main(int argc, char *argv[])
+{
+        uint32_t olen;
+        long ilen;
+        unsigned long offs;
+        FILE *f;
+
+        if (argc < 2) {
+                fprintf(stderr, "Usage: %s compressed_file\n", argv[0]);
+                return 1;
+        }
+
+        /* Get the information for the compressed kernel image first */
+
+        f = fopen(argv[1], "r");
+        if (!f) {
+                perror(argv[1]);
+                return 1;
+        }
+
+
+        if (fseek(f, -4L, SEEK_END)) {
+                perror(argv[1]);
+        }
+        fread(&olen, sizeof olen, 1, f);
+        ilen = ftell(f);
+        olen = getle32(&olen);
+        fclose(f);
+
+        /*
+         * Now we have the input (compressed) and output (uncompressed)
+         * sizes, compute the necessary decompression offset...
+         */
+
+        offs = (olen > ilen) ? olen - ilen : 0;
+        offs += olen >> 12;     /* Add 8 bytes for each 32K block */
+        offs += 32*1024 + 18;   /* Add 32K + 18 bytes slack */
+        offs = (offs+4095) & ~4095; /* Round to a 4K boundary */
+
+        printf(".section \".rodata.compressed\",\"a\",@progbits\n");
+        printf(".globl z_input_len\n");
+        printf("z_input_len = %lu\n", ilen);
+        printf(".globl z_output_len\n");
+        printf("z_output_len = %lu\n", (unsigned long)olen);
+        printf(".globl z_extract_offset\n");
+        printf("z_extract_offset = 0x%lx\n", offs);
+        /* z_extract_offset_negative allows simplification of head_32.S */
+        printf(".globl z_extract_offset_negative\n");
+        printf("z_extract_offset_negative = -0x%lx\n", offs);
+
+        printf(".globl input_data, input_data_end\n");
+        printf("input_data:\n");
+        printf(".incbin \"%s\"\n", argv[1]);
+        printf("input_data_end:\n");
+
+        return 0;
+}
diff --git a/arch/x86/boot/compressed/vmlinux_64.lds b/arch/x86/boot/compressed/vmlinux.lds.S
index bef1ac891bce..cc353e1b3ffd 100644
--- a/arch/x86/boot/compressed/vmlinux_64.lds
+++ b/arch/x86/boot/compressed/vmlinux.lds.S
@@ -1,6 +1,17 @@
-OUTPUT_FORMAT("elf64-x86-64", "elf64-x86-64", "elf64-x86-64")
+OUTPUT_FORMAT(CONFIG_OUTPUT_FORMAT, CONFIG_OUTPUT_FORMAT, CONFIG_OUTPUT_FORMAT)
+
+#undef i386
+
+#include <asm/page_types.h>
+
+#ifdef CONFIG_X86_64
 OUTPUT_ARCH(i386:x86-64)
 ENTRY(startup_64)
+#else
+OUTPUT_ARCH(i386)
+ENTRY(startup_32)
+#endif
+
 SECTIONS
 {
         /* Be careful parts of head_64.S assume startup_32 is at
@@ -33,16 +44,22 @@ SECTIONS
                 *(.data.*)
                 _edata = . ;
         }
+        . = ALIGN(CONFIG_X86_L1_CACHE_BYTES);
         .bss : {
                 _bss = . ;
                 *(.bss)
                 *(.bss.*)
                 *(COMMON)
-                . = ALIGN(8);
-                _end_before_pgt = . ;
-                . = ALIGN(4096);
-                pgtable = . ;
-                . = . + 4096 * 6;
+                . = ALIGN(8);   /* For convenience during zeroing */
                 _ebss = .;
         }
+#ifdef CONFIG_X86_64
+       . = ALIGN(PAGE_SIZE);
+       .pgtable : {
+                _pgtable = . ;
+                *(.pgtable)
+                _epgtable = . ;
+        }
+#endif
+        _end = .;
 }
diff --git a/arch/x86/boot/compressed/vmlinux.scr b/arch/x86/boot/compressed/vmlinux.scr
deleted file mode 100644
index f02382ae5c48..000000000000
--- a/arch/x86/boot/compressed/vmlinux.scr
+++ /dev/null
@@ -1,10 +0,0 @@
-SECTIONS
-{
-  .rodata.compressed : {
-        input_len = .;
-        LONG(input_data_end - input_data) input_data = .;
-        *(.data)
-        output_len = . - 4;
-        input_data_end = .;
-        }
-}
diff --git a/arch/x86/boot/compressed/vmlinux_32.lds b/arch/x86/boot/compressed/vmlinux_32.lds
deleted file mode 100644
index bb3c48379c40..000000000000
--- a/arch/x86/boot/compressed/vmlinux_32.lds
+++ /dev/null
@@ -1,43 +0,0 @@
-OUTPUT_FORMAT("elf32-i386", "elf32-i386", "elf32-i386")
-OUTPUT_ARCH(i386)
-ENTRY(startup_32)
-SECTIONS
-{
-        /* Be careful parts of head_32.S assume startup_32 is at
-         * address 0.
-         */
-        . = 0;
-        .text.head : {
-                _head = . ;
-                *(.text.head)
-                _ehead = . ;
-        }
-        .rodata.compressed : {
-                *(.rodata.compressed)
-        }
-        .text : {
-                _text = .;      /* Text */
-                *(.text)
-                *(.text.*)
-                _etext = . ;
-        }
-        .rodata : {
-                _rodata = . ;
-                *(.rodata)       /* read-only data */
-                *(.rodata.*)
-                _erodata = . ;
-        }
-        .data : {
-                _data = . ;
-                *(.data)
-                *(.data.*)
-                _edata = . ;
-        }
-        .bss : {
-                _bss = . ;
-                *(.bss)
-                *(.bss.*)
-                *(COMMON)
-                _end = . ;
-        }
-}
diff --git a/arch/x86/boot/edd.c b/arch/x86/boot/edd.c
index 1aae8f3e5ca1..c501a5b466f8 100644
--- a/arch/x86/boot/edd.c
+++ b/arch/x86/boot/edd.c
@@ -2,6 +2,7 @@
  *
  *   Copyright (C) 1991, 1992 Linus Torvalds
  *   Copyright 2007 rPath, Inc. - All Rights Reserved
+ *   Copyright 2009 Intel Corporation; author H. Peter Anvin
  *
  *   This file is part of the Linux kernel, and is made available under
  *   the terms of the GNU General Public License version 2.
@@ -22,17 +23,17 @@
  */
 static int read_mbr(u8 devno, void *buf)
 {
-        u16 ax, bx, cx, dx;
+        struct biosregs ireg, oreg;
 
-        ax = 0x0201;            /* Legacy Read, one sector */
-        cx = 0x0001;            /* Sector 0-0-1 */
-        dx = devno;
-        bx = (size_t)buf;
-        asm volatile("pushfl; stc; int $0x13; setc %%al; popfl"
-                     : "+a" (ax), "+c" (cx), "+d" (dx), "+b" (bx)
-                     : : "esi", "edi", "memory");
+        initregs(&ireg);
+        ireg.ax = 0x0201;               /* Legacy Read, one sector */
+        ireg.cx = 0x0001;               /* Sector 0-0-1 */
+        ireg.dl = devno;
+        ireg.bx = (size_t)buf;
 
-        return -(u8)ax;         /* 0 or -1 */
+        intcall(0x13, &ireg, &oreg);
+
+        return -(oreg.eflags & X86_EFLAGS_CF); /* 0 or -1 */
 }
 
 static u32 read_mbr_sig(u8 devno, struct edd_info *ei, u32 *mbrsig)
@@ -72,56 +73,46 @@ static u32 read_mbr_sig(u8 devno, struct edd_info *ei, u32 *mbrsig)
 
 static int get_edd_info(u8 devno, struct edd_info *ei)
 {
-        u16 ax, bx, cx, dx, di;
+        struct biosregs ireg, oreg;
 
         memset(ei, 0, sizeof *ei);
 
         /* Check Extensions Present */
 
-        ax = 0x4100;
-        bx = EDDMAGIC1;
-        dx = devno;
-        asm("pushfl; stc; int $0x13; setc %%al; popfl"
-            : "+a" (ax), "+b" (bx), "=c" (cx), "+d" (dx)
-            : : "esi", "edi");
+        initregs(&ireg);
+        ireg.ah = 0x41;
+        ireg.bx = EDDMAGIC1;
+        ireg.dl = devno;
+        intcall(0x13, &ireg, &oreg);
 
-        if ((u8)ax)
+        if (oreg.eflags & X86_EFLAGS_CF)
                 return -1;      /* No extended information */
 
-        if (bx != EDDMAGIC2)
+        if (oreg.bx != EDDMAGIC2)
                 return -1;
 
         ei->device  = devno;
-        ei->version = ax >> 8;  /* EDD version number */
-        ei->interface_support = cx; /* EDD functionality subsets */
+        ei->version = oreg.ah;           /* EDD version number */
+        ei->interface_support = oreg.cx; /* EDD functionality subsets */
 
         /* Extended Get Device Parameters */
 
         ei->params.length = sizeof(ei->params);
-        ax = 0x4800;
-        dx = devno;
-        asm("pushfl; int $0x13; popfl"
-            : "+a" (ax), "+d" (dx), "=m" (ei->params)
-            : "S" (&ei->params)
-            : "ebx", "ecx", "edi");
+        ireg.ah = 0x48;
+        ireg.si = (size_t)&ei->params;
+        intcall(0x13, &ireg, &oreg);
 
         /* Get legacy CHS parameters */
 
         /* Ralf Brown recommends setting ES:DI to 0:0 */
-        ax = 0x0800;
-        dx = devno;
-        di = 0;
-        asm("pushw %%es; "
-            "movw %%di,%%es; "
-            "pushfl; stc; int $0x13; setc %%al; popfl; "
-            "popw %%es"
-            : "+a" (ax), "=b" (bx), "=c" (cx), "+d" (dx), "+D" (di)
-            : : "esi");
-
-        if ((u8)ax == 0) {
-                ei->legacy_max_cylinder = (cx >> 8) + ((cx & 0xc0) << 2);
-                ei->legacy_max_head = dx >> 8;
-                ei->legacy_sectors_per_track = cx & 0x3f;
+        ireg.ah = 0x08;
+        ireg.es = 0;
+        intcall(0x13, &ireg, &oreg);
+
+        if (!(oreg.eflags & X86_EFLAGS_CF)) {
+                ei->legacy_max_cylinder = oreg.ch + ((oreg.cl & 0xc0) << 2);
+                ei->legacy_max_head = oreg.dh;
+                ei->legacy_sectors_per_track = oreg.cl & 0x3f;
         }
 
         return 0;
diff --git a/arch/x86/boot/header.S b/arch/x86/boot/header.S
index 5d84d1c74e4c..b31cc54b4641 100644
--- a/arch/x86/boot/header.S
+++ b/arch/x86/boot/header.S
@@ -22,7 +22,8 @@
 #include <asm/page_types.h>
 #include <asm/setup.h>
 #include "boot.h"
-#include "offsets.h"
+#include "voffset.h"
+#include "zoffset.h"
 
 BOOTSEG         = 0x07C0                /* original address of boot-sector */
 SYSSEG          = 0x1000                /* historical load address >> 4 */
@@ -115,7 +116,7 @@ _start:
         # Part 2 of the header, from the old setup.S
 
                 .ascii  "HdrS"          # header signature
-                .word   0x0209          # header version number (>= 0x0105)
+                .word   0x020a          # header version number (>= 0x0105)
                                         # or else old loadlin-1.5 will fail)
                 .globl realmode_swtch
 realmode_swtch: .word   0, 0            # default_switch, SETUPSEG
@@ -168,7 +169,11 @@ heap_end_ptr:	.word	_end+STACK_SIZE-512
                                         # end of setup code can be used by setup
                                         # for local heap purposes.
 
-pad1:           .word   0
+ext_loader_ver:
+                .byte   0               # Extended boot loader version
+ext_loader_type:
+                .byte   0               # Extended boot loader type
+
 cmd_line_ptr:   .long   0               # (Header version 0x0202 or later)
                                         # If nonzero, a 32-bit pointer
                                         # to the kernel command line.
@@ -200,7 +205,7 @@ relocatable_kernel:    .byte 1
 #else
 relocatable_kernel:    .byte 0
 #endif
-pad2:                   .byte 0
+min_alignment:          .byte MIN_KERNEL_ALIGN_LG2      # minimum alignment
 pad3:                   .word 0
 
 cmdline_size:   .long   COMMAND_LINE_SIZE-1     #length of the command line,
@@ -212,16 +217,27 @@ hardware_subarch:	.long 0			# subarchitecture, added with 2.07
 
 hardware_subarch_data:  .quad 0
 
-payload_offset:         .long input_data
-payload_length:         .long input_data_end-input_data
+payload_offset:         .long ZO_input_data
+payload_length:         .long ZO_z_input_len
 
 setup_data:             .quad 0                 # 64-bit physical pointer to
                                                 # single linked list of
                                                 # struct setup_data
 
+pref_address:           .quad LOAD_PHYSICAL_ADDR        # preferred load addr
+
+#define ZO_INIT_SIZE    (ZO__end - ZO_startup_32 + ZO_z_extract_offset)
+#define VO_INIT_SIZE    (VO__end - VO__text)
+#if ZO_INIT_SIZE > VO_INIT_SIZE
+#define INIT_SIZE ZO_INIT_SIZE
+#else
+#define INIT_SIZE VO_INIT_SIZE
+#endif
+init_size:              .long INIT_SIZE         # kernel initialization size
+
 # End of setup header #####################################################
 
-        .section ".inittext", "ax"
+        .section ".entrytext", "ax"
 start_of_setup:
 #ifdef SAFE_RESET_DISK_CONTROLLER
 # Reset the disk controller.
diff --git a/arch/x86/boot/main.c b/arch/x86/boot/main.c
index 58f0415d3ae0..140172b895bd 100644
--- a/arch/x86/boot/main.c
+++ b/arch/x86/boot/main.c
@@ -2,6 +2,7 @@
  *
  *   Copyright (C) 1991, 1992 Linus Torvalds
  *   Copyright 2007 rPath, Inc. - All Rights Reserved
+ *   Copyright 2009 Intel Corporation; author H. Peter Anvin
  *
  *   This file is part of the Linux kernel, and is made available under
  *   the terms of the GNU General Public License version 2.
@@ -61,11 +62,10 @@ static void copy_boot_params(void)
  */
 static void keyboard_set_repeat(void)
 {
-        u16 ax = 0x0305;
-        u16 bx = 0;
-        asm volatile("int $0x16"
-                     : "+a" (ax), "+b" (bx)
-                     : : "ecx", "edx", "esi", "edi");
+        struct biosregs ireg;
+        initregs(&ireg);
+        ireg.ax = 0x0305;
+        intcall(0x16, &ireg, NULL);
 }
 
 /*
@@ -73,18 +73,22 @@ static void keyboard_set_repeat(void)
  */
 static void query_ist(void)
 {
+        struct biosregs ireg, oreg;
+
         /* Some older BIOSes apparently crash on this call, so filter
            it from machines too old to have SpeedStep at all. */
         if (cpu.level < 6)
                 return;
 
-        asm("int $0x15"
-            : "=a" (boot_params.ist_info.signature),
-              "=b" (boot_params.ist_info.command),
-              "=c" (boot_params.ist_info.event),
-              "=d" (boot_params.ist_info.perf_level)
-            : "a" (0x0000e980),  /* IST Support */
-              "d" (0x47534943)); /* Request value */
+        initregs(&ireg);
+        ireg.ax  = 0xe980;       /* IST Support */
+        ireg.edx = 0x47534943;   /* Request value */
+        intcall(0x15, &ireg, &oreg);
+
+        boot_params.ist_info.signature  = oreg.eax;
+        boot_params.ist_info.command    = oreg.ebx;
+        boot_params.ist_info.event      = oreg.ecx;
+        boot_params.ist_info.perf_level = oreg.edx;
 }
 
 /*
@@ -93,13 +97,12 @@ static void query_ist(void)
 static void set_bios_mode(void)
 {
 #ifdef CONFIG_X86_64
-        u32 eax, ebx;
+        struct biosregs ireg;
 
-        eax = 0xec00;
-        ebx = 2;
-        asm volatile("int $0x15"
-                     : "+a" (eax), "+b" (ebx)
-                     : : "ecx", "edx", "esi", "edi");
+        initregs(&ireg);
+        ireg.ax = 0xec00;
+        ireg.bx = 2;
+        intcall(0x15, &ireg, NULL);
 #endif
 }
 
diff --git a/arch/x86/boot/mca.c b/arch/x86/boot/mca.c
index 911eaae5d696..a95a531148ef 100644
--- a/arch/x86/boot/mca.c
+++ b/arch/x86/boot/mca.c
@@ -2,6 +2,7 @@
  *
  *   Copyright (C) 1991, 1992 Linus Torvalds
  *   Copyright 2007 rPath, Inc. - All Rights Reserved
+ *   Copyright 2009 Intel Corporation; author H. Peter Anvin
  *
  *   This file is part of the Linux kernel, and is made available under
  *   the terms of the GNU General Public License version 2.
@@ -16,26 +17,22 @@
 
 int query_mca(void)
 {
-        u8 err;
-        u16 es, bx, len;
-
-        asm("pushw %%es ; "
-            "int $0x15 ; "
-            "setc %0 ; "
-            "movw %%es, %1 ; "
-            "popw %%es"
-            : "=acd" (err), "=acdSD" (es), "=b" (bx)
-            : "a" (0xc000));
-
-        if (err)
+        struct biosregs ireg, oreg;
+        u16 len;
+
+        initregs(&ireg);
+        ireg.ah = 0xc0;
+        intcall(0x15, &ireg, &oreg);
+
+        if (oreg.eflags & X86_EFLAGS_CF)
                 return -1;      /* No MCA present */
 
-        set_fs(es);
-        len = rdfs16(bx);
+        set_fs(oreg.es);
+        len = rdfs16(oreg.bx);
 
         if (len > sizeof(boot_params.sys_desc_table))
                 len = sizeof(boot_params.sys_desc_table);
 
-        copy_from_fs(&boot_params.sys_desc_table, bx, len);
+        copy_from_fs(&boot_params.sys_desc_table, oreg.bx, len);
         return 0;
 }
diff --git a/arch/x86/boot/memory.c b/arch/x86/boot/memory.c
index 74b3d2ba84e9..cae3feb1035e 100644
--- a/arch/x86/boot/memory.c
+++ b/arch/x86/boot/memory.c
@@ -20,12 +20,16 @@
 static int detect_memory_e820(void)
 {
         int count = 0;
-        u32 next = 0;
-        u32 size, id, edi;
-        u8 err;
+        struct biosregs ireg, oreg;
         struct e820entry *desc = boot_params.e820_map;
         static struct e820entry buf; /* static so it is zeroed */
 
+        initregs(&ireg);
+        ireg.ax  = 0xe820;
+        ireg.cx  = sizeof buf;
+        ireg.edx = SMAP;
+        ireg.di  = (size_t)&buf;
+
         /*
          * Note: at least one BIOS is known which assumes that the
          * buffer pointed to by one e820 call is the same one as
@@ -41,22 +45,13 @@ static int detect_memory_e820(void)
          */
 
         do {
-                size = sizeof buf;
-
-                /* Important: %edx and %esi are clobbered by some BIOSes,
-                   so they must be either used for the error output
-                   or explicitly marked clobbered.  Given that, assume there
-                   is something out there clobbering %ebp and %edi, too. */
-                asm("pushl %%ebp; int $0x15; popl %%ebp; setc %0"
-                    : "=d" (err), "+b" (next), "=a" (id), "+c" (size),
-                      "=D" (edi), "+m" (buf)
-                    : "D" (&buf), "d" (SMAP), "a" (0xe820)
-                    : "esi");
+                intcall(0x15, &ireg, &oreg);
+                ireg.ebx = oreg.ebx; /* for next iteration... */
 
                 /* BIOSes which terminate the chain with CF = 1 as opposed
                    to %ebx = 0 don't always report the SMAP signature on
                    the final, failing, probe. */
-                if (err)
+                if (oreg.eflags & X86_EFLAGS_CF)
                         break;
 
                 /* Some BIOSes stop returning SMAP in the middle of
@@ -64,60 +59,64 @@ static int detect_memory_e820(void)
                    screwed up the map at that point, we might have a
                    partial map, the full map, or complete garbage, so
                    just return failure. */
-                if (id != SMAP) {
+                if (oreg.eax != SMAP) {
                         count = 0;
                         break;
                 }
 
                 *desc++ = buf;
                 count++;
-        } while (next && count < ARRAY_SIZE(boot_params.e820_map));
+        } while (ireg.ebx && count < ARRAY_SIZE(boot_params.e820_map));
 
         return boot_params.e820_entries = count;
 }
 
 static int detect_memory_e801(void)
 {
-        u16 ax, bx, cx, dx;
-        u8 err;
+        struct biosregs ireg, oreg;
 
-        bx = cx = dx = 0;
-        ax = 0xe801;
-        asm("stc; int $0x15; setc %0"
-            : "=m" (err), "+a" (ax), "+b" (bx), "+c" (cx), "+d" (dx));
+        initregs(&ireg);
+        ireg.ax = 0xe801;
+        intcall(0x15, &ireg, &oreg);
 
-        if (err)
+        if (oreg.eflags & X86_EFLAGS_CF)
                 return -1;
 
         /* Do we really need to do this? */
-        if (cx || dx) {
-                ax = cx;
-                bx = dx;
+        if (oreg.cx || oreg.dx) {
+                oreg.ax = oreg.cx;
+                oreg.bx = oreg.dx;
         }
 
-        if (ax > 15*1024)
+        if (oreg.ax > 15*1024) {
                 return -1;      /* Bogus! */
-
-        /* This ignores memory above 16MB if we have a memory hole
-           there.  If someone actually finds a machine with a memory
-           hole at 16MB and no support for 0E820h they should probably
-           generate a fake e820 map. */
-        boot_params.alt_mem_k = (ax == 15*1024) ? (dx << 6)+ax : ax;
+        } else if (oreg.ax == 15*1024) {
+                boot_params.alt_mem_k = (oreg.dx << 6) + oreg.ax;
+        } else {
+                /*
+                 * This ignores memory above 16MB if we have a memory
+                 * hole there.  If someone actually finds a machine
+                 * with a memory hole at 16MB and no support for
+                 * 0E820h they should probably generate a fake e820
+                 * map.
+                 */
+                boot_params.alt_mem_k = oreg.ax;
+        }
 
         return 0;
 }
 
 static int detect_memory_88(void)
 {
-        u16 ax;
-        u8 err;
+        struct biosregs ireg, oreg;
 
-        ax = 0x8800;
-        asm("stc; int $0x15; setc %0" : "=bcdm" (err), "+a" (ax));
+        initregs(&ireg);
+        ireg.ah = 0x88;
+        intcall(0x15, &ireg, &oreg);
 
-        boot_params.screen_info.ext_mem_k = ax;
+        boot_params.screen_info.ext_mem_k = oreg.ax;
 
-        return -err;
+        return -(oreg.eflags & X86_EFLAGS_CF); /* 0 or -1 */
 }
 
 int detect_memory(void)
diff --git a/arch/x86/boot/regs.c b/arch/x86/boot/regs.c
new file mode 100644
index 000000000000..958019b1cfa5
--- /dev/null
+++ b/arch/x86/boot/regs.c
@@ -0,0 +1,29 @@
+/* -----------------------------------------------------------------------
+ *
+ *   Copyright 2009 Intel Corporation; author H. Peter Anvin
+ *
+ *   This file is part of the Linux kernel, and is made available under
+ *   the terms of the GNU General Public License version 2 or (at your
+ *   option) any later version; incorporated herein by reference.
+ *
+ * ----------------------------------------------------------------------- */
+
+/*
+ * Simple helper function for initializing a register set.
+ *
+ * Note that this sets EFLAGS_CF in the input register set; this
+ * makes it easier to catch functions which do nothing but don't
+ * explicitly set CF.
+ */
+
+#include "boot.h"
+
+void initregs(struct biosregs *reg)
+{
+        memset(reg, 0, sizeof *reg);
+        reg->eflags |= X86_EFLAGS_CF;
+        reg->ds = ds();
+        reg->es = ds();
+        reg->fs = fs();
+        reg->gs = gs();
+}
diff --git a/arch/x86/boot/setup.ld b/arch/x86/boot/setup.ld
index bb8dc2de7969..0f6ec455a2b1 100644
--- a/arch/x86/boot/setup.ld
+++ b/arch/x86/boot/setup.ld
@@ -15,8 +15,11 @@ SECTIONS
 
         . = 497;
         .header         : { *(.header) }
+        .entrytext      : { *(.entrytext) }
         .inittext       : { *(.inittext) }
         .initdata       : { *(.initdata) }
+        __end_init = .;
+
         .text           : { *(.text) }
         .text32         : { *(.text32) }
 
@@ -52,4 +55,7 @@ SECTIONS
 
         . = ASSERT(_end <= 0x8000, "Setup too big!");
         . = ASSERT(hdr == 0x1f1, "The setup header has the wrong offset!");
+        /* Necessary for the very-old-loader check to work... */
+        . = ASSERT(__end_init <= 5*512, "init sections too big!");
+
 }
diff --git a/arch/x86/boot/tty.c b/arch/x86/boot/tty.c
index 7e8e8b25f5f6..01ec69c901c7 100644
--- a/arch/x86/boot/tty.c
+++ b/arch/x86/boot/tty.c
@@ -2,6 +2,7 @@
  *
  *   Copyright (C) 1991, 1992 Linus Torvalds
  *   Copyright 2007 rPath, Inc. - All Rights Reserved
+ *   Copyright 2009 Intel Corporation; author H. Peter Anvin
  *
  *   This file is part of the Linux kernel, and is made available under
  *   the terms of the GNU General Public License version 2.
@@ -22,24 +23,23 @@
 
 void __attribute__((section(".inittext"))) putchar(int ch)
 {
-        unsigned char c = ch;
+        struct biosregs ireg;
 
-        if (c == '\n')
+        if (ch == '\n')
                 putchar('\r');  /* \n -> \r\n */
 
-        /* int $0x10 is known to have bugs involving touching registers
-           it shouldn't.  Be extra conservative... */
-        asm volatile("pushal; pushw %%ds; int $0x10; popw %%ds; popal"
-                     : : "b" (0x0007), "c" (0x0001), "a" (0x0e00|ch));
+        initregs(&ireg);
+        ireg.bx = 0x0007;
+        ireg.cx = 0x0001;
+        ireg.ah = 0x0e;
+        ireg.al = ch;
+        intcall(0x10, &ireg, NULL);
 }
 
 void __attribute__((section(".inittext"))) puts(const char *str)
 {
-        int n = 0;
-        while (*str) {
+        while (*str)
                 putchar(*str++);
-                n++;
-        }
 }
 
 /*
@@ -49,14 +49,13 @@ void __attribute__((section(".inittext"))) puts(const char *str)
 
 static u8 gettime(void)
 {
-        u16 ax = 0x0200;
-        u16 cx, dx;
+        struct biosregs ireg, oreg;
 
-        asm volatile("int $0x1a"
-                     : "+a" (ax), "=c" (cx), "=d" (dx)
-                     : : "ebx", "esi", "edi");
+        initregs(&ireg);
+        ireg.ah = 0x02;
+        intcall(0x1a, &ireg, &oreg);
 
-        return dx >> 8;
+        return oreg.dh;
 }
 
 /*
@@ -64,19 +63,24 @@ static u8 gettime(void)
  */
 int getchar(void)
 {
-        u16 ax = 0;
-        asm volatile("int $0x16" : "+a" (ax));
+        struct biosregs ireg, oreg;
+
+        initregs(&ireg);
+        /* ireg.ah = 0x00; */
+        intcall(0x16, &ireg, &oreg);
 
-        return ax & 0xff;
+        return oreg.al;
 }
 
 static int kbd_pending(void)
 {
-        u8 pending;
-        asm volatile("int $0x16; setnz %0"
-                     : "=qm" (pending)
-                     : "a" (0x0100));
-        return pending;
+        struct biosregs ireg, oreg;
+
+        initregs(&ireg);
+        ireg.ah = 0x01;
+        intcall(0x16, &ireg, &oreg);
+
+        return !(oreg.eflags & X86_EFLAGS_ZF);
 }
 
 void kbd_flush(void)
diff --git a/arch/x86/boot/video-bios.c b/arch/x86/boot/video-bios.c
index 3fa979c9c363..d660be492363 100644
--- a/arch/x86/boot/video-bios.c
+++ b/arch/x86/boot/video-bios.c
@@ -2,6 +2,7 @@
  *
  *   Copyright (C) 1991, 1992 Linus Torvalds
  *   Copyright 2007 rPath, Inc. - All Rights Reserved
+ *   Copyright 2009 Intel Corporation; author H. Peter Anvin
  *
  *   This file is part of the Linux kernel, and is made available under
  *   the terms of the GNU General Public License version 2.
@@ -29,21 +30,21 @@ static int bios_set_mode(struct mode_info *mi)
 
 static int set_bios_mode(u8 mode)
 {
-        u16 ax;
+        struct biosregs ireg, oreg;
         u8 new_mode;
 
-        ax = mode;              /* AH=0x00 Set Video Mode */
-        asm volatile(INT10
-                     : "+a" (ax)
-                     : : "ebx", "ecx", "edx", "esi", "edi");
+        initregs(&ireg);
+        ireg.al = mode;         /* AH=0x00 Set Video Mode */
+        intcall(0x10, &ireg, NULL);
 
-        ax = 0x0f00;            /* Get Current Video Mode */
-        asm volatile(INT10
-                     : "+a" (ax)
-                     : : "ebx", "ecx", "edx", "esi", "edi");
+
+        ireg.ah = 0x0f;         /* Get Current Video Mode */
+        intcall(0x10, &ireg, &oreg);
 
         do_restore = 1;         /* Assume video contents were lost */
-        new_mode = ax & 0x7f;   /* Not all BIOSes are clean with the top bit */
+
+        /* Not all BIOSes are clean with the top bit */
+        new_mode = ireg.al & 0x7f;
 
         if (new_mode == mode)
                 return 0;       /* Mode change OK */
@@ -53,10 +54,8 @@ static int set_bios_mode(u8 mode)
                 /* Mode setting failed, but we didn't end up where we
                    started.  That's bad.  Try to revert to the original
                    video mode. */
-                ax = boot_params.screen_info.orig_video_mode;
-                asm volatile(INT10
-                             : "+a" (ax)
-                             : : "ebx", "ecx", "edx", "esi", "edi");
+                ireg.ax = boot_params.screen_info.orig_video_mode;
+                intcall(0x10, &ireg, NULL);
         }
 #endif
         return -1;
diff --git a/arch/x86/boot/video-vesa.c b/arch/x86/boot/video-vesa.c
index 4a58c8ce3f69..c700147d6ffb 100644
--- a/arch/x86/boot/video-vesa.c
+++ b/arch/x86/boot/video-vesa.c
@@ -2,6 +2,7 @@
  *
  *   Copyright (C) 1991, 1992 Linus Torvalds
  *   Copyright 2007 rPath, Inc. - All Rights Reserved
+ *   Copyright 2009 Intel Corporation; author H. Peter Anvin
  *
  *   This file is part of the Linux kernel, and is made available under
  *   the terms of the GNU General Public License version 2.
@@ -31,7 +32,7 @@ static inline void vesa_store_mode_params_graphics(void) {}
 static int vesa_probe(void)
 {
 #if defined(CONFIG_VIDEO_VESA) || defined(CONFIG_FIRMWARE_EDID)
-        u16 ax, cx, di;
+        struct biosregs ireg, oreg;
         u16 mode;
         addr_t mode_ptr;
         struct mode_info *mi;
@@ -39,13 +40,12 @@ static int vesa_probe(void)
 
         video_vesa.modes = GET_HEAP(struct mode_info, 0);
 
-        ax = 0x4f00;
-        di = (size_t)&vginfo;
-        asm(INT10
-            : "+a" (ax), "+D" (di), "=m" (vginfo)
-            : : "ebx", "ecx", "edx", "esi");
+        initregs(&ireg);
+        ireg.ax = 0x4f00;
+        ireg.di = (size_t)&vginfo;
+        intcall(0x10, &ireg, &oreg);
 
-        if (ax != 0x004f ||
+        if (ireg.ax != 0x004f ||
             vginfo.signature != VESA_MAGIC ||
             vginfo.version < 0x0102)
                 return 0;       /* Not present */
@@ -65,14 +65,12 @@ static int vesa_probe(void)
 
                 memset(&vminfo, 0, sizeof vminfo); /* Just in case... */
 
-                ax = 0x4f01;
-                cx = mode;
-                di = (size_t)&vminfo;
-                asm(INT10
-                    : "+a" (ax), "+c" (cx), "+D" (di), "=m" (vminfo)
-                    : : "ebx", "edx", "esi");
+                ireg.ax = 0x4f01;
+                ireg.cx = mode;
+                ireg.di = (size_t)&vminfo;
+                intcall(0x10, &ireg, &oreg);
 
-                if (ax != 0x004f)
+                if (ireg.ax != 0x004f)
                         continue;
 
                 if ((vminfo.mode_attr & 0x15) == 0x05) {
@@ -111,20 +109,19 @@ static int vesa_probe(void)
 
 static int vesa_set_mode(struct mode_info *mode)
 {
-        u16 ax, bx, cx, di;
+        struct biosregs ireg, oreg;
         int is_graphic;
         u16 vesa_mode = mode->mode - VIDEO_FIRST_VESA;
 
         memset(&vminfo, 0, sizeof vminfo); /* Just in case... */
 
-        ax = 0x4f01;
-        cx = vesa_mode;
-        di = (size_t)&vminfo;
-        asm(INT10
-            : "+a" (ax), "+c" (cx), "+D" (di), "=m" (vminfo)
-            : : "ebx", "edx", "esi");
+        initregs(&ireg);
+        ireg.ax = 0x4f01;
+        ireg.cx = vesa_mode;
+        ireg.di = (size_t)&vminfo;
+        intcall(0x10, &ireg, &oreg);
 
-        if (ax != 0x004f)
+        if (oreg.ax != 0x004f)
                 return -1;
 
         if ((vminfo.mode_attr & 0x15) == 0x05) {
@@ -141,14 +138,12 @@ static int vesa_set_mode(struct mode_info *mode)
         }
 
 
-        ax = 0x4f02;
-        bx = vesa_mode;
-        di = 0;
-        asm volatile(INT10
-                     : "+a" (ax), "+b" (bx), "+D" (di)
-                     : : "ecx", "edx", "esi");
+        initregs(&ireg);
+        ireg.ax = 0x4f02;
+        ireg.bx = vesa_mode;
+        intcall(0x10, &ireg, &oreg);
 
-        if (ax != 0x004f)
+        if (oreg.ax != 0x004f)
                 return -1;
 
         graphic_mode = is_graphic;
@@ -171,50 +166,45 @@ static int vesa_set_mode(struct mode_info *mode)
 /* Switch DAC to 8-bit mode */
 static void vesa_dac_set_8bits(void)
 {
+        struct biosregs ireg, oreg;
         u8 dac_size = 6;
 
         /* If possible, switch the DAC to 8-bit mode */
         if (vginfo.capabilities & 1) {
-                u16 ax, bx;
-
-                ax = 0x4f08;
-                bx = 0x0800;
-                asm volatile(INT10
-                             : "+a" (ax), "+b" (bx)
-                             : : "ecx", "edx", "esi", "edi");
-
-                if (ax == 0x004f)
-                        dac_size = bx >> 8;
+                initregs(&ireg);
+                ireg.ax = 0x4f08;
+                ireg.bh = 0x08;
+                intcall(0x10, &ireg, &oreg);
+                if (oreg.ax == 0x004f)
+                        dac_size = oreg.bh;
         }
 
         /* Set the color sizes to the DAC size, and offsets to 0 */
-        boot_params.screen_info.red_size = dac_size;
+        boot_params.screen_info.red_size   = dac_size;
         boot_params.screen_info.green_size = dac_size;
-        boot_params.screen_info.blue_size = dac_size;
-        boot_params.screen_info.rsvd_size = dac_size;
+        boot_params.screen_info.blue_size  = dac_size;
+        boot_params.screen_info.rsvd_size  = dac_size;
 
-        boot_params.screen_info.red_pos = 0;
-        boot_params.screen_info.green_pos = 0;
-        boot_params.screen_info.blue_pos = 0;
-        boot_params.screen_info.rsvd_pos = 0;
+        boot_params.screen_info.red_pos    = 0;
+        boot_params.screen_info.green_pos  = 0;
+        boot_params.screen_info.blue_pos   = 0;
+        boot_params.screen_info.rsvd_pos   = 0;
 }
 
 /* Save the VESA protected mode info */
 static void vesa_store_pm_info(void)
 {
-        u16 ax, bx, di, es;
+        struct biosregs ireg, oreg;
 
-        ax = 0x4f0a;
-        bx = di = 0;
-        asm("pushw %%es; "INT10"; movw %%es,%0; popw %%es"
-            : "=d" (es), "+a" (ax), "+b" (bx), "+D" (di)
-            : : "ecx", "esi");
+        initregs(&ireg);
+        ireg.ax = 0x4f0a;
+        intcall(0x10, &ireg, &oreg);
 
-        if (ax != 0x004f)
+        if (oreg.ax != 0x004f)
                 return;
 
-        boot_params.screen_info.vesapm_seg = es;
-        boot_params.screen_info.vesapm_off = di;
+        boot_params.screen_info.vesapm_seg = oreg.es;
+        boot_params.screen_info.vesapm_off = oreg.di;
 }
 
 /*
@@ -252,7 +242,7 @@ static void vesa_store_mode_params_graphics(void)
 void vesa_store_edid(void)
 {
 #ifdef CONFIG_FIRMWARE_EDID
-        u16 ax, bx, cx, dx, di;
+        struct biosregs ireg, oreg;
 
         /* Apparently used as a nonsense token... */
         memset(&boot_params.edid_info, 0x13, sizeof boot_params.edid_info);
@@ -260,33 +250,26 @@ void vesa_store_edid(void)
         if (vginfo.version < 0x0200)
                 return;         /* EDID requires VBE 2.0+ */
 
-        ax = 0x4f15;            /* VBE DDC */
-        bx = 0x0000;            /* Report DDC capabilities */
-        cx = 0;                 /* Controller 0 */
-        di = 0;                 /* ES:DI must be 0 by spec */
-
-        /* Note: The VBE DDC spec is different from the main VESA spec;
-           we genuinely have to assume all registers are destroyed here. */
-
-        asm("pushw %%es; movw %2,%%es; "INT10"; popw %%es"
-            : "+a" (ax), "+b" (bx), "+c" (cx), "+D" (di)
-            : : "esi", "edx");
+        initregs(&ireg);
+        ireg.ax = 0x4f15;               /* VBE DDC */
+        /* ireg.bx = 0x0000; */         /* Report DDC capabilities */
+        /* ireg.cx = 0; */              /* Controller 0 */
+        ireg.es = 0;                    /* ES:DI must be 0 by spec */
+        intcall(0x10, &ireg, &oreg);
 
-        if (ax != 0x004f)
+        if (oreg.ax != 0x004f)
                 return;         /* No EDID */
 
         /* BH = time in seconds to transfer EDD information */
         /* BL = DDC level supported */
 
-        ax = 0x4f15;            /* VBE DDC */
-        bx = 0x0001;            /* Read EDID */
-        cx = 0;                 /* Controller 0 */
-        dx = 0;                 /* EDID block number */
-        di =(size_t) &boot_params.edid_info; /* (ES:)Pointer to block */
-        asm(INT10
-            : "+a" (ax), "+b" (bx), "+d" (dx), "=m" (boot_params.edid_info),
-              "+c" (cx), "+D" (di)
-            : : "esi");
+        ireg.ax = 0x4f15;               /* VBE DDC */
+        ireg.bx = 0x0001;               /* Read EDID */
+        /* ireg.cx = 0; */              /* Controller 0 */
+        /* ireg.dx = 0; */              /* EDID block number */
+        ireg.es = ds();
+        ireg.di =(size_t)&boot_params.edid_info; /* (ES:)Pointer to block */
+        intcall(0x10, &ireg, &oreg);
 #endif /* CONFIG_FIRMWARE_EDID */
 }
 
diff --git a/arch/x86/boot/video-vga.c b/arch/x86/boot/video-vga.c
index 9e0587a37768..8f8d827e254d 100644
--- a/arch/x86/boot/video-vga.c
+++ b/arch/x86/boot/video-vga.c
@@ -2,6 +2,7 @@
  *
  *   Copyright (C) 1991, 1992 Linus Torvalds
  *   Copyright 2007 rPath, Inc. - All Rights Reserved
+ *   Copyright 2009 Intel Corporation; author H. Peter Anvin
  *
  *   This file is part of the Linux kernel, and is made available under
  *   the terms of the GNU General Public License version 2.
@@ -39,30 +40,30 @@ static __videocard video_vga;
 /* Set basic 80x25 mode */
 static u8 vga_set_basic_mode(void)
 {
+        struct biosregs ireg, oreg;
         u16 ax;
         u8 rows;
         u8 mode;
 
+        initregs(&ireg);
+
 #ifdef CONFIG_VIDEO_400_HACK
         if (adapter >= ADAPTER_VGA) {
-                asm volatile(INT10
-                             : : "a" (0x1202), "b" (0x0030)
-                             : "ecx", "edx", "esi", "edi");
+                ireg.ax = 0x1202;
+                ireg.bx = 0x0030;
+                intcall(0x10, &ireg, NULL);
         }
 #endif
 
         ax = 0x0f00;
-        asm volatile(INT10
-                     : "+a" (ax)
-                     : : "ebx", "ecx", "edx", "esi", "edi");
-
-        mode = (u8)ax;
+        intcall(0x10, &ireg, &oreg);
+        mode = oreg.al;
 
         set_fs(0);
         rows = rdfs8(0x484);    /* rows minus one */
 
 #ifndef CONFIG_VIDEO_400_HACK
-        if ((ax == 0x5003 || ax == 0x5007) &&
+        if ((oreg.ax == 0x5003 || oreg.ax == 0x5007) &&
             (rows == 0 || rows == 24))
                 return mode;
 #endif
@@ -71,10 +72,8 @@ static u8 vga_set_basic_mode(void)
                 mode = 3;
 
         /* Set the mode */
-        ax = mode;
-        asm volatile(INT10
-                     : "+a" (ax)
-                     : : "ebx", "ecx", "edx", "esi", "edi");
+        ireg.ax = mode;         /* AH=0: set mode */
+        intcall(0x10, &ireg, NULL);
         do_restore = 1;
         return mode;
 }
@@ -82,43 +81,69 @@ static u8 vga_set_basic_mode(void)
 static void vga_set_8font(void)
 {
         /* Set 8x8 font - 80x43 on EGA, 80x50 on VGA */
+        struct biosregs ireg;
+
+        initregs(&ireg);
 
         /* Set 8x8 font */
-        asm volatile(INT10 : : "a" (0x1112), "b" (0));
+        ireg.ax = 0x1112;
+        /* ireg.bl = 0; */
+        intcall(0x10, &ireg, NULL);
 
         /* Use alternate print screen */
-        asm volatile(INT10 : : "a" (0x1200), "b" (0x20));
+        ireg.ax = 0x1200;
+        ireg.bl = 0x20;
+        intcall(0x10, &ireg, NULL);
 
         /* Turn off cursor emulation */
-        asm volatile(INT10 : : "a" (0x1201), "b" (0x34));
+        ireg.ax = 0x1201;
+        ireg.bl = 0x34;
+        intcall(0x10, &ireg, NULL);
 
         /* Cursor is scan lines 6-7 */
-        asm volatile(INT10 : : "a" (0x0100), "c" (0x0607));
+        ireg.ax = 0x0100;
+        ireg.cx = 0x0607;
+        intcall(0x10, &ireg, NULL);
 }
 
 static void vga_set_14font(void)
 {
         /* Set 9x14 font - 80x28 on VGA */
+        struct biosregs ireg;
+
+        initregs(&ireg);
 
         /* Set 9x14 font */
-        asm volatile(INT10 : : "a" (0x1111), "b" (0));
+        ireg.ax = 0x1111;
+        /* ireg.bl = 0; */
+        intcall(0x10, &ireg, NULL);
 
         /* Turn off cursor emulation */
-        asm volatile(INT10 : : "a" (0x1201), "b" (0x34));
+        ireg.ax = 0x1201;
+        ireg.bl = 0x34;
+        intcall(0x10, &ireg, NULL);
 
         /* Cursor is scan lines 11-12 */
-        asm volatile(INT10 : : "a" (0x0100), "c" (0x0b0c));
+        ireg.ax = 0x0100;
+        ireg.cx = 0x0b0c;
+        intcall(0x10, &ireg, NULL);
 }
 
 static void vga_set_80x43(void)
 {
         /* Set 80x43 mode on VGA (not EGA) */
+        struct biosregs ireg;
+
+        initregs(&ireg);
 
         /* Set 350 scans */
-        asm volatile(INT10 : : "a" (0x1201), "b" (0x30));
+        ireg.ax = 0x1201;
+        ireg.bl = 0x30;
+        intcall(0x10, &ireg, NULL);
 
         /* Reset video mode */
-        asm volatile(INT10 : : "a" (0x0003));
+        ireg.ax = 0x0003;
+        intcall(0x10, &ireg, NULL);
 
         vga_set_8font();
 }
@@ -225,8 +250,6 @@ static int vga_set_mode(struct mode_info *mode)
  */
 static int vga_probe(void)
 {
-        u16 ega_bx;
-
         static const char *card_name[] = {
                 "CGA/MDA/HGC", "EGA", "VGA"
         };
@@ -240,26 +263,26 @@ static int vga_probe(void)
                 sizeof(ega_modes)/sizeof(struct mode_info),
                 sizeof(vga_modes)/sizeof(struct mode_info),
         };
-        u8 vga_flag;
 
-        asm(INT10
-            : "=b" (ega_bx)
-            : "a" (0x1200), "b" (0x10) /* Check EGA/VGA */
-            : "ecx", "edx", "esi", "edi");
+        struct biosregs ireg, oreg;
+
+        initregs(&ireg);
+
+        ireg.ax = 0x1200;
+        ireg.bl = 0x10;         /* Check EGA/VGA */
+        intcall(0x10, &ireg, &oreg);
 
 #ifndef _WAKEUP
-        boot_params.screen_info.orig_video_ega_bx = ega_bx;
+        boot_params.screen_info.orig_video_ega_bx = oreg.bx;
 #endif
 
         /* If we have MDA/CGA/HGC then BL will be unchanged at 0x10 */
-        if ((u8)ega_bx != 0x10) {
+        if (oreg.bl != 0x10) {
                 /* EGA/VGA */
-                asm(INT10
-                    : "=a" (vga_flag)
-                    : "a" (0x1a00)
-                    : "ebx", "ecx", "edx", "esi", "edi");
+                ireg.ax = 0x1a00;
+                intcall(0x10, &ireg, &oreg);
 
-                if (vga_flag == 0x1a) {
+                if (oreg.al == 0x1a) {
                         adapter = ADAPTER_VGA;
 #ifndef _WAKEUP
                         boot_params.screen_info.orig_video_isVGA = 1;
diff --git a/arch/x86/boot/video.c b/arch/x86/boot/video.c
index 3bef2c1febe9..bad728b76fc2 100644
--- a/arch/x86/boot/video.c
+++ b/arch/x86/boot/video.c
@@ -2,6 +2,7 @@
  *
  *   Copyright (C) 1991, 1992 Linus Torvalds
  *   Copyright 2007 rPath, Inc. - All Rights Reserved
+ *   Copyright 2009 Intel Corporation; author H. Peter Anvin
  *
  *   This file is part of the Linux kernel, and is made available under
  *   the terms of the GNU General Public License version 2.
@@ -18,33 +19,29 @@
 
 static void store_cursor_position(void)
 {
-        u16 curpos;
-        u16 ax, bx;
+        struct biosregs ireg, oreg;
 
-        ax = 0x0300;
-        bx = 0;
-        asm(INT10
-            : "=d" (curpos), "+a" (ax), "+b" (bx)
-            : : "ecx", "esi", "edi");
+        initregs(&ireg);
+        ireg.ah = 0x03;
+        intcall(0x10, &ireg, &oreg);
 
-        boot_params.screen_info.orig_x = curpos;
-        boot_params.screen_info.orig_y = curpos >> 8;
+        boot_params.screen_info.orig_x = oreg.dl;
+        boot_params.screen_info.orig_y = oreg.dh;
 }
 
 static void store_video_mode(void)
 {
-        u16 ax, page;
+        struct biosregs ireg, oreg;
 
         /* N.B.: the saving of the video page here is a bit silly,
            since we pretty much assume page 0 everywhere. */
-        ax = 0x0f00;
-        asm(INT10
-            : "+a" (ax), "=b" (page)
-            : : "ecx", "edx", "esi", "edi");
+        initregs(&ireg);
+        ireg.ah = 0x0f;
+        intcall(0x10, &ireg, &oreg);
 
         /* Not all BIOSes are clean with respect to the top bit */
-        boot_params.screen_info.orig_video_mode = ax & 0x7f;
-        boot_params.screen_info.orig_video_page = page >> 8;
+        boot_params.screen_info.orig_video_mode = oreg.al & 0x7f;
+        boot_params.screen_info.orig_video_page = oreg.bh;
 }
 
 /*
@@ -257,7 +254,7 @@ static void restore_screen(void)
         int y;
         addr_t dst = 0;
         u16 *src = saved.data;
-        u16 ax, bx, dx;
+        struct biosregs ireg;
 
         if (graphic_mode)
                 return;         /* Can't restore onto a graphic mode */
@@ -296,12 +293,11 @@ static void restore_screen(void)
         }
 
         /* Restore cursor position */
-        ax = 0x0200;            /* Set cursor position */
-        bx = 0;                 /* Page number (<< 8) */
-        dx = (saved.cury << 8)+saved.curx;
-        asm volatile(INT10
-                     : "+a" (ax), "+b" (bx), "+d" (dx)
-                     : : "ecx", "esi", "edi");
+        initregs(&ireg);
+        ireg.ah = 0x02;         /* Set cursor position */
+        ireg.dh = saved.cury;
+        ireg.dl = saved.curx;
+        intcall(0x10, &ireg, NULL);
 }
 #else
 #define save_screen()           ((void)0)
diff --git a/arch/x86/boot/video.h b/arch/x86/boot/video.h
index ee63f5d14461..5bb174a997fc 100644
--- a/arch/x86/boot/video.h
+++ b/arch/x86/boot/video.h
@@ -112,20 +112,6 @@ extern int force_x, force_y;	/* Don't query the BIOS for cols/rows */
 extern int do_restore;          /* Restore screen contents */
 extern int graphic_mode;        /* Graphics mode with linear frame buffer */
 
-/*
- * int $0x10 is notorious for touching registers it shouldn't.
- * gcc doesn't like %ebp being clobbered, so define it as a push/pop
- * sequence here.
- *
- * A number of systems, including the original PC can clobber %bp in
- * certain circumstances, like when scrolling.  There exists at least
- * one Trident video card which could clobber DS under a set of
- * circumstances that we are unlikely to encounter (scrolling when
- * using an extended graphics mode of more than 800x600 pixels), but
- * it's cheap insurance to deal with that here.
- */
-#define INT10 "pushl %%ebp; pushw %%ds; int $0x10; popw %%ds; popl %%ebp"
-
 /* Accessing VGA indexed registers */
 static inline u8 in_idx(u16 port, u8 index)
 {
diff --git a/arch/x86/configs/i386_defconfig b/arch/x86/configs/i386_defconfig
index 235b81d0f6f2..edb992ebef92 100644
--- a/arch/x86/configs/i386_defconfig
+++ b/arch/x86/configs/i386_defconfig
@@ -1,12 +1,13 @@
 #
 # Automatically generated make config: don't edit
-# Linux kernel version: 2.6.29-rc4
-# Tue Feb 24 15:50:58 2009
+# Linux kernel version: 2.6.30-rc2
+# Mon May 11 16:21:55 2009
 #
 # CONFIG_64BIT is not set
 CONFIG_X86_32=y
 # CONFIG_X86_64 is not set
 CONFIG_X86=y
+CONFIG_OUTPUT_FORMAT="elf32-i386"
 CONFIG_ARCH_DEFCONFIG="arch/x86/configs/i386_defconfig"
 CONFIG_GENERIC_TIME=y
 CONFIG_GENERIC_CMOS_UPDATE=y
@@ -33,6 +34,7 @@ CONFIG_ARCH_HAS_CPU_RELAX=y
 CONFIG_ARCH_HAS_DEFAULT_IDLE=y
 CONFIG_ARCH_HAS_CACHE_LINE_SIZE=y
 CONFIG_HAVE_SETUP_PER_CPU_AREA=y
+CONFIG_HAVE_DYNAMIC_PER_CPU_AREA=y
 # CONFIG_HAVE_CPUMASK_OF_CPU_MAP is not set
 CONFIG_ARCH_HIBERNATION_POSSIBLE=y
 CONFIG_ARCH_SUSPEND_POSSIBLE=y
@@ -40,15 +42,16 @@ CONFIG_ARCH_SUSPEND_POSSIBLE=y
 CONFIG_ARCH_POPULATES_NODE_MAP=y
 # CONFIG_AUDIT_ARCH is not set
 CONFIG_ARCH_SUPPORTS_OPTIMIZED_INLINING=y
+CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC=y
 CONFIG_GENERIC_HARDIRQS=y
+CONFIG_GENERIC_HARDIRQS_NO__DO_IRQ=y
 CONFIG_GENERIC_IRQ_PROBE=y
 CONFIG_GENERIC_PENDING_IRQ=y
-CONFIG_X86_SMP=y
 CONFIG_USE_GENERIC_SMP_HELPERS=y
 CONFIG_X86_32_SMP=y
 CONFIG_X86_HT=y
-CONFIG_X86_BIOS_REBOOT=y
 CONFIG_X86_TRAMPOLINE=y
+CONFIG_X86_32_LAZY_GS=y
 CONFIG_KTIME_SCALAR=y
 CONFIG_DEFCONFIG_LIST="/lib/modules/$UNAME_RELEASE/.config"
 
@@ -60,10 +63,17 @@ CONFIG_LOCK_KERNEL=y
 CONFIG_INIT_ENV_ARG_LIMIT=32
 CONFIG_LOCALVERSION=""
 # CONFIG_LOCALVERSION_AUTO is not set
+CONFIG_HAVE_KERNEL_GZIP=y
+CONFIG_HAVE_KERNEL_BZIP2=y
+CONFIG_HAVE_KERNEL_LZMA=y
+CONFIG_KERNEL_GZIP=y
+# CONFIG_KERNEL_BZIP2 is not set
+# CONFIG_KERNEL_LZMA is not set
 CONFIG_SWAP=y
 CONFIG_SYSVIPC=y
 CONFIG_SYSVIPC_SYSCTL=y
 CONFIG_POSIX_MQUEUE=y
+CONFIG_POSIX_MQUEUE_SYSCTL=y
 CONFIG_BSD_PROCESS_ACCT=y
 # CONFIG_BSD_PROCESS_ACCT_V3 is not set
 CONFIG_TASKSTATS=y
@@ -113,23 +123,26 @@ CONFIG_PID_NS=y
 CONFIG_NET_NS=y
 CONFIG_BLK_DEV_INITRD=y
 CONFIG_INITRAMFS_SOURCE=""
+CONFIG_RD_GZIP=y
+CONFIG_RD_BZIP2=y
+CONFIG_RD_LZMA=y
 CONFIG_CC_OPTIMIZE_FOR_SIZE=y
 CONFIG_SYSCTL=y
+CONFIG_ANON_INODES=y
 # CONFIG_EMBEDDED is not set
 CONFIG_UID16=y
 CONFIG_SYSCTL_SYSCALL=y
 CONFIG_KALLSYMS=y
 CONFIG_KALLSYMS_ALL=y
 CONFIG_KALLSYMS_EXTRA_PASS=y
+# CONFIG_STRIP_ASM_SYMS is not set
 CONFIG_HOTPLUG=y
 CONFIG_PRINTK=y
 CONFIG_BUG=y
 CONFIG_ELF_CORE=y
 CONFIG_PCSPKR_PLATFORM=y
-# CONFIG_COMPAT_BRK is not set
 CONFIG_BASE_FULL=y
 CONFIG_FUTEX=y
-CONFIG_ANON_INODES=y
 CONFIG_EPOLL=y
 CONFIG_SIGNALFD=y
 CONFIG_TIMERFD=y
@@ -139,6 +152,7 @@ CONFIG_AIO=y
 CONFIG_VM_EVENT_COUNTERS=y
 CONFIG_PCI_QUIRKS=y
 CONFIG_SLUB_DEBUG=y
+# CONFIG_COMPAT_BRK is not set
 # CONFIG_SLAB is not set
 CONFIG_SLUB=y
 # CONFIG_SLOB is not set
@@ -154,6 +168,8 @@ CONFIG_HAVE_IOREMAP_PROT=y
 CONFIG_HAVE_KPROBES=y
 CONFIG_HAVE_KRETPROBES=y
 CONFIG_HAVE_ARCH_TRACEHOOK=y
+CONFIG_HAVE_DMA_API_DEBUG=y
+# CONFIG_SLOW_WORK is not set
 CONFIG_HAVE_GENERIC_DMA_COHERENT=y
 CONFIG_SLABINFO=y
 CONFIG_RT_MUTEXES=y
@@ -167,7 +183,6 @@ CONFIG_MODULE_FORCE_UNLOAD=y
 CONFIG_STOP_MACHINE=y
 CONFIG_BLOCK=y
 # CONFIG_LBD is not set
-CONFIG_BLK_DEV_IO_TRACE=y
 CONFIG_BLK_DEV_BSG=y
 # CONFIG_BLK_DEV_INTEGRITY is not set
 
@@ -194,12 +209,12 @@ CONFIG_HIGH_RES_TIMERS=y
 CONFIG_GENERIC_CLOCKEVENTS_BUILD=y
 CONFIG_SMP=y
 CONFIG_SPARSE_IRQ=y
-CONFIG_X86_FIND_SMP_CONFIG=y
 CONFIG_X86_MPPARSE=y
+# CONFIG_X86_BIGSMP is not set
+CONFIG_X86_EXTENDED_PLATFORM=y
 # CONFIG_X86_ELAN is not set
-# CONFIG_X86_GENERICARCH is not set
-# CONFIG_X86_VSMP is not set
 # CONFIG_X86_RDC321X is not set
+# CONFIG_X86_32_NON_STANDARD is not set
 CONFIG_SCHED_OMIT_FRAME_POINTER=y
 # CONFIG_PARAVIRT_GUEST is not set
 # CONFIG_MEMTEST is not set
@@ -230,8 +245,10 @@ CONFIG_M686=y
 # CONFIG_GENERIC_CPU is not set
 CONFIG_X86_GENERIC=y
 CONFIG_X86_CPU=y
+CONFIG_X86_L1_CACHE_BYTES=64
+CONFIG_X86_INTERNODE_CACHE_BYTES=64
 CONFIG_X86_CMPXCHG=y
-CONFIG_X86_L1_CACHE_SHIFT=7
+CONFIG_X86_L1_CACHE_SHIFT=5
 CONFIG_X86_XADD=y
 # CONFIG_X86_PPRO_FENCE is not set
 CONFIG_X86_WP_WORKS_OK=y
@@ -247,7 +264,7 @@ CONFIG_X86_DEBUGCTLMSR=y
 CONFIG_CPU_SUP_INTEL=y
 CONFIG_CPU_SUP_CYRIX_32=y
 CONFIG_CPU_SUP_AMD=y
-CONFIG_CPU_SUP_CENTAUR_32=y
+CONFIG_CPU_SUP_CENTAUR=y
 CONFIG_CPU_SUP_TRANSMETA_32=y
 CONFIG_CPU_SUP_UMC_32=y
 CONFIG_X86_DS=y
@@ -279,6 +296,7 @@ CONFIG_MICROCODE_AMD=y
 CONFIG_MICROCODE_OLD_INTERFACE=y
 CONFIG_X86_MSR=y
 CONFIG_X86_CPUID=y
+# CONFIG_X86_CPU_DEBUG is not set
 # CONFIG_NOHIGHMEM is not set
 CONFIG_HIGHMEM4G=y
 # CONFIG_HIGHMEM64G is not set
@@ -302,6 +320,8 @@ CONFIG_ZONE_DMA_FLAG=1
 CONFIG_BOUNCE=y
 CONFIG_VIRT_TO_BUS=y
 CONFIG_UNEVICTABLE_LRU=y
+CONFIG_HAVE_MLOCK=y
+CONFIG_HAVE_MLOCKED_PAGE_BIT=y
 CONFIG_HIGHPTE=y
 CONFIG_X86_CHECK_BIOS_CORRUPTION=y
 CONFIG_X86_BOOTPARAM_MEMORY_CORRUPTION_CHECK=y
@@ -312,6 +332,7 @@ CONFIG_MTRR=y
 CONFIG_X86_PAT=y
 CONFIG_EFI=y
 CONFIG_SECCOMP=y
+# CONFIG_CC_STACKPROTECTOR is not set
 # CONFIG_HZ_100 is not set
 # CONFIG_HZ_250 is not set
 # CONFIG_HZ_300 is not set
@@ -322,8 +343,9 @@ CONFIG_KEXEC=y
 CONFIG_CRASH_DUMP=y
 # CONFIG_KEXEC_JUMP is not set
 CONFIG_PHYSICAL_START=0x1000000
-# CONFIG_RELOCATABLE is not set
-CONFIG_PHYSICAL_ALIGN=0x200000
+CONFIG_RELOCATABLE=y
+CONFIG_X86_NEED_RELOCS=y
+CONFIG_PHYSICAL_ALIGN=0x1000000
 CONFIG_HOTPLUG_CPU=y
 # CONFIG_COMPAT_VDSO is not set
 # CONFIG_CMDLINE_BOOL is not set
@@ -363,7 +385,6 @@ CONFIG_ACPI_THERMAL=y
 CONFIG_ACPI_BLACKLIST_YEAR=0
 # CONFIG_ACPI_DEBUG is not set
 # CONFIG_ACPI_PCI_SLOT is not set
-CONFIG_ACPI_SYSTEM=y
 CONFIG_X86_PM_TIMER=y
 CONFIG_ACPI_CONTAINER=y
 # CONFIG_ACPI_SBS is not set
@@ -425,6 +446,7 @@ CONFIG_PCI_BIOS=y
 CONFIG_PCI_DIRECT=y
 CONFIG_PCI_MMCONFIG=y
 CONFIG_PCI_DOMAINS=y
+# CONFIG_DMAR is not set
 CONFIG_PCIEPORTBUS=y
 # CONFIG_HOTPLUG_PCI_PCIE is not set
 CONFIG_PCIEAER=y
@@ -435,6 +457,7 @@ CONFIG_PCI_MSI=y
 # CONFIG_PCI_DEBUG is not set
 # CONFIG_PCI_STUB is not set
 CONFIG_HT_IRQ=y
+# CONFIG_PCI_IOV is not set
 CONFIG_ISA_DMA_API=y
 # CONFIG_ISA is not set
 # CONFIG_MCA is not set
@@ -481,7 +504,6 @@ CONFIG_NET=y
 #
 # Networking options
 #
-CONFIG_COMPAT_NET_DEV_OPS=y
 CONFIG_PACKET=y
 CONFIG_PACKET_MMAP=y
 CONFIG_UNIX=y
@@ -639,6 +661,7 @@ CONFIG_LLC=y
 # CONFIG_LAPB is not set
 # CONFIG_ECONET is not set
 # CONFIG_WAN_ROUTER is not set
+# CONFIG_PHONET is not set
 CONFIG_NET_SCHED=y
 
 #
@@ -696,6 +719,7 @@ CONFIG_NET_SCH_FIFO=y
 #
 # CONFIG_NET_PKTGEN is not set
 # CONFIG_NET_TCPPROBE is not set
+# CONFIG_NET_DROP_MONITOR is not set
 CONFIG_HAMRADIO=y
 
 #
@@ -706,12 +730,10 @@ CONFIG_HAMRADIO=y
 # CONFIG_IRDA is not set
 # CONFIG_BT is not set
 # CONFIG_AF_RXRPC is not set
-# CONFIG_PHONET is not set
 CONFIG_FIB_RULES=y
 CONFIG_WIRELESS=y
 CONFIG_CFG80211=y
 # CONFIG_CFG80211_REG_DEBUG is not set
-CONFIG_NL80211=y
 CONFIG_WIRELESS_OLD_REGULATORY=y
 CONFIG_WIRELESS_EXT=y
 CONFIG_WIRELESS_EXT_SYSFS=y
@@ -789,6 +811,7 @@ CONFIG_MISC_DEVICES=y
 # CONFIG_ICS932S401 is not set
 # CONFIG_ENCLOSURE_SERVICES is not set
 # CONFIG_HP_ILO is not set
+# CONFIG_ISL29003 is not set
 # CONFIG_C2PORT is not set
 
 #
@@ -842,6 +865,7 @@ CONFIG_SCSI_SPI_ATTRS=y
 # CONFIG_SCSI_LOWLEVEL is not set
 # CONFIG_SCSI_LOWLEVEL_PCMCIA is not set
 # CONFIG_SCSI_DH is not set
+# CONFIG_SCSI_OSD_INITIATOR is not set
 CONFIG_ATA=y
 # CONFIG_ATA_NONSTANDARD is not set
 CONFIG_ATA_ACPI=y
@@ -940,6 +964,7 @@ CONFIG_DM_ZERO=y
 CONFIG_MACINTOSH_DRIVERS=y
 CONFIG_MAC_EMUMOUSEBTN=y
 CONFIG_NETDEVICES=y
+CONFIG_COMPAT_NET_DEV_OPS=y
 # CONFIG_IFB is not set
 # CONFIG_DUMMY is not set
 # CONFIG_BONDING is not set
@@ -977,6 +1002,8 @@ CONFIG_MII=y
 CONFIG_NET_VENDOR_3COM=y
 # CONFIG_VORTEX is not set
 # CONFIG_TYPHOON is not set
+# CONFIG_ETHOC is not set
+# CONFIG_DNET is not set
 CONFIG_NET_TULIP=y
 # CONFIG_DE2104X is not set
 # CONFIG_TULIP is not set
@@ -1026,6 +1053,7 @@ CONFIG_E1000=y
 CONFIG_E1000E=y
 # CONFIG_IP1000 is not set
 # CONFIG_IGB is not set
+# CONFIG_IGBVF is not set
 # CONFIG_NS83820 is not set
 # CONFIG_HAMACHI is not set
 # CONFIG_YELLOWFIN is not set
@@ -1040,6 +1068,7 @@ CONFIG_BNX2=y
 # CONFIG_QLA3XXX is not set
 # CONFIG_ATL1 is not set
 # CONFIG_ATL1E is not set
+# CONFIG_ATL1C is not set
 # CONFIG_JME is not set
 CONFIG_NETDEV_10000=y
 # CONFIG_CHELSIO_T1 is not set
@@ -1049,6 +1078,7 @@ CONFIG_CHELSIO_T3_DEPENDS=y
 # CONFIG_IXGBE is not set
 # CONFIG_IXGB is not set
 # CONFIG_S2IO is not set
+# CONFIG_VXGE is not set
 # CONFIG_MYRI10GE is not set
 # CONFIG_NETXEN_NIC is not set
 # CONFIG_NIU is not set
@@ -1058,6 +1088,7 @@ CONFIG_CHELSIO_T3_DEPENDS=y
 # CONFIG_BNX2X is not set
 # CONFIG_QLGE is not set
 # CONFIG_SFC is not set
+# CONFIG_BE2NET is not set
 CONFIG_TR=y
 # CONFIG_IBMOL is not set
 # CONFIG_IBMLS is not set
@@ -1073,8 +1104,8 @@ CONFIG_WLAN_80211=y
 # CONFIG_LIBERTAS is not set
 # CONFIG_LIBERTAS_THINFIRM is not set
 # CONFIG_AIRO is not set
-# CONFIG_HERMES is not set
 # CONFIG_ATMEL is not set
+# CONFIG_AT76C50X_USB is not set
 # CONFIG_AIRO_CS is not set
 # CONFIG_PCMCIA_WL3501 is not set
 # CONFIG_PRISM54 is not set
@@ -1084,21 +1115,21 @@ CONFIG_WLAN_80211=y
 # CONFIG_RTL8187 is not set
 # CONFIG_ADM8211 is not set
 # CONFIG_MAC80211_HWSIM is not set
+# CONFIG_MWL8K is not set
 # CONFIG_P54_COMMON is not set
 CONFIG_ATH5K=y
 # CONFIG_ATH5K_DEBUG is not set
 # CONFIG_ATH9K is not set
+# CONFIG_AR9170_USB is not set
 # CONFIG_IPW2100 is not set
 # CONFIG_IPW2200 is not set
-# CONFIG_IWLCORE is not set
-# CONFIG_IWLWIFI_LEDS is not set
-# CONFIG_IWLAGN is not set
-# CONFIG_IWL3945 is not set
+# CONFIG_IWLWIFI is not set
 # CONFIG_HOSTAP is not set
 # CONFIG_B43 is not set
 # CONFIG_B43LEGACY is not set
 # CONFIG_ZD1211RW is not set
 # CONFIG_RT2X00 is not set
+# CONFIG_HERMES is not set
 
 #
 # Enable WiMAX (Networking options) to see the WiMAX drivers
@@ -1209,6 +1240,8 @@ CONFIG_INPUT_TABLET=y
 # CONFIG_TABLET_USB_KBTAB is not set
 # CONFIG_TABLET_USB_WACOM is not set
 CONFIG_INPUT_TOUCHSCREEN=y
+# CONFIG_TOUCHSCREEN_AD7879_I2C is not set
+# CONFIG_TOUCHSCREEN_AD7879 is not set
 # CONFIG_TOUCHSCREEN_FUJITSU is not set
 # CONFIG_TOUCHSCREEN_GUNZE is not set
 # CONFIG_TOUCHSCREEN_ELO is not set
@@ -1303,6 +1336,7 @@ CONFIG_UNIX98_PTYS=y
 # CONFIG_LEGACY_PTYS is not set
 # CONFIG_IPMI_HANDLER is not set
 CONFIG_HW_RANDOM=y
+# CONFIG_HW_RANDOM_TIMERIOMEM is not set
 CONFIG_HW_RANDOM_INTEL=y
 CONFIG_HW_RANDOM_AMD=y
 CONFIG_HW_RANDOM_GEODE=y
@@ -1390,7 +1424,6 @@ CONFIG_I2C_I801=y
 # CONFIG_SENSORS_PCF8574 is not set
 # CONFIG_PCF8575 is not set
 # CONFIG_SENSORS_PCA9539 is not set
-# CONFIG_SENSORS_PCF8591 is not set
 # CONFIG_SENSORS_MAX6875 is not set
 # CONFIG_SENSORS_TSL2550 is not set
 # CONFIG_I2C_DEBUG_CORE is not set
@@ -1424,6 +1457,7 @@ CONFIG_HWMON=y
 # CONFIG_SENSORS_ADT7475 is not set
 # CONFIG_SENSORS_K8TEMP is not set
 # CONFIG_SENSORS_ASB100 is not set
+# CONFIG_SENSORS_ATK0110 is not set
 # CONFIG_SENSORS_ATXP1 is not set
 # CONFIG_SENSORS_DS1621 is not set
 # CONFIG_SENSORS_I5K_AMB is not set
@@ -1433,6 +1467,7 @@ CONFIG_HWMON=y
 # CONFIG_SENSORS_FSCHER is not set
 # CONFIG_SENSORS_FSCPOS is not set
 # CONFIG_SENSORS_FSCHMD is not set
+# CONFIG_SENSORS_G760A is not set
 # CONFIG_SENSORS_GL518SM is not set
 # CONFIG_SENSORS_GL520SM is not set
 # CONFIG_SENSORS_CORETEMP is not set
@@ -1448,11 +1483,14 @@ CONFIG_HWMON=y
 # CONFIG_SENSORS_LM90 is not set
 # CONFIG_SENSORS_LM92 is not set
 # CONFIG_SENSORS_LM93 is not set
+# CONFIG_SENSORS_LTC4215 is not set
 # CONFIG_SENSORS_LTC4245 is not set
+# CONFIG_SENSORS_LM95241 is not set
 # CONFIG_SENSORS_MAX1619 is not set
 # CONFIG_SENSORS_MAX6650 is not set
 # CONFIG_SENSORS_PC87360 is not set
 # CONFIG_SENSORS_PC87427 is not set
+# CONFIG_SENSORS_PCF8591 is not set
 # CONFIG_SENSORS_SIS5595 is not set
 # CONFIG_SENSORS_DME1737 is not set
 # CONFIG_SENSORS_SMSC47M1 is not set
@@ -1643,7 +1681,6 @@ CONFIG_FB_EFI=y
 # CONFIG_FB_3DFX is not set
 # CONFIG_FB_VOODOO1 is not set
 # CONFIG_FB_VT8623 is not set
-# CONFIG_FB_CYBLA is not set
 # CONFIG_FB_TRIDENT is not set
 # CONFIG_FB_ARK is not set
 # CONFIG_FB_PM3 is not set
@@ -1652,6 +1689,7 @@ CONFIG_FB_EFI=y
 # CONFIG_FB_VIRTUAL is not set
 # CONFIG_FB_METRONOME is not set
 # CONFIG_FB_MB862XX is not set
+# CONFIG_FB_BROADSHEET is not set
 CONFIG_BACKLIGHT_LCD_SUPPORT=y
 # CONFIG_LCD_CLASS_DEVICE is not set
 CONFIG_BACKLIGHT_CLASS_DEVICE=y
@@ -1738,6 +1776,8 @@ CONFIG_SND_PCI=y
 # CONFIG_SND_INDIGO is not set
 # CONFIG_SND_INDIGOIO is not set
 # CONFIG_SND_INDIGODJ is not set
+# CONFIG_SND_INDIGOIOX is not set
+# CONFIG_SND_INDIGODJX is not set
 # CONFIG_SND_EMU10K1 is not set
 # CONFIG_SND_EMU10K1X is not set
 # CONFIG_SND_ENS1370 is not set
@@ -1811,15 +1851,17 @@ CONFIG_USB_HIDDEV=y
 #
 # Special HID drivers
 #
-CONFIG_HID_COMPAT=y
 CONFIG_HID_A4TECH=y
 CONFIG_HID_APPLE=y
 CONFIG_HID_BELKIN=y
 CONFIG_HID_CHERRY=y
 CONFIG_HID_CHICONY=y
 CONFIG_HID_CYPRESS=y
+# CONFIG_DRAGONRISE_FF is not set
 CONFIG_HID_EZKEY=y
+CONFIG_HID_KYE=y
 CONFIG_HID_GYRATION=y
+CONFIG_HID_KENSINGTON=y
 CONFIG_HID_LOGITECH=y
 CONFIG_LOGITECH_FF=y
 # CONFIG_LOGIRUMBLEPAD2_FF is not set
@@ -1885,11 +1927,11 @@ CONFIG_USB_PRINTER=y
 # CONFIG_USB_TMC is not set
 
 #
-# NOTE: USB_STORAGE depends on SCSI but BLK_DEV_SD may also be needed;
+# NOTE: USB_STORAGE depends on SCSI but BLK_DEV_SD may
 #
 
 #
-# see USB_STORAGE Help for more information
+# also be needed; see USB_STORAGE Help for more info
 #
 CONFIG_USB_STORAGE=y
 # CONFIG_USB_STORAGE_DEBUG is not set
@@ -1931,7 +1973,6 @@ CONFIG_USB_LIBUSUAL=y
 # CONFIG_USB_LED is not set
 # CONFIG_USB_CYPRESS_CY7C63 is not set
 # CONFIG_USB_CYTHERM is not set
-# CONFIG_USB_PHIDGET is not set
 # CONFIG_USB_IDMOUSE is not set
 # CONFIG_USB_FTDI_ELAN is not set
 # CONFIG_USB_APPLEDISPLAY is not set
@@ -1947,6 +1988,7 @@ CONFIG_USB_LIBUSUAL=y
 #
 # OTG and related infrastructure
 #
+# CONFIG_NOP_USB_XCEIV is not set
 # CONFIG_UWB is not set
 # CONFIG_MMC is not set
 # CONFIG_MEMSTICK is not set
@@ -1958,8 +2000,10 @@ CONFIG_LEDS_CLASS=y
 #
 # CONFIG_LEDS_ALIX2 is not set
 # CONFIG_LEDS_PCA9532 is not set
+# CONFIG_LEDS_LP5521 is not set
 # CONFIG_LEDS_CLEVO_MAIL is not set
 # CONFIG_LEDS_PCA955X is not set
+# CONFIG_LEDS_BD2802 is not set
 
 #
 # LED Triggers
@@ -1969,6 +2013,10 @@ CONFIG_LEDS_TRIGGERS=y
 # CONFIG_LEDS_TRIGGER_HEARTBEAT is not set
 # CONFIG_LEDS_TRIGGER_BACKLIGHT is not set
 # CONFIG_LEDS_TRIGGER_DEFAULT_ON is not set
+
+#
+# iptables trigger is under Netfilter config (LED target)
+#
 # CONFIG_ACCESSIBILITY is not set
 # CONFIG_INFINIBAND is not set
 CONFIG_EDAC=y
@@ -2037,6 +2085,7 @@ CONFIG_DMADEVICES=y
 # DMA Devices
 #
 # CONFIG_INTEL_IOATDMA is not set
+# CONFIG_AUXDISPLAY is not set
 # CONFIG_UIO is not set
 # CONFIG_STAGING is not set
 CONFIG_X86_PLATFORM_DEVICES=y
@@ -2071,6 +2120,7 @@ CONFIG_DMIID=y
 #
 # CONFIG_EXT2_FS is not set
 CONFIG_EXT3_FS=y
+# CONFIG_EXT3_DEFAULTS_TO_ORDERED is not set
 CONFIG_EXT3_FS_XATTR=y
 CONFIG_EXT3_FS_POSIX_ACL=y
 CONFIG_EXT3_FS_SECURITY=y
@@ -2101,6 +2151,11 @@ CONFIG_AUTOFS4_FS=y
 CONFIG_GENERIC_ACL=y
 
 #
+# Caches
+#
+# CONFIG_FSCACHE is not set
+
+#
 # CD-ROM/DVD Filesystems
 #
 CONFIG_ISO9660_FS=y
@@ -2151,6 +2206,7 @@ CONFIG_MISC_FILESYSTEMS=y
 # CONFIG_ROMFS_FS is not set
 # CONFIG_SYSV_FS is not set
 # CONFIG_UFS_FS is not set
+# CONFIG_NILFS2_FS is not set
 CONFIG_NETWORK_FILESYSTEMS=y
 CONFIG_NFS_FS=y
 CONFIG_NFS_V3=y
@@ -2164,7 +2220,6 @@ CONFIG_NFS_ACL_SUPPORT=y
 CONFIG_NFS_COMMON=y
 CONFIG_SUNRPC=y
 CONFIG_SUNRPC_GSS=y
-# CONFIG_SUNRPC_REGISTER_V4 is not set
 CONFIG_RPCSEC_GSS_KRB5=y
 # CONFIG_RPCSEC_GSS_SPKM3 is not set
 # CONFIG_SMB_FS is not set
@@ -2251,6 +2306,7 @@ CONFIG_DEBUG_FS=y
 CONFIG_DEBUG_KERNEL=y
 # CONFIG_DEBUG_SHIRQ is not set
 # CONFIG_DETECT_SOFTLOCKUP is not set
+# CONFIG_DETECT_HUNG_TASK is not set
 # CONFIG_SCHED_DEBUG is not set
 CONFIG_SCHEDSTATS=y
 CONFIG_TIMER_STATS=y
@@ -2266,6 +2322,7 @@ CONFIG_TIMER_STATS=y
 # CONFIG_LOCK_STAT is not set
 # CONFIG_DEBUG_SPINLOCK_SLEEP is not set
 # CONFIG_DEBUG_LOCKING_API_SELFTESTS is not set
+CONFIG_STACKTRACE=y
 # CONFIG_DEBUG_KOBJECT is not set
 # CONFIG_DEBUG_HIGHMEM is not set
 CONFIG_DEBUG_BUGVERBOSE=y
@@ -2289,13 +2346,19 @@ CONFIG_FRAME_POINTER=y
 # CONFIG_FAULT_INJECTION is not set
 # CONFIG_LATENCYTOP is not set
 CONFIG_SYSCTL_SYSCALL_CHECK=y
+# CONFIG_DEBUG_PAGEALLOC is not set
 CONFIG_USER_STACKTRACE_SUPPORT=y
+CONFIG_NOP_TRACER=y
 CONFIG_HAVE_FUNCTION_TRACER=y
 CONFIG_HAVE_FUNCTION_GRAPH_TRACER=y
 CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST=y
 CONFIG_HAVE_DYNAMIC_FTRACE=y
 CONFIG_HAVE_FTRACE_MCOUNT_RECORD=y
 CONFIG_HAVE_HW_BRANCH_TRACER=y
+CONFIG_HAVE_FTRACE_SYSCALLS=y
+CONFIG_RING_BUFFER=y
+CONFIG_TRACING=y
+CONFIG_TRACING_SUPPORT=y
 
 #
 # Tracers
@@ -2305,13 +2368,21 @@ CONFIG_HAVE_HW_BRANCH_TRACER=y
 # CONFIG_SYSPROF_TRACER is not set
 # CONFIG_SCHED_TRACER is not set
 # CONFIG_CONTEXT_SWITCH_TRACER is not set
+# CONFIG_EVENT_TRACER is not set
+# CONFIG_FTRACE_SYSCALLS is not set
 # CONFIG_BOOT_TRACER is not set
 # CONFIG_TRACE_BRANCH_PROFILING is not set
 # CONFIG_POWER_TRACER is not set
 # CONFIG_STACK_TRACER is not set
 # CONFIG_HW_BRANCH_TRACER is not set
+# CONFIG_KMEMTRACE is not set
+# CONFIG_WORKQUEUE_TRACER is not set
+CONFIG_BLK_DEV_IO_TRACE=y
+# CONFIG_FTRACE_STARTUP_TEST is not set
+# CONFIG_MMIOTRACE is not set
 CONFIG_PROVIDE_OHCI1394_DMA_INIT=y
-# CONFIG_DYNAMIC_PRINTK_DEBUG is not set
+# CONFIG_DYNAMIC_DEBUG is not set
+# CONFIG_DMA_API_DEBUG is not set
 # CONFIG_SAMPLES is not set
 CONFIG_HAVE_ARCH_KGDB=y
 # CONFIG_KGDB is not set
@@ -2321,7 +2392,6 @@ CONFIG_EARLY_PRINTK=y
 CONFIG_EARLY_PRINTK_DBGP=y
 CONFIG_DEBUG_STACKOVERFLOW=y
 CONFIG_DEBUG_STACK_USAGE=y
-# CONFIG_DEBUG_PAGEALLOC is not set
 # CONFIG_DEBUG_PER_CPU_MAPS is not set
 # CONFIG_X86_PTDUMP is not set
 CONFIG_DEBUG_RODATA=y
@@ -2329,7 +2399,7 @@ CONFIG_DEBUG_RODATA=y
 CONFIG_DEBUG_NX_TEST=m
 # CONFIG_4KSTACKS is not set
 CONFIG_DOUBLEFAULT=y
-# CONFIG_MMIOTRACE is not set
+CONFIG_HAVE_MMIOTRACE_SUPPORT=y
 CONFIG_IO_DELAY_TYPE_0X80=0
 CONFIG_IO_DELAY_TYPE_0XED=1
 CONFIG_IO_DELAY_TYPE_UDELAY=2
@@ -2365,6 +2435,8 @@ CONFIG_SECURITY_SELINUX_AVC_STATS=y
 CONFIG_SECURITY_SELINUX_CHECKREQPROT_VALUE=1
 # CONFIG_SECURITY_SELINUX_POLICYDB_VERSION_MAX is not set
 # CONFIG_SECURITY_SMACK is not set
+# CONFIG_SECURITY_TOMOYO is not set
+# CONFIG_IMA is not set
 CONFIG_CRYPTO=y
 
 #
@@ -2380,10 +2452,12 @@ CONFIG_CRYPTO_BLKCIPHER2=y
 CONFIG_CRYPTO_HASH=y
 CONFIG_CRYPTO_HASH2=y
 CONFIG_CRYPTO_RNG2=y
+CONFIG_CRYPTO_PCOMP=y
 CONFIG_CRYPTO_MANAGER=y
 CONFIG_CRYPTO_MANAGER2=y
 # CONFIG_CRYPTO_GF128MUL is not set
 # CONFIG_CRYPTO_NULL is not set
+CONFIG_CRYPTO_WORKQUEUE=y
 # CONFIG_CRYPTO_CRYPTD is not set
 CONFIG_CRYPTO_AUTHENC=y
 # CONFIG_CRYPTO_TEST is not set
@@ -2456,6 +2530,7 @@ CONFIG_CRYPTO_DES=y
 # Compression
 #
 # CONFIG_CRYPTO_DEFLATE is not set
+# CONFIG_CRYPTO_ZLIB is not set
 # CONFIG_CRYPTO_LZO is not set
 
 #
@@ -2467,11 +2542,13 @@ CONFIG_CRYPTO_HW=y
 # CONFIG_CRYPTO_DEV_GEODE is not set
 # CONFIG_CRYPTO_DEV_HIFN_795X is not set
 CONFIG_HAVE_KVM=y
+CONFIG_HAVE_KVM_IRQCHIP=y
 CONFIG_VIRTUALIZATION=y
 # CONFIG_KVM is not set
 # CONFIG_LGUEST is not set
 # CONFIG_VIRTIO_PCI is not set
 # CONFIG_VIRTIO_BALLOON is not set
+CONFIG_BINARY_PRINTF=y
 
 #
 # Library routines
@@ -2489,7 +2566,10 @@ CONFIG_CRC32=y
 # CONFIG_LIBCRC32C is not set
 CONFIG_AUDIT_GENERIC=y
 CONFIG_ZLIB_INFLATE=y
-CONFIG_PLIST=y
+CONFIG_DECOMPRESS_GZIP=y
+CONFIG_DECOMPRESS_BZIP2=y
+CONFIG_DECOMPRESS_LZMA=y
 CONFIG_HAS_IOMEM=y
 CONFIG_HAS_IOPORT=y
 CONFIG_HAS_DMA=y
+CONFIG_NLATTR=y
diff --git a/arch/x86/configs/x86_64_defconfig b/arch/x86/configs/x86_64_defconfig
index 27b8ce0f5908..cee1dd2e69b2 100644
--- a/arch/x86/configs/x86_64_defconfig
+++ b/arch/x86/configs/x86_64_defconfig
@@ -1,12 +1,13 @@
 #
 # Automatically generated make config: don't edit
-# Linux kernel version: 2.6.29-rc4
-# Tue Feb 24 15:44:16 2009
+# Linux kernel version: 2.6.30-rc2
+# Mon May 11 16:22:00 2009
 #
 CONFIG_64BIT=y
 # CONFIG_X86_32 is not set
 CONFIG_X86_64=y
 CONFIG_X86=y
+CONFIG_OUTPUT_FORMAT="elf64-x86-64"
 CONFIG_ARCH_DEFCONFIG="arch/x86/configs/x86_64_defconfig"
 CONFIG_GENERIC_TIME=y
 CONFIG_GENERIC_CMOS_UPDATE=y
@@ -34,6 +35,7 @@ CONFIG_ARCH_HAS_CPU_RELAX=y
 CONFIG_ARCH_HAS_DEFAULT_IDLE=y
 CONFIG_ARCH_HAS_CACHE_LINE_SIZE=y
 CONFIG_HAVE_SETUP_PER_CPU_AREA=y
+CONFIG_HAVE_DYNAMIC_PER_CPU_AREA=y
 CONFIG_HAVE_CPUMASK_OF_CPU_MAP=y
 CONFIG_ARCH_HIBERNATION_POSSIBLE=y
 CONFIG_ARCH_SUSPEND_POSSIBLE=y
@@ -41,14 +43,14 @@ CONFIG_ZONE_DMA32=y
 CONFIG_ARCH_POPULATES_NODE_MAP=y
 CONFIG_AUDIT_ARCH=y
 CONFIG_ARCH_SUPPORTS_OPTIMIZED_INLINING=y
+CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC=y
 CONFIG_GENERIC_HARDIRQS=y
+CONFIG_GENERIC_HARDIRQS_NO__DO_IRQ=y
 CONFIG_GENERIC_IRQ_PROBE=y
 CONFIG_GENERIC_PENDING_IRQ=y
-CONFIG_X86_SMP=y
 CONFIG_USE_GENERIC_SMP_HELPERS=y
 CONFIG_X86_64_SMP=y
 CONFIG_X86_HT=y
-CONFIG_X86_BIOS_REBOOT=y
 CONFIG_X86_TRAMPOLINE=y
 # CONFIG_KTIME_SCALAR is not set
 CONFIG_DEFCONFIG_LIST="/lib/modules/$UNAME_RELEASE/.config"
@@ -61,10 +63,17 @@ CONFIG_LOCK_KERNEL=y
 CONFIG_INIT_ENV_ARG_LIMIT=32
 CONFIG_LOCALVERSION=""
 # CONFIG_LOCALVERSION_AUTO is not set
+CONFIG_HAVE_KERNEL_GZIP=y
+CONFIG_HAVE_KERNEL_BZIP2=y
+CONFIG_HAVE_KERNEL_LZMA=y
+CONFIG_KERNEL_GZIP=y
+# CONFIG_KERNEL_BZIP2 is not set
+# CONFIG_KERNEL_LZMA is not set
 CONFIG_SWAP=y
 CONFIG_SYSVIPC=y
 CONFIG_SYSVIPC_SYSCTL=y
 CONFIG_POSIX_MQUEUE=y
+CONFIG_POSIX_MQUEUE_SYSCTL=y
 CONFIG_BSD_PROCESS_ACCT=y
 # CONFIG_BSD_PROCESS_ACCT_V3 is not set
 CONFIG_TASKSTATS=y
@@ -114,23 +123,26 @@ CONFIG_PID_NS=y
 CONFIG_NET_NS=y
 CONFIG_BLK_DEV_INITRD=y
 CONFIG_INITRAMFS_SOURCE=""
+CONFIG_RD_GZIP=y
+CONFIG_RD_BZIP2=y
+CONFIG_RD_LZMA=y
 CONFIG_CC_OPTIMIZE_FOR_SIZE=y
 CONFIG_SYSCTL=y
+CONFIG_ANON_INODES=y
 # CONFIG_EMBEDDED is not set
 CONFIG_UID16=y
 CONFIG_SYSCTL_SYSCALL=y
 CONFIG_KALLSYMS=y
 CONFIG_KALLSYMS_ALL=y
 CONFIG_KALLSYMS_EXTRA_PASS=y
+# CONFIG_STRIP_ASM_SYMS is not set
 CONFIG_HOTPLUG=y
 CONFIG_PRINTK=y
 CONFIG_BUG=y
 CONFIG_ELF_CORE=y
 CONFIG_PCSPKR_PLATFORM=y
-# CONFIG_COMPAT_BRK is not set
 CONFIG_BASE_FULL=y
 CONFIG_FUTEX=y
-CONFIG_ANON_INODES=y
 CONFIG_EPOLL=y
 CONFIG_SIGNALFD=y
 CONFIG_TIMERFD=y
@@ -140,6 +152,7 @@ CONFIG_AIO=y
 CONFIG_VM_EVENT_COUNTERS=y
 CONFIG_PCI_QUIRKS=y
 CONFIG_SLUB_DEBUG=y
+# CONFIG_COMPAT_BRK is not set
 # CONFIG_SLAB is not set
 CONFIG_SLUB=y
 # CONFIG_SLOB is not set
@@ -155,6 +168,8 @@ CONFIG_HAVE_IOREMAP_PROT=y
 CONFIG_HAVE_KPROBES=y
 CONFIG_HAVE_KRETPROBES=y
 CONFIG_HAVE_ARCH_TRACEHOOK=y
+CONFIG_HAVE_DMA_API_DEBUG=y
+# CONFIG_SLOW_WORK is not set
 # CONFIG_HAVE_GENERIC_DMA_COHERENT is not set
 CONFIG_SLABINFO=y
 CONFIG_RT_MUTEXES=y
@@ -167,7 +182,6 @@ CONFIG_MODULE_FORCE_UNLOAD=y
 # CONFIG_MODULE_SRCVERSION_ALL is not set
 CONFIG_STOP_MACHINE=y
 CONFIG_BLOCK=y
-CONFIG_BLK_DEV_IO_TRACE=y
 CONFIG_BLK_DEV_BSG=y
 # CONFIG_BLK_DEV_INTEGRITY is not set
 CONFIG_BLOCK_COMPAT=y
@@ -195,11 +209,10 @@ CONFIG_HIGH_RES_TIMERS=y
 CONFIG_GENERIC_CLOCKEVENTS_BUILD=y
 CONFIG_SMP=y
 CONFIG_SPARSE_IRQ=y
-CONFIG_X86_FIND_SMP_CONFIG=y
 CONFIG_X86_MPPARSE=y
-# CONFIG_X86_ELAN is not set
-# CONFIG_X86_GENERICARCH is not set
+CONFIG_X86_EXTENDED_PLATFORM=y
 # CONFIG_X86_VSMP is not set
+# CONFIG_X86_UV is not set
 CONFIG_SCHED_OMIT_FRAME_POINTER=y
 # CONFIG_PARAVIRT_GUEST is not set
 # CONFIG_MEMTEST is not set
@@ -229,10 +242,10 @@ CONFIG_SCHED_OMIT_FRAME_POINTER=y
 # CONFIG_MCORE2 is not set
 CONFIG_GENERIC_CPU=y
 CONFIG_X86_CPU=y
-CONFIG_X86_L1_CACHE_BYTES=128
-CONFIG_X86_INTERNODE_CACHE_BYTES=128
+CONFIG_X86_L1_CACHE_BYTES=64
+CONFIG_X86_INTERNODE_CACHE_BYTES=64
 CONFIG_X86_CMPXCHG=y
-CONFIG_X86_L1_CACHE_SHIFT=7
+CONFIG_X86_L1_CACHE_SHIFT=6
 CONFIG_X86_WP_WORKS_OK=y
 CONFIG_X86_TSC=y
 CONFIG_X86_CMPXCHG64=y
@@ -241,7 +254,7 @@ CONFIG_X86_MINIMUM_CPU_FAMILY=64
 CONFIG_X86_DEBUGCTLMSR=y
 CONFIG_CPU_SUP_INTEL=y
 CONFIG_CPU_SUP_AMD=y
-CONFIG_CPU_SUP_CENTAUR_64=y
+CONFIG_CPU_SUP_CENTAUR=y
 CONFIG_X86_DS=y
 CONFIG_X86_PTRACE_BTS=y
 CONFIG_HPET_TIMER=y
@@ -268,6 +281,7 @@ CONFIG_X86_REROUTE_FOR_BROKEN_BOOT_IRQS=y
 CONFIG_X86_MCE=y
 CONFIG_X86_MCE_INTEL=y
 CONFIG_X86_MCE_AMD=y
+CONFIG_X86_MCE_THRESHOLD=y
 # CONFIG_I8K is not set
 CONFIG_MICROCODE=y
 CONFIG_MICROCODE_INTEL=y
@@ -275,6 +289,7 @@ CONFIG_MICROCODE_AMD=y
 CONFIG_MICROCODE_OLD_INTERFACE=y
 CONFIG_X86_MSR=y
 CONFIG_X86_CPUID=y
+# CONFIG_X86_CPU_DEBUG is not set
 CONFIG_ARCH_PHYS_ADDR_T_64BIT=y
 CONFIG_DIRECT_GBPAGES=y
 CONFIG_NUMA=y
@@ -308,6 +323,8 @@ CONFIG_ZONE_DMA_FLAG=1
 CONFIG_BOUNCE=y
 CONFIG_VIRT_TO_BUS=y
 CONFIG_UNEVICTABLE_LRU=y
+CONFIG_HAVE_MLOCK=y
+CONFIG_HAVE_MLOCKED_PAGE_BIT=y
 CONFIG_X86_CHECK_BIOS_CORRUPTION=y
 CONFIG_X86_BOOTPARAM_MEMORY_CORRUPTION_CHECK=y
 CONFIG_X86_RESERVE_LOW_64K=y
@@ -316,6 +333,7 @@ CONFIG_MTRR=y
 CONFIG_X86_PAT=y
 CONFIG_EFI=y
 CONFIG_SECCOMP=y
+# CONFIG_CC_STACKPROTECTOR is not set
 # CONFIG_HZ_100 is not set
 # CONFIG_HZ_250 is not set
 # CONFIG_HZ_300 is not set
@@ -324,9 +342,10 @@ CONFIG_HZ=1000
 CONFIG_SCHED_HRTICK=y
 CONFIG_KEXEC=y
 CONFIG_CRASH_DUMP=y
+# CONFIG_KEXEC_JUMP is not set
 CONFIG_PHYSICAL_START=0x1000000
-# CONFIG_RELOCATABLE is not set
-CONFIG_PHYSICAL_ALIGN=0x200000
+CONFIG_RELOCATABLE=y
+CONFIG_PHYSICAL_ALIGN=0x1000000
 CONFIG_HOTPLUG_CPU=y
 # CONFIG_COMPAT_VDSO is not set
 # CONFIG_CMDLINE_BOOL is not set
@@ -369,7 +388,6 @@ CONFIG_ACPI_NUMA=y
 CONFIG_ACPI_BLACKLIST_YEAR=0
 # CONFIG_ACPI_DEBUG is not set
 # CONFIG_ACPI_PCI_SLOT is not set
-CONFIG_ACPI_SYSTEM=y
 CONFIG_X86_PM_TIMER=y
 CONFIG_ACPI_CONTAINER=y
 # CONFIG_ACPI_SBS is not set
@@ -435,6 +453,7 @@ CONFIG_PCI_MSI=y
 # CONFIG_PCI_DEBUG is not set
 # CONFIG_PCI_STUB is not set
 CONFIG_HT_IRQ=y
+# CONFIG_PCI_IOV is not set
 CONFIG_ISA_DMA_API=y
 CONFIG_K8_NB=y
 CONFIG_PCCARD=y
@@ -480,7 +499,6 @@ CONFIG_NET=y
 #
 # Networking options
 #
-CONFIG_COMPAT_NET_DEV_OPS=y
 CONFIG_PACKET=y
 CONFIG_PACKET_MMAP=y
 CONFIG_UNIX=y
@@ -638,6 +656,7 @@ CONFIG_LLC=y
 # CONFIG_LAPB is not set
 # CONFIG_ECONET is not set
 # CONFIG_WAN_ROUTER is not set
+# CONFIG_PHONET is not set
 CONFIG_NET_SCHED=y
 
 #
@@ -695,6 +714,7 @@ CONFIG_NET_SCH_FIFO=y
 #
 # CONFIG_NET_PKTGEN is not set
 # CONFIG_NET_TCPPROBE is not set
+# CONFIG_NET_DROP_MONITOR is not set
 CONFIG_HAMRADIO=y
 
 #
@@ -705,12 +725,10 @@ CONFIG_HAMRADIO=y
 # CONFIG_IRDA is not set
 # CONFIG_BT is not set
 # CONFIG_AF_RXRPC is not set
-# CONFIG_PHONET is not set
 CONFIG_FIB_RULES=y
 CONFIG_WIRELESS=y
 CONFIG_CFG80211=y
 # CONFIG_CFG80211_REG_DEBUG is not set
-CONFIG_NL80211=y
 CONFIG_WIRELESS_OLD_REGULATORY=y
 CONFIG_WIRELESS_EXT=y
 CONFIG_WIRELESS_EXT_SYSFS=y
@@ -787,9 +805,8 @@ CONFIG_MISC_DEVICES=y
 # CONFIG_TIFM_CORE is not set
 # CONFIG_ICS932S401 is not set
 # CONFIG_ENCLOSURE_SERVICES is not set
-# CONFIG_SGI_XP is not set
 # CONFIG_HP_ILO is not set
-# CONFIG_SGI_GRU is not set
+# CONFIG_ISL29003 is not set
 # CONFIG_C2PORT is not set
 
 #
@@ -843,6 +860,7 @@ CONFIG_SCSI_SPI_ATTRS=y
 # CONFIG_SCSI_LOWLEVEL is not set
 # CONFIG_SCSI_LOWLEVEL_PCMCIA is not set
 # CONFIG_SCSI_DH is not set
+# CONFIG_SCSI_OSD_INITIATOR is not set
 CONFIG_ATA=y
 # CONFIG_ATA_NONSTANDARD is not set
 CONFIG_ATA_ACPI=y
@@ -939,6 +957,7 @@ CONFIG_DM_ZERO=y
 CONFIG_MACINTOSH_DRIVERS=y
 CONFIG_MAC_EMUMOUSEBTN=y
 CONFIG_NETDEVICES=y
+CONFIG_COMPAT_NET_DEV_OPS=y
 # CONFIG_IFB is not set
 # CONFIG_DUMMY is not set
 # CONFIG_BONDING is not set
@@ -976,6 +995,8 @@ CONFIG_MII=y
 CONFIG_NET_VENDOR_3COM=y
 # CONFIG_VORTEX is not set
 # CONFIG_TYPHOON is not set
+# CONFIG_ETHOC is not set
+# CONFIG_DNET is not set
 CONFIG_NET_TULIP=y
 # CONFIG_DE2104X is not set
 # CONFIG_TULIP is not set
@@ -1025,6 +1046,7 @@ CONFIG_E1000=y
 # CONFIG_E1000E is not set
 # CONFIG_IP1000 is not set
 # CONFIG_IGB is not set
+# CONFIG_IGBVF is not set
 # CONFIG_NS83820 is not set
 # CONFIG_HAMACHI is not set
 # CONFIG_YELLOWFIN is not set
@@ -1039,6 +1061,7 @@ CONFIG_TIGON3=y
 # CONFIG_QLA3XXX is not set
 # CONFIG_ATL1 is not set
 # CONFIG_ATL1E is not set
+# CONFIG_ATL1C is not set
 # CONFIG_JME is not set
 CONFIG_NETDEV_10000=y
 # CONFIG_CHELSIO_T1 is not set
@@ -1048,6 +1071,7 @@ CONFIG_CHELSIO_T3_DEPENDS=y
 # CONFIG_IXGBE is not set
 # CONFIG_IXGB is not set
 # CONFIG_S2IO is not set
+# CONFIG_VXGE is not set
 # CONFIG_MYRI10GE is not set
 # CONFIG_NETXEN_NIC is not set
 # CONFIG_NIU is not set
@@ -1057,6 +1081,7 @@ CONFIG_CHELSIO_T3_DEPENDS=y
 # CONFIG_BNX2X is not set
 # CONFIG_QLGE is not set
 # CONFIG_SFC is not set
+# CONFIG_BE2NET is not set
 CONFIG_TR=y
 # CONFIG_IBMOL is not set
 # CONFIG_3C359 is not set
@@ -1071,8 +1096,8 @@ CONFIG_WLAN_80211=y
 # CONFIG_LIBERTAS is not set
 # CONFIG_LIBERTAS_THINFIRM is not set
 # CONFIG_AIRO is not set
-# CONFIG_HERMES is not set
 # CONFIG_ATMEL is not set
+# CONFIG_AT76C50X_USB is not set
 # CONFIG_AIRO_CS is not set
 # CONFIG_PCMCIA_WL3501 is not set
 # CONFIG_PRISM54 is not set
@@ -1082,21 +1107,21 @@ CONFIG_WLAN_80211=y
 # CONFIG_RTL8187 is not set
 # CONFIG_ADM8211 is not set
 # CONFIG_MAC80211_HWSIM is not set
+# CONFIG_MWL8K is not set
 # CONFIG_P54_COMMON is not set
 CONFIG_ATH5K=y
 # CONFIG_ATH5K_DEBUG is not set
 # CONFIG_ATH9K is not set
+# CONFIG_AR9170_USB is not set
 # CONFIG_IPW2100 is not set
 # CONFIG_IPW2200 is not set
-# CONFIG_IWLCORE is not set
-# CONFIG_IWLWIFI_LEDS is not set
-# CONFIG_IWLAGN is not set
-# CONFIG_IWL3945 is not set
+# CONFIG_IWLWIFI is not set
 # CONFIG_HOSTAP is not set
 # CONFIG_B43 is not set
 # CONFIG_B43LEGACY is not set
 # CONFIG_ZD1211RW is not set
 # CONFIG_RT2X00 is not set
+# CONFIG_HERMES is not set
 
 #
 # Enable WiMAX (Networking options) to see the WiMAX drivers
@@ -1207,6 +1232,8 @@ CONFIG_INPUT_TABLET=y
 # CONFIG_TABLET_USB_KBTAB is not set
 # CONFIG_TABLET_USB_WACOM is not set
 CONFIG_INPUT_TOUCHSCREEN=y
+# CONFIG_TOUCHSCREEN_AD7879_I2C is not set
+# CONFIG_TOUCHSCREEN_AD7879 is not set
 # CONFIG_TOUCHSCREEN_FUJITSU is not set
 # CONFIG_TOUCHSCREEN_GUNZE is not set
 # CONFIG_TOUCHSCREEN_ELO is not set
@@ -1300,6 +1327,7 @@ CONFIG_UNIX98_PTYS=y
 # CONFIG_LEGACY_PTYS is not set
 # CONFIG_IPMI_HANDLER is not set
 CONFIG_HW_RANDOM=y
+# CONFIG_HW_RANDOM_TIMERIOMEM is not set
 # CONFIG_HW_RANDOM_INTEL is not set
 # CONFIG_HW_RANDOM_AMD is not set
 CONFIG_NVRAM=y
@@ -1381,7 +1409,6 @@ CONFIG_I2C_I801=y
 # CONFIG_SENSORS_PCF8574 is not set
 # CONFIG_PCF8575 is not set
 # CONFIG_SENSORS_PCA9539 is not set
-# CONFIG_SENSORS_PCF8591 is not set
 # CONFIG_SENSORS_MAX6875 is not set
 # CONFIG_SENSORS_TSL2550 is not set
 # CONFIG_I2C_DEBUG_CORE is not set
@@ -1415,6 +1442,7 @@ CONFIG_HWMON=y
 # CONFIG_SENSORS_ADT7475 is not set
 # CONFIG_SENSORS_K8TEMP is not set
 # CONFIG_SENSORS_ASB100 is not set
+# CONFIG_SENSORS_ATK0110 is not set
 # CONFIG_SENSORS_ATXP1 is not set
 # CONFIG_SENSORS_DS1621 is not set
 # CONFIG_SENSORS_I5K_AMB is not set
@@ -1424,6 +1452,7 @@ CONFIG_HWMON=y
 # CONFIG_SENSORS_FSCHER is not set
 # CONFIG_SENSORS_FSCPOS is not set
 # CONFIG_SENSORS_FSCHMD is not set
+# CONFIG_SENSORS_G760A is not set
 # CONFIG_SENSORS_GL518SM is not set
 # CONFIG_SENSORS_GL520SM is not set
 # CONFIG_SENSORS_CORETEMP is not set
@@ -1439,11 +1468,14 @@ CONFIG_HWMON=y
 # CONFIG_SENSORS_LM90 is not set
 # CONFIG_SENSORS_LM92 is not set
 # CONFIG_SENSORS_LM93 is not set
+# CONFIG_SENSORS_LTC4215 is not set
 # CONFIG_SENSORS_LTC4245 is not set
+# CONFIG_SENSORS_LM95241 is not set
 # CONFIG_SENSORS_MAX1619 is not set
 # CONFIG_SENSORS_MAX6650 is not set
 # CONFIG_SENSORS_PC87360 is not set
 # CONFIG_SENSORS_PC87427 is not set
+# CONFIG_SENSORS_PCF8591 is not set
 # CONFIG_SENSORS_SIS5595 is not set
 # CONFIG_SENSORS_DME1737 is not set
 # CONFIG_SENSORS_SMSC47M1 is not set
@@ -1634,6 +1666,7 @@ CONFIG_FB_EFI=y
 # CONFIG_FB_VIRTUAL is not set
 # CONFIG_FB_METRONOME is not set
 # CONFIG_FB_MB862XX is not set
+# CONFIG_FB_BROADSHEET is not set
 CONFIG_BACKLIGHT_LCD_SUPPORT=y
 # CONFIG_LCD_CLASS_DEVICE is not set
 CONFIG_BACKLIGHT_CLASS_DEVICE=y
@@ -1719,6 +1752,8 @@ CONFIG_SND_PCI=y
 # CONFIG_SND_INDIGO is not set
 # CONFIG_SND_INDIGOIO is not set
 # CONFIG_SND_INDIGODJ is not set
+# CONFIG_SND_INDIGOIOX is not set
+# CONFIG_SND_INDIGODJX is not set
 # CONFIG_SND_EMU10K1 is not set
 # CONFIG_SND_EMU10K1X is not set
 # CONFIG_SND_ENS1370 is not set
@@ -1791,15 +1826,17 @@ CONFIG_USB_HIDDEV=y
 #
 # Special HID drivers
 #
-CONFIG_HID_COMPAT=y
 CONFIG_HID_A4TECH=y
 CONFIG_HID_APPLE=y
 CONFIG_HID_BELKIN=y
 CONFIG_HID_CHERRY=y
 CONFIG_HID_CHICONY=y
 CONFIG_HID_CYPRESS=y
+# CONFIG_DRAGONRISE_FF is not set
 CONFIG_HID_EZKEY=y
+CONFIG_HID_KYE=y
 CONFIG_HID_GYRATION=y
+CONFIG_HID_KENSINGTON=y
 CONFIG_HID_LOGITECH=y
 CONFIG_LOGITECH_FF=y
 # CONFIG_LOGIRUMBLEPAD2_FF is not set
@@ -1865,11 +1902,11 @@ CONFIG_USB_PRINTER=y
 # CONFIG_USB_TMC is not set
 
 #
-# NOTE: USB_STORAGE depends on SCSI but BLK_DEV_SD may also be needed;
+# NOTE: USB_STORAGE depends on SCSI but BLK_DEV_SD may
 #
 
 #
-# see USB_STORAGE Help for more information
+# also be needed; see USB_STORAGE Help for more info
 #
 CONFIG_USB_STORAGE=y
 # CONFIG_USB_STORAGE_DEBUG is not set
@@ -1911,7 +1948,6 @@ CONFIG_USB_LIBUSUAL=y
 # CONFIG_USB_LED is not set
 # CONFIG_USB_CYPRESS_CY7C63 is not set
 # CONFIG_USB_CYTHERM is not set
-# CONFIG_USB_PHIDGET is not set
 # CONFIG_USB_IDMOUSE is not set
 # CONFIG_USB_FTDI_ELAN is not set
 # CONFIG_USB_APPLEDISPLAY is not set
@@ -1927,6 +1963,7 @@ CONFIG_USB_LIBUSUAL=y
 #
 # OTG and related infrastructure
 #
+# CONFIG_NOP_USB_XCEIV is not set
 # CONFIG_UWB is not set
 # CONFIG_MMC is not set
 # CONFIG_MEMSTICK is not set
@@ -1938,8 +1975,10 @@ CONFIG_LEDS_CLASS=y
 #
 # CONFIG_LEDS_ALIX2 is not set
 # CONFIG_LEDS_PCA9532 is not set
+# CONFIG_LEDS_LP5521 is not set
 # CONFIG_LEDS_CLEVO_MAIL is not set
 # CONFIG_LEDS_PCA955X is not set
+# CONFIG_LEDS_BD2802 is not set
 
 #
 # LED Triggers
@@ -1949,6 +1988,10 @@ CONFIG_LEDS_TRIGGERS=y
 # CONFIG_LEDS_TRIGGER_HEARTBEAT is not set
 # CONFIG_LEDS_TRIGGER_BACKLIGHT is not set
 # CONFIG_LEDS_TRIGGER_DEFAULT_ON is not set
+
+#
+# iptables trigger is under Netfilter config (LED target)
+#
 # CONFIG_ACCESSIBILITY is not set
 # CONFIG_INFINIBAND is not set
 CONFIG_EDAC=y
@@ -2017,6 +2060,7 @@ CONFIG_DMADEVICES=y
 # DMA Devices
 #
 # CONFIG_INTEL_IOATDMA is not set
+# CONFIG_AUXDISPLAY is not set
 # CONFIG_UIO is not set
 # CONFIG_STAGING is not set
 CONFIG_X86_PLATFORM_DEVICES=y
@@ -2050,6 +2094,7 @@ CONFIG_DMIID=y
 #
 # CONFIG_EXT2_FS is not set
 CONFIG_EXT3_FS=y
+# CONFIG_EXT3_DEFAULTS_TO_ORDERED is not set
 CONFIG_EXT3_FS_XATTR=y
 CONFIG_EXT3_FS_POSIX_ACL=y
 CONFIG_EXT3_FS_SECURITY=y
@@ -2081,6 +2126,11 @@ CONFIG_AUTOFS4_FS=y
 CONFIG_GENERIC_ACL=y
 
 #
+# Caches
+#
+# CONFIG_FSCACHE is not set
+
+#
 # CD-ROM/DVD Filesystems
 #
 CONFIG_ISO9660_FS=y
@@ -2131,6 +2181,7 @@ CONFIG_MISC_FILESYSTEMS=y
 # CONFIG_ROMFS_FS is not set
 # CONFIG_SYSV_FS is not set
 # CONFIG_UFS_FS is not set
+# CONFIG_NILFS2_FS is not set
 CONFIG_NETWORK_FILESYSTEMS=y
 CONFIG_NFS_FS=y
 CONFIG_NFS_V3=y
@@ -2144,7 +2195,6 @@ CONFIG_NFS_ACL_SUPPORT=y
 CONFIG_NFS_COMMON=y
 CONFIG_SUNRPC=y
 CONFIG_SUNRPC_GSS=y
-# CONFIG_SUNRPC_REGISTER_V4 is not set
 CONFIG_RPCSEC_GSS_KRB5=y
 # CONFIG_RPCSEC_GSS_SPKM3 is not set
 # CONFIG_SMB_FS is not set
@@ -2231,6 +2281,7 @@ CONFIG_DEBUG_FS=y
 CONFIG_DEBUG_KERNEL=y
 # CONFIG_DEBUG_SHIRQ is not set
 # CONFIG_DETECT_SOFTLOCKUP is not set
+# CONFIG_DETECT_HUNG_TASK is not set
 # CONFIG_SCHED_DEBUG is not set
 CONFIG_SCHEDSTATS=y
 CONFIG_TIMER_STATS=y
@@ -2246,6 +2297,7 @@ CONFIG_TIMER_STATS=y
 # CONFIG_LOCK_STAT is not set
 # CONFIG_DEBUG_SPINLOCK_SLEEP is not set
 # CONFIG_DEBUG_LOCKING_API_SELFTESTS is not set
+CONFIG_STACKTRACE=y
 # CONFIG_DEBUG_KOBJECT is not set
 CONFIG_DEBUG_BUGVERBOSE=y
 # CONFIG_DEBUG_INFO is not set
@@ -2268,13 +2320,19 @@ CONFIG_FRAME_POINTER=y
 # CONFIG_FAULT_INJECTION is not set
 # CONFIG_LATENCYTOP is not set
 CONFIG_SYSCTL_SYSCALL_CHECK=y
+# CONFIG_DEBUG_PAGEALLOC is not set
 CONFIG_USER_STACKTRACE_SUPPORT=y
+CONFIG_NOP_TRACER=y
 CONFIG_HAVE_FUNCTION_TRACER=y
 CONFIG_HAVE_FUNCTION_GRAPH_TRACER=y
 CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST=y
 CONFIG_HAVE_DYNAMIC_FTRACE=y
 CONFIG_HAVE_FTRACE_MCOUNT_RECORD=y
 CONFIG_HAVE_HW_BRANCH_TRACER=y
+CONFIG_HAVE_FTRACE_SYSCALLS=y
+CONFIG_RING_BUFFER=y
+CONFIG_TRACING=y
+CONFIG_TRACING_SUPPORT=y
 
 #
 # Tracers
@@ -2284,13 +2342,21 @@ CONFIG_HAVE_HW_BRANCH_TRACER=y
 # CONFIG_SYSPROF_TRACER is not set
 # CONFIG_SCHED_TRACER is not set
 # CONFIG_CONTEXT_SWITCH_TRACER is not set
+# CONFIG_EVENT_TRACER is not set
+# CONFIG_FTRACE_SYSCALLS is not set
 # CONFIG_BOOT_TRACER is not set
 # CONFIG_TRACE_BRANCH_PROFILING is not set
 # CONFIG_POWER_TRACER is not set
 # CONFIG_STACK_TRACER is not set
 # CONFIG_HW_BRANCH_TRACER is not set
+# CONFIG_KMEMTRACE is not set
+# CONFIG_WORKQUEUE_TRACER is not set
+CONFIG_BLK_DEV_IO_TRACE=y
+# CONFIG_FTRACE_STARTUP_TEST is not set
+# CONFIG_MMIOTRACE is not set
 CONFIG_PROVIDE_OHCI1394_DMA_INIT=y
-# CONFIG_DYNAMIC_PRINTK_DEBUG is not set
+# CONFIG_DYNAMIC_DEBUG is not set
+# CONFIG_DMA_API_DEBUG is not set
 # CONFIG_SAMPLES is not set
 CONFIG_HAVE_ARCH_KGDB=y
 # CONFIG_KGDB is not set
@@ -2300,14 +2366,13 @@ CONFIG_EARLY_PRINTK=y
 CONFIG_EARLY_PRINTK_DBGP=y
 CONFIG_DEBUG_STACKOVERFLOW=y
 CONFIG_DEBUG_STACK_USAGE=y
-# CONFIG_DEBUG_PAGEALLOC is not set
 # CONFIG_DEBUG_PER_CPU_MAPS is not set
 # CONFIG_X86_PTDUMP is not set
 CONFIG_DEBUG_RODATA=y
 # CONFIG_DEBUG_RODATA_TEST is not set
 CONFIG_DEBUG_NX_TEST=m
 # CONFIG_IOMMU_DEBUG is not set
-# CONFIG_MMIOTRACE is not set
+CONFIG_HAVE_MMIOTRACE_SUPPORT=y
 CONFIG_IO_DELAY_TYPE_0X80=0
 CONFIG_IO_DELAY_TYPE_0XED=1
 CONFIG_IO_DELAY_TYPE_UDELAY=2
@@ -2343,6 +2408,8 @@ CONFIG_SECURITY_SELINUX_AVC_STATS=y
 CONFIG_SECURITY_SELINUX_CHECKREQPROT_VALUE=1
 # CONFIG_SECURITY_SELINUX_POLICYDB_VERSION_MAX is not set
 # CONFIG_SECURITY_SMACK is not set
+# CONFIG_SECURITY_TOMOYO is not set
+# CONFIG_IMA is not set
 CONFIG_CRYPTO=y
 
 #
@@ -2358,10 +2425,12 @@ CONFIG_CRYPTO_BLKCIPHER2=y
 CONFIG_CRYPTO_HASH=y
 CONFIG_CRYPTO_HASH2=y
 CONFIG_CRYPTO_RNG2=y
+CONFIG_CRYPTO_PCOMP=y
 CONFIG_CRYPTO_MANAGER=y
 CONFIG_CRYPTO_MANAGER2=y
 # CONFIG_CRYPTO_GF128MUL is not set
 # CONFIG_CRYPTO_NULL is not set
+CONFIG_CRYPTO_WORKQUEUE=y
 # CONFIG_CRYPTO_CRYPTD is not set
 CONFIG_CRYPTO_AUTHENC=y
 # CONFIG_CRYPTO_TEST is not set
@@ -2413,6 +2482,7 @@ CONFIG_CRYPTO_SHA1=y
 #
 CONFIG_CRYPTO_AES=y
 # CONFIG_CRYPTO_AES_X86_64 is not set
+# CONFIG_CRYPTO_AES_NI_INTEL is not set
 # CONFIG_CRYPTO_ANUBIS is not set
 CONFIG_CRYPTO_ARC4=y
 # CONFIG_CRYPTO_BLOWFISH is not set
@@ -2434,6 +2504,7 @@ CONFIG_CRYPTO_DES=y
 # Compression
 #
 # CONFIG_CRYPTO_DEFLATE is not set
+# CONFIG_CRYPTO_ZLIB is not set
 # CONFIG_CRYPTO_LZO is not set
 
 #
@@ -2443,10 +2514,12 @@ CONFIG_CRYPTO_DES=y
 CONFIG_CRYPTO_HW=y
 # CONFIG_CRYPTO_DEV_HIFN_795X is not set
 CONFIG_HAVE_KVM=y
+CONFIG_HAVE_KVM_IRQCHIP=y
 CONFIG_VIRTUALIZATION=y
 # CONFIG_KVM is not set
 # CONFIG_VIRTIO_PCI is not set
 # CONFIG_VIRTIO_BALLOON is not set
+CONFIG_BINARY_PRINTF=y
 
 #
 # Library routines
@@ -2463,7 +2536,10 @@ CONFIG_CRC32=y
 # CONFIG_CRC7 is not set
 # CONFIG_LIBCRC32C is not set
 CONFIG_ZLIB_INFLATE=y
-CONFIG_PLIST=y
+CONFIG_DECOMPRESS_GZIP=y
+CONFIG_DECOMPRESS_BZIP2=y
+CONFIG_DECOMPRESS_LZMA=y
 CONFIG_HAS_IOMEM=y
 CONFIG_HAS_IOPORT=y
 CONFIG_HAS_DMA=y
+CONFIG_NLATTR=y
diff --git a/arch/x86/ia32/ia32entry.S b/arch/x86/ia32/ia32entry.S
index a505202086e8..e590261ba059 100644
--- a/arch/x86/ia32/ia32entry.S
+++ b/arch/x86/ia32/ia32entry.S
@@ -825,9 +825,11 @@ ia32_sys_call_table:
         .quad compat_sys_signalfd4
         .quad sys_eventfd2
         .quad sys_epoll_create1
-        .quad sys_dup3                  /* 330 */
+        .quad sys_dup3                          /* 330 */
         .quad sys_pipe2
         .quad sys_inotify_init1
         .quad compat_sys_preadv
         .quad compat_sys_pwritev
+        .quad compat_sys_rt_tgsigqueueinfo      /* 335 */
+        .quad sys_perf_counter_open
 ia32_syscall_end:
diff --git a/arch/x86/include/asm/alternative.h b/arch/x86/include/asm/alternative.h
index f6aa18eadf71..1a37bcdc8606 100644
--- a/arch/x86/include/asm/alternative.h
+++ b/arch/x86/include/asm/alternative.h
@@ -3,6 +3,7 @@
 
 #include <linux/types.h>
 #include <linux/stddef.h>
+#include <linux/stringify.h>
 #include <asm/asm.h>
 
 /*
@@ -74,6 +75,22 @@ static inline void alternatives_smp_switch(int smp) {}
 
 const unsigned char *const *find_nop_table(void);
 
+/* alternative assembly primitive: */
+#define ALTERNATIVE(oldinstr, newinstr, feature)                        \
+                                                                        \
+      "661:\n\t" oldinstr "\n662:\n"                                    \
+      ".section .altinstructions,\"a\"\n"                               \
+      _ASM_ALIGN "\n"                                                   \
+      _ASM_PTR "661b\n"                         /* label           */   \
+      _ASM_PTR "663f\n"                         /* new instruction */   \
+      "  .byte " __stringify(feature) "\n"      /* feature bit     */   \
+      "  .byte 662b-661b\n"                     /* sourcelen       */   \
+      "  .byte 664f-663f\n"                     /* replacementlen  */   \
+      ".previous\n"                                                     \
+      ".section .altinstr_replacement, \"ax\"\n"                        \
+      "663:\n\t" newinstr "\n664:\n"            /* replacement     */   \
+      ".previous"
+
 /*
  * Alternative instructions for different CPU types or capabilities.
  *
@@ -87,18 +104,7 @@ const unsigned char *const *find_nop_table(void);
  * without volatile and memory clobber.
  */
 #define alternative(oldinstr, newinstr, feature)                        \
-        asm volatile ("661:\n\t" oldinstr "\n662:\n"                    \
-                      ".section .altinstructions,\"a\"\n"               \
-                      _ASM_ALIGN "\n"                                   \
-                      _ASM_PTR "661b\n"         /* label */             \
-                      _ASM_PTR "663f\n"         /* new instruction */   \
-                      "  .byte %c0\n"           /* feature bit */       \
-                      "  .byte 662b-661b\n"     /* sourcelen */         \
-                      "  .byte 664f-663f\n"     /* replacementlen */    \
-                      ".previous\n"                                     \
-                      ".section .altinstr_replacement,\"ax\"\n"         \
-                      "663:\n\t" newinstr "\n664:\n"  /* replacement */ \
-                      ".previous" :: "i" (feature) : "memory")
+        asm volatile (ALTERNATIVE(oldinstr, newinstr, feature) : : : "memory")
 
 /*
  * Alternative inline assembly with input.
@@ -109,35 +115,16 @@ const unsigned char *const *find_nop_table(void);
  * Best is to use constraints that are fixed size (like (%1) ... "r")
  * If you use variable sized constraints like "m" or "g" in the
  * replacement make sure to pad to the worst case length.
+ * Leaving an unused argument 0 to keep API compatibility.
  */
 #define alternative_input(oldinstr, newinstr, feature, input...)        \
-        asm volatile ("661:\n\t" oldinstr "\n662:\n"                    \
-                      ".section .altinstructions,\"a\"\n"               \
-                      _ASM_ALIGN "\n"                                   \
-                      _ASM_PTR "661b\n"         /* label */             \
-                      _ASM_PTR "663f\n"         /* new instruction */   \
-                      "  .byte %c0\n"           /* feature bit */       \
-                      "  .byte 662b-661b\n"     /* sourcelen */         \
-                      "  .byte 664f-663f\n"     /* replacementlen */    \
-                      ".previous\n"                                     \
-                      ".section .altinstr_replacement,\"ax\"\n"         \
-                      "663:\n\t" newinstr "\n664:\n"  /* replacement */ \
-                      ".previous" :: "i" (feature), ##input)
+        asm volatile (ALTERNATIVE(oldinstr, newinstr, feature)          \
+                : : "i" (0), ## input)
 
 /* Like alternative_input, but with a single output argument */
 #define alternative_io(oldinstr, newinstr, feature, output, input...)   \
-        asm volatile ("661:\n\t" oldinstr "\n662:\n"                    \
-                      ".section .altinstructions,\"a\"\n"               \
-                      _ASM_ALIGN "\n"                                   \
-                      _ASM_PTR "661b\n"         /* label */             \
-                      _ASM_PTR "663f\n"         /* new instruction */   \
-                      "  .byte %c[feat]\n"      /* feature bit */       \
-                      "  .byte 662b-661b\n"     /* sourcelen */         \
-                      "  .byte 664f-663f\n"     /* replacementlen */    \
-                      ".previous\n"                                     \
-                      ".section .altinstr_replacement,\"ax\"\n"         \
-                      "663:\n\t" newinstr "\n664:\n"  /* replacement */ \
-                      ".previous" : output : [feat] "i" (feature), ##input)
+        asm volatile (ALTERNATIVE(oldinstr, newinstr, feature)          \
+                : output : "i" (0), ## input)
 
 /*
  * use this macro(s) if you need more than one output parameter
diff --git a/arch/x86/include/asm/amd_iommu.h b/arch/x86/include/asm/amd_iommu.h
index f712344329bc..262e02820049 100644
--- a/arch/x86/include/asm/amd_iommu.h
+++ b/arch/x86/include/asm/amd_iommu.h
@@ -27,6 +27,8 @@ extern int amd_iommu_init(void);
 extern int amd_iommu_init_dma_ops(void);
 extern void amd_iommu_detect(void);
 extern irqreturn_t amd_iommu_int_handler(int irq, void *data);
+extern void amd_iommu_flush_all_domains(void);
+extern void amd_iommu_flush_all_devices(void);
 #else
 static inline int amd_iommu_init(void) { return -ENODEV; }
 static inline void amd_iommu_detect(void) { }
diff --git a/arch/x86/include/asm/amd_iommu_types.h b/arch/x86/include/asm/amd_iommu_types.h
index 95c8cd9d22b5..0c878caaa0a2 100644
--- a/arch/x86/include/asm/amd_iommu_types.h
+++ b/arch/x86/include/asm/amd_iommu_types.h
@@ -194,6 +194,27 @@
 #define PD_DMA_OPS_MASK         (1UL << 0) /* domain used for dma_ops */
 #define PD_DEFAULT_MASK         (1UL << 1) /* domain is a default dma_ops
                                               domain for an IOMMU */
+extern bool amd_iommu_dump;
+#define DUMP_printk(format, arg...)                                     \
+        do {                                                            \
+                if (amd_iommu_dump)                                             \
+                        printk(KERN_INFO "AMD IOMMU: " format, ## arg); \
+        } while(0);
+
+/*
+ * Make iterating over all IOMMUs easier
+ */
+#define for_each_iommu(iommu) \
+        list_for_each_entry((iommu), &amd_iommu_list, list)
+#define for_each_iommu_safe(iommu, next) \
+        list_for_each_entry_safe((iommu), (next), &amd_iommu_list, list)
+
+#define APERTURE_RANGE_SHIFT    27      /* 128 MB */
+#define APERTURE_RANGE_SIZE     (1ULL << APERTURE_RANGE_SHIFT)
+#define APERTURE_RANGE_PAGES    (APERTURE_RANGE_SIZE >> PAGE_SHIFT)
+#define APERTURE_MAX_RANGES     32      /* allows 4GB of DMA address space */
+#define APERTURE_RANGE_INDEX(a) ((a) >> APERTURE_RANGE_SHIFT)
+#define APERTURE_PAGE_INDEX(a)  (((a) >> 21) & 0x3fULL)
 
 /*
  * This structure contains generic data for  IOMMU protection domains
@@ -210,6 +231,26 @@ struct protection_domain {
 };
 
 /*
+ * For dynamic growth the aperture size is split into ranges of 128MB of
+ * DMA address space each. This struct represents one such range.
+ */
+struct aperture_range {
+
+        /* address allocation bitmap */
+        unsigned long *bitmap;
+
+        /*
+         * Array of PTE pages for the aperture. In this array we save all the
+         * leaf pages of the domain page table used for the aperture. This way
+         * we don't need to walk the page table to find a specific PTE. We can
+         * just calculate its address in constant time.
+         */
+        u64 *pte_pages[64];
+
+        unsigned long offset;
+};
+
+/*
  * Data container for a dma_ops specific protection domain
  */
 struct dma_ops_domain {
@@ -222,18 +263,10 @@ struct dma_ops_domain {
         unsigned long aperture_size;
 
         /* address we start to search for free addresses */
-        unsigned long next_bit;
-
-        /* address allocation bitmap */
-        unsigned long *bitmap;
+        unsigned long next_address;
 
-        /*
-         * Array of PTE pages for the aperture. In this array we save all the
-         * leaf pages of the domain page table used for the aperture. This way
-         * we don't need to walk the page table to find a specific PTE. We can
-         * just calculate its address in constant time.
-         */
-        u64 **pte_pages;
+        /* address space relevant data */
+        struct aperture_range *aperture[APERTURE_MAX_RANGES];
 
         /* This will be set to true when TLB needs to be flushed */
         bool need_flush;
diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h
index 3738438a91f5..bb7d47925847 100644
--- a/arch/x86/include/asm/apic.h
+++ b/arch/x86/include/asm/apic.h
@@ -402,7 +402,7 @@ static inline unsigned default_get_apic_id(unsigned long x)
 {
         unsigned int ver = GET_APIC_VERSION(apic_read(APIC_LVR));
 
-        if (APIC_XAPIC(ver))
+        if (APIC_XAPIC(ver) || boot_cpu_has(X86_FEATURE_EXTD_APICID))
                 return (x >> 24) & 0xFF;
         else
                 return (x >> 24) & 0x0F;
@@ -470,6 +470,9 @@ static inline unsigned int read_apic_id(void)
 extern void default_setup_apic_routing(void);
 
 #ifdef CONFIG_X86_32
+
+extern struct apic apic_default;
+
 /*
  * Set up the logical destination ID.
  *
diff --git a/arch/x86/include/asm/atomic_32.h b/arch/x86/include/asm/atomic_32.h
index 85b46fba4229..aff9f1fcdcd7 100644
--- a/arch/x86/include/asm/atomic_32.h
+++ b/arch/x86/include/asm/atomic_32.h
@@ -247,5 +247,241 @@ static inline int atomic_add_unless(atomic_t *v, int a, int u)
 #define smp_mb__before_atomic_inc()     barrier()
 #define smp_mb__after_atomic_inc()      barrier()
 
+/* An 64bit atomic type */
+
+typedef struct {
+        unsigned long long counter;
+} atomic64_t;
+
+#define ATOMIC64_INIT(val)      { (val) }
+
+/**
+ * atomic64_read - read atomic64 variable
+ * @v: pointer of type atomic64_t
+ *
+ * Atomically reads the value of @v.
+ * Doesn't imply a read memory barrier.
+ */
+#define __atomic64_read(ptr)            ((ptr)->counter)
+
+static inline unsigned long long
+cmpxchg8b(unsigned long long *ptr, unsigned long long old, unsigned long long new)
+{
+        asm volatile(
+
+                LOCK_PREFIX "cmpxchg8b (%[ptr])\n"
+
+                     :          "=A" (old)
+
+                     : [ptr]    "D" (ptr),
+                                "A" (old),
+                                "b" (ll_low(new)),
+                                "c" (ll_high(new))
+
+                     : "memory");
+
+        return old;
+}
+
+static inline unsigned long long
+atomic64_cmpxchg(atomic64_t *ptr, unsigned long long old_val,
+                 unsigned long long new_val)
+{
+        return cmpxchg8b(&ptr->counter, old_val, new_val);
+}
+
+/**
+ * atomic64_xchg - xchg atomic64 variable
+ * @ptr:      pointer to type atomic64_t
+ * @new_val:  value to assign
+ * @old_val:  old value that was there
+ *
+ * Atomically xchgs the value of @ptr to @new_val and returns
+ * the old value.
+ */
+
+static inline unsigned long long
+atomic64_xchg(atomic64_t *ptr, unsigned long long new_val)
+{
+        unsigned long long old_val;
+
+        do {
+                old_val = atomic_read(ptr);
+        } while (atomic64_cmpxchg(ptr, old_val, new_val) != old_val);
+
+        return old_val;
+}
+
+/**
+ * atomic64_set - set atomic64 variable
+ * @ptr:      pointer to type atomic64_t
+ * @new_val:  value to assign
+ *
+ * Atomically sets the value of @ptr to @new_val.
+ */
+static inline void atomic64_set(atomic64_t *ptr, unsigned long long new_val)
+{
+        atomic64_xchg(ptr, new_val);
+}
+
+/**
+ * atomic64_read - read atomic64 variable
+ * @ptr:      pointer to type atomic64_t
+ *
+ * Atomically reads the value of @ptr and returns it.
+ */
+static inline unsigned long long atomic64_read(atomic64_t *ptr)
+{
+        unsigned long long curr_val;
+
+        do {
+                curr_val = __atomic64_read(ptr);
+        } while (atomic64_cmpxchg(ptr, curr_val, curr_val) != curr_val);
+
+        return curr_val;
+}
+
+/**
+ * atomic64_add_return - add and return
+ * @delta: integer value to add
+ * @ptr:   pointer to type atomic64_t
+ *
+ * Atomically adds @delta to @ptr and returns @delta + *@ptr
+ */
+static inline unsigned long long
+atomic64_add_return(unsigned long long delta, atomic64_t *ptr)
+{
+        unsigned long long old_val, new_val;
+
+        do {
+                old_val = atomic_read(ptr);
+                new_val = old_val + delta;
+
+        } while (atomic64_cmpxchg(ptr, old_val, new_val) != old_val);
+
+        return new_val;
+}
+
+static inline long atomic64_sub_return(unsigned long long delta, atomic64_t *ptr)
+{
+        return atomic64_add_return(-delta, ptr);
+}
+
+static inline long atomic64_inc_return(atomic64_t *ptr)
+{
+        return atomic64_add_return(1, ptr);
+}
+
+static inline long atomic64_dec_return(atomic64_t *ptr)
+{
+        return atomic64_sub_return(1, ptr);
+}
+
+/**
+ * atomic64_add - add integer to atomic64 variable
+ * @delta: integer value to add
+ * @ptr:   pointer to type atomic64_t
+ *
+ * Atomically adds @delta to @ptr.
+ */
+static inline void atomic64_add(unsigned long long delta, atomic64_t *ptr)
+{
+        atomic64_add_return(delta, ptr);
+}
+
+/**
+ * atomic64_sub - subtract the atomic64 variable
+ * @delta: integer value to subtract
+ * @ptr:   pointer to type atomic64_t
+ *
+ * Atomically subtracts @delta from @ptr.
+ */
+static inline void atomic64_sub(unsigned long long delta, atomic64_t *ptr)
+{
+        atomic64_add(-delta, ptr);
+}
+
+/**
+ * atomic64_sub_and_test - subtract value from variable and test result
+ * @delta: integer value to subtract
+ * @ptr:   pointer to type atomic64_t
+ *
+ * Atomically subtracts @delta from @ptr and returns
+ * true if the result is zero, or false for all
+ * other cases.
+ */
+static inline int
+atomic64_sub_and_test(unsigned long long delta, atomic64_t *ptr)
+{
+        unsigned long long old_val = atomic64_sub_return(delta, ptr);
+
+        return old_val == 0;
+}
+
+/**
+ * atomic64_inc - increment atomic64 variable
+ * @ptr: pointer to type atomic64_t
+ *
+ * Atomically increments @ptr by 1.
+ */
+static inline void atomic64_inc(atomic64_t *ptr)
+{
+        atomic64_add(1, ptr);
+}
+
+/**
+ * atomic64_dec - decrement atomic64 variable
+ * @ptr: pointer to type atomic64_t
+ *
+ * Atomically decrements @ptr by 1.
+ */
+static inline void atomic64_dec(atomic64_t *ptr)
+{
+        atomic64_sub(1, ptr);
+}
+
+/**
+ * atomic64_dec_and_test - decrement and test
+ * @ptr: pointer to type atomic64_t
+ *
+ * Atomically decrements @ptr by 1 and
+ * returns true if the result is 0, or false for all other
+ * cases.
+ */
+static inline int atomic64_dec_and_test(atomic64_t *ptr)
+{
+        return atomic64_sub_and_test(1, ptr);
+}
+
+/**
+ * atomic64_inc_and_test - increment and test
+ * @ptr: pointer to type atomic64_t
+ *
+ * Atomically increments @ptr by 1
+ * and returns true if the result is zero, or false for all
+ * other cases.
+ */
+static inline int atomic64_inc_and_test(atomic64_t *ptr)
+{
+        return atomic64_sub_and_test(-1, ptr);
+}
+
+/**
+ * atomic64_add_negative - add and test if negative
+ * @delta: integer value to add
+ * @ptr:   pointer to type atomic64_t
+ *
+ * Atomically adds @delta to @ptr and returns true
+ * if the result is negative, or false when
+ * result is greater than or equal to zero.
+ */
+static inline int
+atomic64_add_negative(unsigned long long delta, atomic64_t *ptr)
+{
+        long long old_val = atomic64_add_return(delta, ptr);
+
+        return old_val < 0;
+}
+
 #include <asm-generic/atomic.h>
 #endif /* _ASM_X86_ATOMIC_32_H */
diff --git a/arch/x86/include/asm/boot.h b/arch/x86/include/asm/boot.h
index 6ba23dd9fc92..418e632d4a80 100644
--- a/arch/x86/include/asm/boot.h
+++ b/arch/x86/include/asm/boot.h
@@ -8,11 +8,26 @@
 
 #ifdef __KERNEL__
 
+#include <asm/page_types.h>
+
 /* Physical address where kernel should be loaded. */
 #define LOAD_PHYSICAL_ADDR ((CONFIG_PHYSICAL_START \
                                 + (CONFIG_PHYSICAL_ALIGN - 1)) \
                                 & ~(CONFIG_PHYSICAL_ALIGN - 1))
 
+/* Minimum kernel alignment, as a power of two */
+#ifdef CONFIG_x86_64
+#define MIN_KERNEL_ALIGN_LG2    PMD_SHIFT
+#else
+#define MIN_KERNEL_ALIGN_LG2    (PAGE_SHIFT+1)
+#endif
+#define MIN_KERNEL_ALIGN        (_AC(1, UL) << MIN_KERNEL_ALIGN_LG2)
+
+#if (CONFIG_PHYSICAL_ALIGN & (CONFIG_PHYSICAL_ALIGN-1)) || \
+        (CONFIG_PHYSICAL_ALIGN < (_AC(1, UL) << MIN_KERNEL_ALIGN_LG2))
+#error "Invalid value for CONFIG_PHYSICAL_ALIGN"
+#endif
+
 #ifdef CONFIG_KERNEL_BZIP2
 #define BOOT_HEAP_SIZE             0x400000
 #else /* !CONFIG_KERNEL_BZIP2 */
diff --git a/arch/x86/include/asm/bootparam.h b/arch/x86/include/asm/bootparam.h
index 433adaebf9b6..1724e8de317c 100644
--- a/arch/x86/include/asm/bootparam.h
+++ b/arch/x86/include/asm/bootparam.h
@@ -50,7 +50,8 @@ struct setup_header {
         __u32   ramdisk_size;
         __u32   bootsect_kludge;
         __u16   heap_end_ptr;
-        __u16   _pad1;
+        __u8    ext_loader_ver;
+        __u8    ext_loader_type;
         __u32   cmd_line_ptr;
         __u32   initrd_addr_max;
         __u32   kernel_alignment;
diff --git a/arch/x86/include/asm/cpu_debug.h b/arch/x86/include/asm/cpu_debug.h
index 222802029fa6..d96c1ee3a95c 100644
--- a/arch/x86/include/asm/cpu_debug.h
+++ b/arch/x86/include/asm/cpu_debug.h
@@ -86,105 +86,7 @@ enum cpu_file_bit {
         CPU_VALUE_BIT,                          /* value                */
 };
 
-#define CPU_FILE_VALUE                  (1 << CPU_VALUE_BIT)
-
-/*
- * DisplayFamily_DisplayModel   Processor Families/Processor Number Series
- * --------------------------   ------------------------------------------
- * 05_01, 05_02, 05_04          Pentium, Pentium with MMX
- *
- * 06_01                        Pentium Pro
- * 06_03, 06_05                 Pentium II Xeon, Pentium II
- * 06_07, 06_08, 06_0A, 06_0B   Pentium III Xeon, Pentum III
- *
- * 06_09, 060D                  Pentium M
- *
- * 06_0E                        Core Duo, Core Solo
- *
- * 06_0F                        Xeon 3000, 3200, 5100, 5300, 7300 series,
- *                              Core 2 Quad, Core 2 Extreme, Core 2 Duo,
- *                              Pentium dual-core
- * 06_17                        Xeon 5200, 5400 series, Core 2 Quad Q9650
- *
- * 06_1C                        Atom
- *
- * 0F_00, 0F_01, 0F_02          Xeon, Xeon MP, Pentium 4
- * 0F_03, 0F_04                 Xeon, Xeon MP, Pentium 4, Pentium D
- *
- * 0F_06                        Xeon 7100, 5000 Series, Xeon MP,
- *                              Pentium 4, Pentium D
- */
-
-/* Register processors bits */
-enum cpu_processor_bit {
-        CPU_NONE,
-/* Intel */
-        CPU_INTEL_PENTIUM_BIT,
-        CPU_INTEL_P6_BIT,
-        CPU_INTEL_PENTIUM_M_BIT,
-        CPU_INTEL_CORE_BIT,
-        CPU_INTEL_CORE2_BIT,
-        CPU_INTEL_ATOM_BIT,
-        CPU_INTEL_XEON_P4_BIT,
-        CPU_INTEL_XEON_MP_BIT,
-/* AMD */
-        CPU_AMD_K6_BIT,
-        CPU_AMD_K7_BIT,
-        CPU_AMD_K8_BIT,
-        CPU_AMD_0F_BIT,
-        CPU_AMD_10_BIT,
-        CPU_AMD_11_BIT,
-};
-
-#define CPU_INTEL_PENTIUM       (1 << CPU_INTEL_PENTIUM_BIT)
-#define CPU_INTEL_P6            (1 << CPU_INTEL_P6_BIT)
-#define CPU_INTEL_PENTIUM_M     (1 << CPU_INTEL_PENTIUM_M_BIT)
-#define CPU_INTEL_CORE          (1 << CPU_INTEL_CORE_BIT)
-#define CPU_INTEL_CORE2         (1 << CPU_INTEL_CORE2_BIT)
-#define CPU_INTEL_ATOM          (1 << CPU_INTEL_ATOM_BIT)
-#define CPU_INTEL_XEON_P4       (1 << CPU_INTEL_XEON_P4_BIT)
-#define CPU_INTEL_XEON_MP       (1 << CPU_INTEL_XEON_MP_BIT)
-
-#define CPU_INTEL_PX            (CPU_INTEL_P6 | CPU_INTEL_PENTIUM_M)
-#define CPU_INTEL_COREX         (CPU_INTEL_CORE | CPU_INTEL_CORE2)
-#define CPU_INTEL_XEON          (CPU_INTEL_XEON_P4 | CPU_INTEL_XEON_MP)
-#define CPU_CO_AT               (CPU_INTEL_CORE | CPU_INTEL_ATOM)
-#define CPU_C2_AT               (CPU_INTEL_CORE2 | CPU_INTEL_ATOM)
-#define CPU_CX_AT               (CPU_INTEL_COREX | CPU_INTEL_ATOM)
-#define CPU_CX_XE               (CPU_INTEL_COREX | CPU_INTEL_XEON)
-#define CPU_P6_XE               (CPU_INTEL_P6 | CPU_INTEL_XEON)
-#define CPU_PM_CO_AT            (CPU_INTEL_PENTIUM_M | CPU_CO_AT)
-#define CPU_C2_AT_XE            (CPU_C2_AT | CPU_INTEL_XEON)
-#define CPU_CX_AT_XE            (CPU_CX_AT | CPU_INTEL_XEON)
-#define CPU_P6_CX_AT            (CPU_INTEL_P6 | CPU_CX_AT)
-#define CPU_P6_CX_XE            (CPU_P6_XE | CPU_INTEL_COREX)
-#define CPU_P6_CX_AT_XE         (CPU_INTEL_P6 | CPU_CX_AT_XE)
-#define CPU_PM_CX_AT_XE         (CPU_INTEL_PENTIUM_M | CPU_CX_AT_XE)
-#define CPU_PM_CX_AT            (CPU_INTEL_PENTIUM_M | CPU_CX_AT)
-#define CPU_PM_CX_XE            (CPU_INTEL_PENTIUM_M | CPU_CX_XE)
-#define CPU_PX_CX_AT            (CPU_INTEL_PX | CPU_CX_AT)
-#define CPU_PX_CX_AT_XE         (CPU_INTEL_PX | CPU_CX_AT_XE)
-
-/* Select all supported Intel CPUs */
-#define CPU_INTEL_ALL           (CPU_INTEL_PENTIUM | CPU_PX_CX_AT_XE)
-
-#define CPU_AMD_K6              (1 << CPU_AMD_K6_BIT)
-#define CPU_AMD_K7              (1 << CPU_AMD_K7_BIT)
-#define CPU_AMD_K8              (1 << CPU_AMD_K8_BIT)
-#define CPU_AMD_0F              (1 << CPU_AMD_0F_BIT)
-#define CPU_AMD_10              (1 << CPU_AMD_10_BIT)
-#define CPU_AMD_11              (1 << CPU_AMD_11_BIT)
-
-#define CPU_K10_PLUS            (CPU_AMD_10 | CPU_AMD_11)
-#define CPU_K0F_PLUS            (CPU_AMD_0F | CPU_K10_PLUS)
-#define CPU_K8_PLUS             (CPU_AMD_K8 | CPU_K0F_PLUS)
-#define CPU_K7_PLUS             (CPU_AMD_K7 | CPU_K8_PLUS)
-
-/* Select all supported AMD CPUs */
-#define CPU_AMD_ALL             (CPU_AMD_K6 | CPU_K7_PLUS)
-
-/* Select all supported CPUs */
-#define CPU_ALL                 (CPU_INTEL_ALL | CPU_AMD_ALL)
+#define CPU_FILE_VALUE          (1 << CPU_VALUE_BIT)
 
 #define MAX_CPU_FILES           512
 
@@ -220,7 +122,6 @@ struct cpu_debug_range {
         unsigned                min;            /* Register range min   */
         unsigned                max;            /* Register range max   */
         unsigned                flag;           /* Supported flags      */
-        unsigned                model;          /* Supported models     */
 };
 
 #endif /* _ASM_X86_CPU_DEBUG_H */
diff --git a/arch/x86/include/asm/cpufeature.h b/arch/x86/include/asm/cpufeature.h
index 13cc6a503a02..4a28d22d4793 100644
--- a/arch/x86/include/asm/cpufeature.h
+++ b/arch/x86/include/asm/cpufeature.h
@@ -94,6 +94,7 @@
 #define X86_FEATURE_TSC_RELIABLE (3*32+23) /* TSC is known to be reliable */
 #define X86_FEATURE_NONSTOP_TSC (3*32+24) /* TSC does not stop in C states */
 #define X86_FEATURE_CLFLUSH_MONITOR (3*32+25) /* "" clflush reqd with monitor */
+#define X86_FEATURE_EXTD_APICID (3*32+26) /* has extended APICID (8 bits) */
 
 /* Intel-defined CPU features, CPUID level 0x00000001 (ecx), word 4 */
 #define X86_FEATURE_XMM3        (4*32+ 0) /* "pni" SSE-3 */
@@ -115,6 +116,8 @@
 #define X86_FEATURE_XMM4_1      (4*32+19) /* "sse4_1" SSE-4.1 */
 #define X86_FEATURE_XMM4_2      (4*32+20) /* "sse4_2" SSE-4.2 */
 #define X86_FEATURE_X2APIC      (4*32+21) /* x2APIC */
+#define X86_FEATURE_MOVBE       (4*32+22) /* MOVBE instruction */
+#define X86_FEATURE_POPCNT      (4*32+23) /* POPCNT instruction */
 #define X86_FEATURE_AES         (4*32+25) /* AES instructions */
 #define X86_FEATURE_XSAVE       (4*32+26) /* XSAVE/XRSTOR/XSETBV/XGETBV */
 #define X86_FEATURE_OSXSAVE     (4*32+27) /* "" XSAVE enabled in the OS */
diff --git a/arch/x86/include/asm/ds.h b/arch/x86/include/asm/ds.h
index a8f672ba100c..70dac199b093 100644
--- a/arch/x86/include/asm/ds.h
+++ b/arch/x86/include/asm/ds.h
@@ -15,8 +15,8 @@
  * - buffer allocation (memory accounting)
  *
  *
- * Copyright (C) 2007-2008 Intel Corporation.
- * Markus Metzger <markus.t.metzger@intel.com>, 2007-2008
+ * Copyright (C) 2007-2009 Intel Corporation.
+ * Markus Metzger <markus.t.metzger@intel.com>, 2007-2009
  */
 
 #ifndef _ASM_X86_DS_H
@@ -83,8 +83,10 @@ enum ds_feature {
  * The interrupt threshold is independent from the overflow callback
  * to allow users to use their own overflow interrupt handling mechanism.
  *
- * task: the task to request recording for;
- *       NULL for per-cpu recording on the current cpu
+ * The function might sleep.
+ *
+ * task: the task to request recording for
+ * cpu:  the cpu to request recording for
  * base: the base pointer for the (non-pageable) buffer;
  * size: the size of the provided buffer in bytes
  * ovfl: pointer to a function to be called on buffer overflow;
@@ -93,19 +95,28 @@ enum ds_feature {
  *     -1 if no interrupt threshold is requested.
  * flags: a bit-mask of the above flags
  */
-extern struct bts_tracer *ds_request_bts(struct task_struct *task,
-                                         void *base, size_t size,
-                                         bts_ovfl_callback_t ovfl,
-                                         size_t th, unsigned int flags);
-extern struct pebs_tracer *ds_request_pebs(struct task_struct *task,
-                                           void *base, size_t size,
-                                           pebs_ovfl_callback_t ovfl,
-                                           size_t th, unsigned int flags);
+extern struct bts_tracer *ds_request_bts_task(struct task_struct *task,
+                                              void *base, size_t size,
+                                              bts_ovfl_callback_t ovfl,
+                                              size_t th, unsigned int flags);
+extern struct bts_tracer *ds_request_bts_cpu(int cpu, void *base, size_t size,
+                                             bts_ovfl_callback_t ovfl,
+                                             size_t th, unsigned int flags);
+extern struct pebs_tracer *ds_request_pebs_task(struct task_struct *task,
+                                                void *base, size_t size,
+                                                pebs_ovfl_callback_t ovfl,
+                                                size_t th, unsigned int flags);
+extern struct pebs_tracer *ds_request_pebs_cpu(int cpu,
+                                               void *base, size_t size,
+                                               pebs_ovfl_callback_t ovfl,
+                                               size_t th, unsigned int flags);
 
 /*
  * Release BTS or PEBS resources
  * Suspend and resume BTS or PEBS tracing
  *
+ * Must be called with irq's enabled.
+ *
  * tracer: the tracer handle returned from ds_request_~()
  */
 extern void ds_release_bts(struct bts_tracer *tracer);
@@ -115,6 +126,28 @@ extern void ds_release_pebs(struct pebs_tracer *tracer);
 extern void ds_suspend_pebs(struct pebs_tracer *tracer);
 extern void ds_resume_pebs(struct pebs_tracer *tracer);
 
+/*
+ * Release BTS or PEBS resources
+ * Suspend and resume BTS or PEBS tracing
+ *
+ * Cpu tracers must call this on the traced cpu.
+ * Task tracers must call ds_release_~_noirq() for themselves.
+ *
+ * May be called with irq's disabled.
+ *
+ * Returns 0 if successful;
+ * -EPERM if the cpu tracer does not trace the current cpu.
+ * -EPERM if the task tracer does not trace itself.
+ *
+ * tracer: the tracer handle returned from ds_request_~()
+ */
+extern int ds_release_bts_noirq(struct bts_tracer *tracer);
+extern int ds_suspend_bts_noirq(struct bts_tracer *tracer);
+extern int ds_resume_bts_noirq(struct bts_tracer *tracer);
+extern int ds_release_pebs_noirq(struct pebs_tracer *tracer);
+extern int ds_suspend_pebs_noirq(struct pebs_tracer *tracer);
+extern int ds_resume_pebs_noirq(struct pebs_tracer *tracer);
+
 
 /*
  * The raw DS buffer state as it is used for BTS and PEBS recording.
@@ -170,9 +203,9 @@ struct bts_struct {
                 } lbr;
                 /* BTS_TASK_ARRIVES or BTS_TASK_DEPARTS */
                 struct {
-                        __u64 jiffies;
+                        __u64 clock;
                         pid_t pid;
-                } timestamp;
+                } event;
         } variant;
 };
 
@@ -201,8 +234,12 @@ struct bts_trace {
 struct pebs_trace {
         struct ds_trace ds;
 
-        /* the PEBS reset value */
-        unsigned long long reset_value;
+        /* the number of valid counters in the below array */
+        unsigned int counters;
+
+#define MAX_PEBS_COUNTERS 4
+        /* the counter reset value */
+        unsigned long long counter_reset[MAX_PEBS_COUNTERS];
 };
 
 
@@ -237,9 +274,11 @@ extern int ds_reset_pebs(struct pebs_tracer *tracer);
  * Returns 0 on success; -Eerrno on error
  *
  * tracer: the tracer handle returned from ds_request_pebs()
+ * counter: the index of the counter
  * value: the new counter reset value
  */
-extern int ds_set_pebs_reset(struct pebs_tracer *tracer, u64 value);
+extern int ds_set_pebs_reset(struct pebs_tracer *tracer,
+                             unsigned int counter, u64 value);
 
 /*
  * Initialization
@@ -252,21 +291,12 @@ extern void __cpuinit ds_init_intel(struct cpuinfo_x86 *);
  */
 extern void ds_switch_to(struct task_struct *prev, struct task_struct *next);
 
-/*
- * Task clone/init and cleanup work
- */
-extern void ds_copy_thread(struct task_struct *tsk, struct task_struct *father);
-extern void ds_exit_thread(struct task_struct *tsk);
-
 #else /* CONFIG_X86_DS */
 
 struct cpuinfo_x86;
 static inline void __cpuinit ds_init_intel(struct cpuinfo_x86 *ignored) {}
 static inline void ds_switch_to(struct task_struct *prev,
                                 struct task_struct *next) {}
-static inline void ds_copy_thread(struct task_struct *tsk,
-                                  struct task_struct *father) {}
-static inline void ds_exit_thread(struct task_struct *tsk) {}
 
 #endif /* CONFIG_X86_DS */
 #endif /* _ASM_X86_DS_H */
diff --git a/arch/x86/include/asm/entry_arch.h b/arch/x86/include/asm/entry_arch.h
index 69f886805ecb..ff8cbfa07851 100644
--- a/arch/x86/include/asm/entry_arch.h
+++ b/arch/x86/include/asm/entry_arch.h
@@ -50,7 +50,7 @@ BUILD_INTERRUPT(error_interrupt,ERROR_APIC_VECTOR)
 BUILD_INTERRUPT(spurious_interrupt,SPURIOUS_APIC_VECTOR)
 
 #ifdef CONFIG_PERF_COUNTERS
-BUILD_INTERRUPT(perf_counter_interrupt, LOCAL_PERF_VECTOR)
+BUILD_INTERRUPT(perf_pending_interrupt, LOCAL_PENDING_VECTOR)
 #endif
 
 #ifdef CONFIG_X86_THERMAL_VECTOR
diff --git a/arch/x86/include/asm/hardirq.h b/arch/x86/include/asm/hardirq.h
index 922ee7c29693..82e3e8f01043 100644
--- a/arch/x86/include/asm/hardirq.h
+++ b/arch/x86/include/asm/hardirq.h
@@ -13,6 +13,8 @@ typedef struct {
         unsigned int irq_spurious_count;
 #endif
         unsigned int generic_irqs;      /* arch dependent */
+        unsigned int apic_perf_irqs;
+        unsigned int apic_pending_irqs;
 #ifdef CONFIG_SMP
         unsigned int irq_resched_count;
         unsigned int irq_call_count;
diff --git a/arch/x86/include/asm/hw_irq.h b/arch/x86/include/asm/hw_irq.h
index 1c8f28a63058..ba180d93b08c 100644
--- a/arch/x86/include/asm/hw_irq.h
+++ b/arch/x86/include/asm/hw_irq.h
@@ -29,6 +29,8 @@
 extern void apic_timer_interrupt(void);
 extern void generic_interrupt(void);
 extern void error_interrupt(void);
+extern void perf_pending_interrupt(void);
+
 extern void spurious_interrupt(void);
 extern void thermal_interrupt(void);
 extern void reschedule_interrupt(void);
@@ -99,7 +101,11 @@ extern void eisa_set_level_irq(unsigned int irq);
 /* SMP */
 extern void smp_apic_timer_interrupt(struct pt_regs *);
 extern void smp_spurious_interrupt(struct pt_regs *);
+extern void smp_generic_interrupt(struct pt_regs *);
 extern void smp_error_interrupt(struct pt_regs *);
+#ifdef CONFIG_X86_IO_APIC
+extern asmlinkage void smp_irq_move_cleanup_interrupt(void);
+#endif
 #ifdef CONFIG_SMP
 extern void smp_reschedule_interrupt(struct pt_regs *);
 extern void smp_call_function_interrupt(struct pt_regs *);
diff --git a/arch/x86/include/asm/i387.h b/arch/x86/include/asm/i387.h
index 71c9e5183982..175adf58dd4f 100644
--- a/arch/x86/include/asm/i387.h
+++ b/arch/x86/include/asm/i387.h
@@ -67,7 +67,7 @@ static inline int fxrstor_checking(struct i387_fxsave_struct *fx)
                      ".previous\n"
                      _ASM_EXTABLE(1b, 3b)
                      : [err] "=r" (err)
-#if 0 /* See comment in __save_init_fpu() below. */
+#if 0 /* See comment in fxsave() below. */
                      : [fx] "r" (fx), "m" (*fx), "0" (0));
 #else
                      : [fx] "cdaSDb" (fx), "m" (*fx), "0" (0));
@@ -75,14 +75,6 @@ static inline int fxrstor_checking(struct i387_fxsave_struct *fx)
         return err;
 }
 
-static inline int restore_fpu_checking(struct task_struct *tsk)
-{
-        if (task_thread_info(tsk)->status & TS_XSAVE)
-                return xrstor_checking(&tsk->thread.xstate->xsave);
-        else
-                return fxrstor_checking(&tsk->thread.xstate->fxsave);
-}
-
 /* AMD CPUs don't save/restore FDP/FIP/FOP unless an exception
    is pending. Clear the x87 state here by setting it to fixed
    values. The kernel data segment can be sometimes 0 and sometimes
@@ -120,7 +112,7 @@ static inline int fxsave_user(struct i387_fxsave_struct __user *fx)
                      ".previous\n"
                      _ASM_EXTABLE(1b, 3b)
                      : [err] "=r" (err), "=m" (*fx)
-#if 0 /* See comment in __fxsave_clear() below. */
+#if 0 /* See comment in fxsave() below. */
                      : [fx] "r" (fx), "0" (0));
 #else
                      : [fx] "cdaSDb" (fx), "0" (0));
@@ -185,12 +177,9 @@ static inline void tolerant_fwait(void)
         asm volatile("fnclex ; fwait");
 }
 
-static inline void restore_fpu(struct task_struct *tsk)
+/* perform fxrstor iff the processor has extended states, otherwise frstor */
+static inline int fxrstor_checking(struct i387_fxsave_struct *fx)
 {
-        if (task_thread_info(tsk)->status & TS_XSAVE) {
-                xrstor_checking(&tsk->thread.xstate->xsave);
-                return;
-        }
         /*
          * The "nop" is needed to make the instructions the same
          * length.
@@ -199,7 +188,9 @@ static inline void restore_fpu(struct task_struct *tsk)
                 "nop ; frstor %1",
                 "fxrstor %1",
                 X86_FEATURE_FXSR,
-                "m" (tsk->thread.xstate->fxsave));
+                "m" (*fx));
+
+        return 0;
 }
 
 /* We need a safe address that is cheap to find and that is already
@@ -262,6 +253,14 @@ end:
 
 #endif  /* CONFIG_X86_64 */
 
+static inline int restore_fpu_checking(struct task_struct *tsk)
+{
+        if (task_thread_info(tsk)->status & TS_XSAVE)
+                return xrstor_checking(&tsk->thread.xstate->xsave);
+        else
+                return fxrstor_checking(&tsk->thread.xstate->fxsave);
+}
+
 /*
  * Signal frame handlers...
  */
@@ -305,18 +304,18 @@ static inline void kernel_fpu_end(void)
 /*
  * Some instructions like VIA's padlock instructions generate a spurious
  * DNA fault but don't modify SSE registers. And these instructions
- * get used from interrupt context aswell. To prevent these kernel instructions
- * in interrupt context interact wrongly with other user/kernel fpu usage, we
+ * get used from interrupt context as well. To prevent these kernel instructions
+ * in interrupt context interacting wrongly with other user/kernel fpu usage, we
  * should use them only in the context of irq_ts_save/restore()
  */
 static inline int irq_ts_save(void)
 {
         /*
-         * If we are in process context, we are ok to take a spurious DNA fault.
-         * Otherwise, doing clts() in process context require pre-emption to
-         * be disabled or some heavy lifting like kernel_fpu_begin()
+         * If in process context and not atomic, we can take a spurious DNA fault.
+         * Otherwise, doing clts() in process context requires disabling preemption
+         * or some heavy lifting like kernel_fpu_begin()
          */
-        if (!in_interrupt())
+        if (!in_atomic())
                 return 0;
 
         if (read_cr0() & X86_CR0_TS) {
diff --git a/arch/x86/include/asm/intel_arch_perfmon.h b/arch/x86/include/asm/intel_arch_perfmon.h
deleted file mode 100644
index fa0fd068bc2e..000000000000
--- a/arch/x86/include/asm/intel_arch_perfmon.h
+++ /dev/null
@@ -1,31 +0,0 @@
-#ifndef _ASM_X86_INTEL_ARCH_PERFMON_H
-#define _ASM_X86_INTEL_ARCH_PERFMON_H
-
-#define MSR_ARCH_PERFMON_PERFCTR0               0xc1
-#define MSR_ARCH_PERFMON_PERFCTR1               0xc2
-
-#define MSR_ARCH_PERFMON_EVENTSEL0              0x186
-#define MSR_ARCH_PERFMON_EVENTSEL1              0x187
-
-#define ARCH_PERFMON_EVENTSEL0_ENABLE   (1 << 22)
-#define ARCH_PERFMON_EVENTSEL_INT       (1 << 20)
-#define ARCH_PERFMON_EVENTSEL_OS        (1 << 17)
-#define ARCH_PERFMON_EVENTSEL_USR       (1 << 16)
-
-#define ARCH_PERFMON_UNHALTED_CORE_CYCLES_SEL   (0x3c)
-#define ARCH_PERFMON_UNHALTED_CORE_CYCLES_UMASK (0x00 << 8)
-#define ARCH_PERFMON_UNHALTED_CORE_CYCLES_INDEX (0)
-#define ARCH_PERFMON_UNHALTED_CORE_CYCLES_PRESENT \
-        (1 << (ARCH_PERFMON_UNHALTED_CORE_CYCLES_INDEX))
-
-union cpuid10_eax {
-        struct {
-                unsigned int version_id:8;
-                unsigned int num_counters:8;
-                unsigned int bit_width:8;
-                unsigned int mask_length:8;
-        } split;
-        unsigned int full;
-};
-
-#endif /* _ASM_X86_INTEL_ARCH_PERFMON_H */
diff --git a/arch/x86/include/asm/iomap.h b/arch/x86/include/asm/iomap.h
index 86af26091d6c..0e9fe1d9d971 100644
--- a/arch/x86/include/asm/iomap.h
+++ b/arch/x86/include/asm/iomap.h
@@ -1,3 +1,6 @@
+#ifndef _ASM_X86_IOMAP_H
+#define _ASM_X86_IOMAP_H
+
 /*
  * Copyright © 2008 Ingo Molnar
  *
@@ -31,3 +34,5 @@ iomap_atomic_prot_pfn(unsigned long pfn, enum km_type type, pgprot_t prot);
 
 void
 iounmap_atomic(void *kvaddr, enum km_type type);
+
+#endif /* _ASM_X86_IOMAP_H */
diff --git a/arch/x86/include/asm/irq_vectors.h b/arch/x86/include/asm/irq_vectors.h
index 1b35c4357ea8..5b21f0ec3df2 100644
--- a/arch/x86/include/asm/irq_vectors.h
+++ b/arch/x86/include/asm/irq_vectors.h
@@ -104,14 +104,14 @@
 #define LOCAL_TIMER_VECTOR              0xef
 
 /*
- * Performance monitoring interrupt vector:
+ * Generic system vector for platform specific use
  */
-#define LOCAL_PERF_VECTOR               0xee
+#define GENERIC_INTERRUPT_VECTOR        0xed
 
 /*
- * Generic system vector for platform specific use
+ * Performance monitoring pending work vector:
  */
-#define GENERIC_INTERRUPT_VECTOR        0xed
+#define LOCAL_PENDING_VECTOR            0xec
 
 #define UV_BAU_MESSAGE                  0xec
 
diff --git a/arch/x86/include/asm/k8.h b/arch/x86/include/asm/k8.h
index 54c8cc53b24d..c2d1f3b58e5f 100644
--- a/arch/x86/include/asm/k8.h
+++ b/arch/x86/include/asm/k8.h
@@ -12,4 +12,17 @@ extern int cache_k8_northbridges(void);
 extern void k8_flush_garts(void);
 extern int k8_scan_nodes(unsigned long start, unsigned long end);
 
+#ifdef CONFIG_K8_NB
+static inline struct pci_dev *node_to_k8_nb_misc(int node)
+{
+        return (node < num_k8_northbridges) ? k8_northbridges[node] : NULL;
+}
+#else
+static inline struct pci_dev *node_to_k8_nb_misc(int node)
+{
+        return NULL;
+}
+#endif
+
+
 #endif /* _ASM_X86_K8_H */
diff --git a/arch/x86/include/asm/kvm.h b/arch/x86/include/asm/kvm.h
index dc3f6cf11704..125be8b19568 100644
--- a/arch/x86/include/asm/kvm.h
+++ b/arch/x86/include/asm/kvm.h
@@ -16,6 +16,7 @@
 #define __KVM_HAVE_MSI
 #define __KVM_HAVE_USER_NMI
 #define __KVM_HAVE_GUEST_DEBUG
+#define __KVM_HAVE_MSIX
 
 /* Architectural interrupt line count. */
 #define KVM_NR_INTERRUPTS 256
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index f0faf58044ff..eabdc1cfab5c 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -185,6 +185,7 @@ union kvm_mmu_page_role {
                 unsigned access:3;
                 unsigned invalid:1;
                 unsigned cr4_pge:1;
+                unsigned nxe:1;
         };
 };
 
@@ -212,7 +213,6 @@ struct kvm_mmu_page {
         int multimapped;         /* More than one parent_pte? */
         int root_count;          /* Currently serving as active root */
         bool unsync;
-        bool global;
         unsigned int unsync_children;
         union {
                 u64 *parent_pte;               /* !multimapped */
@@ -261,13 +261,11 @@ struct kvm_mmu {
         union kvm_mmu_page_role base_role;
 
         u64 *pae_root;
+        u64 rsvd_bits_mask[2][4];
 };
 
 struct kvm_vcpu_arch {
         u64 host_tsc;
-        int interrupt_window_open;
-        unsigned long irq_summary; /* bit vector: 1 per word in irq_pending */
-        DECLARE_BITMAP(irq_pending, KVM_NR_INTERRUPTS);
         /*
          * rip and regs accesses must go through
          * kvm_{register,rip}_{read,write} functions.
@@ -286,6 +284,7 @@ struct kvm_vcpu_arch {
         u64 shadow_efer;
         u64 apic_base;
         struct kvm_lapic *apic;    /* kernel irqchip context */
+        int32_t apic_arb_prio;
         int mp_state;
         int sipi_vector;
         u64 ia32_misc_enable_msr;
@@ -320,6 +319,8 @@ struct kvm_vcpu_arch {
         struct kvm_pio_request pio;
         void *pio_data;
 
+        u8 event_exit_inst_len;
+
         struct kvm_queued_exception {
                 bool pending;
                 bool has_error_code;
@@ -329,11 +330,12 @@ struct kvm_vcpu_arch {
 
         struct kvm_queued_interrupt {
                 bool pending;
+                bool soft;
                 u8 nr;
         } interrupt;
 
         struct {
-                int active;
+                int vm86_active;
                 u8 save_iopl;
                 struct kvm_save_segment {
                         u16 selector;
@@ -356,9 +358,9 @@ struct kvm_vcpu_arch {
         unsigned int time_offset;
         struct page *time_page;
 
+        bool singlestep; /* guest is single stepped by KVM */
         bool nmi_pending;
         bool nmi_injected;
-        bool nmi_window_open;
 
         struct mtrr_state_type mtrr_state;
         u32 pat;
@@ -392,15 +394,14 @@ struct kvm_arch{
          */
         struct list_head active_mmu_pages;
         struct list_head assigned_dev_head;
-        struct list_head oos_global_pages;
         struct iommu_domain *iommu_domain;
+        int iommu_flags;
         struct kvm_pic *vpic;
         struct kvm_ioapic *vioapic;
         struct kvm_pit *vpit;
         struct hlist_head irq_ack_notifier_list;
         int vapics_in_nmi_mode;
 
-        int round_robin_prev_vcpu;
         unsigned int tss_addr;
         struct page *apic_access_page;
 
@@ -423,7 +424,6 @@ struct kvm_vm_stat {
         u32 mmu_recycled;
         u32 mmu_cache_miss;
         u32 mmu_unsync;
-        u32 mmu_unsync_global;
         u32 remote_tlb_flush;
         u32 lpages;
 };
@@ -443,7 +443,6 @@ struct kvm_vcpu_stat {
         u32 halt_exits;
         u32 halt_wakeup;
         u32 request_irq_exits;
-        u32 request_nmi_exits;
         u32 irq_exits;
         u32 host_state_reload;
         u32 efer_reload;
@@ -511,20 +510,22 @@ struct kvm_x86_ops {
         void (*run)(struct kvm_vcpu *vcpu, struct kvm_run *run);
         int (*handle_exit)(struct kvm_run *run, struct kvm_vcpu *vcpu);
         void (*skip_emulated_instruction)(struct kvm_vcpu *vcpu);
+        void (*set_interrupt_shadow)(struct kvm_vcpu *vcpu, int mask);
+        u32 (*get_interrupt_shadow)(struct kvm_vcpu *vcpu, int mask);
         void (*patch_hypercall)(struct kvm_vcpu *vcpu,
                                 unsigned char *hypercall_addr);
-        int (*get_irq)(struct kvm_vcpu *vcpu);
-        void (*set_irq)(struct kvm_vcpu *vcpu, int vec);
+        void (*set_irq)(struct kvm_vcpu *vcpu);
+        void (*set_nmi)(struct kvm_vcpu *vcpu);
         void (*queue_exception)(struct kvm_vcpu *vcpu, unsigned nr,
                                 bool has_error_code, u32 error_code);
-        bool (*exception_injected)(struct kvm_vcpu *vcpu);
-        void (*inject_pending_irq)(struct kvm_vcpu *vcpu);
-        void (*inject_pending_vectors)(struct kvm_vcpu *vcpu,
-                                       struct kvm_run *run);
-
+        int (*interrupt_allowed)(struct kvm_vcpu *vcpu);
+        int (*nmi_allowed)(struct kvm_vcpu *vcpu);
+        void (*enable_nmi_window)(struct kvm_vcpu *vcpu);
+        void (*enable_irq_window)(struct kvm_vcpu *vcpu);
+        void (*update_cr8_intercept)(struct kvm_vcpu *vcpu, int tpr, int irr);
         int (*set_tss_addr)(struct kvm *kvm, unsigned int addr);
         int (*get_tdp_level)(void);
-        int (*get_mt_mask_shift)(void);
+        u64 (*get_mt_mask)(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio);
 };
 
 extern struct kvm_x86_ops *kvm_x86_ops;
@@ -538,7 +539,7 @@ int kvm_mmu_setup(struct kvm_vcpu *vcpu);
 void kvm_mmu_set_nonpresent_ptes(u64 trap_pte, u64 notrap_pte);
 void kvm_mmu_set_base_ptes(u64 base_pte);
 void kvm_mmu_set_mask_ptes(u64 user_mask, u64 accessed_mask,
-                u64 dirty_mask, u64 nx_mask, u64 x_mask, u64 mt_mask);
+                u64 dirty_mask, u64 nx_mask, u64 x_mask);
 
 int kvm_mmu_reset_context(struct kvm_vcpu *vcpu);
 void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot);
@@ -552,6 +553,7 @@ int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
                           const void *val, int bytes);
 int kvm_pv_mmu_op(struct kvm_vcpu *vcpu, unsigned long bytes,
                   gpa_t addr, unsigned long *ret);
+u8 kvm_get_guest_memory_type(struct kvm_vcpu *vcpu, gfn_t gfn);
 
 extern bool tdp_enabled;
 
@@ -563,6 +565,7 @@ enum emulation_result {
 
 #define EMULTYPE_NO_DECODE          (1 << 0)
 #define EMULTYPE_TRAP_UD            (1 << 1)
+#define EMULTYPE_SKIP               (1 << 2)
 int emulate_instruction(struct kvm_vcpu *vcpu, struct kvm_run *run,
                         unsigned long cr2, u16 error_code, int emulation_type);
 void kvm_report_emulation_failure(struct kvm_vcpu *cvpu, const char *context);
@@ -638,7 +641,6 @@ void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu);
 int kvm_mmu_load(struct kvm_vcpu *vcpu);
 void kvm_mmu_unload(struct kvm_vcpu *vcpu);
 void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu);
-void kvm_mmu_sync_global(struct kvm_vcpu *vcpu);
 
 int kvm_emulate_hypercall(struct kvm_vcpu *vcpu);
 
@@ -769,6 +771,8 @@ enum {
 #define HF_GIF_MASK             (1 << 0)
 #define HF_HIF_MASK             (1 << 1)
 #define HF_VINTR_MASK           (1 << 2)
+#define HF_NMI_MASK             (1 << 3)
+#define HF_IRET_MASK            (1 << 4)
 
 /*
  * Hardware virtualization extension instructions may fault if a
@@ -791,5 +795,6 @@ asmlinkage void kvm_handle_fault_on_reboot(void);
 #define KVM_ARCH_WANT_MMU_NOTIFIER
 int kvm_unmap_hva(struct kvm *kvm, unsigned long hva);
 int kvm_age_hva(struct kvm *kvm, unsigned long hva);
+int cpuid_maxphyaddr(struct kvm_vcpu *vcpu);
 
 #endif /* _ASM_X86_KVM_HOST_H */
diff --git a/arch/x86/include/asm/kvm_x86_emulate.h b/arch/x86/include/asm/kvm_x86_emulate.h
index 6a159732881a..b7ed2c423116 100644
--- a/arch/x86/include/asm/kvm_x86_emulate.h
+++ b/arch/x86/include/asm/kvm_x86_emulate.h
@@ -143,6 +143,9 @@ struct decode_cache {
         struct fetch_cache fetch;
 };
 
+#define X86_SHADOW_INT_MOV_SS  1
+#define X86_SHADOW_INT_STI     2
+
 struct x86_emulate_ctxt {
         /* Register state before/after emulation. */
         struct kvm_vcpu *vcpu;
@@ -152,6 +155,9 @@ struct x86_emulate_ctxt {
         int mode;
         u32 cs_base;
 
+        /* interruptibility state, as a result of execution of STI or MOV SS */
+        int interruptibility;
+
         /* decode cache */
         struct decode_cache decode;
 };
diff --git a/arch/x86/include/asm/microcode.h b/arch/x86/include/asm/microcode.h
index c882664716c1..ef51b501e22a 100644
--- a/arch/x86/include/asm/microcode.h
+++ b/arch/x86/include/asm/microcode.h
@@ -9,20 +9,31 @@ struct cpu_signature {
 
 struct device;
 
+enum ucode_state { UCODE_ERROR, UCODE_OK, UCODE_NFOUND };
+
 struct microcode_ops {
-        int  (*request_microcode_user) (int cpu, const void __user *buf, size_t size);
-        int  (*request_microcode_fw) (int cpu, struct device *device);
+        enum ucode_state (*request_microcode_user) (int cpu,
+                                const void __user *buf, size_t size);
 
-        void (*apply_microcode) (int cpu);
+        enum ucode_state (*request_microcode_fw) (int cpu,
+                                struct device *device);
 
-        int  (*collect_cpu_info) (int cpu, struct cpu_signature *csig);
         void (*microcode_fini_cpu) (int cpu);
+
+        /*
+         * The generic 'microcode_core' part guarantees that
+         * the callbacks below run on a target cpu when they
+         * are being called.
+         * See also the "Synchronization" section in microcode_core.c.
+         */
+        int (*apply_microcode) (int cpu);
+        int (*collect_cpu_info) (int cpu, struct cpu_signature *csig);
 };
 
 struct ucode_cpu_info {
-        struct cpu_signature cpu_sig;
-        int valid;
-        void *mc;
+        struct cpu_signature    cpu_sig;
+        int                     valid;
+        void                    *mc;
 };
 extern struct ucode_cpu_info ucode_cpu_info[];
 
diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h
index c86404695083..1692fb5050e3 100644
--- a/arch/x86/include/asm/msr-index.h
+++ b/arch/x86/include/asm/msr-index.h
@@ -121,7 +121,6 @@
 #define MSR_K8_TOP_MEM1                 0xc001001a
 #define MSR_K8_TOP_MEM2                 0xc001001d
 #define MSR_K8_SYSCFG                   0xc0010010
-#define MSR_K8_HWCR                     0xc0010015
 #define MSR_K8_INT_PENDING_MSG          0xc0010055
 /* C1E active bits in int pending message */
 #define K8_INTP_C1E_ACTIVE_MASK         0x18000000
diff --git a/arch/x86/include/asm/msr.h b/arch/x86/include/asm/msr.h
index 638bf6241807..22603764e7db 100644
--- a/arch/x86/include/asm/msr.h
+++ b/arch/x86/include/asm/msr.h
@@ -12,6 +12,17 @@
 
 #include <asm/asm.h>
 #include <asm/errno.h>
+#include <asm/cpumask.h>
+
+struct msr {
+        union {
+                struct {
+                        u32 l;
+                        u32 h;
+                };
+                u64 q;
+        };
+};
 
 static inline unsigned long long native_read_tscp(unsigned int *aux)
 {
@@ -216,6 +227,8 @@ do {                                                            \
 #ifdef CONFIG_SMP
 int rdmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h);
 int wrmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h);
+void rdmsr_on_cpus(const cpumask_t *mask, u32 msr_no, struct msr *msrs);
+void wrmsr_on_cpus(const cpumask_t *mask, u32 msr_no, struct msr *msrs);
 int rdmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h);
 int wrmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h);
 #else  /*  CONFIG_SMP  */
@@ -229,6 +242,16 @@ static inline int wrmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h)
         wrmsr(msr_no, l, h);
         return 0;
 }
+static inline void rdmsr_on_cpus(const cpumask_t *m, u32 msr_no,
+                                struct msr *msrs)
+{
+       rdmsr_on_cpu(0, msr_no, &(msrs[0].l), &(msrs[0].h));
+}
+static inline void wrmsr_on_cpus(const cpumask_t *m, u32 msr_no,
+                                struct msr *msrs)
+{
+       wrmsr_on_cpu(0, msr_no, msrs[0].l, msrs[0].h);
+}
 static inline int rdmsr_safe_on_cpu(unsigned int cpu, u32 msr_no,
                                     u32 *l, u32 *h)
 {
diff --git a/arch/x86/include/asm/nmi.h b/arch/x86/include/asm/nmi.h
index c45a0a568dff..c97264409934 100644
--- a/arch/x86/include/asm/nmi.h
+++ b/arch/x86/include/asm/nmi.h
@@ -64,7 +64,7 @@ static inline int nmi_watchdog_active(void)
          * but since they are power of two we could use a
          * cheaper way --cvg
          */
-        return nmi_watchdog & 0x3;
+        return nmi_watchdog & (NMI_LOCAL_APIC | NMI_IO_APIC);
 }
 #endif
 
diff --git a/arch/x86/include/asm/numa_64.h b/arch/x86/include/asm/numa_64.h
index 064ed6df4cbe..c4ae822e415f 100644
--- a/arch/x86/include/asm/numa_64.h
+++ b/arch/x86/include/asm/numa_64.h
@@ -17,9 +17,6 @@ extern int compute_hash_shift(struct bootnode *nodes, int numblks,
 extern void numa_init_array(void);
 extern int numa_off;
 
-extern void srat_reserve_add_area(int nodeid);
-extern int hotadd_percent;
-
 extern s16 apicid_to_node[MAX_LOCAL_APIC];
 
 extern unsigned long numa_free_all_bootmem(void);
@@ -27,6 +24,13 @@ extern void setup_node_bootmem(int nodeid, unsigned long start,
                                unsigned long end);
 
 #ifdef CONFIG_NUMA
+/*
+ * Too small node sizes may confuse the VM badly. Usually they
+ * result from BIOS bugs. So dont recognize nodes as standalone
+ * NUMA entities that have less than this amount of RAM listed:
+ */
+#define NODE_MIN_SIZE (4*1024*1024)
+
 extern void __init init_cpu_to_node(void);
 extern void __cpuinit numa_set_node(int cpu, int node);
 extern void __cpuinit numa_clear_node(int cpu);
diff --git a/arch/x86/include/asm/page_32_types.h b/arch/x86/include/asm/page_32_types.h
index 0f915ae649a7..6f1b7331313f 100644
--- a/arch/x86/include/asm/page_32_types.h
+++ b/arch/x86/include/asm/page_32_types.h
@@ -54,10 +54,6 @@ extern unsigned int __VMALLOC_RESERVE;
 extern int sysctl_legacy_va_layout;
 
 extern void find_low_pfn_range(void);
-extern unsigned long init_memory_mapping(unsigned long start,
-                                         unsigned long end);
-extern void initmem_init(unsigned long, unsigned long);
-extern void free_initmem(void);
 extern void setup_bootmem_allocator(void);
 
 #endif  /* !__ASSEMBLY__ */
diff --git a/arch/x86/include/asm/page_64_types.h b/arch/x86/include/asm/page_64_types.h
index d38c91b70248..8d382d3abf38 100644
--- a/arch/x86/include/asm/page_64_types.h
+++ b/arch/x86/include/asm/page_64_types.h
@@ -32,22 +32,14 @@
  */
 #define __PAGE_OFFSET           _AC(0xffff880000000000, UL)
 
-#define __PHYSICAL_START        CONFIG_PHYSICAL_START
-#define __KERNEL_ALIGN          0x200000
-
-/*
- * Make sure kernel is aligned to 2MB address. Catching it at compile
- * time is better. Change your config file and compile the kernel
- * for a 2MB aligned address (CONFIG_PHYSICAL_START)
- */
-#if (CONFIG_PHYSICAL_START % __KERNEL_ALIGN) != 0
-#error "CONFIG_PHYSICAL_START must be a multiple of 2MB"
-#endif
+#define __PHYSICAL_START        ((CONFIG_PHYSICAL_START +               \
+                                  (CONFIG_PHYSICAL_ALIGN - 1)) &        \
+                                 ~(CONFIG_PHYSICAL_ALIGN - 1))
 
 #define __START_KERNEL          (__START_KERNEL_map + __PHYSICAL_START)
 #define __START_KERNEL_map      _AC(0xffffffff80000000, UL)
 
-/* See Documentation/x86_64/mm.txt for a description of the memory map. */
+/* See Documentation/x86/x86_64/mm.txt for a description of the memory map. */
 #define __PHYSICAL_MASK_SHIFT   46
 #define __VIRTUAL_MASK_SHIFT    48
 
@@ -71,12 +63,6 @@ extern unsigned long __phys_addr(unsigned long);
 
 #define vmemmap ((struct page *)VMEMMAP_START)
 
-extern unsigned long init_memory_mapping(unsigned long start,
-                                         unsigned long end);
-
-extern void initmem_init(unsigned long start_pfn, unsigned long end_pfn);
-extern void free_initmem(void);
-
 extern void init_extra_mapping_uc(unsigned long phys, unsigned long size);
 extern void init_extra_mapping_wb(unsigned long phys, unsigned long size);
 
diff --git a/arch/x86/include/asm/page_types.h b/arch/x86/include/asm/page_types.h
index 826ad37006ab..6473f5ccff85 100644
--- a/arch/x86/include/asm/page_types.h
+++ b/arch/x86/include/asm/page_types.h
@@ -46,6 +46,12 @@ extern int devmem_is_allowed(unsigned long pagenr);
 extern unsigned long max_low_pfn_mapped;
 extern unsigned long max_pfn_mapped;
 
+extern unsigned long init_memory_mapping(unsigned long start,
+                                         unsigned long end);
+
+extern void initmem_init(unsigned long start_pfn, unsigned long end_pfn);
+extern void free_initmem(void);
+
 #endif  /* !__ASSEMBLY__ */
 
 #endif  /* _ASM_X86_PAGE_DEFS_H */
diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h
index a53da004e08e..4fb37c8a0832 100644
--- a/arch/x86/include/asm/paravirt.h
+++ b/arch/x86/include/asm/paravirt.h
@@ -56,6 +56,7 @@ struct desc_ptr;
 struct tss_struct;
 struct mm_struct;
 struct desc_struct;
+struct task_struct;
 
 /*
  * Wrapper type for pointers to code which uses the non-standard
@@ -203,7 +204,8 @@ struct pv_cpu_ops {
 
         void (*swapgs)(void);
 
-        struct pv_lazy_ops lazy_mode;
+        void (*start_context_switch)(struct task_struct *prev);
+        void (*end_context_switch)(struct task_struct *next);
 };
 
 struct pv_irq_ops {
@@ -1399,25 +1401,23 @@ enum paravirt_lazy_mode {
 };
 
 enum paravirt_lazy_mode paravirt_get_lazy_mode(void);
-void paravirt_enter_lazy_cpu(void);
-void paravirt_leave_lazy_cpu(void);
+void paravirt_start_context_switch(struct task_struct *prev);
+void paravirt_end_context_switch(struct task_struct *next);
+
 void paravirt_enter_lazy_mmu(void);
 void paravirt_leave_lazy_mmu(void);
-void paravirt_leave_lazy(enum paravirt_lazy_mode mode);
 
-#define  __HAVE_ARCH_ENTER_LAZY_CPU_MODE
-static inline void arch_enter_lazy_cpu_mode(void)
+#define  __HAVE_ARCH_START_CONTEXT_SWITCH
+static inline void arch_start_context_switch(struct task_struct *prev)
 {
-        PVOP_VCALL0(pv_cpu_ops.lazy_mode.enter);
+        PVOP_VCALL1(pv_cpu_ops.start_context_switch, prev);
 }
 
-static inline void arch_leave_lazy_cpu_mode(void)
+static inline void arch_end_context_switch(struct task_struct *next)
 {
-        PVOP_VCALL0(pv_cpu_ops.lazy_mode.leave);
+        PVOP_VCALL1(pv_cpu_ops.end_context_switch, next);
 }
 
-void arch_flush_lazy_cpu_mode(void);
-
 #define  __HAVE_ARCH_ENTER_LAZY_MMU_MODE
 static inline void arch_enter_lazy_mmu_mode(void)
 {
diff --git a/arch/x86/include/asm/perf_counter.h b/arch/x86/include/asm/perf_counter.h
new file mode 100644
index 000000000000..876ed97147b3
--- /dev/null
+++ b/arch/x86/include/asm/perf_counter.h
@@ -0,0 +1,100 @@
+#ifndef _ASM_X86_PERF_COUNTER_H
+#define _ASM_X86_PERF_COUNTER_H
+
+/*
+ * Performance counter hw details:
+ */
+
+#define X86_PMC_MAX_GENERIC                                     8
+#define X86_PMC_MAX_FIXED                                       3
+
+#define X86_PMC_IDX_GENERIC                                     0
+#define X86_PMC_IDX_FIXED                                      32
+#define X86_PMC_IDX_MAX                                        64
+
+#define MSR_ARCH_PERFMON_PERFCTR0                             0xc1
+#define MSR_ARCH_PERFMON_PERFCTR1                             0xc2
+
+#define MSR_ARCH_PERFMON_EVENTSEL0                           0x186
+#define MSR_ARCH_PERFMON_EVENTSEL1                           0x187
+
+#define ARCH_PERFMON_EVENTSEL0_ENABLE                     (1 << 22)
+#define ARCH_PERFMON_EVENTSEL_INT                         (1 << 20)
+#define ARCH_PERFMON_EVENTSEL_OS                          (1 << 17)
+#define ARCH_PERFMON_EVENTSEL_USR                         (1 << 16)
+
+/*
+ * Includes eventsel and unit mask as well:
+ */
+#define ARCH_PERFMON_EVENT_MASK                             0xffff
+
+#define ARCH_PERFMON_UNHALTED_CORE_CYCLES_SEL                 0x3c
+#define ARCH_PERFMON_UNHALTED_CORE_CYCLES_UMASK         (0x00 << 8)
+#define ARCH_PERFMON_UNHALTED_CORE_CYCLES_INDEX                  0
+#define ARCH_PERFMON_UNHALTED_CORE_CYCLES_PRESENT \
+                (1 << (ARCH_PERFMON_UNHALTED_CORE_CYCLES_INDEX))
+
+#define ARCH_PERFMON_BRANCH_MISSES_RETIRED                       6
+
+/*
+ * Intel "Architectural Performance Monitoring" CPUID
+ * detection/enumeration details:
+ */
+union cpuid10_eax {
+        struct {
+                unsigned int version_id:8;
+                unsigned int num_counters:8;
+                unsigned int bit_width:8;
+                unsigned int mask_length:8;
+        } split;
+        unsigned int full;
+};
+
+union cpuid10_edx {
+        struct {
+                unsigned int num_counters_fixed:4;
+                unsigned int reserved:28;
+        } split;
+        unsigned int full;
+};
+
+
+/*
+ * Fixed-purpose performance counters:
+ */
+
+/*
+ * All 3 fixed-mode PMCs are configured via this single MSR:
+ */
+#define MSR_ARCH_PERFMON_FIXED_CTR_CTRL                 0x38d
+
+/*
+ * The counts are available in three separate MSRs:
+ */
+
+/* Instr_Retired.Any: */
+#define MSR_ARCH_PERFMON_FIXED_CTR0                     0x309
+#define X86_PMC_IDX_FIXED_INSTRUCTIONS                  (X86_PMC_IDX_FIXED + 0)
+
+/* CPU_CLK_Unhalted.Core: */
+#define MSR_ARCH_PERFMON_FIXED_CTR1                     0x30a
+#define X86_PMC_IDX_FIXED_CPU_CYCLES                    (X86_PMC_IDX_FIXED + 1)
+
+/* CPU_CLK_Unhalted.Ref: */
+#define MSR_ARCH_PERFMON_FIXED_CTR2                     0x30b
+#define X86_PMC_IDX_FIXED_BUS_CYCLES                    (X86_PMC_IDX_FIXED + 2)
+
+extern void set_perf_counter_pending(void);
+
+#define clear_perf_counter_pending()    do { } while (0)
+#define test_perf_counter_pending()     (0)
+
+#ifdef CONFIG_PERF_COUNTERS
+extern void init_hw_perf_counters(void);
+extern void perf_counters_lapic_init(void);
+#else
+static inline void init_hw_perf_counters(void)          { }
+static inline void perf_counters_lapic_init(void)       { }
+#endif
+
+#endif /* _ASM_X86_PERF_COUNTER_H */
diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h
index 29d96d168bc0..18ef7ebf2631 100644
--- a/arch/x86/include/asm/pgtable.h
+++ b/arch/x86/include/asm/pgtable.h
@@ -81,6 +81,8 @@ static inline void __init paravirt_pagetable_setup_done(pgd_t *base)
 #define pte_val(x)      native_pte_val(x)
 #define __pte(x)        native_make_pte(x)
 
+#define arch_end_context_switch(prev)   do {} while(0)
+
 #endif  /* CONFIG_PARAVIRT */
 
 /*
@@ -503,6 +505,8 @@ static inline int pgd_none(pgd_t pgd)
 
 #ifndef __ASSEMBLY__
 
+extern int direct_gbpages;
+
 /* local pte updates need not use xchg for locking */
 static inline pte_t native_local_ptep_get_and_clear(pte_t *ptep)
 {
diff --git a/arch/x86/include/asm/pgtable_64.h b/arch/x86/include/asm/pgtable_64.h
index 6b87bc6d5018..abde308fdb0f 100644
--- a/arch/x86/include/asm/pgtable_64.h
+++ b/arch/x86/include/asm/pgtable_64.h
@@ -25,10 +25,6 @@ extern pgd_t init_level4_pgt[];
 
 extern void paging_init(void);
 
-#endif /* !__ASSEMBLY__ */
-
-#ifndef __ASSEMBLY__
-
 #define pte_ERROR(e)                                    \
         printk("%s:%d: bad pte %p(%016lx).\n",          \
                __FILE__, __LINE__, &(e), pte_val(e))
@@ -135,8 +131,6 @@ static inline int pgd_large(pgd_t pgd) { return 0; }
 
 #define update_mmu_cache(vma, address, pte) do { } while (0)
 
-extern int direct_gbpages;
-
 /* Encode and de-code a swap entry */
 #if _PAGE_BIT_FILE < _PAGE_BIT_PROTNONE
 #define SWP_TYPE_BITS (_PAGE_BIT_FILE - _PAGE_BIT_PRESENT - 1)
diff --git a/arch/x86/include/asm/pgtable_64_types.h b/arch/x86/include/asm/pgtable_64_types.h
index fbf42b8e0383..766ea16fbbbd 100644
--- a/arch/x86/include/asm/pgtable_64_types.h
+++ b/arch/x86/include/asm/pgtable_64_types.h
@@ -51,11 +51,11 @@ typedef struct { pteval_t pte; } pte_t;
 #define PGDIR_SIZE      (_AC(1, UL) << PGDIR_SHIFT)
 #define PGDIR_MASK      (~(PGDIR_SIZE - 1))
 
-
+/* See Documentation/x86/x86_64/mm.txt for a description of the memory map. */
 #define MAXMEM           _AC(__AC(1, UL) << MAX_PHYSMEM_BITS, UL)
-#define VMALLOC_START    _AC(0xffffc20000000000, UL)
-#define VMALLOC_END      _AC(0xffffe1ffffffffff, UL)
-#define VMEMMAP_START    _AC(0xffffe20000000000, UL)
+#define VMALLOC_START    _AC(0xffffc90000000000, UL)
+#define VMALLOC_END      _AC(0xffffe8ffffffffff, UL)
+#define VMEMMAP_START    _AC(0xffffea0000000000, UL)
 #define MODULES_VADDR    _AC(0xffffffffa0000000, UL)
 #define MODULES_END      _AC(0xffffffffff000000, UL)
 #define MODULES_LEN   (MODULES_END - MODULES_VADDR)
diff --git a/arch/x86/include/asm/pgtable_types.h b/arch/x86/include/asm/pgtable_types.h
index b8238dc8786d..4d258ad76a0f 100644
--- a/arch/x86/include/asm/pgtable_types.h
+++ b/arch/x86/include/asm/pgtable_types.h
@@ -273,7 +273,6 @@ typedef struct page *pgtable_t;
 
 extern pteval_t __supported_pte_mask;
 extern int nx_enabled;
-extern void set_nx(void);
 
 #define pgprot_writecombine     pgprot_writecombine
 extern pgprot_t pgprot_writecombine(pgprot_t prot);
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index fed93fec9764..c7768269b1cf 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -410,9 +410,6 @@ DECLARE_PER_CPU(unsigned long, stack_canary);
 extern unsigned int xstate_size;
 extern void free_thread_xstate(struct task_struct *);
 extern struct kmem_cache *task_xstate_cachep;
-extern void init_scattered_cpuid_features(struct cpuinfo_x86 *c);
-extern unsigned int init_intel_cacheinfo(struct cpuinfo_x86 *c);
-extern unsigned short num_cache_leaves;
 
 struct thread_struct {
         /* Cached TLS descriptors: */
@@ -428,8 +425,12 @@ struct thread_struct {
         unsigned short          fsindex;
         unsigned short          gsindex;
 #endif
+#ifdef CONFIG_X86_32
         unsigned long           ip;
+#endif
+#ifdef CONFIG_X86_64
         unsigned long           fs;
+#endif
         unsigned long           gs;
         /* Hardware debugging registers: */
         unsigned long           debugreg0;
@@ -461,14 +462,8 @@ struct thread_struct {
         unsigned                io_bitmap_max;
 /* MSR_IA32_DEBUGCTLMSR value to switch in if TIF_DEBUGCTLMSR is set.  */
         unsigned long   debugctlmsr;
-#ifdef CONFIG_X86_DS
-/* Debug Store context; see include/asm-x86/ds.h; goes into MSR_IA32_DS_AREA */
+        /* Debug Store context; see asm/ds.h */
         struct ds_context       *ds_ctx;
-#endif /* CONFIG_X86_DS */
-#ifdef CONFIG_X86_PTRACE_BTS
-/* the signal to send on a bts buffer overflow */
-        unsigned int    bts_ovfl_signal;
-#endif /* CONFIG_X86_PTRACE_BTS */
 };
 
 static inline unsigned long native_get_debugreg(int regno)
@@ -796,6 +791,21 @@ static inline unsigned long get_debugctlmsr(void)
     return debugctlmsr;
 }
 
+static inline unsigned long get_debugctlmsr_on_cpu(int cpu)
+{
+        u64 debugctlmsr = 0;
+        u32 val1, val2;
+
+#ifndef CONFIG_X86_DEBUGCTLMSR
+        if (boot_cpu_data.x86 < 6)
+                return 0;
+#endif
+        rdmsr_on_cpu(cpu, MSR_IA32_DEBUGCTLMSR, &val1, &val2);
+        debugctlmsr = val1 | ((u64)val2 << 32);
+
+        return debugctlmsr;
+}
+
 static inline void update_debugctlmsr(unsigned long debugctlmsr)
 {
 #ifndef CONFIG_X86_DEBUGCTLMSR
@@ -805,6 +815,18 @@ static inline void update_debugctlmsr(unsigned long debugctlmsr)
         wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctlmsr);
 }
 
+static inline void update_debugctlmsr_on_cpu(int cpu,
+                                             unsigned long debugctlmsr)
+{
+#ifndef CONFIG_X86_DEBUGCTLMSR
+        if (boot_cpu_data.x86 < 6)
+                return;
+#endif
+        wrmsr_on_cpu(cpu, MSR_IA32_DEBUGCTLMSR,
+                     (u32)((u64)debugctlmsr),
+                     (u32)((u64)debugctlmsr >> 32));
+}
+
 /*
  * from system description table in BIOS. Mostly for MCA use, but
  * others may find it useful:
@@ -815,6 +837,7 @@ extern unsigned int		BIOS_revision;
 
 /* Boot loader type from the setup header: */
 extern int                      bootloader_type;
+extern int                      bootloader_version;
 
 extern char                     ignore_fpu_irq;
 
@@ -875,7 +898,6 @@ static inline void spin_lock_prefetch(const void *x)
         .vm86_info              = NULL,                                   \
         .sysenter_cs            = __KERNEL_CS,                            \
         .io_bitmap_ptr          = NULL,                                   \
-        .fs                     = __KERNEL_PERCPU,                        \
 }
 
 /*
diff --git a/arch/x86/include/asm/ptrace.h b/arch/x86/include/asm/ptrace.h
index 624f133943ed..0f0d908349aa 100644
--- a/arch/x86/include/asm/ptrace.h
+++ b/arch/x86/include/asm/ptrace.h
@@ -236,12 +236,11 @@ extern int do_get_thread_area(struct task_struct *p, int idx,
 extern int do_set_thread_area(struct task_struct *p, int idx,
                               struct user_desc __user *info, int can_allocate);
 
-extern void x86_ptrace_untrace(struct task_struct *);
-extern void x86_ptrace_fork(struct task_struct *child,
-                            unsigned long clone_flags);
+#ifdef CONFIG_X86_PTRACE_BTS
+extern void ptrace_bts_untrace(struct task_struct *tsk);
 
-#define arch_ptrace_untrace(tsk) x86_ptrace_untrace(tsk)
-#define arch_ptrace_fork(child, flags) x86_ptrace_fork(child, flags)
+#define arch_ptrace_untrace(tsk)        ptrace_bts_untrace(tsk)
+#endif /* CONFIG_X86_PTRACE_BTS */
 
 #endif /* __KERNEL__ */
 
diff --git a/arch/x86/include/asm/required-features.h b/arch/x86/include/asm/required-features.h
index a4737dddfd58..64cf2d24fad1 100644
--- a/arch/x86/include/asm/required-features.h
+++ b/arch/x86/include/asm/required-features.h
@@ -48,9 +48,15 @@
 #endif
 
 #ifdef CONFIG_X86_64
+#ifdef CONFIG_PARAVIRT
+/* Paravirtualized systems may not have PSE or PGE available */
 #define NEED_PSE        0
-#define NEED_MSR        (1<<(X86_FEATURE_MSR & 31))
 #define NEED_PGE        0
+#else
+#define NEED_PSE        (1<<(X86_FEATURE_PSE) & 31)
+#define NEED_PGE        (1<<(X86_FEATURE_PGE) & 31)
+#endif
+#define NEED_MSR        (1<<(X86_FEATURE_MSR & 31))
 #define NEED_FXSR       (1<<(X86_FEATURE_FXSR & 31))
 #define NEED_XMM        (1<<(X86_FEATURE_XMM & 31))
 #define NEED_XMM2       (1<<(X86_FEATURE_XMM2 & 31))
diff --git a/arch/x86/include/asm/sparsemem.h b/arch/x86/include/asm/sparsemem.h
index e3cc3c063ec5..4517d6b93188 100644
--- a/arch/x86/include/asm/sparsemem.h
+++ b/arch/x86/include/asm/sparsemem.h
@@ -27,7 +27,7 @@
 #else /* CONFIG_X86_32 */
 # define SECTION_SIZE_BITS      27 /* matt - 128 is convenient right now */
 # define MAX_PHYSADDR_BITS      44
-# define MAX_PHYSMEM_BITS       44 /* Can be max 45 bits */
+# define MAX_PHYSMEM_BITS       46
 #endif
 
 #endif /* CONFIG_SPARSEMEM */
diff --git a/arch/x86/include/asm/svm.h b/arch/x86/include/asm/svm.h
index 82ada75f3ebf..85574b7c1bc1 100644
--- a/arch/x86/include/asm/svm.h
+++ b/arch/x86/include/asm/svm.h
@@ -225,6 +225,7 @@ struct __attribute__ ((__packed__)) vmcb {
 #define SVM_EVTINJ_VALID_ERR (1 << 11)
 
 #define SVM_EXITINTINFO_VEC_MASK SVM_EVTINJ_VEC_MASK
+#define SVM_EXITINTINFO_TYPE_MASK SVM_EVTINJ_TYPE_MASK
 
 #define SVM_EXITINTINFO_TYPE_INTR SVM_EVTINJ_TYPE_INTR
 #define SVM_EXITINTINFO_TYPE_NMI SVM_EVTINJ_TYPE_NMI
diff --git a/arch/x86/include/asm/syscalls.h b/arch/x86/include/asm/syscalls.h
index 7043408f6904..372b76edd63f 100644
--- a/arch/x86/include/asm/syscalls.h
+++ b/arch/x86/include/asm/syscalls.h
@@ -1,7 +1,7 @@
 /*
  * syscalls.h - Linux syscall interfaces (arch-specific)
  *
- * Copyright (c) 2008 Jaswinder Singh
+ * Copyright (c) 2008 Jaswinder Singh Rajput
  *
  * This file is released under the GPLv2.
  * See the file COPYING for more details.
@@ -12,50 +12,55 @@
 
 #include <linux/compiler.h>
 #include <linux/linkage.h>
-#include <linux/types.h>
 #include <linux/signal.h>
+#include <linux/types.h>
 
 /* Common in X86_32 and X86_64 */
 /* kernel/ioport.c */
 asmlinkage long sys_ioperm(unsigned long, unsigned long, int);
 
+/* kernel/process.c */
+int sys_fork(struct pt_regs *);
+int sys_vfork(struct pt_regs *);
+
 /* kernel/ldt.c */
 asmlinkage int sys_modify_ldt(int, void __user *, unsigned long);
 
+/* kernel/signal.c */
+long sys_rt_sigreturn(struct pt_regs *);
+
 /* kernel/tls.c */
 asmlinkage int sys_set_thread_area(struct user_desc __user *);
 asmlinkage int sys_get_thread_area(struct user_desc __user *);
 
 /* X86_32 only */
 #ifdef CONFIG_X86_32
+/* kernel/ioport.c */
+long sys_iopl(struct pt_regs *);
+
 /* kernel/process_32.c */
-int sys_fork(struct pt_regs *);
 int sys_clone(struct pt_regs *);
-int sys_vfork(struct pt_regs *);
 int sys_execve(struct pt_regs *);
 
-/* kernel/signal_32.c */
+/* kernel/signal.c */
 asmlinkage int sys_sigsuspend(int, int, old_sigset_t);
 asmlinkage int sys_sigaction(int, const struct old_sigaction __user *,
                              struct old_sigaction __user *);
 int sys_sigaltstack(struct pt_regs *);
 unsigned long sys_sigreturn(struct pt_regs *);
-long sys_rt_sigreturn(struct pt_regs *);
-
-/* kernel/ioport.c */
-long sys_iopl(struct pt_regs *);
 
 /* kernel/sys_i386_32.c */
+struct mmap_arg_struct;
+struct sel_arg_struct;
+struct oldold_utsname;
+struct old_utsname;
+
 asmlinkage long sys_mmap2(unsigned long, unsigned long, unsigned long,
                           unsigned long, unsigned long, unsigned long);
-struct mmap_arg_struct;
 asmlinkage int old_mmap(struct mmap_arg_struct __user *);
-struct sel_arg_struct;
 asmlinkage int old_select(struct sel_arg_struct __user *);
 asmlinkage int sys_ipc(uint, int, int, int, void __user *, long);
-struct old_utsname;
 asmlinkage int sys_uname(struct old_utsname __user *);
-struct oldold_utsname;
 asmlinkage int sys_olduname(struct oldold_utsname __user *);
 
 /* kernel/vm86_32.c */
@@ -65,29 +70,27 @@ int sys_vm86(struct pt_regs *);
 #else /* CONFIG_X86_32 */
 
 /* X86_64 only */
+/* kernel/ioport.c */
+asmlinkage long sys_iopl(unsigned int, struct pt_regs *);
+
 /* kernel/process_64.c */
-asmlinkage long sys_fork(struct pt_regs *);
 asmlinkage long sys_clone(unsigned long, unsigned long,
                           void __user *, void __user *,
                           struct pt_regs *);
-asmlinkage long sys_vfork(struct pt_regs *);
 asmlinkage long sys_execve(char __user *, char __user * __user *,
                            char __user * __user *,
                            struct pt_regs *);
 long sys_arch_prctl(int, unsigned long);
 
-/* kernel/ioport.c */
-asmlinkage long sys_iopl(unsigned int, struct pt_regs *);
-
-/* kernel/signal_64.c */
+/* kernel/signal.c */
 asmlinkage long sys_sigaltstack(const stack_t __user *, stack_t __user *,
                                 struct pt_regs *);
-long sys_rt_sigreturn(struct pt_regs *);
 
 /* kernel/sys_x86_64.c */
+struct new_utsname;
+
 asmlinkage long sys_mmap(unsigned long, unsigned long, unsigned long,
                          unsigned long, unsigned long, unsigned long);
-struct new_utsname;
 asmlinkage long sys_uname(struct new_utsname __user *);
 
 #endif /* CONFIG_X86_32 */
diff --git a/arch/x86/include/asm/termios.h b/arch/x86/include/asm/termios.h
index f72956331c49..c4ee8056baca 100644
--- a/arch/x86/include/asm/termios.h
+++ b/arch/x86/include/asm/termios.h
@@ -67,6 +67,7 @@ static inline int user_termio_to_kernel_termios(struct ktermios *termios,
         SET_LOW_TERMIOS_BITS(termios, termio, c_oflag);
         SET_LOW_TERMIOS_BITS(termios, termio, c_cflag);
         SET_LOW_TERMIOS_BITS(termios, termio, c_lflag);
+        get_user(termios->c_line, &termio->c_line);
         return copy_from_user(termios->c_cc, termio->c_cc, NCC);
 }
 
diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h
index 8820a73ae090..602c769fc98c 100644
--- a/arch/x86/include/asm/thread_info.h
+++ b/arch/x86/include/asm/thread_info.h
@@ -94,7 +94,8 @@ struct thread_info {
 #define TIF_FORCED_TF           24      /* true if TF in eflags artificially */
 #define TIF_DEBUGCTLMSR         25      /* uses thread_struct.debugctlmsr */
 #define TIF_DS_AREA_MSR         26      /* uses thread_struct.ds_area_msr */
-#define TIF_SYSCALL_FTRACE      27      /* for ftrace syscall instrumentation */
+#define TIF_LAZY_MMU_UPDATES    27      /* task is updating the mmu lazily */
+#define TIF_SYSCALL_FTRACE      28      /* for ftrace syscall instrumentation */
 
 #define _TIF_SYSCALL_TRACE      (1 << TIF_SYSCALL_TRACE)
 #define _TIF_NOTIFY_RESUME      (1 << TIF_NOTIFY_RESUME)
@@ -116,6 +117,7 @@ struct thread_info {
 #define _TIF_FORCED_TF          (1 << TIF_FORCED_TF)
 #define _TIF_DEBUGCTLMSR        (1 << TIF_DEBUGCTLMSR)
 #define _TIF_DS_AREA_MSR        (1 << TIF_DS_AREA_MSR)
+#define _TIF_LAZY_MMU_UPDATES   (1 << TIF_LAZY_MMU_UPDATES)
 #define _TIF_SYSCALL_FTRACE     (1 << TIF_SYSCALL_FTRACE)
 
 /* work to do in syscall_trace_enter() */
diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h
index 16a5c84b0329..a5ecc9c33e92 100644
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -17,7 +17,7 @@
 
 static inline void __native_flush_tlb(void)
 {
-        write_cr3(read_cr3());
+        native_write_cr3(native_read_cr3());
 }
 
 static inline void __native_flush_tlb_global(void)
@@ -32,11 +32,11 @@ static inline void __native_flush_tlb_global(void)
          */
         raw_local_irq_save(flags);
 
-        cr4 = read_cr4();
+        cr4 = native_read_cr4();
         /* clear PGE */
-        write_cr4(cr4 & ~X86_CR4_PGE);
+        native_write_cr4(cr4 & ~X86_CR4_PGE);
         /* write old PGE again and flush TLBs */
-        write_cr4(cr4);
+        native_write_cr4(cr4);
 
         raw_local_irq_restore(flags);
 }
diff --git a/arch/x86/include/asm/topology.h b/arch/x86/include/asm/topology.h
index f44b49abca49..066ef590d7e0 100644
--- a/arch/x86/include/asm/topology.h
+++ b/arch/x86/include/asm/topology.h
@@ -203,7 +203,8 @@ struct pci_bus;
 void x86_pci_root_bus_res_quirks(struct pci_bus *b);
 
 #ifdef CONFIG_SMP
-#define mc_capable()    (cpumask_weight(cpu_core_mask(0)) != nr_cpu_ids)
+#define mc_capable()    ((boot_cpu_data.x86_max_cores > 1) && \
+                        (cpumask_weight(cpu_core_mask(0)) != nr_cpu_ids))
 #define smt_capable()                   (smp_num_siblings > 1)
 #endif
 
diff --git a/arch/x86/include/asm/traps.h b/arch/x86/include/asm/traps.h
index 0d5342515b86..bfd74c032fca 100644
--- a/arch/x86/include/asm/traps.h
+++ b/arch/x86/include/asm/traps.h
@@ -2,6 +2,7 @@
 #define _ASM_X86_TRAPS_H
 
 #include <asm/debugreg.h>
+#include <asm/siginfo.h>                        /* TRAP_TRACE, ... */
 
 #ifdef CONFIG_X86_32
 #define dotraplinkage
@@ -13,6 +14,9 @@ asmlinkage void divide_error(void);
 asmlinkage void debug(void);
 asmlinkage void nmi(void);
 asmlinkage void int3(void);
+asmlinkage void xen_debug(void);
+asmlinkage void xen_int3(void);
+asmlinkage void xen_stack_segment(void);
 asmlinkage void overflow(void);
 asmlinkage void bounds(void);
 asmlinkage void invalid_op(void);
@@ -74,7 +78,6 @@ static inline int get_si_code(unsigned long condition)
 }
 
 extern int panic_on_unrecovered_nmi;
-extern int kstack_depth_to_print;
 
 void math_error(void __user *);
 void math_emulate(struct math_emu_info *);
diff --git a/arch/x86/include/asm/unistd_32.h b/arch/x86/include/asm/unistd_32.h
index 6e72d74cf8dc..732a30706153 100644
--- a/arch/x86/include/asm/unistd_32.h
+++ b/arch/x86/include/asm/unistd_32.h
@@ -340,6 +340,8 @@
 #define __NR_inotify_init1      332
 #define __NR_preadv             333
 #define __NR_pwritev            334
+#define __NR_rt_tgsigqueueinfo  335
+#define __NR_perf_counter_open  336
 
 #ifdef __KERNEL__
 
diff --git a/arch/x86/include/asm/unistd_64.h b/arch/x86/include/asm/unistd_64.h
index f81829462325..900e1617e672 100644
--- a/arch/x86/include/asm/unistd_64.h
+++ b/arch/x86/include/asm/unistd_64.h
@@ -657,7 +657,10 @@ __SYSCALL(__NR_inotify_init1, sys_inotify_init1)
 __SYSCALL(__NR_preadv, sys_preadv)
 #define __NR_pwritev                            296
 __SYSCALL(__NR_pwritev, sys_pwritev)
-
+#define __NR_rt_tgsigqueueinfo                  297
+__SYSCALL(__NR_rt_tgsigqueueinfo, sys_rt_tgsigqueueinfo)
+#define __NR_perf_counter_open                  298
+__SYSCALL(__NR_perf_counter_open, sys_perf_counter_open)
 
 #ifndef __NO_STUBS
 #define __ARCH_WANT_OLD_READDIR
diff --git a/arch/x86/include/asm/uv/uv_bau.h b/arch/x86/include/asm/uv/uv_bau.h
index 9b0e61bf7a88..bddd44f2f0ab 100644
--- a/arch/x86/include/asm/uv/uv_bau.h
+++ b/arch/x86/include/asm/uv/uv_bau.h
@@ -37,7 +37,7 @@
 #define UV_CPUS_PER_ACT_STATUS          32
 #define UV_ACT_STATUS_MASK              0x3
 #define UV_ACT_STATUS_SIZE              2
-#define UV_ACTIVATION_DESCRIPTOR_SIZE   32
+#define UV_ADP_SIZE                     32
 #define UV_DISTRIBUTION_SIZE            256
 #define UV_SW_ACK_NPENDING              8
 #define UV_NET_ENDPOINT_INTD            0x38
diff --git a/arch/x86/include/asm/uv/uv_hub.h b/arch/x86/include/asm/uv/uv_hub.h
index d3a98ea1062e..341070f7ad5c 100644
--- a/arch/x86/include/asm/uv/uv_hub.h
+++ b/arch/x86/include/asm/uv/uv_hub.h
@@ -133,6 +133,7 @@ struct uv_scir_s {
 struct uv_hub_info_s {
         unsigned long           global_mmr_base;
         unsigned long           gpa_mask;
+        unsigned int            gnode_extra;
         unsigned long           gnode_upper;
         unsigned long           lowmem_remap_top;
         unsigned long           lowmem_remap_base;
@@ -159,7 +160,8 @@ DECLARE_PER_CPU(struct uv_hub_info_s, __uv_hub_info);
  *              p -  PNODE (local part of nsids, right shifted 1)
  */
 #define UV_NASID_TO_PNODE(n)            (((n) >> 1) & uv_hub_info->pnode_mask)
-#define UV_PNODE_TO_NASID(p)            (((p) << 1) | uv_hub_info->gnode_upper)
+#define UV_PNODE_TO_GNODE(p)            ((p) |uv_hub_info->gnode_extra)
+#define UV_PNODE_TO_NASID(p)            (UV_PNODE_TO_GNODE(p) << 1)
 
 #define UV_LOCAL_MMR_BASE               0xf4000000UL
 #define UV_GLOBAL_MMR32_BASE            0xf8000000UL
@@ -173,7 +175,7 @@ DECLARE_PER_CPU(struct uv_hub_info_s, __uv_hub_info);
 #define UV_GLOBAL_MMR32_PNODE_BITS(p)   ((p) << (UV_GLOBAL_MMR32_PNODE_SHIFT))
 
 #define UV_GLOBAL_MMR64_PNODE_BITS(p)                                   \
-        ((unsigned long)(p) << UV_GLOBAL_MMR64_PNODE_SHIFT)
+        ((unsigned long)(UV_PNODE_TO_GNODE(p)) << UV_GLOBAL_MMR64_PNODE_SHIFT)
 
 #define UV_APIC_PNODE_SHIFT     6
 
diff --git a/arch/x86/include/asm/vmx.h b/arch/x86/include/asm/vmx.h
index 498f944010b9..11be5ad2e0e9 100644
--- a/arch/x86/include/asm/vmx.h
+++ b/arch/x86/include/asm/vmx.h
@@ -247,6 +247,7 @@ enum vmcs_field {
 #define EXIT_REASON_MSR_READ            31
 #define EXIT_REASON_MSR_WRITE           32
 #define EXIT_REASON_MWAIT_INSTRUCTION   36
+#define EXIT_REASON_MCE_DURING_VMENTRY   41
 #define EXIT_REASON_TPR_BELOW_THRESHOLD 43
 #define EXIT_REASON_APIC_ACCESS         44
 #define EXIT_REASON_EPT_VIOLATION       48
diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile
index 235f5927bb97..4f78bd682125 100644
--- a/arch/x86/kernel/Makefile
+++ b/arch/x86/kernel/Makefile
@@ -44,6 +44,7 @@ obj-y				+= process.o
 obj-y                           += i387.o xsave.o
 obj-y                           += ptrace.o
 obj-$(CONFIG_X86_DS)            += ds.o
+obj-$(CONFIG_X86_DS_SELFTEST)           += ds_selftest.o
 obj-$(CONFIG_X86_32)            += tls.o
 obj-$(CONFIG_IA32_EMULATION)    += tls.o
 obj-y                           += step.o
diff --git a/arch/x86/kernel/acpi/boot.c b/arch/x86/kernel/acpi/boot.c
index 844e5e25213b..631086159c53 100644
--- a/arch/x86/kernel/acpi/boot.c
+++ b/arch/x86/kernel/acpi/boot.c
@@ -985,11 +985,8 @@ void __init mp_register_ioapic(int id, u32 address, u32 gsi_base)
 
         set_fixmap_nocache(FIX_IO_APIC_BASE_0 + idx, address);
         mp_ioapics[idx].apicid = uniq_ioapic_id(id);
-#ifdef CONFIG_X86_32
         mp_ioapics[idx].apicver = io_apic_get_version(idx);
-#else
-        mp_ioapics[idx].apicver = 0;
-#endif
+
         /*
          * Build basic GSI lookup table to facilitate gsi->io_apic lookups
          * and to prevent reprogramming of IOAPIC pins (PCI GSIs).
diff --git a/arch/x86/kernel/acpi/realmode/Makefile b/arch/x86/kernel/acpi/realmode/Makefile
index 1c31cc0e9def..167bc16ce0e5 100644
--- a/arch/x86/kernel/acpi/realmode/Makefile
+++ b/arch/x86/kernel/acpi/realmode/Makefile
@@ -9,7 +9,7 @@
 always          := wakeup.bin
 targets         := wakeup.elf wakeup.lds
 
-wakeup-y        += wakeup.o wakemain.o video-mode.o copy.o
+wakeup-y        += wakeup.o wakemain.o video-mode.o copy.o bioscall.o regs.o
 
 # The link order of the video-*.o modules can matter.  In particular,
 # video-vga.o *must* be listed first, followed by video-vesa.o.
diff --git a/arch/x86/kernel/acpi/realmode/bioscall.S b/arch/x86/kernel/acpi/realmode/bioscall.S
new file mode 100644
index 000000000000..f51eb0bb56ce
--- /dev/null
+++ b/arch/x86/kernel/acpi/realmode/bioscall.S
@@ -0,0 +1 @@
+#include "../../../boot/bioscall.S"
diff --git a/arch/x86/kernel/acpi/realmode/regs.c b/arch/x86/kernel/acpi/realmode/regs.c
new file mode 100644
index 000000000000..6206033ba202
--- /dev/null
+++ b/arch/x86/kernel/acpi/realmode/regs.c
@@ -0,0 +1 @@
+#include "../../../boot/regs.c"
diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c
index a97db99dad52..1c60554537c3 100644
--- a/arch/x86/kernel/amd_iommu.c
+++ b/arch/x86/kernel/amd_iommu.c
@@ -55,7 +55,16 @@ struct iommu_cmd {
 static int dma_ops_unity_map(struct dma_ops_domain *dma_dom,
                              struct unity_map_entry *e);
 static struct dma_ops_domain *find_protection_domain(u16 devid);
+static u64* alloc_pte(struct protection_domain *dom,
+                      unsigned long address, u64
+                      **pte_page, gfp_t gfp);
+static void dma_ops_reserve_addresses(struct dma_ops_domain *dom,
+                                      unsigned long start_page,
+                                      unsigned int pages);
 
+#ifndef BUS_NOTIFY_UNBOUND_DRIVER
+#define BUS_NOTIFY_UNBOUND_DRIVER 0x0005
+#endif
 
 #ifdef CONFIG_AMD_IOMMU_STATS
 
@@ -213,7 +222,7 @@ irqreturn_t amd_iommu_int_handler(int irq, void *data)
 {
         struct amd_iommu *iommu;
 
-        list_for_each_entry(iommu, &amd_iommu_list, list)
+        for_each_iommu(iommu)
                 iommu_poll_events(iommu);
 
         return IRQ_HANDLED;
@@ -440,7 +449,7 @@ static void iommu_flush_domain(u16 domid)
         __iommu_build_inv_iommu_pages(&cmd, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,
                                       domid, 1, 1);
 
-        list_for_each_entry(iommu, &amd_iommu_list, list) {
+        for_each_iommu(iommu) {
                 spin_lock_irqsave(&iommu->lock, flags);
                 __iommu_queue_command(iommu, &cmd);
                 __iommu_completion_wait(iommu);
@@ -449,6 +458,35 @@ static void iommu_flush_domain(u16 domid)
         }
 }
 
+void amd_iommu_flush_all_domains(void)
+{
+        int i;
+
+        for (i = 1; i < MAX_DOMAIN_ID; ++i) {
+                if (!test_bit(i, amd_iommu_pd_alloc_bitmap))
+                        continue;
+                iommu_flush_domain(i);
+        }
+}
+
+void amd_iommu_flush_all_devices(void)
+{
+        struct amd_iommu *iommu;
+        int i;
+
+        for (i = 0; i <= amd_iommu_last_bdf; ++i) {
+                if (amd_iommu_pd_table[i] == NULL)
+                        continue;
+
+                iommu = amd_iommu_rlookup_table[i];
+                if (!iommu)
+                        continue;
+
+                iommu_queue_inv_dev_entry(iommu, i);
+                iommu_completion_wait(iommu);
+        }
+}
+
 /****************************************************************************
  *
  * The functions below are used the create the page table mappings for
@@ -468,7 +506,7 @@ static int iommu_map_page(struct protection_domain *dom,
                           unsigned long phys_addr,
                           int prot)
 {
-        u64 __pte, *pte, *page;
+        u64 __pte, *pte;
 
         bus_addr  = PAGE_ALIGN(bus_addr);
         phys_addr = PAGE_ALIGN(phys_addr);
@@ -477,27 +515,7 @@ static int iommu_map_page(struct protection_domain *dom,
         if (bus_addr > IOMMU_MAP_SIZE_L3 || !(prot & IOMMU_PROT_MASK))
                 return -EINVAL;
 
-        pte = &dom->pt_root[IOMMU_PTE_L2_INDEX(bus_addr)];
-
-        if (!IOMMU_PTE_PRESENT(*pte)) {
-                page = (u64 *)get_zeroed_page(GFP_KERNEL);
-                if (!page)
-                        return -ENOMEM;
-                *pte = IOMMU_L2_PDE(virt_to_phys(page));
-        }
-
-        pte = IOMMU_PTE_PAGE(*pte);
-        pte = &pte[IOMMU_PTE_L1_INDEX(bus_addr)];
-
-        if (!IOMMU_PTE_PRESENT(*pte)) {
-                page = (u64 *)get_zeroed_page(GFP_KERNEL);
-                if (!page)
-                        return -ENOMEM;
-                *pte = IOMMU_L1_PDE(virt_to_phys(page));
-        }
-
-        pte = IOMMU_PTE_PAGE(*pte);
-        pte = &pte[IOMMU_PTE_L0_INDEX(bus_addr)];
+        pte = alloc_pte(dom, bus_addr, NULL, GFP_KERNEL);
 
         if (IOMMU_PTE_PRESENT(*pte))
                 return -EBUSY;
@@ -595,7 +613,8 @@ static int dma_ops_unity_map(struct dma_ops_domain *dma_dom,
                  * as allocated in the aperture
                  */
                 if (addr < dma_dom->aperture_size)
-                        __set_bit(addr >> PAGE_SHIFT, dma_dom->bitmap);
+                        __set_bit(addr >> PAGE_SHIFT,
+                                  dma_dom->aperture[0]->bitmap);
         }
 
         return 0;
@@ -632,42 +651,191 @@ static int init_unity_mappings_for_device(struct dma_ops_domain *dma_dom,
  ****************************************************************************/
 
 /*
- * The address allocator core function.
+ * The address allocator core functions.
  *
  * called with domain->lock held
  */
+
+/*
+ * This function checks if there is a PTE for a given dma address. If
+ * there is one, it returns the pointer to it.
+ */
+static u64* fetch_pte(struct protection_domain *domain,
+                      unsigned long address)
+{
+        u64 *pte;
+
+        pte = &domain->pt_root[IOMMU_PTE_L2_INDEX(address)];
+
+        if (!IOMMU_PTE_PRESENT(*pte))
+                return NULL;
+
+        pte = IOMMU_PTE_PAGE(*pte);
+        pte = &pte[IOMMU_PTE_L1_INDEX(address)];
+
+        if (!IOMMU_PTE_PRESENT(*pte))
+                return NULL;
+
+        pte = IOMMU_PTE_PAGE(*pte);
+        pte = &pte[IOMMU_PTE_L0_INDEX(address)];
+
+        return pte;
+}
+
+/*
+ * This function is used to add a new aperture range to an existing
+ * aperture in case of dma_ops domain allocation or address allocation
+ * failure.
+ */
+static int alloc_new_range(struct amd_iommu *iommu,
+                           struct dma_ops_domain *dma_dom,
+                           bool populate, gfp_t gfp)
+{
+        int index = dma_dom->aperture_size >> APERTURE_RANGE_SHIFT;
+        int i;
+
+#ifdef CONFIG_IOMMU_STRESS
+        populate = false;
+#endif
+
+        if (index >= APERTURE_MAX_RANGES)
+                return -ENOMEM;
+
+        dma_dom->aperture[index] = kzalloc(sizeof(struct aperture_range), gfp);
+        if (!dma_dom->aperture[index])
+                return -ENOMEM;
+
+        dma_dom->aperture[index]->bitmap = (void *)get_zeroed_page(gfp);
+        if (!dma_dom->aperture[index]->bitmap)
+                goto out_free;
+
+        dma_dom->aperture[index]->offset = dma_dom->aperture_size;
+
+        if (populate) {
+                unsigned long address = dma_dom->aperture_size;
+                int i, num_ptes = APERTURE_RANGE_PAGES / 512;
+                u64 *pte, *pte_page;
+
+                for (i = 0; i < num_ptes; ++i) {
+                        pte = alloc_pte(&dma_dom->domain, address,
+                                        &pte_page, gfp);
+                        if (!pte)
+                                goto out_free;
+
+                        dma_dom->aperture[index]->pte_pages[i] = pte_page;
+
+                        address += APERTURE_RANGE_SIZE / 64;
+                }
+        }
+
+        dma_dom->aperture_size += APERTURE_RANGE_SIZE;
+
+        /* Intialize the exclusion range if necessary */
+        if (iommu->exclusion_start &&
+            iommu->exclusion_start >= dma_dom->aperture[index]->offset &&
+            iommu->exclusion_start < dma_dom->aperture_size) {
+                unsigned long startpage = iommu->exclusion_start >> PAGE_SHIFT;
+                int pages = iommu_num_pages(iommu->exclusion_start,
+                                            iommu->exclusion_length,
+                                            PAGE_SIZE);
+                dma_ops_reserve_addresses(dma_dom, startpage, pages);
+        }
+
+        /*
+         * Check for areas already mapped as present in the new aperture
+         * range and mark those pages as reserved in the allocator. Such
+         * mappings may already exist as a result of requested unity
+         * mappings for devices.
+         */
+        for (i = dma_dom->aperture[index]->offset;
+             i < dma_dom->aperture_size;
+             i += PAGE_SIZE) {
+                u64 *pte = fetch_pte(&dma_dom->domain, i);
+                if (!pte || !IOMMU_PTE_PRESENT(*pte))
+                        continue;
+
+                dma_ops_reserve_addresses(dma_dom, i << PAGE_SHIFT, 1);
+        }
+
+        return 0;
+
+out_free:
+        free_page((unsigned long)dma_dom->aperture[index]->bitmap);
+
+        kfree(dma_dom->aperture[index]);
+        dma_dom->aperture[index] = NULL;
+
+        return -ENOMEM;
+}
+
+static unsigned long dma_ops_area_alloc(struct device *dev,
+                                        struct dma_ops_domain *dom,
+                                        unsigned int pages,
+                                        unsigned long align_mask,
+                                        u64 dma_mask,
+                                        unsigned long start)
+{
+        unsigned long next_bit = dom->next_address % APERTURE_RANGE_SIZE;
+        int max_index = dom->aperture_size >> APERTURE_RANGE_SHIFT;
+        int i = start >> APERTURE_RANGE_SHIFT;
+        unsigned long boundary_size;
+        unsigned long address = -1;
+        unsigned long limit;
+
+        next_bit >>= PAGE_SHIFT;
+
+        boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,
+                        PAGE_SIZE) >> PAGE_SHIFT;
+
+        for (;i < max_index; ++i) {
+                unsigned long offset = dom->aperture[i]->offset >> PAGE_SHIFT;
+
+                if (dom->aperture[i]->offset >= dma_mask)
+                        break;
+
+                limit = iommu_device_max_index(APERTURE_RANGE_PAGES, offset,
+                                               dma_mask >> PAGE_SHIFT);
+
+                address = iommu_area_alloc(dom->aperture[i]->bitmap,
+                                           limit, next_bit, pages, 0,
+                                            boundary_size, align_mask);
+                if (address != -1) {
+                        address = dom->aperture[i]->offset +
+                                  (address << PAGE_SHIFT);
+                        dom->next_address = address + (pages << PAGE_SHIFT);
+                        break;
+                }
+
+                next_bit = 0;
+        }
+
+        return address;
+}
+
 static unsigned long dma_ops_alloc_addresses(struct device *dev,
                                              struct dma_ops_domain *dom,
                                              unsigned int pages,
                                              unsigned long align_mask,
                                              u64 dma_mask)
 {
-        unsigned long limit;
         unsigned long address;
-        unsigned long boundary_size;
 
-        boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,
-                        PAGE_SIZE) >> PAGE_SHIFT;
-        limit = iommu_device_max_index(dom->aperture_size >> PAGE_SHIFT, 0,
-                                       dma_mask >> PAGE_SHIFT);
+#ifdef CONFIG_IOMMU_STRESS
+        dom->next_address = 0;
+        dom->need_flush = true;
+#endif
 
-        if (dom->next_bit >= limit) {
-                dom->next_bit = 0;
-                dom->need_flush = true;
-        }
+        address = dma_ops_area_alloc(dev, dom, pages, align_mask,
+                                     dma_mask, dom->next_address);
 
-        address = iommu_area_alloc(dom->bitmap, limit, dom->next_bit, pages,
-                                   0 , boundary_size, align_mask);
         if (address == -1) {
-                address = iommu_area_alloc(dom->bitmap, limit, 0, pages,
-                                0, boundary_size, align_mask);
+                dom->next_address = 0;
+                address = dma_ops_area_alloc(dev, dom, pages, align_mask,
+                                             dma_mask, 0);
                 dom->need_flush = true;
         }
 
-        if (likely(address != -1)) {
-                dom->next_bit = address + pages;
-                address <<= PAGE_SHIFT;
-        } else
+        if (unlikely(address == -1))
                 address = bad_dma_address;
 
         WARN_ON((address + (PAGE_SIZE*pages)) > dom->aperture_size);
@@ -684,11 +852,23 @@ static void dma_ops_free_addresses(struct dma_ops_domain *dom,
                                    unsigned long address,
                                    unsigned int pages)
 {
-        address >>= PAGE_SHIFT;
-        iommu_area_free(dom->bitmap, address, pages);
+        unsigned i = address >> APERTURE_RANGE_SHIFT;
+        struct aperture_range *range = dom->aperture[i];
 
-        if (address >= dom->next_bit)
+        BUG_ON(i >= APERTURE_MAX_RANGES || range == NULL);
+
+#ifdef CONFIG_IOMMU_STRESS
+        if (i < 4)
+                return;
+#endif
+
+        if (address >= dom->next_address)
                 dom->need_flush = true;
+
+        address = (address % APERTURE_RANGE_SIZE) >> PAGE_SHIFT;
+
+        iommu_area_free(range->bitmap, address, pages);
+
 }
 
 /****************************************************************************
@@ -736,12 +916,16 @@ static void dma_ops_reserve_addresses(struct dma_ops_domain *dom,
                                       unsigned long start_page,
                                       unsigned int pages)
 {
-        unsigned int last_page = dom->aperture_size >> PAGE_SHIFT;
+        unsigned int i, last_page = dom->aperture_size >> PAGE_SHIFT;
 
         if (start_page + pages > last_page)
                 pages = last_page - start_page;
 
-        iommu_area_reserve(dom->bitmap, start_page, pages);
+        for (i = start_page; i < start_page + pages; ++i) {
+                int index = i / APERTURE_RANGE_PAGES;
+                int page  = i % APERTURE_RANGE_PAGES;
+                __set_bit(page, dom->aperture[index]->bitmap);
+        }
 }
 
 static void free_pagetable(struct protection_domain *domain)
@@ -780,14 +964,19 @@ static void free_pagetable(struct protection_domain *domain)
  */
 static void dma_ops_domain_free(struct dma_ops_domain *dom)
 {
+        int i;
+
         if (!dom)
                 return;
 
         free_pagetable(&dom->domain);
 
-        kfree(dom->pte_pages);
-
-        kfree(dom->bitmap);
+        for (i = 0; i < APERTURE_MAX_RANGES; ++i) {
+                if (!dom->aperture[i])
+                        continue;
+                free_page((unsigned long)dom->aperture[i]->bitmap);
+                kfree(dom->aperture[i]);
+        }
 
         kfree(dom);
 }
@@ -797,19 +986,9 @@ static void dma_ops_domain_free(struct dma_ops_domain *dom)
  * It also intializes the page table and the address allocator data
  * structures required for the dma_ops interface
  */
-static struct dma_ops_domain *dma_ops_domain_alloc(struct amd_iommu *iommu,
-                                                   unsigned order)
+static struct dma_ops_domain *dma_ops_domain_alloc(struct amd_iommu *iommu)
 {
         struct dma_ops_domain *dma_dom;
-        unsigned i, num_pte_pages;
-        u64 *l2_pde;
-        u64 address;
-
-        /*
-         * Currently the DMA aperture must be between 32 MB and 1GB in size
-         */
-        if ((order < 25) || (order > 30))
-                return NULL;
 
         dma_dom = kzalloc(sizeof(struct dma_ops_domain), GFP_KERNEL);
         if (!dma_dom)
@@ -826,55 +1005,20 @@ static struct dma_ops_domain *dma_ops_domain_alloc(struct amd_iommu *iommu,
         dma_dom->domain.priv = dma_dom;
         if (!dma_dom->domain.pt_root)
                 goto free_dma_dom;
-        dma_dom->aperture_size = (1ULL << order);
-        dma_dom->bitmap = kzalloc(dma_dom->aperture_size / (PAGE_SIZE * 8),
-                                  GFP_KERNEL);
-        if (!dma_dom->bitmap)
-                goto free_dma_dom;
-        /*
-         * mark the first page as allocated so we never return 0 as
-         * a valid dma-address. So we can use 0 as error value
-         */
-        dma_dom->bitmap[0] = 1;
-        dma_dom->next_bit = 0;
 
         dma_dom->need_flush = false;
         dma_dom->target_dev = 0xffff;
 
-        /* Intialize the exclusion range if necessary */
-        if (iommu->exclusion_start &&
-            iommu->exclusion_start < dma_dom->aperture_size) {
-                unsigned long startpage = iommu->exclusion_start >> PAGE_SHIFT;
-                int pages = iommu_num_pages(iommu->exclusion_start,
-                                            iommu->exclusion_length,
-                                            PAGE_SIZE);
-                dma_ops_reserve_addresses(dma_dom, startpage, pages);
-        }
+        if (alloc_new_range(iommu, dma_dom, true, GFP_KERNEL))
+                goto free_dma_dom;
 
         /*
-         * At the last step, build the page tables so we don't need to
-         * allocate page table pages in the dma_ops mapping/unmapping
-         * path.
+         * mark the first page as allocated so we never return 0 as
+         * a valid dma-address. So we can use 0 as error value
          */
-        num_pte_pages = dma_dom->aperture_size / (PAGE_SIZE * 512);
-        dma_dom->pte_pages = kzalloc(num_pte_pages * sizeof(void *),
-                        GFP_KERNEL);
-        if (!dma_dom->pte_pages)
-                goto free_dma_dom;
-
-        l2_pde = (u64 *)get_zeroed_page(GFP_KERNEL);
-        if (l2_pde == NULL)
-                goto free_dma_dom;
+        dma_dom->aperture[0]->bitmap[0] = 1;
+        dma_dom->next_address = 0;
 
-        dma_dom->domain.pt_root[0] = IOMMU_L2_PDE(virt_to_phys(l2_pde));
-
-        for (i = 0; i < num_pte_pages; ++i) {
-                dma_dom->pte_pages[i] = (u64 *)get_zeroed_page(GFP_KERNEL);
-                if (!dma_dom->pte_pages[i])
-                        goto free_dma_dom;
-                address = virt_to_phys(dma_dom->pte_pages[i]);
-                l2_pde[i] = IOMMU_L1_PDE(address);
-        }
 
         return dma_dom;
 
@@ -983,7 +1127,6 @@ static int device_change_notifier(struct notifier_block *nb,
         struct protection_domain *domain;
         struct dma_ops_domain *dma_domain;
         struct amd_iommu *iommu;
-        int order = amd_iommu_aperture_order;
         unsigned long flags;
 
         if (devid > amd_iommu_last_bdf)
@@ -1002,17 +1145,7 @@ static int device_change_notifier(struct notifier_block *nb,
                           "to a non-dma-ops domain\n", dev_name(dev));
 
         switch (action) {
-        case BUS_NOTIFY_BOUND_DRIVER:
-                if (domain)
-                        goto out;
-                dma_domain = find_protection_domain(devid);
-                if (!dma_domain)
-                        dma_domain = iommu->default_dom;
-                attach_device(iommu, &dma_domain->domain, devid);
-                printk(KERN_INFO "AMD IOMMU: Using protection domain %d for "
-                       "device %s\n", dma_domain->domain.id, dev_name(dev));
-                break;
-        case BUS_NOTIFY_UNBIND_DRIVER:
+        case BUS_NOTIFY_UNBOUND_DRIVER:
                 if (!domain)
                         goto out;
                 detach_device(domain, devid);
@@ -1022,7 +1155,7 @@ static int device_change_notifier(struct notifier_block *nb,
                 dma_domain = find_protection_domain(devid);
                 if (dma_domain)
                         goto out;
-                dma_domain = dma_ops_domain_alloc(iommu, order);
+                dma_domain = dma_ops_domain_alloc(iommu);
                 if (!dma_domain)
                         goto out;
                 dma_domain->target_dev = devid;
@@ -1133,8 +1266,8 @@ static int get_device_resources(struct device *dev,
                         dma_dom = (*iommu)->default_dom;
                 *domain = &dma_dom->domain;
                 attach_device(*iommu, *domain, *bdf);
-                printk(KERN_INFO "AMD IOMMU: Using protection domain %d for "
-                                "device %s\n", (*domain)->id, dev_name(dev));
+                DUMP_printk("Using protection domain %d for device %s\n",
+                            (*domain)->id, dev_name(dev));
         }
 
         if (domain_for_device(_bdf) == NULL)
@@ -1144,6 +1277,66 @@ static int get_device_resources(struct device *dev,
 }
 
 /*
+ * If the pte_page is not yet allocated this function is called
+ */
+static u64* alloc_pte(struct protection_domain *dom,
+                      unsigned long address, u64 **pte_page, gfp_t gfp)
+{
+        u64 *pte, *page;
+
+        pte = &dom->pt_root[IOMMU_PTE_L2_INDEX(address)];
+
+        if (!IOMMU_PTE_PRESENT(*pte)) {
+                page = (u64 *)get_zeroed_page(gfp);
+                if (!page)
+                        return NULL;
+                *pte = IOMMU_L2_PDE(virt_to_phys(page));
+        }
+
+        pte = IOMMU_PTE_PAGE(*pte);
+        pte = &pte[IOMMU_PTE_L1_INDEX(address)];
+
+        if (!IOMMU_PTE_PRESENT(*pte)) {
+                page = (u64 *)get_zeroed_page(gfp);
+                if (!page)
+                        return NULL;
+                *pte = IOMMU_L1_PDE(virt_to_phys(page));
+        }
+
+        pte = IOMMU_PTE_PAGE(*pte);
+
+        if (pte_page)
+                *pte_page = pte;
+
+        pte = &pte[IOMMU_PTE_L0_INDEX(address)];
+
+        return pte;
+}
+
+/*
+ * This function fetches the PTE for a given address in the aperture
+ */
+static u64* dma_ops_get_pte(struct dma_ops_domain *dom,
+                            unsigned long address)
+{
+        struct aperture_range *aperture;
+        u64 *pte, *pte_page;
+
+        aperture = dom->aperture[APERTURE_RANGE_INDEX(address)];
+        if (!aperture)
+                return NULL;
+
+        pte = aperture->pte_pages[APERTURE_PAGE_INDEX(address)];
+        if (!pte) {
+                pte = alloc_pte(&dom->domain, address, &pte_page, GFP_ATOMIC);
+                aperture->pte_pages[APERTURE_PAGE_INDEX(address)] = pte_page;
+        } else
+                pte += IOMMU_PTE_L0_INDEX(address);
+
+        return pte;
+}
+
+/*
  * This is the generic map function. It maps one 4kb page at paddr to
  * the given address in the DMA address space for the domain.
  */
@@ -1159,8 +1352,9 @@ static dma_addr_t dma_ops_domain_map(struct amd_iommu *iommu,
 
         paddr &= PAGE_MASK;
 
-        pte  = dom->pte_pages[IOMMU_PTE_L1_INDEX(address)];
-        pte += IOMMU_PTE_L0_INDEX(address);
+        pte  = dma_ops_get_pte(dom, address);
+        if (!pte)
+                return bad_dma_address;
 
         __pte = paddr | IOMMU_PTE_P | IOMMU_PTE_FC;
 
@@ -1185,14 +1379,20 @@ static void dma_ops_domain_unmap(struct amd_iommu *iommu,
                                  struct dma_ops_domain *dom,
                                  unsigned long address)
 {
+        struct aperture_range *aperture;
         u64 *pte;
 
         if (address >= dom->aperture_size)
                 return;
 
-        WARN_ON(address & ~PAGE_MASK || address >= dom->aperture_size);
+        aperture = dom->aperture[APERTURE_RANGE_INDEX(address)];
+        if (!aperture)
+                return;
+
+        pte  = aperture->pte_pages[APERTURE_PAGE_INDEX(address)];
+        if (!pte)
+                return;
 
-        pte  = dom->pte_pages[IOMMU_PTE_L1_INDEX(address)];
         pte += IOMMU_PTE_L0_INDEX(address);
 
         WARN_ON(!*pte);
@@ -1216,7 +1416,7 @@ static dma_addr_t __map_single(struct device *dev,
                                u64 dma_mask)
 {
         dma_addr_t offset = paddr & ~PAGE_MASK;
-        dma_addr_t address, start;
+        dma_addr_t address, start, ret;
         unsigned int pages;
         unsigned long align_mask = 0;
         int i;
@@ -1232,14 +1432,33 @@ static dma_addr_t __map_single(struct device *dev,
         if (align)
                 align_mask = (1UL << get_order(size)) - 1;
 
+retry:
         address = dma_ops_alloc_addresses(dev, dma_dom, pages, align_mask,
                                           dma_mask);
-        if (unlikely(address == bad_dma_address))
-                goto out;
+        if (unlikely(address == bad_dma_address)) {
+                /*
+                 * setting next_address here will let the address
+                 * allocator only scan the new allocated range in the
+                 * first run. This is a small optimization.
+                 */
+                dma_dom->next_address = dma_dom->aperture_size;
+
+                if (alloc_new_range(iommu, dma_dom, false, GFP_ATOMIC))
+                        goto out;
+
+                /*
+                 * aperture was sucessfully enlarged by 128 MB, try
+                 * allocation again
+                 */
+                goto retry;
+        }
 
         start = address;
         for (i = 0; i < pages; ++i) {
-                dma_ops_domain_map(iommu, dma_dom, start, paddr, dir);
+                ret = dma_ops_domain_map(iommu, dma_dom, start, paddr, dir);
+                if (ret == bad_dma_address)
+                        goto out_unmap;
+
                 paddr += PAGE_SIZE;
                 start += PAGE_SIZE;
         }
@@ -1255,6 +1474,17 @@ static dma_addr_t __map_single(struct device *dev,
 
 out:
         return address;
+
+out_unmap:
+
+        for (--i; i >= 0; --i) {
+                start -= PAGE_SIZE;
+                dma_ops_domain_unmap(iommu, dma_dom, start);
+        }
+
+        dma_ops_free_addresses(dma_dom, address, pages);
+
+        return bad_dma_address;
 }
 
 /*
@@ -1537,8 +1767,10 @@ static void *alloc_coherent(struct device *dev, size_t size,
         *dma_addr = __map_single(dev, iommu, domain->priv, paddr,
                                  size, DMA_BIDIRECTIONAL, true, dma_mask);
 
-        if (*dma_addr == bad_dma_address)
+        if (*dma_addr == bad_dma_address) {
+                spin_unlock_irqrestore(&domain->lock, flags);
                 goto out_free;
+        }
 
         iommu_completion_wait(iommu);
 
@@ -1625,7 +1857,6 @@ static void prealloc_protection_domains(void)
         struct pci_dev *dev = NULL;
         struct dma_ops_domain *dma_dom;
         struct amd_iommu *iommu;
-        int order = amd_iommu_aperture_order;
         u16 devid;
 
         while ((dev = pci_get_device(PCI_ANY_ID, PCI_ANY_ID, dev)) != NULL) {
@@ -1638,7 +1869,7 @@ static void prealloc_protection_domains(void)
                 iommu = amd_iommu_rlookup_table[devid];
                 if (!iommu)
                         continue;
-                dma_dom = dma_ops_domain_alloc(iommu, order);
+                dma_dom = dma_ops_domain_alloc(iommu);
                 if (!dma_dom)
                         continue;
                 init_unity_mappings_for_device(dma_dom, devid);
@@ -1664,7 +1895,6 @@ static struct dma_map_ops amd_iommu_dma_ops = {
 int __init amd_iommu_init_dma_ops(void)
 {
         struct amd_iommu *iommu;
-        int order = amd_iommu_aperture_order;
         int ret;
 
         /*
@@ -1672,8 +1902,8 @@ int __init amd_iommu_init_dma_ops(void)
          * found in the system. Devices not assigned to any other
          * protection domain will be assigned to the default one.
          */
-        list_for_each_entry(iommu, &amd_iommu_list, list) {
-                iommu->default_dom = dma_ops_domain_alloc(iommu, order);
+        for_each_iommu(iommu) {
+                iommu->default_dom = dma_ops_domain_alloc(iommu);
                 if (iommu->default_dom == NULL)
                         return -ENOMEM;
                 iommu->default_dom->domain.flags |= PD_DEFAULT_MASK;
@@ -1710,7 +1940,7 @@ int __init amd_iommu_init_dma_ops(void)
 
 free_domains:
 
-        list_for_each_entry(iommu, &amd_iommu_list, list) {
+        for_each_iommu(iommu) {
                 if (iommu->default_dom)
                         dma_ops_domain_free(iommu->default_dom);
         }
@@ -1842,7 +2072,7 @@ static int amd_iommu_attach_device(struct iommu_domain *dom,
 
         old_domain = domain_for_device(devid);
         if (old_domain)
-                return -EBUSY;
+                detach_device(old_domain, devid);
 
         attach_device(iommu, domain, devid);
 
diff --git a/arch/x86/kernel/amd_iommu_init.c b/arch/x86/kernel/amd_iommu_init.c
index 8c0be0902dac..238989ec077d 100644
--- a/arch/x86/kernel/amd_iommu_init.c
+++ b/arch/x86/kernel/amd_iommu_init.c
@@ -115,15 +115,21 @@ struct ivmd_header {
         u64 range_length;
 } __attribute__((packed));
 
+bool amd_iommu_dump;
+
 static int __initdata amd_iommu_detected;
 
 u16 amd_iommu_last_bdf;                 /* largest PCI device id we have
                                            to handle */
 LIST_HEAD(amd_iommu_unity_map);         /* a list of required unity mappings
                                            we find in ACPI */
-unsigned amd_iommu_aperture_order = 26; /* size of aperture in power of 2 */
+#ifdef CONFIG_IOMMU_STRESS
+bool amd_iommu_isolate = false;
+#else
 bool amd_iommu_isolate = true;          /* if true, device isolation is
                                            enabled */
+#endif
+
 bool amd_iommu_unmap_flush;             /* if true, flush on every unmap */
 
 LIST_HEAD(amd_iommu_list);              /* list of all AMD IOMMUs in the
@@ -175,7 +181,7 @@ static inline void update_last_devid(u16 devid)
 static inline unsigned long tbl_size(int entry_size)
 {
         unsigned shift = PAGE_SHIFT +
-                         get_order(amd_iommu_last_bdf * entry_size);
+                         get_order(((int)amd_iommu_last_bdf + 1) * entry_size);
 
         return 1UL << shift;
 }
@@ -193,7 +199,7 @@ static inline unsigned long tbl_size(int entry_size)
  * This function set the exclusion range in the IOMMU. DMA accesses to the
  * exclusion range are passed through untranslated
  */
-static void __init iommu_set_exclusion_range(struct amd_iommu *iommu)
+static void iommu_set_exclusion_range(struct amd_iommu *iommu)
 {
         u64 start = iommu->exclusion_start & PAGE_MASK;
         u64 limit = (start + iommu->exclusion_length) & PAGE_MASK;
@@ -225,7 +231,7 @@ static void __init iommu_set_device_table(struct amd_iommu *iommu)
 }
 
 /* Generic functions to enable/disable certain features of the IOMMU. */
-static void __init iommu_feature_enable(struct amd_iommu *iommu, u8 bit)
+static void iommu_feature_enable(struct amd_iommu *iommu, u8 bit)
 {
         u32 ctrl;
 
@@ -244,7 +250,7 @@ static void __init iommu_feature_disable(struct amd_iommu *iommu, u8 bit)
 }
 
 /* Function to enable the hardware */
-static void __init iommu_enable(struct amd_iommu *iommu)
+static void iommu_enable(struct amd_iommu *iommu)
 {
         printk(KERN_INFO "AMD IOMMU: Enabling IOMMU at %s cap 0x%hx\n",
                dev_name(&iommu->dev->dev), iommu->cap_ptr);
@@ -252,11 +258,9 @@ static void __init iommu_enable(struct amd_iommu *iommu)
         iommu_feature_enable(iommu, CONTROL_IOMMU_EN);
 }
 
-/* Function to enable IOMMU event logging and event interrupts */
-static void __init iommu_enable_event_logging(struct amd_iommu *iommu)
+static void iommu_disable(struct amd_iommu *iommu)
 {
-        iommu_feature_enable(iommu, CONTROL_EVT_LOG_EN);
-        iommu_feature_enable(iommu, CONTROL_EVT_INT_EN);
+        iommu_feature_disable(iommu, CONTROL_IOMMU_EN);
 }
 
 /*
@@ -413,25 +417,36 @@ static u8 * __init alloc_command_buffer(struct amd_iommu *iommu)
 {
         u8 *cmd_buf = (u8 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
                         get_order(CMD_BUFFER_SIZE));
-        u64 entry;
 
         if (cmd_buf == NULL)
                 return NULL;
 
         iommu->cmd_buf_size = CMD_BUFFER_SIZE;
 
-        entry = (u64)virt_to_phys(cmd_buf);
+        return cmd_buf;
+}
+
+/*
+ * This function writes the command buffer address to the hardware and
+ * enables it.
+ */
+static void iommu_enable_command_buffer(struct amd_iommu *iommu)
+{
+        u64 entry;
+
+        BUG_ON(iommu->cmd_buf == NULL);
+
+        entry = (u64)virt_to_phys(iommu->cmd_buf);
         entry |= MMIO_CMD_SIZE_512;
+
         memcpy_toio(iommu->mmio_base + MMIO_CMD_BUF_OFFSET,
-                        &entry, sizeof(entry));
+                    &entry, sizeof(entry));
 
         /* set head and tail to zero manually */
         writel(0x00, iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);
         writel(0x00, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
 
         iommu_feature_enable(iommu, CONTROL_CMDBUF_EN);
-
-        return cmd_buf;
 }
 
 static void __init free_command_buffer(struct amd_iommu *iommu)
@@ -443,20 +458,27 @@ static void __init free_command_buffer(struct amd_iommu *iommu)
 /* allocates the memory where the IOMMU will log its events to */
 static u8 * __init alloc_event_buffer(struct amd_iommu *iommu)
 {
-        u64 entry;
         iommu->evt_buf = (u8 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
                                                 get_order(EVT_BUFFER_SIZE));
 
         if (iommu->evt_buf == NULL)
                 return NULL;
 
+        return iommu->evt_buf;
+}
+
+static void iommu_enable_event_buffer(struct amd_iommu *iommu)
+{
+        u64 entry;
+
+        BUG_ON(iommu->evt_buf == NULL);
+
         entry = (u64)virt_to_phys(iommu->evt_buf) | EVT_LEN_MASK;
+
         memcpy_toio(iommu->mmio_base + MMIO_EVT_BUF_OFFSET,
                     &entry, sizeof(entry));
 
-        iommu->evt_buf_size = EVT_BUFFER_SIZE;
-
-        return iommu->evt_buf;
+        iommu_feature_enable(iommu, CONTROL_EVT_LOG_EN);
 }
 
 static void __init free_event_buffer(struct amd_iommu *iommu)
@@ -596,32 +618,83 @@ static void __init init_iommu_from_acpi(struct amd_iommu *iommu,
         p += sizeof(struct ivhd_header);
         end += h->length;
 
+
         while (p < end) {
                 e = (struct ivhd_entry *)p;
                 switch (e->type) {
                 case IVHD_DEV_ALL:
+
+                        DUMP_printk("  DEV_ALL\t\t\t first devid: %02x:%02x.%x"
+                                    " last device %02x:%02x.%x flags: %02x\n",
+                                    PCI_BUS(iommu->first_device),
+                                    PCI_SLOT(iommu->first_device),
+                                    PCI_FUNC(iommu->first_device),
+                                    PCI_BUS(iommu->last_device),
+                                    PCI_SLOT(iommu->last_device),
+                                    PCI_FUNC(iommu->last_device),
+                                    e->flags);
+
                         for (dev_i = iommu->first_device;
                                         dev_i <= iommu->last_device; ++dev_i)
                                 set_dev_entry_from_acpi(iommu, dev_i,
                                                         e->flags, 0);
                         break;
                 case IVHD_DEV_SELECT:
+
+                        DUMP_printk("  DEV_SELECT\t\t\t devid: %02x:%02x.%x "
+                                    "flags: %02x\n",
+                                    PCI_BUS(e->devid),
+                                    PCI_SLOT(e->devid),
+                                    PCI_FUNC(e->devid),
+                                    e->flags);
+
                         devid = e->devid;
                         set_dev_entry_from_acpi(iommu, devid, e->flags, 0);
                         break;
                 case IVHD_DEV_SELECT_RANGE_START:
+
+                        DUMP_printk("  DEV_SELECT_RANGE_START\t "
+                                    "devid: %02x:%02x.%x flags: %02x\n",
+                                    PCI_BUS(e->devid),
+                                    PCI_SLOT(e->devid),
+                                    PCI_FUNC(e->devid),
+                                    e->flags);
+
                         devid_start = e->devid;
                         flags = e->flags;
                         ext_flags = 0;
                         alias = false;
                         break;
                 case IVHD_DEV_ALIAS:
+
+                        DUMP_printk("  DEV_ALIAS\t\t\t devid: %02x:%02x.%x "
+                                    "flags: %02x devid_to: %02x:%02x.%x\n",
+                                    PCI_BUS(e->devid),
+                                    PCI_SLOT(e->devid),
+                                    PCI_FUNC(e->devid),
+                                    e->flags,
+                                    PCI_BUS(e->ext >> 8),
+                                    PCI_SLOT(e->ext >> 8),
+                                    PCI_FUNC(e->ext >> 8));
+
                         devid = e->devid;
                         devid_to = e->ext >> 8;
-                        set_dev_entry_from_acpi(iommu, devid, e->flags, 0);
+                        set_dev_entry_from_acpi(iommu, devid_to, e->flags, 0);
                         amd_iommu_alias_table[devid] = devid_to;
                         break;
                 case IVHD_DEV_ALIAS_RANGE:
+
+                        DUMP_printk("  DEV_ALIAS_RANGE\t\t "
+                                    "devid: %02x:%02x.%x flags: %02x "
+                                    "devid_to: %02x:%02x.%x\n",
+                                    PCI_BUS(e->devid),
+                                    PCI_SLOT(e->devid),
+                                    PCI_FUNC(e->devid),
+                                    e->flags,
+                                    PCI_BUS(e->ext >> 8),
+                                    PCI_SLOT(e->ext >> 8),
+                                    PCI_FUNC(e->ext >> 8));
+
                         devid_start = e->devid;
                         flags = e->flags;
                         devid_to = e->ext >> 8;
@@ -629,17 +702,39 @@ static void __init init_iommu_from_acpi(struct amd_iommu *iommu,
                         alias = true;
                         break;
                 case IVHD_DEV_EXT_SELECT:
+
+                        DUMP_printk("  DEV_EXT_SELECT\t\t devid: %02x:%02x.%x "
+                                    "flags: %02x ext: %08x\n",
+                                    PCI_BUS(e->devid),
+                                    PCI_SLOT(e->devid),
+                                    PCI_FUNC(e->devid),
+                                    e->flags, e->ext);
+
                         devid = e->devid;
                         set_dev_entry_from_acpi(iommu, devid, e->flags,
                                                 e->ext);
                         break;
                 case IVHD_DEV_EXT_SELECT_RANGE:
+
+                        DUMP_printk("  DEV_EXT_SELECT_RANGE\t devid: "
+                                    "%02x:%02x.%x flags: %02x ext: %08x\n",
+                                    PCI_BUS(e->devid),
+                                    PCI_SLOT(e->devid),
+                                    PCI_FUNC(e->devid),
+                                    e->flags, e->ext);
+
                         devid_start = e->devid;
                         flags = e->flags;
                         ext_flags = e->ext;
                         alias = false;
                         break;
                 case IVHD_DEV_RANGE_END:
+
+                        DUMP_printk("  DEV_RANGE_END\t\t devid: %02x:%02x.%x\n",
+                                    PCI_BUS(e->devid),
+                                    PCI_SLOT(e->devid),
+                                    PCI_FUNC(e->devid));
+
                         devid = e->devid;
                         for (dev_i = devid_start; dev_i <= devid; ++dev_i) {
                                 if (alias)
@@ -679,7 +774,7 @@ static void __init free_iommu_all(void)
 {
         struct amd_iommu *iommu, *next;
 
-        list_for_each_entry_safe(iommu, next, &amd_iommu_list, list) {
+        for_each_iommu_safe(iommu, next) {
                 list_del(&iommu->list);
                 free_iommu_one(iommu);
                 kfree(iommu);
@@ -710,7 +805,6 @@ static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h)
         if (!iommu->mmio_base)
                 return -ENOMEM;
 
-        iommu_set_device_table(iommu);
         iommu->cmd_buf = alloc_command_buffer(iommu);
         if (!iommu->cmd_buf)
                 return -ENOMEM;
@@ -746,6 +840,15 @@ static int __init init_iommu_all(struct acpi_table_header *table)
                 h = (struct ivhd_header *)p;
                 switch (*p) {
                 case ACPI_IVHD_TYPE:
+
+                        DUMP_printk("IOMMU: device: %02x:%02x.%01x cap: %04x "
+                                    "seg: %d flags: %01x info %04x\n",
+                                    PCI_BUS(h->devid), PCI_SLOT(h->devid),
+                                    PCI_FUNC(h->devid), h->cap_ptr,
+                                    h->pci_seg, h->flags, h->info);
+                        DUMP_printk("       mmio-addr: %016llx\n",
+                                    h->mmio_phys);
+
                         iommu = kzalloc(sizeof(struct amd_iommu), GFP_KERNEL);
                         if (iommu == NULL)
                                 return -ENOMEM;
@@ -773,56 +876,9 @@ static int __init init_iommu_all(struct acpi_table_header *table)
  *
  ****************************************************************************/
 
-static int __init iommu_setup_msix(struct amd_iommu *iommu)
-{
-        struct amd_iommu *curr;
-        struct msix_entry entries[32]; /* only 32 supported by AMD IOMMU */
-        int nvec = 0, i;
-
-        list_for_each_entry(curr, &amd_iommu_list, list) {
-                if (curr->dev == iommu->dev) {
-                        entries[nvec].entry = curr->evt_msi_num;
-                        entries[nvec].vector = 0;
-                        curr->int_enabled = true;
-                        nvec++;
-                }
-        }
-
-        if (pci_enable_msix(iommu->dev, entries, nvec)) {
-                pci_disable_msix(iommu->dev);
-                return 1;
-        }
-
-        for (i = 0; i < nvec; ++i) {
-                int r = request_irq(entries->vector, amd_iommu_int_handler,
-                                    IRQF_SAMPLE_RANDOM,
-                                    "AMD IOMMU",
-                                    NULL);
-                if (r)
-                        goto out_free;
-        }
-
-        return 0;
-
-out_free:
-        for (i -= 1; i >= 0; --i)
-                free_irq(entries->vector, NULL);
-
-        pci_disable_msix(iommu->dev);
-
-        return 1;
-}
-
 static int __init iommu_setup_msi(struct amd_iommu *iommu)
 {
         int r;
-        struct amd_iommu *curr;
-
-        list_for_each_entry(curr, &amd_iommu_list, list) {
-                if (curr->dev == iommu->dev)
-                        curr->int_enabled = true;
-        }
-
 
         if (pci_enable_msi(iommu->dev))
                 return 1;
@@ -837,17 +893,18 @@ static int __init iommu_setup_msi(struct amd_iommu *iommu)
                 return 1;
         }
 
+        iommu->int_enabled = true;
+        iommu_feature_enable(iommu, CONTROL_EVT_INT_EN);
+
         return 0;
 }
 
-static int __init iommu_init_msi(struct amd_iommu *iommu)
+static int iommu_init_msi(struct amd_iommu *iommu)
 {
         if (iommu->int_enabled)
                 return 0;
 
-        if (pci_find_capability(iommu->dev, PCI_CAP_ID_MSIX))
-                return iommu_setup_msix(iommu);
-        else if (pci_find_capability(iommu->dev, PCI_CAP_ID_MSI))
+        if (pci_find_capability(iommu->dev, PCI_CAP_ID_MSI))
                 return iommu_setup_msi(iommu);
 
         return 1;
@@ -899,6 +956,7 @@ static int __init init_exclusion_range(struct ivmd_header *m)
 static int __init init_unity_map_range(struct ivmd_header *m)
 {
         struct unity_map_entry *e = 0;
+        char *s;
 
         e = kzalloc(sizeof(*e), GFP_KERNEL);
         if (e == NULL)
@@ -906,14 +964,19 @@ static int __init init_unity_map_range(struct ivmd_header *m)
 
         switch (m->type) {
         default:
+                kfree(e);
+                return 0;
         case ACPI_IVMD_TYPE:
+                s = "IVMD_TYPEi\t\t\t";
                 e->devid_start = e->devid_end = m->devid;
                 break;
         case ACPI_IVMD_TYPE_ALL:
+                s = "IVMD_TYPE_ALL\t\t";
                 e->devid_start = 0;
                 e->devid_end = amd_iommu_last_bdf;
                 break;
         case ACPI_IVMD_TYPE_RANGE:
+                s = "IVMD_TYPE_RANGE\t\t";
                 e->devid_start = m->devid;
                 e->devid_end = m->aux;
                 break;
@@ -922,6 +985,13 @@ static int __init init_unity_map_range(struct ivmd_header *m)
         e->address_end = e->address_start + PAGE_ALIGN(m->range_length);
         e->prot = m->flags >> 1;
 
+        DUMP_printk("%s devid_start: %02x:%02x.%x devid_end: %02x:%02x.%x"
+                    " range_start: %016llx range_end: %016llx flags: %x\n", s,
+                    PCI_BUS(e->devid_start), PCI_SLOT(e->devid_start),
+                    PCI_FUNC(e->devid_start), PCI_BUS(e->devid_end),
+                    PCI_SLOT(e->devid_end), PCI_FUNC(e->devid_end),
+                    e->address_start, e->address_end, m->flags);
+
         list_add_tail(&e->list, &amd_iommu_unity_map);
 
         return 0;
@@ -967,18 +1037,28 @@ static void init_device_table(void)
  * This function finally enables all IOMMUs found in the system after
  * they have been initialized
  */
-static void __init enable_iommus(void)
+static void enable_iommus(void)
 {
         struct amd_iommu *iommu;
 
-        list_for_each_entry(iommu, &amd_iommu_list, list) {
+        for_each_iommu(iommu) {
+                iommu_set_device_table(iommu);
+                iommu_enable_command_buffer(iommu);
+                iommu_enable_event_buffer(iommu);
                 iommu_set_exclusion_range(iommu);
                 iommu_init_msi(iommu);
-                iommu_enable_event_logging(iommu);
                 iommu_enable(iommu);
         }
 }
 
+static void disable_iommus(void)
+{
+        struct amd_iommu *iommu;
+
+        for_each_iommu(iommu)
+                iommu_disable(iommu);
+}
+
 /*
  * Suspend/Resume support
  * disable suspend until real resume implemented
@@ -986,12 +1066,31 @@ static void __init enable_iommus(void)
 
 static int amd_iommu_resume(struct sys_device *dev)
 {
+        /*
+         * Disable IOMMUs before reprogramming the hardware registers.
+         * IOMMU is still enabled from the resume kernel.
+         */
+        disable_iommus();
+
+        /* re-load the hardware */
+        enable_iommus();
+
+        /*
+         * we have to flush after the IOMMUs are enabled because a
+         * disabled IOMMU will never execute the commands we send
+         */
+        amd_iommu_flush_all_domains();
+        amd_iommu_flush_all_devices();
+
         return 0;
 }
 
 static int amd_iommu_suspend(struct sys_device *dev, pm_message_t state)
 {
-        return -EINVAL;
+        /* disable IOMMUs to go out of the way for BIOS */
+        disable_iommus();
+
+        return 0;
 }
 
 static struct sysdev_class amd_iommu_sysdev_class = {
@@ -1137,9 +1236,6 @@ int __init amd_iommu_init(void)
 
         enable_iommus();
 
-        printk(KERN_INFO "AMD IOMMU: aperture size is %d MB\n",
-                        (1 << (amd_iommu_aperture_order-20)));
-
         printk(KERN_INFO "AMD IOMMU: device isolation ");
         if (amd_iommu_isolate)
                 printk("enabled\n");
@@ -1211,6 +1307,13 @@ void __init amd_iommu_detect(void)
  *
  ****************************************************************************/
 
+static int __init parse_amd_iommu_dump(char *str)
+{
+        amd_iommu_dump = true;
+
+        return 1;
+}
+
 static int __init parse_amd_iommu_options(char *str)
 {
         for (; *str; ++str) {
@@ -1225,15 +1328,5 @@ static int __init parse_amd_iommu_options(char *str)
         return 1;
 }
 
-static int __init parse_amd_iommu_size_options(char *str)
-{
-        unsigned order = PAGE_SHIFT + get_order(memparse(str, &str));
-
-        if ((order > 24) && (order < 31))
-                amd_iommu_aperture_order = order;
-
-        return 1;
-}
-
+__setup("amd_iommu_dump", parse_amd_iommu_dump);
 __setup("amd_iommu=", parse_amd_iommu_options);
-__setup("amd_iommu_size=", parse_amd_iommu_size_options);
diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c
index ee75d2a9b9cd..8c7c042ecad1 100644
--- a/arch/x86/kernel/apic/apic.c
+++ b/arch/x86/kernel/apic/apic.c
@@ -14,6 +14,7 @@
  *      Mikael Pettersson       :       PM converted to driver model.
  */
 
+#include <linux/perf_counter.h>
 #include <linux/kernel_stat.h>
 #include <linux/mc146818rtc.h>
 #include <linux/acpi_pmtmr.h>
@@ -34,6 +35,7 @@
 #include <linux/smp.h>
 #include <linux/mm.h>
 
+#include <asm/perf_counter.h>
 #include <asm/pgalloc.h>
 #include <asm/atomic.h>
 #include <asm/mpspec.h>
@@ -249,7 +251,7 @@ static void native_apic_write_dummy(u32 reg, u32 v)
 
 static u32 native_apic_read_dummy(u32 reg)
 {
-        WARN_ON_ONCE((cpu_has_apic || !disable_apic));
+        WARN_ON_ONCE((cpu_has_apic && !disable_apic));
         return 0;
 }
 
@@ -1187,6 +1189,7 @@ void __cpuinit setup_local_APIC(void)
                 apic_write(APIC_ESR, 0);
         }
 #endif
+        perf_counters_lapic_init();
 
         preempt_disable();
 
@@ -1609,6 +1612,13 @@ void __init init_apic_mappings(void)
         new_apicid = read_apic_id();
         if (boot_cpu_physical_apicid != new_apicid) {
                 boot_cpu_physical_apicid = new_apicid;
+                /*
+                 * yeah -- we lie about apic_version
+                 * in case if apic was disabled via boot option
+                 * but it's not a problem for SMP compiled kernel
+                 * since smp_sanity_check is prepared for such a case
+                 * and disable smp mode
+                 */
                 apic_version[new_apicid] =
                          GET_APIC_VERSION(apic_read(APIC_LVR));
         }
@@ -2027,7 +2037,7 @@ static int lapic_resume(struct sys_device *dev)
         unsigned int l, h;
         unsigned long flags;
         int maxlvt;
-        int ret;
+        int ret = 0;
         struct IO_APIC_route_entry **ioapic_entries = NULL;
 
         if (!apic_pm_state.active)
@@ -2038,14 +2048,15 @@ static int lapic_resume(struct sys_device *dev)
                 ioapic_entries = alloc_ioapic_entries();
                 if (!ioapic_entries) {
                         WARN(1, "Alloc ioapic_entries in lapic resume failed.");
-                        return -ENOMEM;
+                        ret = -ENOMEM;
+                        goto restore;
                 }
 
                 ret = save_IO_APIC_setup(ioapic_entries);
                 if (ret) {
                         WARN(1, "Saving IO-APIC state failed: %d\n", ret);
                         free_ioapic_entries(ioapic_entries);
-                        return ret;
+                        goto restore;
                 }
 
                 mask_IO_APIC_setup(ioapic_entries);
@@ -2097,10 +2108,10 @@ static int lapic_resume(struct sys_device *dev)
                 restore_IO_APIC_setup(ioapic_entries);
                 free_ioapic_entries(ioapic_entries);
         }
-
+restore:
         local_irq_restore(flags);
 
-        return 0;
+        return ret;
 }
 
 /*
diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c
index ac7f3b6ad583..94605e7f6a54 100644
--- a/arch/x86/kernel/apic/io_apic.c
+++ b/arch/x86/kernel/apic/io_apic.c
@@ -59,6 +59,7 @@
 #include <asm/setup.h>
 #include <asm/irq_remapping.h>
 #include <asm/hpet.h>
+#include <asm/hw_irq.h>
 #include <asm/uv/uv_hub.h>
 #include <asm/uv/uv_irq.h>
 
@@ -176,16 +177,18 @@ int __init arch_early_irq_init(void)
         struct irq_cfg *cfg;
         struct irq_desc *desc;
         int count;
+        int node;
         int i;
 
         cfg = irq_cfgx;
         count = ARRAY_SIZE(irq_cfgx);
+        node= cpu_to_node(boot_cpu_id);
 
         for (i = 0; i < count; i++) {
                 desc = irq_to_desc(i);
                 desc->chip_data = &cfg[i];
-                alloc_bootmem_cpumask_var(&cfg[i].domain);
-                alloc_bootmem_cpumask_var(&cfg[i].old_domain);
+                alloc_cpumask_var_node(&cfg[i].domain, GFP_NOWAIT, node);
+                alloc_cpumask_var_node(&cfg[i].old_domain, GFP_NOWAIT, node);
                 if (i < NR_IRQS_LEGACY)
                         cpumask_setall(cfg[i].domain);
         }
@@ -4012,6 +4015,7 @@ int __init io_apic_get_unique_id(int ioapic, int apic_id)
 
         return apic_id;
 }
+#endif
 
 int __init io_apic_get_version(int ioapic)
 {
@@ -4024,7 +4028,6 @@ int __init io_apic_get_version(int ioapic)
 
         return reg_01.bits.version;
 }
-#endif
 
 int acpi_get_override_irq(int bus_irq, int *trigger, int *polarity)
 {
diff --git a/arch/x86/kernel/apic/nmi.c b/arch/x86/kernel/apic/nmi.c
index c4762276c17e..b3025b43b63a 100644
--- a/arch/x86/kernel/apic/nmi.c
+++ b/arch/x86/kernel/apic/nmi.c
@@ -104,7 +104,7 @@ static __init void nmi_cpu_busy(void *data)
 }
 #endif
 
-static void report_broken_nmi(int cpu, int *prev_nmi_count)
+static void report_broken_nmi(int cpu, unsigned int *prev_nmi_count)
 {
         printk(KERN_CONT "\n");
 
diff --git a/arch/x86/kernel/apic/probe_32.c b/arch/x86/kernel/apic/probe_32.c
index 01eda2ac65e4..440a8bccd91a 100644
--- a/arch/x86/kernel/apic/probe_32.c
+++ b/arch/x86/kernel/apic/probe_32.c
@@ -160,7 +160,6 @@ extern struct apic apic_summit;
 extern struct apic apic_bigsmp;
 extern struct apic apic_es7000;
 extern struct apic apic_es7000_cluster;
-extern struct apic apic_default;
 
 struct apic *apic = &apic_default;
 EXPORT_SYMBOL_GPL(apic);
diff --git a/arch/x86/kernel/apic/x2apic_cluster.c b/arch/x86/kernel/apic/x2apic_cluster.c
index 4a903e2f0d17..8e4cbb255c38 100644
--- a/arch/x86/kernel/apic/x2apic_cluster.c
+++ b/arch/x86/kernel/apic/x2apic_cluster.c
@@ -10,7 +10,7 @@
 #include <asm/apic.h>
 #include <asm/ipi.h>
 
-DEFINE_PER_CPU(u32, x86_cpu_to_logical_apicid);
+static DEFINE_PER_CPU(u32, x86_cpu_to_logical_apicid);
 
 static int x2apic_acpi_madt_oem_check(char *oem_id, char *oem_table_id)
 {
diff --git a/arch/x86/kernel/apic/x2apic_uv_x.c b/arch/x86/kernel/apic/x2apic_uv_x.c
index 780a733a5e7a..ef0ae207a7c8 100644
--- a/arch/x86/kernel/apic/x2apic_uv_x.c
+++ b/arch/x86/kernel/apic/x2apic_uv_x.c
@@ -562,7 +562,7 @@ void __init uv_system_init(void)
         union uvh_node_id_u node_id;
         unsigned long gnode_upper, lowmem_redir_base, lowmem_redir_size;
         int bytes, nid, cpu, lcpu, pnode, blade, i, j, m_val, n_val;
-        int max_pnode = 0;
+        int gnode_extra, max_pnode = 0;
         unsigned long mmr_base, present, paddr;
         unsigned short pnode_mask;
 
@@ -574,6 +574,13 @@ void __init uv_system_init(void)
         mmr_base =
             uv_read_local_mmr(UVH_RH_GAM_MMR_OVERLAY_CONFIG_MMR) &
             ~UV_MMR_ENABLE;
+        pnode_mask = (1 << n_val) - 1;
+        node_id.v = uv_read_local_mmr(UVH_NODE_ID);
+        gnode_extra = (node_id.s.node_id & ~((1 << n_val) - 1)) >> 1;
+        gnode_upper = ((unsigned long)gnode_extra  << m_val);
+        printk(KERN_DEBUG "UV: N %d, M %d, gnode_upper 0x%lx, gnode_extra 0x%x\n",
+                        n_val, m_val, gnode_upper, gnode_extra);
+
         printk(KERN_DEBUG "UV: global MMR base 0x%lx\n", mmr_base);
 
         for(i = 0; i < UVH_NODE_PRESENT_TABLE_DEPTH; i++)
@@ -610,11 +617,6 @@ void __init uv_system_init(void)
                 }
         }
 
-        pnode_mask = (1 << n_val) - 1;
-        node_id.v = uv_read_local_mmr(UVH_NODE_ID);
-        gnode_upper = (((unsigned long)node_id.s.node_id) &
-                       ~((1 << n_val) - 1)) << m_val;
-
         uv_bios_init();
         uv_bios_get_sn_info(0, &uv_type, &sn_partition_id,
                             &sn_coherency_id, &sn_region_size);
@@ -637,6 +639,7 @@ void __init uv_system_init(void)
                 uv_cpu_hub_info(cpu)->pnode_mask = pnode_mask;
                 uv_cpu_hub_info(cpu)->gpa_mask = (1 << (m_val + n_val)) - 1;
                 uv_cpu_hub_info(cpu)->gnode_upper = gnode_upper;
+                uv_cpu_hub_info(cpu)->gnode_extra = gnode_extra;
                 uv_cpu_hub_info(cpu)->global_mmr_base = mmr_base;
                 uv_cpu_hub_info(cpu)->coherency_domain_number = sn_coherency_id;
                 uv_cpu_hub_info(cpu)->scir.offset = SCIR_LOCAL_MMR_BASE + lcpu;
diff --git a/arch/x86/kernel/asm-offsets_32.c b/arch/x86/kernel/asm-offsets_32.c
index 5a6aa1c1162f..1a830cbd7015 100644
--- a/arch/x86/kernel/asm-offsets_32.c
+++ b/arch/x86/kernel/asm-offsets_32.c
@@ -146,4 +146,5 @@ void foo(void)
         OFFSET(BP_loadflags, boot_params, hdr.loadflags);
         OFFSET(BP_hardware_subarch, boot_params, hdr.hardware_subarch);
         OFFSET(BP_version, boot_params, hdr.version);
+        OFFSET(BP_kernel_alignment, boot_params, hdr.kernel_alignment);
 }
diff --git a/arch/x86/kernel/asm-offsets_64.c b/arch/x86/kernel/asm-offsets_64.c
index e72f062fb4b5..898ecc47e129 100644
--- a/arch/x86/kernel/asm-offsets_64.c
+++ b/arch/x86/kernel/asm-offsets_64.c
@@ -125,6 +125,7 @@ int main(void)
         OFFSET(BP_loadflags, boot_params, hdr.loadflags);
         OFFSET(BP_hardware_subarch, boot_params, hdr.hardware_subarch);
         OFFSET(BP_version, boot_params, hdr.version);
+        OFFSET(BP_kernel_alignment, boot_params, hdr.kernel_alignment);
 
         BLANK();
         DEFINE(PAGE_SIZE_asm, PAGE_SIZE);
diff --git a/arch/x86/kernel/cpu/Makefile b/arch/x86/kernel/cpu/Makefile
index 4e242f9a06e4..3efcb2b96a15 100644
--- a/arch/x86/kernel/cpu/Makefile
+++ b/arch/x86/kernel/cpu/Makefile
@@ -1,5 +1,5 @@
 #
-# Makefile for x86-compatible CPU details and quirks
+# Makefile for x86-compatible CPU details, features and quirks
 #
 
 # Don't trace early stages of a secondary CPU boot
@@ -23,11 +23,13 @@ obj-$(CONFIG_CPU_SUP_CENTAUR)		+= centaur.o
 obj-$(CONFIG_CPU_SUP_TRANSMETA_32)      += transmeta.o
 obj-$(CONFIG_CPU_SUP_UMC_32)            += umc.o
 
-obj-$(CONFIG_X86_MCE)   += mcheck/
-obj-$(CONFIG_MTRR)      += mtrr/
-obj-$(CONFIG_CPU_FREQ)  += cpufreq/
+obj-$(CONFIG_PERF_COUNTERS)             += perf_counter.o
 
-obj-$(CONFIG_X86_LOCAL_APIC) += perfctr-watchdog.o
+obj-$(CONFIG_X86_MCE)                   += mcheck/
+obj-$(CONFIG_MTRR)                      += mtrr/
+obj-$(CONFIG_CPU_FREQ)                  += cpufreq/
+
+obj-$(CONFIG_X86_LOCAL_APIC)            += perfctr-watchdog.o
 
 quiet_cmd_mkcapflags = MKCAP   $@
       cmd_mkcapflags = $(PERL) $(srctree)/$(src)/mkcapflags.pl $< $@
diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c
index 728b3750a3e8..e5b27d8f1b47 100644
--- a/arch/x86/kernel/cpu/amd.c
+++ b/arch/x86/kernel/cpu/amd.c
@@ -6,6 +6,7 @@
 #include <asm/processor.h>
 #include <asm/apic.h>
 #include <asm/cpu.h>
+#include <asm/pci-direct.h>
 
 #ifdef CONFIG_X86_64
 # include <asm/numa_64.h>
@@ -351,6 +352,15 @@ static void __cpuinit early_init_amd(struct cpuinfo_x86 *c)
                     (c->x86_model == 8 && c->x86_mask >= 8))
                         set_cpu_cap(c, X86_FEATURE_K6_MTRR);
 #endif
+#if defined(CONFIG_X86_LOCAL_APIC) && defined(CONFIG_PCI)
+        /* check CPU config space for extended APIC ID */
+        if (c->x86 >= 0xf) {
+                unsigned int val;
+                val = read_pci_config(0, 24, 0, 0x68);
+                if ((val & ((1 << 17) | (1 << 18))) == ((1 << 17) | (1 << 18)))
+                        set_cpu_cap(c, X86_FEATURE_EXTD_APICID);
+        }
+#endif
 }
 
 static void __cpuinit init_amd(struct cpuinfo_x86 *c)
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index b0517aa2bd3b..3ffdcfa9abdf 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -13,6 +13,7 @@
 #include <linux/io.h>
 
 #include <asm/stackprotector.h>
+#include <asm/perf_counter.h>
 #include <asm/mmu_context.h>
 #include <asm/hypervisor.h>
 #include <asm/processor.h>
@@ -874,6 +875,7 @@ void __init identify_boot_cpu(void)
 #else
         vgetcpu_set_mode();
 #endif
+        init_hw_perf_counters();
 }
 
 void __cpuinit identify_secondary_cpu(struct cpuinfo_x86 *c)
diff --git a/arch/x86/kernel/cpu/cpu_debug.c b/arch/x86/kernel/cpu/cpu_debug.c
index 2fc4f6bb9ca5..6b2a52dd0403 100644
--- a/arch/x86/kernel/cpu/cpu_debug.c
+++ b/arch/x86/kernel/cpu/cpu_debug.c
@@ -32,9 +32,7 @@
 
 static DEFINE_PER_CPU(struct cpu_cpuX_base, cpu_arr[CPU_REG_ALL_BIT]);
 static DEFINE_PER_CPU(struct cpu_private *, priv_arr[MAX_CPU_FILES]);
-static DEFINE_PER_CPU(unsigned, cpu_modelflag);
 static DEFINE_PER_CPU(int, cpu_priv_count);
-static DEFINE_PER_CPU(unsigned, cpu_model);
 
 static DEFINE_MUTEX(cpu_debug_lock);
 
@@ -80,302 +78,102 @@ static struct cpu_file_base cpu_file[] = {
         { "value",      CPU_REG_ALL,    1       },
 };
 
-/* Intel Registers Range */
-static struct cpu_debug_range cpu_intel_range[] = {
-        { 0x00000000, 0x00000001, CPU_MC,       CPU_INTEL_ALL           },
-        { 0x00000006, 0x00000007, CPU_MONITOR,  CPU_CX_AT_XE            },
-        { 0x00000010, 0x00000010, CPU_TIME,     CPU_INTEL_ALL           },
-        { 0x00000011, 0x00000013, CPU_PMC,      CPU_INTEL_PENTIUM       },
-        { 0x00000017, 0x00000017, CPU_PLATFORM, CPU_PX_CX_AT_XE         },
-        { 0x0000001B, 0x0000001B, CPU_APIC,     CPU_P6_CX_AT_XE         },
-
-        { 0x0000002A, 0x0000002A, CPU_POWERON,  CPU_PX_CX_AT_XE         },
-        { 0x0000002B, 0x0000002B, CPU_POWERON,  CPU_INTEL_XEON          },
-        { 0x0000002C, 0x0000002C, CPU_FREQ,     CPU_INTEL_XEON          },
-        { 0x0000003A, 0x0000003A, CPU_CONTROL,  CPU_CX_AT_XE            },
-
-        { 0x00000040, 0x00000043, CPU_LBRANCH,  CPU_PM_CX_AT_XE         },
-        { 0x00000044, 0x00000047, CPU_LBRANCH,  CPU_PM_CO_AT            },
-        { 0x00000060, 0x00000063, CPU_LBRANCH,  CPU_C2_AT               },
-        { 0x00000064, 0x00000067, CPU_LBRANCH,  CPU_INTEL_ATOM          },
-
-        { 0x00000079, 0x00000079, CPU_BIOS,     CPU_P6_CX_AT_XE         },
-        { 0x00000088, 0x0000008A, CPU_CACHE,    CPU_INTEL_P6            },
-        { 0x0000008B, 0x0000008B, CPU_BIOS,     CPU_P6_CX_AT_XE         },
-        { 0x0000009B, 0x0000009B, CPU_MONITOR,  CPU_INTEL_XEON          },
-
-        { 0x000000C1, 0x000000C2, CPU_PMC,      CPU_P6_CX_AT            },
-        { 0x000000CD, 0x000000CD, CPU_FREQ,     CPU_CX_AT               },
-        { 0x000000E7, 0x000000E8, CPU_PERF,     CPU_CX_AT               },
-        { 0x000000FE, 0x000000FE, CPU_MTRR,     CPU_P6_CX_XE            },
-
-        { 0x00000116, 0x00000116, CPU_CACHE,    CPU_INTEL_P6            },
-        { 0x00000118, 0x00000118, CPU_CACHE,    CPU_INTEL_P6            },
-        { 0x00000119, 0x00000119, CPU_CACHE,    CPU_INTEL_PX            },
-        { 0x0000011A, 0x0000011B, CPU_CACHE,    CPU_INTEL_P6            },
-        { 0x0000011E, 0x0000011E, CPU_CACHE,    CPU_PX_CX_AT            },
-
-        { 0x00000174, 0x00000176, CPU_SYSENTER, CPU_P6_CX_AT_XE         },
-        { 0x00000179, 0x0000017A, CPU_MC,       CPU_PX_CX_AT_XE         },
-        { 0x0000017B, 0x0000017B, CPU_MC,       CPU_P6_XE               },
-        { 0x00000186, 0x00000187, CPU_PMC,      CPU_P6_CX_AT            },
-        { 0x00000198, 0x00000199, CPU_PERF,     CPU_PM_CX_AT_XE         },
-        { 0x0000019A, 0x0000019A, CPU_TIME,     CPU_PM_CX_AT_XE         },
-        { 0x0000019B, 0x0000019D, CPU_THERM,    CPU_PM_CX_AT_XE         },
-        { 0x000001A0, 0x000001A0, CPU_MISC,     CPU_PM_CX_AT_XE         },
-
-        { 0x000001C9, 0x000001C9, CPU_LBRANCH,  CPU_PM_CX_AT            },
-        { 0x000001D7, 0x000001D8, CPU_LBRANCH,  CPU_INTEL_XEON          },
-        { 0x000001D9, 0x000001D9, CPU_DEBUG,    CPU_CX_AT_XE            },
-        { 0x000001DA, 0x000001DA, CPU_LBRANCH,  CPU_INTEL_XEON          },
-        { 0x000001DB, 0x000001DB, CPU_LBRANCH,  CPU_P6_XE               },
-        { 0x000001DC, 0x000001DC, CPU_LBRANCH,  CPU_INTEL_P6            },
-        { 0x000001DD, 0x000001DE, CPU_LBRANCH,  CPU_PX_CX_AT_XE         },
-        { 0x000001E0, 0x000001E0, CPU_LBRANCH,  CPU_INTEL_P6            },
-
-        { 0x00000200, 0x0000020F, CPU_MTRR,     CPU_P6_CX_XE            },
-        { 0x00000250, 0x00000250, CPU_MTRR,     CPU_P6_CX_XE            },
-        { 0x00000258, 0x00000259, CPU_MTRR,     CPU_P6_CX_XE            },
-        { 0x00000268, 0x0000026F, CPU_MTRR,     CPU_P6_CX_XE            },
-        { 0x00000277, 0x00000277, CPU_PAT,      CPU_C2_AT_XE            },
-        { 0x000002FF, 0x000002FF, CPU_MTRR,     CPU_P6_CX_XE            },
-
-        { 0x00000300, 0x00000308, CPU_PMC,      CPU_INTEL_XEON          },
-        { 0x00000309, 0x0000030B, CPU_PMC,      CPU_C2_AT_XE            },
-        { 0x0000030C, 0x00000311, CPU_PMC,      CPU_INTEL_XEON          },
-        { 0x00000345, 0x00000345, CPU_PMC,      CPU_C2_AT               },
-        { 0x00000360, 0x00000371, CPU_PMC,      CPU_INTEL_XEON          },
-        { 0x0000038D, 0x00000390, CPU_PMC,      CPU_C2_AT               },
-        { 0x000003A0, 0x000003BE, CPU_PMC,      CPU_INTEL_XEON          },
-        { 0x000003C0, 0x000003CD, CPU_PMC,      CPU_INTEL_XEON          },
-        { 0x000003E0, 0x000003E1, CPU_PMC,      CPU_INTEL_XEON          },
-        { 0x000003F0, 0x000003F0, CPU_PMC,      CPU_INTEL_XEON          },
-        { 0x000003F1, 0x000003F1, CPU_PMC,      CPU_C2_AT_XE            },
-        { 0x000003F2, 0x000003F2, CPU_PMC,      CPU_INTEL_XEON          },
-
-        { 0x00000400, 0x00000402, CPU_MC,       CPU_PM_CX_AT_XE         },
-        { 0x00000403, 0x00000403, CPU_MC,       CPU_INTEL_XEON          },
-        { 0x00000404, 0x00000406, CPU_MC,       CPU_PM_CX_AT_XE         },
-        { 0x00000407, 0x00000407, CPU_MC,       CPU_INTEL_XEON          },
-        { 0x00000408, 0x0000040A, CPU_MC,       CPU_PM_CX_AT_XE         },
-        { 0x0000040B, 0x0000040B, CPU_MC,       CPU_INTEL_XEON          },
-        { 0x0000040C, 0x0000040E, CPU_MC,       CPU_PM_CX_XE            },
-        { 0x0000040F, 0x0000040F, CPU_MC,       CPU_INTEL_XEON          },
-        { 0x00000410, 0x00000412, CPU_MC,       CPU_PM_CX_AT_XE         },
-        { 0x00000413, 0x00000417, CPU_MC,       CPU_CX_AT_XE            },
-        { 0x00000480, 0x0000048B, CPU_VMX,      CPU_CX_AT_XE            },
-
-        { 0x00000600, 0x00000600, CPU_DEBUG,    CPU_PM_CX_AT_XE         },
-        { 0x00000680, 0x0000068F, CPU_LBRANCH,  CPU_INTEL_XEON          },
-        { 0x000006C0, 0x000006CF, CPU_LBRANCH,  CPU_INTEL_XEON          },
-
-        { 0x000107CC, 0x000107D3, CPU_PMC,      CPU_INTEL_XEON_MP       },
-
-        { 0xC0000080, 0xC0000080, CPU_FEATURES, CPU_INTEL_XEON          },
-        { 0xC0000081, 0xC0000082, CPU_CALL,     CPU_INTEL_XEON          },
-        { 0xC0000084, 0xC0000084, CPU_CALL,     CPU_INTEL_XEON          },
-        { 0xC0000100, 0xC0000102, CPU_BASE,     CPU_INTEL_XEON          },
+/* CPU Registers Range */
+static struct cpu_debug_range cpu_reg_range[] = {
+        { 0x00000000, 0x00000001, CPU_MC,       },
+        { 0x00000006, 0x00000007, CPU_MONITOR,  },
+        { 0x00000010, 0x00000010, CPU_TIME,     },
+        { 0x00000011, 0x00000013, CPU_PMC,      },
+        { 0x00000017, 0x00000017, CPU_PLATFORM, },
+        { 0x0000001B, 0x0000001B, CPU_APIC,     },
+        { 0x0000002A, 0x0000002B, CPU_POWERON,  },
+        { 0x0000002C, 0x0000002C, CPU_FREQ,     },
+        { 0x0000003A, 0x0000003A, CPU_CONTROL,  },
+        { 0x00000040, 0x00000047, CPU_LBRANCH,  },
+        { 0x00000060, 0x00000067, CPU_LBRANCH,  },
+        { 0x00000079, 0x00000079, CPU_BIOS,     },
+        { 0x00000088, 0x0000008A, CPU_CACHE,    },
+        { 0x0000008B, 0x0000008B, CPU_BIOS,     },
+        { 0x0000009B, 0x0000009B, CPU_MONITOR,  },
+        { 0x000000C1, 0x000000C4, CPU_PMC,      },
+        { 0x000000CD, 0x000000CD, CPU_FREQ,     },
+        { 0x000000E7, 0x000000E8, CPU_PERF,     },
+        { 0x000000FE, 0x000000FE, CPU_MTRR,     },
+
+        { 0x00000116, 0x0000011E, CPU_CACHE,    },
+        { 0x00000174, 0x00000176, CPU_SYSENTER, },
+        { 0x00000179, 0x0000017B, CPU_MC,       },
+        { 0x00000186, 0x00000189, CPU_PMC,      },
+        { 0x00000198, 0x00000199, CPU_PERF,     },
+        { 0x0000019A, 0x0000019A, CPU_TIME,     },
+        { 0x0000019B, 0x0000019D, CPU_THERM,    },
+        { 0x000001A0, 0x000001A0, CPU_MISC,     },
+        { 0x000001C9, 0x000001C9, CPU_LBRANCH,  },
+        { 0x000001D7, 0x000001D8, CPU_LBRANCH,  },
+        { 0x000001D9, 0x000001D9, CPU_DEBUG,    },
+        { 0x000001DA, 0x000001E0, CPU_LBRANCH,  },
+
+        { 0x00000200, 0x0000020F, CPU_MTRR,     },
+        { 0x00000250, 0x00000250, CPU_MTRR,     },
+        { 0x00000258, 0x00000259, CPU_MTRR,     },
+        { 0x00000268, 0x0000026F, CPU_MTRR,     },
+        { 0x00000277, 0x00000277, CPU_PAT,      },
+        { 0x000002FF, 0x000002FF, CPU_MTRR,     },
+
+        { 0x00000300, 0x00000311, CPU_PMC,      },
+        { 0x00000345, 0x00000345, CPU_PMC,      },
+        { 0x00000360, 0x00000371, CPU_PMC,      },
+        { 0x0000038D, 0x00000390, CPU_PMC,      },
+        { 0x000003A0, 0x000003BE, CPU_PMC,      },
+        { 0x000003C0, 0x000003CD, CPU_PMC,      },
+        { 0x000003E0, 0x000003E1, CPU_PMC,      },
+        { 0x000003F0, 0x000003F2, CPU_PMC,      },
+
+        { 0x00000400, 0x00000417, CPU_MC,       },
+        { 0x00000480, 0x0000048B, CPU_VMX,      },
+
+        { 0x00000600, 0x00000600, CPU_DEBUG,    },
+        { 0x00000680, 0x0000068F, CPU_LBRANCH,  },
+        { 0x000006C0, 0x000006CF, CPU_LBRANCH,  },
+
+        { 0x000107CC, 0x000107D3, CPU_PMC,      },
+
+        { 0xC0000080, 0xC0000080, CPU_FEATURES, },
+        { 0xC0000081, 0xC0000084, CPU_CALL,     },
+        { 0xC0000100, 0xC0000102, CPU_BASE,     },
+        { 0xC0000103, 0xC0000103, CPU_TIME,     },
+
+        { 0xC0010000, 0xC0010007, CPU_PMC,      },
+        { 0xC0010010, 0xC0010010, CPU_CONF,     },
+        { 0xC0010015, 0xC0010015, CPU_CONF,     },
+        { 0xC0010016, 0xC001001A, CPU_MTRR,     },
+        { 0xC001001D, 0xC001001D, CPU_MTRR,     },
+        { 0xC001001F, 0xC001001F, CPU_CONF,     },
+        { 0xC0010030, 0xC0010035, CPU_BIOS,     },
+        { 0xC0010044, 0xC0010048, CPU_MC,       },
+        { 0xC0010050, 0xC0010056, CPU_SMM,      },
+        { 0xC0010058, 0xC0010058, CPU_CONF,     },
+        { 0xC0010060, 0xC0010060, CPU_CACHE,    },
+        { 0xC0010061, 0xC0010068, CPU_SMM,      },
+        { 0xC0010069, 0xC001006B, CPU_SMM,      },
+        { 0xC0010070, 0xC0010071, CPU_SMM,      },
+        { 0xC0010111, 0xC0010113, CPU_SMM,      },
+        { 0xC0010114, 0xC0010118, CPU_SVM,      },
+        { 0xC0010140, 0xC0010141, CPU_OSVM,     },
+        { 0xC0011022, 0xC0011023, CPU_CONF,     },
 };
 
-/* AMD Registers Range */
-static struct cpu_debug_range cpu_amd_range[] = {
-        { 0x00000000, 0x00000001, CPU_MC,       CPU_K10_PLUS,           },
-        { 0x00000010, 0x00000010, CPU_TIME,     CPU_K8_PLUS,            },
-        { 0x0000001B, 0x0000001B, CPU_APIC,     CPU_K8_PLUS,            },
-        { 0x0000002A, 0x0000002A, CPU_POWERON,  CPU_K7_PLUS             },
-        { 0x0000008B, 0x0000008B, CPU_VER,      CPU_K8_PLUS             },
-        { 0x000000FE, 0x000000FE, CPU_MTRR,     CPU_K8_PLUS,            },
-
-        { 0x00000174, 0x00000176, CPU_SYSENTER, CPU_K8_PLUS,            },
-        { 0x00000179, 0x0000017B, CPU_MC,       CPU_K8_PLUS,            },
-        { 0x000001D9, 0x000001D9, CPU_DEBUG,    CPU_K8_PLUS,            },
-        { 0x000001DB, 0x000001DE, CPU_LBRANCH,  CPU_K8_PLUS,            },
-
-        { 0x00000200, 0x0000020F, CPU_MTRR,     CPU_K8_PLUS,            },
-        { 0x00000250, 0x00000250, CPU_MTRR,     CPU_K8_PLUS,            },
-        { 0x00000258, 0x00000259, CPU_MTRR,     CPU_K8_PLUS,            },
-        { 0x00000268, 0x0000026F, CPU_MTRR,     CPU_K8_PLUS,            },
-        { 0x00000277, 0x00000277, CPU_PAT,      CPU_K8_PLUS,            },
-        { 0x000002FF, 0x000002FF, CPU_MTRR,     CPU_K8_PLUS,            },
-
-        { 0x00000400, 0x00000413, CPU_MC,       CPU_K8_PLUS,            },
-
-        { 0xC0000080, 0xC0000080, CPU_FEATURES, CPU_AMD_ALL,            },
-        { 0xC0000081, 0xC0000084, CPU_CALL,     CPU_K8_PLUS,            },
-        { 0xC0000100, 0xC0000102, CPU_BASE,     CPU_K8_PLUS,            },
-        { 0xC0000103, 0xC0000103, CPU_TIME,     CPU_K10_PLUS,           },
-
-        { 0xC0010000, 0xC0010007, CPU_PMC,      CPU_K8_PLUS,            },
-        { 0xC0010010, 0xC0010010, CPU_CONF,     CPU_K7_PLUS,            },
-        { 0xC0010015, 0xC0010015, CPU_CONF,     CPU_K7_PLUS,            },
-        { 0xC0010016, 0xC001001A, CPU_MTRR,     CPU_K8_PLUS,            },
-        { 0xC001001D, 0xC001001D, CPU_MTRR,     CPU_K8_PLUS,            },
-        { 0xC001001F, 0xC001001F, CPU_CONF,     CPU_K8_PLUS,            },
-        { 0xC0010030, 0xC0010035, CPU_BIOS,     CPU_K8_PLUS,            },
-        { 0xC0010044, 0xC0010048, CPU_MC,       CPU_K8_PLUS,            },
-        { 0xC0010050, 0xC0010056, CPU_SMM,      CPU_K0F_PLUS,           },
-        { 0xC0010058, 0xC0010058, CPU_CONF,     CPU_K10_PLUS,           },
-        { 0xC0010060, 0xC0010060, CPU_CACHE,    CPU_AMD_11,             },
-        { 0xC0010061, 0xC0010068, CPU_SMM,      CPU_K10_PLUS,           },
-        { 0xC0010069, 0xC001006B, CPU_SMM,      CPU_AMD_11,             },
-        { 0xC0010070, 0xC0010071, CPU_SMM,      CPU_K10_PLUS,           },
-        { 0xC0010111, 0xC0010113, CPU_SMM,      CPU_K8_PLUS,            },
-        { 0xC0010114, 0xC0010118, CPU_SVM,      CPU_K10_PLUS,           },
-        { 0xC0010140, 0xC0010141, CPU_OSVM,     CPU_K10_PLUS,           },
-        { 0xC0011022, 0xC0011023, CPU_CONF,     CPU_K10_PLUS,           },
-};
-
-
-/* Intel */
-static int get_intel_modelflag(unsigned model)
-{
-        int flag;
-
-        switch (model) {
-        case 0x0501:
-        case 0x0502:
-        case 0x0504:
-                flag = CPU_INTEL_PENTIUM;
-                break;
-        case 0x0601:
-        case 0x0603:
-        case 0x0605:
-        case 0x0607:
-        case 0x0608:
-        case 0x060A:
-        case 0x060B:
-                flag = CPU_INTEL_P6;
-                break;
-        case 0x0609:
-        case 0x060D:
-                flag = CPU_INTEL_PENTIUM_M;
-                break;
-        case 0x060E:
-                flag = CPU_INTEL_CORE;
-                break;
-        case 0x060F:
-        case 0x0617:
-                flag = CPU_INTEL_CORE2;
-                break;
-        case 0x061C:
-                flag = CPU_INTEL_ATOM;
-                break;
-        case 0x0F00:
-        case 0x0F01:
-        case 0x0F02:
-        case 0x0F03:
-        case 0x0F04:
-                flag = CPU_INTEL_XEON_P4;
-                break;
-        case 0x0F06:
-                flag = CPU_INTEL_XEON_MP;
-                break;
-        default:
-                flag = CPU_NONE;
-                break;
-        }
-
-        return flag;
-}
-
-/* AMD */
-static int get_amd_modelflag(unsigned model)
-{
-        int flag;
-
-        switch (model >> 8) {
-        case 0x6:
-                flag = CPU_AMD_K6;
-                break;
-        case 0x7:
-                flag = CPU_AMD_K7;
-                break;
-        case 0x8:
-                flag = CPU_AMD_K8;
-                break;
-        case 0xf:
-                flag = CPU_AMD_0F;
-                break;
-        case 0x10:
-                flag = CPU_AMD_10;
-                break;
-        case 0x11:
-                flag = CPU_AMD_11;
-                break;
-        default:
-                flag = CPU_NONE;
-                break;
-        }
-
-        return flag;
-}
-
-static int get_cpu_modelflag(unsigned cpu)
-{
-        int flag;
-
-        flag = per_cpu(cpu_model, cpu);
-
-        switch (flag >> 16) {
-        case X86_VENDOR_INTEL:
-                flag = get_intel_modelflag(flag);
-                break;
-        case X86_VENDOR_AMD:
-                flag = get_amd_modelflag(flag & 0xffff);
-                break;
-        default:
-                flag = CPU_NONE;
-                break;
-        }
-
-        return flag;
-}
-
-static int get_cpu_range_count(unsigned cpu)
-{
-        int index;
-
-        switch (per_cpu(cpu_model, cpu) >> 16) {
-        case X86_VENDOR_INTEL:
-                index = ARRAY_SIZE(cpu_intel_range);
-                break;
-        case X86_VENDOR_AMD:
-                index = ARRAY_SIZE(cpu_amd_range);
-                break;
-        default:
-                index = 0;
-                break;
-        }
-
-        return index;
-}
-
 static int is_typeflag_valid(unsigned cpu, unsigned flag)
 {
-        unsigned vendor, modelflag;
-        int i, index;
+        int i;
 
         /* Standard Registers should be always valid */
         if (flag >= CPU_TSS)
                 return 1;
 
-        modelflag = per_cpu(cpu_modelflag, cpu);
-        vendor = per_cpu(cpu_model, cpu) >> 16;
-        index = get_cpu_range_count(cpu);
-
-        for (i = 0; i < index; i++) {
-                switch (vendor) {
-                case X86_VENDOR_INTEL:
-                        if ((cpu_intel_range[i].model & modelflag) &&
-                            (cpu_intel_range[i].flag & flag))
-                                return 1;
-                        break;
-                case X86_VENDOR_AMD:
-                        if ((cpu_amd_range[i].model & modelflag) &&
-                            (cpu_amd_range[i].flag & flag))
-                                return 1;
-                        break;
-                }
+        for (i = 0; i < ARRAY_SIZE(cpu_reg_range); i++) {
+                if (cpu_reg_range[i].flag == flag)
+                        return 1;
         }
 
         /* Invalid */
@@ -385,26 +183,11 @@ static int is_typeflag_valid(unsigned cpu, unsigned flag)
 static unsigned get_cpu_range(unsigned cpu, unsigned *min, unsigned *max,
                               int index, unsigned flag)
 {
-        unsigned modelflag;
-
-        modelflag = per_cpu(cpu_modelflag, cpu);
-        *max = 0;
-        switch (per_cpu(cpu_model, cpu) >> 16) {
-        case X86_VENDOR_INTEL:
-                if ((cpu_intel_range[index].model & modelflag) &&
-                    (cpu_intel_range[index].flag & flag)) {
-                        *min = cpu_intel_range[index].min;
-                        *max = cpu_intel_range[index].max;
-                }
-                break;
-        case X86_VENDOR_AMD:
-                if ((cpu_amd_range[index].model & modelflag) &&
-                    (cpu_amd_range[index].flag & flag)) {
-                        *min = cpu_amd_range[index].min;
-                        *max = cpu_amd_range[index].max;
-                }
-                break;
-        }
+        if (cpu_reg_range[index].flag == flag) {
+                *min = cpu_reg_range[index].min;
+                *max = cpu_reg_range[index].max;
+        } else
+                *max = 0;
 
         return *max;
 }
@@ -434,7 +217,7 @@ static void print_msr(struct seq_file *seq, unsigned cpu, unsigned flag)
         unsigned msr, msr_min, msr_max;
         struct cpu_private *priv;
         u32 low, high;
-        int i, range;
+        int i;
 
         if (seq) {
                 priv = seq->private;
@@ -446,9 +229,7 @@ static void print_msr(struct seq_file *seq, unsigned cpu, unsigned flag)
                 }
         }
 
-        range = get_cpu_range_count(cpu);
-
-        for (i = 0; i < range; i++) {
+        for (i = 0; i < ARRAY_SIZE(cpu_reg_range); i++) {
                 if (!get_cpu_range(cpu, &msr_min, &msr_max, i, flag))
                         continue;
 
@@ -800,13 +581,11 @@ static int cpu_init_msr(unsigned cpu, unsigned type, struct dentry *dentry)
 {
         struct dentry *cpu_dentry = NULL;
         unsigned reg, reg_min, reg_max;
-        int i, range, err = 0;
+        int i, err = 0;
         char reg_dir[12];
         u32 low, high;
 
-        range = get_cpu_range_count(cpu);
-
-        for (i = 0; i < range; i++) {
+        for (i = 0; i < ARRAY_SIZE(cpu_reg_range); i++) {
                 if (!get_cpu_range(cpu, &reg_min, &reg_max, i,
                                    cpu_base[type].flag))
                         continue;
@@ -862,10 +641,6 @@ static int cpu_init_cpu(void)
                 cpui = &cpu_data(cpu);
                 if (!cpu_has(cpui, X86_FEATURE_MSR))
                         continue;
-                per_cpu(cpu_model, cpu) = ((cpui->x86_vendor << 16) |
-                                           (cpui->x86 << 8) |
-                                           (cpui->x86_model));
-                per_cpu(cpu_modelflag, cpu) = get_cpu_modelflag(cpu);
 
                 sprintf(cpu_dir, "cpu%d", cpu);
                 cpu_dentry = debugfs_create_dir(cpu_dir, cpu_debugfs_dir);
diff --git a/arch/x86/kernel/cpu/cpufreq/Kconfig b/arch/x86/kernel/cpu/cpufreq/Kconfig
index 52c839875478..f138c6c389b9 100644
--- a/arch/x86/kernel/cpu/cpufreq/Kconfig
+++ b/arch/x86/kernel/cpu/cpufreq/Kconfig
@@ -220,11 +220,14 @@ config X86_LONGHAUL
           If in doubt, say N.
 
 config X86_E_POWERSAVER
-        tristate "VIA C7 Enhanced PowerSaver"
+        tristate "VIA C7 Enhanced PowerSaver (DANGEROUS)"
         select CPU_FREQ_TABLE
-        depends on X86_32
+        depends on X86_32 && EXPERIMENTAL
         help
-          This adds the CPUFreq driver for VIA C7 processors.
+          This adds the CPUFreq driver for VIA C7 processors.  However, this driver
+          does not have any safeguards to prevent operating the CPU out of spec
+          and is thus considered dangerous.  Please use the regular ACPI cpufreq
+          driver, enabled by CONFIG_X86_ACPI_CPUFREQ.
 
           If in doubt, say N.
 
diff --git a/arch/x86/kernel/cpu/cpufreq/acpi-cpufreq.c b/arch/x86/kernel/cpu/cpufreq/acpi-cpufreq.c
index 54b6de2cd947..ae9b503220ca 100644
--- a/arch/x86/kernel/cpu/cpufreq/acpi-cpufreq.c
+++ b/arch/x86/kernel/cpu/cpufreq/acpi-cpufreq.c
@@ -90,11 +90,7 @@ static int check_est_cpu(unsigned int cpuid)
 {
         struct cpuinfo_x86 *cpu = &cpu_data(cpuid);
 
-        if (cpu->x86_vendor != X86_VENDOR_INTEL ||
-            !cpu_has(cpu, X86_FEATURE_EST))
-                return 0;
-
-        return 1;
+        return cpu_has(cpu, X86_FEATURE_EST);
 }
 
 static unsigned extract_io(u32 value, struct acpi_cpufreq_data *data)
@@ -550,7 +546,7 @@ static int __init acpi_cpufreq_early_init(void)
                 return -ENOMEM;
         }
         for_each_possible_cpu(i) {
-                if (!alloc_cpumask_var_node(
+                if (!zalloc_cpumask_var_node(
                         &per_cpu_ptr(acpi_perf_data, i)->shared_cpu_map,
                         GFP_KERNEL, cpu_to_node(i))) {
 
diff --git a/arch/x86/kernel/cpu/cpufreq/powernow-k7.c b/arch/x86/kernel/cpu/cpufreq/powernow-k7.c
index a8363e5be4ef..d47c775eb0ab 100644
--- a/arch/x86/kernel/cpu/cpufreq/powernow-k7.c
+++ b/arch/x86/kernel/cpu/cpufreq/powernow-k7.c
@@ -322,7 +322,7 @@ static int powernow_acpi_init(void)
                 goto err0;
         }
 
-        if (!alloc_cpumask_var(&acpi_processor_perf->shared_cpu_map,
+        if (!zalloc_cpumask_var(&acpi_processor_perf->shared_cpu_map,
                                                                 GFP_KERNEL)) {
                 retval = -ENOMEM;
                 goto err05;
diff --git a/arch/x86/kernel/cpu/cpufreq/powernow-k8.c b/arch/x86/kernel/cpu/cpufreq/powernow-k8.c
index f6b32d112357..cf52215d9eb1 100644
--- a/arch/x86/kernel/cpu/cpufreq/powernow-k8.c
+++ b/arch/x86/kernel/cpu/cpufreq/powernow-k8.c
@@ -835,7 +835,7 @@ static int powernow_k8_cpu_init_acpi(struct powernow_k8_data *data)
 {
         struct cpufreq_frequency_table *powernow_table;
         int ret_val = -ENODEV;
-        acpi_integer space_id;
+        acpi_integer control, status;
 
         if (acpi_processor_register_performance(&data->acpi_data, data->cpu)) {
                 dprintk("register performance failed: bad ACPI data\n");
@@ -848,12 +848,13 @@ static int powernow_k8_cpu_init_acpi(struct powernow_k8_data *data)
                 goto err_out;
         }
 
-        space_id = data->acpi_data.control_register.space_id;
-        if ((space_id != ACPI_ADR_SPACE_FIXED_HARDWARE) ||
-                (space_id != ACPI_ADR_SPACE_FIXED_HARDWARE)) {
+        control = data->acpi_data.control_register.space_id;
+        status = data->acpi_data.status_register.space_id;
+
+        if ((control != ACPI_ADR_SPACE_FIXED_HARDWARE) ||
+            (status != ACPI_ADR_SPACE_FIXED_HARDWARE)) {
                 dprintk("Invalid control/status registers (%x - %x)\n",
-                        data->acpi_data.control_register.space_id,
-                        space_id);
+                        control, status);
                 goto err_out;
         }
 
@@ -886,7 +887,7 @@ static int powernow_k8_cpu_init_acpi(struct powernow_k8_data *data)
         /* notify BIOS that we exist */
         acpi_processor_notify_smm(THIS_MODULE);
 
-        if (!alloc_cpumask_var(&data->acpi_data.shared_cpu_map, GFP_KERNEL)) {
+        if (!zalloc_cpumask_var(&data->acpi_data.shared_cpu_map, GFP_KERNEL)) {
                 printk(KERN_ERR PFX
                                 "unable to alloc powernow_k8_data cpumask\n");
                 ret_val = -ENOMEM;
diff --git a/arch/x86/kernel/cpu/cpufreq/speedstep-centrino.c b/arch/x86/kernel/cpu/cpufreq/speedstep-centrino.c
index c9f1fdc02830..55c831ed71ce 100644
--- a/arch/x86/kernel/cpu/cpufreq/speedstep-centrino.c
+++ b/arch/x86/kernel/cpu/cpufreq/speedstep-centrino.c
@@ -471,7 +471,7 @@ static int centrino_target (struct cpufreq_policy *policy,
 
         if (unlikely(!alloc_cpumask_var(&saved_mask, GFP_KERNEL)))
                 return -ENOMEM;
-        if (unlikely(!alloc_cpumask_var(&covered_cpus, GFP_KERNEL))) {
+        if (unlikely(!zalloc_cpumask_var(&covered_cpus, GFP_KERNEL))) {
                 free_cpumask_var(saved_mask);
                 return -ENOMEM;
         }
diff --git a/arch/x86/kernel/cpu/intel_cacheinfo.c b/arch/x86/kernel/cpu/intel_cacheinfo.c
index 483eda96e102..789efe217e1a 100644
--- a/arch/x86/kernel/cpu/intel_cacheinfo.c
+++ b/arch/x86/kernel/cpu/intel_cacheinfo.c
@@ -17,6 +17,7 @@
 
 #include <asm/processor.h>
 #include <asm/smp.h>
+#include <asm/k8.h>
 
 #define LVL_1_INST      1
 #define LVL_1_DATA      2
@@ -159,14 +160,6 @@ struct _cpuid4_info_regs {
         unsigned long can_disable;
 };
 
-#if defined(CONFIG_PCI) && defined(CONFIG_SYSFS)
-static struct pci_device_id k8_nb_id[] = {
-        { PCI_DEVICE(PCI_VENDOR_ID_AMD, 0x1103) },
-        { PCI_DEVICE(PCI_VENDOR_ID_AMD, 0x1203) },
-        {}
-};
-#endif
-
 unsigned short                  num_cache_leaves;
 
 /* AMD doesn't have CPUID4. Emulate it here to report the same
@@ -207,10 +200,17 @@ union l3_cache {
 };
 
 static const unsigned short __cpuinitconst assocs[] = {
-        [1] = 1, [2] = 2, [4] = 4, [6] = 8,
-        [8] = 16, [0xa] = 32, [0xb] = 48,
+        [1] = 1,
+        [2] = 2,
+        [4] = 4,
+        [6] = 8,
+        [8] = 16,
+        [0xa] = 32,
+        [0xb] = 48,
         [0xc] = 64,
-        [0xf] = 0xffff // ??
+        [0xd] = 96,
+        [0xe] = 128,
+        [0xf] = 0xffff /* fully associative - no way to show this currently */
 };
 
 static const unsigned char __cpuinitconst levels[] = { 1, 1, 2, 3 };
@@ -271,7 +271,8 @@ amd_cpuid4(int leaf, union _cpuid4_leaf_eax *eax,
         eax->split.type = types[leaf];
         eax->split.level = levels[leaf];
         if (leaf == 3)
-                eax->split.num_threads_sharing = current_cpu_data.x86_max_cores - 1;
+                eax->split.num_threads_sharing =
+                        current_cpu_data.x86_max_cores - 1;
         else
                 eax->split.num_threads_sharing = 0;
         eax->split.num_cores_on_die = current_cpu_data.x86_max_cores - 1;
@@ -291,6 +292,14 @@ amd_check_l3_disable(int index, struct _cpuid4_info_regs *this_leaf)
 {
         if (index < 3)
                 return;
+
+        if (boot_cpu_data.x86 == 0x11)
+                return;
+
+        /* see erratum #382 */
+        if ((boot_cpu_data.x86 == 0x10) && (boot_cpu_data.x86_model < 0x8))
+                return;
+
         this_leaf->can_disable = 1;
 }
 
@@ -696,97 +705,75 @@ static ssize_t show_type(struct _cpuid4_info *this_leaf, char *buf)
 #define to_object(k)    container_of(k, struct _index_kobject, kobj)
 #define to_attr(a)      container_of(a, struct _cache_attr, attr)
 
-#ifdef CONFIG_PCI
-static struct pci_dev *get_k8_northbridge(int node)
-{
-        struct pci_dev *dev = NULL;
-        int i;
-
-        for (i = 0; i <= node; i++) {
-                do {
-                        dev = pci_get_device(PCI_ANY_ID, PCI_ANY_ID, dev);
-                        if (!dev)
-                                break;
-                } while (!pci_match_id(&k8_nb_id[0], dev));
-                if (!dev)
-                        break;
-        }
-        return dev;
-}
-#else
-static struct pci_dev *get_k8_northbridge(int node)
-{
-        return NULL;
-}
-#endif
-
-static ssize_t show_cache_disable(struct _cpuid4_info *this_leaf, char *buf)
+static ssize_t show_cache_disable(struct _cpuid4_info *this_leaf, char *buf,
+                                  unsigned int index)
 {
-        const struct cpumask *mask = to_cpumask(this_leaf->shared_cpu_map);
-        int node = cpu_to_node(cpumask_first(mask));
-        struct pci_dev *dev = NULL;
-        ssize_t ret = 0;
-        int i;
+        int cpu = cpumask_first(to_cpumask(this_leaf->shared_cpu_map));
+        int node = cpu_to_node(cpu);
+        struct pci_dev *dev = node_to_k8_nb_misc(node);
+        unsigned int reg = 0;
 
         if (!this_leaf->can_disable)
-                return sprintf(buf, "Feature not enabled\n");
-
-        dev = get_k8_northbridge(node);
-        if (!dev) {
-                printk(KERN_ERR "Attempting AMD northbridge operation on a system with no northbridge\n");
                 return -EINVAL;
-        }
 
-        for (i = 0; i < 2; i++) {
-                unsigned int reg;
+        if (!dev)
+                return -EINVAL;
 
-                pci_read_config_dword(dev, 0x1BC + i * 4, &reg);
+        pci_read_config_dword(dev, 0x1BC + index * 4, &reg);
+        return sprintf(buf, "%x\n", reg);
+}
 
-                ret += sprintf(buf, "%sEntry: %d\n", buf, i);
-                ret += sprintf(buf, "%sReads:  %s\tNew Entries: %s\n",  
-                        buf,
-                        reg & 0x80000000 ? "Disabled" : "Allowed",
-                        reg & 0x40000000 ? "Disabled" : "Allowed");
-                ret += sprintf(buf, "%sSubCache: %x\tIndex: %x\n",
-                        buf, (reg & 0x30000) >> 16, reg & 0xfff);
-        }
-        return ret;
+#define SHOW_CACHE_DISABLE(index)                                       \
+static ssize_t                                                          \
+show_cache_disable_##index(struct _cpuid4_info *this_leaf, char *buf)   \
+{                                                                       \
+        return show_cache_disable(this_leaf, buf, index);               \
 }
+SHOW_CACHE_DISABLE(0)
+SHOW_CACHE_DISABLE(1)
 
-static ssize_t
-store_cache_disable(struct _cpuid4_info *this_leaf, const char *buf,
-                    size_t count)
+static ssize_t store_cache_disable(struct _cpuid4_info *this_leaf,
+        const char *buf, size_t count, unsigned int index)
 {
-        const struct cpumask *mask = to_cpumask(this_leaf->shared_cpu_map);
-        int node = cpu_to_node(cpumask_first(mask));
-        struct pci_dev *dev = NULL;
-        unsigned int ret, index, val;
+        int cpu = cpumask_first(to_cpumask(this_leaf->shared_cpu_map));
+        int node = cpu_to_node(cpu);
+        struct pci_dev *dev = node_to_k8_nb_misc(node);
+        unsigned long val = 0;
+        unsigned int scrubber = 0;
 
         if (!this_leaf->can_disable)
-                return 0;
-
-        if (strlen(buf) > 15)
                 return -EINVAL;
 
-        ret = sscanf(buf, "%x %x", &index, &val);
-        if (ret != 2)
+        if (!capable(CAP_SYS_ADMIN))
+                return -EPERM;
+
+        if (!dev)
                 return -EINVAL;
-        if (index > 1)
+
+        if (strict_strtoul(buf, 10, &val) < 0)
                 return -EINVAL;
 
         val |= 0xc0000000;
-        dev = get_k8_northbridge(node);
-        if (!dev) {
-                printk(KERN_ERR "Attempting AMD northbridge operation on a system with no northbridge\n");
-                return -EINVAL;
-        }
+
+        pci_read_config_dword(dev, 0x58, &scrubber);
+        scrubber &= ~0x1f000000;
+        pci_write_config_dword(dev, 0x58, scrubber);
 
         pci_write_config_dword(dev, 0x1BC + index * 4, val & ~0x40000000);
         wbinvd();
         pci_write_config_dword(dev, 0x1BC + index * 4, val);
+        return count;
+}
 
-        return 1;
+#define STORE_CACHE_DISABLE(index)                                      \
+static ssize_t                                                          \
+store_cache_disable_##index(struct _cpuid4_info *this_leaf,             \
+                            const char *buf, size_t count)              \
+{                                                                       \
+        return store_cache_disable(this_leaf, buf, count, index);       \
 }
+STORE_CACHE_DISABLE(0)
+STORE_CACHE_DISABLE(1)
 
 struct _cache_attr {
         struct attribute attr;
@@ -808,7 +795,10 @@ define_one_ro(size);
 define_one_ro(shared_cpu_map);
 define_one_ro(shared_cpu_list);
 
-static struct _cache_attr cache_disable = __ATTR(cache_disable, 0644, show_cache_disable, store_cache_disable);
+static struct _cache_attr cache_disable_0 = __ATTR(cache_disable_0, 0644,
+                show_cache_disable_0, store_cache_disable_0);
+static struct _cache_attr cache_disable_1 = __ATTR(cache_disable_1, 0644,
+                show_cache_disable_1, store_cache_disable_1);
 
 static struct attribute * default_attrs[] = {
         &type.attr,
@@ -820,7 +810,8 @@ static struct attribute * default_attrs[] = {
         &size.attr,
         &shared_cpu_map.attr,
         &shared_cpu_list.attr,
-        &cache_disable.attr,
+        &cache_disable_0.attr,
+        &cache_disable_1.attr,
         NULL
 };
 
diff --git a/arch/x86/kernel/cpu/mcheck/mce_intel_64.c b/arch/x86/kernel/cpu/mcheck/mce_intel_64.c
index 046087e9808f..f2ef6952c400 100644
--- a/arch/x86/kernel/cpu/mcheck/mce_intel_64.c
+++ b/arch/x86/kernel/cpu/mcheck/mce_intel_64.c
@@ -15,7 +15,6 @@
 #include <asm/hw_irq.h>
 #include <asm/idle.h>
 #include <asm/therm_throt.h>
-#include <asm/apic.h>
 
 #include "mce.h"
 
diff --git a/arch/x86/kernel/cpu/mtrr/cleanup.c b/arch/x86/kernel/cpu/mtrr/cleanup.c
index ce0fe4b5c04f..1d584a18a50d 100644
--- a/arch/x86/kernel/cpu/mtrr/cleanup.c
+++ b/arch/x86/kernel/cpu/mtrr/cleanup.c
@@ -808,7 +808,7 @@ int __init mtrr_cleanup(unsigned address_bits)
 
         if (!is_cpu(INTEL) || enable_mtrr_cleanup < 1)
                 return 0;
-        rdmsr(MTRRdefType_MSR, def, dummy);
+        rdmsr(MSR_MTRRdefType, def, dummy);
         def &= 0xff;
         if (def != MTRR_TYPE_UNCACHABLE)
                 return 0;
@@ -1003,7 +1003,7 @@ int __init mtrr_trim_uncached_memory(unsigned long end_pfn)
          */
         if (!is_cpu(INTEL) || disable_mtrr_trim)
                 return 0;
-        rdmsr(MTRRdefType_MSR, def, dummy);
+        rdmsr(MSR_MTRRdefType, def, dummy);
         def &= 0xff;
         if (def != MTRR_TYPE_UNCACHABLE)
                 return 0;
diff --git a/arch/x86/kernel/cpu/mtrr/generic.c b/arch/x86/kernel/cpu/mtrr/generic.c
index d21d4fb161f7..0543f69f0b27 100644
--- a/arch/x86/kernel/cpu/mtrr/generic.c
+++ b/arch/x86/kernel/cpu/mtrr/generic.c
@@ -20,9 +20,9 @@ struct fixed_range_block {
 };
 
 static struct fixed_range_block fixed_range_blocks[] = {
-        { MTRRfix64K_00000_MSR, 1 }, /* one  64k MTRR  */
-        { MTRRfix16K_80000_MSR, 2 }, /* two  16k MTRRs */
-        { MTRRfix4K_C0000_MSR,  8 }, /* eight 4k MTRRs */
+        { MSR_MTRRfix64K_00000, 1 }, /* one  64k MTRR  */
+        { MSR_MTRRfix16K_80000, 2 }, /* two  16k MTRRs */
+        { MSR_MTRRfix4K_C0000,  8 }, /* eight 4k MTRRs */
         {}
 };
 
@@ -194,12 +194,12 @@ get_fixed_ranges(mtrr_type * frs)
 
         k8_check_syscfg_dram_mod_en();
 
-        rdmsr(MTRRfix64K_00000_MSR, p[0], p[1]);
+        rdmsr(MSR_MTRRfix64K_00000, p[0], p[1]);
 
         for (i = 0; i < 2; i++)
-                rdmsr(MTRRfix16K_80000_MSR + i, p[2 + i * 2], p[3 + i * 2]);
+                rdmsr(MSR_MTRRfix16K_80000 + i, p[2 + i * 2], p[3 + i * 2]);
         for (i = 0; i < 8; i++)
-                rdmsr(MTRRfix4K_C0000_MSR + i, p[6 + i * 2], p[7 + i * 2]);
+                rdmsr(MSR_MTRRfix4K_C0000 + i, p[6 + i * 2], p[7 + i * 2]);
 }
 
 void mtrr_save_fixed_ranges(void *info)
@@ -310,7 +310,7 @@ void __init get_mtrr_state(void)
 
         vrs = mtrr_state.var_ranges;
 
-        rdmsr(MTRRcap_MSR, lo, dummy);
+        rdmsr(MSR_MTRRcap, lo, dummy);
         mtrr_state.have_fixed = (lo >> 8) & 1;
 
         for (i = 0; i < num_var_ranges; i++)
@@ -318,7 +318,7 @@ void __init get_mtrr_state(void)
         if (mtrr_state.have_fixed)
                 get_fixed_ranges(mtrr_state.fixed_ranges);
 
-        rdmsr(MTRRdefType_MSR, lo, dummy);
+        rdmsr(MSR_MTRRdefType, lo, dummy);
         mtrr_state.def_type = (lo & 0xff);
         mtrr_state.enabled = (lo & 0xc00) >> 10;
 
@@ -583,10 +583,10 @@ static void prepare_set(void) __acquires(set_atomicity_lock)
         __flush_tlb();
 
         /*  Save MTRR state */
-        rdmsr(MTRRdefType_MSR, deftype_lo, deftype_hi);
+        rdmsr(MSR_MTRRdefType, deftype_lo, deftype_hi);
 
         /*  Disable MTRRs, and set the default type to uncached  */
-        mtrr_wrmsr(MTRRdefType_MSR, deftype_lo & ~0xcff, deftype_hi);
+        mtrr_wrmsr(MSR_MTRRdefType, deftype_lo & ~0xcff, deftype_hi);
 }
 
 static void post_set(void) __releases(set_atomicity_lock)
@@ -595,7 +595,7 @@ static void post_set(void) __releases(set_atomicity_lock)
         __flush_tlb();
 
         /* Intel (P6) standard MTRRs */
-        mtrr_wrmsr(MTRRdefType_MSR, deftype_lo, deftype_hi);
+        mtrr_wrmsr(MSR_MTRRdefType, deftype_lo, deftype_hi);
                 
         /*  Enable caches  */
         write_cr0(read_cr0() & 0xbfffffff);
@@ -707,7 +707,7 @@ int generic_validate_add_page(unsigned long base, unsigned long size, unsigned i
 static int generic_have_wrcomb(void)
 {
         unsigned long config, dummy;
-        rdmsr(MTRRcap_MSR, config, dummy);
+        rdmsr(MSR_MTRRcap, config, dummy);
         return (config & (1 << 10));
 }
 
diff --git a/arch/x86/kernel/cpu/mtrr/main.c b/arch/x86/kernel/cpu/mtrr/main.c
index 03cda01f57c7..8fc248b5aeaf 100644
--- a/arch/x86/kernel/cpu/mtrr/main.c
+++ b/arch/x86/kernel/cpu/mtrr/main.c
@@ -104,7 +104,7 @@ static void __init set_num_var_ranges(void)
         unsigned long config = 0, dummy;
 
         if (use_intel()) {
-                rdmsr(MTRRcap_MSR, config, dummy);
+                rdmsr(MSR_MTRRcap, config, dummy);
         } else if (is_cpu(AMD))
                 config = 2;
         else if (is_cpu(CYRIX) || is_cpu(CENTAUR))
diff --git a/arch/x86/kernel/cpu/mtrr/mtrr.h b/arch/x86/kernel/cpu/mtrr/mtrr.h
index 77f67f7b347a..7538b767f206 100644
--- a/arch/x86/kernel/cpu/mtrr/mtrr.h
+++ b/arch/x86/kernel/cpu/mtrr/mtrr.h
@@ -5,21 +5,6 @@
 #include <linux/types.h>
 #include <linux/stddef.h>
 
-#define MTRRcap_MSR     0x0fe
-#define MTRRdefType_MSR 0x2ff
-
-#define MTRRfix64K_00000_MSR 0x250
-#define MTRRfix16K_80000_MSR 0x258
-#define MTRRfix16K_A0000_MSR 0x259
-#define MTRRfix4K_C0000_MSR 0x268
-#define MTRRfix4K_C8000_MSR 0x269
-#define MTRRfix4K_D0000_MSR 0x26a
-#define MTRRfix4K_D8000_MSR 0x26b
-#define MTRRfix4K_E0000_MSR 0x26c
-#define MTRRfix4K_E8000_MSR 0x26d
-#define MTRRfix4K_F0000_MSR 0x26e
-#define MTRRfix4K_F8000_MSR 0x26f
-
 #define MTRR_CHANGE_MASK_FIXED     0x01
 #define MTRR_CHANGE_MASK_VARIABLE  0x02
 #define MTRR_CHANGE_MASK_DEFTYPE   0x04
diff --git a/arch/x86/kernel/cpu/mtrr/state.c b/arch/x86/kernel/cpu/mtrr/state.c
index 7f7e2753685b..1f5fb1588d1f 100644
--- a/arch/x86/kernel/cpu/mtrr/state.c
+++ b/arch/x86/kernel/cpu/mtrr/state.c
@@ -35,7 +35,7 @@ void set_mtrr_prepare_save(struct set_mtrr_context *ctxt)
 
                 if (use_intel())
                         /*  Save MTRR state */
-                        rdmsr(MTRRdefType_MSR, ctxt->deftype_lo, ctxt->deftype_hi);
+                        rdmsr(MSR_MTRRdefType, ctxt->deftype_lo, ctxt->deftype_hi);
                 else
                         /* Cyrix ARRs - everything else were excluded at the top */
                         ctxt->ccr3 = getCx86(CX86_CCR3);
@@ -46,7 +46,7 @@ void set_mtrr_cache_disable(struct set_mtrr_context *ctxt)
 {
         if (use_intel())
                 /*  Disable MTRRs, and set the default type to uncached  */
-                mtrr_wrmsr(MTRRdefType_MSR, ctxt->deftype_lo & 0xf300UL,
+                mtrr_wrmsr(MSR_MTRRdefType, ctxt->deftype_lo & 0xf300UL,
                       ctxt->deftype_hi);
         else if (is_cpu(CYRIX))
                 /* Cyrix ARRs - everything else were excluded at the top */
@@ -64,7 +64,7 @@ void set_mtrr_done(struct set_mtrr_context *ctxt)
                 /*  Restore MTRRdefType  */
                 if (use_intel())
                         /* Intel (P6) standard MTRRs */
-                        mtrr_wrmsr(MTRRdefType_MSR, ctxt->deftype_lo, ctxt->deftype_hi);
+                        mtrr_wrmsr(MSR_MTRRdefType, ctxt->deftype_lo, ctxt->deftype_hi);
                 else
                         /* Cyrix ARRs - everything else was excluded at the top */
                         setCx86(CX86_CCR3, ctxt->ccr3);
diff --git a/arch/x86/kernel/cpu/perf_counter.c b/arch/x86/kernel/cpu/perf_counter.c
new file mode 100644
index 000000000000..895c82e78455
--- /dev/null
+++ b/arch/x86/kernel/cpu/perf_counter.c
@@ -0,0 +1,1704 @@
+/*
+ * Performance counter x86 architecture code
+ *
+ *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
+ *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
+ *  Copyright (C) 2009 Jaswinder Singh Rajput
+ *  Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
+ *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
+ *
+ *  For licencing details see kernel-base/COPYING
+ */
+
+#include <linux/perf_counter.h>
+#include <linux/capability.h>
+#include <linux/notifier.h>
+#include <linux/hardirq.h>
+#include <linux/kprobes.h>
+#include <linux/module.h>
+#include <linux/kdebug.h>
+#include <linux/sched.h>
+#include <linux/uaccess.h>
+
+#include <asm/apic.h>
+#include <asm/stacktrace.h>
+#include <asm/nmi.h>
+
+static u64 perf_counter_mask __read_mostly;
+
+struct cpu_hw_counters {
+        struct perf_counter     *counters[X86_PMC_IDX_MAX];
+        unsigned long           used_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
+        unsigned long           active_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
+        unsigned long           interrupts;
+        int                     enabled;
+};
+
+/*
+ * struct x86_pmu - generic x86 pmu
+ */
+struct x86_pmu {
+        const char      *name;
+        int             version;
+        int             (*handle_irq)(struct pt_regs *);
+        void            (*disable_all)(void);
+        void            (*enable_all)(void);
+        void            (*enable)(struct hw_perf_counter *, int);
+        void            (*disable)(struct hw_perf_counter *, int);
+        unsigned        eventsel;
+        unsigned        perfctr;
+        u64             (*event_map)(int);
+        u64             (*raw_event)(u64);
+        int             max_events;
+        int             num_counters;
+        int             num_counters_fixed;
+        int             counter_bits;
+        u64             counter_mask;
+        u64             max_period;
+        u64             intel_ctrl;
+};
+
+static struct x86_pmu x86_pmu __read_mostly;
+
+static DEFINE_PER_CPU(struct cpu_hw_counters, cpu_hw_counters) = {
+        .enabled = 1,
+};
+
+/*
+ * Intel PerfMon v3. Used on Core2 and later.
+ */
+static const u64 intel_perfmon_event_map[] =
+{
+  [PERF_COUNT_HW_CPU_CYCLES]            = 0x003c,
+  [PERF_COUNT_HW_INSTRUCTIONS]          = 0x00c0,
+  [PERF_COUNT_HW_CACHE_REFERENCES]      = 0x4f2e,
+  [PERF_COUNT_HW_CACHE_MISSES]          = 0x412e,
+  [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]   = 0x00c4,
+  [PERF_COUNT_HW_BRANCH_MISSES]         = 0x00c5,
+  [PERF_COUNT_HW_BUS_CYCLES]            = 0x013c,
+};
+
+static u64 intel_pmu_event_map(int event)
+{
+        return intel_perfmon_event_map[event];
+}
+
+/*
+ * Generalized hw caching related event table, filled
+ * in on a per model basis. A value of 0 means
+ * 'not supported', -1 means 'event makes no sense on
+ * this CPU', any other value means the raw event
+ * ID.
+ */
+
+#define C(x) PERF_COUNT_HW_CACHE_##x
+
+static u64 __read_mostly hw_cache_event_ids
+                                [PERF_COUNT_HW_CACHE_MAX]
+                                [PERF_COUNT_HW_CACHE_OP_MAX]
+                                [PERF_COUNT_HW_CACHE_RESULT_MAX];
+
+static const u64 nehalem_hw_cache_event_ids
+                                [PERF_COUNT_HW_CACHE_MAX]
+                                [PERF_COUNT_HW_CACHE_OP_MAX]
+                                [PERF_COUNT_HW_CACHE_RESULT_MAX] =
+{
+ [ C(L1D) ] = {
+        [ C(OP_READ) ] = {
+                [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI            */
+                [ C(RESULT_MISS)   ] = 0x0140, /* L1D_CACHE_LD.I_STATE         */
+        },
+        [ C(OP_WRITE) ] = {
+                [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI            */
+                [ C(RESULT_MISS)   ] = 0x0141, /* L1D_CACHE_ST.I_STATE         */
+        },
+        [ C(OP_PREFETCH) ] = {
+                [ C(RESULT_ACCESS) ] = 0x014e, /* L1D_PREFETCH.REQUESTS        */
+                [ C(RESULT_MISS)   ] = 0x024e, /* L1D_PREFETCH.MISS            */
+        },
+ },
+ [ C(L1I ) ] = {
+        [ C(OP_READ) ] = {
+                [ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS                    */
+                [ C(RESULT_MISS)   ] = 0x0280, /* L1I.MISSES                   */
+        },
+        [ C(OP_WRITE) ] = {
+                [ C(RESULT_ACCESS) ] = -1,
+                [ C(RESULT_MISS)   ] = -1,
+        },
+        [ C(OP_PREFETCH) ] = {
+                [ C(RESULT_ACCESS) ] = 0x0,
+                [ C(RESULT_MISS)   ] = 0x0,
+        },
+ },
+ [ C(LL  ) ] = {
+        [ C(OP_READ) ] = {
+                [ C(RESULT_ACCESS) ] = 0x0324, /* L2_RQSTS.LOADS               */
+                [ C(RESULT_MISS)   ] = 0x0224, /* L2_RQSTS.LD_MISS             */
+        },
+        [ C(OP_WRITE) ] = {
+                [ C(RESULT_ACCESS) ] = 0x0c24, /* L2_RQSTS.RFOS                */
+                [ C(RESULT_MISS)   ] = 0x0824, /* L2_RQSTS.RFO_MISS            */
+        },
+        [ C(OP_PREFETCH) ] = {
+                [ C(RESULT_ACCESS) ] = 0x4f2e, /* LLC Reference                */
+                [ C(RESULT_MISS)   ] = 0x412e, /* LLC Misses                   */
+        },
+ },
+ [ C(DTLB) ] = {
+        [ C(OP_READ) ] = {
+                [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI   (alias)  */
+                [ C(RESULT_MISS)   ] = 0x0108, /* DTLB_LOAD_MISSES.ANY         */
+        },
+        [ C(OP_WRITE) ] = {
+                [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI   (alias)  */
+                [ C(RESULT_MISS)   ] = 0x010c, /* MEM_STORE_RETIRED.DTLB_MISS  */
+        },
+        [ C(OP_PREFETCH) ] = {
+                [ C(RESULT_ACCESS) ] = 0x0,
+                [ C(RESULT_MISS)   ] = 0x0,
+        },
+ },
+ [ C(ITLB) ] = {
+        [ C(OP_READ) ] = {
+                [ C(RESULT_ACCESS) ] = 0x01c0, /* INST_RETIRED.ANY_P           */
+                [ C(RESULT_MISS)   ] = 0x20c8, /* ITLB_MISS_RETIRED            */
+        },
+        [ C(OP_WRITE) ] = {
+                [ C(RESULT_ACCESS) ] = -1,
+                [ C(RESULT_MISS)   ] = -1,
+        },
+        [ C(OP_PREFETCH) ] = {
+                [ C(RESULT_ACCESS) ] = -1,
+                [ C(RESULT_MISS)   ] = -1,
+        },
+ },
+ [ C(BPU ) ] = {
+        [ C(OP_READ) ] = {
+                [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ALL_BRANCHES */
+                [ C(RESULT_MISS)   ] = 0x03e8, /* BPU_CLEARS.ANY               */
+        },
+        [ C(OP_WRITE) ] = {
+                [ C(RESULT_ACCESS) ] = -1,
+                [ C(RESULT_MISS)   ] = -1,
+        },
+        [ C(OP_PREFETCH) ] = {
+                [ C(RESULT_ACCESS) ] = -1,
+                [ C(RESULT_MISS)   ] = -1,
+        },
+ },
+};
+
+static const u64 core2_hw_cache_event_ids
+                                [PERF_COUNT_HW_CACHE_MAX]
+                                [PERF_COUNT_HW_CACHE_OP_MAX]
+                                [PERF_COUNT_HW_CACHE_RESULT_MAX] =
+{
+ [ C(L1D) ] = {
+        [ C(OP_READ) ] = {
+                [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI          */
+                [ C(RESULT_MISS)   ] = 0x0140, /* L1D_CACHE_LD.I_STATE       */
+        },
+        [ C(OP_WRITE) ] = {
+                [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI          */
+                [ C(RESULT_MISS)   ] = 0x0141, /* L1D_CACHE_ST.I_STATE       */
+        },
+        [ C(OP_PREFETCH) ] = {
+                [ C(RESULT_ACCESS) ] = 0x104e, /* L1D_PREFETCH.REQUESTS      */
+                [ C(RESULT_MISS)   ] = 0,
+        },
+ },
+ [ C(L1I ) ] = {
+        [ C(OP_READ) ] = {
+                [ C(RESULT_ACCESS) ] = 0x0080, /* L1I.READS                  */
+                [ C(RESULT_MISS)   ] = 0x0081, /* L1I.MISSES                 */
+        },
+        [ C(OP_WRITE) ] = {
+                [ C(RESULT_ACCESS) ] = -1,
+                [ C(RESULT_MISS)   ] = -1,
+        },
+        [ C(OP_PREFETCH) ] = {
+                [ C(RESULT_ACCESS) ] = 0,
+                [ C(RESULT_MISS)   ] = 0,
+        },
+ },
+ [ C(LL  ) ] = {
+        [ C(OP_READ) ] = {
+                [ C(RESULT_ACCESS) ] = 0x4f29, /* L2_LD.MESI                 */
+                [ C(RESULT_MISS)   ] = 0x4129, /* L2_LD.ISTATE               */
+        },
+        [ C(OP_WRITE) ] = {
+                [ C(RESULT_ACCESS) ] = 0x4f2A, /* L2_ST.MESI                 */
+                [ C(RESULT_MISS)   ] = 0x412A, /* L2_ST.ISTATE               */
+        },
+        [ C(OP_PREFETCH) ] = {
+                [ C(RESULT_ACCESS) ] = 0,
+                [ C(RESULT_MISS)   ] = 0,
+        },
+ },
+ [ C(DTLB) ] = {
+        [ C(OP_READ) ] = {
+                [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI  (alias) */
+                [ C(RESULT_MISS)   ] = 0x0208, /* DTLB_MISSES.MISS_LD        */
+        },
+        [ C(OP_WRITE) ] = {
+                [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI  (alias) */
+                [ C(RESULT_MISS)   ] = 0x0808, /* DTLB_MISSES.MISS_ST        */
+        },
+        [ C(OP_PREFETCH) ] = {
+                [ C(RESULT_ACCESS) ] = 0,
+                [ C(RESULT_MISS)   ] = 0,
+        },
+ },
+ [ C(ITLB) ] = {
+        [ C(OP_READ) ] = {
+                [ C(RESULT_ACCESS) ] = 0x00c0, /* INST_RETIRED.ANY_P         */
+                [ C(RESULT_MISS)   ] = 0x1282, /* ITLBMISSES                 */
+        },
+        [ C(OP_WRITE) ] = {
+                [ C(RESULT_ACCESS) ] = -1,
+                [ C(RESULT_MISS)   ] = -1,
+        },
+        [ C(OP_PREFETCH) ] = {
+                [ C(RESULT_ACCESS) ] = -1,
+                [ C(RESULT_MISS)   ] = -1,
+        },
+ },
+ [ C(BPU ) ] = {
+        [ C(OP_READ) ] = {
+                [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ANY        */
+                [ C(RESULT_MISS)   ] = 0x00c5, /* BP_INST_RETIRED.MISPRED    */
+        },
+        [ C(OP_WRITE) ] = {
+                [ C(RESULT_ACCESS) ] = -1,
+                [ C(RESULT_MISS)   ] = -1,
+        },
+        [ C(OP_PREFETCH) ] = {
+                [ C(RESULT_ACCESS) ] = -1,
+                [ C(RESULT_MISS)   ] = -1,
+        },
+ },
+};
+
+static const u64 atom_hw_cache_event_ids
+                                [PERF_COUNT_HW_CACHE_MAX]
+                                [PERF_COUNT_HW_CACHE_OP_MAX]
+                                [PERF_COUNT_HW_CACHE_RESULT_MAX] =
+{
+ [ C(L1D) ] = {
+        [ C(OP_READ) ] = {
+                [ C(RESULT_ACCESS) ] = 0x2140, /* L1D_CACHE.LD               */
+                [ C(RESULT_MISS)   ] = 0,
+        },
+        [ C(OP_WRITE) ] = {
+                [ C(RESULT_ACCESS) ] = 0x2240, /* L1D_CACHE.ST               */
+                [ C(RESULT_MISS)   ] = 0,
+        },
+        [ C(OP_PREFETCH) ] = {
+                [ C(RESULT_ACCESS) ] = 0x0,
+                [ C(RESULT_MISS)   ] = 0,
+        },
+ },
+ [ C(L1I ) ] = {
+        [ C(OP_READ) ] = {
+                [ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS                  */
+                [ C(RESULT_MISS)   ] = 0x0280, /* L1I.MISSES                 */
+        },
+        [ C(OP_WRITE) ] = {
+                [ C(RESULT_ACCESS) ] = -1,
+                [ C(RESULT_MISS)   ] = -1,
+        },
+        [ C(OP_PREFETCH) ] = {
+                [ C(RESULT_ACCESS) ] = 0,
+                [ C(RESULT_MISS)   ] = 0,
+        },
+ },
+ [ C(LL  ) ] = {
+        [ C(OP_READ) ] = {
+                [ C(RESULT_ACCESS) ] = 0x4f29, /* L2_LD.MESI                 */
+                [ C(RESULT_MISS)   ] = 0x4129, /* L2_LD.ISTATE               */
+        },
+        [ C(OP_WRITE) ] = {
+                [ C(RESULT_ACCESS) ] = 0x4f2A, /* L2_ST.MESI                 */
+                [ C(RESULT_MISS)   ] = 0x412A, /* L2_ST.ISTATE               */
+        },
+        [ C(OP_PREFETCH) ] = {
+                [ C(RESULT_ACCESS) ] = 0,
+                [ C(RESULT_MISS)   ] = 0,
+        },
+ },
+ [ C(DTLB) ] = {
+        [ C(OP_READ) ] = {
+                [ C(RESULT_ACCESS) ] = 0x2140, /* L1D_CACHE_LD.MESI  (alias) */
+                [ C(RESULT_MISS)   ] = 0x0508, /* DTLB_MISSES.MISS_LD        */
+        },
+        [ C(OP_WRITE) ] = {
+                [ C(RESULT_ACCESS) ] = 0x2240, /* L1D_CACHE_ST.MESI  (alias) */
+                [ C(RESULT_MISS)   ] = 0x0608, /* DTLB_MISSES.MISS_ST        */
+        },
+        [ C(OP_PREFETCH) ] = {
+                [ C(RESULT_ACCESS) ] = 0,
+                [ C(RESULT_MISS)   ] = 0,
+        },
+ },
+ [ C(ITLB) ] = {
+        [ C(OP_READ) ] = {
+                [ C(RESULT_ACCESS) ] = 0x00c0, /* INST_RETIRED.ANY_P         */
+                [ C(RESULT_MISS)   ] = 0x0282, /* ITLB.MISSES                */
+        },
+        [ C(OP_WRITE) ] = {
+                [ C(RESULT_ACCESS) ] = -1,
+                [ C(RESULT_MISS)   ] = -1,
+        },
+        [ C(OP_PREFETCH) ] = {
+                [ C(RESULT_ACCESS) ] = -1,
+                [ C(RESULT_MISS)   ] = -1,
+        },
+ },
+ [ C(BPU ) ] = {
+        [ C(OP_READ) ] = {
+                [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ANY        */
+                [ C(RESULT_MISS)   ] = 0x00c5, /* BP_INST_RETIRED.MISPRED    */
+        },
+        [ C(OP_WRITE) ] = {
+                [ C(RESULT_ACCESS) ] = -1,
+                [ C(RESULT_MISS)   ] = -1,
+        },
+        [ C(OP_PREFETCH) ] = {
+                [ C(RESULT_ACCESS) ] = -1,
+                [ C(RESULT_MISS)   ] = -1,
+        },
+ },
+};
+
+static u64 intel_pmu_raw_event(u64 event)
+{
+#define CORE_EVNTSEL_EVENT_MASK         0x000000FFULL
+#define CORE_EVNTSEL_UNIT_MASK          0x0000FF00ULL
+#define CORE_EVNTSEL_EDGE_MASK          0x00040000ULL
+#define CORE_EVNTSEL_INV_MASK           0x00800000ULL
+#define CORE_EVNTSEL_COUNTER_MASK       0xFF000000ULL
+
+#define CORE_EVNTSEL_MASK               \
+        (CORE_EVNTSEL_EVENT_MASK |      \
+         CORE_EVNTSEL_UNIT_MASK  |      \
+         CORE_EVNTSEL_EDGE_MASK  |      \
+         CORE_EVNTSEL_INV_MASK  |       \
+         CORE_EVNTSEL_COUNTER_MASK)
+
+        return event & CORE_EVNTSEL_MASK;
+}
+
+static const u64 amd_0f_hw_cache_event_ids
+                                [PERF_COUNT_HW_CACHE_MAX]
+                                [PERF_COUNT_HW_CACHE_OP_MAX]
+                                [PERF_COUNT_HW_CACHE_RESULT_MAX] =
+{
+ [ C(L1D) ] = {
+        [ C(OP_READ) ] = {
+                [ C(RESULT_ACCESS) ] = 0,
+                [ C(RESULT_MISS)   ] = 0,
+        },
+        [ C(OP_WRITE) ] = {
+                [ C(RESULT_ACCESS) ] = 0,
+                [ C(RESULT_MISS)   ] = 0,
+        },
+        [ C(OP_PREFETCH) ] = {
+                [ C(RESULT_ACCESS) ] = 0,
+                [ C(RESULT_MISS)   ] = 0,
+        },
+ },
+ [ C(L1I ) ] = {
+        [ C(OP_READ) ] = {
+                [ C(RESULT_ACCESS) ] = 0x0080, /* Instruction cache fetches  */
+                [ C(RESULT_MISS)   ] = 0x0081, /* Instruction cache misses   */
+        },
+        [ C(OP_WRITE) ] = {
+                [ C(RESULT_ACCESS) ] = -1,
+                [ C(RESULT_MISS)   ] = -1,
+        },
+        [ C(OP_PREFETCH) ] = {
+                [ C(RESULT_ACCESS) ] = 0,
+                [ C(RESULT_MISS)   ] = 0,
+        },
+ },
+ [ C(LL  ) ] = {
+        [ C(OP_READ) ] = {
+                [ C(RESULT_ACCESS) ] = 0,
+                [ C(RESULT_MISS)   ] = 0,
+        },
+        [ C(OP_WRITE) ] = {
+                [ C(RESULT_ACCESS) ] = 0,
+                [ C(RESULT_MISS)   ] = 0,
+        },
+        [ C(OP_PREFETCH) ] = {
+                [ C(RESULT_ACCESS) ] = 0,
+                [ C(RESULT_MISS)   ] = 0,
+        },
+ },
+ [ C(DTLB) ] = {
+        [ C(OP_READ) ] = {
+                [ C(RESULT_ACCESS) ] = 0,
+                [ C(RESULT_MISS)   ] = 0,
+        },
+        [ C(OP_WRITE) ] = {
+                [ C(RESULT_ACCESS) ] = 0,
+                [ C(RESULT_MISS)   ] = 0,
+        },
+        [ C(OP_PREFETCH) ] = {
+                [ C(RESULT_ACCESS) ] = 0,
+                [ C(RESULT_MISS)   ] = 0,
+        },
+ },
+ [ C(ITLB) ] = {
+        [ C(OP_READ) ] = {
+                [ C(RESULT_ACCESS) ] = 0x0080, /* Instruction fecthes        */
+                [ C(RESULT_MISS)   ] = 0x0085, /* Instr. fetch ITLB misses   */
+        },
+        [ C(OP_WRITE) ] = {
+                [ C(RESULT_ACCESS) ] = -1,
+                [ C(RESULT_MISS)   ] = -1,
+        },
+        [ C(OP_PREFETCH) ] = {
+                [ C(RESULT_ACCESS) ] = -1,
+                [ C(RESULT_MISS)   ] = -1,
+        },
+ },
+ [ C(BPU ) ] = {
+        [ C(OP_READ) ] = {
+                [ C(RESULT_ACCESS) ] = 0x00c2, /* Retired Branch Instr.      */
+                [ C(RESULT_MISS)   ] = 0x00c3, /* Retired Mispredicted BI    */
+        },
+        [ C(OP_WRITE) ] = {
+                [ C(RESULT_ACCESS) ] = -1,
+                [ C(RESULT_MISS)   ] = -1,
+        },
+        [ C(OP_PREFETCH) ] = {
+                [ C(RESULT_ACCESS) ] = -1,
+                [ C(RESULT_MISS)   ] = -1,
+        },
+ },
+};
+
+/*
+ * AMD Performance Monitor K7 and later.
+ */
+static const u64 amd_perfmon_event_map[] =
+{
+  [PERF_COUNT_HW_CPU_CYCLES]            = 0x0076,
+  [PERF_COUNT_HW_INSTRUCTIONS]          = 0x00c0,
+  [PERF_COUNT_HW_CACHE_REFERENCES]      = 0x0080,
+  [PERF_COUNT_HW_CACHE_MISSES]          = 0x0081,
+  [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]   = 0x00c4,
+  [PERF_COUNT_HW_BRANCH_MISSES]         = 0x00c5,
+};
+
+static u64 amd_pmu_event_map(int event)
+{
+        return amd_perfmon_event_map[event];
+}
+
+static u64 amd_pmu_raw_event(u64 event)
+{
+#define K7_EVNTSEL_EVENT_MASK   0x7000000FFULL
+#define K7_EVNTSEL_UNIT_MASK    0x00000FF00ULL
+#define K7_EVNTSEL_EDGE_MASK    0x000040000ULL
+#define K7_EVNTSEL_INV_MASK     0x000800000ULL
+#define K7_EVNTSEL_COUNTER_MASK 0x0FF000000ULL
+
+#define K7_EVNTSEL_MASK                 \
+        (K7_EVNTSEL_EVENT_MASK |        \
+         K7_EVNTSEL_UNIT_MASK  |        \
+         K7_EVNTSEL_EDGE_MASK  |        \
+         K7_EVNTSEL_INV_MASK   |        \
+         K7_EVNTSEL_COUNTER_MASK)
+
+        return event & K7_EVNTSEL_MASK;
+}
+
+/*
+ * Propagate counter elapsed time into the generic counter.
+ * Can only be executed on the CPU where the counter is active.
+ * Returns the delta events processed.
+ */
+static u64
+x86_perf_counter_update(struct perf_counter *counter,
+                        struct hw_perf_counter *hwc, int idx)
+{
+        int shift = 64 - x86_pmu.counter_bits;
+        u64 prev_raw_count, new_raw_count;
+        s64 delta;
+
+        /*
+         * Careful: an NMI might modify the previous counter value.
+         *
+         * Our tactic to handle this is to first atomically read and
+         * exchange a new raw count - then add that new-prev delta
+         * count to the generic counter atomically:
+         */
+again:
+        prev_raw_count = atomic64_read(&hwc->prev_count);
+        rdmsrl(hwc->counter_base + idx, new_raw_count);
+
+        if (atomic64_cmpxchg(&hwc->prev_count, prev_raw_count,
+                                        new_raw_count) != prev_raw_count)
+                goto again;
+
+        /*
+         * Now we have the new raw value and have updated the prev
+         * timestamp already. We can now calculate the elapsed delta
+         * (counter-)time and add that to the generic counter.
+         *
+         * Careful, not all hw sign-extends above the physical width
+         * of the count.
+         */
+        delta = (new_raw_count << shift) - (prev_raw_count << shift);
+        delta >>= shift;
+
+        atomic64_add(delta, &counter->count);
+        atomic64_sub(delta, &hwc->period_left);
+
+        return new_raw_count;
+}
+
+static atomic_t active_counters;
+static DEFINE_MUTEX(pmc_reserve_mutex);
+
+static bool reserve_pmc_hardware(void)
+{
+        int i;
+
+        if (nmi_watchdog == NMI_LOCAL_APIC)
+                disable_lapic_nmi_watchdog();
+
+        for (i = 0; i < x86_pmu.num_counters; i++) {
+                if (!reserve_perfctr_nmi(x86_pmu.perfctr + i))
+                        goto perfctr_fail;
+        }
+
+        for (i = 0; i < x86_pmu.num_counters; i++) {
+                if (!reserve_evntsel_nmi(x86_pmu.eventsel + i))
+                        goto eventsel_fail;
+        }
+
+        return true;
+
+eventsel_fail:
+        for (i--; i >= 0; i--)
+                release_evntsel_nmi(x86_pmu.eventsel + i);
+
+        i = x86_pmu.num_counters;
+
+perfctr_fail:
+        for (i--; i >= 0; i--)
+                release_perfctr_nmi(x86_pmu.perfctr + i);
+
+        if (nmi_watchdog == NMI_LOCAL_APIC)
+                enable_lapic_nmi_watchdog();
+
+        return false;
+}
+
+static void release_pmc_hardware(void)
+{
+        int i;
+
+        for (i = 0; i < x86_pmu.num_counters; i++) {
+                release_perfctr_nmi(x86_pmu.perfctr + i);
+                release_evntsel_nmi(x86_pmu.eventsel + i);
+        }
+
+        if (nmi_watchdog == NMI_LOCAL_APIC)
+                enable_lapic_nmi_watchdog();
+}
+
+static void hw_perf_counter_destroy(struct perf_counter *counter)
+{
+        if (atomic_dec_and_mutex_lock(&active_counters, &pmc_reserve_mutex)) {
+                release_pmc_hardware();
+                mutex_unlock(&pmc_reserve_mutex);
+        }
+}
+
+static inline int x86_pmu_initialized(void)
+{
+        return x86_pmu.handle_irq != NULL;
+}
+
+static inline int
+set_ext_hw_attr(struct hw_perf_counter *hwc, struct perf_counter_attr *attr)
+{
+        unsigned int cache_type, cache_op, cache_result;
+        u64 config, val;
+
+        config = attr->config;
+
+        cache_type = (config >>  0) & 0xff;
+        if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
+                return -EINVAL;
+
+        cache_op = (config >>  8) & 0xff;
+        if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
+                return -EINVAL;
+
+        cache_result = (config >> 16) & 0xff;
+        if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
+                return -EINVAL;
+
+        val = hw_cache_event_ids[cache_type][cache_op][cache_result];
+
+        if (val == 0)
+                return -ENOENT;
+
+        if (val == -1)
+                return -EINVAL;
+
+        hwc->config |= val;
+
+        return 0;
+}
+
+/*
+ * Setup the hardware configuration for a given attr_type
+ */
+static int __hw_perf_counter_init(struct perf_counter *counter)
+{
+        struct perf_counter_attr *attr = &counter->attr;
+        struct hw_perf_counter *hwc = &counter->hw;
+        int err;
+
+        if (!x86_pmu_initialized())
+                return -ENODEV;
+
+        err = 0;
+        if (!atomic_inc_not_zero(&active_counters)) {
+                mutex_lock(&pmc_reserve_mutex);
+                if (atomic_read(&active_counters) == 0 && !reserve_pmc_hardware())
+                        err = -EBUSY;
+                else
+                        atomic_inc(&active_counters);
+                mutex_unlock(&pmc_reserve_mutex);
+        }
+        if (err)
+                return err;
+
+        /*
+         * Generate PMC IRQs:
+         * (keep 'enabled' bit clear for now)
+         */
+        hwc->config = ARCH_PERFMON_EVENTSEL_INT;
+
+        /*
+         * Count user and OS events unless requested not to.
+         */
+        if (!attr->exclude_user)
+                hwc->config |= ARCH_PERFMON_EVENTSEL_USR;
+        if (!attr->exclude_kernel)
+                hwc->config |= ARCH_PERFMON_EVENTSEL_OS;
+
+        if (!hwc->sample_period) {
+                hwc->sample_period = x86_pmu.max_period;
+                hwc->last_period = hwc->sample_period;
+                atomic64_set(&hwc->period_left, hwc->sample_period);
+        }
+
+        counter->destroy = hw_perf_counter_destroy;
+
+        /*
+         * Raw event type provide the config in the event structure
+         */
+        if (attr->type == PERF_TYPE_RAW) {
+                hwc->config |= x86_pmu.raw_event(attr->config);
+                return 0;
+        }
+
+        if (attr->type == PERF_TYPE_HW_CACHE)
+                return set_ext_hw_attr(hwc, attr);
+
+        if (attr->config >= x86_pmu.max_events)
+                return -EINVAL;
+        /*
+         * The generic map:
+         */
+        hwc->config |= x86_pmu.event_map(attr->config);
+
+        return 0;
+}
+
+static void intel_pmu_disable_all(void)
+{
+        wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0);
+}
+
+static void amd_pmu_disable_all(void)
+{
+        struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
+        int idx;
+
+        if (!cpuc->enabled)
+                return;
+
+        cpuc->enabled = 0;
+        /*
+         * ensure we write the disable before we start disabling the
+         * counters proper, so that amd_pmu_enable_counter() does the
+         * right thing.
+         */
+        barrier();
+
+        for (idx = 0; idx < x86_pmu.num_counters; idx++) {
+                u64 val;
+
+                if (!test_bit(idx, cpuc->active_mask))
+                        continue;
+                rdmsrl(MSR_K7_EVNTSEL0 + idx, val);
+                if (!(val & ARCH_PERFMON_EVENTSEL0_ENABLE))
+                        continue;
+                val &= ~ARCH_PERFMON_EVENTSEL0_ENABLE;
+                wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
+        }
+}
+
+void hw_perf_disable(void)
+{
+        if (!x86_pmu_initialized())
+                return;
+        return x86_pmu.disable_all();
+}
+
+static void intel_pmu_enable_all(void)
+{
+        wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, x86_pmu.intel_ctrl);
+}
+
+static void amd_pmu_enable_all(void)
+{
+        struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
+        int idx;
+
+        if (cpuc->enabled)
+                return;
+
+        cpuc->enabled = 1;
+        barrier();
+
+        for (idx = 0; idx < x86_pmu.num_counters; idx++) {
+                u64 val;
+
+                if (!test_bit(idx, cpuc->active_mask))
+                        continue;
+                rdmsrl(MSR_K7_EVNTSEL0 + idx, val);
+                if (val & ARCH_PERFMON_EVENTSEL0_ENABLE)
+                        continue;
+                val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
+                wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
+        }
+}
+
+void hw_perf_enable(void)
+{
+        if (!x86_pmu_initialized())
+                return;
+        x86_pmu.enable_all();
+}
+
+static inline u64 intel_pmu_get_status(void)
+{
+        u64 status;
+
+        rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
+
+        return status;
+}
+
+static inline void intel_pmu_ack_status(u64 ack)
+{
+        wrmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, ack);
+}
+
+static inline void x86_pmu_enable_counter(struct hw_perf_counter *hwc, int idx)
+{
+        int err;
+        err = checking_wrmsrl(hwc->config_base + idx,
+                              hwc->config | ARCH_PERFMON_EVENTSEL0_ENABLE);
+}
+
+static inline void x86_pmu_disable_counter(struct hw_perf_counter *hwc, int idx)
+{
+        int err;
+        err = checking_wrmsrl(hwc->config_base + idx,
+                              hwc->config);
+}
+
+static inline void
+intel_pmu_disable_fixed(struct hw_perf_counter *hwc, int __idx)
+{
+        int idx = __idx - X86_PMC_IDX_FIXED;
+        u64 ctrl_val, mask;
+        int err;
+
+        mask = 0xfULL << (idx * 4);
+
+        rdmsrl(hwc->config_base, ctrl_val);
+        ctrl_val &= ~mask;
+        err = checking_wrmsrl(hwc->config_base, ctrl_val);
+}
+
+static inline void
+intel_pmu_disable_counter(struct hw_perf_counter *hwc, int idx)
+{
+        if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
+                intel_pmu_disable_fixed(hwc, idx);
+                return;
+        }
+
+        x86_pmu_disable_counter(hwc, idx);
+}
+
+static inline void
+amd_pmu_disable_counter(struct hw_perf_counter *hwc, int idx)
+{
+        x86_pmu_disable_counter(hwc, idx);
+}
+
+static DEFINE_PER_CPU(u64, prev_left[X86_PMC_IDX_MAX]);
+
+/*
+ * Set the next IRQ period, based on the hwc->period_left value.
+ * To be called with the counter disabled in hw:
+ */
+static int
+x86_perf_counter_set_period(struct perf_counter *counter,
+                             struct hw_perf_counter *hwc, int idx)
+{
+        s64 left = atomic64_read(&hwc->period_left);
+        s64 period = hwc->sample_period;
+        int err, ret = 0;
+
+        /*
+         * If we are way outside a reasoable range then just skip forward:
+         */
+        if (unlikely(left <= -period)) {
+                left = period;
+                atomic64_set(&hwc->period_left, left);
+                hwc->last_period = period;
+                ret = 1;
+        }
+
+        if (unlikely(left <= 0)) {
+                left += period;
+                atomic64_set(&hwc->period_left, left);
+                hwc->last_period = period;
+                ret = 1;
+        }
+        /*
+         * Quirk: certain CPUs dont like it if just 1 event is left:
+         */
+        if (unlikely(left < 2))
+                left = 2;
+
+        if (left > x86_pmu.max_period)
+                left = x86_pmu.max_period;
+
+        per_cpu(prev_left[idx], smp_processor_id()) = left;
+
+        /*
+         * The hw counter starts counting from this counter offset,
+         * mark it to be able to extra future deltas:
+         */
+        atomic64_set(&hwc->prev_count, (u64)-left);
+
+        err = checking_wrmsrl(hwc->counter_base + idx,
+                             (u64)(-left) & x86_pmu.counter_mask);
+
+        return ret;
+}
+
+static inline void
+intel_pmu_enable_fixed(struct hw_perf_counter *hwc, int __idx)
+{
+        int idx = __idx - X86_PMC_IDX_FIXED;
+        u64 ctrl_val, bits, mask;
+        int err;
+
+        /*
+         * Enable IRQ generation (0x8),
+         * and enable ring-3 counting (0x2) and ring-0 counting (0x1)
+         * if requested:
+         */
+        bits = 0x8ULL;
+        if (hwc->config & ARCH_PERFMON_EVENTSEL_USR)
+                bits |= 0x2;
+        if (hwc->config & ARCH_PERFMON_EVENTSEL_OS)
+                bits |= 0x1;
+        bits <<= (idx * 4);
+        mask = 0xfULL << (idx * 4);
+
+        rdmsrl(hwc->config_base, ctrl_val);
+        ctrl_val &= ~mask;
+        ctrl_val |= bits;
+        err = checking_wrmsrl(hwc->config_base, ctrl_val);
+}
+
+static void intel_pmu_enable_counter(struct hw_perf_counter *hwc, int idx)
+{
+        if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
+                intel_pmu_enable_fixed(hwc, idx);
+                return;
+        }
+
+        x86_pmu_enable_counter(hwc, idx);
+}
+
+static void amd_pmu_enable_counter(struct hw_perf_counter *hwc, int idx)
+{
+        struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
+
+        if (cpuc->enabled)
+                x86_pmu_enable_counter(hwc, idx);
+        else
+                x86_pmu_disable_counter(hwc, idx);
+}
+
+static int
+fixed_mode_idx(struct perf_counter *counter, struct hw_perf_counter *hwc)
+{
+        unsigned int event;
+
+        if (!x86_pmu.num_counters_fixed)
+                return -1;
+
+        event = hwc->config & ARCH_PERFMON_EVENT_MASK;
+
+        if (unlikely(event == x86_pmu.event_map(PERF_COUNT_HW_INSTRUCTIONS)))
+                return X86_PMC_IDX_FIXED_INSTRUCTIONS;
+        if (unlikely(event == x86_pmu.event_map(PERF_COUNT_HW_CPU_CYCLES)))
+                return X86_PMC_IDX_FIXED_CPU_CYCLES;
+        if (unlikely(event == x86_pmu.event_map(PERF_COUNT_HW_BUS_CYCLES)))
+                return X86_PMC_IDX_FIXED_BUS_CYCLES;
+
+        return -1;
+}
+
+/*
+ * Find a PMC slot for the freshly enabled / scheduled in counter:
+ */
+static int x86_pmu_enable(struct perf_counter *counter)
+{
+        struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
+        struct hw_perf_counter *hwc = &counter->hw;
+        int idx;
+
+        idx = fixed_mode_idx(counter, hwc);
+        if (idx >= 0) {
+                /*
+                 * Try to get the fixed counter, if that is already taken
+                 * then try to get a generic counter:
+                 */
+                if (test_and_set_bit(idx, cpuc->used_mask))
+                        goto try_generic;
+
+                hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
+                /*
+                 * We set it so that counter_base + idx in wrmsr/rdmsr maps to
+                 * MSR_ARCH_PERFMON_FIXED_CTR0 ... CTR2:
+                 */
+                hwc->counter_base =
+                        MSR_ARCH_PERFMON_FIXED_CTR0 - X86_PMC_IDX_FIXED;
+                hwc->idx = idx;
+        } else {
+                idx = hwc->idx;
+                /* Try to get the previous generic counter again */
+                if (test_and_set_bit(idx, cpuc->used_mask)) {
+try_generic:
+                        idx = find_first_zero_bit(cpuc->used_mask,
+                                                  x86_pmu.num_counters);
+                        if (idx == x86_pmu.num_counters)
+                                return -EAGAIN;
+
+                        set_bit(idx, cpuc->used_mask);
+                        hwc->idx = idx;
+                }
+                hwc->config_base  = x86_pmu.eventsel;
+                hwc->counter_base = x86_pmu.perfctr;
+        }
+
+        perf_counters_lapic_init();
+
+        x86_pmu.disable(hwc, idx);
+
+        cpuc->counters[idx] = counter;
+        set_bit(idx, cpuc->active_mask);
+
+        x86_perf_counter_set_period(counter, hwc, idx);
+        x86_pmu.enable(hwc, idx);
+
+        return 0;
+}
+
+static void x86_pmu_unthrottle(struct perf_counter *counter)
+{
+        struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
+        struct hw_perf_counter *hwc = &counter->hw;
+
+        if (WARN_ON_ONCE(hwc->idx >= X86_PMC_IDX_MAX ||
+                                cpuc->counters[hwc->idx] != counter))
+                return;
+
+        x86_pmu.enable(hwc, hwc->idx);
+}
+
+void perf_counter_print_debug(void)
+{
+        u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed;
+        struct cpu_hw_counters *cpuc;
+        unsigned long flags;
+        int cpu, idx;
+
+        if (!x86_pmu.num_counters)
+                return;
+
+        local_irq_save(flags);
+
+        cpu = smp_processor_id();
+        cpuc = &per_cpu(cpu_hw_counters, cpu);
+
+        if (x86_pmu.version >= 2) {
+                rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
+                rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
+                rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow);
+                rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed);
+
+                pr_info("\n");
+                pr_info("CPU#%d: ctrl:       %016llx\n", cpu, ctrl);
+                pr_info("CPU#%d: status:     %016llx\n", cpu, status);
+                pr_info("CPU#%d: overflow:   %016llx\n", cpu, overflow);
+                pr_info("CPU#%d: fixed:      %016llx\n", cpu, fixed);
+        }
+        pr_info("CPU#%d: used:       %016llx\n", cpu, *(u64 *)cpuc->used_mask);
+
+        for (idx = 0; idx < x86_pmu.num_counters; idx++) {
+                rdmsrl(x86_pmu.eventsel + idx, pmc_ctrl);
+                rdmsrl(x86_pmu.perfctr  + idx, pmc_count);
+
+                prev_left = per_cpu(prev_left[idx], cpu);
+
+                pr_info("CPU#%d:   gen-PMC%d ctrl:  %016llx\n",
+                        cpu, idx, pmc_ctrl);
+                pr_info("CPU#%d:   gen-PMC%d count: %016llx\n",
+                        cpu, idx, pmc_count);
+                pr_info("CPU#%d:   gen-PMC%d left:  %016llx\n",
+                        cpu, idx, prev_left);
+        }
+        for (idx = 0; idx < x86_pmu.num_counters_fixed; idx++) {
+                rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count);
+
+                pr_info("CPU#%d: fixed-PMC%d count: %016llx\n",
+                        cpu, idx, pmc_count);
+        }
+        local_irq_restore(flags);
+}
+
+static void x86_pmu_disable(struct perf_counter *counter)
+{
+        struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
+        struct hw_perf_counter *hwc = &counter->hw;
+        int idx = hwc->idx;
+
+        /*
+         * Must be done before we disable, otherwise the nmi handler
+         * could reenable again:
+         */
+        clear_bit(idx, cpuc->active_mask);
+        x86_pmu.disable(hwc, idx);
+
+        /*
+         * Make sure the cleared pointer becomes visible before we
+         * (potentially) free the counter:
+         */
+        barrier();
+
+        /*
+         * Drain the remaining delta count out of a counter
+         * that we are disabling:
+         */
+        x86_perf_counter_update(counter, hwc, idx);
+        cpuc->counters[idx] = NULL;
+        clear_bit(idx, cpuc->used_mask);
+}
+
+/*
+ * Save and restart an expired counter. Called by NMI contexts,
+ * so it has to be careful about preempting normal counter ops:
+ */
+static int intel_pmu_save_and_restart(struct perf_counter *counter)
+{
+        struct hw_perf_counter *hwc = &counter->hw;
+        int idx = hwc->idx;
+        int ret;
+
+        x86_perf_counter_update(counter, hwc, idx);
+        ret = x86_perf_counter_set_period(counter, hwc, idx);
+
+        if (counter->state == PERF_COUNTER_STATE_ACTIVE)
+                intel_pmu_enable_counter(hwc, idx);
+
+        return ret;
+}
+
+static void intel_pmu_reset(void)
+{
+        unsigned long flags;
+        int idx;
+
+        if (!x86_pmu.num_counters)
+                return;
+
+        local_irq_save(flags);
+
+        printk("clearing PMU state on CPU#%d\n", smp_processor_id());
+
+        for (idx = 0; idx < x86_pmu.num_counters; idx++) {
+                checking_wrmsrl(x86_pmu.eventsel + idx, 0ull);
+                checking_wrmsrl(x86_pmu.perfctr  + idx, 0ull);
+        }
+        for (idx = 0; idx < x86_pmu.num_counters_fixed; idx++) {
+                checking_wrmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, 0ull);
+        }
+
+        local_irq_restore(flags);
+}
+
+
+/*
+ * This handler is triggered by the local APIC, so the APIC IRQ handling
+ * rules apply:
+ */
+static int intel_pmu_handle_irq(struct pt_regs *regs)
+{
+        struct perf_sample_data data;
+        struct cpu_hw_counters *cpuc;
+        int bit, cpu, loops;
+        u64 ack, status;
+
+        data.regs = regs;
+        data.addr = 0;
+
+        cpu = smp_processor_id();
+        cpuc = &per_cpu(cpu_hw_counters, cpu);
+
+        perf_disable();
+        status = intel_pmu_get_status();
+        if (!status) {
+                perf_enable();
+                return 0;
+        }
+
+        loops = 0;
+again:
+        if (++loops > 100) {
+                WARN_ONCE(1, "perfcounters: irq loop stuck!\n");
+                perf_counter_print_debug();
+                intel_pmu_reset();
+                perf_enable();
+                return 1;
+        }
+
+        inc_irq_stat(apic_perf_irqs);
+        ack = status;
+        for_each_bit(bit, (unsigned long *)&status, X86_PMC_IDX_MAX) {
+                struct perf_counter *counter = cpuc->counters[bit];
+
+                clear_bit(bit, (unsigned long *) &status);
+                if (!test_bit(bit, cpuc->active_mask))
+                        continue;
+
+                if (!intel_pmu_save_and_restart(counter))
+                        continue;
+
+                if (perf_counter_overflow(counter, 1, &data))
+                        intel_pmu_disable_counter(&counter->hw, bit);
+        }
+
+        intel_pmu_ack_status(ack);
+
+        /*
+         * Repeat if there is more work to be done:
+         */
+        status = intel_pmu_get_status();
+        if (status)
+                goto again;
+
+        perf_enable();
+
+        return 1;
+}
+
+static int amd_pmu_handle_irq(struct pt_regs *regs)
+{
+        struct perf_sample_data data;
+        struct cpu_hw_counters *cpuc;
+        struct perf_counter *counter;
+        struct hw_perf_counter *hwc;
+        int cpu, idx, handled = 0;
+        u64 val;
+
+        data.regs = regs;
+        data.addr = 0;
+
+        cpu = smp_processor_id();
+        cpuc = &per_cpu(cpu_hw_counters, cpu);
+
+        for (idx = 0; idx < x86_pmu.num_counters; idx++) {
+                if (!test_bit(idx, cpuc->active_mask))
+                        continue;
+
+                counter = cpuc->counters[idx];
+                hwc = &counter->hw;
+
+                val = x86_perf_counter_update(counter, hwc, idx);
+                if (val & (1ULL << (x86_pmu.counter_bits - 1)))
+                        continue;
+
+                /*
+                 * counter overflow
+                 */
+                handled         = 1;
+                data.period     = counter->hw.last_period;
+
+                if (!x86_perf_counter_set_period(counter, hwc, idx))
+                        continue;
+
+                if (perf_counter_overflow(counter, 1, &data))
+                        amd_pmu_disable_counter(hwc, idx);
+        }
+
+        if (handled)
+                inc_irq_stat(apic_perf_irqs);
+
+        return handled;
+}
+
+void smp_perf_pending_interrupt(struct pt_regs *regs)
+{
+        irq_enter();
+        ack_APIC_irq();
+        inc_irq_stat(apic_pending_irqs);
+        perf_counter_do_pending();
+        irq_exit();
+}
+
+void set_perf_counter_pending(void)
+{
+        apic->send_IPI_self(LOCAL_PENDING_VECTOR);
+}
+
+void perf_counters_lapic_init(void)
+{
+        if (!x86_pmu_initialized())
+                return;
+
+        /*
+         * Always use NMI for PMU
+         */
+        apic_write(APIC_LVTPC, APIC_DM_NMI);
+}
+
+static int __kprobes
+perf_counter_nmi_handler(struct notifier_block *self,
+                         unsigned long cmd, void *__args)
+{
+        struct die_args *args = __args;
+        struct pt_regs *regs;
+
+        if (!atomic_read(&active_counters))
+                return NOTIFY_DONE;
+
+        switch (cmd) {
+        case DIE_NMI:
+        case DIE_NMI_IPI:
+                break;
+
+        default:
+                return NOTIFY_DONE;
+        }
+
+        regs = args->regs;
+
+        apic_write(APIC_LVTPC, APIC_DM_NMI);
+        /*
+         * Can't rely on the handled return value to say it was our NMI, two
+         * counters could trigger 'simultaneously' raising two back-to-back NMIs.
+         *
+         * If the first NMI handles both, the latter will be empty and daze
+         * the CPU.
+         */
+        x86_pmu.handle_irq(regs);
+
+        return NOTIFY_STOP;
+}
+
+static __read_mostly struct notifier_block perf_counter_nmi_notifier = {
+        .notifier_call          = perf_counter_nmi_handler,
+        .next                   = NULL,
+        .priority               = 1
+};
+
+static struct x86_pmu intel_pmu = {
+        .name                   = "Intel",
+        .handle_irq             = intel_pmu_handle_irq,
+        .disable_all            = intel_pmu_disable_all,
+        .enable_all             = intel_pmu_enable_all,
+        .enable                 = intel_pmu_enable_counter,
+        .disable                = intel_pmu_disable_counter,
+        .eventsel               = MSR_ARCH_PERFMON_EVENTSEL0,
+        .perfctr                = MSR_ARCH_PERFMON_PERFCTR0,
+        .event_map              = intel_pmu_event_map,
+        .raw_event              = intel_pmu_raw_event,
+        .max_events             = ARRAY_SIZE(intel_perfmon_event_map),
+        /*
+         * Intel PMCs cannot be accessed sanely above 32 bit width,
+         * so we install an artificial 1<<31 period regardless of
+         * the generic counter period:
+         */
+        .max_period             = (1ULL << 31) - 1,
+};
+
+static struct x86_pmu amd_pmu = {
+        .name                   = "AMD",
+        .handle_irq             = amd_pmu_handle_irq,
+        .disable_all            = amd_pmu_disable_all,
+        .enable_all             = amd_pmu_enable_all,
+        .enable                 = amd_pmu_enable_counter,
+        .disable                = amd_pmu_disable_counter,
+        .eventsel               = MSR_K7_EVNTSEL0,
+        .perfctr                = MSR_K7_PERFCTR0,
+        .event_map              = amd_pmu_event_map,
+        .raw_event              = amd_pmu_raw_event,
+        .max_events             = ARRAY_SIZE(amd_perfmon_event_map),
+        .num_counters           = 4,
+        .counter_bits           = 48,
+        .counter_mask           = (1ULL << 48) - 1,
+        /* use highest bit to detect overflow */
+        .max_period             = (1ULL << 47) - 1,
+};
+
+static int intel_pmu_init(void)
+{
+        union cpuid10_edx edx;
+        union cpuid10_eax eax;
+        unsigned int unused;
+        unsigned int ebx;
+        int version;
+
+        if (!cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON))
+                return -ENODEV;
+
+        /*
+         * Check whether the Architectural PerfMon supports
+         * Branch Misses Retired Event or not.
+         */
+        cpuid(10, &eax.full, &ebx, &unused, &edx.full);
+        if (eax.split.mask_length <= ARCH_PERFMON_BRANCH_MISSES_RETIRED)
+                return -ENODEV;
+
+        version = eax.split.version_id;
+        if (version < 2)
+                return -ENODEV;
+
+        x86_pmu                         = intel_pmu;
+        x86_pmu.version                 = version;
+        x86_pmu.num_counters            = eax.split.num_counters;
+        x86_pmu.counter_bits            = eax.split.bit_width;
+        x86_pmu.counter_mask            = (1ULL << eax.split.bit_width) - 1;
+
+        /*
+         * Quirk: v2 perfmon does not report fixed-purpose counters, so
+         * assume at least 3 counters:
+         */
+        x86_pmu.num_counters_fixed      = max((int)edx.split.num_counters_fixed, 3);
+
+        rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, x86_pmu.intel_ctrl);
+
+        /*
+         * Install the hw-cache-events table:
+         */
+        switch (boot_cpu_data.x86_model) {
+        case 15: /* original 65 nm celeron/pentium/core2/xeon, "Merom"/"Conroe" */
+        case 22: /* single-core 65 nm celeron/core2solo "Merom-L"/"Conroe-L" */
+        case 23: /* current 45 nm celeron/core2/xeon "Penryn"/"Wolfdale" */
+        case 29: /* six-core 45 nm xeon "Dunnington" */
+                memcpy(hw_cache_event_ids, core2_hw_cache_event_ids,
+                       sizeof(hw_cache_event_ids));
+
+                pr_cont("Core2 events, ");
+                break;
+        default:
+        case 26:
+                memcpy(hw_cache_event_ids, nehalem_hw_cache_event_ids,
+                       sizeof(hw_cache_event_ids));
+
+                pr_cont("Nehalem/Corei7 events, ");
+                break;
+        case 28:
+                memcpy(hw_cache_event_ids, atom_hw_cache_event_ids,
+                       sizeof(hw_cache_event_ids));
+
+                pr_cont("Atom events, ");
+                break;
+        }
+        return 0;
+}
+
+static int amd_pmu_init(void)
+{
+        x86_pmu = amd_pmu;
+
+        switch (boot_cpu_data.x86) {
+        case 0x0f:
+        case 0x10:
+        case 0x11:
+                memcpy(hw_cache_event_ids, amd_0f_hw_cache_event_ids,
+                       sizeof(hw_cache_event_ids));
+
+                pr_cont("AMD Family 0f/10/11 events, ");
+                break;
+        }
+        return 0;
+}
+
+void __init init_hw_perf_counters(void)
+{
+        int err;
+
+        pr_info("Performance Counters: ");
+
+        switch (boot_cpu_data.x86_vendor) {
+        case X86_VENDOR_INTEL:
+                err = intel_pmu_init();
+                break;
+        case X86_VENDOR_AMD:
+                err = amd_pmu_init();
+                break;
+        default:
+                return;
+        }
+        if (err != 0) {
+                pr_cont("no PMU driver, software counters only.\n");
+                return;
+        }
+
+        pr_cont("%s PMU driver.\n", x86_pmu.name);
+
+        if (x86_pmu.num_counters > X86_PMC_MAX_GENERIC) {
+                x86_pmu.num_counters = X86_PMC_MAX_GENERIC;
+                WARN(1, KERN_ERR "hw perf counters %d > max(%d), clipping!",
+                     x86_pmu.num_counters, X86_PMC_MAX_GENERIC);
+        }
+        perf_counter_mask = (1 << x86_pmu.num_counters) - 1;
+        perf_max_counters = x86_pmu.num_counters;
+
+        if (x86_pmu.num_counters_fixed > X86_PMC_MAX_FIXED) {
+                x86_pmu.num_counters_fixed = X86_PMC_MAX_FIXED;
+                WARN(1, KERN_ERR "hw perf counters fixed %d > max(%d), clipping!",
+                     x86_pmu.num_counters_fixed, X86_PMC_MAX_FIXED);
+        }
+
+        perf_counter_mask |=
+                ((1LL << x86_pmu.num_counters_fixed)-1) << X86_PMC_IDX_FIXED;
+
+        perf_counters_lapic_init();
+        register_die_notifier(&perf_counter_nmi_notifier);
+
+        pr_info("... version:                 %d\n",     x86_pmu.version);
+        pr_info("... bit width:               %d\n",     x86_pmu.counter_bits);
+        pr_info("... generic counters:        %d\n",     x86_pmu.num_counters);
+        pr_info("... value mask:              %016Lx\n", x86_pmu.counter_mask);
+        pr_info("... max period:              %016Lx\n", x86_pmu.max_period);
+        pr_info("... fixed-purpose counters:  %d\n",     x86_pmu.num_counters_fixed);
+        pr_info("... counter mask:            %016Lx\n", perf_counter_mask);
+}
+
+static inline void x86_pmu_read(struct perf_counter *counter)
+{
+        x86_perf_counter_update(counter, &counter->hw, counter->hw.idx);
+}
+
+static const struct pmu pmu = {
+        .enable         = x86_pmu_enable,
+        .disable        = x86_pmu_disable,
+        .read           = x86_pmu_read,
+        .unthrottle     = x86_pmu_unthrottle,
+};
+
+const struct pmu *hw_perf_counter_init(struct perf_counter *counter)
+{
+        int err;
+
+        err = __hw_perf_counter_init(counter);
+        if (err)
+                return ERR_PTR(err);
+
+        return &pmu;
+}
+
+/*
+ * callchain support
+ */
+
+static inline
+void callchain_store(struct perf_callchain_entry *entry, unsigned long ip)
+{
+        if (entry->nr < MAX_STACK_DEPTH)
+                entry->ip[entry->nr++] = ip;
+}
+
+static DEFINE_PER_CPU(struct perf_callchain_entry, irq_entry);
+static DEFINE_PER_CPU(struct perf_callchain_entry, nmi_entry);
+
+
+static void
+backtrace_warning_symbol(void *data, char *msg, unsigned long symbol)
+{
+        /* Ignore warnings */
+}
+
+static void backtrace_warning(void *data, char *msg)
+{
+        /* Ignore warnings */
+}
+
+static int backtrace_stack(void *data, char *name)
+{
+        /* Don't bother with IRQ stacks for now */
+        return -1;
+}
+
+static void backtrace_address(void *data, unsigned long addr, int reliable)
+{
+        struct perf_callchain_entry *entry = data;
+
+        if (reliable)
+                callchain_store(entry, addr);
+}
+
+static const struct stacktrace_ops backtrace_ops = {
+        .warning                = backtrace_warning,
+        .warning_symbol         = backtrace_warning_symbol,
+        .stack                  = backtrace_stack,
+        .address                = backtrace_address,
+};
+
+static void
+perf_callchain_kernel(struct pt_regs *regs, struct perf_callchain_entry *entry)
+{
+        unsigned long bp;
+        char *stack;
+        int nr = entry->nr;
+
+        callchain_store(entry, instruction_pointer(regs));
+
+        stack = ((char *)regs + sizeof(struct pt_regs));
+#ifdef CONFIG_FRAME_POINTER
+        bp = frame_pointer(regs);
+#else
+        bp = 0;
+#endif
+
+        dump_trace(NULL, regs, (void *)stack, bp, &backtrace_ops, entry);
+
+        entry->kernel = entry->nr - nr;
+}
+
+
+struct stack_frame {
+        const void __user       *next_fp;
+        unsigned long           return_address;
+};
+
+static int copy_stack_frame(const void __user *fp, struct stack_frame *frame)
+{
+        int ret;
+
+        if (!access_ok(VERIFY_READ, fp, sizeof(*frame)))
+                return 0;
+
+        ret = 1;
+        pagefault_disable();
+        if (__copy_from_user_inatomic(frame, fp, sizeof(*frame)))
+                ret = 0;
+        pagefault_enable();
+
+        return ret;
+}
+
+static void
+perf_callchain_user(struct pt_regs *regs, struct perf_callchain_entry *entry)
+{
+        struct stack_frame frame;
+        const void __user *fp;
+        int nr = entry->nr;
+
+        regs = (struct pt_regs *)current->thread.sp0 - 1;
+        fp   = (void __user *)regs->bp;
+
+        callchain_store(entry, regs->ip);
+
+        while (entry->nr < MAX_STACK_DEPTH) {
+                frame.next_fp        = NULL;
+                frame.return_address = 0;
+
+                if (!copy_stack_frame(fp, &frame))
+                        break;
+
+                if ((unsigned long)fp < user_stack_pointer(regs))
+                        break;
+
+                callchain_store(entry, frame.return_address);
+                fp = frame.next_fp;
+        }
+
+        entry->user = entry->nr - nr;
+}
+
+static void
+perf_do_callchain(struct pt_regs *regs, struct perf_callchain_entry *entry)
+{
+        int is_user;
+
+        if (!regs)
+                return;
+
+        is_user = user_mode(regs);
+
+        if (!current || current->pid == 0)
+                return;
+
+        if (is_user && current->state != TASK_RUNNING)
+                return;
+
+        if (!is_user)
+                perf_callchain_kernel(regs, entry);
+
+        if (current->mm)
+                perf_callchain_user(regs, entry);
+}
+
+struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
+{
+        struct perf_callchain_entry *entry;
+
+        if (in_nmi())
+                entry = &__get_cpu_var(nmi_entry);
+        else
+                entry = &__get_cpu_var(irq_entry);
+
+        entry->nr = 0;
+        entry->hv = 0;
+        entry->kernel = 0;
+        entry->user = 0;
+
+        perf_do_callchain(regs, entry);
+
+        return entry;
+}
diff --git a/arch/x86/kernel/cpu/perfctr-watchdog.c b/arch/x86/kernel/cpu/perfctr-watchdog.c
index f6c70a164e32..d6f5b9fbde32 100644
--- a/arch/x86/kernel/cpu/perfctr-watchdog.c
+++ b/arch/x86/kernel/cpu/perfctr-watchdog.c
@@ -19,8 +19,8 @@
 #include <linux/nmi.h>
 #include <linux/kprobes.h>
 
-#include <asm/genapic.h>
-#include <asm/intel_arch_perfmon.h>
+#include <asm/apic.h>
+#include <asm/perf_counter.h>
 
 struct nmi_watchdog_ctlblk {
         unsigned int cccr_msr;
diff --git a/arch/x86/kernel/ds.c b/arch/x86/kernel/ds.c
index 87b67e3a765a..48bfe1386038 100644
--- a/arch/x86/kernel/ds.c
+++ b/arch/x86/kernel/ds.c
@@ -19,45 +19,61 @@
  * Markus Metzger <markus.t.metzger@intel.com>, 2007-2009
  */
 
-
-#include <asm/ds.h>
-
-#include <linux/errno.h>
+#include <linux/kernel.h>
 #include <linux/string.h>
-#include <linux/slab.h>
+#include <linux/errno.h>
 #include <linux/sched.h>
+#include <linux/slab.h>
 #include <linux/mm.h>
-#include <linux/kernel.h>
+#include <linux/trace_clock.h>
+
+#include <asm/ds.h>
 
+#include "ds_selftest.h"
 
 /*
- * The configuration for a particular DS hardware implementation.
+ * The configuration for a particular DS hardware implementation:
  */
 struct ds_configuration {
-        /* the name of the configuration */
-        const char *name;
-        /* the size of one pointer-typed field in the DS structure and
-           in the BTS and PEBS buffers in bytes;
-           this covers the first 8 DS fields related to buffer management. */
-        unsigned char  sizeof_field;
-        /* the size of a BTS/PEBS record in bytes */
-        unsigned char  sizeof_rec[2];
-        /* a series of bit-masks to control various features indexed
-         * by enum ds_feature */
-        unsigned long ctl[dsf_ctl_max];
+        /* The name of the configuration: */
+        const char              *name;
+
+        /* The size of pointer-typed fields in DS, BTS, and PEBS: */
+        unsigned char           sizeof_ptr_field;
+
+        /* The size of a BTS/PEBS record in bytes: */
+        unsigned char           sizeof_rec[2];
+
+        /* The number of pebs counter reset values in the DS structure. */
+        unsigned char           nr_counter_reset;
+
+        /* Control bit-masks indexed by enum ds_feature: */
+        unsigned long           ctl[dsf_ctl_max];
 };
-static DEFINE_PER_CPU(struct ds_configuration, ds_cfg_array);
+static struct ds_configuration ds_cfg __read_mostly;
+
+
+/* Maximal size of a DS configuration: */
+#define MAX_SIZEOF_DS           0x80
 
-#define ds_cfg per_cpu(ds_cfg_array, smp_processor_id())
+/* Maximal size of a BTS record: */
+#define MAX_SIZEOF_BTS          (3 * 8)
 
-#define MAX_SIZEOF_DS (12 * 8)  /* maximal size of a DS configuration */
-#define MAX_SIZEOF_BTS (3 * 8)  /* maximal size of a BTS record */
-#define DS_ALIGNMENT (1 << 3)   /* BTS and PEBS buffer alignment */
+/* BTS and PEBS buffer alignment: */
+#define DS_ALIGNMENT            (1 << 3)
 
-#define BTS_CONTROL \
- (ds_cfg.ctl[dsf_bts] | ds_cfg.ctl[dsf_bts_kernel] | ds_cfg.ctl[dsf_bts_user] |\
-  ds_cfg.ctl[dsf_bts_overflow])
+/* Number of buffer pointers in DS: */
+#define NUM_DS_PTR_FIELDS       8
 
+/* Size of a pebs reset value in DS: */
+#define PEBS_RESET_FIELD_SIZE   8
+
+/* Mask of control bits in the DS MSR register: */
+#define BTS_CONTROL                               \
+        ( ds_cfg.ctl[dsf_bts]                   | \
+          ds_cfg.ctl[dsf_bts_kernel]            | \
+          ds_cfg.ctl[dsf_bts_user]              | \
+          ds_cfg.ctl[dsf_bts_overflow] )
 
 /*
  * A BTS or PEBS tracer.
@@ -66,29 +82,36 @@ static DEFINE_PER_CPU(struct ds_configuration, ds_cfg_array);
  * to identify tracers.
  */
 struct ds_tracer {
-        /* the DS context (partially) owned by this tracer */
-        struct ds_context *context;
-        /* the buffer provided on ds_request() and its size in bytes */
-        void *buffer;
-        size_t size;
+        /* The DS context (partially) owned by this tracer. */
+        struct ds_context       *context;
+        /* The buffer provided on ds_request() and its size in bytes. */
+        void                    *buffer;
+        size_t                  size;
 };
 
 struct bts_tracer {
-        /* the common DS part */
-        struct ds_tracer ds;
-        /* the trace including the DS configuration */
-        struct bts_trace trace;
-        /* buffer overflow notification function */
-        bts_ovfl_callback_t ovfl;
+        /* The common DS part: */
+        struct ds_tracer        ds;
+
+        /* The trace including the DS configuration: */
+        struct bts_trace        trace;
+
+        /* Buffer overflow notification function: */
+        bts_ovfl_callback_t     ovfl;
+
+        /* Active flags affecting trace collection. */
+        unsigned int            flags;
 };
 
 struct pebs_tracer {
-        /* the common DS part */
-        struct ds_tracer ds;
-        /* the trace including the DS configuration */
-        struct pebs_trace trace;
-        /* buffer overflow notification function */
-        pebs_ovfl_callback_t ovfl;
+        /* The common DS part: */
+        struct ds_tracer        ds;
+
+        /* The trace including the DS configuration: */
+        struct pebs_trace       trace;
+
+        /* Buffer overflow notification function: */
+        pebs_ovfl_callback_t    ovfl;
 };
 
 /*
@@ -97,6 +120,7 @@ struct pebs_tracer {
  *
  * The DS configuration consists of the following fields; different
  * architetures vary in the size of those fields.
+ *
  * - double-word aligned base linear address of the BTS buffer
  * - write pointer into the BTS buffer
  * - end linear address of the BTS buffer (one byte beyond the end of
@@ -135,21 +159,22 @@ enum ds_field {
 };
 
 enum ds_qualifier {
-        ds_bts  = 0,
+        ds_bts = 0,
         ds_pebs
 };
 
-static inline unsigned long ds_get(const unsigned char *base,
-                                   enum ds_qualifier qual, enum ds_field field)
+static inline unsigned long
+ds_get(const unsigned char *base, enum ds_qualifier qual, enum ds_field field)
 {
-        base += (ds_cfg.sizeof_field * (field + (4 * qual)));
+        base += (ds_cfg.sizeof_ptr_field * (field + (4 * qual)));
         return *(unsigned long *)base;
 }
 
-static inline void ds_set(unsigned char *base, enum ds_qualifier qual,
-                          enum ds_field field, unsigned long value)
+static inline void
+ds_set(unsigned char *base, enum ds_qualifier qual, enum ds_field field,
+       unsigned long value)
 {
-        base += (ds_cfg.sizeof_field * (field + (4 * qual)));
+        base += (ds_cfg.sizeof_ptr_field * (field + (4 * qual)));
         (*(unsigned long *)base) = value;
 }
 
@@ -159,7 +184,6 @@ static inline void ds_set(unsigned char *base, enum ds_qualifier qual,
  */
 static DEFINE_SPINLOCK(ds_lock);
 
-
 /*
  * We either support (system-wide) per-cpu or per-thread allocation.
  * We distinguish the two based on the task_struct pointer, where a
@@ -178,12 +202,28 @@ static DEFINE_SPINLOCK(ds_lock);
  */
 static atomic_t tracers = ATOMIC_INIT(0);
 
-static inline void get_tracer(struct task_struct *task)
+static inline int get_tracer(struct task_struct *task)
 {
-        if (task)
+        int error;
+
+        spin_lock_irq(&ds_lock);
+
+        if (task) {
+                error = -EPERM;
+                if (atomic_read(&tracers) < 0)
+                        goto out;
                 atomic_inc(&tracers);
-        else
+        } else {
+                error = -EPERM;
+                if (atomic_read(&tracers) > 0)
+                        goto out;
                 atomic_dec(&tracers);
+        }
+
+        error = 0;
+out:
+        spin_unlock_irq(&ds_lock);
+        return error;
 }
 
 static inline void put_tracer(struct task_struct *task)
@@ -194,14 +234,6 @@ static inline void put_tracer(struct task_struct *task)
                 atomic_inc(&tracers);
 }
 
-static inline int check_tracer(struct task_struct *task)
-{
-        return task ?
-                (atomic_read(&tracers) >= 0) :
-                (atomic_read(&tracers) <= 0);
-}
-
-
 /*
  * The DS context is either attached to a thread or to a cpu:
  * - in the former case, the thread_struct contains a pointer to the
@@ -213,61 +245,58 @@ static inline int check_tracer(struct task_struct *task)
  * deallocated when the last user puts the context.
  */
 struct ds_context {
-        /* pointer to the DS configuration; goes into MSR_IA32_DS_AREA */
-        unsigned char ds[MAX_SIZEOF_DS];
-        /* the owner of the BTS and PEBS configuration, respectively */
-        struct bts_tracer *bts_master;
-        struct pebs_tracer *pebs_master;
-        /* use count */
-        unsigned long count;
-        /* a pointer to the context location inside the thread_struct
-         * or the per_cpu context array */
-        struct ds_context **this;
-        /* a pointer to the task owning this context, or NULL, if the
-         * context is owned by a cpu */
-        struct task_struct *task;
-};
+        /* The DS configuration; goes into MSR_IA32_DS_AREA: */
+        unsigned char           ds[MAX_SIZEOF_DS];
+
+        /* The owner of the BTS and PEBS configuration, respectively: */
+        struct bts_tracer       *bts_master;
+        struct pebs_tracer      *pebs_master;
 
-static DEFINE_PER_CPU(struct ds_context *, system_context_array);
+        /* Use count: */
+        unsigned long           count;
 
-#define system_context per_cpu(system_context_array, smp_processor_id())
+        /* Pointer to the context pointer field: */
+        struct ds_context       **this;
+
+        /* The traced task; NULL for cpu tracing: */
+        struct task_struct      *task;
+
+        /* The traced cpu; only valid if task is NULL: */
+        int                     cpu;
+};
 
+static DEFINE_PER_CPU(struct ds_context *, cpu_context);
 
-static inline struct ds_context *ds_get_context(struct task_struct *task)
+
+static struct ds_context *ds_get_context(struct task_struct *task, int cpu)
 {
         struct ds_context **p_context =
-                (task ? &task->thread.ds_ctx : &system_context);
+                (task ? &task->thread.ds_ctx : &per_cpu(cpu_context, cpu));
         struct ds_context *context = NULL;
         struct ds_context *new_context = NULL;
-        unsigned long irq;
 
         /* Chances are small that we already have a context. */
         new_context = kzalloc(sizeof(*new_context), GFP_KERNEL);
         if (!new_context)
                 return NULL;
 
-        spin_lock_irqsave(&ds_lock, irq);
+        spin_lock_irq(&ds_lock);
 
         context = *p_context;
-        if (!context) {
+        if (likely(!context)) {
                 context = new_context;
 
                 context->this = p_context;
                 context->task = task;
+                context->cpu = cpu;
                 context->count = 0;
 
-                if (task)
-                        set_tsk_thread_flag(task, TIF_DS_AREA_MSR);
-
-                if (!task || (task == current))
-                        wrmsrl(MSR_IA32_DS_AREA, (unsigned long)context->ds);
-
                 *p_context = context;
         }
 
         context->count++;
 
-        spin_unlock_irqrestore(&ds_lock, irq);
+        spin_unlock_irq(&ds_lock);
 
         if (context != new_context)
                 kfree(new_context);
@@ -275,8 +304,9 @@ static inline struct ds_context *ds_get_context(struct task_struct *task)
         return context;
 }
 
-static inline void ds_put_context(struct ds_context *context)
+static void ds_put_context(struct ds_context *context)
 {
+        struct task_struct *task;
         unsigned long irq;
 
         if (!context)
@@ -291,17 +321,55 @@ static inline void ds_put_context(struct ds_context *context)
 
         *(context->this) = NULL;
 
-        if (context->task)
-                clear_tsk_thread_flag(context->task, TIF_DS_AREA_MSR);
+        task = context->task;
+
+        if (task)
+                clear_tsk_thread_flag(task, TIF_DS_AREA_MSR);
 
-        if (!context->task || (context->task == current))
-                wrmsrl(MSR_IA32_DS_AREA, 0);
+        /*
+         * We leave the (now dangling) pointer to the DS configuration in
+         * the DS_AREA msr. This is as good or as bad as replacing it with
+         * NULL - the hardware would crash if we enabled tracing.
+         *
+         * This saves us some problems with having to write an msr on a
+         * different cpu while preventing others from doing the same for the
+         * next context for that same cpu.
+         */
 
         spin_unlock_irqrestore(&ds_lock, irq);
 
+        /* The context might still be in use for context switching. */
+        if (task && (task != current))
+                wait_task_context_switch(task);
+
         kfree(context);
 }
 
+static void ds_install_ds_area(struct ds_context *context)
+{
+        unsigned long ds;
+
+        ds = (unsigned long)context->ds;
+
+        /*
+         * There is a race between the bts master and the pebs master.
+         *
+         * The thread/cpu access is synchronized via get/put_cpu() for
+         * task tracing and via wrmsr_on_cpu for cpu tracing.
+         *
+         * If bts and pebs are collected for the same task or same cpu,
+         * the same confiuration is written twice.
+         */
+        if (context->task) {
+                get_cpu();
+                if (context->task == current)
+                        wrmsrl(MSR_IA32_DS_AREA, ds);
+                set_tsk_thread_flag(context->task, TIF_DS_AREA_MSR);
+                put_cpu();
+        } else
+                wrmsr_on_cpu(context->cpu, MSR_IA32_DS_AREA,
+                             (u32)((u64)ds), (u32)((u64)ds >> 32));
+}
 
 /*
  * Call the tracer's callback on a buffer overflow.
@@ -332,9 +400,9 @@ static void ds_overflow(struct ds_context *context, enum ds_qualifier qual)
  * The remainder of any partially written record is zeroed out.
  *
  * context: the DS context
- * qual: the buffer type
- * record: the data to write
- * size: the size of the data
+ * qual:    the buffer type
+ * record:  the data to write
+ * size:    the size of the data
  */
 static int ds_write(struct ds_context *context, enum ds_qualifier qual,
                     const void *record, size_t size)
@@ -349,14 +417,14 @@ static int ds_write(struct ds_context *context, enum ds_qualifier qual,
                 unsigned long write_size, adj_write_size;
 
                 /*
-                 * write as much as possible without producing an
+                 * Write as much as possible without producing an
                  * overflow interrupt.
                  *
-                 * interrupt_threshold must either be
+                 * Interrupt_threshold must either be
                  * - bigger than absolute_maximum or
                  * - point to a record between buffer_base and absolute_maximum
                  *
-                 * index points to a valid record.
+                 * Index points to a valid record.
                  */
                 base   = ds_get(context->ds, qual, ds_buffer_base);
                 index  = ds_get(context->ds, qual, ds_index);
@@ -365,8 +433,10 @@ static int ds_write(struct ds_context *context, enum ds_qualifier qual,
 
                 write_end = min(end, int_th);
 
-                /* if we are already beyond the interrupt threshold,
-                 * we fill the entire buffer */
+                /*
+                 * If we are already beyond the interrupt threshold,
+                 * we fill the entire buffer.
+                 */
                 if (write_end <= index)
                         write_end = end;
 
@@ -383,7 +453,7 @@ static int ds_write(struct ds_context *context, enum ds_qualifier qual,
                 adj_write_size = write_size / ds_cfg.sizeof_rec[qual];
                 adj_write_size *= ds_cfg.sizeof_rec[qual];
 
-                /* zero out trailing bytes */
+                /* Zero out trailing bytes. */
                 memset((char *)index + write_size, 0,
                        adj_write_size - write_size);
                 index += adj_write_size;
@@ -410,7 +480,7 @@ static int ds_write(struct ds_context *context, enum ds_qualifier qual,
  * Later architectures use 64bit pointers throughout, whereas earlier
  * architectures use 32bit pointers in 32bit mode.
  *
- * We compute the base address for the first 8 fields based on:
+ * We compute the base address for the fields based on:
  * - the field size stored in the DS configuration
  * - the relative field position
  *
@@ -431,23 +501,23 @@ enum bts_field {
         bts_to,
         bts_flags,
 
-        bts_qual = bts_from,
-        bts_jiffies = bts_to,
-        bts_pid = bts_flags,
+        bts_qual                = bts_from,
+        bts_clock               = bts_to,
+        bts_pid                 = bts_flags,
 
-        bts_qual_mask = (bts_qual_max - 1),
-        bts_escape = ((unsigned long)-1 & ~bts_qual_mask)
+        bts_qual_mask           = (bts_qual_max - 1),
+        bts_escape              = ((unsigned long)-1 & ~bts_qual_mask)
 };
 
 static inline unsigned long bts_get(const char *base, enum bts_field field)
 {
-        base += (ds_cfg.sizeof_field * field);
+        base += (ds_cfg.sizeof_ptr_field * field);
         return *(unsigned long *)base;
 }
 
 static inline void bts_set(char *base, enum bts_field field, unsigned long val)
 {
-        base += (ds_cfg.sizeof_field * field);;
+        base += (ds_cfg.sizeof_ptr_field * field);;
         (*(unsigned long *)base) = val;
 }
 
@@ -463,8 +533,8 @@ static inline void bts_set(char *base, enum bts_field field, unsigned long val)
  *
  * return: bytes read/written on success; -Eerrno, otherwise
  */
-static int bts_read(struct bts_tracer *tracer, const void *at,
-                    struct bts_struct *out)
+static int
+bts_read(struct bts_tracer *tracer, const void *at, struct bts_struct *out)
 {
         if (!tracer)
                 return -EINVAL;
@@ -478,8 +548,8 @@ static int bts_read(struct bts_tracer *tracer, const void *at,
         memset(out, 0, sizeof(*out));
         if ((bts_get(at, bts_qual) & ~bts_qual_mask) == bts_escape) {
                 out->qualifier = (bts_get(at, bts_qual) & bts_qual_mask);
-                out->variant.timestamp.jiffies = bts_get(at, bts_jiffies);
-                out->variant.timestamp.pid = bts_get(at, bts_pid);
+                out->variant.event.clock = bts_get(at, bts_clock);
+                out->variant.event.pid = bts_get(at, bts_pid);
         } else {
                 out->qualifier = bts_branch;
                 out->variant.lbr.from = bts_get(at, bts_from);
@@ -516,8 +586,8 @@ static int bts_write(struct bts_tracer *tracer, const struct bts_struct *in)
         case bts_task_arrives:
         case bts_task_departs:
                 bts_set(raw, bts_qual, (bts_escape | in->qualifier));
-                bts_set(raw, bts_jiffies, in->variant.timestamp.jiffies);
-                bts_set(raw, bts_pid, in->variant.timestamp.pid);
+                bts_set(raw, bts_clock, in->variant.event.clock);
+                bts_set(raw, bts_pid, in->variant.event.pid);
                 break;
         default:
                 return -EINVAL;
@@ -555,7 +625,8 @@ static void ds_init_ds_trace(struct ds_trace *trace, enum ds_qualifier qual,
                              unsigned int flags) {
         unsigned long buffer, adj;
 
-        /* adjust the buffer address and size to meet alignment
+        /*
+         * Adjust the buffer address and size to meet alignment
          * constraints:
          * - buffer is double-word aligned
          * - size is multiple of record size
@@ -577,9 +648,11 @@ static void ds_init_ds_trace(struct ds_trace *trace, enum ds_qualifier qual,
         trace->begin = (void *)buffer;
         trace->top = trace->begin;
         trace->end = (void *)(buffer + size);
-        /* The value for 'no threshold' is -1, which will set the
+        /*
+         * The value for 'no threshold' is -1, which will set the
          * threshold outside of the buffer, just like we want it.
          */
+        ith *= ds_cfg.sizeof_rec[qual];
         trace->ith = (void *)(buffer + size - ith);
 
         trace->flags = flags;
@@ -588,18 +661,27 @@ static void ds_init_ds_trace(struct ds_trace *trace, enum ds_qualifier qual,
 
 static int ds_request(struct ds_tracer *tracer, struct ds_trace *trace,
                       enum ds_qualifier qual, struct task_struct *task,
-                      void *base, size_t size, size_t th, unsigned int flags)
+                      int cpu, void *base, size_t size, size_t th)
 {
         struct ds_context *context;
         int error;
+        size_t req_size;
+
+        error = -EOPNOTSUPP;
+        if (!ds_cfg.sizeof_rec[qual])
+                goto out;
 
         error = -EINVAL;
         if (!base)
                 goto out;
 
-        /* we require some space to do alignment adjustments below */
+        req_size = ds_cfg.sizeof_rec[qual];
+        /* We might need space for alignment adjustments. */
+        if (!IS_ALIGNED((unsigned long)base, DS_ALIGNMENT))
+                req_size += DS_ALIGNMENT;
+
         error = -EINVAL;
-        if (size < (DS_ALIGNMENT + ds_cfg.sizeof_rec[qual]))
+        if (size < req_size)
                 goto out;
 
         if (th != (size_t)-1) {
@@ -614,182 +696,318 @@ static int ds_request(struct ds_tracer *tracer, struct ds_trace *trace,
         tracer->size = size;
 
         error = -ENOMEM;
-        context = ds_get_context(task);
+        context = ds_get_context(task, cpu);
         if (!context)
                 goto out;
         tracer->context = context;
 
-        ds_init_ds_trace(trace, qual, base, size, th, flags);
+        /*
+         * Defer any tracer-specific initialization work for the context until
+         * context ownership has been clarified.
+         */
 
         error = 0;
  out:
         return error;
 }
 
-struct bts_tracer *ds_request_bts(struct task_struct *task,
-                                  void *base, size_t size,
-                                  bts_ovfl_callback_t ovfl, size_t th,
-                                  unsigned int flags)
+static struct bts_tracer *ds_request_bts(struct task_struct *task, int cpu,
+                                         void *base, size_t size,
+                                         bts_ovfl_callback_t ovfl, size_t th,
+                                         unsigned int flags)
 {
         struct bts_tracer *tracer;
-        unsigned long irq;
         int error;
 
+        /* Buffer overflow notification is not yet implemented. */
         error = -EOPNOTSUPP;
-        if (!ds_cfg.ctl[dsf_bts])
+        if (ovfl)
                 goto out;
 
-        /* buffer overflow notification is not yet implemented */
-        error = -EOPNOTSUPP;
-        if (ovfl)
+        error = get_tracer(task);
+        if (error < 0)
                 goto out;
 
         error = -ENOMEM;
         tracer = kzalloc(sizeof(*tracer), GFP_KERNEL);
         if (!tracer)
-                goto out;
+                goto out_put_tracer;
         tracer->ovfl = ovfl;
 
+        /* Do some more error checking and acquire a tracing context. */
         error = ds_request(&tracer->ds, &tracer->trace.ds,
-                           ds_bts, task, base, size, th, flags);
+                           ds_bts, task, cpu, base, size, th);
         if (error < 0)
                 goto out_tracer;
 
-
-        spin_lock_irqsave(&ds_lock, irq);
-
-        error = -EPERM;
-        if (!check_tracer(task))
-                goto out_unlock;
-        get_tracer(task);
+        /* Claim the bts part of the tracing context we acquired above. */
+        spin_lock_irq(&ds_lock);
 
         error = -EPERM;
         if (tracer->ds.context->bts_master)
-                goto out_put_tracer;
+                goto out_unlock;
         tracer->ds.context->bts_master = tracer;
 
-        spin_unlock_irqrestore(&ds_lock, irq);
+        spin_unlock_irq(&ds_lock);
 
+        /*
+         * Now that we own the bts part of the context, let's complete the
+         * initialization for that part.
+         */
+        ds_init_ds_trace(&tracer->trace.ds, ds_bts, base, size, th, flags);
+        ds_write_config(tracer->ds.context, &tracer->trace.ds, ds_bts);
+        ds_install_ds_area(tracer->ds.context);
 
         tracer->trace.read  = bts_read;
         tracer->trace.write = bts_write;
 
-        ds_write_config(tracer->ds.context, &tracer->trace.ds, ds_bts);
+        /* Start tracing. */
         ds_resume_bts(tracer);
 
         return tracer;
 
- out_put_tracer:
-        put_tracer(task);
  out_unlock:
-        spin_unlock_irqrestore(&ds_lock, irq);
+        spin_unlock_irq(&ds_lock);
         ds_put_context(tracer->ds.context);
  out_tracer:
         kfree(tracer);
+ out_put_tracer:
+        put_tracer(task);
  out:
         return ERR_PTR(error);
 }
 
-struct pebs_tracer *ds_request_pebs(struct task_struct *task,
-                                    void *base, size_t size,
-                                    pebs_ovfl_callback_t ovfl, size_t th,
-                                    unsigned int flags)
+struct bts_tracer *ds_request_bts_task(struct task_struct *task,
+                                       void *base, size_t size,
+                                       bts_ovfl_callback_t ovfl,
+                                       size_t th, unsigned int flags)
+{
+        return ds_request_bts(task, 0, base, size, ovfl, th, flags);
+}
+
+struct bts_tracer *ds_request_bts_cpu(int cpu, void *base, size_t size,
+                                      bts_ovfl_callback_t ovfl,
+                                      size_t th, unsigned int flags)
+{
+        return ds_request_bts(NULL, cpu, base, size, ovfl, th, flags);
+}
+
+static struct pebs_tracer *ds_request_pebs(struct task_struct *task, int cpu,
+                                           void *base, size_t size,
+                                           pebs_ovfl_callback_t ovfl, size_t th,
+                                           unsigned int flags)
 {
         struct pebs_tracer *tracer;
-        unsigned long irq;
         int error;
 
-        /* buffer overflow notification is not yet implemented */
+        /* Buffer overflow notification is not yet implemented. */
         error = -EOPNOTSUPP;
         if (ovfl)
                 goto out;
 
+        error = get_tracer(task);
+        if (error < 0)
+                goto out;
+
         error = -ENOMEM;
         tracer = kzalloc(sizeof(*tracer), GFP_KERNEL);
         if (!tracer)
-                goto out;
+                goto out_put_tracer;
         tracer->ovfl = ovfl;
 
+        /* Do some more error checking and acquire a tracing context. */
         error = ds_request(&tracer->ds, &tracer->trace.ds,
-                           ds_pebs, task, base, size, th, flags);
+                           ds_pebs, task, cpu, base, size, th);
         if (error < 0)
                 goto out_tracer;
 
-        spin_lock_irqsave(&ds_lock, irq);
-
-        error = -EPERM;
-        if (!check_tracer(task))
-                goto out_unlock;
-        get_tracer(task);
+        /* Claim the pebs part of the tracing context we acquired above. */
+        spin_lock_irq(&ds_lock);
 
         error = -EPERM;
         if (tracer->ds.context->pebs_master)
-                goto out_put_tracer;
+                goto out_unlock;
         tracer->ds.context->pebs_master = tracer;
 
-        spin_unlock_irqrestore(&ds_lock, irq);
+        spin_unlock_irq(&ds_lock);
 
+        /*
+         * Now that we own the pebs part of the context, let's complete the
+         * initialization for that part.
+         */
+        ds_init_ds_trace(&tracer->trace.ds, ds_pebs, base, size, th, flags);
         ds_write_config(tracer->ds.context, &tracer->trace.ds, ds_pebs);
+        ds_install_ds_area(tracer->ds.context);
+
+        /* Start tracing. */
         ds_resume_pebs(tracer);
 
         return tracer;
 
- out_put_tracer:
-        put_tracer(task);
  out_unlock:
-        spin_unlock_irqrestore(&ds_lock, irq);
+        spin_unlock_irq(&ds_lock);
         ds_put_context(tracer->ds.context);
  out_tracer:
         kfree(tracer);
+ out_put_tracer:
+        put_tracer(task);
  out:
         return ERR_PTR(error);
 }
 
-void ds_release_bts(struct bts_tracer *tracer)
+struct pebs_tracer *ds_request_pebs_task(struct task_struct *task,
+                                         void *base, size_t size,
+                                         pebs_ovfl_callback_t ovfl,
+                                         size_t th, unsigned int flags)
 {
-        if (!tracer)
-                return;
+        return ds_request_pebs(task, 0, base, size, ovfl, th, flags);
+}
 
-        ds_suspend_bts(tracer);
+struct pebs_tracer *ds_request_pebs_cpu(int cpu, void *base, size_t size,
+                                        pebs_ovfl_callback_t ovfl,
+                                        size_t th, unsigned int flags)
+{
+        return ds_request_pebs(NULL, cpu, base, size, ovfl, th, flags);
+}
+
+static void ds_free_bts(struct bts_tracer *tracer)
+{
+        struct task_struct *task;
+
+        task = tracer->ds.context->task;
 
         WARN_ON_ONCE(tracer->ds.context->bts_master != tracer);
         tracer->ds.context->bts_master = NULL;
 
-        put_tracer(tracer->ds.context->task);
+        /* Make sure tracing stopped and the tracer is not in use. */
+        if (task && (task != current))
+                wait_task_context_switch(task);
+
         ds_put_context(tracer->ds.context);
+        put_tracer(task);
 
         kfree(tracer);
 }
 
+void ds_release_bts(struct bts_tracer *tracer)
+{
+        might_sleep();
+
+        if (!tracer)
+                return;
+
+        ds_suspend_bts(tracer);
+        ds_free_bts(tracer);
+}
+
+int ds_release_bts_noirq(struct bts_tracer *tracer)
+{
+        struct task_struct *task;
+        unsigned long irq;
+        int error;
+
+        if (!tracer)
+                return 0;
+
+        task = tracer->ds.context->task;
+
+        local_irq_save(irq);
+
+        error = -EPERM;
+        if (!task &&
+            (tracer->ds.context->cpu != smp_processor_id()))
+                goto out;
+
+        error = -EPERM;
+        if (task && (task != current))
+                goto out;
+
+        ds_suspend_bts_noirq(tracer);
+        ds_free_bts(tracer);
+
+        error = 0;
+ out:
+        local_irq_restore(irq);
+        return error;
+}
+
+static void update_task_debugctlmsr(struct task_struct *task,
+                                    unsigned long debugctlmsr)
+{
+        task->thread.debugctlmsr = debugctlmsr;
+
+        get_cpu();
+        if (task == current)
+                update_debugctlmsr(debugctlmsr);
+        put_cpu();
+}
+
 void ds_suspend_bts(struct bts_tracer *tracer)
 {
         struct task_struct *task;
+        unsigned long debugctlmsr;
+        int cpu;
 
         if (!tracer)
                 return;
 
+        tracer->flags = 0;
+
         task = tracer->ds.context->task;
+        cpu  = tracer->ds.context->cpu;
 
-        if (!task || (task == current))
-                update_debugctlmsr(get_debugctlmsr() & ~BTS_CONTROL);
+        WARN_ON(!task && irqs_disabled());
 
-        if (task) {
-                task->thread.debugctlmsr &= ~BTS_CONTROL;
+        debugctlmsr = (task ?
+                       task->thread.debugctlmsr :
+                       get_debugctlmsr_on_cpu(cpu));
+        debugctlmsr &= ~BTS_CONTROL;
 
-                if (!task->thread.debugctlmsr)
-                        clear_tsk_thread_flag(task, TIF_DEBUGCTLMSR);
-        }
+        if (task)
+                update_task_debugctlmsr(task, debugctlmsr);
+        else
+                update_debugctlmsr_on_cpu(cpu, debugctlmsr);
 }
 
-void ds_resume_bts(struct bts_tracer *tracer)
+int ds_suspend_bts_noirq(struct bts_tracer *tracer)
 {
         struct task_struct *task;
-        unsigned long control;
+        unsigned long debugctlmsr, irq;
+        int cpu, error = 0;
 
         if (!tracer)
-                return;
+                return 0;
+
+        tracer->flags = 0;
 
         task = tracer->ds.context->task;
+        cpu  = tracer->ds.context->cpu;
+
+        local_irq_save(irq);
+
+        error = -EPERM;
+        if (!task && (cpu != smp_processor_id()))
+                goto out;
+
+        debugctlmsr = (task ?
+                       task->thread.debugctlmsr :
+                       get_debugctlmsr());
+        debugctlmsr &= ~BTS_CONTROL;
+
+        if (task)
+                update_task_debugctlmsr(task, debugctlmsr);
+        else
+                update_debugctlmsr(debugctlmsr);
+
+        error = 0;
+ out:
+        local_irq_restore(irq);
+        return error;
+}
+
+static unsigned long ds_bts_control(struct bts_tracer *tracer)
+{
+        unsigned long control;
 
         control = ds_cfg.ctl[dsf_bts];
         if (!(tracer->trace.ds.flags & BTS_KERNEL))
@@ -797,41 +1015,149 @@ void ds_resume_bts(struct bts_tracer *tracer)
         if (!(tracer->trace.ds.flags & BTS_USER))
                 control |= ds_cfg.ctl[dsf_bts_user];
 
-        if (task) {
-                task->thread.debugctlmsr |= control;
-                set_tsk_thread_flag(task, TIF_DEBUGCTLMSR);
-        }
-
-        if (!task || (task == current))
-                update_debugctlmsr(get_debugctlmsr() | control);
+        return control;
 }
 
-void ds_release_pebs(struct pebs_tracer *tracer)
+void ds_resume_bts(struct bts_tracer *tracer)
 {
+        struct task_struct *task;
+        unsigned long debugctlmsr;
+        int cpu;
+
         if (!tracer)
                 return;
 
-        ds_suspend_pebs(tracer);
+        tracer->flags = tracer->trace.ds.flags;
+
+        task = tracer->ds.context->task;
+        cpu  = tracer->ds.context->cpu;
+
+        WARN_ON(!task && irqs_disabled());
+
+        debugctlmsr = (task ?
+                       task->thread.debugctlmsr :
+                       get_debugctlmsr_on_cpu(cpu));
+        debugctlmsr |= ds_bts_control(tracer);
+
+        if (task)
+                update_task_debugctlmsr(task, debugctlmsr);
+        else
+                update_debugctlmsr_on_cpu(cpu, debugctlmsr);
+}
+
+int ds_resume_bts_noirq(struct bts_tracer *tracer)
+{
+        struct task_struct *task;
+        unsigned long debugctlmsr, irq;
+        int cpu, error = 0;
+
+        if (!tracer)
+                return 0;
+
+        tracer->flags = tracer->trace.ds.flags;
+
+        task = tracer->ds.context->task;
+        cpu  = tracer->ds.context->cpu;
+
+        local_irq_save(irq);
+
+        error = -EPERM;
+        if (!task && (cpu != smp_processor_id()))
+                goto out;
+
+        debugctlmsr = (task ?
+                       task->thread.debugctlmsr :
+                       get_debugctlmsr());
+        debugctlmsr |= ds_bts_control(tracer);
+
+        if (task)
+                update_task_debugctlmsr(task, debugctlmsr);
+        else
+                update_debugctlmsr(debugctlmsr);
+
+        error = 0;
+ out:
+        local_irq_restore(irq);
+        return error;
+}
+
+static void ds_free_pebs(struct pebs_tracer *tracer)
+{
+        struct task_struct *task;
+
+        task = tracer->ds.context->task;
 
         WARN_ON_ONCE(tracer->ds.context->pebs_master != tracer);
         tracer->ds.context->pebs_master = NULL;
 
-        put_tracer(tracer->ds.context->task);
         ds_put_context(tracer->ds.context);
+        put_tracer(task);
 
         kfree(tracer);
 }
 
+void ds_release_pebs(struct pebs_tracer *tracer)
+{
+        might_sleep();
+
+        if (!tracer)
+                return;
+
+        ds_suspend_pebs(tracer);
+        ds_free_pebs(tracer);
+}
+
+int ds_release_pebs_noirq(struct pebs_tracer *tracer)
+{
+        struct task_struct *task;
+        unsigned long irq;
+        int error;
+
+        if (!tracer)
+                return 0;
+
+        task = tracer->ds.context->task;
+
+        local_irq_save(irq);
+
+        error = -EPERM;
+        if (!task &&
+            (tracer->ds.context->cpu != smp_processor_id()))
+                goto out;
+
+        error = -EPERM;
+        if (task && (task != current))
+                goto out;
+
+        ds_suspend_pebs_noirq(tracer);
+        ds_free_pebs(tracer);
+
+        error = 0;
+ out:
+        local_irq_restore(irq);
+        return error;
+}
+
 void ds_suspend_pebs(struct pebs_tracer *tracer)
 {
 
 }
 
+int ds_suspend_pebs_noirq(struct pebs_tracer *tracer)
+{
+        return 0;
+}
+
 void ds_resume_pebs(struct pebs_tracer *tracer)
 {
 
 }
 
+int ds_resume_pebs_noirq(struct pebs_tracer *tracer)
+{
+        return 0;
+}
+
 const struct bts_trace *ds_read_bts(struct bts_tracer *tracer)
 {
         if (!tracer)
@@ -847,8 +1173,12 @@ const struct pebs_trace *ds_read_pebs(struct pebs_tracer *tracer)
                 return NULL;
 
         ds_read_config(tracer->ds.context, &tracer->trace.ds, ds_pebs);
-        tracer->trace.reset_value =
-                *(u64 *)(tracer->ds.context->ds + (ds_cfg.sizeof_field * 8));
+
+        tracer->trace.counters = ds_cfg.nr_counter_reset;
+        memcpy(tracer->trace.counter_reset,
+               tracer->ds.context->ds +
+               (NUM_DS_PTR_FIELDS * ds_cfg.sizeof_ptr_field),
+               ds_cfg.nr_counter_reset * PEBS_RESET_FIELD_SIZE);
 
         return &tracer->trace;
 }
@@ -873,18 +1203,24 @@ int ds_reset_pebs(struct pebs_tracer *tracer)
 
         tracer->trace.ds.top = tracer->trace.ds.begin;
 
-        ds_set(tracer->ds.context->ds, ds_bts, ds_index,
+        ds_set(tracer->ds.context->ds, ds_pebs, ds_index,
                (unsigned long)tracer->trace.ds.top);
 
         return 0;
 }
 
-int ds_set_pebs_reset(struct pebs_tracer *tracer, u64 value)
+int ds_set_pebs_reset(struct pebs_tracer *tracer,
+                      unsigned int counter, u64 value)
 {
         if (!tracer)
                 return -EINVAL;
 
-        *(u64 *)(tracer->ds.context->ds + (ds_cfg.sizeof_field * 8)) = value;
+        if (ds_cfg.nr_counter_reset < counter)
+                return -EINVAL;
+
+        *(u64 *)(tracer->ds.context->ds +
+                 (NUM_DS_PTR_FIELDS * ds_cfg.sizeof_ptr_field) +
+                 (counter * PEBS_RESET_FIELD_SIZE)) = value;
 
         return 0;
 }
@@ -894,73 +1230,117 @@ static const struct ds_configuration ds_cfg_netburst = {
         .ctl[dsf_bts]           = (1 << 2) | (1 << 3),
         .ctl[dsf_bts_kernel]    = (1 << 5),
         .ctl[dsf_bts_user]      = (1 << 6),
-
-        .sizeof_field           = sizeof(long),
-        .sizeof_rec[ds_bts]     = sizeof(long) * 3,
-#ifdef __i386__
-        .sizeof_rec[ds_pebs]    = sizeof(long) * 10,
-#else
-        .sizeof_rec[ds_pebs]    = sizeof(long) * 18,
-#endif
+        .nr_counter_reset       = 1,
 };
 static const struct ds_configuration ds_cfg_pentium_m = {
         .name = "Pentium M",
         .ctl[dsf_bts]           = (1 << 6) | (1 << 7),
-
-        .sizeof_field           = sizeof(long),
-        .sizeof_rec[ds_bts]     = sizeof(long) * 3,
-#ifdef __i386__
-        .sizeof_rec[ds_pebs]    = sizeof(long) * 10,
-#else
-        .sizeof_rec[ds_pebs]    = sizeof(long) * 18,
-#endif
+        .nr_counter_reset       = 1,
 };
 static const struct ds_configuration ds_cfg_core2_atom = {
         .name = "Core 2/Atom",
         .ctl[dsf_bts]           = (1 << 6) | (1 << 7),
         .ctl[dsf_bts_kernel]    = (1 << 9),
         .ctl[dsf_bts_user]      = (1 << 10),
-
-        .sizeof_field           = 8,
-        .sizeof_rec[ds_bts]     = 8 * 3,
-        .sizeof_rec[ds_pebs]    = 8 * 18,
+        .nr_counter_reset       = 1,
+};
+static const struct ds_configuration ds_cfg_core_i7 = {
+        .name = "Core i7",
+        .ctl[dsf_bts]           = (1 << 6) | (1 << 7),
+        .ctl[dsf_bts_kernel]    = (1 << 9),
+        .ctl[dsf_bts_user]      = (1 << 10),
+        .nr_counter_reset       = 4,
 };
 
 static void
-ds_configure(const struct ds_configuration *cfg)
+ds_configure(const struct ds_configuration *cfg,
+             struct cpuinfo_x86 *cpu)
 {
+        unsigned long nr_pebs_fields = 0;
+
+        printk(KERN_INFO "[ds] using %s configuration\n", cfg->name);
+
+#ifdef __i386__
+        nr_pebs_fields = 10;
+#else
+        nr_pebs_fields = 18;
+#endif
+
+        /*
+         * Starting with version 2, architectural performance
+         * monitoring supports a format specifier.
+         */
+        if ((cpuid_eax(0xa) & 0xff) > 1) {
+                unsigned long perf_capabilities, format;
+
+                rdmsrl(MSR_IA32_PERF_CAPABILITIES, perf_capabilities);
+
+                format = (perf_capabilities >> 8) & 0xf;
+
+                switch (format) {
+                case 0:
+                        nr_pebs_fields = 18;
+                        break;
+                case 1:
+                        nr_pebs_fields = 22;
+                        break;
+                default:
+                        printk(KERN_INFO
+                               "[ds] unknown PEBS format: %lu\n", format);
+                        nr_pebs_fields = 0;
+                        break;
+                }
+        }
+
         memset(&ds_cfg, 0, sizeof(ds_cfg));
         ds_cfg = *cfg;
 
-        printk(KERN_INFO "[ds] using %s configuration\n", ds_cfg.name);
+        ds_cfg.sizeof_ptr_field =
+                (cpu_has(cpu, X86_FEATURE_DTES64) ? 8 : 4);
+
+        ds_cfg.sizeof_rec[ds_bts]  = ds_cfg.sizeof_ptr_field * 3;
+        ds_cfg.sizeof_rec[ds_pebs] = ds_cfg.sizeof_ptr_field * nr_pebs_fields;
 
-        if (!cpu_has_bts) {
-                ds_cfg.ctl[dsf_bts] = 0;
+        if (!cpu_has(cpu, X86_FEATURE_BTS)) {
+                ds_cfg.sizeof_rec[ds_bts] = 0;
                 printk(KERN_INFO "[ds] bts not available\n");
         }
-        if (!cpu_has_pebs)
+        if (!cpu_has(cpu, X86_FEATURE_PEBS)) {
+                ds_cfg.sizeof_rec[ds_pebs] = 0;
                 printk(KERN_INFO "[ds] pebs not available\n");
+        }
+
+        printk(KERN_INFO "[ds] sizes: address: %u bit, ",
+               8 * ds_cfg.sizeof_ptr_field);
+        printk("bts/pebs record: %u/%u bytes\n",
+               ds_cfg.sizeof_rec[ds_bts], ds_cfg.sizeof_rec[ds_pebs]);
 
-        WARN_ON_ONCE(MAX_SIZEOF_DS < (12 * ds_cfg.sizeof_field));
+        WARN_ON_ONCE(MAX_PEBS_COUNTERS < ds_cfg.nr_counter_reset);
 }
 
 void __cpuinit ds_init_intel(struct cpuinfo_x86 *c)
 {
+        /* Only configure the first cpu. Others are identical. */
+        if (ds_cfg.name)
+                return;
+
         switch (c->x86) {
         case 0x6:
                 switch (c->x86_model) {
                 case 0x9:
                 case 0xd: /* Pentium M */
-                        ds_configure(&ds_cfg_pentium_m);
+                        ds_configure(&ds_cfg_pentium_m, c);
                         break;
                 case 0xf:
                 case 0x17: /* Core2 */
                 case 0x1c: /* Atom */
-                        ds_configure(&ds_cfg_core2_atom);
+                        ds_configure(&ds_cfg_core2_atom, c);
+                        break;
+                case 0x1a: /* Core i7 */
+                        ds_configure(&ds_cfg_core_i7, c);
                         break;
-                case 0x1a: /* i7 */
                 default:
-                        /* sorry, don't know about them */
+                        /* Sorry, don't know about them. */
                         break;
                 }
                 break;
@@ -969,64 +1349,89 @@ void __cpuinit ds_init_intel(struct cpuinfo_x86 *c)
                 case 0x0:
                 case 0x1:
                 case 0x2: /* Netburst */
-                        ds_configure(&ds_cfg_netburst);
+                        ds_configure(&ds_cfg_netburst, c);
                         break;
                 default:
-                        /* sorry, don't know about them */
+                        /* Sorry, don't know about them. */
                         break;
                 }
                 break;
         default:
-                /* sorry, don't know about them */
+                /* Sorry, don't know about them. */
                 break;
         }
 }
 
+static inline void ds_take_timestamp(struct ds_context *context,
+                                     enum bts_qualifier qualifier,
+                                     struct task_struct *task)
+{
+        struct bts_tracer *tracer = context->bts_master;
+        struct bts_struct ts;
+
+        /* Prevent compilers from reading the tracer pointer twice. */
+        barrier();
+
+        if (!tracer || !(tracer->flags & BTS_TIMESTAMPS))
+                return;
+
+        memset(&ts, 0, sizeof(ts));
+        ts.qualifier            = qualifier;
+        ts.variant.event.clock  = trace_clock_global();
+        ts.variant.event.pid    = task->pid;
+
+        bts_write(tracer, &ts);
+}
+
 /*
  * Change the DS configuration from tracing prev to tracing next.
  */
 void ds_switch_to(struct task_struct *prev, struct task_struct *next)
 {
-        struct ds_context *prev_ctx = prev->thread.ds_ctx;
-        struct ds_context *next_ctx = next->thread.ds_ctx;
+        struct ds_context *prev_ctx     = prev->thread.ds_ctx;
+        struct ds_context *next_ctx     = next->thread.ds_ctx;
+        unsigned long debugctlmsr       = next->thread.debugctlmsr;
+
+        /* Make sure all data is read before we start. */
+        barrier();
 
         if (prev_ctx) {
                 update_debugctlmsr(0);
 
-                if (prev_ctx->bts_master &&
-                    (prev_ctx->bts_master->trace.ds.flags & BTS_TIMESTAMPS)) {
-                        struct bts_struct ts = {
-                                .qualifier = bts_task_departs,
-                                .variant.timestamp.jiffies = jiffies_64,
-                                .variant.timestamp.pid = prev->pid
-                        };
-                        bts_write(prev_ctx->bts_master, &ts);
-                }
+                ds_take_timestamp(prev_ctx, bts_task_departs, prev);
         }
 
         if (next_ctx) {
-                if (next_ctx->bts_master &&
-                    (next_ctx->bts_master->trace.ds.flags & BTS_TIMESTAMPS)) {
-                        struct bts_struct ts = {
-                                .qualifier = bts_task_arrives,
-                                .variant.timestamp.jiffies = jiffies_64,
-                                .variant.timestamp.pid = next->pid
-                        };
-                        bts_write(next_ctx->bts_master, &ts);
-                }
+                ds_take_timestamp(next_ctx, bts_task_arrives, next);
 
                 wrmsrl(MSR_IA32_DS_AREA, (unsigned long)next_ctx->ds);
         }
 
-        update_debugctlmsr(next->thread.debugctlmsr);
+        update_debugctlmsr(debugctlmsr);
 }
 
-void ds_copy_thread(struct task_struct *tsk, struct task_struct *father)
+static __init int ds_selftest(void)
 {
-        clear_tsk_thread_flag(tsk, TIF_DS_AREA_MSR);
-        tsk->thread.ds_ctx = NULL;
-}
+        if (ds_cfg.sizeof_rec[ds_bts]) {
+                int error;
 
-void ds_exit_thread(struct task_struct *tsk)
-{
+                error = ds_selftest_bts();
+                if (error) {
+                        WARN(1, "[ds] selftest failed. disabling bts.\n");
+                        ds_cfg.sizeof_rec[ds_bts] = 0;
+                }
+        }
+
+        if (ds_cfg.sizeof_rec[ds_pebs]) {
+                int error;
+
+                error = ds_selftest_pebs();
+                if (error) {
+                        WARN(1, "[ds] selftest failed. disabling pebs.\n");
+                        ds_cfg.sizeof_rec[ds_pebs] = 0;
+                }
+        }
+
+        return 0;
 }
+device_initcall(ds_selftest);
diff --git a/arch/x86/kernel/ds_selftest.c b/arch/x86/kernel/ds_selftest.c
new file mode 100644
index 000000000000..6bc7c199ab99
--- /dev/null
+++ b/arch/x86/kernel/ds_selftest.c
@@ -0,0 +1,408 @@
+/*
+ * Debug Store support - selftest
+ *
+ *
+ * Copyright (C) 2009 Intel Corporation.
+ * Markus Metzger <markus.t.metzger@intel.com>, 2009
+ */
+
+#include "ds_selftest.h"
+
+#include <linux/kernel.h>
+#include <linux/string.h>
+#include <linux/smp.h>
+#include <linux/cpu.h>
+
+#include <asm/ds.h>
+
+
+#define BUFFER_SIZE             521     /* Intentionally chose an odd size. */
+#define SMALL_BUFFER_SIZE       24      /* A single bts entry. */
+
+struct ds_selftest_bts_conf {
+        struct bts_tracer *tracer;
+        int error;
+        int (*suspend)(struct bts_tracer *);
+        int (*resume)(struct bts_tracer *);
+};
+
+static int ds_selftest_bts_consistency(const struct bts_trace *trace)
+{
+        int error = 0;
+
+        if (!trace) {
+                printk(KERN_CONT "failed to access trace...");
+                /* Bail out. Other tests are pointless. */
+                return -1;
+        }
+
+        if (!trace->read) {
+                printk(KERN_CONT "bts read not available...");
+                error = -1;
+        }
+
+        /* Do some sanity checks on the trace configuration. */
+        if (!trace->ds.n) {
+                printk(KERN_CONT "empty bts buffer...");
+                error = -1;
+        }
+        if (!trace->ds.size) {
+                printk(KERN_CONT "bad bts trace setup...");
+                error = -1;
+        }
+        if (trace->ds.end !=
+            (char *)trace->ds.begin + (trace->ds.n * trace->ds.size)) {
+                printk(KERN_CONT "bad bts buffer setup...");
+                error = -1;
+        }
+        /*
+         * We allow top in [begin; end], since its not clear when the
+         * overflow adjustment happens: after the increment or before the
+         * write.
+         */
+        if ((trace->ds.top < trace->ds.begin) ||
+            (trace->ds.end < trace->ds.top)) {
+                printk(KERN_CONT "bts top out of bounds...");
+                error = -1;
+        }
+
+        return error;
+}
+
+static int ds_selftest_bts_read(struct bts_tracer *tracer,
+                                const struct bts_trace *trace,
+                                const void *from, const void *to)
+{
+        const unsigned char *at;
+
+        /*
+         * Check a few things which do not belong to this test.
+         * They should be covered by other tests.
+         */
+        if (!trace)
+                return -1;
+
+        if (!trace->read)
+                return -1;
+
+        if (to < from)
+                return -1;
+
+        if (from < trace->ds.begin)
+                return -1;
+
+        if (trace->ds.end < to)
+                return -1;
+
+        if (!trace->ds.size)
+                return -1;
+
+        /* Now to the test itself. */
+        for (at = from; (void *)at < to; at += trace->ds.size) {
+                struct bts_struct bts;
+                unsigned long index;
+                int error;
+
+                if (((void *)at - trace->ds.begin) % trace->ds.size) {
+                        printk(KERN_CONT
+                               "read from non-integer index...");
+                        return -1;
+                }
+                index = ((void *)at - trace->ds.begin) / trace->ds.size;
+
+                memset(&bts, 0, sizeof(bts));
+                error = trace->read(tracer, at, &bts);
+                if (error < 0) {
+                        printk(KERN_CONT
+                               "error reading bts trace at [%lu] (0x%p)...",
+                               index, at);
+                        return error;
+                }
+
+                switch (bts.qualifier) {
+                case BTS_BRANCH:
+                        break;
+                default:
+                        printk(KERN_CONT
+                               "unexpected bts entry %llu at [%lu] (0x%p)...",
+                               bts.qualifier, index, at);
+                        return -1;
+                }
+        }
+
+        return 0;
+}
+
+static void ds_selftest_bts_cpu(void *arg)
+{
+        struct ds_selftest_bts_conf *conf = arg;
+        const struct bts_trace *trace;
+        void *top;
+
+        if (IS_ERR(conf->tracer)) {
+                conf->error = PTR_ERR(conf->tracer);
+                conf->tracer = NULL;
+
+                printk(KERN_CONT
+                       "initialization failed (err: %d)...", conf->error);
+                return;
+        }
+
+        /* We should meanwhile have enough trace. */
+        conf->error = conf->suspend(conf->tracer);
+        if (conf->error < 0)
+                return;
+
+        /* Let's see if we can access the trace. */
+        trace = ds_read_bts(conf->tracer);
+
+        conf->error = ds_selftest_bts_consistency(trace);
+        if (conf->error < 0)
+                return;
+
+        /* If everything went well, we should have a few trace entries. */
+        if (trace->ds.top == trace->ds.begin) {
+                /*
+                 * It is possible but highly unlikely that we got a
+                 * buffer overflow and end up at exactly the same
+                 * position we started from.
+                 * Let's issue a warning, but continue.
+                 */
+                printk(KERN_CONT "no trace/overflow...");
+        }
+
+        /* Let's try to read the trace we collected. */
+        conf->error =
+                ds_selftest_bts_read(conf->tracer, trace,
+                                     trace->ds.begin, trace->ds.top);
+        if (conf->error < 0)
+                return;
+
+        /*
+         * Let's read the trace again.
+         * Since we suspended tracing, we should get the same result.
+         */
+        top = trace->ds.top;
+
+        trace = ds_read_bts(conf->tracer);
+        conf->error = ds_selftest_bts_consistency(trace);
+        if (conf->error < 0)
+                return;
+
+        if (top != trace->ds.top) {
+                printk(KERN_CONT "suspend not working...");
+                conf->error = -1;
+                return;
+        }
+
+        /* Let's collect some more trace - see if resume is working. */
+        conf->error = conf->resume(conf->tracer);
+        if (conf->error < 0)
+                return;
+
+        conf->error = conf->suspend(conf->tracer);
+        if (conf->error < 0)
+                return;
+
+        trace = ds_read_bts(conf->tracer);
+
+        conf->error = ds_selftest_bts_consistency(trace);
+        if (conf->error < 0)
+                return;
+
+        if (trace->ds.top == top) {
+                /*
+                 * It is possible but highly unlikely that we got a
+                 * buffer overflow and end up at exactly the same
+                 * position we started from.
+                 * Let's issue a warning and check the full trace.
+                 */
+                printk(KERN_CONT
+                       "no resume progress/overflow...");
+
+                conf->error =
+                        ds_selftest_bts_read(conf->tracer, trace,
+                                             trace->ds.begin, trace->ds.end);
+        } else if (trace->ds.top < top) {
+                /*
+                 * We had a buffer overflow - the entire buffer should
+                 * contain trace records.
+                 */
+                conf->error =
+                        ds_selftest_bts_read(conf->tracer, trace,
+                                             trace->ds.begin, trace->ds.end);
+        } else {
+                /*
+                 * It is quite likely that the buffer did not overflow.
+                 * Let's just check the delta trace.
+                 */
+                conf->error =
+                        ds_selftest_bts_read(conf->tracer, trace, top,
+                                             trace->ds.top);
+        }
+        if (conf->error < 0)
+                return;
+
+        conf->error = 0;
+}
+
+static int ds_suspend_bts_wrap(struct bts_tracer *tracer)
+{
+        ds_suspend_bts(tracer);
+        return 0;
+}
+
+static int ds_resume_bts_wrap(struct bts_tracer *tracer)
+{
+        ds_resume_bts(tracer);
+        return 0;
+}
+
+static void ds_release_bts_noirq_wrap(void *tracer)
+{
+        (void)ds_release_bts_noirq(tracer);
+}
+
+static int ds_selftest_bts_bad_release_noirq(int cpu,
+                                             struct bts_tracer *tracer)
+{
+        int error = -EPERM;
+
+        /* Try to release the tracer on the wrong cpu. */
+        get_cpu();
+        if (cpu != smp_processor_id()) {
+                error = ds_release_bts_noirq(tracer);
+                if (error != -EPERM)
+                        printk(KERN_CONT "release on wrong cpu...");
+        }
+        put_cpu();
+
+        return error ? 0 : -1;
+}
+
+static int ds_selftest_bts_bad_request_cpu(int cpu, void *buffer)
+{
+        struct bts_tracer *tracer;
+        int error;
+
+        /* Try to request cpu tracing while task tracing is active. */
+        tracer = ds_request_bts_cpu(cpu, buffer, BUFFER_SIZE, NULL,
+                                    (size_t)-1, BTS_KERNEL);
+        error = PTR_ERR(tracer);
+        if (!IS_ERR(tracer)) {
+                ds_release_bts(tracer);
+                error = 0;
+        }
+
+        if (error != -EPERM)
+                printk(KERN_CONT "cpu/task tracing overlap...");
+
+        return error ? 0 : -1;
+}
+
+static int ds_selftest_bts_bad_request_task(void *buffer)
+{
+        struct bts_tracer *tracer;
+        int error;
+
+        /* Try to request cpu tracing while task tracing is active. */
+        tracer = ds_request_bts_task(current, buffer, BUFFER_SIZE, NULL,
+                                    (size_t)-1, BTS_KERNEL);
+        error = PTR_ERR(tracer);
+        if (!IS_ERR(tracer)) {
+                error = 0;
+                ds_release_bts(tracer);
+        }
+
+        if (error != -EPERM)
+                printk(KERN_CONT "task/cpu tracing overlap...");
+
+        return error ? 0 : -1;
+}
+
+int ds_selftest_bts(void)
+{
+        struct ds_selftest_bts_conf conf;
+        unsigned char buffer[BUFFER_SIZE], *small_buffer;
+        unsigned long irq;
+        int cpu;
+
+        printk(KERN_INFO "[ds] bts selftest...");
+        conf.error = 0;
+
+        small_buffer = (unsigned char *)ALIGN((unsigned long)buffer, 8) + 8;
+
+        get_online_cpus();
+        for_each_online_cpu(cpu) {
+                conf.suspend = ds_suspend_bts_wrap;
+                conf.resume = ds_resume_bts_wrap;
+                conf.tracer =
+                        ds_request_bts_cpu(cpu, buffer, BUFFER_SIZE,
+                                           NULL, (size_t)-1, BTS_KERNEL);
+                ds_selftest_bts_cpu(&conf);
+                if (conf.error >= 0)
+                        conf.error = ds_selftest_bts_bad_request_task(buffer);
+                ds_release_bts(conf.tracer);
+                if (conf.error < 0)
+                        goto out;
+
+                conf.suspend = ds_suspend_bts_noirq;
+                conf.resume = ds_resume_bts_noirq;
+                conf.tracer =
+                        ds_request_bts_cpu(cpu, buffer, BUFFER_SIZE,
+                                           NULL, (size_t)-1, BTS_KERNEL);
+                smp_call_function_single(cpu, ds_selftest_bts_cpu, &conf, 1);
+                if (conf.error >= 0) {
+                        conf.error =
+                                ds_selftest_bts_bad_release_noirq(cpu,
+                                                                  conf.tracer);
+                        /* We must not release the tracer twice. */
+                        if (conf.error < 0)
+                                conf.tracer = NULL;
+                }
+                if (conf.error >= 0)
+                        conf.error = ds_selftest_bts_bad_request_task(buffer);
+                smp_call_function_single(cpu, ds_release_bts_noirq_wrap,
+                                         conf.tracer, 1);
+                if (conf.error < 0)
+                        goto out;
+        }
+
+        conf.suspend = ds_suspend_bts_wrap;
+        conf.resume = ds_resume_bts_wrap;
+        conf.tracer =
+                ds_request_bts_task(current, buffer, BUFFER_SIZE,
+                                    NULL, (size_t)-1, BTS_KERNEL);
+        ds_selftest_bts_cpu(&conf);
+        if (conf.error >= 0)
+                conf.error = ds_selftest_bts_bad_request_cpu(0, buffer);
+        ds_release_bts(conf.tracer);
+        if (conf.error < 0)
+                goto out;
+
+        conf.suspend = ds_suspend_bts_noirq;
+        conf.resume = ds_resume_bts_noirq;
+        conf.tracer =
+                ds_request_bts_task(current, small_buffer, SMALL_BUFFER_SIZE,
+                                   NULL, (size_t)-1, BTS_KERNEL);
+        local_irq_save(irq);
+        ds_selftest_bts_cpu(&conf);
+        if (conf.error >= 0)
+                conf.error = ds_selftest_bts_bad_request_cpu(0, buffer);
+        ds_release_bts_noirq(conf.tracer);
+        local_irq_restore(irq);
+        if (conf.error < 0)
+                goto out;
+
+        conf.error = 0;
+ out:
+        put_online_cpus();
+        printk(KERN_CONT "%s.\n", (conf.error ? "failed" : "passed"));
+
+        return conf.error;
+}
+
+int ds_selftest_pebs(void)
+{
+        return 0;
+}
diff --git a/arch/x86/kernel/ds_selftest.h b/arch/x86/kernel/ds_selftest.h
new file mode 100644
index 000000000000..2ba8745c6663
--- /dev/null
+++ b/arch/x86/kernel/ds_selftest.h
@@ -0,0 +1,15 @@
+/*
+ * Debug Store support - selftest
+ *
+ *
+ * Copyright (C) 2009 Intel Corporation.
+ * Markus Metzger <markus.t.metzger@intel.com>, 2009
+ */
+
+#ifdef CONFIG_X86_DS_SELFTEST
+extern int ds_selftest_bts(void);
+extern int ds_selftest_pebs(void);
+#else
+static inline int ds_selftest_bts(void) { return 0; }
+static inline int ds_selftest_pebs(void) { return 0; }
+#endif
diff --git a/arch/x86/kernel/dumpstack.h b/arch/x86/kernel/dumpstack.h
index da87590b8698..81086c227ab7 100644
--- a/arch/x86/kernel/dumpstack.h
+++ b/arch/x86/kernel/dumpstack.h
@@ -29,7 +29,6 @@ show_stack_log_lvl(struct task_struct *task, struct pt_regs *regs,
                 unsigned long *sp, unsigned long bp, char *log_lvl);
 
 extern unsigned int code_bytes;
-extern int kstack_depth_to_print;
 
 /* The form of the top of the frame on the stack */
 struct stack_frame {
diff --git a/arch/x86/kernel/e820.c b/arch/x86/kernel/e820.c
index 006281302925..7271fa33d791 100644
--- a/arch/x86/kernel/e820.c
+++ b/arch/x86/kernel/e820.c
@@ -617,7 +617,7 @@ __init int e820_search_gap(unsigned long *gapstart, unsigned long *gapsize,
  */
 __init void e820_setup_gap(void)
 {
-        unsigned long gapstart, gapsize, round;
+        unsigned long gapstart, gapsize;
         int found;
 
         gapstart = 0x10000000;
@@ -635,14 +635,9 @@ __init void e820_setup_gap(void)
 #endif
 
         /*
-         * See how much we want to round up: start off with
-         * rounding to the next 1MB area.
+         * e820_reserve_resources_late protect stolen RAM already
          */
-        round = 0x100000;
-        while ((gapsize >> 4) > round)
-                round += round;
-        /* Fun with two's complement */
-        pci_mem_start = (gapstart + round) & -round;
+        pci_mem_start = gapstart;
 
         printk(KERN_INFO
                "Allocating PCI resources starting at %lx (gap: %lx:%lx)\n",
@@ -1371,6 +1366,23 @@ void __init e820_reserve_resources(void)
         }
 }
 
+/* How much should we pad RAM ending depending on where it is? */
+static unsigned long ram_alignment(resource_size_t pos)
+{
+        unsigned long mb = pos >> 20;
+
+        /* To 64kB in the first megabyte */
+        if (!mb)
+                return 64*1024;
+
+        /* To 1MB in the first 16MB */
+        if (mb < 16)
+                return 1024*1024;
+
+        /* To 32MB for anything above that */
+        return 32*1024*1024;
+}
+
 void __init e820_reserve_resources_late(void)
 {
         int i;
@@ -1382,6 +1394,24 @@ void __init e820_reserve_resources_late(void)
                         insert_resource_expand_to_fit(&iomem_resource, res);
                 res++;
         }
+
+        /*
+         * Try to bump up RAM regions to reasonable boundaries to
+         * avoid stolen RAM:
+         */
+        for (i = 0; i < e820.nr_map; i++) {
+                struct e820entry *entry = &e820_saved.map[i];
+                resource_size_t start, end;
+
+                if (entry->type != E820_RAM)
+                        continue;
+                start = entry->addr + entry->size;
+                end = round_up(start, ram_alignment(start));
+                if (start == end)
+                        continue;
+                reserve_region_with_split(&iomem_resource, start,
+                                                  end - 1, "RAM buffer");
+        }
 }
 
 char *__init default_machine_specific_memory_setup(void)
diff --git a/arch/x86/kernel/early-quirks.c b/arch/x86/kernel/early-quirks.c
index 76b8cd953dee..ebdb85cf2686 100644
--- a/arch/x86/kernel/early-quirks.c
+++ b/arch/x86/kernel/early-quirks.c
@@ -97,6 +97,7 @@ static void __init nvidia_bugs(int num, int slot, int func)
 }
 
 #if defined(CONFIG_ACPI) && defined(CONFIG_X86_IO_APIC)
+#if defined(CONFIG_ACPI) && defined(CONFIG_X86_IO_APIC)
 static u32 __init ati_ixp4x0_rev(int num, int slot, int func)
 {
         u32 d;
@@ -114,6 +115,7 @@ static u32 __init ati_ixp4x0_rev(int num, int slot, int func)
         d &= 0xff;
         return d;
 }
+#endif
 
 static void __init ati_bugs(int num, int slot, int func)
 {
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index 4234b1235652..de74f0a3e0ed 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -147,27 +147,14 @@ END(ftrace_graph_caller)
 GLOBAL(return_to_handler)
         subq  $80, %rsp
 
+        /* Save the return values */
         movq %rax, (%rsp)
-        movq %rcx, 8(%rsp)
-        movq %rdx, 16(%rsp)
-        movq %rsi, 24(%rsp)
-        movq %rdi, 32(%rsp)
-        movq %r8, 40(%rsp)
-        movq %r9, 48(%rsp)
-        movq %r10, 56(%rsp)
-        movq %r11, 64(%rsp)
+        movq %rdx, 8(%rsp)
 
         call ftrace_return_to_handler
 
         movq %rax, 72(%rsp)
-        movq 64(%rsp), %r11
-        movq 56(%rsp), %r10
-        movq 48(%rsp), %r9
-        movq 40(%rsp), %r8
-        movq 32(%rsp), %rdi
-        movq 24(%rsp), %rsi
-        movq 16(%rsp), %rdx
-        movq 8(%rsp), %rcx
+        movq 8(%rsp), %rdx
         movq (%rsp), %rax
         addq $72, %rsp
         retq
@@ -1032,6 +1019,11 @@ apicinterrupt ERROR_APIC_VECTOR \
 apicinterrupt SPURIOUS_APIC_VECTOR \
         spurious_interrupt smp_spurious_interrupt
 
+#ifdef CONFIG_PERF_COUNTERS
+apicinterrupt LOCAL_PENDING_VECTOR \
+        perf_pending_interrupt smp_perf_pending_interrupt
+#endif
+
 /*
  * Exception entry points.
  */
@@ -1386,6 +1378,11 @@ END(xen_failsafe_callback)
 paranoidzeroentry_ist debug do_debug DEBUG_STACK
 paranoidzeroentry_ist int3 do_int3 DEBUG_STACK
 paranoiderrorentry stack_segment do_stack_segment
+#ifdef CONFIG_XEN
+zeroentry xen_debug do_debug
+zeroentry xen_int3 do_int3
+errorentry xen_stack_segment do_stack_segment
+#endif
 errorentry general_protection do_general_protection
 errorentry page_fault do_page_fault
 #ifdef CONFIG_X86_MCE
diff --git a/arch/x86/kernel/head_32.S b/arch/x86/kernel/head_32.S
index 30683883e0cd..dc5ed4bdd88d 100644
--- a/arch/x86/kernel/head_32.S
+++ b/arch/x86/kernel/head_32.S
@@ -608,13 +608,6 @@ ignore_int:
 ENTRY(initial_code)
         .long i386_start_kernel
 
-.section .text
-/*
- * Real beginning of normal "text" segment
- */
-ENTRY(stext)
-ENTRY(_stext)
-
 /*
  * BSS section
  */
diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c
index 9773395aa758..b0cdde6932f5 100644
--- a/arch/x86/kernel/irq.c
+++ b/arch/x86/kernel/irq.c
@@ -13,6 +13,7 @@
 #include <asm/irq.h>
 #include <asm/idle.h>
 #include <asm/mce.h>
+#include <asm/hw_irq.h>
 
 atomic_t irq_err_count;
 
@@ -62,6 +63,14 @@ static int show_other_interrupts(struct seq_file *p, int prec)
         for_each_online_cpu(j)
                 seq_printf(p, "%10u ", irq_stats(j)->irq_spurious_count);
         seq_printf(p, "  Spurious interrupts\n");
+        seq_printf(p, "%*s: ", prec, "CNT");
+        for_each_online_cpu(j)
+                seq_printf(p, "%10u ", irq_stats(j)->apic_perf_irqs);
+        seq_printf(p, "  Performance counter interrupts\n");
+        seq_printf(p, "%*s: ", prec, "PND");
+        for_each_online_cpu(j)
+                seq_printf(p, "%10u ", irq_stats(j)->apic_pending_irqs);
+        seq_printf(p, "  Performance pending work\n");
 #endif
         if (generic_interrupt_extension) {
                 seq_printf(p, "%*s: ", prec, "PLT");
@@ -175,6 +184,8 @@ u64 arch_irq_stat_cpu(unsigned int cpu)
 #ifdef CONFIG_X86_LOCAL_APIC
         sum += irq_stats(cpu)->apic_timer_irqs;
         sum += irq_stats(cpu)->irq_spurious_count;
+        sum += irq_stats(cpu)->apic_perf_irqs;
+        sum += irq_stats(cpu)->apic_pending_irqs;
 #endif
         if (generic_interrupt_extension)
                 sum += irq_stats(cpu)->generic_irqs;
diff --git a/arch/x86/kernel/irqinit.c b/arch/x86/kernel/irqinit.c
index 4a69ec55be3d..696f0e475c2d 100644
--- a/arch/x86/kernel/irqinit.c
+++ b/arch/x86/kernel/irqinit.c
@@ -207,7 +207,6 @@ static void __init apic_intr_init(void)
 
         /* Performance monitoring interrupts: */
 # ifdef CONFIG_PERF_COUNTERS
-        alloc_intr_gate(LOCAL_PERF_VECTOR, perf_counter_interrupt);
         alloc_intr_gate(LOCAL_PENDING_VECTOR, perf_pending_interrupt);
 # endif
 
diff --git a/arch/x86/kernel/kgdb.c b/arch/x86/kernel/kgdb.c
index b1f4dffb919e..8d82a77a3f3b 100644
--- a/arch/x86/kernel/kgdb.c
+++ b/arch/x86/kernel/kgdb.c
@@ -142,7 +142,7 @@ void sleeping_thread_to_gdb_regs(unsigned long *gdb_regs, struct task_struct *p)
         gdb_regs32[GDB_PS]      = *(unsigned long *)(p->thread.sp + 8);
         gdb_regs32[GDB_CS]      = __KERNEL_CS;
         gdb_regs32[GDB_SS]      = __KERNEL_DS;
-        gdb_regs[GDB_PC]        = p->thread.ip;
+        gdb_regs[GDB_PC]        = 0;
         gdb_regs[GDB_R8]        = 0;
         gdb_regs[GDB_R9]        = 0;
         gdb_regs[GDB_R10]       = 0;
diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c
index 33019ddb56b4..a78ecad0c900 100644
--- a/arch/x86/kernel/kvm.c
+++ b/arch/x86/kernel/kvm.c
@@ -27,6 +27,7 @@
 #include <linux/mm.h>
 #include <linux/highmem.h>
 #include <linux/hardirq.h>
+#include <asm/timer.h>
 
 #define MMU_QUEUE_SIZE 1024
 
@@ -195,7 +196,7 @@ static void kvm_leave_lazy_mmu(void)
         struct kvm_para_state *state = kvm_para_state();
 
         mmu_queue_flush(state);
-        paravirt_leave_lazy(paravirt_get_lazy_mode());
+        paravirt_leave_lazy_mmu();
         state->mode = paravirt_get_lazy_mode();
 }
 
@@ -230,6 +231,9 @@ static void paravirt_ops_setup(void)
                 pv_mmu_ops.lazy_mode.enter = kvm_enter_lazy_mmu;
                 pv_mmu_ops.lazy_mode.leave = kvm_leave_lazy_mmu;
         }
+#ifdef CONFIG_X86_IO_APIC
+        no_timer_check = 1;
+#endif
 }
 
 void __init kvm_guest_init(void)
diff --git a/arch/x86/kernel/microcode_amd.c b/arch/x86/kernel/microcode_amd.c
index 453b5795a5c6..366baa179913 100644
--- a/arch/x86/kernel/microcode_amd.c
+++ b/arch/x86/kernel/microcode_amd.c
@@ -13,25 +13,13 @@
  *  Licensed under the terms of the GNU General Public
  *  License version 2. See file COPYING for details.
  */
-#include <linux/platform_device.h>
-#include <linux/capability.h>
-#include <linux/miscdevice.h>
 #include <linux/firmware.h>
-#include <linux/spinlock.h>
-#include <linux/cpumask.h>
 #include <linux/pci_ids.h>
 #include <linux/uaccess.h>
 #include <linux/vmalloc.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
-#include <linux/mutex.h>
-#include <linux/sched.h>
-#include <linux/init.h>
-#include <linux/slab.h>
-#include <linux/cpu.h>
 #include <linux/pci.h>
-#include <linux/fs.h>
-#include <linux/mm.h>
 
 #include <asm/microcode.h>
 #include <asm/processor.h>
@@ -79,9 +67,6 @@ struct microcode_amd {
 #define UCODE_CONTAINER_SECTION_HDR     8
 #define UCODE_CONTAINER_HEADER_SIZE     12
 
-/* serialize access to the physical write */
-static DEFINE_SPINLOCK(microcode_update_lock);
-
 static struct equiv_cpu_entry *equiv_cpu_table;
 
 static int collect_cpu_info_amd(int cpu, struct cpu_signature *csig)
@@ -144,9 +129,8 @@ static int get_matching_microcode(int cpu, void *mc, int rev)
         return 1;
 }
 
-static void apply_microcode_amd(int cpu)
+static int apply_microcode_amd(int cpu)
 {
-        unsigned long flags;
         u32 rev, dummy;
         int cpu_num = raw_smp_processor_id();
         struct ucode_cpu_info *uci = ucode_cpu_info + cpu_num;
@@ -156,25 +140,25 @@ static void apply_microcode_amd(int cpu)
         BUG_ON(cpu_num != cpu);
 
         if (mc_amd == NULL)
-                return;
+                return 0;
 
-        spin_lock_irqsave(&microcode_update_lock, flags);
         wrmsrl(MSR_AMD64_PATCH_LOADER, (u64)(long)&mc_amd->hdr.data_code);
         /* get patch id after patching */
         rdmsr(MSR_AMD64_PATCH_LEVEL, rev, dummy);
-        spin_unlock_irqrestore(&microcode_update_lock, flags);
 
         /* check current patch id and patch's id for match */
         if (rev != mc_amd->hdr.patch_id) {
                 printk(KERN_ERR "microcode: CPU%d: update failed "
                        "(for patch_level=0x%x)\n", cpu, mc_amd->hdr.patch_id);
-                return;
+                return -1;
         }
 
         printk(KERN_INFO "microcode: CPU%d: updated (new patch_level=0x%x)\n",
                cpu, rev);
 
         uci->cpu_sig.rev = rev;
+
+        return 0;
 }
 
 static int get_ucode_data(void *to, const u8 *from, size_t n)
@@ -257,13 +241,12 @@ static int install_equiv_cpu_table(const u8 *buf)
 
 static void free_equiv_cpu_table(void)
 {
-        if (equiv_cpu_table) {
-                vfree(equiv_cpu_table);
-                equiv_cpu_table = NULL;
-        }
+        vfree(equiv_cpu_table);
+        equiv_cpu_table = NULL;
 }
 
-static int generic_load_microcode(int cpu, const u8 *data, size_t size)
+static enum ucode_state
+generic_load_microcode(int cpu, const u8 *data, size_t size)
 {
         struct ucode_cpu_info *uci = ucode_cpu_info + cpu;
         const u8 *ucode_ptr = data;
@@ -272,12 +255,13 @@ static int generic_load_microcode(int cpu, const u8 *data, size_t size)
         int new_rev = uci->cpu_sig.rev;
         unsigned int leftover;
         unsigned long offset;
+        enum ucode_state state = UCODE_OK;
 
         offset = install_equiv_cpu_table(ucode_ptr);
         if (!offset) {
                 printk(KERN_ERR "microcode: failed to create "
                        "equivalent cpu table\n");
-                return -EINVAL;
+                return UCODE_ERROR;
         }
 
         ucode_ptr += offset;
@@ -293,8 +277,7 @@ static int generic_load_microcode(int cpu, const u8 *data, size_t size)
 
                 mc_header = (struct microcode_header_amd *)mc;
                 if (get_matching_microcode(cpu, mc, new_rev)) {
-                        if (new_mc)
-                                vfree(new_mc);
+                        vfree(new_mc);
                         new_rev = mc_header->patch_id;
                         new_mc  = mc;
                 } else
@@ -306,34 +289,32 @@ static int generic_load_microcode(int cpu, const u8 *data, size_t size)
 
         if (new_mc) {
                 if (!leftover) {
-                        if (uci->mc)
-                                vfree(uci->mc);
+                        vfree(uci->mc);
                         uci->mc = new_mc;
                         pr_debug("microcode: CPU%d found a matching microcode "
                                  "update with version 0x%x (current=0x%x)\n",
                                  cpu, new_rev, uci->cpu_sig.rev);
-                } else
+                } else {
                         vfree(new_mc);
-        }
+                        state = UCODE_ERROR;
+                }
+        } else
+                state = UCODE_NFOUND;
 
         free_equiv_cpu_table();
 
-        return (int)leftover;
+        return state;
 }
 
-static int request_microcode_fw(int cpu, struct device *device)
+static enum ucode_state request_microcode_fw(int cpu, struct device *device)
 {
         const char *fw_name = "amd-ucode/microcode_amd.bin";
         const struct firmware *firmware;
-        int ret;
-
-        /* We should bind the task to the CPU */
-        BUG_ON(cpu != raw_smp_processor_id());
+        enum ucode_state ret;
 
-        ret = request_firmware(&firmware, fw_name, device);
-        if (ret) {
+        if (request_firmware(&firmware, fw_name, device)) {
                 printk(KERN_ERR "microcode: failed to load file %s\n", fw_name);
-                return ret;
+                return UCODE_NFOUND;
         }
 
         ret = generic_load_microcode(cpu, firmware->data, firmware->size);
@@ -343,11 +324,12 @@ static int request_microcode_fw(int cpu, struct device *device)
         return ret;
 }
 
-static int request_microcode_user(int cpu, const void __user *buf, size_t size)
+static enum ucode_state
+request_microcode_user(int cpu, const void __user *buf, size_t size)
 {
         printk(KERN_INFO "microcode: AMD microcode update via "
                "/dev/cpu/microcode not supported\n");
-        return -1;
+        return UCODE_ERROR;
 }
 
 static void microcode_fini_cpu_amd(int cpu)
diff --git a/arch/x86/kernel/microcode_core.c b/arch/x86/kernel/microcode_core.c
index 98c470c069d1..9c4461501fcb 100644
--- a/arch/x86/kernel/microcode_core.c
+++ b/arch/x86/kernel/microcode_core.c
@@ -71,27 +71,18 @@
  *              Thanks to Stuart Swales for pointing out this bug.
  */
 #include <linux/platform_device.h>
-#include <linux/capability.h>
 #include <linux/miscdevice.h>
-#include <linux/firmware.h>
+#include <linux/capability.h>
 #include <linux/smp_lock.h>
-#include <linux/spinlock.h>
-#include <linux/cpumask.h>
-#include <linux/uaccess.h>
-#include <linux/vmalloc.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/mutex.h>
-#include <linux/sched.h>
-#include <linux/init.h>
-#include <linux/slab.h>
 #include <linux/cpu.h>
 #include <linux/fs.h>
 #include <linux/mm.h>
 
 #include <asm/microcode.h>
 #include <asm/processor.h>
-#include <asm/msr.h>
 
 MODULE_DESCRIPTION("Microcode Update Driver");
 MODULE_AUTHOR("Tigran Aivazian <tigran@aivazian.fsnet.co.uk>");
@@ -101,36 +92,110 @@ MODULE_LICENSE("GPL");
 
 static struct microcode_ops     *microcode_ops;
 
-/* no concurrent ->write()s are allowed on /dev/cpu/microcode */
+/*
+ * Synchronization.
+ *
+ * All non cpu-hotplug-callback call sites use:
+ *
+ * - microcode_mutex to synchronize with each other;
+ * - get/put_online_cpus() to synchronize with
+ *   the cpu-hotplug-callback call sites.
+ *
+ * We guarantee that only a single cpu is being
+ * updated at any particular moment of time.
+ */
 static DEFINE_MUTEX(microcode_mutex);
 
 struct ucode_cpu_info           ucode_cpu_info[NR_CPUS];
 EXPORT_SYMBOL_GPL(ucode_cpu_info);
 
+/*
+ * Operations that are run on a target cpu:
+ */
+
+struct cpu_info_ctx {
+        struct cpu_signature    *cpu_sig;
+        int                     err;
+};
+
+static void collect_cpu_info_local(void *arg)
+{
+        struct cpu_info_ctx *ctx = arg;
+
+        ctx->err = microcode_ops->collect_cpu_info(smp_processor_id(),
+                                                   ctx->cpu_sig);
+}
+
+static int collect_cpu_info_on_target(int cpu, struct cpu_signature *cpu_sig)
+{
+        struct cpu_info_ctx ctx = { .cpu_sig = cpu_sig, .err = 0 };
+        int ret;
+
+        ret = smp_call_function_single(cpu, collect_cpu_info_local, &ctx, 1);
+        if (!ret)
+                ret = ctx.err;
+
+        return ret;
+}
+
+static int collect_cpu_info(int cpu)
+{
+        struct ucode_cpu_info *uci = ucode_cpu_info + cpu;
+        int ret;
+
+        memset(uci, 0, sizeof(*uci));
+
+        ret = collect_cpu_info_on_target(cpu, &uci->cpu_sig);
+        if (!ret)
+                uci->valid = 1;
+
+        return ret;
+}
+
+struct apply_microcode_ctx {
+        int err;
+};
+
+static void apply_microcode_local(void *arg)
+{
+        struct apply_microcode_ctx *ctx = arg;
+
+        ctx->err = microcode_ops->apply_microcode(smp_processor_id());
+}
+
+static int apply_microcode_on_target(int cpu)
+{
+        struct apply_microcode_ctx ctx = { .err = 0 };
+        int ret;
+
+        ret = smp_call_function_single(cpu, apply_microcode_local, &ctx, 1);
+        if (!ret)
+                ret = ctx.err;
+
+        return ret;
+}
+
 #ifdef CONFIG_MICROCODE_OLD_INTERFACE
 static int do_microcode_update(const void __user *buf, size_t size)
 {
-        cpumask_t old;
         int error = 0;
         int cpu;
 
-        old = current->cpus_allowed;
-
         for_each_online_cpu(cpu) {
                 struct ucode_cpu_info *uci = ucode_cpu_info + cpu;
+                enum ucode_state ustate;
 
                 if (!uci->valid)
                         continue;
 
-                set_cpus_allowed_ptr(current, &cpumask_of_cpu(cpu));
-                error = microcode_ops->request_microcode_user(cpu, buf, size);
-                if (error < 0)
-                        goto out;
-                if (!error)
-                        microcode_ops->apply_microcode(cpu);
+                ustate = microcode_ops->request_microcode_user(cpu, buf, size);
+                if (ustate == UCODE_ERROR) {
+                        error = -1;
+                        break;
+                } else if (ustate == UCODE_OK)
+                        apply_microcode_on_target(cpu);
         }
-out:
-        set_cpus_allowed_ptr(current, &old);
+
         return error;
 }
 
@@ -143,19 +208,17 @@ static int microcode_open(struct inode *unused1, struct file *unused2)
 static ssize_t microcode_write(struct file *file, const char __user *buf,
                                size_t len, loff_t *ppos)
 {
-        ssize_t ret;
+        ssize_t ret = -EINVAL;
 
         if ((len >> PAGE_SHIFT) > num_physpages) {
-                printk(KERN_ERR "microcode: too much data (max %ld pages)\n",
-                       num_physpages);
-                return -EINVAL;
+                pr_err("microcode: too much data (max %ld pages)\n", num_physpages);
+                return ret;
         }
 
         get_online_cpus();
         mutex_lock(&microcode_mutex);
 
-        ret = do_microcode_update(buf, len);
-        if (!ret)
+        if (do_microcode_update(buf, len) == 0)
                 ret = (ssize_t)len;
 
         mutex_unlock(&microcode_mutex);
@@ -165,15 +228,15 @@ static ssize_t microcode_write(struct file *file, const char __user *buf,
 }
 
 static const struct file_operations microcode_fops = {
-        .owner          = THIS_MODULE,
-        .write          = microcode_write,
-        .open           = microcode_open,
+        .owner                  = THIS_MODULE,
+        .write                  = microcode_write,
+        .open                   = microcode_open,
 };
 
 static struct miscdevice microcode_dev = {
-        .minor          = MICROCODE_MINOR,
-        .name           = "microcode",
-        .fops           = &microcode_fops,
+        .minor                  = MICROCODE_MINOR,
+        .name                   = "microcode",
+        .fops                   = &microcode_fops,
 };
 
 static int __init microcode_dev_init(void)
@@ -182,9 +245,7 @@ static int __init microcode_dev_init(void)
 
         error = misc_register(&microcode_dev);
         if (error) {
-                printk(KERN_ERR
-                        "microcode: can't misc_register on minor=%d\n",
-                        MICROCODE_MINOR);
+                pr_err("microcode: can't misc_register on minor=%d\n", MICROCODE_MINOR);
                 return error;
         }
 
@@ -205,42 +266,51 @@ MODULE_ALIAS_MISCDEV(MICROCODE_MINOR);
 /* fake device for request_firmware */
 static struct platform_device   *microcode_pdev;
 
-static long reload_for_cpu(void *unused)
+static int reload_for_cpu(int cpu)
 {
-        struct ucode_cpu_info *uci = ucode_cpu_info + smp_processor_id();
+        struct ucode_cpu_info *uci = ucode_cpu_info + cpu;
         int err = 0;
 
         mutex_lock(&microcode_mutex);
         if (uci->valid) {
-                err = microcode_ops->request_microcode_fw(smp_processor_id(),
-                                                          &microcode_pdev->dev);
-                if (!err)
-                        microcode_ops->apply_microcode(smp_processor_id());
+                enum ucode_state ustate;
+
+                ustate = microcode_ops->request_microcode_fw(cpu, &microcode_pdev->dev);
+                if (ustate == UCODE_OK)
+                        apply_microcode_on_target(cpu);
+                else
+                        if (ustate == UCODE_ERROR)
+                                err = -EINVAL;
         }
         mutex_unlock(&microcode_mutex);
+
         return err;
 }
 
 static ssize_t reload_store(struct sys_device *dev,
                             struct sysdev_attribute *attr,
-                            const char *buf, size_t sz)
+                            const char *buf, size_t size)
 {
-        char *end;
-        unsigned long val = simple_strtoul(buf, &end, 0);
-        int err = 0;
+        unsigned long val;
         int cpu = dev->id;
+        int ret = 0;
+        char *end;
 
+        val = simple_strtoul(buf, &end, 0);
         if (end == buf)
                 return -EINVAL;
+
         if (val == 1) {
                 get_online_cpus();
                 if (cpu_online(cpu))
-                        err = work_on_cpu(cpu, reload_for_cpu, NULL);
+                        ret = reload_for_cpu(cpu);
                 put_online_cpus();
         }
-        if (err)
-                return err;
-        return sz;
+
+        if (!ret)
+                ret = size;
+
+        return ret;
 }
 
 static ssize_t version_show(struct sys_device *dev,
@@ -271,11 +341,11 @@ static struct attribute *mc_default_attrs[] = {
 };
 
 static struct attribute_group mc_attr_group = {
-        .attrs          = mc_default_attrs,
-        .name           = "microcode",
+        .attrs                  = mc_default_attrs,
+        .name                   = "microcode",
 };
 
-static void __microcode_fini_cpu(int cpu)
+static void microcode_fini_cpu(int cpu)
 {
         struct ucode_cpu_info *uci = ucode_cpu_info + cpu;
 
@@ -283,103 +353,68 @@ static void __microcode_fini_cpu(int cpu)
         uci->valid = 0;
 }
 
-static void microcode_fini_cpu(int cpu)
-{
-        mutex_lock(&microcode_mutex);
-        __microcode_fini_cpu(cpu);
-        mutex_unlock(&microcode_mutex);
-}
-
-static void collect_cpu_info(int cpu)
+static enum ucode_state microcode_resume_cpu(int cpu)
 {
         struct ucode_cpu_info *uci = ucode_cpu_info + cpu;
 
-        memset(uci, 0, sizeof(*uci));
-        if (!microcode_ops->collect_cpu_info(cpu, &uci->cpu_sig))
-                uci->valid = 1;
+        if (!uci->mc)
+                return UCODE_NFOUND;
+
+        pr_debug("microcode: CPU%d updated upon resume\n", cpu);
+        apply_microcode_on_target(cpu);
+
+        return UCODE_OK;
 }
 
-static int microcode_resume_cpu(int cpu)
+static enum ucode_state microcode_init_cpu(int cpu)
 {
-        struct ucode_cpu_info *uci = ucode_cpu_info + cpu;
-        struct cpu_signature nsig;
+        enum ucode_state ustate;
 
-        pr_debug("microcode: CPU%d resumed\n", cpu);
+        if (collect_cpu_info(cpu))
+                return UCODE_ERROR;
 
-        if (!uci->mc)
-                return 1;
+        /* --dimm. Trigger a delayed update? */
+        if (system_state != SYSTEM_RUNNING)
+                return UCODE_NFOUND;
 
-        /*
-         * Let's verify that the 'cached' ucode does belong
-         * to this cpu (a bit of paranoia):
-         */
-        if (microcode_ops->collect_cpu_info(cpu, &nsig)) {
-                __microcode_fini_cpu(cpu);
-                printk(KERN_ERR "failed to collect_cpu_info for resuming cpu #%d\n",
-                                cpu);
-                return -1;
-        }
+        ustate = microcode_ops->request_microcode_fw(cpu, &microcode_pdev->dev);
 
-        if ((nsig.sig != uci->cpu_sig.sig) || (nsig.pf != uci->cpu_sig.pf)) {
-                __microcode_fini_cpu(cpu);
-                printk(KERN_ERR "cached ucode doesn't match the resuming cpu #%d\n",
-                                cpu);
-                /* Should we look for a new ucode here? */
-                return 1;
+        if (ustate == UCODE_OK) {
+                pr_debug("microcode: CPU%d updated upon init\n", cpu);
+                apply_microcode_on_target(cpu);
         }
 
-        return 0;
+        return ustate;
 }
 
-static long microcode_update_cpu(void *unused)
+static enum ucode_state microcode_update_cpu(int cpu)
 {
-        struct ucode_cpu_info *uci = ucode_cpu_info + smp_processor_id();
-        int err = 0;
+        struct ucode_cpu_info *uci = ucode_cpu_info + cpu;
+        enum ucode_state ustate;
 
-        /*
-         * Check if the system resume is in progress (uci->valid != NULL),
-         * otherwise just request a firmware:
-         */
-        if (uci->valid) {
-                err = microcode_resume_cpu(smp_processor_id());
-        } else {
-                collect_cpu_info(smp_processor_id());
-                if (uci->valid && system_state == SYSTEM_RUNNING)
-                        err = microcode_ops->request_microcode_fw(
-                                        smp_processor_id(),
-                                        &microcode_pdev->dev);
-        }
-        if (!err)
-                microcode_ops->apply_microcode(smp_processor_id());
-        return err;
-}
+        if (uci->valid)
+                ustate = microcode_resume_cpu(cpu);
+        else
+                ustate = microcode_init_cpu(cpu);
 
-static int microcode_init_cpu(int cpu)
-{
-        int err;
-        mutex_lock(&microcode_mutex);
-        err = work_on_cpu(cpu, microcode_update_cpu, NULL);
-        mutex_unlock(&microcode_mutex);
-
-        return err;
+        return ustate;
 }
 
 static int mc_sysdev_add(struct sys_device *sys_dev)
 {
         int err, cpu = sys_dev->id;
-        struct ucode_cpu_info *uci = ucode_cpu_info + cpu;
 
         if (!cpu_online(cpu))
                 return 0;
 
         pr_debug("microcode: CPU%d added\n", cpu);
-        memset(uci, 0, sizeof(*uci));
 
         err = sysfs_create_group(&sys_dev->kobj, &mc_attr_group);
         if (err)
                 return err;
 
-        err = microcode_init_cpu(cpu);
+        if (microcode_init_cpu(cpu) == UCODE_ERROR)
+                err = -EINVAL;
 
         return err;
 }
@@ -400,19 +435,30 @@ static int mc_sysdev_remove(struct sys_device *sys_dev)
 static int mc_sysdev_resume(struct sys_device *dev)
 {
         int cpu = dev->id;
+        struct ucode_cpu_info *uci = ucode_cpu_info + cpu;
 
         if (!cpu_online(cpu))
                 return 0;
 
-        /* only CPU 0 will apply ucode here */
-        microcode_update_cpu(NULL);
+        /*
+         * All non-bootup cpus are still disabled,
+         * so only CPU 0 will apply ucode here.
+         *
+         * Moreover, there can be no concurrent
+         * updates from any other places at this point.
+         */
+        WARN_ON(cpu != 0);
+
+        if (uci->valid && uci->mc)
+                microcode_ops->apply_microcode(cpu);
+
         return 0;
 }
 
 static struct sysdev_driver mc_sysdev_driver = {
-        .add            = mc_sysdev_add,
-        .remove         = mc_sysdev_remove,
-        .resume         = mc_sysdev_resume,
+        .add                    = mc_sysdev_add,
+        .remove                 = mc_sysdev_remove,
+        .resume                 = mc_sysdev_resume,
 };
 
 static __cpuinit int
@@ -425,15 +471,12 @@ mc_cpu_callback(struct notifier_block *nb, unsigned long action, void *hcpu)
         switch (action) {
         case CPU_ONLINE:
         case CPU_ONLINE_FROZEN:
-                if (microcode_init_cpu(cpu))
-                        printk(KERN_ERR "microcode: failed to init CPU%d\n",
-                               cpu);
+                microcode_update_cpu(cpu);
         case CPU_DOWN_FAILED:
         case CPU_DOWN_FAILED_FROZEN:
                 pr_debug("microcode: CPU%d added\n", cpu);
                 if (sysfs_create_group(&sys_dev->kobj, &mc_attr_group))
-                        printk(KERN_ERR "microcode: Failed to create the sysfs "
-                                "group for CPU%d\n", cpu);
+                        pr_err("microcode: Failed to create group for CPU%d\n", cpu);
                 break;
         case CPU_DOWN_PREPARE:
         case CPU_DOWN_PREPARE_FROZEN:
@@ -465,13 +508,10 @@ static int __init microcode_init(void)
                 microcode_ops = init_amd_microcode();
 
         if (!microcode_ops) {
-                printk(KERN_ERR "microcode: no support for this CPU vendor\n");
+                pr_err("microcode: no support for this CPU vendor\n");
                 return -ENODEV;
         }
 
-        error = microcode_dev_init();
-        if (error)
-                return error;
         microcode_pdev = platform_device_register_simple("microcode", -1,
                                                          NULL, 0);
         if (IS_ERR(microcode_pdev)) {
@@ -480,23 +520,31 @@ static int __init microcode_init(void)
         }
 
         get_online_cpus();
+        mutex_lock(&microcode_mutex);
+
         error = sysdev_driver_register(&cpu_sysdev_class, &mc_sysdev_driver);
+
+        mutex_unlock(&microcode_mutex);
         put_online_cpus();
+
         if (error) {
-                microcode_dev_exit();
                 platform_device_unregister(microcode_pdev);
                 return error;
         }
 
+        error = microcode_dev_init();
+        if (error)
+                return error;
+
         register_hotcpu_notifier(&mc_cpu_notifier);
 
-        printk(KERN_INFO
-               "Microcode Update Driver: v" MICROCODE_VERSION
+        pr_info("Microcode Update Driver: v" MICROCODE_VERSION
                " <tigran@aivazian.fsnet.co.uk>,"
                " Peter Oruba\n");
 
         return 0;
 }
+module_init(microcode_init);
 
 static void __exit microcode_exit(void)
 {
@@ -505,16 +553,17 @@ static void __exit microcode_exit(void)
         unregister_hotcpu_notifier(&mc_cpu_notifier);
 
         get_online_cpus();
+        mutex_lock(&microcode_mutex);
+
         sysdev_driver_unregister(&cpu_sysdev_class, &mc_sysdev_driver);
+
+        mutex_unlock(&microcode_mutex);
         put_online_cpus();
 
         platform_device_unregister(microcode_pdev);
 
         microcode_ops = NULL;
 
-        printk(KERN_INFO
-               "Microcode Update Driver: v" MICROCODE_VERSION " removed.\n");
+        pr_info("Microcode Update Driver: v" MICROCODE_VERSION " removed.\n");
 }
-
-module_init(microcode_init);
 module_exit(microcode_exit);
diff --git a/arch/x86/kernel/microcode_intel.c b/arch/x86/kernel/microcode_intel.c
index 149b9ec7c1ab..0d334ddd0a96 100644
--- a/arch/x86/kernel/microcode_intel.c
+++ b/arch/x86/kernel/microcode_intel.c
@@ -70,24 +70,11 @@
  *              Fix sigmatch() macro to handle old CPUs with pf == 0.
  *              Thanks to Stuart Swales for pointing out this bug.
  */
-#include <linux/platform_device.h>
-#include <linux/capability.h>
-#include <linux/miscdevice.h>
 #include <linux/firmware.h>
-#include <linux/smp_lock.h>
-#include <linux/spinlock.h>
-#include <linux/cpumask.h>
 #include <linux/uaccess.h>
-#include <linux/vmalloc.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
-#include <linux/mutex.h>
-#include <linux/sched.h>
-#include <linux/init.h>
-#include <linux/slab.h>
-#include <linux/cpu.h>
-#include <linux/fs.h>
-#include <linux/mm.h>
+#include <linux/vmalloc.h>
 
 #include <asm/microcode.h>
 #include <asm/processor.h>
@@ -150,13 +137,9 @@ struct extended_sigtable {
 
 #define exttable_size(et) ((et)->count * EXT_SIGNATURE_SIZE + EXT_HEADER_SIZE)
 
-/* serialize access to the physical write to MSR 0x79 */
-static DEFINE_SPINLOCK(microcode_update_lock);
-
 static int collect_cpu_info(int cpu_num, struct cpu_signature *csig)
 {
         struct cpuinfo_x86 *c = &cpu_data(cpu_num);
-        unsigned long flags;
         unsigned int val[2];
 
         memset(csig, 0, sizeof(*csig));
@@ -176,18 +159,14 @@ static int collect_cpu_info(int cpu_num, struct cpu_signature *csig)
                 csig->pf = 1 << ((val[1] >> 18) & 7);
         }
 
-        /* serialize access to the physical write to MSR 0x79 */
-        spin_lock_irqsave(&microcode_update_lock, flags);
-
         wrmsr(MSR_IA32_UCODE_REV, 0, 0);
         /* see notes above for revision 1.07.  Apparent chip bug */
         sync_core();
         /* get the current revision from MSR 0x8B */
         rdmsr(MSR_IA32_UCODE_REV, val[0], csig->rev);
-        spin_unlock_irqrestore(&microcode_update_lock, flags);
 
-        pr_debug("microcode: collect_cpu_info : sig=0x%x, pf=0x%x, rev=0x%x\n",
-                        csig->sig, csig->pf, csig->rev);
+        printk(KERN_INFO "microcode: CPU%d sig=0x%x, pf=0x%x, revision=0x%x\n",
+                        cpu_num, csig->sig, csig->pf, csig->rev);
 
         return 0;
 }
@@ -318,11 +297,10 @@ get_matching_microcode(struct cpu_signature *cpu_sig, void *mc, int rev)
         return 0;
 }
 
-static void apply_microcode(int cpu)
+static int apply_microcode(int cpu)
 {
         struct microcode_intel *mc_intel;
         struct ucode_cpu_info *uci;
-        unsigned long flags;
         unsigned int val[2];
         int cpu_num;
 
@@ -334,10 +312,7 @@ static void apply_microcode(int cpu)
         BUG_ON(cpu_num != cpu);
 
         if (mc_intel == NULL)
-                return;
-
-        /* serialize access to the physical write to MSR 0x79 */
-        spin_lock_irqsave(&microcode_update_lock, flags);
+                return 0;
 
         /* write microcode via MSR 0x79 */
         wrmsr(MSR_IA32_UCODE_WRITE,
@@ -351,30 +326,32 @@ static void apply_microcode(int cpu)
         /* get the current revision from MSR 0x8B */
         rdmsr(MSR_IA32_UCODE_REV, val[0], val[1]);
 
-        spin_unlock_irqrestore(&microcode_update_lock, flags);
         if (val[1] != mc_intel->hdr.rev) {
-                printk(KERN_ERR "microcode: CPU%d update from revision "
-                                "0x%x to 0x%x failed\n",
-                        cpu_num, uci->cpu_sig.rev, val[1]);
-                return;
+                printk(KERN_ERR "microcode: CPU%d update "
+                                "to revision 0x%x failed\n",
+                        cpu_num, mc_intel->hdr.rev);
+                return -1;
         }
-        printk(KERN_INFO "microcode: CPU%d updated from revision "
-                         "0x%x to 0x%x, date = %04x-%02x-%02x \n",
-                cpu_num, uci->cpu_sig.rev, val[1],
+        printk(KERN_INFO "microcode: CPU%d updated to revision "
+                         "0x%x, date = %04x-%02x-%02x \n",
+                cpu_num, val[1],
                 mc_intel->hdr.date & 0xffff,
                 mc_intel->hdr.date >> 24,
                 (mc_intel->hdr.date >> 16) & 0xff);
 
         uci->cpu_sig.rev = val[1];
+
+        return 0;
 }
 
-static int generic_load_microcode(int cpu, void *data, size_t size,
-                int (*get_ucode_data)(void *, const void *, size_t))
+static enum ucode_state generic_load_microcode(int cpu, void *data, size_t size,
+                                int (*get_ucode_data)(void *, const void *, size_t))
 {
         struct ucode_cpu_info *uci = ucode_cpu_info + cpu;
         u8 *ucode_ptr = data, *new_mc = NULL, *mc;
         int new_rev = uci->cpu_sig.rev;
         unsigned int leftover = size;
+        enum ucode_state state = UCODE_OK;
 
         while (leftover) {
                 struct microcode_header_intel mc_header;
@@ -412,11 +389,15 @@ static int generic_load_microcode(int cpu, void *data, size_t size,
                 leftover  -= mc_size;
         }
 
-        if (!new_mc)
+        if (leftover) {
+                if (new_mc)
+                        vfree(new_mc);
+                state = UCODE_ERROR;
                 goto out;
+        }
 
-        if (leftover) {
-                vfree(new_mc);
+        if (!new_mc) {
+                state = UCODE_NFOUND;
                 goto out;
         }
 
@@ -427,9 +408,8 @@ static int generic_load_microcode(int cpu, void *data, size_t size,
         pr_debug("microcode: CPU%d found a matching microcode update with"
                  " version 0x%x (current=0x%x)\n",
                         cpu, new_rev, uci->cpu_sig.rev);
-
- out:
-        return (int)leftover;
+out:
+        return state;
 }
 
 static int get_ucode_fw(void *to, const void *from, size_t n)
@@ -438,21 +418,19 @@ static int get_ucode_fw(void *to, const void *from, size_t n)
         return 0;
 }
 
-static int request_microcode_fw(int cpu, struct device *device)
+static enum ucode_state request_microcode_fw(int cpu, struct device *device)
 {
         char name[30];
         struct cpuinfo_x86 *c = &cpu_data(cpu);
         const struct firmware *firmware;
-        int ret;
+        enum ucode_state ret;
 
-        /* We should bind the task to the CPU */
-        BUG_ON(cpu != raw_smp_processor_id());
         sprintf(name, "intel-ucode/%02x-%02x-%02x",
                 c->x86, c->x86_model, c->x86_mask);
-        ret = request_firmware(&firmware, name, device);
-        if (ret) {
+
+        if (request_firmware(&firmware, name, device)) {
                 pr_debug("microcode: data file %s load failed\n", name);
-                return ret;
+                return UCODE_NFOUND;
         }
 
         ret = generic_load_microcode(cpu, (void *)firmware->data,
@@ -468,11 +446,9 @@ static int get_ucode_user(void *to, const void *from, size_t n)
         return copy_from_user(to, from, n);
 }
 
-static int request_microcode_user(int cpu, const void __user *buf, size_t size)
+static enum ucode_state
+request_microcode_user(int cpu, const void __user *buf, size_t size)
 {
-        /* We should bind the task to the CPU */
-        BUG_ON(cpu != raw_smp_processor_id());
-
         return generic_load_microcode(cpu, (void *)buf, size, &get_ucode_user);
 }
 
diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c
index 9faf43bea336..70ec9b951d76 100644
--- a/arch/x86/kernel/paravirt.c
+++ b/arch/x86/kernel/paravirt.c
@@ -248,18 +248,16 @@ static DEFINE_PER_CPU(enum paravirt_lazy_mode, paravirt_lazy_mode) = PARAVIRT_LA
 
 static inline void enter_lazy(enum paravirt_lazy_mode mode)
 {
-        BUG_ON(__get_cpu_var(paravirt_lazy_mode) != PARAVIRT_LAZY_NONE);
-        BUG_ON(preemptible());
+        BUG_ON(percpu_read(paravirt_lazy_mode) != PARAVIRT_LAZY_NONE);
 
-        __get_cpu_var(paravirt_lazy_mode) = mode;
+        percpu_write(paravirt_lazy_mode, mode);
 }
 
-void paravirt_leave_lazy(enum paravirt_lazy_mode mode)
+static void leave_lazy(enum paravirt_lazy_mode mode)
 {
-        BUG_ON(__get_cpu_var(paravirt_lazy_mode) != mode);
-        BUG_ON(preemptible());
+        BUG_ON(percpu_read(paravirt_lazy_mode) != mode);
 
-        __get_cpu_var(paravirt_lazy_mode) = PARAVIRT_LAZY_NONE;
+        percpu_write(paravirt_lazy_mode, PARAVIRT_LAZY_NONE);
 }
 
 void paravirt_enter_lazy_mmu(void)
@@ -269,22 +267,36 @@ void paravirt_enter_lazy_mmu(void)
 
 void paravirt_leave_lazy_mmu(void)
 {
-        paravirt_leave_lazy(PARAVIRT_LAZY_MMU);
+        leave_lazy(PARAVIRT_LAZY_MMU);
 }
 
-void paravirt_enter_lazy_cpu(void)
+void paravirt_start_context_switch(struct task_struct *prev)
 {
+        BUG_ON(preemptible());
+
+        if (percpu_read(paravirt_lazy_mode) == PARAVIRT_LAZY_MMU) {
+                arch_leave_lazy_mmu_mode();
+                set_ti_thread_flag(task_thread_info(prev), TIF_LAZY_MMU_UPDATES);
+        }
         enter_lazy(PARAVIRT_LAZY_CPU);
 }
 
-void paravirt_leave_lazy_cpu(void)
+void paravirt_end_context_switch(struct task_struct *next)
 {
-        paravirt_leave_lazy(PARAVIRT_LAZY_CPU);
+        BUG_ON(preemptible());
+
+        leave_lazy(PARAVIRT_LAZY_CPU);
+
+        if (test_and_clear_ti_thread_flag(task_thread_info(next), TIF_LAZY_MMU_UPDATES))
+                arch_enter_lazy_mmu_mode();
 }
 
 enum paravirt_lazy_mode paravirt_get_lazy_mode(void)
 {
-        return __get_cpu_var(paravirt_lazy_mode);
+        if (in_interrupt())
+                return PARAVIRT_LAZY_NONE;
+
+        return percpu_read(paravirt_lazy_mode);
 }
 
 void arch_flush_lazy_mmu_mode(void)
@@ -292,7 +304,6 @@ void arch_flush_lazy_mmu_mode(void)
         preempt_disable();
 
         if (paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU) {
-                WARN_ON(preempt_count() == 1);
                 arch_leave_lazy_mmu_mode();
                 arch_enter_lazy_mmu_mode();
         }
@@ -300,19 +311,6 @@ void arch_flush_lazy_mmu_mode(void)
         preempt_enable();
 }
 
-void arch_flush_lazy_cpu_mode(void)
-{
-        preempt_disable();
-
-        if (paravirt_get_lazy_mode() == PARAVIRT_LAZY_CPU) {
-                WARN_ON(preempt_count() == 1);
-                arch_leave_lazy_cpu_mode();
-                arch_enter_lazy_cpu_mode();
-        }
-
-        preempt_enable();
-}
-
 struct pv_info pv_info = {
         .name = "bare hardware",
         .paravirt_enabled = 0,
@@ -404,10 +402,8 @@ struct pv_cpu_ops pv_cpu_ops = {
         .set_iopl_mask = native_set_iopl_mask,
         .io_delay = native_io_delay,
 
-        .lazy_mode = {
-                .enter = paravirt_nop,
-                .leave = paravirt_nop,
-        },
+        .start_context_switch = paravirt_nop,
+        .end_context_switch = paravirt_nop,
 };
 
 struct pv_apic_ops pv_apic_ops = {
diff --git a/arch/x86/kernel/pci-calgary_64.c b/arch/x86/kernel/pci-calgary_64.c
index 755c21e906f3..971a3bec47a8 100644
--- a/arch/x86/kernel/pci-calgary_64.c
+++ b/arch/x86/kernel/pci-calgary_64.c
@@ -186,37 +186,6 @@ static struct cal_chipset_ops calioc2_chip_ops = {
 
 static struct calgary_bus_info bus_info[MAX_PHB_BUS_NUM] = { { NULL, 0, 0 }, };
 
-/* enable this to stress test the chip's TCE cache */
-#ifdef CONFIG_IOMMU_DEBUG
-static int debugging = 1;
-
-static inline unsigned long verify_bit_range(unsigned long* bitmap,
-        int expected, unsigned long start, unsigned long end)
-{
-        unsigned long idx = start;
-
-        BUG_ON(start >= end);
-
-        while (idx < end) {
-                if (!!test_bit(idx, bitmap) != expected)
-                        return idx;
-                ++idx;
-        }
-
-        /* all bits have the expected value */
-        return ~0UL;
-}
-#else /* debugging is disabled */
-static int debugging;
-
-static inline unsigned long verify_bit_range(unsigned long* bitmap,
-        int expected, unsigned long start, unsigned long end)
-{
-        return ~0UL;
-}
-
-#endif /* CONFIG_IOMMU_DEBUG */
-
 static inline int translation_enabled(struct iommu_table *tbl)
 {
         /* only PHBs with translation enabled have an IOMMU table */
@@ -228,7 +197,6 @@ static void iommu_range_reserve(struct iommu_table *tbl,
 {
         unsigned long index;
         unsigned long end;
-        unsigned long badbit;
         unsigned long flags;
 
         index = start_addr >> PAGE_SHIFT;
@@ -243,14 +211,6 @@ static void iommu_range_reserve(struct iommu_table *tbl,
 
         spin_lock_irqsave(&tbl->it_lock, flags);
 
-        badbit = verify_bit_range(tbl->it_map, 0, index, end);
-        if (badbit != ~0UL) {
-                if (printk_ratelimit())
-                        printk(KERN_ERR "Calgary: entry already allocated at "
-                               "0x%lx tbl %p dma 0x%lx npages %u\n",
-                               badbit, tbl, start_addr, npages);
-        }
-
         iommu_area_reserve(tbl->it_map, index, npages);
 
         spin_unlock_irqrestore(&tbl->it_lock, flags);
@@ -326,7 +286,6 @@ static void iommu_free(struct iommu_table *tbl, dma_addr_t dma_addr,
         unsigned int npages)
 {
         unsigned long entry;
-        unsigned long badbit;
         unsigned long badend;
         unsigned long flags;
 
@@ -346,14 +305,6 @@ static void iommu_free(struct iommu_table *tbl, dma_addr_t dma_addr,
 
         spin_lock_irqsave(&tbl->it_lock, flags);
 
-        badbit = verify_bit_range(tbl->it_map, 1, entry, entry + npages);
-        if (badbit != ~0UL) {
-                if (printk_ratelimit())
-                        printk(KERN_ERR "Calgary: bit is off at 0x%lx "
-                               "tbl %p dma 0x%Lx entry 0x%lx npages %u\n",
-                               badbit, tbl, dma_addr, entry, npages);
-        }
-
         iommu_area_free(tbl->it_map, entry, npages);
 
         spin_unlock_irqrestore(&tbl->it_lock, flags);
@@ -1488,9 +1439,8 @@ void __init detect_calgary(void)
                 iommu_detected = 1;
                 calgary_detected = 1;
                 printk(KERN_INFO "PCI-DMA: Calgary IOMMU detected.\n");
-                printk(KERN_INFO "PCI-DMA: Calgary TCE table spec is %d, "
-                       "CONFIG_IOMMU_DEBUG is %s.\n", specified_table_size,
-                       debugging ? "enabled" : "disabled");
+                printk(KERN_INFO "PCI-DMA: Calgary TCE table spec is %d\n",
+                       specified_table_size);
 
                 /* swiotlb for devices that aren't behind the Calgary. */
                 if (max_pfn > MAX_DMA32_PFN)
diff --git a/arch/x86/kernel/pci-gart_64.c b/arch/x86/kernel/pci-gart_64.c
index b284b58c035c..cfd9f9063896 100644
--- a/arch/x86/kernel/pci-gart_64.c
+++ b/arch/x86/kernel/pci-gart_64.c
@@ -144,48 +144,21 @@ static void flush_gart(void)
 }
 
 #ifdef CONFIG_IOMMU_LEAK
-
-#define SET_LEAK(x)                                                     \
-        do {                                                            \
-                if (iommu_leak_tab)                                     \
-                        iommu_leak_tab[x] = __builtin_return_address(0);\
-        } while (0)
-
-#define CLEAR_LEAK(x)                                                   \
-        do {                                                            \
-                if (iommu_leak_tab)                                     \
-                        iommu_leak_tab[x] = NULL;                       \
-        } while (0)
-
 /* Debugging aid for drivers that don't free their IOMMU tables */
-static void **iommu_leak_tab;
 static int leak_trace;
 static int iommu_leak_pages = 20;
 
 static void dump_leak(void)
 {
-        int i;
         static int dump;
 
-        if (dump || !iommu_leak_tab)
+        if (dump)
                 return;
         dump = 1;
-        show_stack(NULL, NULL);
 
-        /* Very crude. dump some from the end of the table too */
-        printk(KERN_DEBUG "Dumping %d pages from end of IOMMU:\n",
-               iommu_leak_pages);
-        for (i = 0; i < iommu_leak_pages; i += 2) {
-                printk(KERN_DEBUG "%lu: ", iommu_pages-i);
-                printk_address((unsigned long) iommu_leak_tab[iommu_pages-i],
-                                0);
-                printk(KERN_CONT "%c", (i+1)%2 == 0 ? '\n' : ' ');
-        }
-        printk(KERN_DEBUG "\n");
+        show_stack(NULL, NULL);
+        debug_dma_dump_mappings(NULL);
 }
-#else
-# define SET_LEAK(x)
-# define CLEAR_LEAK(x)
 #endif
 
 static void iommu_full(struct device *dev, size_t size, int dir)
@@ -248,7 +221,6 @@ static dma_addr_t dma_map_area(struct device *dev, dma_addr_t phys_mem,
 
         for (i = 0; i < npages; i++) {
                 iommu_gatt_base[iommu_page + i] = GPTE_ENCODE(phys_mem);
-                SET_LEAK(iommu_page + i);
                 phys_mem += PAGE_SIZE;
         }
         return iommu_bus_base + iommu_page*PAGE_SIZE + (phys_mem & ~PAGE_MASK);
@@ -294,7 +266,6 @@ static void gart_unmap_page(struct device *dev, dma_addr_t dma_addr,
         npages = iommu_num_pages(dma_addr, size, PAGE_SIZE);
         for (i = 0; i < npages; i++) {
                 iommu_gatt_base[iommu_page + i] = gart_unmapped_entry;
-                CLEAR_LEAK(iommu_page + i);
         }
         free_iommu(iommu_page, npages);
 }
@@ -377,7 +348,6 @@ static int __dma_map_cont(struct device *dev, struct scatterlist *start,
                 pages = iommu_num_pages(s->offset, s->length, PAGE_SIZE);
                 while (pages--) {
                         iommu_gatt_base[iommu_page] = GPTE_ENCODE(addr);
-                        SET_LEAK(iommu_page);
                         addr += PAGE_SIZE;
                         iommu_page++;
                 }
@@ -688,8 +658,6 @@ static __init int init_k8_gatt(struct agp_kern_info *info)
 
         agp_gatt_table = gatt;
 
-        enable_gart_translations();
-
         error = sysdev_class_register(&gart_sysdev_class);
         if (!error)
                 error = sysdev_register(&device_gart);
@@ -801,11 +769,12 @@ void __init gart_iommu_init(void)
 
 #ifdef CONFIG_IOMMU_LEAK
         if (leak_trace) {
-                iommu_leak_tab = (void *)__get_free_pages(GFP_KERNEL|__GFP_ZERO,
-                                  get_order(iommu_pages*sizeof(void *)));
-                if (!iommu_leak_tab)
+                int ret;
+
+                ret = dma_debug_resize_entries(iommu_pages);
+                if (ret)
                         printk(KERN_DEBUG
-                               "PCI-DMA: Cannot allocate leak trace area\n");
+                               "PCI-DMA: Cannot trace all the entries\n");
         }
 #endif
 
@@ -845,6 +814,14 @@ void __init gart_iommu_init(void)
          * the pages as Not-Present:
          */
         wbinvd();
+        
+        /*
+         * Now all caches are flushed and we can safely enable
+         * GART hardware.  Doing it early leaves the possibility
+         * of stale cache entries that can lead to GART PTE
+         * errors.
+         */
+        enable_gart_translations();
 
         /*
          * Try to workaround a bug (thanks to BenH):
diff --git a/arch/x86/kernel/pci-swiotlb.c b/arch/x86/kernel/pci-swiotlb.c
index 221a3853e268..a1712f2b50f1 100644
--- a/arch/x86/kernel/pci-swiotlb.c
+++ b/arch/x86/kernel/pci-swiotlb.c
@@ -28,7 +28,7 @@ dma_addr_t swiotlb_phys_to_bus(struct device *hwdev, phys_addr_t paddr)
         return paddr;
 }
 
-phys_addr_t swiotlb_bus_to_phys(dma_addr_t baddr)
+phys_addr_t swiotlb_bus_to_phys(struct device *hwdev, dma_addr_t baddr)
 {
         return baddr;
 }
diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c
index ca989158e847..3bb2be1649bd 100644
--- a/arch/x86/kernel/process.c
+++ b/arch/x86/kernel/process.c
@@ -8,12 +8,15 @@
 #include <linux/module.h>
 #include <linux/pm.h>
 #include <linux/clockchips.h>
+#include <linux/random.h>
 #include <trace/power.h>
 #include <asm/system.h>
 #include <asm/apic.h>
+#include <asm/syscalls.h>
 #include <asm/idle.h>
 #include <asm/uaccess.h>
 #include <asm/i387.h>
+#include <asm/ds.h>
 
 unsigned long idle_halt;
 EXPORT_SYMBOL(idle_halt);
@@ -45,6 +48,8 @@ void free_thread_xstate(struct task_struct *tsk)
                 kmem_cache_free(task_xstate_cachep, tsk->thread.xstate);
                 tsk->thread.xstate = NULL;
         }
+
+        WARN(tsk->thread.ds_ctx, "leaking DS context\n");
 }
 
 void free_thread_info(struct thread_info *ti)
@@ -83,8 +88,6 @@ void exit_thread(void)
                 put_cpu();
                 kfree(bp);
         }
-
-        ds_exit_thread(current);
 }
 
 void flush_thread(void)
@@ -613,3 +616,16 @@ static int __init idle_setup(char *str)
 }
 early_param("idle", idle_setup);
 
+unsigned long arch_align_stack(unsigned long sp)
+{
+        if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space)
+                sp -= get_random_int() % 8192;
+        return sp & ~0xf;
+}
+
+unsigned long arch_randomize_brk(struct mm_struct *mm)
+{
+        unsigned long range_end = mm->brk + 0x02000000;
+        return randomize_range(mm->brk, range_end, 0) ? : mm->brk;
+}
+
diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c
index 76f8f84043a2..59f4524984af 100644
--- a/arch/x86/kernel/process_32.c
+++ b/arch/x86/kernel/process_32.c
@@ -9,8 +9,6 @@
  * This file handles the architecture-dependent parts of process handling..
  */
 
-#include <stdarg.h>
-
 #include <linux/stackprotector.h>
 #include <linux/cpu.h>
 #include <linux/errno.h>
@@ -33,7 +31,6 @@
 #include <linux/module.h>
 #include <linux/kallsyms.h>
 #include <linux/ptrace.h>
-#include <linux/random.h>
 #include <linux/personality.h>
 #include <linux/tick.h>
 #include <linux/percpu.h>
@@ -290,7 +287,8 @@ int copy_thread(unsigned long clone_flags, unsigned long sp,
                 p->thread.io_bitmap_max = 0;
         }
 
-        ds_copy_thread(p, current);
+        clear_tsk_thread_flag(p, TIF_DS_AREA_MSR);
+        p->thread.ds_ctx = NULL;
 
         clear_tsk_thread_flag(p, TIF_DEBUGCTLMSR);
         p->thread.debugctlmsr = 0;
@@ -407,7 +405,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
          * done before math_state_restore, so the TS bit is up
          * to date.
          */
-        arch_leave_lazy_cpu_mode();
+        arch_end_context_switch(next_p);
 
         /* If the task has used fpu the last 5 timeslices, just do a full
          * restore of the math state immediately to avoid the trap; the
@@ -497,15 +495,3 @@ unsigned long get_wchan(struct task_struct *p)
         return 0;
 }
 
-unsigned long arch_align_stack(unsigned long sp)
-{
-        if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space)
-                sp -= get_random_int() % 8192;
-        return sp & ~0xf;
-}
-
-unsigned long arch_randomize_brk(struct mm_struct *mm)
-{
-        unsigned long range_end = mm->brk + 0x02000000;
-        return randomize_range(mm->brk, range_end, 0) ? : mm->brk;
-}
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index b751a41392b1..ebefb5407b9d 100644
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -14,8 +14,6 @@
  * This file handles the architecture-dependent parts of process handling..
  */
 
-#include <stdarg.h>
-
 #include <linux/stackprotector.h>
 #include <linux/cpu.h>
 #include <linux/errno.h>
@@ -32,7 +30,6 @@
 #include <linux/delay.h>
 #include <linux/module.h>
 #include <linux/ptrace.h>
-#include <linux/random.h>
 #include <linux/notifier.h>
 #include <linux/kprobes.h>
 #include <linux/kdebug.h>
@@ -335,7 +332,8 @@ int copy_thread(unsigned long clone_flags, unsigned long sp,
                         goto out;
         }
 
-        ds_copy_thread(p, me);
+        clear_tsk_thread_flag(p, TIF_DS_AREA_MSR);
+        p->thread.ds_ctx = NULL;
 
         clear_tsk_thread_flag(p, TIF_DEBUGCTLMSR);
         p->thread.debugctlmsr = 0;
@@ -428,7 +426,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
          * done before math_state_restore, so the TS bit is up
          * to date.
          */
-        arch_leave_lazy_cpu_mode();
+        arch_end_context_switch(next_p);
 
         /*
          * Switch FS and GS.
@@ -660,15 +658,3 @@ long sys_arch_prctl(int code, unsigned long addr)
         return do_arch_prctl(current, code, addr);
 }
 
-unsigned long arch_align_stack(unsigned long sp)
-{
-        if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space)
-                sp -= get_random_int() % 8192;
-        return sp & ~0xf;
-}
-
-unsigned long arch_randomize_brk(struct mm_struct *mm)
-{
-        unsigned long range_end = mm->brk + 0x02000000;
-        return randomize_range(mm->brk, range_end, 0) ? : mm->brk;
-}
diff --git a/arch/x86/kernel/ptrace.c b/arch/x86/kernel/ptrace.c
index 23b7c8f017e2..09ecbde91c13 100644
--- a/arch/x86/kernel/ptrace.c
+++ b/arch/x86/kernel/ptrace.c
@@ -21,6 +21,7 @@
 #include <linux/audit.h>
 #include <linux/seccomp.h>
 #include <linux/signal.h>
+#include <linux/workqueue.h>
 
 #include <asm/uaccess.h>
 #include <asm/pgtable.h>
@@ -578,17 +579,130 @@ static int ioperm_get(struct task_struct *target,
 }
 
 #ifdef CONFIG_X86_PTRACE_BTS
+/*
+ * A branch trace store context.
+ *
+ * Contexts may only be installed by ptrace_bts_config() and only for
+ * ptraced tasks.
+ *
+ * Contexts are destroyed when the tracee is detached from the tracer.
+ * The actual destruction work requires interrupts enabled, so the
+ * work is deferred and will be scheduled during __ptrace_unlink().
+ *
+ * Contexts hold an additional task_struct reference on the traced
+ * task, as well as a reference on the tracer's mm.
+ *
+ * Ptrace already holds a task_struct for the duration of ptrace operations,
+ * but since destruction is deferred, it may be executed after both
+ * tracer and tracee exited.
+ */
+struct bts_context {
+        /* The branch trace handle. */
+        struct bts_tracer       *tracer;
+
+        /* The buffer used to store the branch trace and its size. */
+        void                    *buffer;
+        unsigned int            size;
+
+        /* The mm that paid for the above buffer. */
+        struct mm_struct        *mm;
+
+        /* The task this context belongs to. */
+        struct task_struct      *task;
+
+        /* The signal to send on a bts buffer overflow. */
+        unsigned int            bts_ovfl_signal;
+
+        /* The work struct to destroy a context. */
+        struct work_struct      work;
+};
+
+static int alloc_bts_buffer(struct bts_context *context, unsigned int size)
+{
+        void *buffer = NULL;
+        int err = -ENOMEM;
+
+        err = account_locked_memory(current->mm, current->signal->rlim, size);
+        if (err < 0)
+                return err;
+
+        buffer = kzalloc(size, GFP_KERNEL);
+        if (!buffer)
+                goto out_refund;
+
+        context->buffer = buffer;
+        context->size = size;
+        context->mm = get_task_mm(current);
+
+        return 0;
+
+ out_refund:
+        refund_locked_memory(current->mm, size);
+        return err;
+}
+
+static inline void free_bts_buffer(struct bts_context *context)
+{
+        if (!context->buffer)
+                return;
+
+        kfree(context->buffer);
+        context->buffer = NULL;
+
+        refund_locked_memory(context->mm, context->size);
+        context->size = 0;
+
+        mmput(context->mm);
+        context->mm = NULL;
+}
+
+static void free_bts_context_work(struct work_struct *w)
+{
+        struct bts_context *context;
+
+        context = container_of(w, struct bts_context, work);
+
+        ds_release_bts(context->tracer);
+        put_task_struct(context->task);
+        free_bts_buffer(context);
+        kfree(context);
+}
+
+static inline void free_bts_context(struct bts_context *context)
+{
+        INIT_WORK(&context->work, free_bts_context_work);
+        schedule_work(&context->work);
+}
+
+static inline struct bts_context *alloc_bts_context(struct task_struct *task)
+{
+        struct bts_context *context = kzalloc(sizeof(*context), GFP_KERNEL);
+        if (context) {
+                context->task = task;
+                task->bts = context;
+
+                get_task_struct(task);
+        }
+
+        return context;
+}
+
 static int ptrace_bts_read_record(struct task_struct *child, size_t index,
                                   struct bts_struct __user *out)
 {
+        struct bts_context *context;
         const struct bts_trace *trace;
         struct bts_struct bts;
         const unsigned char *at;
         int error;
 
-        trace = ds_read_bts(child->bts);
+        context = child->bts;
+        if (!context)
+                return -ESRCH;
+
+        trace = ds_read_bts(context->tracer);
         if (!trace)
-                return -EPERM;
+                return -ESRCH;
 
         at = trace->ds.top - ((index + 1) * trace->ds.size);
         if ((void *)at < trace->ds.begin)
@@ -597,7 +711,7 @@ static int ptrace_bts_read_record(struct task_struct *child, size_t index,
         if (!trace->read)
                 return -EOPNOTSUPP;
 
-        error = trace->read(child->bts, at, &bts);
+        error = trace->read(context->tracer, at, &bts);
         if (error < 0)
                 return error;
 
@@ -611,13 +725,18 @@ static int ptrace_bts_drain(struct task_struct *child,
                             long size,
                             struct bts_struct __user *out)
 {
+        struct bts_context *context;
         const struct bts_trace *trace;
         const unsigned char *at;
         int error, drained = 0;
 
-        trace = ds_read_bts(child->bts);
+        context = child->bts;
+        if (!context)
+                return -ESRCH;
+
+        trace = ds_read_bts(context->tracer);
         if (!trace)
-                return -EPERM;
+                return -ESRCH;
 
         if (!trace->read)
                 return -EOPNOTSUPP;
@@ -628,9 +747,8 @@ static int ptrace_bts_drain(struct task_struct *child,
         for (at = trace->ds.begin; (void *)at < trace->ds.top;
              out++, drained++, at += trace->ds.size) {
                 struct bts_struct bts;
-                int error;
 
-                error = trace->read(child->bts, at, &bts);
+                error = trace->read(context->tracer, at, &bts);
                 if (error < 0)
                         return error;
 
@@ -640,35 +758,18 @@ static int ptrace_bts_drain(struct task_struct *child,
 
         memset(trace->ds.begin, 0, trace->ds.n * trace->ds.size);
 
-        error = ds_reset_bts(child->bts);
+        error = ds_reset_bts(context->tracer);
         if (error < 0)
                 return error;
 
         return drained;
 }
 
-static int ptrace_bts_allocate_buffer(struct task_struct *child, size_t size)
-{
-        child->bts_buffer = alloc_locked_buffer(size);
-        if (!child->bts_buffer)
-                return -ENOMEM;
-
-        child->bts_size = size;
-
-        return 0;
-}
-
-static void ptrace_bts_free_buffer(struct task_struct *child)
-{
-        free_locked_buffer(child->bts_buffer, child->bts_size);
-        child->bts_buffer = NULL;
-        child->bts_size = 0;
-}
-
 static int ptrace_bts_config(struct task_struct *child,
                              long cfg_size,
                              const struct ptrace_bts_config __user *ucfg)
 {
+        struct bts_context *context;
         struct ptrace_bts_config cfg;
         unsigned int flags = 0;
 
@@ -678,28 +779,33 @@ static int ptrace_bts_config(struct task_struct *child,
         if (copy_from_user(&cfg, ucfg, sizeof(cfg)))
                 return -EFAULT;
 
-        if (child->bts) {
-                ds_release_bts(child->bts);
-                child->bts = NULL;
-        }
+        context = child->bts;
+        if (!context)
+                context = alloc_bts_context(child);
+        if (!context)
+                return -ENOMEM;
 
         if (cfg.flags & PTRACE_BTS_O_SIGNAL) {
                 if (!cfg.signal)
                         return -EINVAL;
 
-                child->thread.bts_ovfl_signal = cfg.signal;
                 return -EOPNOTSUPP;
+                context->bts_ovfl_signal = cfg.signal;
         }
 
-        if ((cfg.flags & PTRACE_BTS_O_ALLOC) &&
-            (cfg.size != child->bts_size)) {
-                int error;
+        ds_release_bts(context->tracer);
+        context->tracer = NULL;
 
-                ptrace_bts_free_buffer(child);
+        if ((cfg.flags & PTRACE_BTS_O_ALLOC) && (cfg.size != context->size)) {
+                int err;
 
-                error = ptrace_bts_allocate_buffer(child, cfg.size);
-                if (error < 0)
-                        return error;
+                free_bts_buffer(context);
+                if (!cfg.size)
+                        return 0;
+
+                err = alloc_bts_buffer(context, cfg.size);
+                if (err < 0)
+                        return err;
         }
 
         if (cfg.flags & PTRACE_BTS_O_TRACE)
@@ -708,15 +814,14 @@ static int ptrace_bts_config(struct task_struct *child,
         if (cfg.flags & PTRACE_BTS_O_SCHED)
                 flags |= BTS_TIMESTAMPS;
 
-        child->bts = ds_request_bts(child, child->bts_buffer, child->bts_size,
-                                    /* ovfl = */ NULL, /* th = */ (size_t)-1,
-                                    flags);
-        if (IS_ERR(child->bts)) {
-                int error = PTR_ERR(child->bts);
-
-                ptrace_bts_free_buffer(child);
-                child->bts = NULL;
+        context->tracer =
+                ds_request_bts_task(child, context->buffer, context->size,
+                                    NULL, (size_t)-1, flags);
+        if (unlikely(IS_ERR(context->tracer))) {
+                int error = PTR_ERR(context->tracer);
 
+                free_bts_buffer(context);
+                context->tracer = NULL;
                 return error;
         }
 
@@ -727,20 +832,25 @@ static int ptrace_bts_status(struct task_struct *child,
                              long cfg_size,
                              struct ptrace_bts_config __user *ucfg)
 {
+        struct bts_context *context;
         const struct bts_trace *trace;
         struct ptrace_bts_config cfg;
 
+        context = child->bts;
+        if (!context)
+                return -ESRCH;
+
         if (cfg_size < sizeof(cfg))
                 return -EIO;
 
-        trace = ds_read_bts(child->bts);
+        trace = ds_read_bts(context->tracer);
         if (!trace)
-                return -EPERM;
+                return -ESRCH;
 
         memset(&cfg, 0, sizeof(cfg));
-        cfg.size = trace->ds.end - trace->ds.begin;
-        cfg.signal = child->thread.bts_ovfl_signal;
-        cfg.bts_size = sizeof(struct bts_struct);
+        cfg.size        = trace->ds.end - trace->ds.begin;
+        cfg.signal      = context->bts_ovfl_signal;
+        cfg.bts_size    = sizeof(struct bts_struct);
 
         if (cfg.signal)
                 cfg.flags |= PTRACE_BTS_O_SIGNAL;
@@ -759,80 +869,51 @@ static int ptrace_bts_status(struct task_struct *child,
 
 static int ptrace_bts_clear(struct task_struct *child)
 {
+        struct bts_context *context;
         const struct bts_trace *trace;
 
-        trace = ds_read_bts(child->bts);
+        context = child->bts;
+        if (!context)
+                return -ESRCH;
+
+        trace = ds_read_bts(context->tracer);
         if (!trace)
-                return -EPERM;
+                return -ESRCH;
 
         memset(trace->ds.begin, 0, trace->ds.n * trace->ds.size);
 
-        return ds_reset_bts(child->bts);
+        return ds_reset_bts(context->tracer);
 }
 
 static int ptrace_bts_size(struct task_struct *child)
 {
+        struct bts_context *context;
         const struct bts_trace *trace;
 
-        trace = ds_read_bts(child->bts);
+        context = child->bts;
+        if (!context)
+                return -ESRCH;
+
+        trace = ds_read_bts(context->tracer);
         if (!trace)
-                return -EPERM;
+                return -ESRCH;
 
         return (trace->ds.top - trace->ds.begin) / trace->ds.size;
 }
 
-static void ptrace_bts_fork(struct task_struct *tsk)
-{
-        tsk->bts = NULL;
-        tsk->bts_buffer = NULL;
-        tsk->bts_size = 0;
-        tsk->thread.bts_ovfl_signal = 0;
-}
-
-static void ptrace_bts_untrace(struct task_struct *child)
+/*
+ * Called from __ptrace_unlink() after the child has been moved back
+ * to its original parent.
+ */
+void ptrace_bts_untrace(struct task_struct *child)
 {
         if (unlikely(child->bts)) {
-                ds_release_bts(child->bts);
+                free_bts_context(child->bts);
                 child->bts = NULL;
-
-                /* We cannot update total_vm and locked_vm since
-                   child's mm is already gone. But we can reclaim the
-                   memory. */
-                kfree(child->bts_buffer);
-                child->bts_buffer = NULL;
-                child->bts_size = 0;
         }
 }
-
-static void ptrace_bts_detach(struct task_struct *child)
-{
-        /*
-         * Ptrace_detach() races with ptrace_untrace() in case
-         * the child dies and is reaped by another thread.
-         *
-         * We only do the memory accounting at this point and
-         * leave the buffer deallocation and the bts tracer
-         * release to ptrace_bts_untrace() which will be called
-         * later on with tasklist_lock held.
-         */
-        release_locked_buffer(child->bts_buffer, child->bts_size);
-}
-#else
-static inline void ptrace_bts_fork(struct task_struct *tsk) {}
-static inline void ptrace_bts_detach(struct task_struct *child) {}
-static inline void ptrace_bts_untrace(struct task_struct *child) {}
 #endif /* CONFIG_X86_PTRACE_BTS */
 
-void x86_ptrace_fork(struct task_struct *child, unsigned long clone_flags)
-{
-        ptrace_bts_fork(child);
-}
-
-void x86_ptrace_untrace(struct task_struct *child)
-{
-        ptrace_bts_untrace(child);
-}
-
 /*
  * Called by kernel/ptrace.c when detaching..
  *
@@ -844,7 +925,6 @@ void ptrace_disable(struct task_struct *child)
 #ifdef TIF_SYSCALL_EMU
         clear_tsk_thread_flag(child, TIF_SYSCALL_EMU);
 #endif
-        ptrace_bts_detach(child);
 }
 
 #if defined CONFIG_X86_32 || defined CONFIG_IA32_EMULATION
diff --git a/arch/x86/kernel/quirks.c b/arch/x86/kernel/quirks.c
index 7563b31b4f03..af71d06624bf 100644
--- a/arch/x86/kernel/quirks.c
+++ b/arch/x86/kernel/quirks.c
@@ -491,5 +491,42 @@ void force_hpet_resume(void)
                 break;
         }
 }
+#endif
+
+#if defined(CONFIG_PCI) && defined(CONFIG_NUMA)
+/* Set correct numa_node information for AMD NB functions */
+static void __init quirk_amd_nb_node(struct pci_dev *dev)
+{
+        struct pci_dev *nb_ht;
+        unsigned int devfn;
+        u32 val;
+
+        devfn = PCI_DEVFN(PCI_SLOT(dev->devfn), 0);
+        nb_ht = pci_get_slot(dev->bus, devfn);
+        if (!nb_ht)
+                return;
+
+        pci_read_config_dword(nb_ht, 0x60, &val);
+        set_dev_node(&dev->dev, val & 7);
+        pci_dev_put(dev);
+}
 
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_K8_NB,
+                        quirk_amd_nb_node);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP,
+                        quirk_amd_nb_node);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_K8_NB_MEMCTL,
+                        quirk_amd_nb_node);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_K8_NB_MISC,
+                        quirk_amd_nb_node);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_10H_NB_HT,
+                        quirk_amd_nb_node);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_10H_NB_MAP,
+                        quirk_amd_nb_node);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_10H_NB_DRAM,
+                        quirk_amd_nb_node);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_10H_NB_MISC,
+                        quirk_amd_nb_node);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_10H_NB_LINK,
+                        quirk_amd_nb_node);
 #endif
diff --git a/arch/x86/kernel/reboot.c b/arch/x86/kernel/reboot.c
index 667188e0b5a0..d2d1ce8170f0 100644
--- a/arch/x86/kernel/reboot.c
+++ b/arch/x86/kernel/reboot.c
@@ -192,6 +192,15 @@ static struct dmi_system_id __initdata reboot_dmi_table[] = {
                         DMI_MATCH(DMI_BOARD_NAME, "0KP561"),
                 },
         },
+        {   /* Handle problems with rebooting on Dell Optiplex 360 with 0T656F */
+                .callback = set_bios_reboot,
+                .ident = "Dell OptiPlex 360",
+                .matches = {
+                        DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
+                        DMI_MATCH(DMI_PRODUCT_NAME, "OptiPlex 360"),
+                        DMI_MATCH(DMI_BOARD_NAME, "0T656F"),
+                },
+        },
         {       /* Handle problems with rebooting on Dell 2400's */
                 .callback = set_bios_reboot,
                 .ident = "Dell PowerEdge 2400",
diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c
index 523bb697120d..d1c636bf31a7 100644
--- a/arch/x86/kernel/setup.c
+++ b/arch/x86/kernel/setup.c
@@ -112,6 +112,14 @@
 #define ARCH_SETUP
 #endif
 
+/*
+ * end_pfn only includes RAM, while max_pfn_mapped includes all e820 entries.
+ * The direct mapping extends to max_pfn_mapped, so that we can directly access
+ * apertures, ACPI and other tables without having to play with fixmaps.
+ */
+unsigned long max_low_pfn_mapped;
+unsigned long max_pfn_mapped;
+
 RESERVE_BRK(dmi_alloc, 65536);
 
 unsigned int boot_cpu_id __read_mostly;
@@ -214,8 +222,8 @@ unsigned long mmu_cr4_features;
 unsigned long mmu_cr4_features = X86_CR4_PAE;
 #endif
 
-/* Boot loader ID as an integer, for the benefit of proc_dointvec */
-int bootloader_type;
+/* Boot loader ID and version as integers, for the benefit of proc_dointvec */
+int bootloader_type, bootloader_version;
 
 /*
  * Setup options
@@ -706,6 +714,12 @@ void __init setup_arch(char **cmdline_p)
 #endif
         saved_video_mode = boot_params.hdr.vid_mode;
         bootloader_type = boot_params.hdr.type_of_loader;
+        if ((bootloader_type >> 4) == 0xe) {
+                bootloader_type &= 0xf;
+                bootloader_type |= (boot_params.hdr.ext_loader_type+0x10) << 4;
+        }
+        bootloader_version  = bootloader_type & 0xf;
+        bootloader_version |= boot_params.hdr.ext_loader_ver << 4;
 
 #ifdef CONFIG_BLK_DEV_RAM
         rd_image_start = boot_params.hdr.ram_size & RAMDISK_IMAGE_START_MASK;
@@ -854,12 +868,16 @@ void __init setup_arch(char **cmdline_p)
                 max_low_pfn = max_pfn;
 
         high_memory = (void *)__va(max_pfn * PAGE_SIZE - 1) + 1;
+        max_pfn_mapped = KERNEL_IMAGE_SIZE >> PAGE_SHIFT;
 #endif
 
 #ifdef CONFIG_X86_CHECK_BIOS_CORRUPTION
         setup_bios_corruption_check();
 #endif
 
+        printk(KERN_DEBUG "initial memory mapped : 0 - %08lx\n",
+                        max_pfn_mapped<<PAGE_SHIFT);
+
         reserve_brk();
 
         /* max_pfn_mapped is updated here */
diff --git a/arch/x86/kernel/setup_percpu.c b/arch/x86/kernel/setup_percpu.c
index 8f0e13be36b3..9c3f0823e6aa 100644
--- a/arch/x86/kernel/setup_percpu.c
+++ b/arch/x86/kernel/setup_percpu.c
@@ -425,6 +425,14 @@ void __init setup_per_cpu_areas(void)
         early_per_cpu_ptr(x86_cpu_to_node_map) = NULL;
 #endif
 
+#if defined(CONFIG_X86_64) && defined(CONFIG_NUMA)
+        /*
+         * make sure boot cpu node_number is right, when boot cpu is on the
+         * node that doesn't have mem installed
+         */
+        per_cpu(node_number, boot_cpu_id) = cpu_to_node(boot_cpu_id);
+#endif
+
         /* Setup node to cpumask map */
         setup_node_to_cpumask_map();
 
diff --git a/arch/x86/kernel/signal.c b/arch/x86/kernel/signal.c
index 4976888094f0..4c578751e94e 100644
--- a/arch/x86/kernel/signal.c
+++ b/arch/x86/kernel/signal.c
@@ -6,7 +6,6 @@
  *  2000-06-20  Pentium III FXSR, SSE support by Gareth Hughes
  *  2000-2002   x86-64 support by Andi Kleen
  */
-
 #include <linux/sched.h>
 #include <linux/mm.h>
 #include <linux/smp.h>
diff --git a/arch/x86/kernel/smp.c b/arch/x86/kernel/smp.c
index bf1831aa14fa..ec1de97600e7 100644
--- a/arch/x86/kernel/smp.c
+++ b/arch/x86/kernel/smp.c
@@ -198,6 +198,9 @@ void smp_reschedule_interrupt(struct pt_regs *regs)
 {
         ack_APIC_irq();
         inc_irq_stat(irq_resched_count);
+        /*
+         * KVM uses this interrupt to force a cpu out of guest mode
+         */
 }
 
 void smp_call_function_interrupt(struct pt_regs *regs)
diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index d2e8de958156..7c80007ea5f7 100644
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -992,10 +992,12 @@ static int __init smp_sanity_check(unsigned max_cpus)
          */
         if (APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid]) &&
             !cpu_has_apic) {
-                printk(KERN_ERR "BIOS bug, local APIC #%d not detected!...\n",
-                        boot_cpu_physical_apicid);
-                printk(KERN_ERR "... forcing use of dummy APIC emulation."
+                if (!disable_apic) {
+                        pr_err("BIOS bug, local APIC #%d not detected!...\n",
+                                boot_cpu_physical_apicid);
+                        pr_err("... forcing use of dummy APIC emulation."
                                 "(tell your hw vendor)\n");
+                }
                 smpboot_clear_io_apic();
                 arch_disable_smp_support();
                 return -1;
diff --git a/arch/x86/kernel/stacktrace.c b/arch/x86/kernel/stacktrace.c
index f7bddc2e37d1..4aaf7e48394f 100644
--- a/arch/x86/kernel/stacktrace.c
+++ b/arch/x86/kernel/stacktrace.c
@@ -20,7 +20,7 @@ save_stack_warning_symbol(void *data, char *msg, unsigned long symbol)
 
 static int save_stack_stack(void *data, char *name)
 {
-        return -1;
+        return 0;
 }
 
 static void save_stack_address(void *data, unsigned long addr, int reliable)
diff --git a/arch/x86/kernel/syscall_table_32.S b/arch/x86/kernel/syscall_table_32.S
index ff5c8736b491..d51321ddafda 100644
--- a/arch/x86/kernel/syscall_table_32.S
+++ b/arch/x86/kernel/syscall_table_32.S
@@ -334,3 +334,5 @@ ENTRY(sys_call_table)
         .long sys_inotify_init1
         .long sys_preadv
         .long sys_pwritev
+        .long sys_rt_tgsigqueueinfo     /* 335 */
+        .long sys_perf_counter_open
diff --git a/arch/x86/kernel/tlb_uv.c b/arch/x86/kernel/tlb_uv.c
index ed0c33761e6d..124d40c575df 100644
--- a/arch/x86/kernel/tlb_uv.c
+++ b/arch/x86/kernel/tlb_uv.c
@@ -715,7 +715,12 @@ uv_activation_descriptor_init(int node, int pnode)
         struct bau_desc *adp;
         struct bau_desc *ad2;
 
-        adp = (struct bau_desc *)kmalloc_node(16384, GFP_KERNEL, node);
+        /*
+         * each bau_desc is 64 bytes; there are 8 (UV_ITEMS_PER_DESCRIPTOR)
+         * per cpu; and up to 32 (UV_ADP_SIZE) cpu's per blade
+         */
+        adp = (struct bau_desc *)kmalloc_node(sizeof(struct bau_desc)*
+                UV_ADP_SIZE*UV_ITEMS_PER_DESCRIPTOR, GFP_KERNEL, node);
         BUG_ON(!adp);
 
         pa = uv_gpa(adp); /* need the real nasid*/
@@ -729,7 +734,13 @@ uv_activation_descriptor_init(int node, int pnode)
                                       (n << UV_DESC_BASE_PNODE_SHIFT | m));
         }
 
-        for (i = 0, ad2 = adp; i < UV_ACTIVATION_DESCRIPTOR_SIZE; i++, ad2++) {
+        /*
+         * initializing all 8 (UV_ITEMS_PER_DESCRIPTOR) descriptors for each
+         * cpu even though we only use the first one; one descriptor can
+         * describe a broadcast to 256 nodes.
+         */
+        for (i = 0, ad2 = adp; i < (UV_ADP_SIZE*UV_ITEMS_PER_DESCRIPTOR);
+                i++, ad2++) {
                 memset(ad2, 0, sizeof(struct bau_desc));
                 ad2->header.sw_ack_flag = 1;
                 /*
@@ -832,7 +843,7 @@ static int __init uv_bau_init(void)
                 return 0;
 
         for_each_possible_cpu(cur_cpu)
-                alloc_cpumask_var_node(&per_cpu(uv_flush_tlb_mask, cur_cpu),
+                zalloc_cpumask_var_node(&per_cpu(uv_flush_tlb_mask, cur_cpu),
                                        GFP_KERNEL, cpu_to_node(cur_cpu));
 
         uv_bau_retry_limit = 1;
diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c
index f4d683b630ba..1e1e27b7d438 100644
--- a/arch/x86/kernel/traps.c
+++ b/arch/x86/kernel/traps.c
@@ -839,9 +839,6 @@ asmlinkage void math_state_restore(void)
         }
 
         clts();                         /* Allow maths ops (or we recurse) */
-#ifdef CONFIG_X86_32
-        restore_fpu(tsk);
-#else
         /*
          * Paranoid restore. send a SIGSEGV if we fail to restore the state.
          */
@@ -850,7 +847,7 @@ asmlinkage void math_state_restore(void)
                 force_sig(SIGSEGV, tsk);
                 return;
         }
-#endif
+
         thread->status |= TS_USEDFPU;   /* So we fnsave on switch_to() */
         tsk->fpu_counter++;
 }
@@ -945,8 +942,13 @@ void __init trap_init(void)
 #endif
         set_intr_gate(19, &simd_coprocessor_error);
 
+        /* Reserve all the builtin and the syscall vector: */
+        for (i = 0; i < FIRST_EXTERNAL_VECTOR; i++)
+                set_bit(i, used_vectors);
+
 #ifdef CONFIG_IA32_EMULATION
         set_system_intr_gate(IA32_SYSCALL_VECTOR, ia32_syscall);
+        set_bit(IA32_SYSCALL_VECTOR, used_vectors);
 #endif
 
 #ifdef CONFIG_X86_32
@@ -963,14 +965,9 @@ void __init trap_init(void)
         }
 
         set_system_trap_gate(SYSCALL_VECTOR, &system_call);
+        set_bit(SYSCALL_VECTOR, used_vectors);
 #endif
 
-        /* Reserve all the builtin and the syscall vector: */
-        for (i = 0; i < FIRST_EXTERNAL_VECTOR; i++)
-                set_bit(i, used_vectors);
-
-        set_bit(IA32_SYSCALL_VECTOR, used_vectors);
-
         /*
          * Should be a barrier for any external CPU state:
          */
diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c
index d57de05dc430..3e1c057e98fe 100644
--- a/arch/x86/kernel/tsc.c
+++ b/arch/x86/kernel/tsc.c
@@ -384,13 +384,13 @@ unsigned long native_calibrate_tsc(void)
 {
         u64 tsc1, tsc2, delta, ref1, ref2;
         unsigned long tsc_pit_min = ULONG_MAX, tsc_ref_min = ULONG_MAX;
-        unsigned long flags, latch, ms, fast_calibrate, tsc_khz;
+        unsigned long flags, latch, ms, fast_calibrate, hv_tsc_khz;
         int hpet = is_hpet_enabled(), i, loopmin;
 
-        tsc_khz = get_hypervisor_tsc_freq();
-        if (tsc_khz) {
+        hv_tsc_khz = get_hypervisor_tsc_freq();
+        if (hv_tsc_khz) {
                 printk(KERN_INFO "TSC: Frequency read from the hypervisor\n");
-                return tsc_khz;
+                return hv_tsc_khz;
         }
 
         local_irq_save(flags);
@@ -710,7 +710,16 @@ static cycle_t read_tsc(struct clocksource *cs)
 #ifdef CONFIG_X86_64
 static cycle_t __vsyscall_fn vread_tsc(void)
 {
-        cycle_t ret = (cycle_t)vget_cycles();
+        cycle_t ret;
+
+        /*
+         * Surround the RDTSC by barriers, to make sure it's not
+         * speculated to outside the seqlock critical section and
+         * does not cause time warps:
+         */
+        rdtsc_barrier();
+        ret = (cycle_t)vget_cycles();
+        rdtsc_barrier();
 
         return ret >= __vsyscall_gtod_data.clock.cycle_last ?
                 ret : __vsyscall_gtod_data.clock.cycle_last;
diff --git a/arch/x86/kernel/tsc_sync.c b/arch/x86/kernel/tsc_sync.c
index bf36328f6ef9..027b5b498993 100644
--- a/arch/x86/kernel/tsc_sync.c
+++ b/arch/x86/kernel/tsc_sync.c
@@ -34,6 +34,7 @@ static __cpuinitdata atomic_t stop_count;
  * of a critical section, to be able to prove TSC time-warps:
  */
 static __cpuinitdata raw_spinlock_t sync_lock = __RAW_SPIN_LOCK_UNLOCKED;
+
 static __cpuinitdata cycles_t last_tsc;
 static __cpuinitdata cycles_t max_warp;
 static __cpuinitdata int nr_warps;
@@ -113,13 +114,12 @@ void __cpuinit check_tsc_sync_source(int cpu)
                 return;
 
         if (boot_cpu_has(X86_FEATURE_TSC_RELIABLE)) {
-                printk(KERN_INFO
-                       "Skipping synchronization checks as TSC is reliable.\n");
+                pr_info("Skipping synchronization checks as TSC is reliable.\n");
                 return;
         }
 
-        printk(KERN_INFO "checking TSC synchronization [CPU#%d -> CPU#%d]:",
-                          smp_processor_id(), cpu);
+        pr_info("checking TSC synchronization [CPU#%d -> CPU#%d]:",
+                smp_processor_id(), cpu);
 
         /*
          * Reset it - in case this is a second bootup:
@@ -143,8 +143,8 @@ void __cpuinit check_tsc_sync_source(int cpu)
 
         if (nr_warps) {
                 printk("\n");
-                printk(KERN_WARNING "Measured %Ld cycles TSC warp between CPUs,"
-                                    " turning off TSC clock.\n", max_warp);
+                pr_warning("Measured %Ld cycles TSC warp between CPUs, "
+                           "turning off TSC clock.\n", max_warp);
                 mark_tsc_unstable("check_tsc_sync_source failed");
         } else {
                 printk(" passed.\n");
@@ -195,5 +195,3 @@ void __cpuinit check_tsc_sync_target(void)
         while (atomic_read(&stop_count) != cpus)
                 cpu_relax();
 }
-#undef NR_LOOPS
-
diff --git a/arch/x86/kernel/vm86_32.c b/arch/x86/kernel/vm86_32.c
index d7ac84e7fc1c..9c4e62539058 100644
--- a/arch/x86/kernel/vm86_32.c
+++ b/arch/x86/kernel/vm86_32.c
@@ -287,10 +287,9 @@ static void do_sys_vm86(struct kernel_vm86_struct *info, struct task_struct *tsk
         info->regs.pt.ds = 0;
         info->regs.pt.es = 0;
         info->regs.pt.fs = 0;
-
-/* we are clearing gs later just before "jmp resume_userspace",
- * because it is not saved/restored.
- */
+#ifndef CONFIG_X86_32_LAZY_GS
+        info->regs.pt.gs = 0;
+#endif
 
 /*
  * The flags register is also special: we cannot trust that the user
@@ -318,9 +317,9 @@ static void do_sys_vm86(struct kernel_vm86_struct *info, struct task_struct *tsk
         }
 
 /*
- * Save old state, set default return value (%ax) to 0
+ * Save old state, set default return value (%ax) to 0 (VM86_SIGNAL)
  */
-        info->regs32->ax = 0;
+        info->regs32->ax = VM86_SIGNAL;
         tsk->thread.saved_sp0 = tsk->thread.sp0;
         tsk->thread.saved_fs = info->regs32->fs;
         tsk->thread.saved_gs = get_user_gs(info->regs32);
@@ -343,7 +342,9 @@ static void do_sys_vm86(struct kernel_vm86_struct *info, struct task_struct *tsk
         __asm__ __volatile__(
                 "movl %0,%%esp\n\t"
                 "movl %1,%%ebp\n\t"
+#ifdef CONFIG_X86_32_LAZY_GS
                 "mov  %2, %%gs\n\t"
+#endif
                 "jmp resume_userspace"
                 : /* no outputs */
                 :"r" (&info->regs), "r" (task_thread_info(tsk)), "r" (0));
diff --git a/arch/x86/kernel/vmi_32.c b/arch/x86/kernel/vmi_32.c
index 95deb9f2211e..b263423fbe2a 100644
--- a/arch/x86/kernel/vmi_32.c
+++ b/arch/x86/kernel/vmi_32.c
@@ -462,22 +462,28 @@ vmi_startup_ipi_hook(int phys_apicid, unsigned long start_eip,
 }
 #endif
 
-static void vmi_enter_lazy_cpu(void)
+static void vmi_start_context_switch(struct task_struct *prev)
 {
-        paravirt_enter_lazy_cpu();
+        paravirt_start_context_switch(prev);
         vmi_ops.set_lazy_mode(2);
 }
 
+static void vmi_end_context_switch(struct task_struct *next)
+{
+        vmi_ops.set_lazy_mode(0);
+        paravirt_end_context_switch(next);
+}
+
 static void vmi_enter_lazy_mmu(void)
 {
         paravirt_enter_lazy_mmu();
         vmi_ops.set_lazy_mode(1);
 }
 
-static void vmi_leave_lazy(void)
+static void vmi_leave_lazy_mmu(void)
 {
-        paravirt_leave_lazy(paravirt_get_lazy_mode());
         vmi_ops.set_lazy_mode(0);
+        paravirt_leave_lazy_mmu();
 }
 
 static inline int __init check_vmi_rom(struct vrom_header *rom)
@@ -711,14 +717,14 @@ static inline int __init activate_vmi(void)
         para_fill(pv_cpu_ops.set_iopl_mask, SetIOPLMask);
         para_fill(pv_cpu_ops.io_delay, IODelay);
 
-        para_wrap(pv_cpu_ops.lazy_mode.enter, vmi_enter_lazy_cpu,
+        para_wrap(pv_cpu_ops.start_context_switch, vmi_start_context_switch,
                   set_lazy_mode, SetLazyMode);
-        para_wrap(pv_cpu_ops.lazy_mode.leave, vmi_leave_lazy,
+        para_wrap(pv_cpu_ops.end_context_switch, vmi_end_context_switch,
                   set_lazy_mode, SetLazyMode);
 
         para_wrap(pv_mmu_ops.lazy_mode.enter, vmi_enter_lazy_mmu,
                   set_lazy_mode, SetLazyMode);
-        para_wrap(pv_mmu_ops.lazy_mode.leave, vmi_leave_lazy,
+        para_wrap(pv_mmu_ops.lazy_mode.leave, vmi_leave_lazy_mmu,
                   set_lazy_mode, SetLazyMode);
 
         /* user and kernel flush are just handled with different flags to FlushTLB */
diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S
index 849ee611f013..4c85b2e2bb65 100644
--- a/arch/x86/kernel/vmlinux.lds.S
+++ b/arch/x86/kernel/vmlinux.lds.S
@@ -1,5 +1,431 @@
+/*
+ * ld script for the x86 kernel
+ *
+ * Historic 32-bit version written by Martin Mares <mj@atrey.karlin.mff.cuni.cz>
+ *
+ * Modernisation, unification and other changes and fixes:
+ *   Copyright (C) 2007-2009  Sam Ravnborg <sam@ravnborg.org>
+ *
+ *
+ * Don't define absolute symbols until and unless you know that symbol
+ * value is should remain constant even if kernel image is relocated
+ * at run time. Absolute symbols are not relocated. If symbol value should
+ * change if kernel is relocated, make the symbol section relative and
+ * put it inside the section definition.
+ */
+
 #ifdef CONFIG_X86_32
-# include "vmlinux_32.lds.S"
+#define LOAD_OFFSET __PAGE_OFFSET
 #else
-# include "vmlinux_64.lds.S"
+#define LOAD_OFFSET __START_KERNEL_map
 #endif
+
+#include <asm-generic/vmlinux.lds.h>
+#include <asm/asm-offsets.h>
+#include <asm/thread_info.h>
+#include <asm/page_types.h>
+#include <asm/cache.h>
+#include <asm/boot.h>
+
+#undef i386     /* in case the preprocessor is a 32bit one */
+
+OUTPUT_FORMAT(CONFIG_OUTPUT_FORMAT, CONFIG_OUTPUT_FORMAT, CONFIG_OUTPUT_FORMAT)
+
+#ifdef CONFIG_X86_32
+OUTPUT_ARCH(i386)
+ENTRY(phys_startup_32)
+jiffies = jiffies_64;
+#else
+OUTPUT_ARCH(i386:x86-64)
+ENTRY(phys_startup_64)
+jiffies_64 = jiffies;
+#endif
+
+PHDRS {
+        text PT_LOAD FLAGS(5);          /* R_E */
+        data PT_LOAD FLAGS(7);          /* RWE */
+#ifdef CONFIG_X86_64
+        user PT_LOAD FLAGS(7);          /* RWE */
+        data.init PT_LOAD FLAGS(7);     /* RWE */
+#ifdef CONFIG_SMP
+        percpu PT_LOAD FLAGS(7);        /* RWE */
+#endif
+        data.init2 PT_LOAD FLAGS(7);    /* RWE */
+#endif
+        note PT_NOTE FLAGS(0);          /* ___ */
+}
+
+SECTIONS
+{
+#ifdef CONFIG_X86_32
+        . = LOAD_OFFSET + LOAD_PHYSICAL_ADDR;
+        phys_startup_32 = startup_32 - LOAD_OFFSET;
+#else
+        . = __START_KERNEL;
+        phys_startup_64 = startup_64 - LOAD_OFFSET;
+#endif
+
+        /* Text and read-only data */
+
+        /* bootstrapping code */
+        .text.head : AT(ADDR(.text.head) - LOAD_OFFSET) {
+                _text = .;
+                *(.text.head)
+        } :text = 0x9090
+
+        /* The rest of the text */
+        .text :  AT(ADDR(.text) - LOAD_OFFSET) {
+#ifdef CONFIG_X86_32
+                /* not really needed, already page aligned */
+                . = ALIGN(PAGE_SIZE);
+                *(.text.page_aligned)
+#endif
+                . = ALIGN(8);
+                _stext = .;
+                TEXT_TEXT
+                SCHED_TEXT
+                LOCK_TEXT
+                KPROBES_TEXT
+                IRQENTRY_TEXT
+                *(.fixup)
+                *(.gnu.warning)
+                /* End of text section */
+                _etext = .;
+        } :text = 0x9090
+
+        NOTES :text :note
+
+        /* Exception table */
+        . = ALIGN(16);
+        __ex_table : AT(ADDR(__ex_table) - LOAD_OFFSET) {
+                __start___ex_table = .;
+                *(__ex_table)
+                __stop___ex_table = .;
+        } :text = 0x9090
+
+        RODATA
+
+        /* Data */
+        . = ALIGN(PAGE_SIZE);
+        .data : AT(ADDR(.data) - LOAD_OFFSET) {
+                DATA_DATA
+                CONSTRUCTORS
+
+#ifdef CONFIG_X86_64
+                /* End of data section */
+                _edata = .;
+#endif
+        } :data
+
+#ifdef CONFIG_X86_32
+        /* 32 bit has nosave before _edata */
+        . = ALIGN(PAGE_SIZE);
+        .data_nosave : AT(ADDR(.data_nosave) - LOAD_OFFSET) {
+                __nosave_begin = .;
+                *(.data.nosave)
+                . = ALIGN(PAGE_SIZE);
+                __nosave_end = .;
+        }
+#endif
+
+        . = ALIGN(PAGE_SIZE);
+        .data.page_aligned : AT(ADDR(.data.page_aligned) - LOAD_OFFSET) {
+                *(.data.page_aligned)
+                *(.data.idt)
+        }
+
+#ifdef CONFIG_X86_32
+        . = ALIGN(32);
+#else
+        . = ALIGN(PAGE_SIZE);
+        . = ALIGN(CONFIG_X86_L1_CACHE_BYTES);
+#endif
+        .data.cacheline_aligned :
+                AT(ADDR(.data.cacheline_aligned) - LOAD_OFFSET) {
+                *(.data.cacheline_aligned)
+        }
+
+        /* rarely changed data like cpu maps */
+#ifdef CONFIG_X86_32
+        . = ALIGN(32);
+#else
+        . = ALIGN(CONFIG_X86_INTERNODE_CACHE_BYTES);
+#endif
+        .data.read_mostly : AT(ADDR(.data.read_mostly) - LOAD_OFFSET) {
+                *(.data.read_mostly)
+
+#ifdef CONFIG_X86_32
+                /* End of data section */
+                _edata = .;
+#endif
+        }
+
+#ifdef CONFIG_X86_64
+
+#define VSYSCALL_ADDR (-10*1024*1024)
+#define VSYSCALL_PHYS_ADDR ((LOADADDR(.data.read_mostly) + \
+                            SIZEOF(.data.read_mostly) + 4095) & ~(4095))
+#define VSYSCALL_VIRT_ADDR ((ADDR(.data.read_mostly) + \
+                            SIZEOF(.data.read_mostly) + 4095) & ~(4095))
+
+#define VLOAD_OFFSET (VSYSCALL_ADDR - VSYSCALL_PHYS_ADDR)
+#define VLOAD(x) (ADDR(x) - VLOAD_OFFSET)
+
+#define VVIRT_OFFSET (VSYSCALL_ADDR - VSYSCALL_VIRT_ADDR)
+#define VVIRT(x) (ADDR(x) - VVIRT_OFFSET)
+
+        . = VSYSCALL_ADDR;
+        .vsyscall_0 : AT(VSYSCALL_PHYS_ADDR) {
+                *(.vsyscall_0)
+        } :user
+
+        __vsyscall_0 = VSYSCALL_VIRT_ADDR;
+
+        . = ALIGN(CONFIG_X86_L1_CACHE_BYTES);
+        .vsyscall_fn : AT(VLOAD(.vsyscall_fn)) {
+                *(.vsyscall_fn)
+        }
+
+        . = ALIGN(CONFIG_X86_L1_CACHE_BYTES);
+        .vsyscall_gtod_data : AT(VLOAD(.vsyscall_gtod_data)) {
+                *(.vsyscall_gtod_data)
+        }
+
+        vsyscall_gtod_data = VVIRT(.vsyscall_gtod_data);
+        .vsyscall_clock : AT(VLOAD(.vsyscall_clock)) {
+                *(.vsyscall_clock)
+        }
+        vsyscall_clock = VVIRT(.vsyscall_clock);
+
+
+        .vsyscall_1 ADDR(.vsyscall_0) + 1024: AT(VLOAD(.vsyscall_1)) {
+                *(.vsyscall_1)
+        }
+        .vsyscall_2 ADDR(.vsyscall_0) + 2048: AT(VLOAD(.vsyscall_2)) {
+                *(.vsyscall_2)
+        }
+
+        .vgetcpu_mode : AT(VLOAD(.vgetcpu_mode)) {
+                *(.vgetcpu_mode)
+        }
+        vgetcpu_mode = VVIRT(.vgetcpu_mode);
+
+        . = ALIGN(CONFIG_X86_L1_CACHE_BYTES);
+        .jiffies : AT(VLOAD(.jiffies)) {
+                *(.jiffies)
+        }
+        jiffies = VVIRT(.jiffies);
+
+        .vsyscall_3 ADDR(.vsyscall_0) + 3072: AT(VLOAD(.vsyscall_3)) {
+                *(.vsyscall_3)
+        }
+
+        . = VSYSCALL_VIRT_ADDR + PAGE_SIZE;
+
+#undef VSYSCALL_ADDR
+#undef VSYSCALL_PHYS_ADDR
+#undef VSYSCALL_VIRT_ADDR
+#undef VLOAD_OFFSET
+#undef VLOAD
+#undef VVIRT_OFFSET
+#undef VVIRT
+
+#endif /* CONFIG_X86_64 */
+
+        /* init_task */
+        . = ALIGN(THREAD_SIZE);
+        .data.init_task : AT(ADDR(.data.init_task) - LOAD_OFFSET) {
+                *(.data.init_task)
+        }
+#ifdef CONFIG_X86_64
+         :data.init
+#endif
+
+        /*
+         * smp_locks might be freed after init
+         * start/end must be page aligned
+         */
+        . = ALIGN(PAGE_SIZE);
+        .smp_locks : AT(ADDR(.smp_locks) - LOAD_OFFSET) {
+                __smp_locks = .;
+                *(.smp_locks)
+                __smp_locks_end = .;
+                . = ALIGN(PAGE_SIZE);
+        }
+
+        /* Init code and data - will be freed after init */
+        . = ALIGN(PAGE_SIZE);
+        .init.text : AT(ADDR(.init.text) - LOAD_OFFSET) {
+                __init_begin = .; /* paired with __init_end */
+                _sinittext = .;
+                INIT_TEXT
+                _einittext = .;
+        }
+
+        .init.data : AT(ADDR(.init.data) - LOAD_OFFSET) {
+                INIT_DATA
+        }
+
+        . = ALIGN(16);
+        .init.setup : AT(ADDR(.init.setup) - LOAD_OFFSET) {
+                __setup_start = .;
+                *(.init.setup)
+                __setup_end = .;
+        }
+        .initcall.init : AT(ADDR(.initcall.init) - LOAD_OFFSET) {
+                __initcall_start = .;
+                INITCALLS
+                __initcall_end = .;
+        }
+
+        .con_initcall.init : AT(ADDR(.con_initcall.init) - LOAD_OFFSET) {
+                __con_initcall_start = .;
+                *(.con_initcall.init)
+                __con_initcall_end = .;
+        }
+
+        .x86_cpu_dev.init : AT(ADDR(.x86_cpu_dev.init) - LOAD_OFFSET) {
+                __x86_cpu_dev_start = .;
+                *(.x86_cpu_dev.init)
+                __x86_cpu_dev_end = .;
+        }
+
+        SECURITY_INIT
+
+        . = ALIGN(8);
+        .parainstructions : AT(ADDR(.parainstructions) - LOAD_OFFSET) {
+                __parainstructions = .;
+                *(.parainstructions)
+                __parainstructions_end = .;
+        }
+
+        . = ALIGN(8);
+        .altinstructions : AT(ADDR(.altinstructions) - LOAD_OFFSET) {
+                __alt_instructions = .;
+                *(.altinstructions)
+                __alt_instructions_end = .;
+        }
+
+        .altinstr_replacement : AT(ADDR(.altinstr_replacement) - LOAD_OFFSET) {
+                *(.altinstr_replacement)
+        }
+
+        /*
+         * .exit.text is discard at runtime, not link time, to deal with
+         *  references from .altinstructions and .eh_frame
+         */
+        .exit.text : AT(ADDR(.exit.text) - LOAD_OFFSET) {
+                EXIT_TEXT
+        }
+
+        .exit.data : AT(ADDR(.exit.data) - LOAD_OFFSET) {
+                EXIT_DATA
+        }
+
+#ifdef CONFIG_BLK_DEV_INITRD
+        . = ALIGN(PAGE_SIZE);
+        .init.ramfs : AT(ADDR(.init.ramfs) - LOAD_OFFSET) {
+                __initramfs_start = .;
+                *(.init.ramfs)
+                __initramfs_end = .;
+        }
+#endif
+
+#if defined(CONFIG_X86_64) && defined(CONFIG_SMP)
+        /*
+         * percpu offsets are zero-based on SMP.  PERCPU_VADDR() changes the
+         * output PHDR, so the next output section - __data_nosave - should
+         * start another section data.init2.  Also, pda should be at the head of
+         * percpu area.  Preallocate it and define the percpu offset symbol
+         * so that it can be accessed as a percpu variable.
+         */
+        . = ALIGN(PAGE_SIZE);
+        PERCPU_VADDR(0, :percpu)
+#else
+        PERCPU(PAGE_SIZE)
+#endif
+
+        . = ALIGN(PAGE_SIZE);
+
+        /* freed after init ends here */
+        .init.end : AT(ADDR(.init.end) - LOAD_OFFSET) {
+                __init_end = .;
+        }
+
+#ifdef CONFIG_X86_64
+        .data_nosave : AT(ADDR(.data_nosave) - LOAD_OFFSET) {
+                . = ALIGN(PAGE_SIZE);
+                __nosave_begin = .;
+                *(.data.nosave)
+                . = ALIGN(PAGE_SIZE);
+                __nosave_end = .;
+        } :data.init2
+        /* use another section data.init2, see PERCPU_VADDR() above */
+#endif
+
+        /* BSS */
+        . = ALIGN(PAGE_SIZE);
+        .bss : AT(ADDR(.bss) - LOAD_OFFSET) {
+                __bss_start = .;
+                *(.bss.page_aligned)
+                *(.bss)
+                . = ALIGN(4);
+                __bss_stop = .;
+        }
+
+        . = ALIGN(PAGE_SIZE);
+        .brk : AT(ADDR(.brk) - LOAD_OFFSET) {
+                __brk_base = .;
+                . += 64 * 1024;         /* 64k alignment slop space */
+                *(.brk_reservation)     /* areas brk users have reserved */
+                __brk_limit = .;
+        }
+
+        .end : AT(ADDR(.end) - LOAD_OFFSET) {
+                _end = .;
+        }
+
+        /* Sections to be discarded */
+        /DISCARD/ : {
+                *(.exitcall.exit)
+                *(.eh_frame)
+                *(.discard)
+        }
+
+        STABS_DEBUG
+        DWARF_DEBUG
+}
+
+
+#ifdef CONFIG_X86_32
+ASSERT((_end - LOAD_OFFSET <= KERNEL_IMAGE_SIZE),
+        "kernel image bigger than KERNEL_IMAGE_SIZE")
+#else
+/*
+ * Per-cpu symbols which need to be offset from __per_cpu_load
+ * for the boot processor.
+ */
+#define INIT_PER_CPU(x) init_per_cpu__##x = per_cpu__##x + __per_cpu_load
+INIT_PER_CPU(gdt_page);
+INIT_PER_CPU(irq_stack_union);
+
+/*
+ * Build-time check on the image size:
+ */
+ASSERT((_end - _text <= KERNEL_IMAGE_SIZE),
+        "kernel image bigger than KERNEL_IMAGE_SIZE")
+
+#ifdef CONFIG_SMP
+ASSERT((per_cpu__irq_stack_union == 0),
+        "irq_stack_union is not at start of per-cpu area");
+#endif
+
+#endif /* CONFIG_X86_32 */
+
+#ifdef CONFIG_KEXEC
+#include <asm/kexec.h>
+
+ASSERT(kexec_control_code_size <= KEXEC_CONTROL_CODE_MAX_SIZE,
+       "kexec control code size is too big")
+#endif
+
diff --git a/arch/x86/kernel/vmlinux_32.lds.S b/arch/x86/kernel/vmlinux_32.lds.S
deleted file mode 100644
index 62ad500d55f3..000000000000
--- a/arch/x86/kernel/vmlinux_32.lds.S
+++ /dev/null
@@ -1,229 +0,0 @@
-/* ld script to make i386 Linux kernel
- * Written by Martin Mares <mj@atrey.karlin.mff.cuni.cz>;
- *
- * Don't define absolute symbols until and unless you know that symbol
- * value is should remain constant even if kernel image is relocated
- * at run time. Absolute symbols are not relocated. If symbol value should
- * change if kernel is relocated, make the symbol section relative and
- * put it inside the section definition.
- */
-
-#define LOAD_OFFSET __PAGE_OFFSET
-
-#include <asm-generic/vmlinux.lds.h>
-#include <asm/thread_info.h>
-#include <asm/page_types.h>
-#include <asm/cache.h>
-#include <asm/boot.h>
-
-OUTPUT_FORMAT("elf32-i386", "elf32-i386", "elf32-i386")
-OUTPUT_ARCH(i386)
-ENTRY(phys_startup_32)
-jiffies = jiffies_64;
-
-PHDRS {
-        text PT_LOAD FLAGS(5);  /* R_E */
-        data PT_LOAD FLAGS(7);  /* RWE */
-        note PT_NOTE FLAGS(0);  /* ___ */
-}
-SECTIONS
-{
-  . = LOAD_OFFSET + LOAD_PHYSICAL_ADDR;
-  phys_startup_32 = startup_32 - LOAD_OFFSET;
-
-  .text.head : AT(ADDR(.text.head) - LOAD_OFFSET) {
-        _text = .;                      /* Text and read-only data */
-        *(.text.head)
-  } :text = 0x9090
-
-  /* read-only */
-  .text : AT(ADDR(.text) - LOAD_OFFSET) {
-        . = ALIGN(PAGE_SIZE); /* not really needed, already page aligned */
-        *(.text.page_aligned)
-        TEXT_TEXT
-        SCHED_TEXT
-        LOCK_TEXT
-        KPROBES_TEXT
-        IRQENTRY_TEXT
-        *(.fixup)
-        *(.gnu.warning)
-        _etext = .;                     /* End of text section */
-  } :text = 0x9090
-
-  NOTES :text :note
-
-  . = ALIGN(16);                /* Exception table */
-  __ex_table : AT(ADDR(__ex_table) - LOAD_OFFSET) {
-        __start___ex_table = .;
-         *(__ex_table)
-        __stop___ex_table = .;
-  } :text = 0x9090
-
-  RODATA
-
-  /* writeable */
-  . = ALIGN(PAGE_SIZE);
-  .data : AT(ADDR(.data) - LOAD_OFFSET) {       /* Data */
-        DATA_DATA
-        CONSTRUCTORS
-        } :data
-
-  . = ALIGN(PAGE_SIZE);
-  .data_nosave : AT(ADDR(.data_nosave) - LOAD_OFFSET) {
-        __nosave_begin = .;
-        *(.data.nosave)
-        . = ALIGN(PAGE_SIZE);
-        __nosave_end = .;
-  }
-
-  . = ALIGN(PAGE_SIZE);
-  .data.page_aligned : AT(ADDR(.data.page_aligned) - LOAD_OFFSET) {
-        *(.data.page_aligned)
-        *(.data.idt)
-  }
-
-  . = ALIGN(32);
-  .data.cacheline_aligned : AT(ADDR(.data.cacheline_aligned) - LOAD_OFFSET) {
-        *(.data.cacheline_aligned)
-  }
-
-  /* rarely changed data like cpu maps */
-  . = ALIGN(32);
-  .data.read_mostly : AT(ADDR(.data.read_mostly) - LOAD_OFFSET) {
-        *(.data.read_mostly)
-        _edata = .;             /* End of data section */
-  }
-
-  . = ALIGN(THREAD_SIZE);       /* init_task */
-  .data.init_task : AT(ADDR(.data.init_task) - LOAD_OFFSET) {
-        *(.data.init_task)
-  }
-
-  /* might get freed after init */
-  . = ALIGN(PAGE_SIZE);
-  .smp_locks : AT(ADDR(.smp_locks) - LOAD_OFFSET) {
-        __smp_locks = .;
-        *(.smp_locks)
-        __smp_locks_end = .;
-  }
-  /* will be freed after init
-   * Following ALIGN() is required to make sure no other data falls on the
-   * same page where __smp_alt_end is pointing as that page might be freed
-   * after boot. Always make sure that ALIGN() directive is present after
-   * the section which contains __smp_alt_end.
-   */
-  . = ALIGN(PAGE_SIZE);
-
-  /* will be freed after init */
-  . = ALIGN(PAGE_SIZE);         /* Init code and data */
-  .init.text : AT(ADDR(.init.text) - LOAD_OFFSET) {
-        __init_begin = .;
-        _sinittext = .;
-        INIT_TEXT
-        _einittext = .;
-  }
-  .init.data : AT(ADDR(.init.data) - LOAD_OFFSET) {
-        INIT_DATA
-  }
-  . = ALIGN(16);
-  .init.setup : AT(ADDR(.init.setup) - LOAD_OFFSET) {
-        __setup_start = .;
-        *(.init.setup)
-        __setup_end = .;
-   }
-  .initcall.init : AT(ADDR(.initcall.init) - LOAD_OFFSET) {
-        __initcall_start = .;
-        INITCALLS
-        __initcall_end = .;
-  }
-  .con_initcall.init : AT(ADDR(.con_initcall.init) - LOAD_OFFSET) {
-        __con_initcall_start = .;
-        *(.con_initcall.init)
-        __con_initcall_end = .;
-  }
-  .x86_cpu_dev.init : AT(ADDR(.x86_cpu_dev.init) - LOAD_OFFSET) {
-        __x86_cpu_dev_start = .;
-        *(.x86_cpu_dev.init)
-        __x86_cpu_dev_end = .;
-  }
-  SECURITY_INIT
-  . = ALIGN(4);
-  .altinstructions : AT(ADDR(.altinstructions) - LOAD_OFFSET) {
-        __alt_instructions = .;
-        *(.altinstructions)
-        __alt_instructions_end = .;
-  }
-  .altinstr_replacement : AT(ADDR(.altinstr_replacement) - LOAD_OFFSET) {
-        *(.altinstr_replacement)
-  }
-  . = ALIGN(4);
-  .parainstructions : AT(ADDR(.parainstructions) - LOAD_OFFSET) {
-        __parainstructions = .;
-        *(.parainstructions)
-        __parainstructions_end = .;
-  }
-  /* .exit.text is discard at runtime, not link time, to deal with references
-     from .altinstructions and .eh_frame */
-  .exit.text : AT(ADDR(.exit.text) - LOAD_OFFSET) {
-        EXIT_TEXT
-  }
-  .exit.data : AT(ADDR(.exit.data) - LOAD_OFFSET) {
-        EXIT_DATA
-  }
-#if defined(CONFIG_BLK_DEV_INITRD)
-  . = ALIGN(PAGE_SIZE);
-  .init.ramfs : AT(ADDR(.init.ramfs) - LOAD_OFFSET) {
-        __initramfs_start = .;
-        *(.init.ramfs)
-        __initramfs_end = .;
-  }
-#endif
-  PERCPU(PAGE_SIZE)
-  . = ALIGN(PAGE_SIZE);
-  /* freed after init ends here */
-
-  .bss : AT(ADDR(.bss) - LOAD_OFFSET) {
-        __init_end = .;
-        __bss_start = .;                /* BSS */
-        *(.bss.page_aligned)
-        *(.bss)
-        . = ALIGN(4);
-        __bss_stop = .;
-  }
-
-  .brk : AT(ADDR(.brk) - LOAD_OFFSET) {
-        . = ALIGN(PAGE_SIZE);
-        __brk_base = . ;
-        . += 64 * 1024 ;        /* 64k alignment slop space */
-        *(.brk_reservation)     /* areas brk users have reserved */
-        __brk_limit = . ;
-  }
-
-  .end : AT(ADDR(.end) - LOAD_OFFSET) {
-        _end = . ;
-  }
-
-  /* Sections to be discarded */
-  /DISCARD/ : {
-        *(.exitcall.exit)
-        *(.discard)
-        }
-
-  STABS_DEBUG
-
-  DWARF_DEBUG
-}
-
-/*
- * Build-time check on the image size:
- */
-ASSERT((_end - LOAD_OFFSET <= KERNEL_IMAGE_SIZE),
-        "kernel image bigger than KERNEL_IMAGE_SIZE")
-
-#ifdef CONFIG_KEXEC
-/* Link time checks */
-#include <asm/kexec.h>
-
-ASSERT(kexec_control_code_size <= KEXEC_CONTROL_CODE_MAX_SIZE,
-       "kexec control code size is too big")
-#endif
diff --git a/arch/x86/kernel/vmlinux_64.lds.S b/arch/x86/kernel/vmlinux_64.lds.S
deleted file mode 100644
index c8742507b030..000000000000
--- a/arch/x86/kernel/vmlinux_64.lds.S
+++ /dev/null
@@ -1,298 +0,0 @@
-/* ld script to make x86-64 Linux kernel
- * Written by Martin Mares <mj@atrey.karlin.mff.cuni.cz>;
- */
-
-#define LOAD_OFFSET __START_KERNEL_map
-
-#include <asm-generic/vmlinux.lds.h>
-#include <asm/asm-offsets.h>
-#include <asm/page_types.h>
-
-#undef i386     /* in case the preprocessor is a 32bit one */
-
-OUTPUT_FORMAT("elf64-x86-64", "elf64-x86-64", "elf64-x86-64")
-OUTPUT_ARCH(i386:x86-64)
-ENTRY(phys_startup_64)
-jiffies_64 = jiffies;
-PHDRS {
-        text PT_LOAD FLAGS(5);  /* R_E */
-        data PT_LOAD FLAGS(7);  /* RWE */
-        user PT_LOAD FLAGS(7);  /* RWE */
-        data.init PT_LOAD FLAGS(7);     /* RWE */
-#ifdef CONFIG_SMP
-        percpu PT_LOAD FLAGS(7);        /* RWE */
-#endif
-        data.init2 PT_LOAD FLAGS(7);    /* RWE */
-        note PT_NOTE FLAGS(0);  /* ___ */
-}
-SECTIONS
-{
-  . = __START_KERNEL;
-  phys_startup_64 = startup_64 - LOAD_OFFSET;
-  .text :  AT(ADDR(.text) - LOAD_OFFSET) {
-        _text = .;                      /* Text and read-only data */
-        /* First the code that has to be first for bootstrapping */
-        *(.text.head)
-        _stext = .;
-        /* Then the rest */
-        TEXT_TEXT
-        SCHED_TEXT
-        LOCK_TEXT
-        KPROBES_TEXT
-        IRQENTRY_TEXT
-        *(.fixup)
-        *(.gnu.warning)
-        _etext = .;             /* End of text section */
-  } :text = 0x9090
-
-  NOTES :text :note
-
-  . = ALIGN(16);                /* Exception table */
-  __ex_table : AT(ADDR(__ex_table) - LOAD_OFFSET) {
-        __start___ex_table = .;
-         *(__ex_table)
-        __stop___ex_table = .;
-  } :text = 0x9090
-
-  RODATA
-
-  . = ALIGN(PAGE_SIZE);         /* Align data segment to page size boundary */
-                                /* Data */
-  .data : AT(ADDR(.data) - LOAD_OFFSET) {
-        DATA_DATA
-        CONSTRUCTORS
-        _edata = .;                     /* End of data section */
-        } :data
-
-
-  .data.cacheline_aligned : AT(ADDR(.data.cacheline_aligned) - LOAD_OFFSET) {
-        . = ALIGN(PAGE_SIZE);
-        . = ALIGN(CONFIG_X86_L1_CACHE_BYTES);
-        *(.data.cacheline_aligned)
-  }
-  . = ALIGN(CONFIG_X86_INTERNODE_CACHE_BYTES);
-  .data.read_mostly : AT(ADDR(.data.read_mostly) - LOAD_OFFSET) {
-        *(.data.read_mostly)
-  }
-
-#define VSYSCALL_ADDR (-10*1024*1024)
-#define VSYSCALL_PHYS_ADDR ((LOADADDR(.data.read_mostly) + SIZEOF(.data.read_mostly) + 4095) & ~(4095))
-#define VSYSCALL_VIRT_ADDR ((ADDR(.data.read_mostly) + SIZEOF(.data.read_mostly) + 4095) & ~(4095))
-
-#define VLOAD_OFFSET (VSYSCALL_ADDR - VSYSCALL_PHYS_ADDR)
-#define VLOAD(x) (ADDR(x) - VLOAD_OFFSET)
-
-#define VVIRT_OFFSET (VSYSCALL_ADDR - VSYSCALL_VIRT_ADDR)
-#define VVIRT(x) (ADDR(x) - VVIRT_OFFSET)
-
-  . = VSYSCALL_ADDR;
-  .vsyscall_0 :  AT(VSYSCALL_PHYS_ADDR) { *(.vsyscall_0) } :user
-  __vsyscall_0 = VSYSCALL_VIRT_ADDR;
-
-  . = ALIGN(CONFIG_X86_L1_CACHE_BYTES);
-  .vsyscall_fn : AT(VLOAD(.vsyscall_fn)) { *(.vsyscall_fn) }
-  . = ALIGN(CONFIG_X86_L1_CACHE_BYTES);
-  .vsyscall_gtod_data : AT(VLOAD(.vsyscall_gtod_data))
-                { *(.vsyscall_gtod_data) }
-  vsyscall_gtod_data = VVIRT(.vsyscall_gtod_data);
-  .vsyscall_clock : AT(VLOAD(.vsyscall_clock))
-                { *(.vsyscall_clock) }
-  vsyscall_clock = VVIRT(.vsyscall_clock);
-
-
-  .vsyscall_1 ADDR(.vsyscall_0) + 1024: AT(VLOAD(.vsyscall_1))
-                { *(.vsyscall_1) }
-  .vsyscall_2 ADDR(.vsyscall_0) + 2048: AT(VLOAD(.vsyscall_2))
-                { *(.vsyscall_2) }
-
-  .vgetcpu_mode : AT(VLOAD(.vgetcpu_mode)) { *(.vgetcpu_mode) }
-  vgetcpu_mode = VVIRT(.vgetcpu_mode);
-
-  . = ALIGN(CONFIG_X86_L1_CACHE_BYTES);
-  .jiffies : AT(VLOAD(.jiffies)) { *(.jiffies) }
-  jiffies = VVIRT(.jiffies);
-
-  .vsyscall_3 ADDR(.vsyscall_0) + 3072: AT(VLOAD(.vsyscall_3))
-                { *(.vsyscall_3) }
-
-  . = VSYSCALL_VIRT_ADDR + PAGE_SIZE;
-
-#undef VSYSCALL_ADDR
-#undef VSYSCALL_PHYS_ADDR
-#undef VSYSCALL_VIRT_ADDR
-#undef VLOAD_OFFSET
-#undef VLOAD
-#undef VVIRT_OFFSET
-#undef VVIRT
-
-  .data.init_task : AT(ADDR(.data.init_task) - LOAD_OFFSET) {
-        . = ALIGN(THREAD_SIZE); /* init_task */
-        *(.data.init_task)
-  }:data.init
-
-  .data.page_aligned : AT(ADDR(.data.page_aligned) - LOAD_OFFSET) {
-        . = ALIGN(PAGE_SIZE);
-        *(.data.page_aligned)
-  }
-
-  .smp_locks : AT(ADDR(.smp_locks) - LOAD_OFFSET) {
-        /* might get freed after init */
-        . = ALIGN(PAGE_SIZE);
-        __smp_alt_begin = .;
-        __smp_locks = .;
-        *(.smp_locks)
-        __smp_locks_end = .;
-        . = ALIGN(PAGE_SIZE);
-        __smp_alt_end = .;
-  }
-
-  . = ALIGN(PAGE_SIZE);         /* Init code and data */
-  __init_begin = .;     /* paired with __init_end */
-  .init.text : AT(ADDR(.init.text) - LOAD_OFFSET) {
-        _sinittext = .;
-        INIT_TEXT
-        _einittext = .;
-  }
-  .init.data : AT(ADDR(.init.data) - LOAD_OFFSET) {
-        __initdata_begin = .;
-        INIT_DATA
-        __initdata_end = .;
-   }
-
-  .init.setup : AT(ADDR(.init.setup) - LOAD_OFFSET) {
-        . = ALIGN(16);
-        __setup_start = .;
-        *(.init.setup)
-        __setup_end = .;
-  }
-  .initcall.init : AT(ADDR(.initcall.init) - LOAD_OFFSET) {
-        __initcall_start = .;
-        INITCALLS
-        __initcall_end = .;
-  }
-  .con_initcall.init : AT(ADDR(.con_initcall.init) - LOAD_OFFSET) {
-        __con_initcall_start = .;
-        *(.con_initcall.init)
-        __con_initcall_end = .;
-  }
-  .x86_cpu_dev.init : AT(ADDR(.x86_cpu_dev.init) - LOAD_OFFSET) {
-        __x86_cpu_dev_start = .;
-        *(.x86_cpu_dev.init)
-        __x86_cpu_dev_end = .;
-  }
-  SECURITY_INIT
-
-  . = ALIGN(8);
-  .parainstructions : AT(ADDR(.parainstructions) - LOAD_OFFSET) {
-        __parainstructions = .;
-       *(.parainstructions)
-        __parainstructions_end = .;
-  }
-
-  .altinstructions : AT(ADDR(.altinstructions) - LOAD_OFFSET) {
-        . = ALIGN(8);
-        __alt_instructions = .;
-        *(.altinstructions)
-        __alt_instructions_end = .;
-  }
-  .altinstr_replacement : AT(ADDR(.altinstr_replacement) - LOAD_OFFSET) {
-        *(.altinstr_replacement)
-  }
-  /* .exit.text is discard at runtime, not link time, to deal with references
-     from .altinstructions and .eh_frame */
-  .exit.text : AT(ADDR(.exit.text) - LOAD_OFFSET) {
-        EXIT_TEXT
-  }
-  .exit.data : AT(ADDR(.exit.data) - LOAD_OFFSET) {
-        EXIT_DATA
-  }
-
-#ifdef CONFIG_BLK_DEV_INITRD
-  . = ALIGN(PAGE_SIZE);
-  .init.ramfs : AT(ADDR(.init.ramfs) - LOAD_OFFSET) {
-        __initramfs_start = .;
-        *(.init.ramfs)
-        __initramfs_end = .;
-  }
-#endif
-
-#ifdef CONFIG_SMP
-  /*
-   * percpu offsets are zero-based on SMP.  PERCPU_VADDR() changes the
-   * output PHDR, so the next output section - __data_nosave - should
-   * start another section data.init2.  Also, pda should be at the head of
-   * percpu area.  Preallocate it and define the percpu offset symbol
-   * so that it can be accessed as a percpu variable.
-   */
-  . = ALIGN(PAGE_SIZE);
-  PERCPU_VADDR(0, :percpu)
-#else
-  PERCPU(PAGE_SIZE)
-#endif
-
-  . = ALIGN(PAGE_SIZE);
-  __init_end = .;
-
-  .data_nosave : AT(ADDR(.data_nosave) - LOAD_OFFSET) {
-        . = ALIGN(PAGE_SIZE);
-        __nosave_begin = .;
-        *(.data.nosave)
-        . = ALIGN(PAGE_SIZE);
-        __nosave_end = .;
-  } :data.init2 /* use another section data.init2, see PERCPU_VADDR() above */
-
-  .bss : AT(ADDR(.bss) - LOAD_OFFSET) {
-        . = ALIGN(PAGE_SIZE);
-        __bss_start = .;                /* BSS */
-        *(.bss.page_aligned)
-        *(.bss)
-        __bss_stop = .;
-  }
-
-  .brk : AT(ADDR(.brk) - LOAD_OFFSET) {
-        . = ALIGN(PAGE_SIZE);
-        __brk_base = . ;
-        . += 64 * 1024 ;        /* 64k alignment slop space */
-        *(.brk_reservation)     /* areas brk users have reserved */
-        __brk_limit = . ;
-  }
-
-  _end = . ;
-
-  /* Sections to be discarded */
-  /DISCARD/ : {
-        *(.exitcall.exit)
-        *(.eh_frame)
-        *(.discard)
-        }
-
-  STABS_DEBUG
-
-  DWARF_DEBUG
-}
-
- /*
-  * Per-cpu symbols which need to be offset from __per_cpu_load
-  * for the boot processor.
-  */
-#define INIT_PER_CPU(x) init_per_cpu__##x = per_cpu__##x + __per_cpu_load
-INIT_PER_CPU(gdt_page);
-INIT_PER_CPU(irq_stack_union);
-
-/*
- * Build-time check on the image size:
- */
-ASSERT((_end - _text <= KERNEL_IMAGE_SIZE),
-        "kernel image bigger than KERNEL_IMAGE_SIZE")
-
-#ifdef CONFIG_SMP
-ASSERT((per_cpu__irq_stack_union == 0),
-        "irq_stack_union is not at start of per-cpu area");
-#endif
-
-#ifdef CONFIG_KEXEC
-#include <asm/kexec.h>
-
-ASSERT(kexec_control_code_size <= KEXEC_CONTROL_CODE_MAX_SIZE,
-       "kexec control code size is too big")
-#endif
diff --git a/arch/x86/kernel/vsyscall_64.c b/arch/x86/kernel/vsyscall_64.c
index 44153afc9067..25ee06a80aad 100644
--- a/arch/x86/kernel/vsyscall_64.c
+++ b/arch/x86/kernel/vsyscall_64.c
@@ -132,15 +132,7 @@ static __always_inline void do_vgettimeofday(struct timeval * tv)
                         return;
                 }
 
-                /*
-                 * Surround the RDTSC by barriers, to make sure it's not
-                 * speculated to outside the seqlock critical section and
-                 * does not cause time warps:
-                 */
-                rdtsc_barrier();
                 now = vread();
-                rdtsc_barrier();
-
                 base = __vsyscall_gtod_data.clock.cycle_last;
                 mask = __vsyscall_gtod_data.clock.mask;
                 mult = __vsyscall_gtod_data.clock.mult;
diff --git a/arch/x86/kvm/Kconfig b/arch/x86/kvm/Kconfig
index a58504ea78cc..8600a09e0c6c 100644
--- a/arch/x86/kvm/Kconfig
+++ b/arch/x86/kvm/Kconfig
@@ -50,6 +50,9 @@ config KVM_INTEL
           Provides support for KVM on Intel processors equipped with the VT
           extensions.
 
+          To compile this as a module, choose M here: the module
+          will be called kvm-intel.
+
 config KVM_AMD
         tristate "KVM for AMD processors support"
         depends on KVM
@@ -57,6 +60,9 @@ config KVM_AMD
           Provides support for KVM on AMD processors equipped with the AMD-V
           (SVM) extensions.
 
+          To compile this as a module, choose M here: the module
+          will be called kvm-amd.
+
 config KVM_TRACE
         bool "KVM trace support"
         depends on KVM && SYSFS
diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile
index d3ec292f00f2..b43c4efafe80 100644
--- a/arch/x86/kvm/Makefile
+++ b/arch/x86/kvm/Makefile
@@ -14,7 +14,7 @@ endif
 EXTRA_CFLAGS += -Ivirt/kvm -Iarch/x86/kvm
 
 kvm-objs := $(common-objs) x86.o mmu.o x86_emulate.o i8259.o irq.o lapic.o \
-        i8254.o
+        i8254.o timer.o
 obj-$(CONFIG_KVM) += kvm.o
 kvm-intel-objs = vmx.o
 obj-$(CONFIG_KVM_INTEL) += kvm-intel.o
diff --git a/arch/x86/kvm/i8254.c b/arch/x86/kvm/i8254.c
index c13bb92d3157..4d6f0d293ee2 100644
--- a/arch/x86/kvm/i8254.c
+++ b/arch/x86/kvm/i8254.c
@@ -98,6 +98,37 @@ static int pit_get_gate(struct kvm *kvm, int channel)
         return kvm->arch.vpit->pit_state.channels[channel].gate;
 }
 
+static s64 __kpit_elapsed(struct kvm *kvm)
+{
+        s64 elapsed;
+        ktime_t remaining;
+        struct kvm_kpit_state *ps = &kvm->arch.vpit->pit_state;
+
+        /*
+         * The Counter does not stop when it reaches zero. In
+         * Modes 0, 1, 4, and 5 the Counter ``wraps around'' to
+         * the highest count, either FFFF hex for binary counting
+         * or 9999 for BCD counting, and continues counting.
+         * Modes 2 and 3 are periodic; the Counter reloads
+         * itself with the initial count and continues counting
+         * from there.
+         */
+        remaining = hrtimer_expires_remaining(&ps->pit_timer.timer);
+        elapsed = ps->pit_timer.period - ktime_to_ns(remaining);
+        elapsed = mod_64(elapsed, ps->pit_timer.period);
+
+        return elapsed;
+}
+
+static s64 kpit_elapsed(struct kvm *kvm, struct kvm_kpit_channel_state *c,
+                        int channel)
+{
+        if (channel == 0)
+                return __kpit_elapsed(kvm);
+
+        return ktime_to_ns(ktime_sub(ktime_get(), c->count_load_time));
+}
+
 static int pit_get_count(struct kvm *kvm, int channel)
 {
         struct kvm_kpit_channel_state *c =
@@ -107,7 +138,7 @@ static int pit_get_count(struct kvm *kvm, int channel)
 
         WARN_ON(!mutex_is_locked(&kvm->arch.vpit->pit_state.lock));
 
-        t = ktime_to_ns(ktime_sub(ktime_get(), c->count_load_time));
+        t = kpit_elapsed(kvm, c, channel);
         d = muldiv64(t, KVM_PIT_FREQ, NSEC_PER_SEC);
 
         switch (c->mode) {
@@ -137,7 +168,7 @@ static int pit_get_out(struct kvm *kvm, int channel)
 
         WARN_ON(!mutex_is_locked(&kvm->arch.vpit->pit_state.lock));
 
-        t = ktime_to_ns(ktime_sub(ktime_get(), c->count_load_time));
+        t = kpit_elapsed(kvm, c, channel);
         d = muldiv64(t, KVM_PIT_FREQ, NSEC_PER_SEC);
 
         switch (c->mode) {
@@ -193,28 +224,6 @@ static void pit_latch_status(struct kvm *kvm, int channel)
         }
 }
 
-static int __pit_timer_fn(struct kvm_kpit_state *ps)
-{
-        struct kvm_vcpu *vcpu0 = ps->pit->kvm->vcpus[0];
-        struct kvm_kpit_timer *pt = &ps->pit_timer;
-
-        if (!atomic_inc_and_test(&pt->pending))
-                set_bit(KVM_REQ_PENDING_TIMER, &vcpu0->requests);
-
-        if (!pt->reinject)
-                atomic_set(&pt->pending, 1);
-
-        if (vcpu0 && waitqueue_active(&vcpu0->wq))
-                wake_up_interruptible(&vcpu0->wq);
-
-        hrtimer_add_expires_ns(&pt->timer, pt->period);
-        pt->scheduled = hrtimer_get_expires_ns(&pt->timer);
-        if (pt->period)
-                ps->channels[0].count_load_time = ktime_get();
-
-        return (pt->period == 0 ? 0 : 1);
-}
-
 int pit_has_pending_timer(struct kvm_vcpu *vcpu)
 {
         struct kvm_pit *pit = vcpu->kvm->arch.vpit;
@@ -235,21 +244,6 @@ static void kvm_pit_ack_irq(struct kvm_irq_ack_notifier *kian)
         spin_unlock(&ps->inject_lock);
 }
 
-static enum hrtimer_restart pit_timer_fn(struct hrtimer *data)
-{
-        struct kvm_kpit_state *ps;
-        int restart_timer = 0;
-
-        ps = container_of(data, struct kvm_kpit_state, pit_timer.timer);
-
-        restart_timer = __pit_timer_fn(ps);
-
-        if (restart_timer)
-                return HRTIMER_RESTART;
-        else
-                return HRTIMER_NORESTART;
-}
-
 void __kvm_migrate_pit_timer(struct kvm_vcpu *vcpu)
 {
         struct kvm_pit *pit = vcpu->kvm->arch.vpit;
@@ -263,15 +257,26 @@ void __kvm_migrate_pit_timer(struct kvm_vcpu *vcpu)
                 hrtimer_start_expires(timer, HRTIMER_MODE_ABS);
 }
 
-static void destroy_pit_timer(struct kvm_kpit_timer *pt)
+static void destroy_pit_timer(struct kvm_timer *pt)
 {
         pr_debug("pit: execute del timer!\n");
         hrtimer_cancel(&pt->timer);
 }
 
+static bool kpit_is_periodic(struct kvm_timer *ktimer)
+{
+        struct kvm_kpit_state *ps = container_of(ktimer, struct kvm_kpit_state,
+                                                 pit_timer);
+        return ps->is_periodic;
+}
+
+static struct kvm_timer_ops kpit_ops = {
+        .is_periodic = kpit_is_periodic,
+};
+
 static void create_pit_timer(struct kvm_kpit_state *ps, u32 val, int is_period)
 {
-        struct kvm_kpit_timer *pt = &ps->pit_timer;
+        struct kvm_timer *pt = &ps->pit_timer;
         s64 interval;
 
         interval = muldiv64(val, NSEC_PER_SEC, KVM_PIT_FREQ);
@@ -280,8 +285,14 @@ static void create_pit_timer(struct kvm_kpit_state *ps, u32 val, int is_period)
 
         /* TODO The new value only affected after the retriggered */
         hrtimer_cancel(&pt->timer);
-        pt->period = (is_period == 0) ? 0 : interval;
-        pt->timer.function = pit_timer_fn;
+        pt->period = interval;
+        ps->is_periodic = is_period;
+
+        pt->timer.function = kvm_timer_fn;
+        pt->t_ops = &kpit_ops;
+        pt->kvm = ps->pit->kvm;
+        pt->vcpu_id = 0;
+
         atomic_set(&pt->pending, 0);
         ps->irq_ack = 1;
 
@@ -298,23 +309,23 @@ static void pit_load_count(struct kvm *kvm, int channel, u32 val)
         pr_debug("pit: load_count val is %d, channel is %d\n", val, channel);
 
         /*
-         * Though spec said the state of 8254 is undefined after power-up,
-         * seems some tricky OS like Windows XP depends on IRQ0 interrupt
-         * when booting up.
-         * So here setting initialize rate for it, and not a specific number
+         * The largest possible initial count is 0; this is equivalent
+         * to 216 for binary counting and 104 for BCD counting.
          */
         if (val == 0)
                 val = 0x10000;
 
-        ps->channels[channel].count_load_time = ktime_get();
         ps->channels[channel].count = val;
 
-        if (channel != 0)
+        if (channel != 0) {
+                ps->channels[channel].count_load_time = ktime_get();
                 return;
+        }
 
         /* Two types of timer
          * mode 1 is one shot, mode 2 is period, otherwise del timer */
         switch (ps->channels[0].mode) {
+        case 0:
         case 1:
         /* FIXME: enhance mode 4 precision */
         case 4:
diff --git a/arch/x86/kvm/i8254.h b/arch/x86/kvm/i8254.h
index 6acbe4b505d5..bbd863ff60b7 100644
--- a/arch/x86/kvm/i8254.h
+++ b/arch/x86/kvm/i8254.h
@@ -3,15 +3,6 @@
 
 #include "iodev.h"
 
-struct kvm_kpit_timer {
-        struct hrtimer timer;
-        int irq;
-        s64 period; /* unit: ns */
-        s64 scheduled;
-        atomic_t pending;
-        bool reinject;
-};
-
 struct kvm_kpit_channel_state {
         u32 count; /* can be 65536 */
         u16 latched_count;
@@ -30,7 +21,8 @@ struct kvm_kpit_channel_state {
 
 struct kvm_kpit_state {
         struct kvm_kpit_channel_state channels[3];
-        struct kvm_kpit_timer pit_timer;
+        struct kvm_timer pit_timer;
+        bool is_periodic;
         u32    speaker_data_on;
         struct mutex lock;
         struct kvm_pit *pit;
diff --git a/arch/x86/kvm/irq.c b/arch/x86/kvm/irq.c
index cf17ed52f6fb..96dfbb6ad2a9 100644
--- a/arch/x86/kvm/irq.c
+++ b/arch/x86/kvm/irq.c
@@ -24,6 +24,7 @@
 
 #include "irq.h"
 #include "i8254.h"
+#include "x86.h"
 
 /*
  * check if there are pending timer events
@@ -48,6 +49,9 @@ int kvm_cpu_has_interrupt(struct kvm_vcpu *v)
 {
         struct kvm_pic *s;
 
+        if (!irqchip_in_kernel(v->kvm))
+                return v->arch.interrupt.pending;
+
         if (kvm_apic_has_interrupt(v) == -1) {  /* LAPIC */
                 if (kvm_apic_accept_pic_intr(v)) {
                         s = pic_irqchip(v->kvm);        /* PIC */
@@ -67,6 +71,9 @@ int kvm_cpu_get_interrupt(struct kvm_vcpu *v)
         struct kvm_pic *s;
         int vector;
 
+        if (!irqchip_in_kernel(v->kvm))
+                return v->arch.interrupt.nr;
+
         vector = kvm_get_apic_interrupt(v);     /* APIC */
         if (vector == -1) {
                 if (kvm_apic_accept_pic_intr(v)) {
diff --git a/arch/x86/kvm/kvm_timer.h b/arch/x86/kvm/kvm_timer.h
new file mode 100644
index 000000000000..26bd6ba74e1c
--- /dev/null
+++ b/arch/x86/kvm/kvm_timer.h
@@ -0,0 +1,18 @@
+
+struct kvm_timer {
+        struct hrtimer timer;
+        s64 period;                             /* unit: ns */
+        atomic_t pending;                       /* accumulated triggered timers */
+        bool reinject;
+        struct kvm_timer_ops *t_ops;
+        struct kvm *kvm;
+        int vcpu_id;
+};
+
+struct kvm_timer_ops {
+        bool (*is_periodic)(struct kvm_timer *);
+};
+
+
+enum hrtimer_restart kvm_timer_fn(struct hrtimer *data);
+
diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c
index f0b67f2cdd69..ae99d83f81a3 100644
--- a/arch/x86/kvm/lapic.c
+++ b/arch/x86/kvm/lapic.c
@@ -196,20 +196,15 @@ int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu)
 }
 EXPORT_SYMBOL_GPL(kvm_lapic_find_highest_irr);
 
-int kvm_apic_set_irq(struct kvm_vcpu *vcpu, u8 vec, u8 trig)
+static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode,
+                             int vector, int level, int trig_mode);
+
+int kvm_apic_set_irq(struct kvm_vcpu *vcpu, struct kvm_lapic_irq *irq)
 {
         struct kvm_lapic *apic = vcpu->arch.apic;
 
-        if (!apic_test_and_set_irr(vec, apic)) {
-                /* a new pending irq is set in IRR */
-                if (trig)
-                        apic_set_vector(vec, apic->regs + APIC_TMR);
-                else
-                        apic_clear_vector(vec, apic->regs + APIC_TMR);
-                kvm_vcpu_kick(apic->vcpu);
-                return 1;
-        }
-        return 0;
+        return __apic_accept_irq(apic, irq->delivery_mode, irq->vector,
+                        irq->level, irq->trig_mode);
 }
 
 static inline int apic_find_highest_isr(struct kvm_lapic *apic)
@@ -250,7 +245,7 @@ static void apic_set_tpr(struct kvm_lapic *apic, u32 tpr)
 
 int kvm_apic_match_physical_addr(struct kvm_lapic *apic, u16 dest)
 {
-        return kvm_apic_id(apic) == dest;
+        return dest == 0xff || kvm_apic_id(apic) == dest;
 }
 
 int kvm_apic_match_logical_addr(struct kvm_lapic *apic, u8 mda)
@@ -279,37 +274,34 @@ int kvm_apic_match_logical_addr(struct kvm_lapic *apic, u8 mda)
         return result;
 }
 
-static int apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source,
+int kvm_apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source,
                            int short_hand, int dest, int dest_mode)
 {
         int result = 0;
         struct kvm_lapic *target = vcpu->arch.apic;
 
         apic_debug("target %p, source %p, dest 0x%x, "
-                   "dest_mode 0x%x, short_hand 0x%x",
+                   "dest_mode 0x%x, short_hand 0x%x\n",
                    target, source, dest, dest_mode, short_hand);
 
         ASSERT(!target);
         switch (short_hand) {
         case APIC_DEST_NOSHORT:
-                if (dest_mode == 0) {
+                if (dest_mode == 0)
                         /* Physical mode. */
-                        if ((dest == 0xFF) || (dest == kvm_apic_id(target)))
-                                result = 1;
-                } else
+                        result = kvm_apic_match_physical_addr(target, dest);
+                else
                         /* Logical mode. */
                         result = kvm_apic_match_logical_addr(target, dest);
                 break;
         case APIC_DEST_SELF:
-                if (target == source)
-                        result = 1;
+                result = (target == source);
                 break;
         case APIC_DEST_ALLINC:
                 result = 1;
                 break;
         case APIC_DEST_ALLBUT:
-                if (target != source)
-                        result = 1;
+                result = (target != source);
                 break;
         default:
                 printk(KERN_WARNING "Bad dest shorthand value %x\n",
@@ -327,20 +319,22 @@ static int apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source,
 static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode,
                              int vector, int level, int trig_mode)
 {
-        int orig_irr, result = 0;
+        int result = 0;
         struct kvm_vcpu *vcpu = apic->vcpu;
 
         switch (delivery_mode) {
-        case APIC_DM_FIXED:
         case APIC_DM_LOWEST:
+                vcpu->arch.apic_arb_prio++;
+        case APIC_DM_FIXED:
                 /* FIXME add logic for vcpu on reset */
                 if (unlikely(!apic_enabled(apic)))
                         break;
 
-                orig_irr = apic_test_and_set_irr(vector, apic);
-                if (orig_irr && trig_mode) {
-                        apic_debug("level trig mode repeatedly for vector %d",
-                                   vector);
+                result = !apic_test_and_set_irr(vector, apic);
+                if (!result) {
+                        if (trig_mode)
+                                apic_debug("level trig mode repeatedly for "
+                                                "vector %d", vector);
                         break;
                 }
 
@@ -349,10 +343,7 @@ static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode,
                         apic_set_vector(vector, apic->regs + APIC_TMR);
                 } else
                         apic_clear_vector(vector, apic->regs + APIC_TMR);
-
                 kvm_vcpu_kick(vcpu);
-
-                result = (orig_irr == 0);
                 break;
 
         case APIC_DM_REMRD:
@@ -364,12 +355,14 @@ static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode,
                 break;
 
         case APIC_DM_NMI:
+                result = 1;
                 kvm_inject_nmi(vcpu);
                 kvm_vcpu_kick(vcpu);
                 break;
 
         case APIC_DM_INIT:
                 if (level) {
+                        result = 1;
                         if (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE)
                                 printk(KERN_DEBUG
                                        "INIT on a runnable vcpu %d\n",
@@ -386,6 +379,7 @@ static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode,
                 apic_debug("SIPI to vcpu %d vector 0x%02x\n",
                            vcpu->vcpu_id, vector);
                 if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) {
+                        result = 1;
                         vcpu->arch.sipi_vector = vector;
                         vcpu->arch.mp_state = KVM_MP_STATE_SIPI_RECEIVED;
                         kvm_vcpu_kick(vcpu);
@@ -408,43 +402,9 @@ static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode,
         return result;
 }
 
-static struct kvm_lapic *kvm_apic_round_robin(struct kvm *kvm, u8 vector,
-                                       unsigned long bitmap)
-{
-        int last;
-        int next;
-        struct kvm_lapic *apic = NULL;
-
-        last = kvm->arch.round_robin_prev_vcpu;
-        next = last;
-
-        do {
-                if (++next == KVM_MAX_VCPUS)
-                        next = 0;
-                if (kvm->vcpus[next] == NULL || !test_bit(next, &bitmap))
-                        continue;
-                apic = kvm->vcpus[next]->arch.apic;
-                if (apic && apic_enabled(apic))
-                        break;
-                apic = NULL;
-        } while (next != last);
-        kvm->arch.round_robin_prev_vcpu = next;
-
-        if (!apic)
-                printk(KERN_DEBUG "vcpu not ready for apic_round_robin\n");
-
-        return apic;
-}
-
-struct kvm_vcpu *kvm_get_lowest_prio_vcpu(struct kvm *kvm, u8 vector,
-                unsigned long bitmap)
+int kvm_apic_compare_prio(struct kvm_vcpu *vcpu1, struct kvm_vcpu *vcpu2)
 {
-        struct kvm_lapic *apic;
-
-        apic = kvm_apic_round_robin(kvm, vector, bitmap);
-        if (apic)
-                return apic->vcpu;
-        return NULL;
+        return vcpu1->arch.apic_arb_prio - vcpu2->arch.apic_arb_prio;
 }
 
 static void apic_set_eoi(struct kvm_lapic *apic)
@@ -472,47 +432,24 @@ static void apic_send_ipi(struct kvm_lapic *apic)
 {
         u32 icr_low = apic_get_reg(apic, APIC_ICR);
         u32 icr_high = apic_get_reg(apic, APIC_ICR2);
+        struct kvm_lapic_irq irq;
 
-        unsigned int dest = GET_APIC_DEST_FIELD(icr_high);
-        unsigned int short_hand = icr_low & APIC_SHORT_MASK;
-        unsigned int trig_mode = icr_low & APIC_INT_LEVELTRIG;
-        unsigned int level = icr_low & APIC_INT_ASSERT;
-        unsigned int dest_mode = icr_low & APIC_DEST_MASK;
-        unsigned int delivery_mode = icr_low & APIC_MODE_MASK;
-        unsigned int vector = icr_low & APIC_VECTOR_MASK;
-
-        struct kvm_vcpu *target;
-        struct kvm_vcpu *vcpu;
-        unsigned long lpr_map = 0;
-        int i;
+        irq.vector = icr_low & APIC_VECTOR_MASK;
+        irq.delivery_mode = icr_low & APIC_MODE_MASK;
+        irq.dest_mode = icr_low & APIC_DEST_MASK;
+        irq.level = icr_low & APIC_INT_ASSERT;
+        irq.trig_mode = icr_low & APIC_INT_LEVELTRIG;
+        irq.shorthand = icr_low & APIC_SHORT_MASK;
+        irq.dest_id = GET_APIC_DEST_FIELD(icr_high);
 
         apic_debug("icr_high 0x%x, icr_low 0x%x, "
                    "short_hand 0x%x, dest 0x%x, trig_mode 0x%x, level 0x%x, "
                    "dest_mode 0x%x, delivery_mode 0x%x, vector 0x%x\n",
-                   icr_high, icr_low, short_hand, dest,
-                   trig_mode, level, dest_mode, delivery_mode, vector);
-
-        for (i = 0; i < KVM_MAX_VCPUS; i++) {
-                vcpu = apic->vcpu->kvm->vcpus[i];
-                if (!vcpu)
-                        continue;
-
-                if (vcpu->arch.apic &&
-                    apic_match_dest(vcpu, apic, short_hand, dest, dest_mode)) {
-                        if (delivery_mode == APIC_DM_LOWEST)
-                                set_bit(vcpu->vcpu_id, &lpr_map);
-                        else
-                                __apic_accept_irq(vcpu->arch.apic, delivery_mode,
-                                                  vector, level, trig_mode);
-                }
-        }
+                   icr_high, icr_low, irq.shorthand, irq.dest_id,
+                   irq.trig_mode, irq.level, irq.dest_mode, irq.delivery_mode,
+                   irq.vector);
 
-        if (delivery_mode == APIC_DM_LOWEST) {
-                target = kvm_get_lowest_prio_vcpu(vcpu->kvm, vector, lpr_map);
-                if (target != NULL)
-                        __apic_accept_irq(target->arch.apic, delivery_mode,
-                                          vector, level, trig_mode);
-        }
+        kvm_irq_delivery_to_apic(apic->vcpu->kvm, apic, &irq);
 }
 
 static u32 apic_get_tmcct(struct kvm_lapic *apic)
@@ -527,12 +464,13 @@ static u32 apic_get_tmcct(struct kvm_lapic *apic)
         if (apic_get_reg(apic, APIC_TMICT) == 0)
                 return 0;
 
-        remaining = hrtimer_expires_remaining(&apic->timer.dev);
+        remaining = hrtimer_expires_remaining(&apic->lapic_timer.timer);
         if (ktime_to_ns(remaining) < 0)
                 remaining = ktime_set(0, 0);
 
-        ns = mod_64(ktime_to_ns(remaining), apic->timer.period);
-        tmcct = div64_u64(ns, (APIC_BUS_CYCLE_NS * apic->timer.divide_count));
+        ns = mod_64(ktime_to_ns(remaining), apic->lapic_timer.period);
+        tmcct = div64_u64(ns,
+                         (APIC_BUS_CYCLE_NS * apic->divide_count));
 
         return tmcct;
 }
@@ -619,25 +557,25 @@ static void update_divide_count(struct kvm_lapic *apic)
         tdcr = apic_get_reg(apic, APIC_TDCR);
         tmp1 = tdcr & 0xf;
         tmp2 = ((tmp1 & 0x3) | ((tmp1 & 0x8) >> 1)) + 1;
-        apic->timer.divide_count = 0x1 << (tmp2 & 0x7);
+        apic->divide_count = 0x1 << (tmp2 & 0x7);
 
         apic_debug("timer divide count is 0x%x\n",
-                                   apic->timer.divide_count);
+                                   apic->divide_count);
 }
 
 static void start_apic_timer(struct kvm_lapic *apic)
 {
-        ktime_t now = apic->timer.dev.base->get_time();
+        ktime_t now = apic->lapic_timer.timer.base->get_time();
 
-        apic->timer.period = apic_get_reg(apic, APIC_TMICT) *
-                    APIC_BUS_CYCLE_NS * apic->timer.divide_count;
-        atomic_set(&apic->timer.pending, 0);
+        apic->lapic_timer.period = apic_get_reg(apic, APIC_TMICT) *
+                    APIC_BUS_CYCLE_NS * apic->divide_count;
+        atomic_set(&apic->lapic_timer.pending, 0);
 
-        if (!apic->timer.period)
+        if (!apic->lapic_timer.period)
                 return;
 
-        hrtimer_start(&apic->timer.dev,
-                      ktime_add_ns(now, apic->timer.period),
+        hrtimer_start(&apic->lapic_timer.timer,
+                      ktime_add_ns(now, apic->lapic_timer.period),
                       HRTIMER_MODE_ABS);
 
         apic_debug("%s: bus cycle is %" PRId64 "ns, now 0x%016"
@@ -646,9 +584,9 @@ static void start_apic_timer(struct kvm_lapic *apic)
                            "expire @ 0x%016" PRIx64 ".\n", __func__,
                            APIC_BUS_CYCLE_NS, ktime_to_ns(now),
                            apic_get_reg(apic, APIC_TMICT),
-                           apic->timer.period,
+                           apic->lapic_timer.period,
                            ktime_to_ns(ktime_add_ns(now,
-                                        apic->timer.period)));
+                                        apic->lapic_timer.period)));
 }
 
 static void apic_manage_nmi_watchdog(struct kvm_lapic *apic, u32 lvt0_val)
@@ -730,7 +668,7 @@ static void apic_mmio_write(struct kvm_io_device *this,
                                 apic_set_reg(apic, APIC_LVTT + 0x10 * i,
                                              lvt_val | APIC_LVT_MASKED);
                         }
-                        atomic_set(&apic->timer.pending, 0);
+                        atomic_set(&apic->lapic_timer.pending, 0);
 
                 }
                 break;
@@ -762,7 +700,7 @@ static void apic_mmio_write(struct kvm_io_device *this,
                 break;
 
         case APIC_TMICT:
-                hrtimer_cancel(&apic->timer.dev);
+                hrtimer_cancel(&apic->lapic_timer.timer);
                 apic_set_reg(apic, APIC_TMICT, val);
                 start_apic_timer(apic);
                 return;
@@ -802,7 +740,7 @@ void kvm_free_lapic(struct kvm_vcpu *vcpu)
         if (!vcpu->arch.apic)
                 return;
 
-        hrtimer_cancel(&vcpu->arch.apic->timer.dev);
+        hrtimer_cancel(&vcpu->arch.apic->lapic_timer.timer);
 
         if (vcpu->arch.apic->regs_page)
                 __free_page(vcpu->arch.apic->regs_page);
@@ -880,7 +818,7 @@ void kvm_lapic_reset(struct kvm_vcpu *vcpu)
         ASSERT(apic != NULL);
 
         /* Stop the timer in case it's a reset to an active apic */
-        hrtimer_cancel(&apic->timer.dev);
+        hrtimer_cancel(&apic->lapic_timer.timer);
 
         apic_set_reg(apic, APIC_ID, vcpu->vcpu_id << 24);
         apic_set_reg(apic, APIC_LVR, APIC_VERSION);
@@ -905,11 +843,13 @@ void kvm_lapic_reset(struct kvm_vcpu *vcpu)
                 apic_set_reg(apic, APIC_TMR + 0x10 * i, 0);
         }
         update_divide_count(apic);
-        atomic_set(&apic->timer.pending, 0);
+        atomic_set(&apic->lapic_timer.pending, 0);
         if (vcpu->vcpu_id == 0)
                 vcpu->arch.apic_base |= MSR_IA32_APICBASE_BSP;
         apic_update_ppr(apic);
 
+        vcpu->arch.apic_arb_prio = 0;
+
         apic_debug(KERN_INFO "%s: vcpu=%p, id=%d, base_msr="
                    "0x%016" PRIx64 ", base_address=0x%0lx.\n", __func__,
                    vcpu, kvm_apic_id(apic),
@@ -917,16 +857,14 @@ void kvm_lapic_reset(struct kvm_vcpu *vcpu)
 }
 EXPORT_SYMBOL_GPL(kvm_lapic_reset);
 
-int kvm_lapic_enabled(struct kvm_vcpu *vcpu)
+bool kvm_apic_present(struct kvm_vcpu *vcpu)
 {
-        struct kvm_lapic *apic = vcpu->arch.apic;
-        int ret = 0;
-
-        if (!apic)
-                return 0;
-        ret = apic_enabled(apic);
+        return vcpu->arch.apic && apic_hw_enabled(vcpu->arch.apic);
+}
 
-        return ret;
+int kvm_lapic_enabled(struct kvm_vcpu *vcpu)
+{
+        return kvm_apic_present(vcpu) && apic_sw_enabled(vcpu->arch.apic);
 }
 EXPORT_SYMBOL_GPL(kvm_lapic_enabled);
 
@@ -936,22 +874,11 @@ EXPORT_SYMBOL_GPL(kvm_lapic_enabled);
  *----------------------------------------------------------------------
  */
 
-/* TODO: make sure __apic_timer_fn runs in current pCPU */
-static int __apic_timer_fn(struct kvm_lapic *apic)
+static bool lapic_is_periodic(struct kvm_timer *ktimer)
 {
-        int result = 0;
-        wait_queue_head_t *q = &apic->vcpu->wq;
-
-        if(!atomic_inc_and_test(&apic->timer.pending))
-                set_bit(KVM_REQ_PENDING_TIMER, &apic->vcpu->requests);
-        if (waitqueue_active(q))
-                wake_up_interruptible(q);
-
-        if (apic_lvtt_period(apic)) {
-                result = 1;
-                hrtimer_add_expires_ns(&apic->timer.dev, apic->timer.period);
-        }
-        return result;
+        struct kvm_lapic *apic = container_of(ktimer, struct kvm_lapic,
+                                              lapic_timer);
+        return apic_lvtt_period(apic);
 }
 
 int apic_has_pending_timer(struct kvm_vcpu *vcpu)
@@ -959,7 +886,7 @@ int apic_has_pending_timer(struct kvm_vcpu *vcpu)
         struct kvm_lapic *lapic = vcpu->arch.apic;
 
         if (lapic && apic_enabled(lapic) && apic_lvt_enabled(lapic, APIC_LVTT))
-                return atomic_read(&lapic->timer.pending);
+                return atomic_read(&lapic->lapic_timer.pending);
 
         return 0;
 }
@@ -986,20 +913,9 @@ void kvm_apic_nmi_wd_deliver(struct kvm_vcpu *vcpu)
                 kvm_apic_local_deliver(apic, APIC_LVT0);
 }
 
-static enum hrtimer_restart apic_timer_fn(struct hrtimer *data)
-{
-        struct kvm_lapic *apic;
-        int restart_timer = 0;
-
-        apic = container_of(data, struct kvm_lapic, timer.dev);
-
-        restart_timer = __apic_timer_fn(apic);
-
-        if (restart_timer)
-                return HRTIMER_RESTART;
-        else
-                return HRTIMER_NORESTART;
-}
+static struct kvm_timer_ops lapic_timer_ops = {
+        .is_periodic = lapic_is_periodic,
+};
 
 int kvm_create_lapic(struct kvm_vcpu *vcpu)
 {
@@ -1024,8 +940,13 @@ int kvm_create_lapic(struct kvm_vcpu *vcpu)
         memset(apic->regs, 0, PAGE_SIZE);
         apic->vcpu = vcpu;
 
-        hrtimer_init(&apic->timer.dev, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
-        apic->timer.dev.function = apic_timer_fn;
+        hrtimer_init(&apic->lapic_timer.timer, CLOCK_MONOTONIC,
+                     HRTIMER_MODE_ABS);
+        apic->lapic_timer.timer.function = kvm_timer_fn;
+        apic->lapic_timer.t_ops = &lapic_timer_ops;
+        apic->lapic_timer.kvm = vcpu->kvm;
+        apic->lapic_timer.vcpu_id = vcpu->vcpu_id;
+
         apic->base_address = APIC_DEFAULT_PHYS_BASE;
         vcpu->arch.apic_base = APIC_DEFAULT_PHYS_BASE;
 
@@ -1078,9 +999,9 @@ void kvm_inject_apic_timer_irqs(struct kvm_vcpu *vcpu)
 {
         struct kvm_lapic *apic = vcpu->arch.apic;
 
-        if (apic && atomic_read(&apic->timer.pending) > 0) {
+        if (apic && atomic_read(&apic->lapic_timer.pending) > 0) {
                 if (kvm_apic_local_deliver(apic, APIC_LVTT))
-                        atomic_dec(&apic->timer.pending);
+                        atomic_dec(&apic->lapic_timer.pending);
         }
 }
 
@@ -1106,7 +1027,7 @@ void kvm_apic_post_state_restore(struct kvm_vcpu *vcpu)
                              MSR_IA32_APICBASE_BASE;
         apic_set_reg(apic, APIC_LVR, APIC_VERSION);
         apic_update_ppr(apic);
-        hrtimer_cancel(&apic->timer.dev);
+        hrtimer_cancel(&apic->lapic_timer.timer);
         update_divide_count(apic);
         start_apic_timer(apic);
 }
@@ -1119,7 +1040,7 @@ void __kvm_migrate_apic_timer(struct kvm_vcpu *vcpu)
         if (!apic)
                 return;
 
-        timer = &apic->timer.dev;
+        timer = &apic->lapic_timer.timer;
         if (hrtimer_cancel(timer))
                 hrtimer_start_expires(timer, HRTIMER_MODE_ABS);
 }
diff --git a/arch/x86/kvm/lapic.h b/arch/x86/kvm/lapic.h
index 45ab6ee71209..a587f8349c46 100644
--- a/arch/x86/kvm/lapic.h
+++ b/arch/x86/kvm/lapic.h
@@ -2,18 +2,15 @@
 #define __KVM_X86_LAPIC_H
 
 #include "iodev.h"
+#include "kvm_timer.h"
 
 #include <linux/kvm_host.h>
 
 struct kvm_lapic {
         unsigned long base_address;
         struct kvm_io_device dev;
-        struct {
-                atomic_t pending;
-                s64 period;     /* unit: ns */
-                u32 divide_count;
-                struct hrtimer dev;
-        } timer;
+        struct kvm_timer lapic_timer;
+        u32 divide_count;
         struct kvm_vcpu *vcpu;
         struct page *regs_page;
         void *regs;
@@ -34,12 +31,13 @@ u64 kvm_lapic_get_base(struct kvm_vcpu *vcpu);
 
 int kvm_apic_match_physical_addr(struct kvm_lapic *apic, u16 dest);
 int kvm_apic_match_logical_addr(struct kvm_lapic *apic, u8 mda);
-int kvm_apic_set_irq(struct kvm_vcpu *vcpu, u8 vec, u8 trig);
+int kvm_apic_set_irq(struct kvm_vcpu *vcpu, struct kvm_lapic_irq *irq);
 
 u64 kvm_get_apic_base(struct kvm_vcpu *vcpu);
 void kvm_set_apic_base(struct kvm_vcpu *vcpu, u64 data);
 void kvm_apic_post_state_restore(struct kvm_vcpu *vcpu);
 int kvm_lapic_enabled(struct kvm_vcpu *vcpu);
+bool kvm_apic_present(struct kvm_vcpu *vcpu);
 int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu);
 
 void kvm_lapic_set_vapic_addr(struct kvm_vcpu *vcpu, gpa_t vapic_addr);
diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c
index 32cf11e5728a..5c3d6e81a7dc 100644
--- a/arch/x86/kvm/mmu.c
+++ b/arch/x86/kvm/mmu.c
@@ -126,6 +126,7 @@ module_param(oos_shadow, bool, 0644);
 #define PFERR_PRESENT_MASK (1U << 0)
 #define PFERR_WRITE_MASK (1U << 1)
 #define PFERR_USER_MASK (1U << 2)
+#define PFERR_RSVD_MASK (1U << 3)
 #define PFERR_FETCH_MASK (1U << 4)
 
 #define PT_DIRECTORY_LEVEL 2
@@ -177,7 +178,11 @@ static u64 __read_mostly shadow_x_mask;	/* mutual exclusive with nx_mask */
 static u64 __read_mostly shadow_user_mask;
 static u64 __read_mostly shadow_accessed_mask;
 static u64 __read_mostly shadow_dirty_mask;
-static u64 __read_mostly shadow_mt_mask;
+
+static inline u64 rsvd_bits(int s, int e)
+{
+        return ((1ULL << (e - s + 1)) - 1) << s;
+}
 
 void kvm_mmu_set_nonpresent_ptes(u64 trap_pte, u64 notrap_pte)
 {
@@ -193,14 +198,13 @@ void kvm_mmu_set_base_ptes(u64 base_pte)
 EXPORT_SYMBOL_GPL(kvm_mmu_set_base_ptes);
 
 void kvm_mmu_set_mask_ptes(u64 user_mask, u64 accessed_mask,
-                u64 dirty_mask, u64 nx_mask, u64 x_mask, u64 mt_mask)
+                u64 dirty_mask, u64 nx_mask, u64 x_mask)
 {
         shadow_user_mask = user_mask;
         shadow_accessed_mask = accessed_mask;
         shadow_dirty_mask = dirty_mask;
         shadow_nx_mask = nx_mask;
         shadow_x_mask = x_mask;
-        shadow_mt_mask = mt_mask;
 }
 EXPORT_SYMBOL_GPL(kvm_mmu_set_mask_ptes);
 
@@ -219,11 +223,6 @@ static int is_nx(struct kvm_vcpu *vcpu)
         return vcpu->arch.shadow_efer & EFER_NX;
 }
 
-static int is_present_pte(unsigned long pte)
-{
-        return pte & PT_PRESENT_MASK;
-}
-
 static int is_shadow_present_pte(u64 pte)
 {
         return pte != shadow_trap_nonpresent_pte
@@ -1074,18 +1073,10 @@ static struct kvm_mmu_page *kvm_mmu_lookup_page(struct kvm *kvm, gfn_t gfn)
         return NULL;
 }
 
-static void kvm_unlink_unsync_global(struct kvm *kvm, struct kvm_mmu_page *sp)
-{
-        list_del(&sp->oos_link);
-        --kvm->stat.mmu_unsync_global;
-}
-
 static void kvm_unlink_unsync_page(struct kvm *kvm, struct kvm_mmu_page *sp)
 {
         WARN_ON(!sp->unsync);
         sp->unsync = 0;
-        if (sp->global)
-                kvm_unlink_unsync_global(kvm, sp);
         --kvm->stat.mmu_unsync;
 }
 
@@ -1248,7 +1239,6 @@ static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu,
         pgprintk("%s: adding gfn %lx role %x\n", __func__, gfn, role.word);
         sp->gfn = gfn;
         sp->role = role;
-        sp->global = 0;
         hlist_add_head(&sp->hash_link, bucket);
         if (!direct) {
                 if (rmap_write_protect(vcpu->kvm, gfn))
@@ -1616,7 +1606,7 @@ static int get_mtrr_type(struct mtrr_state_type *mtrr_state,
         return mtrr_state->def_type;
 }
 
-static u8 get_memory_type(struct kvm_vcpu *vcpu, gfn_t gfn)
+u8 kvm_get_guest_memory_type(struct kvm_vcpu *vcpu, gfn_t gfn)
 {
         u8 mtrr;
 
@@ -1626,6 +1616,7 @@ static u8 get_memory_type(struct kvm_vcpu *vcpu, gfn_t gfn)
                 mtrr = MTRR_TYPE_WRBACK;
         return mtrr;
 }
+EXPORT_SYMBOL_GPL(kvm_get_guest_memory_type);
 
 static int kvm_unsync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
 {
@@ -1646,11 +1637,7 @@ static int kvm_unsync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
         ++vcpu->kvm->stat.mmu_unsync;
         sp->unsync = 1;
 
-        if (sp->global) {
-                list_add(&sp->oos_link, &vcpu->kvm->arch.oos_global_pages);
-                ++vcpu->kvm->stat.mmu_unsync_global;
-        } else
-                kvm_mmu_mark_parents_unsync(vcpu, sp);
+        kvm_mmu_mark_parents_unsync(vcpu, sp);
 
         mmu_convert_notrap(sp);
         return 0;
@@ -1677,21 +1664,11 @@ static int mmu_need_write_protect(struct kvm_vcpu *vcpu, gfn_t gfn,
 static int set_spte(struct kvm_vcpu *vcpu, u64 *shadow_pte,
                     unsigned pte_access, int user_fault,
                     int write_fault, int dirty, int largepage,
-                    int global, gfn_t gfn, pfn_t pfn, bool speculative,
+                    gfn_t gfn, pfn_t pfn, bool speculative,
                     bool can_unsync)
 {
         u64 spte;
         int ret = 0;
-        u64 mt_mask = shadow_mt_mask;
-        struct kvm_mmu_page *sp = page_header(__pa(shadow_pte));
-
-        if (!global && sp->global) {
-                sp->global = 0;
-                if (sp->unsync) {
-                        kvm_unlink_unsync_global(vcpu->kvm, sp);
-                        kvm_mmu_mark_parents_unsync(vcpu, sp);
-                }
-        }
 
         /*
          * We don't set the accessed bit, since we sometimes want to see
@@ -1711,16 +1688,9 @@ static int set_spte(struct kvm_vcpu *vcpu, u64 *shadow_pte,
                 spte |= shadow_user_mask;
         if (largepage)
                 spte |= PT_PAGE_SIZE_MASK;
-        if (mt_mask) {
-                if (!kvm_is_mmio_pfn(pfn)) {
-                        mt_mask = get_memory_type(vcpu, gfn) <<
-                                kvm_x86_ops->get_mt_mask_shift();
-                        mt_mask |= VMX_EPT_IGMT_BIT;
-                } else
-                        mt_mask = MTRR_TYPE_UNCACHABLE <<
-                                kvm_x86_ops->get_mt_mask_shift();
-                spte |= mt_mask;
-        }
+        if (tdp_enabled)
+                spte |= kvm_x86_ops->get_mt_mask(vcpu, gfn,
+                        kvm_is_mmio_pfn(pfn));
 
         spte |= (u64)pfn << PAGE_SHIFT;
 
@@ -1765,8 +1735,8 @@ set_pte:
 static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *shadow_pte,
                          unsigned pt_access, unsigned pte_access,
                          int user_fault, int write_fault, int dirty,
-                         int *ptwrite, int largepage, int global,
-                         gfn_t gfn, pfn_t pfn, bool speculative)
+                         int *ptwrite, int largepage, gfn_t gfn,
+                         pfn_t pfn, bool speculative)
 {
         int was_rmapped = 0;
         int was_writeble = is_writeble_pte(*shadow_pte);
@@ -1795,7 +1765,7 @@ static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *shadow_pte,
                         was_rmapped = 1;
         }
         if (set_spte(vcpu, shadow_pte, pte_access, user_fault, write_fault,
-                      dirty, largepage, global, gfn, pfn, speculative, true)) {
+                      dirty, largepage, gfn, pfn, speculative, true)) {
                 if (write_fault)
                         *ptwrite = 1;
                 kvm_x86_ops->tlb_flush(vcpu);
@@ -1843,7 +1813,7 @@ static int __direct_map(struct kvm_vcpu *vcpu, gpa_t v, int write,
                     || (largepage && iterator.level == PT_DIRECTORY_LEVEL)) {
                         mmu_set_spte(vcpu, iterator.sptep, ACC_ALL, ACC_ALL,
                                      0, write, 1, &pt_write,
-                                     largepage, 0, gfn, pfn, false);
+                                     largepage, gfn, pfn, false);
                         ++vcpu->stat.pf_fixed;
                         break;
                 }
@@ -1942,7 +1912,19 @@ static void mmu_free_roots(struct kvm_vcpu *vcpu)
         vcpu->arch.mmu.root_hpa = INVALID_PAGE;
 }
 
-static void mmu_alloc_roots(struct kvm_vcpu *vcpu)
+static int mmu_check_root(struct kvm_vcpu *vcpu, gfn_t root_gfn)
+{
+        int ret = 0;
+
+        if (!kvm_is_visible_gfn(vcpu->kvm, root_gfn)) {
+                set_bit(KVM_REQ_TRIPLE_FAULT, &vcpu->requests);
+                ret = 1;
+        }
+
+        return ret;
+}
+
+static int mmu_alloc_roots(struct kvm_vcpu *vcpu)
 {
         int i;
         gfn_t root_gfn;
@@ -1957,13 +1939,15 @@ static void mmu_alloc_roots(struct kvm_vcpu *vcpu)
                 ASSERT(!VALID_PAGE(root));
                 if (tdp_enabled)
                         direct = 1;
+                if (mmu_check_root(vcpu, root_gfn))
+                        return 1;
                 sp = kvm_mmu_get_page(vcpu, root_gfn, 0,
                                       PT64_ROOT_LEVEL, direct,
                                       ACC_ALL, NULL);
                 root = __pa(sp->spt);
                 ++sp->root_count;
                 vcpu->arch.mmu.root_hpa = root;
-                return;
+                return 0;
         }
         direct = !is_paging(vcpu);
         if (tdp_enabled)
@@ -1980,6 +1964,8 @@ static void mmu_alloc_roots(struct kvm_vcpu *vcpu)
                         root_gfn = vcpu->arch.pdptrs[i] >> PAGE_SHIFT;
                 } else if (vcpu->arch.mmu.root_level == 0)
                         root_gfn = 0;
+                if (mmu_check_root(vcpu, root_gfn))
+                        return 1;
                 sp = kvm_mmu_get_page(vcpu, root_gfn, i << 30,
                                       PT32_ROOT_LEVEL, direct,
                                       ACC_ALL, NULL);
@@ -1988,6 +1974,7 @@ static void mmu_alloc_roots(struct kvm_vcpu *vcpu)
                 vcpu->arch.mmu.pae_root[i] = root | PT_PRESENT_MASK;
         }
         vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.pae_root);
+        return 0;
 }
 
 static void mmu_sync_roots(struct kvm_vcpu *vcpu)
@@ -2006,7 +1993,7 @@ static void mmu_sync_roots(struct kvm_vcpu *vcpu)
         for (i = 0; i < 4; ++i) {
                 hpa_t root = vcpu->arch.mmu.pae_root[i];
 
-                if (root) {
+                if (root && VALID_PAGE(root)) {
                         root &= PT64_BASE_ADDR_MASK;
                         sp = page_header(root);
                         mmu_sync_children(vcpu, sp);
@@ -2014,15 +2001,6 @@ static void mmu_sync_roots(struct kvm_vcpu *vcpu)
         }
 }
 
-static void mmu_sync_global(struct kvm_vcpu *vcpu)
-{
-        struct kvm *kvm = vcpu->kvm;
-        struct kvm_mmu_page *sp, *n;
-
-        list_for_each_entry_safe(sp, n, &kvm->arch.oos_global_pages, oos_link)
-                kvm_sync_page(vcpu, sp);
-}
-
 void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu)
 {
         spin_lock(&vcpu->kvm->mmu_lock);
@@ -2030,13 +2008,6 @@ void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu)
         spin_unlock(&vcpu->kvm->mmu_lock);
 }
 
-void kvm_mmu_sync_global(struct kvm_vcpu *vcpu)
-{
-        spin_lock(&vcpu->kvm->mmu_lock);
-        mmu_sync_global(vcpu);
-        spin_unlock(&vcpu->kvm->mmu_lock);
-}
-
 static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr)
 {
         return vaddr;
@@ -2151,6 +2122,14 @@ static void paging_free(struct kvm_vcpu *vcpu)
         nonpaging_free(vcpu);
 }
 
+static bool is_rsvd_bits_set(struct kvm_vcpu *vcpu, u64 gpte, int level)
+{
+        int bit7;
+
+        bit7 = (gpte >> 7) & 1;
+        return (gpte & vcpu->arch.mmu.rsvd_bits_mask[bit7][level-1]) != 0;
+}
+
 #define PTTYPE 64
 #include "paging_tmpl.h"
 #undef PTTYPE
@@ -2159,6 +2138,59 @@ static void paging_free(struct kvm_vcpu *vcpu)
 #include "paging_tmpl.h"
 #undef PTTYPE
 
+static void reset_rsvds_bits_mask(struct kvm_vcpu *vcpu, int level)
+{
+        struct kvm_mmu *context = &vcpu->arch.mmu;
+        int maxphyaddr = cpuid_maxphyaddr(vcpu);
+        u64 exb_bit_rsvd = 0;
+
+        if (!is_nx(vcpu))
+                exb_bit_rsvd = rsvd_bits(63, 63);
+        switch (level) {
+        case PT32_ROOT_LEVEL:
+                /* no rsvd bits for 2 level 4K page table entries */
+                context->rsvd_bits_mask[0][1] = 0;
+                context->rsvd_bits_mask[0][0] = 0;
+                if (is_cpuid_PSE36())
+                        /* 36bits PSE 4MB page */
+                        context->rsvd_bits_mask[1][1] = rsvd_bits(17, 21);
+                else
+                        /* 32 bits PSE 4MB page */
+                        context->rsvd_bits_mask[1][1] = rsvd_bits(13, 21);
+                context->rsvd_bits_mask[1][0] = ~0ull;
+                break;
+        case PT32E_ROOT_LEVEL:
+                context->rsvd_bits_mask[0][2] =
+                        rsvd_bits(maxphyaddr, 63) |
+                        rsvd_bits(7, 8) | rsvd_bits(1, 2);      /* PDPTE */
+                context->rsvd_bits_mask[0][1] = exb_bit_rsvd |
+                        rsvd_bits(maxphyaddr, 62);      /* PDE */
+                context->rsvd_bits_mask[0][0] = exb_bit_rsvd |
+                        rsvd_bits(maxphyaddr, 62);      /* PTE */
+                context->rsvd_bits_mask[1][1] = exb_bit_rsvd |
+                        rsvd_bits(maxphyaddr, 62) |
+                        rsvd_bits(13, 20);              /* large page */
+                context->rsvd_bits_mask[1][0] = ~0ull;
+                break;
+        case PT64_ROOT_LEVEL:
+                context->rsvd_bits_mask[0][3] = exb_bit_rsvd |
+                        rsvd_bits(maxphyaddr, 51) | rsvd_bits(7, 8);
+                context->rsvd_bits_mask[0][2] = exb_bit_rsvd |
+                        rsvd_bits(maxphyaddr, 51) | rsvd_bits(7, 8);
+                context->rsvd_bits_mask[0][1] = exb_bit_rsvd |
+                        rsvd_bits(maxphyaddr, 51);
+                context->rsvd_bits_mask[0][0] = exb_bit_rsvd |
+                        rsvd_bits(maxphyaddr, 51);
+                context->rsvd_bits_mask[1][3] = context->rsvd_bits_mask[0][3];
+                context->rsvd_bits_mask[1][2] = context->rsvd_bits_mask[0][2];
+                context->rsvd_bits_mask[1][1] = exb_bit_rsvd |
+                        rsvd_bits(maxphyaddr, 51) |
+                        rsvd_bits(13, 20);              /* large page */
+                context->rsvd_bits_mask[1][0] = ~0ull;
+                break;
+        }
+}
+
 static int paging64_init_context_common(struct kvm_vcpu *vcpu, int level)
 {
         struct kvm_mmu *context = &vcpu->arch.mmu;
@@ -2179,6 +2211,7 @@ static int paging64_init_context_common(struct kvm_vcpu *vcpu, int level)
 
 static int paging64_init_context(struct kvm_vcpu *vcpu)
 {
+        reset_rsvds_bits_mask(vcpu, PT64_ROOT_LEVEL);
         return paging64_init_context_common(vcpu, PT64_ROOT_LEVEL);
 }
 
@@ -2186,6 +2219,7 @@ static int paging32_init_context(struct kvm_vcpu *vcpu)
 {
         struct kvm_mmu *context = &vcpu->arch.mmu;
 
+        reset_rsvds_bits_mask(vcpu, PT32_ROOT_LEVEL);
         context->new_cr3 = paging_new_cr3;
         context->page_fault = paging32_page_fault;
         context->gva_to_gpa = paging32_gva_to_gpa;
@@ -2201,6 +2235,7 @@ static int paging32_init_context(struct kvm_vcpu *vcpu)
 
 static int paging32E_init_context(struct kvm_vcpu *vcpu)
 {
+        reset_rsvds_bits_mask(vcpu, PT32E_ROOT_LEVEL);
         return paging64_init_context_common(vcpu, PT32E_ROOT_LEVEL);
 }
 
@@ -2221,12 +2256,15 @@ static int init_kvm_tdp_mmu(struct kvm_vcpu *vcpu)
                 context->gva_to_gpa = nonpaging_gva_to_gpa;
                 context->root_level = 0;
         } else if (is_long_mode(vcpu)) {
+                reset_rsvds_bits_mask(vcpu, PT64_ROOT_LEVEL);
                 context->gva_to_gpa = paging64_gva_to_gpa;
                 context->root_level = PT64_ROOT_LEVEL;
         } else if (is_pae(vcpu)) {
+                reset_rsvds_bits_mask(vcpu, PT32E_ROOT_LEVEL);
                 context->gva_to_gpa = paging64_gva_to_gpa;
                 context->root_level = PT32E_ROOT_LEVEL;
         } else {
+                reset_rsvds_bits_mask(vcpu, PT32_ROOT_LEVEL);
                 context->gva_to_gpa = paging32_gva_to_gpa;
                 context->root_level = PT32_ROOT_LEVEL;
         }
@@ -2290,9 +2328,11 @@ int kvm_mmu_load(struct kvm_vcpu *vcpu)
                 goto out;
         spin_lock(&vcpu->kvm->mmu_lock);
         kvm_mmu_free_some_pages(vcpu);
-        mmu_alloc_roots(vcpu);
+        r = mmu_alloc_roots(vcpu);
         mmu_sync_roots(vcpu);
         spin_unlock(&vcpu->kvm->mmu_lock);
+        if (r)
+                goto out;
         kvm_x86_ops->set_cr3(vcpu, vcpu->arch.mmu.root_hpa);
         kvm_mmu_flush_tlb(vcpu);
 out:
@@ -2638,14 +2678,6 @@ EXPORT_SYMBOL_GPL(kvm_disable_tdp);
 
 static void free_mmu_pages(struct kvm_vcpu *vcpu)
 {
-        struct kvm_mmu_page *sp;
-
-        while (!list_empty(&vcpu->kvm->arch.active_mmu_pages)) {
-                sp = container_of(vcpu->kvm->arch.active_mmu_pages.next,
-                                  struct kvm_mmu_page, link);
-                kvm_mmu_zap_page(vcpu->kvm, sp);
-                cond_resched();
-        }
         free_page((unsigned long)vcpu->arch.mmu.pae_root);
 }
 
@@ -2710,7 +2742,6 @@ void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot)
 {
         struct kvm_mmu_page *sp;
 
-        spin_lock(&kvm->mmu_lock);
         list_for_each_entry(sp, &kvm->arch.active_mmu_pages, link) {
                 int i;
                 u64 *pt;
@@ -2725,7 +2756,6 @@ void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot)
                                 pt[i] &= ~PT_WRITABLE_MASK;
         }
         kvm_flush_remote_tlbs(kvm);
-        spin_unlock(&kvm->mmu_lock);
 }
 
 void kvm_mmu_zap_all(struct kvm *kvm)
@@ -3007,11 +3037,13 @@ static void audit_mappings_page(struct kvm_vcpu *vcpu, u64 page_pte,
                                        " in nonleaf level: levels %d gva %lx"
                                        " level %d pte %llx\n", audit_msg,
                                        vcpu->arch.mmu.root_level, va, level, ent);
-
-                        audit_mappings_page(vcpu, ent, va, level - 1);
+                        else
+                                audit_mappings_page(vcpu, ent, va, level - 1);
                 } else {
                         gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, va);
-                        hpa_t hpa = (hpa_t)gpa_to_pfn(vcpu, gpa) << PAGE_SHIFT;
+                        gfn_t gfn = gpa >> PAGE_SHIFT;
+                        pfn_t pfn = gfn_to_pfn(vcpu->kvm, gfn);
+                        hpa_t hpa = (hpa_t)pfn << PAGE_SHIFT;
 
                         if (is_shadow_present_pte(ent)
                             && (ent & PT64_BASE_ADDR_MASK) != hpa)
diff --git a/arch/x86/kvm/mmu.h b/arch/x86/kvm/mmu.h
index eaab2145f62b..3494a2fb136e 100644
--- a/arch/x86/kvm/mmu.h
+++ b/arch/x86/kvm/mmu.h
@@ -75,4 +75,9 @@ static inline int is_paging(struct kvm_vcpu *vcpu)
         return vcpu->arch.cr0 & X86_CR0_PG;
 }
 
+static inline int is_present_pte(unsigned long pte)
+{
+        return pte & PT_PRESENT_MASK;
+}
+
 #endif
diff --git a/arch/x86/kvm/paging_tmpl.h b/arch/x86/kvm/paging_tmpl.h
index 6bd70206c561..258e4591e1ca 100644
--- a/arch/x86/kvm/paging_tmpl.h
+++ b/arch/x86/kvm/paging_tmpl.h
@@ -123,6 +123,7 @@ static int FNAME(walk_addr)(struct guest_walker *walker,
         gfn_t table_gfn;
         unsigned index, pt_access, pte_access;
         gpa_t pte_gpa;
+        int rsvd_fault = 0;
 
         pgprintk("%s: addr %lx\n", __func__, addr);
 walk:
@@ -157,6 +158,10 @@ walk:
                 if (!is_present_pte(pte))
                         goto not_present;
 
+                rsvd_fault = is_rsvd_bits_set(vcpu, pte, walker->level);
+                if (rsvd_fault)
+                        goto access_error;
+
                 if (write_fault && !is_writeble_pte(pte))
                         if (user_fault || is_write_protection(vcpu))
                                 goto access_error;
@@ -209,7 +214,6 @@ walk:
                 if (ret)
                         goto walk;
                 pte |= PT_DIRTY_MASK;
-                kvm_mmu_pte_write(vcpu, pte_gpa, (u8 *)&pte, sizeof(pte), 0);
                 walker->ptes[walker->level - 1] = pte;
         }
 
@@ -233,6 +237,8 @@ err:
                 walker->error_code |= PFERR_USER_MASK;
         if (fetch_fault)
                 walker->error_code |= PFERR_FETCH_MASK;
+        if (rsvd_fault)
+                walker->error_code |= PFERR_RSVD_MASK;
         return 0;
 }
 
@@ -262,8 +268,7 @@ static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *page,
         kvm_get_pfn(pfn);
         mmu_set_spte(vcpu, spte, page->role.access, pte_access, 0, 0,
                      gpte & PT_DIRTY_MASK, NULL, largepage,
-                     gpte & PT_GLOBAL_MASK, gpte_to_gfn(gpte),
-                     pfn, true);
+                     gpte_to_gfn(gpte), pfn, true);
 }
 
 /*
@@ -297,7 +302,6 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr,
                                      user_fault, write_fault,
                                      gw->ptes[gw->level-1] & PT_DIRTY_MASK,
                                      ptwrite, largepage,
-                                     gw->ptes[gw->level-1] & PT_GLOBAL_MASK,
                                      gw->gfn, pfn, false);
                         break;
                 }
@@ -380,7 +384,7 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr,
                 return r;
 
         /*
-         * Look up the shadow pte for the faulting address.
+         * Look up the guest pte for the faulting address.
          */
         r = FNAME(walk_addr)(&walker, vcpu, addr, write_fault, user_fault,
                              fetch_fault);
@@ -586,7 +590,7 @@ static int FNAME(sync_page)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
                 nr_present++;
                 pte_access = sp->role.access & FNAME(gpte_access)(vcpu, gpte);
                 set_spte(vcpu, &sp->spt[i], pte_access, 0, 0,
-                         is_dirty_pte(gpte), 0, gpte & PT_GLOBAL_MASK, gfn,
+                         is_dirty_pte(gpte), 0, gfn,
                          spte_to_pfn(sp->spt[i]), true, false);
         }
 
diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c
index 1f8510c51d6e..71510e07e69e 100644
--- a/arch/x86/kvm/svm.c
+++ b/arch/x86/kvm/svm.c
@@ -19,6 +19,7 @@
 #include "irq.h"
 #include "mmu.h"
 #include "kvm_cache_regs.h"
+#include "x86.h"
 
 #include <linux/module.h>
 #include <linux/kernel.h>
@@ -69,7 +70,6 @@ module_param(npt, int, S_IRUGO);
 static int nested = 0;
 module_param(nested, int, S_IRUGO);
 
-static void kvm_reput_irq(struct vcpu_svm *svm);
 static void svm_flush_tlb(struct kvm_vcpu *vcpu);
 
 static int nested_svm_exit_handled(struct vcpu_svm *svm, bool kvm_override);
@@ -132,24 +132,6 @@ static inline u32 svm_has(u32 feat)
         return svm_features & feat;
 }
 
-static inline u8 pop_irq(struct kvm_vcpu *vcpu)
-{
-        int word_index = __ffs(vcpu->arch.irq_summary);
-        int bit_index = __ffs(vcpu->arch.irq_pending[word_index]);
-        int irq = word_index * BITS_PER_LONG + bit_index;
-
-        clear_bit(bit_index, &vcpu->arch.irq_pending[word_index]);
-        if (!vcpu->arch.irq_pending[word_index])
-                clear_bit(word_index, &vcpu->arch.irq_summary);
-        return irq;
-}
-
-static inline void push_irq(struct kvm_vcpu *vcpu, u8 irq)
-{
-        set_bit(irq, vcpu->arch.irq_pending);
-        set_bit(irq / BITS_PER_LONG, &vcpu->arch.irq_summary);
-}
-
 static inline void clgi(void)
 {
         asm volatile (__ex(SVM_CLGI));
@@ -214,17 +196,31 @@ static void svm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr,
         svm->vmcb->control.event_inj_err = error_code;
 }
 
-static bool svm_exception_injected(struct kvm_vcpu *vcpu)
+static int is_external_interrupt(u32 info)
+{
+        info &= SVM_EVTINJ_TYPE_MASK | SVM_EVTINJ_VALID;
+        return info == (SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR);
+}
+
+static u32 svm_get_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
 {
         struct vcpu_svm *svm = to_svm(vcpu);
+        u32 ret = 0;
 
-        return !(svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_VALID);
+        if (svm->vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK)
+                ret |= X86_SHADOW_INT_STI | X86_SHADOW_INT_MOV_SS;
+        return ret & mask;
 }
 
-static int is_external_interrupt(u32 info)
+static void svm_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
 {
-        info &= SVM_EVTINJ_TYPE_MASK | SVM_EVTINJ_VALID;
-        return info == (SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR);
+        struct vcpu_svm *svm = to_svm(vcpu);
+
+        if (mask == 0)
+                svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK;
+        else
+                svm->vmcb->control.int_state |= SVM_INTERRUPT_SHADOW_MASK;
+
 }
 
 static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
@@ -232,7 +228,9 @@ static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
         struct vcpu_svm *svm = to_svm(vcpu);
 
         if (!svm->next_rip) {
-                printk(KERN_DEBUG "%s: NOP\n", __func__);
+                if (emulate_instruction(vcpu, vcpu->run, 0, 0, EMULTYPE_SKIP) !=
+                                EMULATE_DONE)
+                        printk(KERN_DEBUG "%s: NOP\n", __func__);
                 return;
         }
         if (svm->next_rip - kvm_rip_read(vcpu) > MAX_INST_SIZE)
@@ -240,9 +238,7 @@ static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
                        __func__, kvm_rip_read(vcpu), svm->next_rip);
 
         kvm_rip_write(vcpu, svm->next_rip);
-        svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK;
-
-        vcpu->arch.interrupt_window_open = (svm->vcpu.arch.hflags & HF_GIF_MASK);
+        svm_set_interrupt_shadow(vcpu, 0);
 }
 
 static int has_svm(void)
@@ -830,6 +826,15 @@ static void svm_get_segment(struct kvm_vcpu *vcpu,
                 if (!var->unusable)
                         var->type |= 0x1;
                 break;
+        case VCPU_SREG_SS:
+                /* On AMD CPUs sometimes the DB bit in the segment
+                 * descriptor is left as 1, although the whole segment has
+                 * been made unusable. Clear it here to pass an Intel VMX
+                 * entry check when cross vendor migrating.
+                 */
+                if (var->unusable)
+                        var->db = 0;
+                break;
         }
 }
 
@@ -960,15 +965,16 @@ static void svm_set_segment(struct kvm_vcpu *vcpu,
 
 }
 
-static int svm_guest_debug(struct kvm_vcpu *vcpu, struct kvm_guest_debug *dbg)
+static void update_db_intercept(struct kvm_vcpu *vcpu)
 {
-        int old_debug = vcpu->guest_debug;
         struct vcpu_svm *svm = to_svm(vcpu);
 
-        vcpu->guest_debug = dbg->control;
-
         svm->vmcb->control.intercept_exceptions &=
                 ~((1 << DB_VECTOR) | (1 << BP_VECTOR));
+
+        if (vcpu->arch.singlestep)
+                svm->vmcb->control.intercept_exceptions |= (1 << DB_VECTOR);
+
         if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) {
                 if (vcpu->guest_debug &
                     (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))
@@ -979,6 +985,16 @@ static int svm_guest_debug(struct kvm_vcpu *vcpu, struct kvm_guest_debug *dbg)
                                 1 << BP_VECTOR;
         } else
                 vcpu->guest_debug = 0;
+}
+
+static int svm_guest_debug(struct kvm_vcpu *vcpu, struct kvm_guest_debug *dbg)
+{
+        int old_debug = vcpu->guest_debug;
+        struct vcpu_svm *svm = to_svm(vcpu);
+
+        vcpu->guest_debug = dbg->control;
+
+        update_db_intercept(vcpu);
 
         if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
                 svm->vmcb->save.dr7 = dbg->arch.debugreg[7];
@@ -993,16 +1009,6 @@ static int svm_guest_debug(struct kvm_vcpu *vcpu, struct kvm_guest_debug *dbg)
         return 0;
 }
 
-static int svm_get_irq(struct kvm_vcpu *vcpu)
-{
-        struct vcpu_svm *svm = to_svm(vcpu);
-        u32 exit_int_info = svm->vmcb->control.exit_int_info;
-
-        if (is_external_interrupt(exit_int_info))
-                return exit_int_info & SVM_EVTINJ_VEC_MASK;
-        return -1;
-}
-
 static void load_host_msrs(struct kvm_vcpu *vcpu)
 {
 #ifdef CONFIG_X86_64
@@ -1107,17 +1113,8 @@ static void svm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long value,
 
 static int pf_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
 {
-        u32 exit_int_info = svm->vmcb->control.exit_int_info;
-        struct kvm *kvm = svm->vcpu.kvm;
         u64 fault_address;
         u32 error_code;
-        bool event_injection = false;
-
-        if (!irqchip_in_kernel(kvm) &&
-            is_external_interrupt(exit_int_info)) {
-                event_injection = true;
-                push_irq(&svm->vcpu, exit_int_info & SVM_EVTINJ_VEC_MASK);
-        }
 
         fault_address  = svm->vmcb->control.exit_info_2;
         error_code = svm->vmcb->control.exit_info_1;
@@ -1137,23 +1134,40 @@ static int pf_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
          */
         if (npt_enabled)
                 svm_flush_tlb(&svm->vcpu);
-
-        if (!npt_enabled && event_injection)
-                kvm_mmu_unprotect_page_virt(&svm->vcpu, fault_address);
+        else {
+                if (kvm_event_needs_reinjection(&svm->vcpu))
+                        kvm_mmu_unprotect_page_virt(&svm->vcpu, fault_address);
+        }
         return kvm_mmu_page_fault(&svm->vcpu, fault_address, error_code);
 }
 
 static int db_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
 {
         if (!(svm->vcpu.guest_debug &
-              (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))) {
+              (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) &&
+                !svm->vcpu.arch.singlestep) {
                 kvm_queue_exception(&svm->vcpu, DB_VECTOR);
                 return 1;
         }
-        kvm_run->exit_reason = KVM_EXIT_DEBUG;
-        kvm_run->debug.arch.pc = svm->vmcb->save.cs.base + svm->vmcb->save.rip;
-        kvm_run->debug.arch.exception = DB_VECTOR;
-        return 0;
+
+        if (svm->vcpu.arch.singlestep) {
+                svm->vcpu.arch.singlestep = false;
+                if (!(svm->vcpu.guest_debug & KVM_GUESTDBG_SINGLESTEP))
+                        svm->vmcb->save.rflags &=
+                                ~(X86_EFLAGS_TF | X86_EFLAGS_RF);
+                update_db_intercept(&svm->vcpu);
+        }
+
+        if (svm->vcpu.guest_debug &
+            (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)){
+                kvm_run->exit_reason = KVM_EXIT_DEBUG;
+                kvm_run->debug.arch.pc =
+                        svm->vmcb->save.cs.base + svm->vmcb->save.rip;
+                kvm_run->debug.arch.exception = DB_VECTOR;
+                return 0;
+        }
+
+        return 1;
 }
 
 static int bp_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
@@ -1842,17 +1856,51 @@ static int task_switch_interception(struct vcpu_svm *svm,
                                     struct kvm_run *kvm_run)
 {
         u16 tss_selector;
+        int reason;
+        int int_type = svm->vmcb->control.exit_int_info &
+                SVM_EXITINTINFO_TYPE_MASK;
+        int int_vec = svm->vmcb->control.exit_int_info & SVM_EVTINJ_VEC_MASK;
+        uint32_t type =
+                svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_TYPE_MASK;
+        uint32_t idt_v =
+                svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_VALID;
 
         tss_selector = (u16)svm->vmcb->control.exit_info_1;
+
         if (svm->vmcb->control.exit_info_2 &
             (1ULL << SVM_EXITINFOSHIFT_TS_REASON_IRET))
-                return kvm_task_switch(&svm->vcpu, tss_selector,
-                                       TASK_SWITCH_IRET);
-        if (svm->vmcb->control.exit_info_2 &
-            (1ULL << SVM_EXITINFOSHIFT_TS_REASON_JMP))
-                return kvm_task_switch(&svm->vcpu, tss_selector,
-                                       TASK_SWITCH_JMP);
-        return kvm_task_switch(&svm->vcpu, tss_selector, TASK_SWITCH_CALL);
+                reason = TASK_SWITCH_IRET;
+        else if (svm->vmcb->control.exit_info_2 &
+                 (1ULL << SVM_EXITINFOSHIFT_TS_REASON_JMP))
+                reason = TASK_SWITCH_JMP;
+        else if (idt_v)
+                reason = TASK_SWITCH_GATE;
+        else
+                reason = TASK_SWITCH_CALL;
+
+        if (reason == TASK_SWITCH_GATE) {
+                switch (type) {
+                case SVM_EXITINTINFO_TYPE_NMI:
+                        svm->vcpu.arch.nmi_injected = false;
+                        break;
+                case SVM_EXITINTINFO_TYPE_EXEPT:
+                        kvm_clear_exception_queue(&svm->vcpu);
+                        break;
+                case SVM_EXITINTINFO_TYPE_INTR:
+                        kvm_clear_interrupt_queue(&svm->vcpu);
+                        break;
+                default:
+                        break;
+                }
+        }
+
+        if (reason != TASK_SWITCH_GATE ||
+            int_type == SVM_EXITINTINFO_TYPE_SOFT ||
+            (int_type == SVM_EXITINTINFO_TYPE_EXEPT &&
+             (int_vec == OF_VECTOR || int_vec == BP_VECTOR)))
+                skip_emulated_instruction(&svm->vcpu);
+
+        return kvm_task_switch(&svm->vcpu, tss_selector, reason);
 }
 
 static int cpuid_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
@@ -1862,6 +1910,14 @@ static int cpuid_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
         return 1;
 }
 
+static int iret_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
+{
+        ++svm->vcpu.stat.nmi_window_exits;
+        svm->vmcb->control.intercept &= ~(1UL << INTERCEPT_IRET);
+        svm->vcpu.arch.hflags |= HF_IRET_MASK;
+        return 1;
+}
+
 static int invlpg_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
 {
         if (emulate_instruction(&svm->vcpu, kvm_run, 0, 0, 0) != EMULATE_DONE)
@@ -1879,8 +1935,14 @@ static int emulate_on_interception(struct vcpu_svm *svm,
 
 static int cr8_write_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
 {
+        u8 cr8_prev = kvm_get_cr8(&svm->vcpu);
+        /* instruction emulation calls kvm_set_cr8() */
         emulate_instruction(&svm->vcpu, NULL, 0, 0, 0);
-        if (irqchip_in_kernel(svm->vcpu.kvm))
+        if (irqchip_in_kernel(svm->vcpu.kvm)) {
+                svm->vmcb->control.intercept_cr_write &= ~INTERCEPT_CR8_MASK;
+                return 1;
+        }
+        if (cr8_prev <= kvm_get_cr8(&svm->vcpu))
                 return 1;
         kvm_run->exit_reason = KVM_EXIT_SET_TPR;
         return 0;
@@ -2090,8 +2152,9 @@ static int interrupt_window_interception(struct vcpu_svm *svm,
          * If the user space waits to inject interrupts, exit as soon as
          * possible
          */
-        if (kvm_run->request_interrupt_window &&
-            !svm->vcpu.arch.irq_summary) {
+        if (!irqchip_in_kernel(svm->vcpu.kvm) &&
+            kvm_run->request_interrupt_window &&
+            !kvm_cpu_has_interrupt(&svm->vcpu)) {
                 ++svm->vcpu.stat.irq_window_exits;
                 kvm_run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;
                 return 0;
@@ -2134,6 +2197,7 @@ static int (*svm_exit_handlers[])(struct vcpu_svm *svm,
         [SVM_EXIT_VINTR]                        = interrupt_window_interception,
         /* [SVM_EXIT_CR0_SEL_WRITE]             = emulate_on_interception, */
         [SVM_EXIT_CPUID]                        = cpuid_interception,
+        [SVM_EXIT_IRET]                         = iret_interception,
         [SVM_EXIT_INVD]                         = emulate_on_interception,
         [SVM_EXIT_HLT]                          = halt_interception,
         [SVM_EXIT_INVLPG]                       = invlpg_interception,
@@ -2194,7 +2258,6 @@ static int handle_exit(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
                 }
         }
 
-        kvm_reput_irq(svm);
 
         if (svm->vmcb->control.exit_code == SVM_EXIT_ERR) {
                 kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY;
@@ -2205,7 +2268,7 @@ static int handle_exit(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
 
         if (is_external_interrupt(svm->vmcb->control.exit_int_info) &&
             exit_code != SVM_EXIT_EXCP_BASE + PF_VECTOR &&
-            exit_code != SVM_EXIT_NPF)
+            exit_code != SVM_EXIT_NPF && exit_code != SVM_EXIT_TASK_SWITCH)
                 printk(KERN_ERR "%s: unexpected exit_ini_info 0x%x "
                        "exit_code 0x%x\n",
                        __func__, svm->vmcb->control.exit_int_info,
@@ -2242,6 +2305,15 @@ static void pre_svm_run(struct vcpu_svm *svm)
                 new_asid(svm, svm_data);
 }
 
+static void svm_inject_nmi(struct kvm_vcpu *vcpu)
+{
+        struct vcpu_svm *svm = to_svm(vcpu);
+
+        svm->vmcb->control.event_inj = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI;
+        vcpu->arch.hflags |= HF_NMI_MASK;
+        svm->vmcb->control.intercept |= (1UL << INTERCEPT_IRET);
+        ++vcpu->stat.nmi_injections;
+}
 
 static inline void svm_inject_irq(struct vcpu_svm *svm, int irq)
 {
@@ -2257,134 +2329,71 @@ static inline void svm_inject_irq(struct vcpu_svm *svm, int irq)
                 ((/*control->int_vector >> 4*/ 0xf) << V_INTR_PRIO_SHIFT);
 }
 
-static void svm_set_irq(struct kvm_vcpu *vcpu, int irq)
+static void svm_queue_irq(struct kvm_vcpu *vcpu, unsigned nr)
 {
         struct vcpu_svm *svm = to_svm(vcpu);
 
-        nested_svm_intr(svm);
-
-        svm_inject_irq(svm, irq);
+        svm->vmcb->control.event_inj = nr |
+                SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR;
 }
 
-static void update_cr8_intercept(struct kvm_vcpu *vcpu)
+static void svm_set_irq(struct kvm_vcpu *vcpu)
 {
         struct vcpu_svm *svm = to_svm(vcpu);
-        struct vmcb *vmcb = svm->vmcb;
-        int max_irr, tpr;
 
-        if (!irqchip_in_kernel(vcpu->kvm) || vcpu->arch.apic->vapic_addr)
-                return;
+        nested_svm_intr(svm);
 
-        vmcb->control.intercept_cr_write &= ~INTERCEPT_CR8_MASK;
+        svm_queue_irq(vcpu, vcpu->arch.interrupt.nr);
+}
 
-        max_irr = kvm_lapic_find_highest_irr(vcpu);
-        if (max_irr == -1)
-                return;
+static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
+{
+        struct vcpu_svm *svm = to_svm(vcpu);
 
-        tpr = kvm_lapic_get_cr8(vcpu) << 4;
+        if (irr == -1)
+                return;
 
-        if (tpr >= (max_irr & 0xf0))
-                vmcb->control.intercept_cr_write |= INTERCEPT_CR8_MASK;
+        if (tpr >= irr)
+                svm->vmcb->control.intercept_cr_write |= INTERCEPT_CR8_MASK;
 }
 
-static void svm_intr_assist(struct kvm_vcpu *vcpu)
+static int svm_nmi_allowed(struct kvm_vcpu *vcpu)
 {
         struct vcpu_svm *svm = to_svm(vcpu);
         struct vmcb *vmcb = svm->vmcb;
-        int intr_vector = -1;
-
-        if ((vmcb->control.exit_int_info & SVM_EVTINJ_VALID) &&
-            ((vmcb->control.exit_int_info & SVM_EVTINJ_TYPE_MASK) == 0)) {
-                intr_vector = vmcb->control.exit_int_info &
-                              SVM_EVTINJ_VEC_MASK;
-                vmcb->control.exit_int_info = 0;
-                svm_inject_irq(svm, intr_vector);
-                goto out;
-        }
-
-        if (vmcb->control.int_ctl & V_IRQ_MASK)
-                goto out;
-
-        if (!kvm_cpu_has_interrupt(vcpu))
-                goto out;
-
-        if (nested_svm_intr(svm))
-                goto out;
-
-        if (!(svm->vcpu.arch.hflags & HF_GIF_MASK))
-                goto out;
-
-        if (!(vmcb->save.rflags & X86_EFLAGS_IF) ||
-            (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) ||
-            (vmcb->control.event_inj & SVM_EVTINJ_VALID)) {
-                /* unable to deliver irq, set pending irq */
-                svm_set_vintr(svm);
-                svm_inject_irq(svm, 0x0);
-                goto out;
-        }
-        /* Okay, we can deliver the interrupt: grab it and update PIC state. */
-        intr_vector = kvm_cpu_get_interrupt(vcpu);
-        svm_inject_irq(svm, intr_vector);
-out:
-        update_cr8_intercept(vcpu);
+        return !(vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) &&
+                !(svm->vcpu.arch.hflags & HF_NMI_MASK);
 }
 
-static void kvm_reput_irq(struct vcpu_svm *svm)
+static int svm_interrupt_allowed(struct kvm_vcpu *vcpu)
 {
-        struct vmcb_control_area *control = &svm->vmcb->control;
-
-        if ((control->int_ctl & V_IRQ_MASK)
-            && !irqchip_in_kernel(svm->vcpu.kvm)) {
-                control->int_ctl &= ~V_IRQ_MASK;
-                push_irq(&svm->vcpu, control->int_vector);
-        }
-
-        svm->vcpu.arch.interrupt_window_open =
-                !(control->int_state & SVM_INTERRUPT_SHADOW_MASK) &&
-                 (svm->vcpu.arch.hflags & HF_GIF_MASK);
+        struct vcpu_svm *svm = to_svm(vcpu);
+        struct vmcb *vmcb = svm->vmcb;
+        return (vmcb->save.rflags & X86_EFLAGS_IF) &&
+                !(vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) &&
+                (svm->vcpu.arch.hflags & HF_GIF_MASK);
 }
 
-static void svm_do_inject_vector(struct vcpu_svm *svm)
+static void enable_irq_window(struct kvm_vcpu *vcpu)
 {
-        struct kvm_vcpu *vcpu = &svm->vcpu;
-        int word_index = __ffs(vcpu->arch.irq_summary);
-        int bit_index = __ffs(vcpu->arch.irq_pending[word_index]);
-        int irq = word_index * BITS_PER_LONG + bit_index;
-
-        clear_bit(bit_index, &vcpu->arch.irq_pending[word_index]);
-        if (!vcpu->arch.irq_pending[word_index])
-                clear_bit(word_index, &vcpu->arch.irq_summary);
-        svm_inject_irq(svm, irq);
+        svm_set_vintr(to_svm(vcpu));
+        svm_inject_irq(to_svm(vcpu), 0x0);
 }
 
-static void do_interrupt_requests(struct kvm_vcpu *vcpu,
-                                       struct kvm_run *kvm_run)
+static void enable_nmi_window(struct kvm_vcpu *vcpu)
 {
         struct vcpu_svm *svm = to_svm(vcpu);
-        struct vmcb_control_area *control = &svm->vmcb->control;
-
-        if (nested_svm_intr(svm))
-                return;
 
-        svm->vcpu.arch.interrupt_window_open =
-                (!(control->int_state & SVM_INTERRUPT_SHADOW_MASK) &&
-                 (svm->vmcb->save.rflags & X86_EFLAGS_IF) &&
-                 (svm->vcpu.arch.hflags & HF_GIF_MASK));
+        if ((svm->vcpu.arch.hflags & (HF_NMI_MASK | HF_IRET_MASK))
+            == HF_NMI_MASK)
+                return; /* IRET will cause a vm exit */
 
-        if (svm->vcpu.arch.interrupt_window_open && svm->vcpu.arch.irq_summary)
-                /*
-                 * If interrupts enabled, and not blocked by sti or mov ss. Good.
-                 */
-                svm_do_inject_vector(svm);
-
-        /*
-         * Interrupts blocked.  Wait for unblock.
-         */
-        if (!svm->vcpu.arch.interrupt_window_open &&
-            (svm->vcpu.arch.irq_summary || kvm_run->request_interrupt_window))
-                svm_set_vintr(svm);
-        else
-                svm_clear_vintr(svm);
+        /* Something prevents NMI from been injected. Single step over
+           possible problem (IRET or exception injection or interrupt
+           shadow) */
+        vcpu->arch.singlestep = true;
+        svm->vmcb->save.rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
+        update_db_intercept(vcpu);
 }
 
 static int svm_set_tss_addr(struct kvm *kvm, unsigned int addr)
@@ -2407,7 +2416,7 @@ static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu)
 
         if (!(svm->vmcb->control.intercept_cr_write & INTERCEPT_CR8_MASK)) {
                 int cr8 = svm->vmcb->control.int_ctl & V_TPR_MASK;
-                kvm_lapic_set_tpr(vcpu, cr8);
+                kvm_set_cr8(vcpu, cr8);
         }
 }
 
@@ -2416,14 +2425,54 @@ static inline void sync_lapic_to_cr8(struct kvm_vcpu *vcpu)
         struct vcpu_svm *svm = to_svm(vcpu);
         u64 cr8;
 
-        if (!irqchip_in_kernel(vcpu->kvm))
-                return;
-
         cr8 = kvm_get_cr8(vcpu);
         svm->vmcb->control.int_ctl &= ~V_TPR_MASK;
         svm->vmcb->control.int_ctl |= cr8 & V_TPR_MASK;
 }
 
+static void svm_complete_interrupts(struct vcpu_svm *svm)
+{
+        u8 vector;
+        int type;
+        u32 exitintinfo = svm->vmcb->control.exit_int_info;
+
+        if (svm->vcpu.arch.hflags & HF_IRET_MASK)
+                svm->vcpu.arch.hflags &= ~(HF_NMI_MASK | HF_IRET_MASK);
+
+        svm->vcpu.arch.nmi_injected = false;
+        kvm_clear_exception_queue(&svm->vcpu);
+        kvm_clear_interrupt_queue(&svm->vcpu);
+
+        if (!(exitintinfo & SVM_EXITINTINFO_VALID))
+                return;
+
+        vector = exitintinfo & SVM_EXITINTINFO_VEC_MASK;
+        type = exitintinfo & SVM_EXITINTINFO_TYPE_MASK;
+
+        switch (type) {
+        case SVM_EXITINTINFO_TYPE_NMI:
+                svm->vcpu.arch.nmi_injected = true;
+                break;
+        case SVM_EXITINTINFO_TYPE_EXEPT:
+                /* In case of software exception do not reinject an exception
+                   vector, but re-execute and instruction instead */
+                if (kvm_exception_is_soft(vector))
+                        break;
+                if (exitintinfo & SVM_EXITINTINFO_VALID_ERR) {
+                        u32 err = svm->vmcb->control.exit_int_info_err;
+                        kvm_queue_exception_e(&svm->vcpu, vector, err);
+
+                } else
+                        kvm_queue_exception(&svm->vcpu, vector);
+                break;
+        case SVM_EXITINTINFO_TYPE_INTR:
+                kvm_queue_interrupt(&svm->vcpu, vector, false);
+                break;
+        default:
+                break;
+        }
+}
+
 #ifdef CONFIG_X86_64
 #define R "r"
 #else
@@ -2552,6 +2601,8 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
         sync_cr8_to_lapic(vcpu);
 
         svm->next_rip = 0;
+
+        svm_complete_interrupts(svm);
 }
 
 #undef R
@@ -2617,7 +2668,7 @@ static int get_npt_level(void)
 #endif
 }
 
-static int svm_get_mt_mask_shift(void)
+static u64 svm_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
 {
         return 0;
 }
@@ -2667,17 +2718,21 @@ static struct kvm_x86_ops svm_x86_ops = {
         .run = svm_vcpu_run,
         .handle_exit = handle_exit,
         .skip_emulated_instruction = skip_emulated_instruction,
+        .set_interrupt_shadow = svm_set_interrupt_shadow,
+        .get_interrupt_shadow = svm_get_interrupt_shadow,
         .patch_hypercall = svm_patch_hypercall,
-        .get_irq = svm_get_irq,
         .set_irq = svm_set_irq,
+        .set_nmi = svm_inject_nmi,
         .queue_exception = svm_queue_exception,
-        .exception_injected = svm_exception_injected,
-        .inject_pending_irq = svm_intr_assist,
-        .inject_pending_vectors = do_interrupt_requests,
+        .interrupt_allowed = svm_interrupt_allowed,
+        .nmi_allowed = svm_nmi_allowed,
+        .enable_nmi_window = enable_nmi_window,
+        .enable_irq_window = enable_irq_window,
+        .update_cr8_intercept = update_cr8_intercept,
 
         .set_tss_addr = svm_set_tss_addr,
         .get_tdp_level = get_npt_level,
-        .get_mt_mask_shift = svm_get_mt_mask_shift,
+        .get_mt_mask = svm_get_mt_mask,
 };
 
 static int __init svm_init(void)
diff --git a/arch/x86/kvm/timer.c b/arch/x86/kvm/timer.c
new file mode 100644
index 000000000000..86dbac072d0c
--- /dev/null
+++ b/arch/x86/kvm/timer.c
@@ -0,0 +1,46 @@
+#include <linux/kvm_host.h>
+#include <linux/kvm.h>
+#include <linux/hrtimer.h>
+#include <asm/atomic.h>
+#include "kvm_timer.h"
+
+static int __kvm_timer_fn(struct kvm_vcpu *vcpu, struct kvm_timer *ktimer)
+{
+        int restart_timer = 0;
+        wait_queue_head_t *q = &vcpu->wq;
+
+        /* FIXME: this code should not know anything about vcpus */
+        if (!atomic_inc_and_test(&ktimer->pending))
+                set_bit(KVM_REQ_PENDING_TIMER, &vcpu->requests);
+
+        if (!ktimer->reinject)
+                atomic_set(&ktimer->pending, 1);
+
+        if (waitqueue_active(q))
+                wake_up_interruptible(q);
+
+        if (ktimer->t_ops->is_periodic(ktimer)) {
+                hrtimer_add_expires_ns(&ktimer->timer, ktimer->period);
+                restart_timer = 1;
+        }
+
+        return restart_timer;
+}
+
+enum hrtimer_restart kvm_timer_fn(struct hrtimer *data)
+{
+        int restart_timer;
+        struct kvm_vcpu *vcpu;
+        struct kvm_timer *ktimer = container_of(data, struct kvm_timer, timer);
+
+        vcpu = ktimer->kvm->vcpus[ktimer->vcpu_id];
+        if (!vcpu)
+                return HRTIMER_NORESTART;
+
+        restart_timer = __kvm_timer_fn(vcpu, ktimer);
+        if (restart_timer)
+                return HRTIMER_RESTART;
+        else
+                return HRTIMER_NORESTART;
+}
+
diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index bb481330716f..32d6ae8fb60e 100644
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -32,26 +32,27 @@
 #include <asm/desc.h>
 #include <asm/vmx.h>
 #include <asm/virtext.h>
+#include <asm/mce.h>
 
 #define __ex(x) __kvm_handle_fault_on_reboot(x)
 
 MODULE_AUTHOR("Qumranet");
 MODULE_LICENSE("GPL");
 
-static int bypass_guest_pf = 1;
-module_param(bypass_guest_pf, bool, 0);
+static int __read_mostly bypass_guest_pf = 1;
+module_param(bypass_guest_pf, bool, S_IRUGO);
 
-static int enable_vpid = 1;
-module_param(enable_vpid, bool, 0);
+static int __read_mostly enable_vpid = 1;
+module_param_named(vpid, enable_vpid, bool, 0444);
 
-static int flexpriority_enabled = 1;
-module_param(flexpriority_enabled, bool, 0);
+static int __read_mostly flexpriority_enabled = 1;
+module_param_named(flexpriority, flexpriority_enabled, bool, S_IRUGO);
 
-static int enable_ept = 1;
-module_param(enable_ept, bool, 0);
+static int __read_mostly enable_ept = 1;
+module_param_named(ept, enable_ept, bool, S_IRUGO);
 
-static int emulate_invalid_guest_state = 0;
-module_param(emulate_invalid_guest_state, bool, 0);
+static int __read_mostly emulate_invalid_guest_state = 0;
+module_param(emulate_invalid_guest_state, bool, S_IRUGO);
 
 struct vmcs {
         u32 revision_id;
@@ -97,6 +98,7 @@ struct vcpu_vmx {
         int soft_vnmi_blocked;
         ktime_t entry_time;
         s64 vnmi_blocked_time;
+        u32 exit_reason;
 };
 
 static inline struct vcpu_vmx *to_vmx(struct kvm_vcpu *vcpu)
@@ -111,9 +113,10 @@ static DEFINE_PER_CPU(struct vmcs *, vmxarea);
 static DEFINE_PER_CPU(struct vmcs *, current_vmcs);
 static DEFINE_PER_CPU(struct list_head, vcpus_on_cpu);
 
-static struct page *vmx_io_bitmap_a;
-static struct page *vmx_io_bitmap_b;
-static struct page *vmx_msr_bitmap;
+static unsigned long *vmx_io_bitmap_a;
+static unsigned long *vmx_io_bitmap_b;
+static unsigned long *vmx_msr_bitmap_legacy;
+static unsigned long *vmx_msr_bitmap_longmode;
 
 static DECLARE_BITMAP(vmx_vpid_bitmap, VMX_NR_VPIDS);
 static DEFINE_SPINLOCK(vmx_vpid_lock);
@@ -213,70 +216,78 @@ static inline int is_external_interrupt(u32 intr_info)
                 == (INTR_TYPE_EXT_INTR | INTR_INFO_VALID_MASK);
 }
 
+static inline int is_machine_check(u32 intr_info)
+{
+        return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
+                             INTR_INFO_VALID_MASK)) ==
+                (INTR_TYPE_HARD_EXCEPTION | MC_VECTOR | INTR_INFO_VALID_MASK);
+}
+
 static inline int cpu_has_vmx_msr_bitmap(void)
 {
-        return (vmcs_config.cpu_based_exec_ctrl & CPU_BASED_USE_MSR_BITMAPS);
+        return vmcs_config.cpu_based_exec_ctrl & CPU_BASED_USE_MSR_BITMAPS;
 }
 
 static inline int cpu_has_vmx_tpr_shadow(void)
 {
-        return (vmcs_config.cpu_based_exec_ctrl & CPU_BASED_TPR_SHADOW);
+        return vmcs_config.cpu_based_exec_ctrl & CPU_BASED_TPR_SHADOW;
 }
 
 static inline int vm_need_tpr_shadow(struct kvm *kvm)
 {
-        return ((cpu_has_vmx_tpr_shadow()) && (irqchip_in_kernel(kvm)));
+        return (cpu_has_vmx_tpr_shadow()) && (irqchip_in_kernel(kvm));
 }
 
 static inline int cpu_has_secondary_exec_ctrls(void)
 {
-        return (vmcs_config.cpu_based_exec_ctrl &
-                CPU_BASED_ACTIVATE_SECONDARY_CONTROLS);
+        return vmcs_config.cpu_based_exec_ctrl &
+                CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
 }
 
 static inline bool cpu_has_vmx_virtualize_apic_accesses(void)
 {
-        return flexpriority_enabled
-                && (vmcs_config.cpu_based_2nd_exec_ctrl &
-                    SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES);
+        return vmcs_config.cpu_based_2nd_exec_ctrl &
+                SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
+}
+
+static inline bool cpu_has_vmx_flexpriority(void)
+{
+        return cpu_has_vmx_tpr_shadow() &&
+                cpu_has_vmx_virtualize_apic_accesses();
 }
 
 static inline int cpu_has_vmx_invept_individual_addr(void)
 {
-        return (!!(vmx_capability.ept & VMX_EPT_EXTENT_INDIVIDUAL_BIT));
+        return !!(vmx_capability.ept & VMX_EPT_EXTENT_INDIVIDUAL_BIT);
 }
 
 static inline int cpu_has_vmx_invept_context(void)
 {
-        return (!!(vmx_capability.ept & VMX_EPT_EXTENT_CONTEXT_BIT));
+        return !!(vmx_capability.ept & VMX_EPT_EXTENT_CONTEXT_BIT);
 }
 
 static inline int cpu_has_vmx_invept_global(void)
 {
-        return (!!(vmx_capability.ept & VMX_EPT_EXTENT_GLOBAL_BIT));
+        return !!(vmx_capability.ept & VMX_EPT_EXTENT_GLOBAL_BIT);
 }
 
 static inline int cpu_has_vmx_ept(void)
 {
-        return (vmcs_config.cpu_based_2nd_exec_ctrl &
-                SECONDARY_EXEC_ENABLE_EPT);
-}
-
-static inline int vm_need_ept(void)
-{
-        return (cpu_has_vmx_ept() && enable_ept);
+        return vmcs_config.cpu_based_2nd_exec_ctrl &
+                SECONDARY_EXEC_ENABLE_EPT;
 }
 
 static inline int vm_need_virtualize_apic_accesses(struct kvm *kvm)
 {
-        return ((cpu_has_vmx_virtualize_apic_accesses()) &&
-                (irqchip_in_kernel(kvm)));
+        return flexpriority_enabled &&
+                (cpu_has_vmx_virtualize_apic_accesses()) &&
+                (irqchip_in_kernel(kvm));
 }
 
 static inline int cpu_has_vmx_vpid(void)
 {
-        return (vmcs_config.cpu_based_2nd_exec_ctrl &
-                SECONDARY_EXEC_ENABLE_VPID);
+        return vmcs_config.cpu_based_2nd_exec_ctrl &
+                SECONDARY_EXEC_ENABLE_VPID;
 }
 
 static inline int cpu_has_virtual_nmis(void)
@@ -284,6 +295,11 @@ static inline int cpu_has_virtual_nmis(void)
         return vmcs_config.pin_based_exec_ctrl & PIN_BASED_VIRTUAL_NMIS;
 }
 
+static inline bool report_flexpriority(void)
+{
+        return flexpriority_enabled;
+}
+
 static int __find_msr_index(struct vcpu_vmx *vmx, u32 msr)
 {
         int i;
@@ -381,7 +397,7 @@ static inline void ept_sync_global(void)
 
 static inline void ept_sync_context(u64 eptp)
 {
-        if (vm_need_ept()) {
+        if (enable_ept) {
                 if (cpu_has_vmx_invept_context())
                         __invept(VMX_EPT_EXTENT_CONTEXT, eptp, 0);
                 else
@@ -391,7 +407,7 @@ static inline void ept_sync_context(u64 eptp)
 
 static inline void ept_sync_individual_addr(u64 eptp, gpa_t gpa)
 {
-        if (vm_need_ept()) {
+        if (enable_ept) {
                 if (cpu_has_vmx_invept_individual_addr())
                         __invept(VMX_EPT_EXTENT_INDIVIDUAL_ADDR,
                                         eptp, gpa);
@@ -478,7 +494,7 @@ static void update_exception_bitmap(struct kvm_vcpu *vcpu)
 {
         u32 eb;
 
-        eb = (1u << PF_VECTOR) | (1u << UD_VECTOR);
+        eb = (1u << PF_VECTOR) | (1u << UD_VECTOR) | (1u << MC_VECTOR);
         if (!vcpu->fpu_active)
                 eb |= 1u << NM_VECTOR;
         if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) {
@@ -488,9 +504,9 @@ static void update_exception_bitmap(struct kvm_vcpu *vcpu)
                 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
                         eb |= 1u << BP_VECTOR;
         }
-        if (vcpu->arch.rmode.active)
+        if (vcpu->arch.rmode.vm86_active)
                 eb = ~0;
-        if (vm_need_ept())
+        if (enable_ept)
                 eb &= ~(1u << PF_VECTOR); /* bypass_guest_pf = 0 */
         vmcs_write32(EXCEPTION_BITMAP, eb);
 }
@@ -724,29 +740,50 @@ static unsigned long vmx_get_rflags(struct kvm_vcpu *vcpu)
 
 static void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
 {
-        if (vcpu->arch.rmode.active)
+        if (vcpu->arch.rmode.vm86_active)
                 rflags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM;
         vmcs_writel(GUEST_RFLAGS, rflags);
 }
 
+static u32 vmx_get_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
+{
+        u32 interruptibility = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
+        int ret = 0;
+
+        if (interruptibility & GUEST_INTR_STATE_STI)
+                ret |= X86_SHADOW_INT_STI;
+        if (interruptibility & GUEST_INTR_STATE_MOV_SS)
+                ret |= X86_SHADOW_INT_MOV_SS;
+
+        return ret & mask;
+}
+
+static void vmx_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
+{
+        u32 interruptibility_old = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
+        u32 interruptibility = interruptibility_old;
+
+        interruptibility &= ~(GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS);
+
+        if (mask & X86_SHADOW_INT_MOV_SS)
+                interruptibility |= GUEST_INTR_STATE_MOV_SS;
+        if (mask & X86_SHADOW_INT_STI)
+                interruptibility |= GUEST_INTR_STATE_STI;
+
+        if ((interruptibility != interruptibility_old))
+                vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, interruptibility);
+}
+
 static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
 {
         unsigned long rip;
-        u32 interruptibility;
 
         rip = kvm_rip_read(vcpu);
         rip += vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
         kvm_rip_write(vcpu, rip);
 
-        /*
-         * We emulated an instruction, so temporary interrupt blocking
-         * should be removed, if set.
-         */
-        interruptibility = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
-        if (interruptibility & 3)
-                vmcs_write32(GUEST_INTERRUPTIBILITY_INFO,
-                             interruptibility & ~3);
-        vcpu->arch.interrupt_window_open = 1;
+        /* skipping an emulated instruction also counts */
+        vmx_set_interrupt_shadow(vcpu, 0);
 }
 
 static void vmx_queue_exception(struct kvm_vcpu *vcpu, unsigned nr,
@@ -760,7 +797,7 @@ static void vmx_queue_exception(struct kvm_vcpu *vcpu, unsigned nr,
                 intr_info |= INTR_INFO_DELIVER_CODE_MASK;
         }
 
-        if (vcpu->arch.rmode.active) {
+        if (vcpu->arch.rmode.vm86_active) {
                 vmx->rmode.irq.pending = true;
                 vmx->rmode.irq.vector = nr;
                 vmx->rmode.irq.rip = kvm_rip_read(vcpu);
@@ -773,8 +810,9 @@ static void vmx_queue_exception(struct kvm_vcpu *vcpu, unsigned nr,
                 return;
         }
 
-        if (nr == BP_VECTOR || nr == OF_VECTOR) {
-                vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, 1);
+        if (kvm_exception_is_soft(nr)) {
+                vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
+                             vmx->vcpu.arch.event_exit_inst_len);
                 intr_info |= INTR_TYPE_SOFT_EXCEPTION;
         } else
                 intr_info |= INTR_TYPE_HARD_EXCEPTION;
@@ -782,11 +820,6 @@ static void vmx_queue_exception(struct kvm_vcpu *vcpu, unsigned nr,
         vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr_info);
 }
 
-static bool vmx_exception_injected(struct kvm_vcpu *vcpu)
-{
-        return false;
-}
-
 /*
  * Swap MSR entry in host/guest MSR entry array.
  */
@@ -812,6 +845,7 @@ static void move_msr_up(struct vcpu_vmx *vmx, int from, int to)
 static void setup_msrs(struct vcpu_vmx *vmx)
 {
         int save_nmsrs;
+        unsigned long *msr_bitmap;
 
         vmx_load_host_state(vmx);
         save_nmsrs = 0;
@@ -847,6 +881,15 @@ static void setup_msrs(struct vcpu_vmx *vmx)
                 __find_msr_index(vmx, MSR_KERNEL_GS_BASE);
 #endif
         vmx->msr_offset_efer = __find_msr_index(vmx, MSR_EFER);
+
+        if (cpu_has_vmx_msr_bitmap()) {
+                if (is_long_mode(&vmx->vcpu))
+                        msr_bitmap = vmx_msr_bitmap_longmode;
+                else
+                        msr_bitmap = vmx_msr_bitmap_legacy;
+
+                vmcs_write64(MSR_BITMAP, __pa(msr_bitmap));
+        }
 }
 
 /*
@@ -1034,13 +1077,6 @@ static int set_guest_debug(struct kvm_vcpu *vcpu, struct kvm_guest_debug *dbg)
         return 0;
 }
 
-static int vmx_get_irq(struct kvm_vcpu *vcpu)
-{
-        if (!vcpu->arch.interrupt.pending)
-                return -1;
-        return vcpu->arch.interrupt.nr;
-}
-
 static __init int cpu_has_kvm_support(void)
 {
         return cpu_has_vmx();
@@ -1294,6 +1330,18 @@ static __init int hardware_setup(void)
         if (boot_cpu_has(X86_FEATURE_NX))
                 kvm_enable_efer_bits(EFER_NX);
 
+        if (!cpu_has_vmx_vpid())
+                enable_vpid = 0;
+
+        if (!cpu_has_vmx_ept())
+                enable_ept = 0;
+
+        if (!cpu_has_vmx_flexpriority())
+                flexpriority_enabled = 0;
+
+        if (!cpu_has_vmx_tpr_shadow())
+                kvm_x86_ops->update_cr8_intercept = NULL;
+
         return alloc_kvm_area();
 }
 
@@ -1324,7 +1372,7 @@ static void enter_pmode(struct kvm_vcpu *vcpu)
         struct vcpu_vmx *vmx = to_vmx(vcpu);
 
         vmx->emulation_required = 1;
-        vcpu->arch.rmode.active = 0;
+        vcpu->arch.rmode.vm86_active = 0;
 
         vmcs_writel(GUEST_TR_BASE, vcpu->arch.rmode.tr.base);
         vmcs_write32(GUEST_TR_LIMIT, vcpu->arch.rmode.tr.limit);
@@ -1386,7 +1434,7 @@ static void enter_rmode(struct kvm_vcpu *vcpu)
         struct vcpu_vmx *vmx = to_vmx(vcpu);
 
         vmx->emulation_required = 1;
-        vcpu->arch.rmode.active = 1;
+        vcpu->arch.rmode.vm86_active = 1;
 
         vcpu->arch.rmode.tr.base = vmcs_readl(GUEST_TR_BASE);
         vmcs_writel(GUEST_TR_BASE, rmode_tss_base(vcpu->kvm));
@@ -1485,7 +1533,7 @@ static void exit_lmode(struct kvm_vcpu *vcpu)
 static void vmx_flush_tlb(struct kvm_vcpu *vcpu)
 {
         vpid_sync_vcpu_all(to_vmx(vcpu));
-        if (vm_need_ept())
+        if (enable_ept)
                 ept_sync_context(construct_eptp(vcpu->arch.mmu.root_hpa));
 }
 
@@ -1555,10 +1603,10 @@ static void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
 
         vmx_fpu_deactivate(vcpu);
 
-        if (vcpu->arch.rmode.active && (cr0 & X86_CR0_PE))
+        if (vcpu->arch.rmode.vm86_active && (cr0 & X86_CR0_PE))
                 enter_pmode(vcpu);
 
-        if (!vcpu->arch.rmode.active && !(cr0 & X86_CR0_PE))
+        if (!vcpu->arch.rmode.vm86_active && !(cr0 & X86_CR0_PE))
                 enter_rmode(vcpu);
 
 #ifdef CONFIG_X86_64
@@ -1570,7 +1618,7 @@ static void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
         }
 #endif
 
-        if (vm_need_ept())
+        if (enable_ept)
                 ept_update_paging_mode_cr0(&hw_cr0, cr0, vcpu);
 
         vmcs_writel(CR0_READ_SHADOW, cr0);
@@ -1599,7 +1647,7 @@ static void vmx_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
         u64 eptp;
 
         guest_cr3 = cr3;
-        if (vm_need_ept()) {
+        if (enable_ept) {
                 eptp = construct_eptp(cr3);
                 vmcs_write64(EPT_POINTER, eptp);
                 ept_sync_context(eptp);
@@ -1616,11 +1664,11 @@ static void vmx_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
 
 static void vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
 {
-        unsigned long hw_cr4 = cr4 | (vcpu->arch.rmode.active ?
+        unsigned long hw_cr4 = cr4 | (vcpu->arch.rmode.vm86_active ?
                     KVM_RMODE_VM_CR4_ALWAYS_ON : KVM_PMODE_VM_CR4_ALWAYS_ON);
 
         vcpu->arch.cr4 = cr4;
-        if (vm_need_ept())
+        if (enable_ept)
                 ept_update_paging_mode_cr4(&hw_cr4, vcpu);
 
         vmcs_writel(CR4_READ_SHADOW, cr4);
@@ -1699,7 +1747,7 @@ static void vmx_set_segment(struct kvm_vcpu *vcpu,
         struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
         u32 ar;
 
-        if (vcpu->arch.rmode.active && seg == VCPU_SREG_TR) {
+        if (vcpu->arch.rmode.vm86_active && seg == VCPU_SREG_TR) {
                 vcpu->arch.rmode.tr.selector = var->selector;
                 vcpu->arch.rmode.tr.base = var->base;
                 vcpu->arch.rmode.tr.limit = var->limit;
@@ -1709,7 +1757,7 @@ static void vmx_set_segment(struct kvm_vcpu *vcpu,
         vmcs_writel(sf->base, var->base);
         vmcs_write32(sf->limit, var->limit);
         vmcs_write16(sf->selector, var->selector);
-        if (vcpu->arch.rmode.active && var->s) {
+        if (vcpu->arch.rmode.vm86_active && var->s) {
                 /*
                  * Hack real-mode segments into vm86 compatibility.
                  */
@@ -1982,7 +2030,7 @@ static int init_rmode_identity_map(struct kvm *kvm)
         pfn_t identity_map_pfn;
         u32 tmp;
 
-        if (!vm_need_ept())
+        if (!enable_ept)
                 return 1;
         if (unlikely(!kvm->arch.ept_identity_pagetable)) {
                 printk(KERN_ERR "EPT: identity-mapping pagetable "
@@ -2071,7 +2119,7 @@ static void allocate_vpid(struct vcpu_vmx *vmx)
         int vpid;
 
         vmx->vpid = 0;
-        if (!enable_vpid || !cpu_has_vmx_vpid())
+        if (!enable_vpid)
                 return;
         spin_lock(&vmx_vpid_lock);
         vpid = find_first_zero_bit(vmx_vpid_bitmap, VMX_NR_VPIDS);
@@ -2082,9 +2130,9 @@ static void allocate_vpid(struct vcpu_vmx *vmx)
         spin_unlock(&vmx_vpid_lock);
 }
 
-static void vmx_disable_intercept_for_msr(struct page *msr_bitmap, u32 msr)
+static void __vmx_disable_intercept_for_msr(unsigned long *msr_bitmap, u32 msr)
 {
-        void *va;
+        int f = sizeof(unsigned long);
 
         if (!cpu_has_vmx_msr_bitmap())
                 return;
@@ -2094,16 +2142,21 @@ static void vmx_disable_intercept_for_msr(struct page *msr_bitmap, u32 msr)
          * have the write-low and read-high bitmap offsets the wrong way round.
          * We can control MSRs 0x00000000-0x00001fff and 0xc0000000-0xc0001fff.
          */
-        va = kmap(msr_bitmap);
         if (msr <= 0x1fff) {
-                __clear_bit(msr, va + 0x000); /* read-low */
-                __clear_bit(msr, va + 0x800); /* write-low */
+                __clear_bit(msr, msr_bitmap + 0x000 / f); /* read-low */
+                __clear_bit(msr, msr_bitmap + 0x800 / f); /* write-low */
         } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
                 msr &= 0x1fff;
-                __clear_bit(msr, va + 0x400); /* read-high */
-                __clear_bit(msr, va + 0xc00); /* write-high */
+                __clear_bit(msr, msr_bitmap + 0x400 / f); /* read-high */
+                __clear_bit(msr, msr_bitmap + 0xc00 / f); /* write-high */
         }
-        kunmap(msr_bitmap);
+}
+
+static void vmx_disable_intercept_for_msr(u32 msr, bool longmode_only)
+{
+        if (!longmode_only)
+                __vmx_disable_intercept_for_msr(vmx_msr_bitmap_legacy, msr);
+        __vmx_disable_intercept_for_msr(vmx_msr_bitmap_longmode, msr);
 }
 
 /*
@@ -2121,11 +2174,11 @@ static int vmx_vcpu_setup(struct vcpu_vmx *vmx)
         u32 exec_control;
 
         /* I/O */
-        vmcs_write64(IO_BITMAP_A, page_to_phys(vmx_io_bitmap_a));
-        vmcs_write64(IO_BITMAP_B, page_to_phys(vmx_io_bitmap_b));
+        vmcs_write64(IO_BITMAP_A, __pa(vmx_io_bitmap_a));
+        vmcs_write64(IO_BITMAP_B, __pa(vmx_io_bitmap_b));
 
         if (cpu_has_vmx_msr_bitmap())
-                vmcs_write64(MSR_BITMAP, page_to_phys(vmx_msr_bitmap));
+                vmcs_write64(MSR_BITMAP, __pa(vmx_msr_bitmap_legacy));
 
         vmcs_write64(VMCS_LINK_POINTER, -1ull); /* 22.3.1.5 */
 
@@ -2141,7 +2194,7 @@ static int vmx_vcpu_setup(struct vcpu_vmx *vmx)
                                 CPU_BASED_CR8_LOAD_EXITING;
 #endif
         }
-        if (!vm_need_ept())
+        if (!enable_ept)
                 exec_control |= CPU_BASED_CR3_STORE_EXITING |
                                 CPU_BASED_CR3_LOAD_EXITING  |
                                 CPU_BASED_INVLPG_EXITING;
@@ -2154,7 +2207,7 @@ static int vmx_vcpu_setup(struct vcpu_vmx *vmx)
                                 ~SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
                 if (vmx->vpid == 0)
                         exec_control &= ~SECONDARY_EXEC_ENABLE_VPID;
-                if (!vm_need_ept())
+                if (!enable_ept)
                         exec_control &= ~SECONDARY_EXEC_ENABLE_EPT;
                 vmcs_write32(SECONDARY_VM_EXEC_CONTROL, exec_control);
         }
@@ -2273,7 +2326,7 @@ static int vmx_vcpu_reset(struct kvm_vcpu *vcpu)
                 goto out;
         }
 
-        vmx->vcpu.arch.rmode.active = 0;
+        vmx->vcpu.arch.rmode.vm86_active = 0;
 
         vmx->soft_vnmi_blocked = 0;
 
@@ -2402,14 +2455,16 @@ static void enable_nmi_window(struct kvm_vcpu *vcpu)
         vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
 }
 
-static void vmx_inject_irq(struct kvm_vcpu *vcpu, int irq)
+static void vmx_inject_irq(struct kvm_vcpu *vcpu)
 {
         struct vcpu_vmx *vmx = to_vmx(vcpu);
+        uint32_t intr;
+        int irq = vcpu->arch.interrupt.nr;
 
         KVMTRACE_1D(INJ_VIRQ, vcpu, (u32)irq, handler);
 
         ++vcpu->stat.irq_injections;
-        if (vcpu->arch.rmode.active) {
+        if (vcpu->arch.rmode.vm86_active) {
                 vmx->rmode.irq.pending = true;
                 vmx->rmode.irq.vector = irq;
                 vmx->rmode.irq.rip = kvm_rip_read(vcpu);
@@ -2419,8 +2474,14 @@ static void vmx_inject_irq(struct kvm_vcpu *vcpu, int irq)
                 kvm_rip_write(vcpu, vmx->rmode.irq.rip - 1);
                 return;
         }
-        vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
-                        irq | INTR_TYPE_EXT_INTR | INTR_INFO_VALID_MASK);
+        intr = irq | INTR_INFO_VALID_MASK;
+        if (vcpu->arch.interrupt.soft) {
+                intr |= INTR_TYPE_SOFT_INTR;
+                vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
+                             vmx->vcpu.arch.event_exit_inst_len);
+        } else
+                intr |= INTR_TYPE_EXT_INTR;
+        vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr);
 }
 
 static void vmx_inject_nmi(struct kvm_vcpu *vcpu)
@@ -2441,7 +2502,7 @@ static void vmx_inject_nmi(struct kvm_vcpu *vcpu)
         }
 
         ++vcpu->stat.nmi_injections;
-        if (vcpu->arch.rmode.active) {
+        if (vcpu->arch.rmode.vm86_active) {
                 vmx->rmode.irq.pending = true;
                 vmx->rmode.irq.vector = NMI_VECTOR;
                 vmx->rmode.irq.rip = kvm_rip_read(vcpu);
@@ -2456,76 +2517,21 @@ static void vmx_inject_nmi(struct kvm_vcpu *vcpu)
                         INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK | NMI_VECTOR);
 }
 
-static void vmx_update_window_states(struct kvm_vcpu *vcpu)
+static int vmx_nmi_allowed(struct kvm_vcpu *vcpu)
 {
-        u32 guest_intr = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
-
-        vcpu->arch.nmi_window_open =
-                !(guest_intr & (GUEST_INTR_STATE_STI |
-                                GUEST_INTR_STATE_MOV_SS |
-                                GUEST_INTR_STATE_NMI));
         if (!cpu_has_virtual_nmis() && to_vmx(vcpu)->soft_vnmi_blocked)
-                vcpu->arch.nmi_window_open = 0;
-
-        vcpu->arch.interrupt_window_open =
-                ((vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_IF) &&
-                 !(guest_intr & (GUEST_INTR_STATE_STI |
-                                 GUEST_INTR_STATE_MOV_SS)));
-}
-
-static void kvm_do_inject_irq(struct kvm_vcpu *vcpu)
-{
-        int word_index = __ffs(vcpu->arch.irq_summary);
-        int bit_index = __ffs(vcpu->arch.irq_pending[word_index]);
-        int irq = word_index * BITS_PER_LONG + bit_index;
+                return 0;
 
-        clear_bit(bit_index, &vcpu->arch.irq_pending[word_index]);
-        if (!vcpu->arch.irq_pending[word_index])
-                clear_bit(word_index, &vcpu->arch.irq_summary);
-        kvm_queue_interrupt(vcpu, irq);
+        return  !(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) &
+                        (GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS |
+                                GUEST_INTR_STATE_NMI));
 }
 
-static void do_interrupt_requests(struct kvm_vcpu *vcpu,
-                                       struct kvm_run *kvm_run)
+static int vmx_interrupt_allowed(struct kvm_vcpu *vcpu)
 {
-        vmx_update_window_states(vcpu);
-
-        if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
-                vmcs_clear_bits(GUEST_INTERRUPTIBILITY_INFO,
-                                GUEST_INTR_STATE_STI |
-                                GUEST_INTR_STATE_MOV_SS);
-
-        if (vcpu->arch.nmi_pending && !vcpu->arch.nmi_injected) {
-                if (vcpu->arch.interrupt.pending) {
-                        enable_nmi_window(vcpu);
-                } else if (vcpu->arch.nmi_window_open) {
-                        vcpu->arch.nmi_pending = false;
-                        vcpu->arch.nmi_injected = true;
-                } else {
-                        enable_nmi_window(vcpu);
-                        return;
-                }
-        }
-        if (vcpu->arch.nmi_injected) {
-                vmx_inject_nmi(vcpu);
-                if (vcpu->arch.nmi_pending)
-                        enable_nmi_window(vcpu);
-                else if (vcpu->arch.irq_summary
-                         || kvm_run->request_interrupt_window)
-                        enable_irq_window(vcpu);
-                return;
-        }
-
-        if (vcpu->arch.interrupt_window_open) {
-                if (vcpu->arch.irq_summary && !vcpu->arch.interrupt.pending)
-                        kvm_do_inject_irq(vcpu);
-
-                if (vcpu->arch.interrupt.pending)
-                        vmx_inject_irq(vcpu, vcpu->arch.interrupt.nr);
-        }
-        if (!vcpu->arch.interrupt_window_open &&
-            (vcpu->arch.irq_summary || kvm_run->request_interrupt_window))
-                enable_irq_window(vcpu);
+        return (vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_IF) &&
+                !(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) &
+                        (GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS));
 }
 
 static int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr)
@@ -2585,6 +2591,31 @@ static int handle_rmode_exception(struct kvm_vcpu *vcpu,
         return 0;
 }
 
+/*
+ * Trigger machine check on the host. We assume all the MSRs are already set up
+ * by the CPU and that we still run on the same CPU as the MCE occurred on.
+ * We pass a fake environment to the machine check handler because we want
+ * the guest to be always treated like user space, no matter what context
+ * it used internally.
+ */
+static void kvm_machine_check(void)
+{
+#if defined(CONFIG_X86_MCE) && defined(CONFIG_X86_64)
+        struct pt_regs regs = {
+                .cs = 3, /* Fake ring 3 no matter what the guest ran on */
+                .flags = X86_EFLAGS_IF,
+        };
+
+        do_machine_check(&regs, 0);
+#endif
+}
+
+static int handle_machine_check(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
+{
+        /* already handled by vcpu_run */
+        return 1;
+}
+
 static int handle_exception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 {
         struct vcpu_vmx *vmx = to_vmx(vcpu);
@@ -2596,17 +2627,14 @@ static int handle_exception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
         vect_info = vmx->idt_vectoring_info;
         intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
 
+        if (is_machine_check(intr_info))
+                return handle_machine_check(vcpu, kvm_run);
+
         if ((vect_info & VECTORING_INFO_VALID_MASK) &&
                                                 !is_page_fault(intr_info))
                 printk(KERN_ERR "%s: unexpected, vectoring info 0x%x "
                        "intr info 0x%x\n", __func__, vect_info, intr_info);
 
-        if (!irqchip_in_kernel(vcpu->kvm) && is_external_interrupt(vect_info)) {
-                int irq = vect_info & VECTORING_INFO_VECTOR_MASK;
-                set_bit(irq, vcpu->arch.irq_pending);
-                set_bit(irq / BITS_PER_LONG, &vcpu->arch.irq_summary);
-        }
-
         if ((intr_info & INTR_INFO_INTR_TYPE_MASK) == INTR_TYPE_NMI_INTR)
                 return 1;  /* already handled by vmx_vcpu_run() */
 
@@ -2628,17 +2656,17 @@ static int handle_exception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
                 error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE);
         if (is_page_fault(intr_info)) {
                 /* EPT won't cause page fault directly */
-                if (vm_need_ept())
+                if (enable_ept)
                         BUG();
                 cr2 = vmcs_readl(EXIT_QUALIFICATION);
                 KVMTRACE_3D(PAGE_FAULT, vcpu, error_code, (u32)cr2,
                             (u32)((u64)cr2 >> 32), handler);
-                if (vcpu->arch.interrupt.pending || vcpu->arch.exception.pending)
+                if (kvm_event_needs_reinjection(vcpu))
                         kvm_mmu_unprotect_page_virt(vcpu, cr2);
                 return kvm_mmu_page_fault(vcpu, cr2, error_code);
         }
 
-        if (vcpu->arch.rmode.active &&
+        if (vcpu->arch.rmode.vm86_active &&
             handle_rmode_exception(vcpu, intr_info & INTR_INFO_VECTOR_MASK,
                                                                 error_code)) {
                 if (vcpu->arch.halt_request) {
@@ -2753,13 +2781,18 @@ static int handle_cr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
                         kvm_set_cr4(vcpu, kvm_register_read(vcpu, reg));
                         skip_emulated_instruction(vcpu);
                         return 1;
-                case 8:
-                        kvm_set_cr8(vcpu, kvm_register_read(vcpu, reg));
-                        skip_emulated_instruction(vcpu);
-                        if (irqchip_in_kernel(vcpu->kvm))
-                                return 1;
-                        kvm_run->exit_reason = KVM_EXIT_SET_TPR;
-                        return 0;
+                case 8: {
+                                u8 cr8_prev = kvm_get_cr8(vcpu);
+                                u8 cr8 = kvm_register_read(vcpu, reg);
+                                kvm_set_cr8(vcpu, cr8);
+                                skip_emulated_instruction(vcpu);
+                                if (irqchip_in_kernel(vcpu->kvm))
+                                        return 1;
+                                if (cr8_prev <= cr8)
+                                        return 1;
+                                kvm_run->exit_reason = KVM_EXIT_SET_TPR;
+                                return 0;
+                        }
                 };
                 break;
         case 2: /* clts */
@@ -2957,8 +2990,9 @@ static int handle_interrupt_window(struct kvm_vcpu *vcpu,
          * If the user space waits to inject interrupts, exit as soon as
          * possible
          */
-        if (kvm_run->request_interrupt_window &&
-            !vcpu->arch.irq_summary) {
+        if (!irqchip_in_kernel(vcpu->kvm) &&
+            kvm_run->request_interrupt_window &&
+            !kvm_cpu_has_interrupt(vcpu)) {
                 kvm_run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;
                 return 0;
         }
@@ -2980,7 +3014,7 @@ static int handle_vmcall(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 
 static int handle_invlpg(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 {
-        u64 exit_qualification = vmcs_read64(EXIT_QUALIFICATION);
+        unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
 
         kvm_mmu_invlpg(vcpu, exit_qualification);
         skip_emulated_instruction(vcpu);
@@ -2996,11 +3030,11 @@ static int handle_wbinvd(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 
 static int handle_apic_access(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 {
-        u64 exit_qualification;
+        unsigned long exit_qualification;
         enum emulation_result er;
         unsigned long offset;
 
-        exit_qualification = vmcs_read64(EXIT_QUALIFICATION);
+        exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
         offset = exit_qualification & 0xffful;
 
         er = emulate_instruction(vcpu, kvm_run, 0, 0, 0);
@@ -3019,22 +3053,41 @@ static int handle_task_switch(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
         struct vcpu_vmx *vmx = to_vmx(vcpu);
         unsigned long exit_qualification;
         u16 tss_selector;
-        int reason;
+        int reason, type, idt_v;
+
+        idt_v = (vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK);
+        type = (vmx->idt_vectoring_info & VECTORING_INFO_TYPE_MASK);
 
         exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
 
         reason = (u32)exit_qualification >> 30;
-        if (reason == TASK_SWITCH_GATE && vmx->vcpu.arch.nmi_injected &&
-            (vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK) &&
-            (vmx->idt_vectoring_info & VECTORING_INFO_TYPE_MASK)
-            == INTR_TYPE_NMI_INTR) {
-                vcpu->arch.nmi_injected = false;
-                if (cpu_has_virtual_nmis())
-                        vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
-                                      GUEST_INTR_STATE_NMI);
+        if (reason == TASK_SWITCH_GATE && idt_v) {
+                switch (type) {
+                case INTR_TYPE_NMI_INTR:
+                        vcpu->arch.nmi_injected = false;
+                        if (cpu_has_virtual_nmis())
+                                vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
+                                              GUEST_INTR_STATE_NMI);
+                        break;
+                case INTR_TYPE_EXT_INTR:
+                case INTR_TYPE_SOFT_INTR:
+                        kvm_clear_interrupt_queue(vcpu);
+                        break;
+                case INTR_TYPE_HARD_EXCEPTION:
+                case INTR_TYPE_SOFT_EXCEPTION:
+                        kvm_clear_exception_queue(vcpu);
+                        break;
+                default:
+                        break;
+                }
         }
         tss_selector = exit_qualification;
 
+        if (!idt_v || (type != INTR_TYPE_HARD_EXCEPTION &&
+                       type != INTR_TYPE_EXT_INTR &&
+                       type != INTR_TYPE_NMI_INTR))
+                skip_emulated_instruction(vcpu);
+
         if (!kvm_task_switch(vcpu, tss_selector, reason))
                 return 0;
 
@@ -3051,11 +3104,11 @@ static int handle_task_switch(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 
 static int handle_ept_violation(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 {
-        u64 exit_qualification;
+        unsigned long exit_qualification;
         gpa_t gpa;
         int gla_validity;
 
-        exit_qualification = vmcs_read64(EXIT_QUALIFICATION);
+        exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
 
         if (exit_qualification & (1 << 6)) {
                 printk(KERN_ERR "EPT: GPA exceeds GAW!\n");
@@ -3067,7 +3120,7 @@ static int handle_ept_violation(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
                 printk(KERN_ERR "EPT: Handling EPT violation failed!\n");
                 printk(KERN_ERR "EPT: GPA: 0x%lx, GVA: 0x%lx\n",
                         (long unsigned int)vmcs_read64(GUEST_PHYSICAL_ADDRESS),
-                        (long unsigned int)vmcs_read64(GUEST_LINEAR_ADDRESS));
+                        vmcs_readl(GUEST_LINEAR_ADDRESS));
                 printk(KERN_ERR "EPT: Exit qualification is 0x%lx\n",
                         (long unsigned int)exit_qualification);
                 kvm_run->exit_reason = KVM_EXIT_UNKNOWN;
@@ -3150,6 +3203,7 @@ static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu,
         [EXIT_REASON_WBINVD]                  = handle_wbinvd,
         [EXIT_REASON_TASK_SWITCH]             = handle_task_switch,
         [EXIT_REASON_EPT_VIOLATION]           = handle_ept_violation,
+        [EXIT_REASON_MCE_DURING_VMENTRY]      = handle_machine_check,
 };
 
 static const int kvm_vmx_max_exit_handlers =
@@ -3159,10 +3213,10 @@ static const int kvm_vmx_max_exit_handlers =
  * The guest has exited.  See if we can fix it or if we need userspace
  * assistance.
  */
-static int kvm_handle_exit(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
+static int vmx_handle_exit(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
 {
-        u32 exit_reason = vmcs_read32(VM_EXIT_REASON);
         struct vcpu_vmx *vmx = to_vmx(vcpu);
+        u32 exit_reason = vmx->exit_reason;
         u32 vectoring_info = vmx->idt_vectoring_info;
 
         KVMTRACE_3D(VMEXIT, vcpu, exit_reason, (u32)kvm_rip_read(vcpu),
@@ -3178,7 +3232,7 @@ static int kvm_handle_exit(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
 
         /* Access CR3 don't cause VMExit in paging mode, so we need
          * to sync with guest real CR3. */
-        if (vm_need_ept() && is_paging(vcpu)) {
+        if (enable_ept && is_paging(vcpu)) {
                 vcpu->arch.cr3 = vmcs_readl(GUEST_CR3);
                 ept_load_pdptrs(vcpu);
         }
@@ -3199,9 +3253,8 @@ static int kvm_handle_exit(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
                        __func__, vectoring_info, exit_reason);
 
         if (unlikely(!cpu_has_virtual_nmis() && vmx->soft_vnmi_blocked)) {
-                if (vcpu->arch.interrupt_window_open) {
+                if (vmx_interrupt_allowed(vcpu)) {
                         vmx->soft_vnmi_blocked = 0;
-                        vcpu->arch.nmi_window_open = 1;
                 } else if (vmx->vnmi_blocked_time > 1000000000LL &&
                            vcpu->arch.nmi_pending) {
                         /*
@@ -3214,7 +3267,6 @@ static int kvm_handle_exit(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
                                "state on VCPU %d after 1 s timeout\n",
                                __func__, vcpu->vcpu_id);
                         vmx->soft_vnmi_blocked = 0;
-                        vmx->vcpu.arch.nmi_window_open = 1;
                 }
         }
 
@@ -3228,122 +3280,107 @@ static int kvm_handle_exit(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
         return 0;
 }
 
-static void update_tpr_threshold(struct kvm_vcpu *vcpu)
+static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
 {
-        int max_irr, tpr;
-
-        if (!vm_need_tpr_shadow(vcpu->kvm))
-                return;
-
-        if (!kvm_lapic_enabled(vcpu) ||
-            ((max_irr = kvm_lapic_find_highest_irr(vcpu)) == -1)) {
+        if (irr == -1 || tpr < irr) {
                 vmcs_write32(TPR_THRESHOLD, 0);
                 return;
         }
 
-        tpr = (kvm_lapic_get_cr8(vcpu) & 0x0f) << 4;
-        vmcs_write32(TPR_THRESHOLD, (max_irr > tpr) ? tpr >> 4 : max_irr >> 4);
+        vmcs_write32(TPR_THRESHOLD, irr);
 }
 
 static void vmx_complete_interrupts(struct vcpu_vmx *vmx)
 {
         u32 exit_intr_info;
-        u32 idt_vectoring_info;
+        u32 idt_vectoring_info = vmx->idt_vectoring_info;
         bool unblock_nmi;
         u8 vector;
         int type;
         bool idtv_info_valid;
-        u32 error;
 
         exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
+
+        vmx->exit_reason = vmcs_read32(VM_EXIT_REASON);
+
+        /* Handle machine checks before interrupts are enabled */
+        if ((vmx->exit_reason == EXIT_REASON_MCE_DURING_VMENTRY)
+            || (vmx->exit_reason == EXIT_REASON_EXCEPTION_NMI
+                && is_machine_check(exit_intr_info)))
+                kvm_machine_check();
+
+        /* We need to handle NMIs before interrupts are enabled */
+        if ((exit_intr_info & INTR_INFO_INTR_TYPE_MASK) == INTR_TYPE_NMI_INTR &&
+            (exit_intr_info & INTR_INFO_VALID_MASK)) {
+                KVMTRACE_0D(NMI, &vmx->vcpu, handler);
+                asm("int $2");
+        }
+
+        idtv_info_valid = idt_vectoring_info & VECTORING_INFO_VALID_MASK;
+
         if (cpu_has_virtual_nmis()) {
                 unblock_nmi = (exit_intr_info & INTR_INFO_UNBLOCK_NMI) != 0;
                 vector = exit_intr_info & INTR_INFO_VECTOR_MASK;
                 /*
-                 * SDM 3: 25.7.1.2
+                 * SDM 3: 27.7.1.2 (September 2008)
                  * Re-set bit "block by NMI" before VM entry if vmexit caused by
                  * a guest IRET fault.
+                 * SDM 3: 23.2.2 (September 2008)
+                 * Bit 12 is undefined in any of the following cases:
+                 *  If the VM exit sets the valid bit in the IDT-vectoring
+                 *   information field.
+                 *  If the VM exit is due to a double fault.
                  */
-                if (unblock_nmi && vector != DF_VECTOR)
+                if ((exit_intr_info & INTR_INFO_VALID_MASK) && unblock_nmi &&
+                    vector != DF_VECTOR && !idtv_info_valid)
                         vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
                                       GUEST_INTR_STATE_NMI);
         } else if (unlikely(vmx->soft_vnmi_blocked))
                 vmx->vnmi_blocked_time +=
                         ktime_to_ns(ktime_sub(ktime_get(), vmx->entry_time));
 
-        idt_vectoring_info = vmx->idt_vectoring_info;
-        idtv_info_valid = idt_vectoring_info & VECTORING_INFO_VALID_MASK;
+        vmx->vcpu.arch.nmi_injected = false;
+        kvm_clear_exception_queue(&vmx->vcpu);
+        kvm_clear_interrupt_queue(&vmx->vcpu);
+
+        if (!idtv_info_valid)
+                return;
+
         vector = idt_vectoring_info & VECTORING_INFO_VECTOR_MASK;
         type = idt_vectoring_info & VECTORING_INFO_TYPE_MASK;
-        if (vmx->vcpu.arch.nmi_injected) {
+
+        switch (type) {
+        case INTR_TYPE_NMI_INTR:
+                vmx->vcpu.arch.nmi_injected = true;
                 /*
-                 * SDM 3: 25.7.1.2
-                 * Clear bit "block by NMI" before VM entry if a NMI delivery
-                 * faulted.
+                 * SDM 3: 27.7.1.2 (September 2008)
+                 * Clear bit "block by NMI" before VM entry if a NMI
+                 * delivery faulted.
                  */
-                if (idtv_info_valid && type == INTR_TYPE_NMI_INTR)
-                        vmcs_clear_bits(GUEST_INTERRUPTIBILITY_INFO,
-                                        GUEST_INTR_STATE_NMI);
-                else
-                        vmx->vcpu.arch.nmi_injected = false;
-        }
-        kvm_clear_exception_queue(&vmx->vcpu);
-        if (idtv_info_valid && (type == INTR_TYPE_HARD_EXCEPTION ||
-                                type == INTR_TYPE_SOFT_EXCEPTION)) {
+                vmcs_clear_bits(GUEST_INTERRUPTIBILITY_INFO,
+                                GUEST_INTR_STATE_NMI);
+                break;
+        case INTR_TYPE_SOFT_EXCEPTION:
+                vmx->vcpu.arch.event_exit_inst_len =
+                        vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
+                /* fall through */
+        case INTR_TYPE_HARD_EXCEPTION:
                 if (idt_vectoring_info & VECTORING_INFO_DELIVER_CODE_MASK) {
-                        error = vmcs_read32(IDT_VECTORING_ERROR_CODE);
-                        kvm_queue_exception_e(&vmx->vcpu, vector, error);
+                        u32 err = vmcs_read32(IDT_VECTORING_ERROR_CODE);
+                        kvm_queue_exception_e(&vmx->vcpu, vector, err);
                 } else
                         kvm_queue_exception(&vmx->vcpu, vector);
-                vmx->idt_vectoring_info = 0;
-        }
-        kvm_clear_interrupt_queue(&vmx->vcpu);
-        if (idtv_info_valid && type == INTR_TYPE_EXT_INTR) {
-                kvm_queue_interrupt(&vmx->vcpu, vector);
-                vmx->idt_vectoring_info = 0;
-        }
-}
-
-static void vmx_intr_assist(struct kvm_vcpu *vcpu)
-{
-        update_tpr_threshold(vcpu);
-
-        vmx_update_window_states(vcpu);
-
-        if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
-                vmcs_clear_bits(GUEST_INTERRUPTIBILITY_INFO,
-                                GUEST_INTR_STATE_STI |
-                                GUEST_INTR_STATE_MOV_SS);
-
-        if (vcpu->arch.nmi_pending && !vcpu->arch.nmi_injected) {
-                if (vcpu->arch.interrupt.pending) {
-                        enable_nmi_window(vcpu);
-                } else if (vcpu->arch.nmi_window_open) {
-                        vcpu->arch.nmi_pending = false;
-                        vcpu->arch.nmi_injected = true;
-                } else {
-                        enable_nmi_window(vcpu);
-                        return;
-                }
-        }
-        if (vcpu->arch.nmi_injected) {
-                vmx_inject_nmi(vcpu);
-                if (vcpu->arch.nmi_pending)
-                        enable_nmi_window(vcpu);
-                else if (kvm_cpu_has_interrupt(vcpu))
-                        enable_irq_window(vcpu);
-                return;
-        }
-        if (!vcpu->arch.interrupt.pending && kvm_cpu_has_interrupt(vcpu)) {
-                if (vcpu->arch.interrupt_window_open)
-                        kvm_queue_interrupt(vcpu, kvm_cpu_get_interrupt(vcpu));
-                else
-                        enable_irq_window(vcpu);
-        }
-        if (vcpu->arch.interrupt.pending) {
-                vmx_inject_irq(vcpu, vcpu->arch.interrupt.nr);
-                if (kvm_cpu_has_interrupt(vcpu))
-                        enable_irq_window(vcpu);
+                break;
+        case INTR_TYPE_SOFT_INTR:
+                vmx->vcpu.arch.event_exit_inst_len =
+                        vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
+                /* fall through */
+        case INTR_TYPE_EXT_INTR:
+                kvm_queue_interrupt(&vmx->vcpu, vector,
+                        type == INTR_TYPE_SOFT_INTR);
+                break;
+        default:
+                break;
         }
 }
 
@@ -3381,7 +3418,6 @@ static void fixup_rmode_irq(struct vcpu_vmx *vmx)
 static void vmx_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 {
         struct vcpu_vmx *vmx = to_vmx(vcpu);
-        u32 intr_info;
 
         /* Record the guest's net vcpu time for enforced NMI injections. */
         if (unlikely(!cpu_has_virtual_nmis() && vmx->soft_vnmi_blocked))
@@ -3505,20 +3541,9 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
         if (vmx->rmode.irq.pending)
                 fixup_rmode_irq(vmx);
 
-        vmx_update_window_states(vcpu);
-
         asm("mov %0, %%ds; mov %0, %%es" : : "r"(__USER_DS));
         vmx->launched = 1;
 
-        intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
-
-        /* We need to handle NMIs before interrupts are enabled */
-        if ((intr_info & INTR_INFO_INTR_TYPE_MASK) == INTR_TYPE_NMI_INTR &&
-            (intr_info & INTR_INFO_VALID_MASK)) {
-                KVMTRACE_0D(NMI, vcpu, handler);
-                asm("int $2");
-        }
-
         vmx_complete_interrupts(vmx);
 }
 
@@ -3593,7 +3618,7 @@ static struct kvm_vcpu *vmx_create_vcpu(struct kvm *kvm, unsigned int id)
                 if (alloc_apic_access_page(kvm) != 0)
                         goto free_vmcs;
 
-        if (vm_need_ept())
+        if (enable_ept)
                 if (alloc_identity_pagetable(kvm) != 0)
                         goto free_vmcs;
 
@@ -3631,9 +3656,32 @@ static int get_ept_level(void)
         return VMX_EPT_DEFAULT_GAW + 1;
 }
 
-static int vmx_get_mt_mask_shift(void)
+static u64 vmx_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
 {
-        return VMX_EPT_MT_EPTE_SHIFT;
+        u64 ret;
+
+        /* For VT-d and EPT combination
+         * 1. MMIO: always map as UC
+         * 2. EPT with VT-d:
+         *   a. VT-d without snooping control feature: can't guarantee the
+         *      result, try to trust guest.
+         *   b. VT-d with snooping control feature: snooping control feature of
+         *      VT-d engine can guarantee the cache correctness. Just set it
+         *      to WB to keep consistent with host. So the same as item 3.
+         * 3. EPT without VT-d: always map as WB and set IGMT=1 to keep
+         *    consistent with host MTRR
+         */
+        if (is_mmio)
+                ret = MTRR_TYPE_UNCACHABLE << VMX_EPT_MT_EPTE_SHIFT;
+        else if (vcpu->kvm->arch.iommu_domain &&
+                !(vcpu->kvm->arch.iommu_flags & KVM_IOMMU_CACHE_COHERENCY))
+                ret = kvm_get_guest_memory_type(vcpu, gfn) <<
+                      VMX_EPT_MT_EPTE_SHIFT;
+        else
+                ret = (MTRR_TYPE_WRBACK << VMX_EPT_MT_EPTE_SHIFT)
+                        | VMX_EPT_IGMT_BIT;
+
+        return ret;
 }
 
 static struct kvm_x86_ops vmx_x86_ops = {
@@ -3644,7 +3692,7 @@ static struct kvm_x86_ops vmx_x86_ops = {
         .check_processor_compatibility = vmx_check_processor_compat,
         .hardware_enable = hardware_enable,
         .hardware_disable = hardware_disable,
-        .cpu_has_accelerated_tpr = cpu_has_vmx_virtualize_apic_accesses,
+        .cpu_has_accelerated_tpr = report_flexpriority,
 
         .vcpu_create = vmx_create_vcpu,
         .vcpu_free = vmx_free_vcpu,
@@ -3678,78 +3726,82 @@ static struct kvm_x86_ops vmx_x86_ops = {
         .tlb_flush = vmx_flush_tlb,
 
         .run = vmx_vcpu_run,
-        .handle_exit = kvm_handle_exit,
+        .handle_exit = vmx_handle_exit,
         .skip_emulated_instruction = skip_emulated_instruction,
+        .set_interrupt_shadow = vmx_set_interrupt_shadow,
+        .get_interrupt_shadow = vmx_get_interrupt_shadow,
         .patch_hypercall = vmx_patch_hypercall,
-        .get_irq = vmx_get_irq,
         .set_irq = vmx_inject_irq,
+        .set_nmi = vmx_inject_nmi,
         .queue_exception = vmx_queue_exception,
-        .exception_injected = vmx_exception_injected,
-        .inject_pending_irq = vmx_intr_assist,
-        .inject_pending_vectors = do_interrupt_requests,
+        .interrupt_allowed = vmx_interrupt_allowed,
+        .nmi_allowed = vmx_nmi_allowed,
+        .enable_nmi_window = enable_nmi_window,
+        .enable_irq_window = enable_irq_window,
+        .update_cr8_intercept = update_cr8_intercept,
 
         .set_tss_addr = vmx_set_tss_addr,
         .get_tdp_level = get_ept_level,
-        .get_mt_mask_shift = vmx_get_mt_mask_shift,
+        .get_mt_mask = vmx_get_mt_mask,
 };
 
 static int __init vmx_init(void)
 {
-        void *va;
         int r;
 
-        vmx_io_bitmap_a = alloc_page(GFP_KERNEL | __GFP_HIGHMEM);
+        vmx_io_bitmap_a = (unsigned long *)__get_free_page(GFP_KERNEL);
         if (!vmx_io_bitmap_a)
                 return -ENOMEM;
 
-        vmx_io_bitmap_b = alloc_page(GFP_KERNEL | __GFP_HIGHMEM);
+        vmx_io_bitmap_b = (unsigned long *)__get_free_page(GFP_KERNEL);
         if (!vmx_io_bitmap_b) {
                 r = -ENOMEM;
                 goto out;
         }
 
-        vmx_msr_bitmap = alloc_page(GFP_KERNEL | __GFP_HIGHMEM);
-        if (!vmx_msr_bitmap) {
+        vmx_msr_bitmap_legacy = (unsigned long *)__get_free_page(GFP_KERNEL);
+        if (!vmx_msr_bitmap_legacy) {
                 r = -ENOMEM;
                 goto out1;
         }
 
+        vmx_msr_bitmap_longmode = (unsigned long *)__get_free_page(GFP_KERNEL);
+        if (!vmx_msr_bitmap_longmode) {
+                r = -ENOMEM;
+                goto out2;
+        }
+
         /*
          * Allow direct access to the PC debug port (it is often used for I/O
          * delays, but the vmexits simply slow things down).
          */
-        va = kmap(vmx_io_bitmap_a);
-        memset(va, 0xff, PAGE_SIZE);
-        clear_bit(0x80, va);
-        kunmap(vmx_io_bitmap_a);
+        memset(vmx_io_bitmap_a, 0xff, PAGE_SIZE);
+        clear_bit(0x80, vmx_io_bitmap_a);
 
-        va = kmap(vmx_io_bitmap_b);
-        memset(va, 0xff, PAGE_SIZE);
-        kunmap(vmx_io_bitmap_b);
+        memset(vmx_io_bitmap_b, 0xff, PAGE_SIZE);
 
-        va = kmap(vmx_msr_bitmap);
-        memset(va, 0xff, PAGE_SIZE);
-        kunmap(vmx_msr_bitmap);
+        memset(vmx_msr_bitmap_legacy, 0xff, PAGE_SIZE);
+        memset(vmx_msr_bitmap_longmode, 0xff, PAGE_SIZE);
 
         set_bit(0, vmx_vpid_bitmap); /* 0 is reserved for host */
 
         r = kvm_init(&vmx_x86_ops, sizeof(struct vcpu_vmx), THIS_MODULE);
         if (r)
-                goto out2;
+                goto out3;
 
-        vmx_disable_intercept_for_msr(vmx_msr_bitmap, MSR_FS_BASE);
-        vmx_disable_intercept_for_msr(vmx_msr_bitmap, MSR_GS_BASE);
-        vmx_disable_intercept_for_msr(vmx_msr_bitmap, MSR_IA32_SYSENTER_CS);
-        vmx_disable_intercept_for_msr(vmx_msr_bitmap, MSR_IA32_SYSENTER_ESP);
-        vmx_disable_intercept_for_msr(vmx_msr_bitmap, MSR_IA32_SYSENTER_EIP);
+        vmx_disable_intercept_for_msr(MSR_FS_BASE, false);
+        vmx_disable_intercept_for_msr(MSR_GS_BASE, false);
+        vmx_disable_intercept_for_msr(MSR_KERNEL_GS_BASE, true);
+        vmx_disable_intercept_for_msr(MSR_IA32_SYSENTER_CS, false);
+        vmx_disable_intercept_for_msr(MSR_IA32_SYSENTER_ESP, false);
+        vmx_disable_intercept_for_msr(MSR_IA32_SYSENTER_EIP, false);
 
-        if (vm_need_ept()) {
+        if (enable_ept) {
                 bypass_guest_pf = 0;
                 kvm_mmu_set_base_ptes(VMX_EPT_READABLE_MASK |
                         VMX_EPT_WRITABLE_MASK);
                 kvm_mmu_set_mask_ptes(0ull, 0ull, 0ull, 0ull,
-                                VMX_EPT_EXECUTABLE_MASK,
-                                VMX_EPT_DEFAULT_MT << VMX_EPT_MT_EPTE_SHIFT);
+                                VMX_EPT_EXECUTABLE_MASK);
                 kvm_enable_tdp();
         } else
                 kvm_disable_tdp();
@@ -3761,20 +3813,23 @@ static int __init vmx_init(void)
 
         return 0;
 
+out3:
+        free_page((unsigned long)vmx_msr_bitmap_longmode);
 out2:
-        __free_page(vmx_msr_bitmap);
+        free_page((unsigned long)vmx_msr_bitmap_legacy);
 out1:
-        __free_page(vmx_io_bitmap_b);
+        free_page((unsigned long)vmx_io_bitmap_b);
 out:
-        __free_page(vmx_io_bitmap_a);
+        free_page((unsigned long)vmx_io_bitmap_a);
         return r;
 }
 
 static void __exit vmx_exit(void)
 {
-        __free_page(vmx_msr_bitmap);
-        __free_page(vmx_io_bitmap_b);
-        __free_page(vmx_io_bitmap_a);
+        free_page((unsigned long)vmx_msr_bitmap_legacy);
+        free_page((unsigned long)vmx_msr_bitmap_longmode);
+        free_page((unsigned long)vmx_io_bitmap_b);
+        free_page((unsigned long)vmx_io_bitmap_a);
 
         kvm_exit();
 }
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 3944e917e794..249540f98513 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -91,7 +91,6 @@ struct kvm_stats_debugfs_item debugfs_entries[] = {
         { "halt_wakeup", VCPU_STAT(halt_wakeup) },
         { "hypercalls", VCPU_STAT(hypercalls) },
         { "request_irq", VCPU_STAT(request_irq_exits) },
-        { "request_nmi", VCPU_STAT(request_nmi_exits) },
         { "irq_exits", VCPU_STAT(irq_exits) },
         { "host_state_reload", VCPU_STAT(host_state_reload) },
         { "efer_reload", VCPU_STAT(efer_reload) },
@@ -108,7 +107,6 @@ struct kvm_stats_debugfs_item debugfs_entries[] = {
         { "mmu_recycled", VM_STAT(mmu_recycled) },
         { "mmu_cache_miss", VM_STAT(mmu_cache_miss) },
         { "mmu_unsync", VM_STAT(mmu_unsync) },
-        { "mmu_unsync_global", VM_STAT(mmu_unsync_global) },
         { "remote_tlb_flush", VM_STAT(remote_tlb_flush) },
         { "largepages", VM_STAT(lpages) },
         { NULL }
@@ -234,7 +232,8 @@ int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3)
                 goto out;
         }
         for (i = 0; i < ARRAY_SIZE(pdpte); ++i) {
-                if ((pdpte[i] & 1) && (pdpte[i] & 0xfffffff0000001e6ull)) {
+                if (is_present_pte(pdpte[i]) &&
+                    (pdpte[i] & vcpu->arch.mmu.rsvd_bits_mask[0][2])) {
                         ret = 0;
                         goto out;
                 }
@@ -321,7 +320,6 @@ void kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
         kvm_x86_ops->set_cr0(vcpu, cr0);
         vcpu->arch.cr0 = cr0;
 
-        kvm_mmu_sync_global(vcpu);
         kvm_mmu_reset_context(vcpu);
         return;
 }
@@ -370,7 +368,6 @@ void kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
         kvm_x86_ops->set_cr4(vcpu, cr4);
         vcpu->arch.cr4 = cr4;
         vcpu->arch.mmu.base_role.cr4_pge = (cr4 & X86_CR4_PGE) && !tdp_enabled;
-        kvm_mmu_sync_global(vcpu);
         kvm_mmu_reset_context(vcpu);
 }
 EXPORT_SYMBOL_GPL(kvm_set_cr4);
@@ -523,6 +520,9 @@ static void set_efer(struct kvm_vcpu *vcpu, u64 efer)
         efer |= vcpu->arch.shadow_efer & EFER_LMA;
 
         vcpu->arch.shadow_efer = efer;
+
+        vcpu->arch.mmu.base_role.nxe = (efer & EFER_NX) && !tdp_enabled;
+        kvm_mmu_reset_context(vcpu);
 }
 
 void kvm_enable_efer_bits(u64 mask)
@@ -630,14 +630,17 @@ static void kvm_write_guest_time(struct kvm_vcpu *v)
         unsigned long flags;
         struct kvm_vcpu_arch *vcpu = &v->arch;
         void *shared_kaddr;
+        unsigned long this_tsc_khz;
 
         if ((!vcpu->time_page))
                 return;
 
-        if (unlikely(vcpu->hv_clock_tsc_khz != __get_cpu_var(cpu_tsc_khz))) {
-                kvm_set_time_scale(__get_cpu_var(cpu_tsc_khz), &vcpu->hv_clock);
-                vcpu->hv_clock_tsc_khz = __get_cpu_var(cpu_tsc_khz);
+        this_tsc_khz = get_cpu_var(cpu_tsc_khz);
+        if (unlikely(vcpu->hv_clock_tsc_khz != this_tsc_khz)) {
+                kvm_set_time_scale(this_tsc_khz, &vcpu->hv_clock);
+                vcpu->hv_clock_tsc_khz = this_tsc_khz;
         }
+        put_cpu_var(cpu_tsc_khz);
 
         /* Keep irq disabled to prevent changes to the clock */
         local_irq_save(flags);
@@ -893,6 +896,8 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
         case MSR_IA32_LASTINTFROMIP:
         case MSR_IA32_LASTINTTOIP:
         case MSR_VM_HSAVE_PA:
+        case MSR_P6_EVNTSEL0:
+        case MSR_P6_EVNTSEL1:
                 data = 0;
                 break;
         case MSR_MTRRcap:
@@ -1024,6 +1029,7 @@ int kvm_dev_ioctl_check_extension(long ext)
         case KVM_CAP_SYNC_MMU:
         case KVM_CAP_REINJECT_CONTROL:
         case KVM_CAP_IRQ_INJECT_STATUS:
+        case KVM_CAP_ASSIGN_DEV_IRQ:
                 r = 1;
                 break;
         case KVM_CAP_COALESCED_MMIO:
@@ -1241,41 +1247,53 @@ static void do_cpuid_1_ent(struct kvm_cpuid_entry2 *entry, u32 function,
         entry->flags = 0;
 }
 
+#define F(x) bit(X86_FEATURE_##x)
+
 static void do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function,
                          u32 index, int *nent, int maxnent)
 {
-        const u32 kvm_supported_word0_x86_features = bit(X86_FEATURE_FPU) |
-                bit(X86_FEATURE_VME) | bit(X86_FEATURE_DE) |
-                bit(X86_FEATURE_PSE) | bit(X86_FEATURE_TSC) |
-                bit(X86_FEATURE_MSR) | bit(X86_FEATURE_PAE) |
-                bit(X86_FEATURE_CX8) | bit(X86_FEATURE_APIC) |
-                bit(X86_FEATURE_SEP) | bit(X86_FEATURE_PGE) |
-                bit(X86_FEATURE_CMOV) | bit(X86_FEATURE_PSE36) |
-                bit(X86_FEATURE_CLFLSH) | bit(X86_FEATURE_MMX) |
-                bit(X86_FEATURE_FXSR) | bit(X86_FEATURE_XMM) |
-                bit(X86_FEATURE_XMM2) | bit(X86_FEATURE_SELFSNOOP);
-        const u32 kvm_supported_word1_x86_features = bit(X86_FEATURE_FPU) |
-                bit(X86_FEATURE_VME) | bit(X86_FEATURE_DE) |
-                bit(X86_FEATURE_PSE) | bit(X86_FEATURE_TSC) |
-                bit(X86_FEATURE_MSR) | bit(X86_FEATURE_PAE) |
-                bit(X86_FEATURE_CX8) | bit(X86_FEATURE_APIC) |
-                bit(X86_FEATURE_PGE) |
-                bit(X86_FEATURE_CMOV) | bit(X86_FEATURE_PSE36) |
-                bit(X86_FEATURE_MMX) | bit(X86_FEATURE_FXSR) |
-                bit(X86_FEATURE_SYSCALL) |
-                (is_efer_nx() ? bit(X86_FEATURE_NX) : 0) |
+        unsigned f_nx = is_efer_nx() ? F(NX) : 0;
 #ifdef CONFIG_X86_64
-                bit(X86_FEATURE_LM) |
+        unsigned f_lm = F(LM);
+#else
+        unsigned f_lm = 0;
 #endif
-                bit(X86_FEATURE_FXSR_OPT) |
-                bit(X86_FEATURE_MMXEXT) |
-                bit(X86_FEATURE_3DNOWEXT) |
-                bit(X86_FEATURE_3DNOW);
-        const u32 kvm_supported_word3_x86_features =
-                bit(X86_FEATURE_XMM3) | bit(X86_FEATURE_CX16);
+
+        /* cpuid 1.edx */
+        const u32 kvm_supported_word0_x86_features =
+                F(FPU) | F(VME) | F(DE) | F(PSE) |
+                F(TSC) | F(MSR) | F(PAE) | F(MCE) |
+                F(CX8) | F(APIC) | 0 /* Reserved */ | F(SEP) |
+                F(MTRR) | F(PGE) | F(MCA) | F(CMOV) |
+                F(PAT) | F(PSE36) | 0 /* PSN */ | F(CLFLSH) |
+                0 /* Reserved, DS, ACPI */ | F(MMX) |
+                F(FXSR) | F(XMM) | F(XMM2) | F(SELFSNOOP) |
+                0 /* HTT, TM, Reserved, PBE */;
+        /* cpuid 0x80000001.edx */
+        const u32 kvm_supported_word1_x86_features =
+                F(FPU) | F(VME) | F(DE) | F(PSE) |
+                F(TSC) | F(MSR) | F(PAE) | F(MCE) |
+                F(CX8) | F(APIC) | 0 /* Reserved */ | F(SYSCALL) |
+                F(MTRR) | F(PGE) | F(MCA) | F(CMOV) |
+                F(PAT) | F(PSE36) | 0 /* Reserved */ |
+                f_nx | 0 /* Reserved */ | F(MMXEXT) | F(MMX) |
+                F(FXSR) | F(FXSR_OPT) | 0 /* GBPAGES */ | 0 /* RDTSCP */ |
+                0 /* Reserved */ | f_lm | F(3DNOWEXT) | F(3DNOW);
+        /* cpuid 1.ecx */
+        const u32 kvm_supported_word4_x86_features =
+                F(XMM3) | 0 /* Reserved, DTES64, MONITOR */ |
+                0 /* DS-CPL, VMX, SMX, EST */ |
+                0 /* TM2 */ | F(SSSE3) | 0 /* CNXT-ID */ | 0 /* Reserved */ |
+                0 /* Reserved */ | F(CX16) | 0 /* xTPR Update, PDCM */ |
+                0 /* Reserved, DCA */ | F(XMM4_1) |
+                F(XMM4_2) | 0 /* x2APIC */ | F(MOVBE) | F(POPCNT) |
+                0 /* Reserved, XSAVE, OSXSAVE */;
+        /* cpuid 0x80000001.ecx */
         const u32 kvm_supported_word6_x86_features =
-                bit(X86_FEATURE_LAHF_LM) | bit(X86_FEATURE_CMP_LEGACY) |
-                bit(X86_FEATURE_SVM);
+                F(LAHF_LM) | F(CMP_LEGACY) | F(SVM) | 0 /* ExtApicSpace */ |
+                F(CR8_LEGACY) | F(ABM) | F(SSE4A) | F(MISALIGNSSE) |
+                F(3DNOWPREFETCH) | 0 /* OSVW */ | 0 /* IBS */ | F(SSE5) |
+                0 /* SKINIT */ | 0 /* WDT */;
 
         /* all calls to cpuid_count() should be made on the same cpu */
         get_cpu();
@@ -1288,7 +1306,7 @@ static void do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function,
                 break;
         case 1:
                 entry->edx &= kvm_supported_word0_x86_features;
-                entry->ecx &= kvm_supported_word3_x86_features;
+                entry->ecx &= kvm_supported_word4_x86_features;
                 break;
         /* function 2 entries are STATEFUL. That is, repeated cpuid commands
          * may return different values. This forces us to get_cpu() before
@@ -1350,6 +1368,8 @@ static void do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function,
         put_cpu();
 }
 
+#undef F
+
 static int kvm_dev_ioctl_get_supported_cpuid(struct kvm_cpuid2 *cpuid,
                                      struct kvm_cpuid_entry2 __user *entries)
 {
@@ -1421,8 +1441,7 @@ static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,
                 return -ENXIO;
         vcpu_load(vcpu);
 
-        set_bit(irq->irq, vcpu->arch.irq_pending);
-        set_bit(irq->irq / BITS_PER_LONG, &vcpu->arch.irq_summary);
+        kvm_queue_interrupt(vcpu, irq->irq, false);
 
         vcpu_put(vcpu);
 
@@ -1584,8 +1603,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp,
                 r = -EINVAL;
         }
 out:
-        if (lapic)
-                kfree(lapic);
+        kfree(lapic);
         return r;
 }
 
@@ -1606,10 +1624,12 @@ static int kvm_vm_ioctl_set_nr_mmu_pages(struct kvm *kvm,
                 return -EINVAL;
 
         down_write(&kvm->slots_lock);
+        spin_lock(&kvm->mmu_lock);
 
         kvm_mmu_change_mmu_pages(kvm, kvm_nr_mmu_pages);
         kvm->arch.n_requested_mmu_pages = kvm_nr_mmu_pages;
 
+        spin_unlock(&kvm->mmu_lock);
         up_write(&kvm->slots_lock);
         return 0;
 }
@@ -1785,7 +1805,9 @@ int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
 
         /* If nothing is dirty, don't bother messing with page tables. */
         if (is_dirty) {
+                spin_lock(&kvm->mmu_lock);
                 kvm_mmu_slot_remove_write_access(kvm, log->slot);
+                spin_unlock(&kvm->mmu_lock);
                 kvm_flush_remote_tlbs(kvm);
                 memslot = &kvm->memslots[log->slot];
                 n = ALIGN(memslot->npages, BITS_PER_LONG) / 8;
@@ -2360,7 +2382,7 @@ int emulate_instruction(struct kvm_vcpu *vcpu,
                         u16 error_code,
                         int emulation_type)
 {
-        int r;
+        int r, shadow_mask;
         struct decode_cache *c;
 
         kvm_clear_exception_queue(vcpu);
@@ -2408,7 +2430,16 @@ int emulate_instruction(struct kvm_vcpu *vcpu,
                 }
         }
 
+        if (emulation_type & EMULTYPE_SKIP) {
+                kvm_rip_write(vcpu, vcpu->arch.emulate_ctxt.decode.eip);
+                return EMULATE_DONE;
+        }
+
         r = x86_emulate_insn(&vcpu->arch.emulate_ctxt, &emulate_ops);
+        shadow_mask = vcpu->arch.emulate_ctxt.interruptibility;
+
+        if (r == 0)
+                kvm_x86_ops->set_interrupt_shadow(vcpu, shadow_mask);
 
         if (vcpu->arch.pio.string)
                 return EMULATE_DO_MMIO;
@@ -2761,7 +2792,7 @@ int kvm_arch_init(void *opaque)
         kvm_mmu_set_nonpresent_ptes(0ull, 0ull);
         kvm_mmu_set_base_ptes(PT_PRESENT_MASK);
         kvm_mmu_set_mask_ptes(PT_USER_MASK, PT_ACCESSED_MASK,
-                        PT_DIRTY_MASK, PT64_NX_MASK, 0, 0);
+                        PT_DIRTY_MASK, PT64_NX_MASK, 0);
 
         for_each_possible_cpu(cpu)
                 per_cpu(cpu_tsc_khz, cpu) = tsc_khz;
@@ -3012,6 +3043,16 @@ struct kvm_cpuid_entry2 *kvm_find_cpuid_entry(struct kvm_vcpu *vcpu,
         return best;
 }
 
+int cpuid_maxphyaddr(struct kvm_vcpu *vcpu)
+{
+        struct kvm_cpuid_entry2 *best;
+
+        best = kvm_find_cpuid_entry(vcpu, 0x80000008, 0);
+        if (best)
+                return best->eax & 0xff;
+        return 36;
+}
+
 void kvm_emulate_cpuid(struct kvm_vcpu *vcpu)
 {
         u32 function, index;
@@ -3048,10 +3089,9 @@ EXPORT_SYMBOL_GPL(kvm_emulate_cpuid);
 static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu,
                                           struct kvm_run *kvm_run)
 {
-        return (!vcpu->arch.irq_summary &&
+        return (!irqchip_in_kernel(vcpu->kvm) && !kvm_cpu_has_interrupt(vcpu) &&
                 kvm_run->request_interrupt_window &&
-                vcpu->arch.interrupt_window_open &&
-                (kvm_x86_ops->get_rflags(vcpu) & X86_EFLAGS_IF));
+                kvm_arch_interrupt_allowed(vcpu));
 }
 
 static void post_kvm_run_save(struct kvm_vcpu *vcpu,
@@ -3064,8 +3104,9 @@ static void post_kvm_run_save(struct kvm_vcpu *vcpu,
                 kvm_run->ready_for_interrupt_injection = 1;
         else
                 kvm_run->ready_for_interrupt_injection =
-                                        (vcpu->arch.interrupt_window_open &&
-                                         vcpu->arch.irq_summary == 0);
+                        kvm_arch_interrupt_allowed(vcpu) &&
+                        !kvm_cpu_has_interrupt(vcpu) &&
+                        !kvm_event_needs_reinjection(vcpu);
 }
 
 static void vapic_enter(struct kvm_vcpu *vcpu)
@@ -3094,9 +3135,63 @@ static void vapic_exit(struct kvm_vcpu *vcpu)
         up_read(&vcpu->kvm->slots_lock);
 }
 
+static void update_cr8_intercept(struct kvm_vcpu *vcpu)
+{
+        int max_irr, tpr;
+
+        if (!kvm_x86_ops->update_cr8_intercept)
+                return;
+
+        if (!vcpu->arch.apic->vapic_addr)
+                max_irr = kvm_lapic_find_highest_irr(vcpu);
+        else
+                max_irr = -1;
+
+        if (max_irr != -1)
+                max_irr >>= 4;
+
+        tpr = kvm_lapic_get_cr8(vcpu);
+
+        kvm_x86_ops->update_cr8_intercept(vcpu, tpr, max_irr);
+}
+
+static void inject_pending_irq(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
+{
+        if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
+                kvm_x86_ops->set_interrupt_shadow(vcpu, 0);
+
+        /* try to reinject previous events if any */
+        if (vcpu->arch.nmi_injected) {
+                kvm_x86_ops->set_nmi(vcpu);
+                return;
+        }
+
+        if (vcpu->arch.interrupt.pending) {
+                kvm_x86_ops->set_irq(vcpu);
+                return;
+        }
+
+        /* try to inject new event if pending */
+        if (vcpu->arch.nmi_pending) {
+                if (kvm_x86_ops->nmi_allowed(vcpu)) {
+                        vcpu->arch.nmi_pending = false;
+                        vcpu->arch.nmi_injected = true;
+                        kvm_x86_ops->set_nmi(vcpu);
+                }
+        } else if (kvm_cpu_has_interrupt(vcpu)) {
+                if (kvm_x86_ops->interrupt_allowed(vcpu)) {
+                        kvm_queue_interrupt(vcpu, kvm_cpu_get_interrupt(vcpu),
+                                            false);
+                        kvm_x86_ops->set_irq(vcpu);
+                }
+        }
+}
+
 static int vcpu_enter_guest(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 {
         int r;
+        bool req_int_win = !irqchip_in_kernel(vcpu->kvm) &&
+                kvm_run->request_interrupt_window;
 
         if (vcpu->requests)
                 if (test_and_clear_bit(KVM_REQ_MMU_RELOAD, &vcpu->requests))
@@ -3128,9 +3223,6 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
                 }
         }
 
-        clear_bit(KVM_REQ_PENDING_TIMER, &vcpu->requests);
-        kvm_inject_pending_timer_irqs(vcpu);
-
         preempt_disable();
 
         kvm_x86_ops->prepare_guest_switch(vcpu);
@@ -3138,6 +3230,9 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 
         local_irq_disable();
 
+        clear_bit(KVM_REQ_KICK, &vcpu->requests);
+        smp_mb__after_clear_bit();
+
         if (vcpu->requests || need_resched() || signal_pending(current)) {
                 local_irq_enable();
                 preempt_enable();
@@ -3145,21 +3240,21 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
                 goto out;
         }
 
-        vcpu->guest_mode = 1;
-        /*
-         * Make sure that guest_mode assignment won't happen after
-         * testing the pending IRQ vector bitmap.
-         */
-        smp_wmb();
-
         if (vcpu->arch.exception.pending)
                 __queue_exception(vcpu);
-        else if (irqchip_in_kernel(vcpu->kvm))
-                kvm_x86_ops->inject_pending_irq(vcpu);
         else
-                kvm_x86_ops->inject_pending_vectors(vcpu, kvm_run);
+                inject_pending_irq(vcpu, kvm_run);
 
-        kvm_lapic_sync_to_vapic(vcpu);
+        /* enable NMI/IRQ window open exits if needed */
+        if (vcpu->arch.nmi_pending)
+                kvm_x86_ops->enable_nmi_window(vcpu);
+        else if (kvm_cpu_has_interrupt(vcpu) || req_int_win)
+                kvm_x86_ops->enable_irq_window(vcpu);
+
+        if (kvm_lapic_enabled(vcpu)) {
+                update_cr8_intercept(vcpu);
+                kvm_lapic_sync_to_vapic(vcpu);
+        }
 
         up_read(&vcpu->kvm->slots_lock);
 
@@ -3193,7 +3288,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
         set_debugreg(vcpu->arch.host_dr6, 6);
         set_debugreg(vcpu->arch.host_dr7, 7);
 
-        vcpu->guest_mode = 0;
+        set_bit(KVM_REQ_KICK, &vcpu->requests);
         local_irq_enable();
 
         ++vcpu->stat.exits;
@@ -3220,8 +3315,6 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
                 profile_hit(KVM_PROFILING, (void *)rip);
         }
 
-        if (vcpu->arch.exception.pending && kvm_x86_ops->exception_injected(vcpu))
-                vcpu->arch.exception.pending = false;
 
         kvm_lapic_sync_from_vapic(vcpu);
 
@@ -3230,6 +3323,7 @@ out:
         return r;
 }
 
+
 static int __vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 {
         int r;
@@ -3256,29 +3350,42 @@ static int __vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
                         kvm_vcpu_block(vcpu);
                         down_read(&vcpu->kvm->slots_lock);
                         if (test_and_clear_bit(KVM_REQ_UNHALT, &vcpu->requests))
-                                if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED)
+                        {
+                                switch(vcpu->arch.mp_state) {
+                                case KVM_MP_STATE_HALTED:
                                         vcpu->arch.mp_state =
-                                                        KVM_MP_STATE_RUNNABLE;
-                        if (vcpu->arch.mp_state != KVM_MP_STATE_RUNNABLE)
-                                r = -EINTR;
+                                                KVM_MP_STATE_RUNNABLE;
+                                case KVM_MP_STATE_RUNNABLE:
+                                        break;
+                                case KVM_MP_STATE_SIPI_RECEIVED:
+                                default:
+                                        r = -EINTR;
+                                        break;
+                                }
+                        }
                 }
 
-                if (r > 0) {
-                        if (dm_request_for_irq_injection(vcpu, kvm_run)) {
-                                r = -EINTR;
-                                kvm_run->exit_reason = KVM_EXIT_INTR;
-                                ++vcpu->stat.request_irq_exits;
-                        }
-                        if (signal_pending(current)) {
-                                r = -EINTR;
-                                kvm_run->exit_reason = KVM_EXIT_INTR;
-                                ++vcpu->stat.signal_exits;
-                        }
-                        if (need_resched()) {
-                                up_read(&vcpu->kvm->slots_lock);
-                                kvm_resched(vcpu);
-                                down_read(&vcpu->kvm->slots_lock);
-                        }
+                if (r <= 0)
+                        break;
+
+                clear_bit(KVM_REQ_PENDING_TIMER, &vcpu->requests);
+                if (kvm_cpu_has_pending_timer(vcpu))
+                        kvm_inject_pending_timer_irqs(vcpu);
+
+                if (dm_request_for_irq_injection(vcpu, kvm_run)) {
+                        r = -EINTR;
+                        kvm_run->exit_reason = KVM_EXIT_INTR;
+                        ++vcpu->stat.request_irq_exits;
+                }
+                if (signal_pending(current)) {
+                        r = -EINTR;
+                        kvm_run->exit_reason = KVM_EXIT_INTR;
+                        ++vcpu->stat.signal_exits;
+                }
+                if (need_resched()) {
+                        up_read(&vcpu->kvm->slots_lock);
+                        kvm_resched(vcpu);
+                        down_read(&vcpu->kvm->slots_lock);
                 }
         }
 
@@ -3442,7 +3549,6 @@ int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
                                   struct kvm_sregs *sregs)
 {
         struct descriptor_table dt;
-        int pending_vec;
 
         vcpu_load(vcpu);
 
@@ -3472,16 +3578,11 @@ int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
         sregs->efer = vcpu->arch.shadow_efer;
         sregs->apic_base = kvm_get_apic_base(vcpu);
 
-        if (irqchip_in_kernel(vcpu->kvm)) {
-                memset(sregs->interrupt_bitmap, 0,
-                       sizeof sregs->interrupt_bitmap);
-                pending_vec = kvm_x86_ops->get_irq(vcpu);
-                if (pending_vec >= 0)
-                        set_bit(pending_vec,
-                                (unsigned long *)sregs->interrupt_bitmap);
-        } else
-                memcpy(sregs->interrupt_bitmap, vcpu->arch.irq_pending,
-                       sizeof sregs->interrupt_bitmap);
+        memset(sregs->interrupt_bitmap, 0, sizeof sregs->interrupt_bitmap);
+
+        if (vcpu->arch.interrupt.pending && !vcpu->arch.interrupt.soft)
+                set_bit(vcpu->arch.interrupt.nr,
+                        (unsigned long *)sregs->interrupt_bitmap);
 
         vcpu_put(vcpu);
 
@@ -3688,7 +3789,6 @@ static void save_state_to_tss32(struct kvm_vcpu *vcpu,
         tss->fs = get_segment_selector(vcpu, VCPU_SREG_FS);
         tss->gs = get_segment_selector(vcpu, VCPU_SREG_GS);
         tss->ldt_selector = get_segment_selector(vcpu, VCPU_SREG_LDTR);
-        tss->prev_task_link = get_segment_selector(vcpu, VCPU_SREG_TR);
 }
 
 static int load_state_from_tss32(struct kvm_vcpu *vcpu,
@@ -3785,8 +3885,8 @@ static int load_state_from_tss16(struct kvm_vcpu *vcpu,
 }
 
 static int kvm_task_switch_16(struct kvm_vcpu *vcpu, u16 tss_selector,
-                       u32 old_tss_base,
-                       struct desc_struct *nseg_desc)
+                              u16 old_tss_sel, u32 old_tss_base,
+                              struct desc_struct *nseg_desc)
 {
         struct tss_segment_16 tss_segment_16;
         int ret = 0;
@@ -3805,6 +3905,16 @@ static int kvm_task_switch_16(struct kvm_vcpu *vcpu, u16 tss_selector,
                            &tss_segment_16, sizeof tss_segment_16))
                 goto out;
 
+        if (old_tss_sel != 0xffff) {
+                tss_segment_16.prev_task_link = old_tss_sel;
+
+                if (kvm_write_guest(vcpu->kvm,
+                                    get_tss_base_addr(vcpu, nseg_desc),
+                                    &tss_segment_16.prev_task_link,
+                                    sizeof tss_segment_16.prev_task_link))
+                        goto out;
+        }
+
         if (load_state_from_tss16(vcpu, &tss_segment_16))
                 goto out;
 
@@ -3814,7 +3924,7 @@ out:
 }
 
 static int kvm_task_switch_32(struct kvm_vcpu *vcpu, u16 tss_selector,
-                       u32 old_tss_base,
+                       u16 old_tss_sel, u32 old_tss_base,
                        struct desc_struct *nseg_desc)
 {
         struct tss_segment_32 tss_segment_32;
@@ -3834,6 +3944,16 @@ static int kvm_task_switch_32(struct kvm_vcpu *vcpu, u16 tss_selector,
                            &tss_segment_32, sizeof tss_segment_32))
                 goto out;
 
+        if (old_tss_sel != 0xffff) {
+                tss_segment_32.prev_task_link = old_tss_sel;
+
+                if (kvm_write_guest(vcpu->kvm,
+                                    get_tss_base_addr(vcpu, nseg_desc),
+                                    &tss_segment_32.prev_task_link,
+                                    sizeof tss_segment_32.prev_task_link))
+                        goto out;
+        }
+
         if (load_state_from_tss32(vcpu, &tss_segment_32))
                 goto out;
 
@@ -3887,14 +4007,22 @@ int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int reason)
                 kvm_x86_ops->set_rflags(vcpu, eflags & ~X86_EFLAGS_NT);
         }
 
-        kvm_x86_ops->skip_emulated_instruction(vcpu);
+        /* set back link to prev task only if NT bit is set in eflags
+           note that old_tss_sel is not used afetr this point */
+        if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
+                old_tss_sel = 0xffff;
+
+        /* set back link to prev task only if NT bit is set in eflags
+           note that old_tss_sel is not used afetr this point */
+        if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
+                old_tss_sel = 0xffff;
 
         if (nseg_desc.type & 8)
-                ret = kvm_task_switch_32(vcpu, tss_selector, old_tss_base,
-                                         &nseg_desc);
+                ret = kvm_task_switch_32(vcpu, tss_selector, old_tss_sel,
+                                         old_tss_base, &nseg_desc);
         else
-                ret = kvm_task_switch_16(vcpu, tss_selector, old_tss_base,
-                                         &nseg_desc);
+                ret = kvm_task_switch_16(vcpu, tss_selector, old_tss_sel,
+                                         old_tss_base, &nseg_desc);
 
         if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE) {
                 u32 eflags = kvm_x86_ops->get_rflags(vcpu);
@@ -3920,7 +4048,7 @@ int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
                                   struct kvm_sregs *sregs)
 {
         int mmu_reset_needed = 0;
-        int i, pending_vec, max_bits;
+        int pending_vec, max_bits;
         struct descriptor_table dt;
 
         vcpu_load(vcpu);
@@ -3934,7 +4062,13 @@ int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
 
         vcpu->arch.cr2 = sregs->cr2;
         mmu_reset_needed |= vcpu->arch.cr3 != sregs->cr3;
-        vcpu->arch.cr3 = sregs->cr3;
+
+        down_read(&vcpu->kvm->slots_lock);
+        if (gfn_to_memslot(vcpu->kvm, sregs->cr3 >> PAGE_SHIFT))
+                vcpu->arch.cr3 = sregs->cr3;
+        else
+                set_bit(KVM_REQ_TRIPLE_FAULT, &vcpu->requests);
+        up_read(&vcpu->kvm->slots_lock);
 
         kvm_set_cr8(vcpu, sregs->cr8);
 
@@ -3956,25 +4090,14 @@ int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
         if (mmu_reset_needed)
                 kvm_mmu_reset_context(vcpu);
 
-        if (!irqchip_in_kernel(vcpu->kvm)) {
-                memcpy(vcpu->arch.irq_pending, sregs->interrupt_bitmap,
-                       sizeof vcpu->arch.irq_pending);
-                vcpu->arch.irq_summary = 0;
-                for (i = 0; i < ARRAY_SIZE(vcpu->arch.irq_pending); ++i)
-                        if (vcpu->arch.irq_pending[i])
-                                __set_bit(i, &vcpu->arch.irq_summary);
-        } else {
-                max_bits = (sizeof sregs->interrupt_bitmap) << 3;
-                pending_vec = find_first_bit(
-                        (const unsigned long *)sregs->interrupt_bitmap,
-                        max_bits);
-                /* Only pending external irq is handled here */
-                if (pending_vec < max_bits) {
-                        kvm_x86_ops->set_irq(vcpu, pending_vec);
-                        pr_debug("Set back pending irq %d\n",
-                                 pending_vec);
-                }
-                kvm_pic_clear_isr_ack(vcpu->kvm);
+        max_bits = (sizeof sregs->interrupt_bitmap) << 3;
+        pending_vec = find_first_bit(
+                (const unsigned long *)sregs->interrupt_bitmap, max_bits);
+        if (pending_vec < max_bits) {
+                kvm_queue_interrupt(vcpu, pending_vec, false);
+                pr_debug("Set back pending irq %d\n", pending_vec);
+                if (irqchip_in_kernel(vcpu->kvm))
+                        kvm_pic_clear_isr_ack(vcpu->kvm);
         }
 
         kvm_set_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
@@ -4308,7 +4431,6 @@ struct  kvm *kvm_arch_create_vm(void)
                 return ERR_PTR(-ENOMEM);
 
         INIT_LIST_HEAD(&kvm->arch.active_mmu_pages);
-        INIT_LIST_HEAD(&kvm->arch.oos_global_pages);
         INIT_LIST_HEAD(&kvm->arch.assigned_dev_head);
 
         /* Reserve bit 0 of irq_sources_bitmap for userspace irq source */
@@ -4411,12 +4533,14 @@ int kvm_arch_set_memory_region(struct kvm *kvm,
                 }
         }
 
+        spin_lock(&kvm->mmu_lock);
         if (!kvm->arch.n_requested_mmu_pages) {
                 unsigned int nr_mmu_pages = kvm_mmu_calculate_mmu_pages(kvm);
                 kvm_mmu_change_mmu_pages(kvm, nr_mmu_pages);
         }
 
         kvm_mmu_slot_remove_write_access(kvm, mem->slot);
+        spin_unlock(&kvm->mmu_lock);
         kvm_flush_remote_tlbs(kvm);
 
         return 0;
@@ -4425,6 +4549,7 @@ int kvm_arch_set_memory_region(struct kvm *kvm,
 void kvm_arch_flush_shadow(struct kvm *kvm)
 {
         kvm_mmu_zap_all(kvm);
+        kvm_reload_remote_mmus(kvm);
 }
 
 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
@@ -4434,28 +4559,24 @@ int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
                || vcpu->arch.nmi_pending;
 }
 
-static void vcpu_kick_intr(void *info)
-{
-#ifdef DEBUG
-        struct kvm_vcpu *vcpu = (struct kvm_vcpu *)info;
-        printk(KERN_DEBUG "vcpu_kick_intr %p \n", vcpu);
-#endif
-}
-
 void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
 {
-        int ipi_pcpu = vcpu->cpu;
-        int cpu = get_cpu();
+        int me;
+        int cpu = vcpu->cpu;
 
         if (waitqueue_active(&vcpu->wq)) {
                 wake_up_interruptible(&vcpu->wq);
                 ++vcpu->stat.halt_wakeup;
         }
-        /*
-         * We may be called synchronously with irqs disabled in guest mode,
-         * So need not to call smp_call_function_single() in that case.
-         */
-        if (vcpu->guest_mode && vcpu->cpu != cpu)
-                smp_call_function_single(ipi_pcpu, vcpu_kick_intr, vcpu, 0);
+
+        me = get_cpu();
+        if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu))
+                if (!test_and_set_bit(KVM_REQ_KICK, &vcpu->requests))
+                        smp_send_reschedule(cpu);
         put_cpu();
 }
+
+int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu)
+{
+        return kvm_x86_ops->interrupt_allowed(vcpu);
+}
diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h
index 6a4be78a7384..4c8e10af78e8 100644
--- a/arch/x86/kvm/x86.h
+++ b/arch/x86/kvm/x86.h
@@ -8,9 +8,11 @@ static inline void kvm_clear_exception_queue(struct kvm_vcpu *vcpu)
         vcpu->arch.exception.pending = false;
 }
 
-static inline void kvm_queue_interrupt(struct kvm_vcpu *vcpu, u8 vector)
+static inline void kvm_queue_interrupt(struct kvm_vcpu *vcpu, u8 vector,
+        bool soft)
 {
         vcpu->arch.interrupt.pending = true;
+        vcpu->arch.interrupt.soft = soft;
         vcpu->arch.interrupt.nr = vector;
 }
 
@@ -19,4 +21,14 @@ static inline void kvm_clear_interrupt_queue(struct kvm_vcpu *vcpu)
         vcpu->arch.interrupt.pending = false;
 }
 
+static inline bool kvm_event_needs_reinjection(struct kvm_vcpu *vcpu)
+{
+        return vcpu->arch.exception.pending || vcpu->arch.interrupt.pending ||
+                vcpu->arch.nmi_injected;
+}
+
+static inline bool kvm_exception_is_soft(unsigned int nr)
+{
+        return (nr == BP_VECTOR) || (nr == OF_VECTOR);
+}
 #endif
diff --git a/arch/x86/kvm/x86_emulate.c b/arch/x86/kvm/x86_emulate.c
index ca91749d2083..c1b6c232e02b 100644
--- a/arch/x86/kvm/x86_emulate.c
+++ b/arch/x86/kvm/x86_emulate.c
@@ -59,13 +59,14 @@
 #define SrcImm      (5<<4)      /* Immediate operand. */
 #define SrcImmByte  (6<<4)      /* 8-bit sign-extended immediate operand. */
 #define SrcOne      (7<<4)      /* Implied '1' */
-#define SrcMask     (7<<4)
+#define SrcImmUByte (8<<4)      /* 8-bit unsigned immediate operand. */
+#define SrcMask     (0xf<<4)
 /* Generic ModRM decode. */
-#define ModRM       (1<<7)
+#define ModRM       (1<<8)
 /* Destination is only written; never read. */
-#define Mov         (1<<8)
-#define BitOp       (1<<9)
-#define MemAbs      (1<<10)      /* Memory operand is absolute displacement */
+#define Mov         (1<<9)
+#define BitOp       (1<<10)
+#define MemAbs      (1<<11)      /* Memory operand is absolute displacement */
 #define String      (1<<12)     /* String instruction (rep capable) */
 #define Stack       (1<<13)     /* Stack instruction (push/pop) */
 #define Group       (1<<14)     /* Bits 3:5 of modrm byte extend opcode */
@@ -76,6 +77,7 @@
 #define Src2CL      (1<<29)
 #define Src2ImmByte (2<<29)
 #define Src2One     (3<<29)
+#define Src2Imm16   (4<<29)
 #define Src2Mask    (7<<29)
 
 enum {
@@ -135,11 +137,11 @@ static u32 opcode_table[256] = {
         SrcNone  | ByteOp  | ImplicitOps, SrcNone  | ImplicitOps, /* insb, insw/insd */
         SrcNone  | ByteOp  | ImplicitOps, SrcNone  | ImplicitOps, /* outsb, outsw/outsd */
         /* 0x70 - 0x77 */
-        ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps,
-        ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps,
+        SrcImmByte, SrcImmByte, SrcImmByte, SrcImmByte,
+        SrcImmByte, SrcImmByte, SrcImmByte, SrcImmByte,
         /* 0x78 - 0x7F */
-        ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps,
-        ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps,
+        SrcImmByte, SrcImmByte, SrcImmByte, SrcImmByte,
+        SrcImmByte, SrcImmByte, SrcImmByte, SrcImmByte,
         /* 0x80 - 0x87 */
         Group | Group1_80, Group | Group1_81,
         Group | Group1_82, Group | Group1_83,
@@ -153,7 +155,8 @@ static u32 opcode_table[256] = {
         /* 0x90 - 0x97 */
         DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg,
         /* 0x98 - 0x9F */
-        0, 0, 0, 0, ImplicitOps | Stack, ImplicitOps | Stack, 0, 0,
+        0, 0, SrcImm | Src2Imm16, 0,
+        ImplicitOps | Stack, ImplicitOps | Stack, 0, 0,
         /* 0xA0 - 0xA7 */
         ByteOp | DstReg | SrcMem | Mov | MemAbs, DstReg | SrcMem | Mov | MemAbs,
         ByteOp | DstMem | SrcReg | Mov | MemAbs, DstMem | SrcReg | Mov | MemAbs,
@@ -178,7 +181,8 @@ static u32 opcode_table[256] = {
         0, ImplicitOps | Stack, 0, 0,
         ByteOp | DstMem | SrcImm | ModRM | Mov, DstMem | SrcImm | ModRM | Mov,
         /* 0xC8 - 0xCF */
-        0, 0, 0, ImplicitOps | Stack, 0, 0, 0, 0,
+        0, 0, 0, ImplicitOps | Stack,
+        ImplicitOps, SrcImmByte, ImplicitOps, ImplicitOps,
         /* 0xD0 - 0xD7 */
         ByteOp | DstMem | SrcImplicit | ModRM, DstMem | SrcImplicit | ModRM,
         ByteOp | DstMem | SrcImplicit | ModRM, DstMem | SrcImplicit | ModRM,
@@ -187,11 +191,11 @@ static u32 opcode_table[256] = {
         0, 0, 0, 0, 0, 0, 0, 0,
         /* 0xE0 - 0xE7 */
         0, 0, 0, 0,
-        SrcNone | ByteOp | ImplicitOps, SrcNone | ImplicitOps,
-        SrcNone | ByteOp | ImplicitOps, SrcNone | ImplicitOps,
+        ByteOp | SrcImmUByte, SrcImmUByte,
+        ByteOp | SrcImmUByte, SrcImmUByte,
         /* 0xE8 - 0xEF */
-        ImplicitOps | Stack, SrcImm | ImplicitOps,
-        ImplicitOps, SrcImmByte | ImplicitOps,
+        SrcImm | Stack, SrcImm | ImplicitOps,
+        SrcImm | Src2Imm16, SrcImmByte | ImplicitOps,
         SrcNone | ByteOp | ImplicitOps, SrcNone | ImplicitOps,
         SrcNone | ByteOp | ImplicitOps, SrcNone | ImplicitOps,
         /* 0xF0 - 0xF7 */
@@ -230,10 +234,8 @@ static u32 twobyte_table[256] = {
         /* 0x70 - 0x7F */
         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
         /* 0x80 - 0x8F */
-        ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps,
-        ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps,
-        ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps,
-        ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps,
+        SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm,
+        SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm,
         /* 0x90 - 0x9F */
         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
         /* 0xA0 - 0xA7 */
@@ -1044,10 +1046,14 @@ done_prefixes:
                 }
                 break;
         case SrcImmByte:
+        case SrcImmUByte:
                 c->src.type = OP_IMM;
                 c->src.ptr = (unsigned long *)c->eip;
                 c->src.bytes = 1;
-                c->src.val = insn_fetch(s8, 1, c->eip);
+                if ((c->d & SrcMask) == SrcImmByte)
+                        c->src.val = insn_fetch(s8, 1, c->eip);
+                else
+                        c->src.val = insn_fetch(u8, 1, c->eip);
                 break;
         case SrcOne:
                 c->src.bytes = 1;
@@ -1072,6 +1078,12 @@ done_prefixes:
                 c->src2.bytes = 1;
                 c->src2.val = insn_fetch(u8, 1, c->eip);
                 break;
+        case Src2Imm16:
+                c->src2.type = OP_IMM;
+                c->src2.ptr = (unsigned long *)c->eip;
+                c->src2.bytes = 2;
+                c->src2.val = insn_fetch(u16, 2, c->eip);
+                break;
         case Src2One:
                 c->src2.bytes = 1;
                 c->src2.val = 1;
@@ -1349,6 +1361,20 @@ static inline int writeback(struct x86_emulate_ctxt *ctxt,
         return 0;
 }
 
+void toggle_interruptibility(struct x86_emulate_ctxt *ctxt, u32 mask)
+{
+        u32 int_shadow = kvm_x86_ops->get_interrupt_shadow(ctxt->vcpu, mask);
+        /*
+         * an sti; sti; sequence only disable interrupts for the first
+         * instruction. So, if the last instruction, be it emulated or
+         * not, left the system with the INT_STI flag enabled, it
+         * means that the last instruction is an sti. We should not
+         * leave the flag on in this case. The same goes for mov ss
+         */
+        if (!(int_shadow & mask))
+                ctxt->interruptibility = mask;
+}
+
 int
 x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
 {
@@ -1360,6 +1386,8 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
         int io_dir_in;
         int rc = 0;
 
+        ctxt->interruptibility = 0;
+
         /* Shadow copy of register state. Committed on successful emulation.
          * NOTE: we can copy them from vcpu as x86_decode_insn() doesn't
          * modify them.
@@ -1531,13 +1559,10 @@ special_insn:
                         return -1;
                 }
                 return 0;
-        case 0x70 ... 0x7f: /* jcc (short) */ {
-                int rel = insn_fetch(s8, 1, c->eip);
-
+        case 0x70 ... 0x7f: /* jcc (short) */
                 if (test_cc(c->b, ctxt->eflags))
-                        jmp_rel(c, rel);
+                        jmp_rel(c, c->src.val);
                 break;
-        }
         case 0x80 ... 0x83:     /* Grp1 */
                 switch (c->modrm_reg) {
                 case 0:
@@ -1609,6 +1634,9 @@ special_insn:
                 int err;
 
                 sel = c->src.val;
+                if (c->modrm_reg == VCPU_SREG_SS)
+                        toggle_interruptibility(ctxt, X86_SHADOW_INT_MOV_SS);
+
                 if (c->modrm_reg <= 5) {
                         type_bits = (c->modrm_reg == 1) ? 9 : 1;
                         err = kvm_load_segment_descriptor(ctxt->vcpu, sel,
@@ -1769,59 +1797,32 @@ special_insn:
                 break;
         case 0xe4:      /* inb */
         case 0xe5:      /* in */
-                port = insn_fetch(u8, 1, c->eip);
+                port = c->src.val;
                 io_dir_in = 1;
                 goto do_io;
         case 0xe6: /* outb */
         case 0xe7: /* out */
-                port = insn_fetch(u8, 1, c->eip);
+                port = c->src.val;
                 io_dir_in = 0;
                 goto do_io;
         case 0xe8: /* call (near) */ {
-                long int rel;
-                switch (c->op_bytes) {
-                case 2:
-                        rel = insn_fetch(s16, 2, c->eip);
-                        break;
-                case 4:
-                        rel = insn_fetch(s32, 4, c->eip);
-                        break;
-                default:
-                        DPRINTF("Call: Invalid op_bytes\n");
-                        goto cannot_emulate;
-                }
+                long int rel = c->src.val;
                 c->src.val = (unsigned long) c->eip;
                 jmp_rel(c, rel);
-                c->op_bytes = c->ad_bytes;
                 emulate_push(ctxt);
                 break;
         }
         case 0xe9: /* jmp rel */
                 goto jmp;
-        case 0xea: /* jmp far */ {
-                uint32_t eip;
-                uint16_t sel;
-
-                switch (c->op_bytes) {
-                case 2:
-                        eip = insn_fetch(u16, 2, c->eip);
-                        break;
-                case 4:
-                        eip = insn_fetch(u32, 4, c->eip);
-                        break;
-                default:
-                        DPRINTF("jmp far: Invalid op_bytes\n");
-                        goto cannot_emulate;
-                }
-                sel = insn_fetch(u16, 2, c->eip);
-                if (kvm_load_segment_descriptor(ctxt->vcpu, sel, 9, VCPU_SREG_CS) < 0) {
+        case 0xea: /* jmp far */
+                if (kvm_load_segment_descriptor(ctxt->vcpu, c->src2.val, 9,
+                                        VCPU_SREG_CS) < 0) {
                         DPRINTF("jmp far: Failed to load CS descriptor\n");
                         goto cannot_emulate;
                 }
 
-                c->eip = eip;
+                c->eip = c->src.val;
                 break;
-        }
         case 0xeb:
               jmp:              /* jmp rel short */
                 jmp_rel(c, c->src.val);
@@ -1865,6 +1866,7 @@ special_insn:
                 c->dst.type = OP_NONE;  /* Disable writeback. */
                 break;
         case 0xfb: /* sti */
+                toggle_interruptibility(ctxt, X86_SHADOW_INT_STI);
                 ctxt->eflags |= X86_EFLAGS_IF;
                 c->dst.type = OP_NONE;  /* Disable writeback. */
                 break;
@@ -2039,28 +2041,11 @@ twobyte_insn:
                 if (!test_cc(c->b, ctxt->eflags))
                         c->dst.type = OP_NONE; /* no writeback */
                 break;
-        case 0x80 ... 0x8f: /* jnz rel, etc*/ {
-                long int rel;
-
-                switch (c->op_bytes) {
-                case 2:
-                        rel = insn_fetch(s16, 2, c->eip);
-                        break;
-                case 4:
-                        rel = insn_fetch(s32, 4, c->eip);
-                        break;
-                case 8:
-                        rel = insn_fetch(s64, 8, c->eip);
-                        break;
-                default:
-                        DPRINTF("jnz: Invalid op_bytes\n");
-                        goto cannot_emulate;
-                }
+        case 0x80 ... 0x8f: /* jnz rel, etc*/
                 if (test_cc(c->b, ctxt->eflags))
-                        jmp_rel(c, rel);
+                        jmp_rel(c, c->src.val);
                 c->dst.type = OP_NONE;
                 break;
-        }
         case 0xa3:
               bt:               /* bt */
                 c->dst.type = OP_NONE;
diff --git a/arch/x86/lguest/Makefile b/arch/x86/lguest/Makefile
index 27f0c9ed7f60..94e0e54056a9 100644
--- a/arch/x86/lguest/Makefile
+++ b/arch/x86/lguest/Makefile
@@ -1 +1,2 @@
 obj-y           := i386_head.o boot.o
+CFLAGS_boot.o   := $(call cc-option, -fno-stack-protector)
diff --git a/arch/x86/lguest/boot.c b/arch/x86/lguest/boot.c
index 45acbcf25683..4e0c26559395 100644
--- a/arch/x86/lguest/boot.c
+++ b/arch/x86/lguest/boot.c
@@ -67,6 +67,7 @@
 #include <asm/mce.h>
 #include <asm/io.h>
 #include <asm/i387.h>
+#include <asm/stackprotector.h>
 #include <asm/reboot.h>         /* for struct machine_ops */
 
 /*G:010 Welcome to the Guest!
@@ -166,10 +167,16 @@ static void lazy_hcall3(unsigned long call,
 
 /* When lazy mode is turned off reset the per-cpu lazy mode variable and then
  * issue the do-nothing hypercall to flush any stored calls. */
-static void lguest_leave_lazy_mode(void)
+static void lguest_leave_lazy_mmu_mode(void)
 {
-        paravirt_leave_lazy(paravirt_get_lazy_mode());
         kvm_hypercall0(LHCALL_FLUSH_ASYNC);
+        paravirt_leave_lazy_mmu();
+}
+
+static void lguest_end_context_switch(struct task_struct *next)
+{
+        kvm_hypercall0(LHCALL_FLUSH_ASYNC);
+        paravirt_end_context_switch(next);
 }
 
 /*G:033
@@ -1053,8 +1060,8 @@ __init void lguest_init(void)
         pv_cpu_ops.write_gdt_entry = lguest_write_gdt_entry;
         pv_cpu_ops.write_idt_entry = lguest_write_idt_entry;
         pv_cpu_ops.wbinvd = lguest_wbinvd;
-        pv_cpu_ops.lazy_mode.enter = paravirt_enter_lazy_cpu;
-        pv_cpu_ops.lazy_mode.leave = lguest_leave_lazy_mode;
+        pv_cpu_ops.start_context_switch = paravirt_start_context_switch;
+        pv_cpu_ops.end_context_switch = lguest_end_context_switch;
 
         /* pagetable management */
         pv_mmu_ops.write_cr3 = lguest_write_cr3;
@@ -1067,7 +1074,7 @@ __init void lguest_init(void)
         pv_mmu_ops.read_cr2 = lguest_read_cr2;
         pv_mmu_ops.read_cr3 = lguest_read_cr3;
         pv_mmu_ops.lazy_mode.enter = paravirt_enter_lazy_mmu;
-        pv_mmu_ops.lazy_mode.leave = lguest_leave_lazy_mode;
+        pv_mmu_ops.lazy_mode.leave = lguest_leave_lazy_mmu_mode;
         pv_mmu_ops.pte_update = lguest_pte_update;
         pv_mmu_ops.pte_update_defer = lguest_pte_update;
 
@@ -1088,13 +1095,21 @@ __init void lguest_init(void)
          * lguest_init() where the rest of the fairly chaotic boot setup
          * occurs. */
 
+        /* The stack protector is a weird thing where gcc places a canary
+         * value on the stack and then checks it on return.  This file is
+         * compiled with -fno-stack-protector it, so we got this far without
+         * problems.  The value of the canary is kept at offset 20 from the
+         * %gs register, so we need to set that up before calling C functions
+         * in other files. */
+        setup_stack_canary_segment(0);
+        /* We could just call load_stack_canary_segment(), but we might as
+         * call switch_to_new_gdt() which loads the whole table and sets up
+         * the per-cpu segment descriptor register %fs as well. */
+        switch_to_new_gdt(0);
+
         /* As described in head_32.S, we map the first 128M of memory. */
         max_pfn_mapped = (128*1024*1024) >> PAGE_SHIFT;
 
-        /* Load the %fs segment register (the per-cpu segment register) with
-         * the normal data segment to get through booting. */
-        asm volatile ("mov %0, %%fs" : : "r" (__KERNEL_DS) : "memory");
-
         /* The Host<->Guest Switcher lives at the top of our address space, and
          * the Host told us how big it is when we made LGUEST_INIT hypercall:
          * it put the answer in lguest_data.reserve_mem  */
diff --git a/arch/x86/lib/Makefile b/arch/x86/lib/Makefile
index 55e11aa6d66c..f9d35632666b 100644
--- a/arch/x86/lib/Makefile
+++ b/arch/x86/lib/Makefile
@@ -2,7 +2,7 @@
 # Makefile for x86 specific library files.
 #
 
-obj-$(CONFIG_SMP) := msr-on-cpu.o
+obj-$(CONFIG_SMP) := msr.o
 
 lib-y := delay.o
 lib-y += thunk_$(BITS).o
diff --git a/arch/x86/lib/msr-on-cpu.c b/arch/x86/lib/msr-on-cpu.c
deleted file mode 100644
index 321cf720dbb6..000000000000
--- a/arch/x86/lib/msr-on-cpu.c
+++ /dev/null
@@ -1,97 +0,0 @@
-#include <linux/module.h>
-#include <linux/preempt.h>
-#include <linux/smp.h>
-#include <asm/msr.h>
-
-struct msr_info {
-        u32 msr_no;
-        u32 l, h;
-        int err;
-};
-
-static void __rdmsr_on_cpu(void *info)
-{
-        struct msr_info *rv = info;
-
-        rdmsr(rv->msr_no, rv->l, rv->h);
-}
-
-static void __wrmsr_on_cpu(void *info)
-{
-        struct msr_info *rv = info;
-
-        wrmsr(rv->msr_no, rv->l, rv->h);
-}
-
-int rdmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h)
-{
-        int err;
-        struct msr_info rv;
-
-        rv.msr_no = msr_no;
-        err = smp_call_function_single(cpu, __rdmsr_on_cpu, &rv, 1);
-        *l = rv.l;
-        *h = rv.h;
-
-        return err;
-}
-
-int wrmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h)
-{
-        int err;
-        struct msr_info rv;
-
-        rv.msr_no = msr_no;
-        rv.l = l;
-        rv.h = h;
-        err = smp_call_function_single(cpu, __wrmsr_on_cpu, &rv, 1);
-
-        return err;
-}
-
-/* These "safe" variants are slower and should be used when the target MSR
-   may not actually exist. */
-static void __rdmsr_safe_on_cpu(void *info)
-{
-        struct msr_info *rv = info;
-
-        rv->err = rdmsr_safe(rv->msr_no, &rv->l, &rv->h);
-}
-
-static void __wrmsr_safe_on_cpu(void *info)
-{
-        struct msr_info *rv = info;
-
-        rv->err = wrmsr_safe(rv->msr_no, rv->l, rv->h);
-}
-
-int rdmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h)
-{
-        int err;
-        struct msr_info rv;
-
-        rv.msr_no = msr_no;
-        err = smp_call_function_single(cpu, __rdmsr_safe_on_cpu, &rv, 1);
-        *l = rv.l;
-        *h = rv.h;
-
-        return err ? err : rv.err;
-}
-
-int wrmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h)
-{
-        int err;
-        struct msr_info rv;
-
-        rv.msr_no = msr_no;
-        rv.l = l;
-        rv.h = h;
-        err = smp_call_function_single(cpu, __wrmsr_safe_on_cpu, &rv, 1);
-
-        return err ? err : rv.err;
-}
-
-EXPORT_SYMBOL(rdmsr_on_cpu);
-EXPORT_SYMBOL(wrmsr_on_cpu);
-EXPORT_SYMBOL(rdmsr_safe_on_cpu);
-EXPORT_SYMBOL(wrmsr_safe_on_cpu);
diff --git a/arch/x86/lib/msr.c b/arch/x86/lib/msr.c
new file mode 100644
index 000000000000..1440b9c0547e
--- /dev/null
+++ b/arch/x86/lib/msr.c
@@ -0,0 +1,183 @@
+#include <linux/module.h>
+#include <linux/preempt.h>
+#include <linux/smp.h>
+#include <asm/msr.h>
+
+struct msr_info {
+        u32 msr_no;
+        struct msr reg;
+        struct msr *msrs;
+        int off;
+        int err;
+};
+
+static void __rdmsr_on_cpu(void *info)
+{
+        struct msr_info *rv = info;
+        struct msr *reg;
+        int this_cpu = raw_smp_processor_id();
+
+        if (rv->msrs)
+                reg = &rv->msrs[this_cpu - rv->off];
+        else
+                reg = &rv->reg;
+
+        rdmsr(rv->msr_no, reg->l, reg->h);
+}
+
+static void __wrmsr_on_cpu(void *info)
+{
+        struct msr_info *rv = info;
+        struct msr *reg;
+        int this_cpu = raw_smp_processor_id();
+
+        if (rv->msrs)
+                reg = &rv->msrs[this_cpu - rv->off];
+        else
+                reg = &rv->reg;
+
+        wrmsr(rv->msr_no, reg->l, reg->h);
+}
+
+int rdmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h)
+{
+        int err;
+        struct msr_info rv;
+
+        memset(&rv, 0, sizeof(rv));
+
+        rv.msr_no = msr_no;
+        err = smp_call_function_single(cpu, __rdmsr_on_cpu, &rv, 1);
+        *l = rv.reg.l;
+        *h = rv.reg.h;
+
+        return err;
+}
+EXPORT_SYMBOL(rdmsr_on_cpu);
+
+int wrmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h)
+{
+        int err;
+        struct msr_info rv;
+
+        memset(&rv, 0, sizeof(rv));
+
+        rv.msr_no = msr_no;
+        rv.reg.l = l;
+        rv.reg.h = h;
+        err = smp_call_function_single(cpu, __wrmsr_on_cpu, &rv, 1);
+
+        return err;
+}
+EXPORT_SYMBOL(wrmsr_on_cpu);
+
+/* rdmsr on a bunch of CPUs
+ *
+ * @mask:       which CPUs
+ * @msr_no:     which MSR
+ * @msrs:       array of MSR values
+ *
+ */
+void rdmsr_on_cpus(const cpumask_t *mask, u32 msr_no, struct msr *msrs)
+{
+        struct msr_info rv;
+        int this_cpu;
+
+        memset(&rv, 0, sizeof(rv));
+
+        rv.off    = cpumask_first(mask);
+        rv.msrs   = msrs;
+        rv.msr_no = msr_no;
+
+        preempt_disable();
+        /*
+         * FIXME: handle the CPU we're executing on separately for now until
+         * smp_call_function_many has been fixed to not skip it.
+         */
+        this_cpu = raw_smp_processor_id();
+        smp_call_function_single(this_cpu, __rdmsr_on_cpu, &rv, 1);
+
+        smp_call_function_many(mask, __rdmsr_on_cpu, &rv, 1);
+        preempt_enable();
+}
+EXPORT_SYMBOL(rdmsr_on_cpus);
+
+/*
+ * wrmsr on a bunch of CPUs
+ *
+ * @mask:       which CPUs
+ * @msr_no:     which MSR
+ * @msrs:       array of MSR values
+ *
+ */
+void wrmsr_on_cpus(const cpumask_t *mask, u32 msr_no, struct msr *msrs)
+{
+        struct msr_info rv;
+        int this_cpu;
+
+        memset(&rv, 0, sizeof(rv));
+
+        rv.off    = cpumask_first(mask);
+        rv.msrs   = msrs;
+        rv.msr_no = msr_no;
+
+        preempt_disable();
+        /*
+         * FIXME: handle the CPU we're executing on separately for now until
+         * smp_call_function_many has been fixed to not skip it.
+         */
+        this_cpu = raw_smp_processor_id();
+        smp_call_function_single(this_cpu, __wrmsr_on_cpu, &rv, 1);
+
+        smp_call_function_many(mask, __wrmsr_on_cpu, &rv, 1);
+        preempt_enable();
+}
+EXPORT_SYMBOL(wrmsr_on_cpus);
+
+/* These "safe" variants are slower and should be used when the target MSR
+   may not actually exist. */
+static void __rdmsr_safe_on_cpu(void *info)
+{
+        struct msr_info *rv = info;
+
+        rv->err = rdmsr_safe(rv->msr_no, &rv->reg.l, &rv->reg.h);
+}
+
+static void __wrmsr_safe_on_cpu(void *info)
+{
+        struct msr_info *rv = info;
+
+        rv->err = wrmsr_safe(rv->msr_no, rv->reg.l, rv->reg.h);
+}
+
+int rdmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h)
+{
+        int err;
+        struct msr_info rv;
+
+        memset(&rv, 0, sizeof(rv));
+
+        rv.msr_no = msr_no;
+        err = smp_call_function_single(cpu, __rdmsr_safe_on_cpu, &rv, 1);
+        *l = rv.reg.l;
+        *h = rv.reg.h;
+
+        return err ? err : rv.err;
+}
+EXPORT_SYMBOL(rdmsr_safe_on_cpu);
+
+int wrmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h)
+{
+        int err;
+        struct msr_info rv;
+
+        memset(&rv, 0, sizeof(rv));
+
+        rv.msr_no = msr_no;
+        rv.reg.l = l;
+        rv.reg.h = h;
+        err = smp_call_function_single(cpu, __wrmsr_safe_on_cpu, &rv, 1);
+
+        return err ? err : rv.err;
+}
+EXPORT_SYMBOL(wrmsr_safe_on_cpu);
diff --git a/arch/x86/mm/dump_pagetables.c b/arch/x86/mm/dump_pagetables.c
index e7277cbcfb40..a725b7f760ae 100644
--- a/arch/x86/mm/dump_pagetables.c
+++ b/arch/x86/mm/dump_pagetables.c
@@ -161,13 +161,14 @@ static void note_page(struct seq_file *m, struct pg_state *st,
                    st->current_address >= st->marker[1].start_address) {
                 const char *unit = units;
                 unsigned long delta;
+                int width = sizeof(unsigned long) * 2;
 
                 /*
                  * Now print the actual finished series
                  */
-                seq_printf(m, "0x%p-0x%p   ",
-                           (void *)st->start_address,
-                           (void *)st->current_address);
+                seq_printf(m, "0x%0*lx-0x%0*lx   ",
+                           width, st->start_address,
+                           width, st->current_address);
 
                 delta = (st->current_address - st->start_address) >> 10;
                 while (!(delta & 1023) && unit[1]) {
diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c
index a03b7279efa0..c6acc6326374 100644
--- a/arch/x86/mm/fault.c
+++ b/arch/x86/mm/fault.c
@@ -3,40 +3,17 @@
  *  Copyright (C) 2001, 2002 Andi Kleen, SuSE Labs.
  *  Copyright (C) 2008-2009, Red Hat Inc., Ingo Molnar
  */
-#include <linux/interrupt.h>
-#include <linux/mmiotrace.h>
-#include <linux/bootmem.h>
-#include <linux/compiler.h>
-#include <linux/highmem.h>
-#include <linux/kprobes.h>
-#include <linux/uaccess.h>
-#include <linux/vmalloc.h>
-#include <linux/vt_kern.h>
-#include <linux/signal.h>
-#include <linux/kernel.h>
-#include <linux/ptrace.h>
-#include <linux/string.h>
-#include <linux/module.h>
-#include <linux/kdebug.h>
-#include <linux/errno.h>
-#include <linux/magic.h>
-#include <linux/sched.h>
-#include <linux/types.h>
-#include <linux/init.h>
-#include <linux/mman.h>
-#include <linux/tty.h>
-#include <linux/smp.h>
-#include <linux/mm.h>
-
-#include <asm-generic/sections.h>
-
-#include <asm/tlbflush.h>
-#include <asm/pgalloc.h>
-#include <asm/segment.h>
-#include <asm/system.h>
-#include <asm/proto.h>
-#include <asm/traps.h>
-#include <asm/desc.h>
+#include <linux/magic.h>                /* STACK_END_MAGIC              */
+#include <linux/sched.h>                /* test_thread_flag(), ...      */
+#include <linux/kdebug.h>               /* oops_begin/end, ...          */
+#include <linux/module.h>               /* search_exception_table       */
+#include <linux/bootmem.h>              /* max_low_pfn                  */
+#include <linux/kprobes.h>              /* __kprobes, ...               */
+#include <linux/mmiotrace.h>            /* kmmio_handler, ...           */
+#include <linux/perf_counter.h>         /* perf_swcounter_event         */
+
+#include <asm/traps.h>                  /* dotraplinkage, ...           */
+#include <asm/pgalloc.h>                /* pgd_*(), ...                 */
 
 /*
  * Page fault error code bits:
@@ -225,12 +202,10 @@ static inline pmd_t *vmalloc_sync_one(pgd_t *pgd, unsigned long address)
         if (!pmd_present(*pmd_k))
                 return NULL;
 
-        if (!pmd_present(*pmd)) {
+        if (!pmd_present(*pmd))
                 set_pmd(pmd, *pmd_k);
-                arch_flush_lazy_mmu_mode();
-        } else {
+        else
                 BUG_ON(pmd_page(*pmd) != pmd_page(*pmd_k));
-        }
 
         return pmd_k;
 }
@@ -538,8 +513,6 @@ bad:
 static int is_errata93(struct pt_regs *regs, unsigned long address)
 {
 #ifdef CONFIG_X86_64
-        static int once;
-
         if (address != regs->ip)
                 return 0;
 
@@ -549,10 +522,7 @@ static int is_errata93(struct pt_regs *regs, unsigned long address)
         address |= 0xffffffffUL << 32;
         if ((address >= (u64)_stext && address <= (u64)_etext) ||
             (address >= MODULES_VADDR && address <= MODULES_END)) {
-                if (!once) {
-                        printk(errata93_warning);
-                        once = 1;
-                }
+                printk_once(errata93_warning);
                 regs->ip = address;
                 return 1;
         }
@@ -1044,6 +1014,8 @@ do_page_fault(struct pt_regs *regs, unsigned long error_code)
         if (unlikely(error_code & PF_RSVD))
                 pgtable_bad(regs, error_code, address);
 
+        perf_swcounter_event(PERF_COUNT_SW_PAGE_FAULTS, 1, 0, regs, address);
+
         /*
          * If we're in an interrupt, have no user context or are running
          * in an atomic region then we must not take the fault:
@@ -1137,10 +1109,15 @@ good_area:
                 return;
         }
 
-        if (fault & VM_FAULT_MAJOR)
+        if (fault & VM_FAULT_MAJOR) {
                 tsk->maj_flt++;
-        else
+                perf_swcounter_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, 0,
+                                     regs, address);
+        } else {
                 tsk->min_flt++;
+                perf_swcounter_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, 0,
+                                     regs, address);
+        }
 
         check_v8086_mode(regs, address, tsk);
 
diff --git a/arch/x86/mm/highmem_32.c b/arch/x86/mm/highmem_32.c
index 8126e8d1a2a4..58f621e81919 100644
--- a/arch/x86/mm/highmem_32.c
+++ b/arch/x86/mm/highmem_32.c
@@ -44,7 +44,6 @@ void *kmap_atomic_prot(struct page *page, enum km_type type, pgprot_t prot)
         vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx);
         BUG_ON(!pte_none(*(kmap_pte-idx)));
         set_pte(kmap_pte-idx, mk_pte(page, prot));
-        arch_flush_lazy_mmu_mode();
 
         return (void *)vaddr;
 }
@@ -74,7 +73,6 @@ void kunmap_atomic(void *kvaddr, enum km_type type)
 #endif
         }
 
-        arch_flush_lazy_mmu_mode();
         pagefault_enable();
 }
 
diff --git a/arch/x86/mm/init.c b/arch/x86/mm/init.c
index ae4f7b5d7104..34c1bfb64f1c 100644
--- a/arch/x86/mm/init.c
+++ b/arch/x86/mm/init.c
@@ -1,3 +1,4 @@
+#include <linux/initrd.h>
 #include <linux/ioport.h>
 #include <linux/swap.h>
 
@@ -10,6 +11,9 @@
 #include <asm/setup.h>
 #include <asm/system.h>
 #include <asm/tlbflush.h>
+#include <asm/tlb.h>
+
+DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
 
 unsigned long __initdata e820_table_start;
 unsigned long __meminitdata e820_table_end;
@@ -23,6 +27,69 @@ int direct_gbpages
 #endif
 ;
 
+int nx_enabled;
+
+#if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
+static int disable_nx __cpuinitdata;
+
+/*
+ * noexec = on|off
+ *
+ * Control non-executable mappings for processes.
+ *
+ * on      Enable
+ * off     Disable
+ */
+static int __init noexec_setup(char *str)
+{
+        if (!str)
+                return -EINVAL;
+        if (!strncmp(str, "on", 2)) {
+                __supported_pte_mask |= _PAGE_NX;
+                disable_nx = 0;
+        } else if (!strncmp(str, "off", 3)) {
+                disable_nx = 1;
+                __supported_pte_mask &= ~_PAGE_NX;
+        }
+        return 0;
+}
+early_param("noexec", noexec_setup);
+#endif
+
+#ifdef CONFIG_X86_PAE
+static void __init set_nx(void)
+{
+        unsigned int v[4], l, h;
+
+        if (cpu_has_pae && (cpuid_eax(0x80000000) > 0x80000001)) {
+                cpuid(0x80000001, &v[0], &v[1], &v[2], &v[3]);
+
+                if ((v[3] & (1 << 20)) && !disable_nx) {
+                        rdmsr(MSR_EFER, l, h);
+                        l |= EFER_NX;
+                        wrmsr(MSR_EFER, l, h);
+                        nx_enabled = 1;
+                        __supported_pte_mask |= _PAGE_NX;
+                }
+        }
+}
+#else
+static inline void set_nx(void)
+{
+}
+#endif
+
+#ifdef CONFIG_X86_64
+void __cpuinit check_efer(void)
+{
+        unsigned long efer;
+
+        rdmsrl(MSR_EFER, efer);
+        if (!(efer & EFER_NX) || disable_nx)
+                __supported_pte_mask &= ~_PAGE_NX;
+}
+#endif
+
 static void __init find_early_table_space(unsigned long end, int use_pse,
                                           int use_gbpages)
 {
@@ -66,12 +133,11 @@ static void __init find_early_table_space(unsigned long end, int use_pse,
          */
 #ifdef CONFIG_X86_32
         start = 0x7000;
-        e820_table_start = find_e820_area(start, max_pfn_mapped<<PAGE_SHIFT,
-                                        tables, PAGE_SIZE);
-#else /* CONFIG_X86_64 */
+#else
         start = 0x8000;
-        e820_table_start = find_e820_area(start, end, tables, PAGE_SIZE);
 #endif
+        e820_table_start = find_e820_area(start, max_pfn_mapped<<PAGE_SHIFT,
+                                        tables, PAGE_SIZE);
         if (e820_table_start == -1UL)
                 panic("Cannot find space for the kernel page tables");
 
@@ -159,12 +225,9 @@ unsigned long __init_refok init_memory_mapping(unsigned long start,
         use_gbpages = direct_gbpages;
 #endif
 
-#ifdef CONFIG_X86_32
-#ifdef CONFIG_X86_PAE
         set_nx();
         if (nx_enabled)
                 printk(KERN_INFO "NX (Execute Disable) protection: active\n");
-#endif
 
         /* Enable PSE if available */
         if (cpu_has_pse)
@@ -175,7 +238,6 @@ unsigned long __init_refok init_memory_mapping(unsigned long start,
                 set_in_cr4(X86_CR4_PGE);
                 __supported_pte_mask |= _PAGE_GLOBAL;
         }
-#endif
 
         if (use_gbpages)
                 page_size_mask |= 1 << PG_LEVEL_1G;
diff --git a/arch/x86/mm/init_32.c b/arch/x86/mm/init_32.c
index 749559ed80f5..949708d7a481 100644
--- a/arch/x86/mm/init_32.c
+++ b/arch/x86/mm/init_32.c
@@ -49,12 +49,9 @@
 #include <asm/paravirt.h>
 #include <asm/setup.h>
 #include <asm/cacheflush.h>
+#include <asm/page_types.h>
 #include <asm/init.h>
 
-unsigned long max_low_pfn_mapped;
-unsigned long max_pfn_mapped;
-
-DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
 unsigned long highstart_pfn, highend_pfn;
 
 static noinline int do_test_wp_bit(void);
@@ -587,61 +584,9 @@ void zap_low_mappings(void)
         flush_tlb_all();
 }
 
-int nx_enabled;
-
 pteval_t __supported_pte_mask __read_mostly = ~(_PAGE_NX | _PAGE_GLOBAL | _PAGE_IOMAP);
 EXPORT_SYMBOL_GPL(__supported_pte_mask);
 
-#ifdef CONFIG_X86_PAE
-
-static int disable_nx __initdata;
-
-/*
- * noexec = on|off
- *
- * Control non executable mappings.
- *
- * on      Enable
- * off     Disable
- */
-static int __init noexec_setup(char *str)
-{
-        if (!str || !strcmp(str, "on")) {
-                if (cpu_has_nx) {
-                        __supported_pte_mask |= _PAGE_NX;
-                        disable_nx = 0;
-                }
-        } else {
-                if (!strcmp(str, "off")) {
-                        disable_nx = 1;
-                        __supported_pte_mask &= ~_PAGE_NX;
-                } else {
-                        return -EINVAL;
-                }
-        }
-
-        return 0;
-}
-early_param("noexec", noexec_setup);
-
-void __init set_nx(void)
-{
-        unsigned int v[4], l, h;
-
-        if (cpu_has_pae && (cpuid_eax(0x80000000) > 0x80000001)) {
-                cpuid(0x80000001, &v[0], &v[1], &v[2], &v[3]);
-
-                if ((v[3] & (1 << 20)) && !disable_nx) {
-                        rdmsr(MSR_EFER, l, h);
-                        l |= EFER_NX;
-                        wrmsr(MSR_EFER, l, h);
-                        nx_enabled = 1;
-                        __supported_pte_mask |= _PAGE_NX;
-                }
-        }
-}
-#endif
-
 /* user-defined highmem size */
 static unsigned int highmem_pages = -1;
 
@@ -761,15 +706,15 @@ void __init initmem_init(unsigned long start_pfn,
         highstart_pfn = highend_pfn = max_pfn;
         if (max_pfn > max_low_pfn)
                 highstart_pfn = max_low_pfn;
-        memory_present(0, 0, highend_pfn);
         e820_register_active_regions(0, 0, highend_pfn);
+        sparse_memory_present_with_active_regions(0);
         printk(KERN_NOTICE "%ldMB HIGHMEM available.\n",
                 pages_to_mb(highend_pfn - highstart_pfn));
         num_physpages = highend_pfn;
         high_memory = (void *) __va(highstart_pfn * PAGE_SIZE - 1) + 1;
 #else
-        memory_present(0, 0, max_low_pfn);
         e820_register_active_regions(0, 0, max_low_pfn);
+        sparse_memory_present_with_active_regions(0);
         num_physpages = max_low_pfn;
         high_memory = (void *) __va(max_low_pfn * PAGE_SIZE - 1) + 1;
 #endif
diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
index 1753e8020df6..52bb9519bb86 100644
--- a/arch/x86/mm/init_64.c
+++ b/arch/x86/mm/init_64.c
@@ -50,18 +50,8 @@
 #include <asm/cacheflush.h>
 #include <asm/init.h>
 
-/*
- * end_pfn only includes RAM, while max_pfn_mapped includes all e820 entries.
- * The direct mapping extends to max_pfn_mapped, so that we can directly access
- * apertures, ACPI and other tables without having to play with fixmaps.
- */
-unsigned long max_low_pfn_mapped;
-unsigned long max_pfn_mapped;
-
 static unsigned long dma_reserve __initdata;
 
-DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
-
 static int __init parse_direct_gbpages_off(char *arg)
 {
         direct_gbpages = 0;
@@ -85,39 +75,6 @@ early_param("gbpages", parse_direct_gbpages_on);
 pteval_t __supported_pte_mask __read_mostly = ~_PAGE_IOMAP;
 EXPORT_SYMBOL_GPL(__supported_pte_mask);
 
-static int disable_nx __cpuinitdata;
-
-/*
- * noexec=on|off
- * Control non-executable mappings for 64-bit processes.
- *
- * on   Enable (default)
- * off  Disable
- */
-static int __init nonx_setup(char *str)
-{
-        if (!str)
-                return -EINVAL;
-        if (!strncmp(str, "on", 2)) {
-                __supported_pte_mask |= _PAGE_NX;
-                disable_nx = 0;
-        } else if (!strncmp(str, "off", 3)) {
-                disable_nx = 1;
-                __supported_pte_mask &= ~_PAGE_NX;
-        }
-        return 0;
-}
-early_param("noexec", nonx_setup);
-
-void __cpuinit check_efer(void)
-{
-        unsigned long efer;
-
-        rdmsrl(MSR_EFER, efer);
-        if (!(efer & EFER_NX) || disable_nx)
-                __supported_pte_mask &= ~_PAGE_NX;
-}
-
 int force_personality32;
 
 /*
@@ -628,6 +585,7 @@ void __init initmem_init(unsigned long start_pfn, unsigned long end_pfn)
         early_res_to_bootmem(0, end_pfn<<PAGE_SHIFT);
         reserve_bootmem(bootmap, bootmap_size, BOOTMEM_DEFAULT);
 }
+#endif
 
 void __init paging_init(void)
 {
@@ -638,11 +596,10 @@ void __init paging_init(void)
         max_zone_pfns[ZONE_DMA32] = MAX_DMA32_PFN;
         max_zone_pfns[ZONE_NORMAL] = max_pfn;
 
-        memory_present(0, 0, max_pfn);
+        sparse_memory_present_with_active_regions(MAX_NUMNODES);
         sparse_init();
         free_area_init_nodes(max_zone_pfns);
 }
-#endif
 
 /*
  * Memory hotplug specific functions
diff --git a/arch/x86/mm/iomap_32.c b/arch/x86/mm/iomap_32.c
index 8056545e2d39..fe6f84ca121e 100644
--- a/arch/x86/mm/iomap_32.c
+++ b/arch/x86/mm/iomap_32.c
@@ -82,7 +82,6 @@ iounmap_atomic(void *kvaddr, enum km_type type)
         if (vaddr == __fix_to_virt(FIX_KMAP_BEGIN+idx))
                 kpte_clear_flush(kmap_pte-idx, vaddr);
 
-        arch_flush_lazy_mmu_mode();
         pagefault_enable();
 }
 EXPORT_SYMBOL_GPL(iounmap_atomic);
diff --git a/arch/x86/mm/kmmio.c b/arch/x86/mm/kmmio.c
index 50dc802a1c46..16ccbd77917f 100644
--- a/arch/x86/mm/kmmio.c
+++ b/arch/x86/mm/kmmio.c
@@ -32,7 +32,7 @@ struct kmmio_fault_page {
         struct list_head list;
         struct kmmio_fault_page *release_next;
         unsigned long page; /* location of the fault page */
-        bool old_presence; /* page presence prior to arming */
+        pteval_t old_presence; /* page presence prior to arming */
         bool armed;
 
         /*
@@ -97,60 +97,62 @@ static struct kmmio_probe *get_kmmio_probe(unsigned long addr)
 static struct kmmio_fault_page *get_kmmio_fault_page(unsigned long page)
 {
         struct list_head *head;
-        struct kmmio_fault_page *p;
+        struct kmmio_fault_page *f;
 
         page &= PAGE_MASK;
         head = kmmio_page_list(page);
-        list_for_each_entry_rcu(p, head, list) {
-                if (p->page == page)
-                        return p;
+        list_for_each_entry_rcu(f, head, list) {
+                if (f->page == page)
+                        return f;
         }
         return NULL;
 }
 
-static void set_pmd_presence(pmd_t *pmd, bool present, bool *old)
+static void clear_pmd_presence(pmd_t *pmd, bool clear, pmdval_t *old)
 {
         pmdval_t v = pmd_val(*pmd);
-        *old = !!(v & _PAGE_PRESENT);
-        v &= ~_PAGE_PRESENT;
-        if (present)
-                v |= _PAGE_PRESENT;
+        if (clear) {
+                *old = v & _PAGE_PRESENT;
+                v &= ~_PAGE_PRESENT;
+        } else  /* presume this has been called with clear==true previously */
+                v |= *old;
         set_pmd(pmd, __pmd(v));
 }
 
-static void set_pte_presence(pte_t *pte, bool present, bool *old)
+static void clear_pte_presence(pte_t *pte, bool clear, pteval_t *old)
 {
         pteval_t v = pte_val(*pte);
-        *old = !!(v & _PAGE_PRESENT);
-        v &= ~_PAGE_PRESENT;
-        if (present)
-                v |= _PAGE_PRESENT;
+        if (clear) {
+                *old = v & _PAGE_PRESENT;
+                v &= ~_PAGE_PRESENT;
+        } else  /* presume this has been called with clear==true previously */
+                v |= *old;
         set_pte_atomic(pte, __pte(v));
 }
 
-static int set_page_presence(unsigned long addr, bool present, bool *old)
+static int clear_page_presence(struct kmmio_fault_page *f, bool clear)
 {
         unsigned int level;
-        pte_t *pte = lookup_address(addr, &level);
+        pte_t *pte = lookup_address(f->page, &level);
 
         if (!pte) {
-                pr_err("kmmio: no pte for page 0x%08lx\n", addr);
+                pr_err("kmmio: no pte for page 0x%08lx\n", f->page);
                 return -1;
         }
 
         switch (level) {
         case PG_LEVEL_2M:
-                set_pmd_presence((pmd_t *)pte, present, old);
+                clear_pmd_presence((pmd_t *)pte, clear, &f->old_presence);
                 break;
         case PG_LEVEL_4K:
-                set_pte_presence(pte, present, old);
+                clear_pte_presence(pte, clear, &f->old_presence);
                 break;
         default:
                 pr_err("kmmio: unexpected page level 0x%x.\n", level);
                 return -1;
         }
 
-        __flush_tlb_one(addr);
+        __flush_tlb_one(f->page);
         return 0;
 }
 
@@ -171,9 +173,9 @@ static int arm_kmmio_fault_page(struct kmmio_fault_page *f)
         WARN_ONCE(f->armed, KERN_ERR "kmmio page already armed.\n");
         if (f->armed) {
                 pr_warning("kmmio double-arm: page 0x%08lx, ref %d, old %d\n",
-                                        f->page, f->count, f->old_presence);
+                                        f->page, f->count, !!f->old_presence);
         }
-        ret = set_page_presence(f->page, false, &f->old_presence);
+        ret = clear_page_presence(f, true);
         WARN_ONCE(ret < 0, KERN_ERR "kmmio arming 0x%08lx failed.\n", f->page);
         f->armed = true;
         return ret;
@@ -182,8 +184,7 @@ static int arm_kmmio_fault_page(struct kmmio_fault_page *f)
 /** Restore the given page to saved presence state. */
 static void disarm_kmmio_fault_page(struct kmmio_fault_page *f)
 {
-        bool tmp;
-        int ret = set_page_presence(f->page, f->old_presence, &tmp);
+        int ret = clear_page_presence(f, false);
         WARN_ONCE(ret < 0,
                         KERN_ERR "kmmio disarming 0x%08lx failed.\n", f->page);
         f->armed = false;
@@ -310,7 +311,12 @@ static int post_kmmio_handler(unsigned long condition, struct pt_regs *regs)
         struct kmmio_context *ctx = &get_cpu_var(kmmio_ctx);
 
         if (!ctx->active) {
-                pr_debug("kmmio: spurious debug trap on CPU %d.\n",
+                /*
+                 * debug traps without an active context are due to either
+                 * something external causing them (f.e. using a debugger while
+                 * mmio tracing enabled), or erroneous behaviour
+                 */
+                pr_warning("kmmio: unexpected debug trap on CPU %d.\n",
                                                         smp_processor_id());
                 goto out;
         }
@@ -439,12 +445,12 @@ static void rcu_free_kmmio_fault_pages(struct rcu_head *head)
                                                 head,
                                                 struct kmmio_delayed_release,
                                                 rcu);
-        struct kmmio_fault_page *p = dr->release_list;
-        while (p) {
-                struct kmmio_fault_page *next = p->release_next;
-                BUG_ON(p->count);
-                kfree(p);
-                p = next;
+        struct kmmio_fault_page *f = dr->release_list;
+        while (f) {
+                struct kmmio_fault_page *next = f->release_next;
+                BUG_ON(f->count);
+                kfree(f);
+                f = next;
         }
         kfree(dr);
 }
@@ -453,19 +459,19 @@ static void remove_kmmio_fault_pages(struct rcu_head *head)
 {
         struct kmmio_delayed_release *dr =
                 container_of(head, struct kmmio_delayed_release, rcu);
-        struct kmmio_fault_page *p = dr->release_list;
+        struct kmmio_fault_page *f = dr->release_list;
         struct kmmio_fault_page **prevp = &dr->release_list;
         unsigned long flags;
 
         spin_lock_irqsave(&kmmio_lock, flags);
-        while (p) {
-                if (!p->count) {
-                        list_del_rcu(&p->list);
-                        prevp = &p->release_next;
+        while (f) {
+                if (!f->count) {
+                        list_del_rcu(&f->list);
+                        prevp = &f->release_next;
                 } else {
-                        *prevp = p->release_next;
+                        *prevp = f->release_next;
                 }
-                p = p->release_next;
+                f = f->release_next;
         }
         spin_unlock_irqrestore(&kmmio_lock, flags);
 
@@ -528,8 +534,8 @@ void unregister_kmmio_probe(struct kmmio_probe *p)
 }
 EXPORT_SYMBOL(unregister_kmmio_probe);
 
-static int kmmio_die_notifier(struct notifier_block *nb, unsigned long val,
-                                                                void *args)
+static int
+kmmio_die_notifier(struct notifier_block *nb, unsigned long val, void *args)
 {
         struct die_args *arg = args;
 
@@ -544,11 +550,23 @@ static struct notifier_block nb_die = {
         .notifier_call = kmmio_die_notifier
 };
 
-static int __init init_kmmio(void)
+int kmmio_init(void)
 {
         int i;
+
         for (i = 0; i < KMMIO_PAGE_TABLE_SIZE; i++)
                 INIT_LIST_HEAD(&kmmio_page_table[i]);
+
         return register_die_notifier(&nb_die);
 }
-fs_initcall(init_kmmio); /* should be before device_initcall() */
+
+void kmmio_cleanup(void)
+{
+        int i;
+
+        unregister_die_notifier(&nb_die);
+        for (i = 0; i < KMMIO_PAGE_TABLE_SIZE; i++) {
+                WARN_ONCE(!list_empty(&kmmio_page_table[i]),
+                        KERN_ERR "kmmio_page_table not empty at cleanup, any further tracing will leak memory.\n");
+        }
+}
diff --git a/arch/x86/mm/memtest.c b/arch/x86/mm/memtest.c
index 605c8be06217..18d244f70205 100644
--- a/arch/x86/mm/memtest.c
+++ b/arch/x86/mm/memtest.c
@@ -40,23 +40,22 @@ static void __init reserve_bad_mem(u64 pattern, u64 start_bad, u64 end_bad)
 
 static void __init memtest(u64 pattern, u64 start_phys, u64 size)
 {
-        u64 i, count;
-        u64 *start;
+        u64 *p, *start, *end;
         u64 start_bad, last_bad;
         u64 start_phys_aligned;
-        size_t incr;
+        const size_t incr = sizeof(pattern);
 
-        incr = sizeof(pattern);
         start_phys_aligned = ALIGN(start_phys, incr);
-        count = (size - (start_phys_aligned - start_phys))/incr;
         start = __va(start_phys_aligned);
+        end = start + (size - (start_phys_aligned - start_phys)) / incr;
         start_bad = 0;
         last_bad = 0;
 
-        for (i = 0; i < count; i++)
-                start[i] = pattern;
-        for (i = 0; i < count; i++, start++, start_phys_aligned += incr) {
-                if (*start == pattern)
+        for (p = start; p < end; p++)
+                *p = pattern;
+
+        for (p = start; p < end; p++, start_phys_aligned += incr) {
+                if (*p == pattern)
                         continue;
                 if (start_phys_aligned == last_bad + incr) {
                         last_bad += incr;
diff --git a/arch/x86/mm/mmio-mod.c b/arch/x86/mm/mmio-mod.c
index c9342ed8b402..132772a8ec57 100644
--- a/arch/x86/mm/mmio-mod.c
+++ b/arch/x86/mm/mmio-mod.c
@@ -451,6 +451,7 @@ void enable_mmiotrace(void)
 
         if (nommiotrace)
                 pr_info(NAME "MMIO tracing disabled.\n");
+        kmmio_init();
         enter_uniprocessor();
         spin_lock_irq(&trace_lock);
         atomic_inc(&mmiotrace_enabled);
@@ -473,6 +474,7 @@ void disable_mmiotrace(void)
 
         clear_trace_list(); /* guarantees: no more kmmio callbacks */
         leave_uniprocessor();
+        kmmio_cleanup();
         pr_info(NAME "disabled.\n");
 out:
         mutex_unlock(&mmiotrace_mutex);
diff --git a/arch/x86/mm/numa_64.c b/arch/x86/mm/numa_64.c
index 2d05a12029dc..459913beac71 100644
--- a/arch/x86/mm/numa_64.c
+++ b/arch/x86/mm/numa_64.c
@@ -179,18 +179,25 @@ static void * __init early_node_mem(int nodeid, unsigned long start,
 }
 
 /* Initialize bootmem allocator for a node */
-void __init setup_node_bootmem(int nodeid, unsigned long start,
-                               unsigned long end)
+void __init
+setup_node_bootmem(int nodeid, unsigned long start, unsigned long end)
 {
         unsigned long start_pfn, last_pfn, bootmap_pages, bootmap_size;
+        const int pgdat_size = roundup(sizeof(pg_data_t), PAGE_SIZE);
         unsigned long bootmap_start, nodedata_phys;
         void *bootmap;
-        const int pgdat_size = roundup(sizeof(pg_data_t), PAGE_SIZE);
         int nid;
 
         if (!end)
                 return;
 
+        /*
+         * Don't confuse VM with a node that doesn't have the
+         * minimum amount of memory:
+         */
+        if (end && (end - start) < NODE_MIN_SIZE)
+                return;
+
         start = roundup(start, ZONE_ALIGN);
 
         printk(KERN_INFO "Bootmem setup node %d %016lx-%016lx\n", nodeid,
@@ -272,9 +279,6 @@ void __init setup_node_bootmem(int nodeid, unsigned long start,
                 reserve_bootmem_node(NODE_DATA(nodeid), bootmap_start,
                                  bootmap_pages<<PAGE_SHIFT, BOOTMEM_DEFAULT);
 
-#ifdef CONFIG_ACPI_NUMA
-        srat_reserve_add_area(nodeid);
-#endif
         node_set_online(nodeid);
 }
 
@@ -578,21 +582,6 @@ unsigned long __init numa_free_all_bootmem(void)
         return pages;
 }
 
-void __init paging_init(void)
-{
-        unsigned long max_zone_pfns[MAX_NR_ZONES];
-
-        memset(max_zone_pfns, 0, sizeof(max_zone_pfns));
-        max_zone_pfns[ZONE_DMA] = MAX_DMA_PFN;
-        max_zone_pfns[ZONE_DMA32] = MAX_DMA32_PFN;
-        max_zone_pfns[ZONE_NORMAL] = max_pfn;
-
-        sparse_memory_present_with_active_regions(MAX_NUMNODES);
-        sparse_init();
-
-        free_area_init_nodes(max_zone_pfns);
-}
-
 static __init int numa_setup(char *opt)
 {
         if (!opt)
@@ -606,8 +595,6 @@ static __init int numa_setup(char *opt)
 #ifdef CONFIG_ACPI_NUMA
         if (!strncmp(opt, "noacpi", 6))
                 acpi_numa = -1;
-        if (!strncmp(opt, "hotadd=", 7))
-                hotadd_percent = simple_strtoul(opt+7, NULL, 10);
 #endif
         return 0;
 }
diff --git a/arch/x86/mm/pageattr.c b/arch/x86/mm/pageattr.c
index e17efed088c5..6ce9518fe2ac 100644
--- a/arch/x86/mm/pageattr.c
+++ b/arch/x86/mm/pageattr.c
@@ -839,13 +839,6 @@ static int change_page_attr_set_clr(unsigned long *addr, int numpages,
 
         vm_unmap_aliases();
 
-        /*
-         * If we're called with lazy mmu updates enabled, the
-         * in-memory pte state may be stale.  Flush pending updates to
-         * bring them up to date.
-         */
-        arch_flush_lazy_mmu_mode();
-
         cpa.vaddr = addr;
         cpa.pages = pages;
         cpa.numpages = numpages;
@@ -890,13 +883,6 @@ static int change_page_attr_set_clr(unsigned long *addr, int numpages,
         } else
                 cpa_flush_all(cache);
 
-        /*
-         * If we've been called with lazy mmu updates enabled, then
-         * make sure that everything gets flushed out before we
-         * return.
-         */
-        arch_flush_lazy_mmu_mode();
-
 out:
         return ret;
 }
diff --git a/arch/x86/mm/srat_64.c b/arch/x86/mm/srat_64.c
index 01765955baaf..2dfcbf9df2ae 100644
--- a/arch/x86/mm/srat_64.c
+++ b/arch/x86/mm/srat_64.c
@@ -31,17 +31,11 @@ static nodemask_t nodes_parsed __initdata;
 static nodemask_t cpu_nodes_parsed __initdata;
 static struct bootnode nodes[MAX_NUMNODES] __initdata;
 static struct bootnode nodes_add[MAX_NUMNODES];
-static int found_add_area __initdata;
-int hotadd_percent __initdata = 0;
 
 static int num_node_memblks __initdata;
 static struct bootnode node_memblk_range[NR_NODE_MEMBLKS] __initdata;
 static int memblk_nodeid[NR_NODE_MEMBLKS] __initdata;
 
-/* Too small nodes confuse the VM badly. Usually they result
-   from BIOS bugs. */
-#define NODE_MIN_SIZE (4*1024*1024)
-
 static __init int setup_node(int pxm)
 {
         return acpi_map_pxm_to_node(pxm);
@@ -66,9 +60,6 @@ static __init void cutoff_node(int i, unsigned long start, unsigned long end)
 {
         struct bootnode *nd = &nodes[i];
 
-        if (found_add_area)
-                return;
-
         if (nd->start < start) {
                 nd->start = start;
                 if (nd->end < nd->start)
@@ -86,7 +77,6 @@ static __init void bad_srat(void)
         int i;
         printk(KERN_ERR "SRAT: SRAT not used.\n");
         acpi_numa = -1;
-        found_add_area = 0;
         for (i = 0; i < MAX_LOCAL_APIC; i++)
                 apicid_to_node[i] = NUMA_NO_NODE;
         for (i = 0; i < MAX_NUMNODES; i++)
@@ -182,24 +172,21 @@ acpi_numa_processor_affinity_init(struct acpi_srat_cpu_affinity *pa)
                pxm, apic_id, node);
 }
 
-static int update_end_of_memory(unsigned long end) {return -1;}
-static int hotadd_enough_memory(struct bootnode *nd) {return 1;}
 #ifdef CONFIG_MEMORY_HOTPLUG_SPARSE
 static inline int save_add_info(void) {return 1;}
 #else
 static inline int save_add_info(void) {return 0;}
 #endif
 /*
- * Update nodes_add and decide if to include add are in the zone.
- * Both SPARSE and RESERVE need nodes_add information.
- * This code supports one contiguous hot add area per node.
+ * Update nodes_add[]
+ * This code supports one contiguous hot add area per node
  */
-static int __init
-reserve_hotadd(int node, unsigned long start, unsigned long end)
+static void __init
+update_nodes_add(int node, unsigned long start, unsigned long end)
 {
         unsigned long s_pfn = start >> PAGE_SHIFT;
         unsigned long e_pfn = end >> PAGE_SHIFT;
-        int ret = 0, changed = 0;
+        int changed = 0;
         struct bootnode *nd = &nodes_add[node];
 
         /* I had some trouble with strange memory hotadd regions breaking
@@ -210,7 +197,7 @@ reserve_hotadd(int node, unsigned long start, unsigned long end)
            mistakes */
         if ((signed long)(end - start) < NODE_MIN_SIZE) {
                 printk(KERN_ERR "SRAT: Hotplug area too small\n");
-                return -1;
+                return;
         }
 
         /* This check might be a bit too strict, but I'm keeping it for now. */
@@ -218,12 +205,7 @@ reserve_hotadd(int node, unsigned long start, unsigned long end)
                 printk(KERN_ERR
                         "SRAT: Hotplug area %lu -> %lu has existing memory\n",
                         s_pfn, e_pfn);
-                return -1;
-        }
-
-        if (!hotadd_enough_memory(&nodes_add[node]))  {
-                printk(KERN_ERR "SRAT: Hotplug area too large\n");
-                return -1;
+                return;
         }
 
         /* Looks good */
@@ -245,11 +227,9 @@ reserve_hotadd(int node, unsigned long start, unsigned long end)
                         printk(KERN_ERR "SRAT: Hotplug zone not continuous. Partly ignored\n");
         }
 
-        ret = update_end_of_memory(nd->end);
-
         if (changed)
-                printk(KERN_INFO "SRAT: hot plug zone found %Lx - %Lx\n", nd->start, nd->end);
-        return ret;
+                printk(KERN_INFO "SRAT: hot plug zone found %Lx - %Lx\n",
+                                 nd->start, nd->end);
 }
 
 /* Callback for parsing of the Proximity Domain <-> Memory Area mappings */
@@ -310,13 +290,10 @@ acpi_numa_memory_affinity_init(struct acpi_srat_mem_affinity *ma)
                start, end);
         e820_register_active_regions(node, start >> PAGE_SHIFT,
                                      end >> PAGE_SHIFT);
-        push_node_boundaries(node, nd->start >> PAGE_SHIFT,
-                                                nd->end >> PAGE_SHIFT);
 
-        if ((ma->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE) &&
-            (reserve_hotadd(node, start, end) < 0)) {
-                /* Ignore hotadd region. Undo damage */
-                printk(KERN_NOTICE "SRAT: Hotplug region ignored\n");
+        if (ma->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE) {
+                update_nodes_add(node, start, end);
+                /* restore nodes[node] */
                 *nd = oldnode;
                 if ((nd->start | nd->end) == 0)
                         node_clear(node, nodes_parsed);
@@ -345,9 +322,9 @@ static int __init nodes_cover_memory(const struct bootnode *nodes)
                         pxmram = 0;
         }
 
-        e820ram = max_pfn - absent_pages_in_range(0, max_pfn);
-        /* We seem to lose 3 pages somewhere. Allow a bit of slack. */
-        if ((long)(e820ram - pxmram) >= 1*1024*1024) {
+        e820ram = max_pfn - (e820_hole_size(0, max_pfn<<PAGE_SHIFT)>>PAGE_SHIFT);
+        /* We seem to lose 3 pages somewhere. Allow 1M of slack. */
+        if ((long)(e820ram - pxmram) >= (1<<(20 - PAGE_SHIFT))) {
                 printk(KERN_ERR
         "SRAT: PXMs only cover %luMB of your %luMB e820 RAM. Not used.\n",
                         (pxmram << PAGE_SHIFT) >> 20,
@@ -357,17 +334,6 @@ static int __init nodes_cover_memory(const struct bootnode *nodes)
         return 1;
 }
 
-static void __init unparse_node(int node)
-{
-        int i;
-        node_clear(node, nodes_parsed);
-        node_clear(node, cpu_nodes_parsed);
-        for (i = 0; i < MAX_LOCAL_APIC; i++) {
-                if (apicid_to_node[i] == node)
-                        apicid_to_node[i] = NUMA_NO_NODE;
-        }
-}
-
 void __init acpi_numa_arch_fixup(void) {}
 
 /* Use the information discovered above to actually set up the nodes. */
@@ -379,18 +345,8 @@ int __init acpi_scan_nodes(unsigned long start, unsigned long end)
                 return -1;
 
         /* First clean up the node list */
-        for (i = 0; i < MAX_NUMNODES; i++) {
+        for (i = 0; i < MAX_NUMNODES; i++)
                 cutoff_node(i, start, end);
-                /*
-                 * don't confuse VM with a node that doesn't have the
-                 * minimum memory.
-                 */
-                if (nodes[i].end &&
-                        (nodes[i].end - nodes[i].start) < NODE_MIN_SIZE) {
-                        unparse_node(i);
-                        node_set_offline(i);
-                }
-        }
 
         if (!nodes_cover_memory(nodes)) {
                 bad_srat();
@@ -423,7 +379,7 @@ int __init acpi_scan_nodes(unsigned long start, unsigned long end)
 
                 if (node == NUMA_NO_NODE)
                         continue;
-                if (!node_isset(node, node_possible_map))
+                if (!node_online(node))
                         numa_clear_node(i);
         }
         numa_init_array();
@@ -510,26 +466,6 @@ static int null_slit_node_compare(int a, int b)
 }
 #endif /* CONFIG_NUMA_EMU */
 
-void __init srat_reserve_add_area(int nodeid)
-{
-        if (found_add_area && nodes_add[nodeid].end) {
-                u64 total_mb;
-
-                printk(KERN_INFO "SRAT: Reserving hot-add memory space "
-                                "for node %d at %Lx-%Lx\n",
-                        nodeid, nodes_add[nodeid].start, nodes_add[nodeid].end);
-                total_mb = (nodes_add[nodeid].end - nodes_add[nodeid].start)
-                                        >> PAGE_SHIFT;
-                total_mb *= sizeof(struct page);
-                total_mb >>= 20;
-                printk(KERN_INFO "SRAT: This will cost you %Lu MB of "
-                                "pre-allocated memory.\n", (unsigned long long)total_mb);
-                reserve_bootmem_node(NODE_DATA(nodeid), nodes_add[nodeid].start,
-                               nodes_add[nodeid].end - nodes_add[nodeid].start,
-                               BOOTMEM_DEFAULT);
-        }
-}
-
 int __node_distance(int a, int b)
 {
         int index;
diff --git a/arch/x86/oprofile/nmi_int.c b/arch/x86/oprofile/nmi_int.c
index 202864ad49a7..b07dd8d0b321 100644
--- a/arch/x86/oprofile/nmi_int.c
+++ b/arch/x86/oprofile/nmi_int.c
@@ -40,8 +40,9 @@ static int profile_exceptions_notify(struct notifier_block *self,
 
         switch (val) {
         case DIE_NMI:
-                if (model->check_ctrs(args->regs, &per_cpu(cpu_msrs, cpu)))
-                        ret = NOTIFY_STOP;
+        case DIE_NMI_IPI:
+                model->check_ctrs(args->regs, &per_cpu(cpu_msrs, cpu));
+                ret = NOTIFY_STOP;
                 break;
         default:
                 break;
@@ -134,7 +135,7 @@ static void nmi_cpu_setup(void *dummy)
 static struct notifier_block profile_exceptions_nb = {
         .notifier_call = profile_exceptions_notify,
         .next = NULL,
-        .priority = 0
+        .priority = 2
 };
 
 static int nmi_setup(void)
@@ -356,14 +357,11 @@ static void exit_sysfs(void)
 #define exit_sysfs() do { } while (0)
 #endif /* CONFIG_PM */
 
-static int p4force;
-module_param(p4force, int, 0);
-
 static int __init p4_init(char **cpu_type)
 {
         __u8 cpu_model = boot_cpu_data.x86_model;
 
-        if (!p4force && (cpu_model > 6 || cpu_model == 5))
+        if (cpu_model > 6 || cpu_model == 5)
                 return 0;
 
 #ifndef CONFIG_SMP
@@ -389,10 +387,25 @@ static int __init p4_init(char **cpu_type)
         return 0;
 }
 
+static int force_arch_perfmon;
+static int force_cpu_type(const char *str, struct kernel_param *kp)
+{
+        if (!strcmp(str, "archperfmon")) {
+                force_arch_perfmon = 1;
+                printk(KERN_INFO "oprofile: forcing architectural perfmon\n");
+        }
+
+        return 0;
+}
+module_param_call(cpu_type, force_cpu_type, NULL, NULL, 0);
+
 static int __init ppro_init(char **cpu_type)
 {
         __u8 cpu_model = boot_cpu_data.x86_model;
 
+        if (force_arch_perfmon && cpu_has_arch_perfmon)
+                return 0;
+
         switch (cpu_model) {
         case 0 ... 2:
                 *cpu_type = "i386/ppro";
@@ -414,6 +427,13 @@ static int __init ppro_init(char **cpu_type)
         case 15: case 23:
                 *cpu_type = "i386/core_2";
                 break;
+        case 26:
+                arch_perfmon_setup_counters();
+                *cpu_type = "i386/core_i7";
+                break;
+        case 28:
+                *cpu_type = "i386/atom";
+                break;
         default:
                 /* Unknown */
                 return 0;
diff --git a/arch/x86/oprofile/op_model_ppro.c b/arch/x86/oprofile/op_model_ppro.c
index 10131fbdaada..4da7230b3d17 100644
--- a/arch/x86/oprofile/op_model_ppro.c
+++ b/arch/x86/oprofile/op_model_ppro.c
@@ -18,7 +18,7 @@
 #include <asm/msr.h>
 #include <asm/apic.h>
 #include <asm/nmi.h>
-#include <asm/intel_arch_perfmon.h>
+#include <asm/perf_counter.h>
 
 #include "op_x86_model.h"
 #include "op_counter.h"
@@ -136,6 +136,13 @@ static int ppro_check_ctrs(struct pt_regs * const regs,
         u64 val;
         int i;
 
+        /*
+         * This can happen if perf counters are in use when
+         * we steal the die notifier NMI.
+         */
+        if (unlikely(!reset_value))
+                goto out;
+
         for (i = 0 ; i < num_counters; ++i) {
                 if (!reset_value[i])
                         continue;
@@ -146,6 +153,7 @@ static int ppro_check_ctrs(struct pt_regs * const regs,
                 }
         }
 
+out:
         /* Only P6 based Pentium M need to re-unmask the apic vector but it
          * doesn't hurt other P6 variant */
         apic_write(APIC_LVTPC, apic_read(APIC_LVTPC) & ~APIC_LVT_MASKED);
diff --git a/arch/x86/pci/mmconfig-shared.c b/arch/x86/pci/mmconfig-shared.c
index 5fa10bb9604f..8766b0e216c5 100644
--- a/arch/x86/pci/mmconfig-shared.c
+++ b/arch/x86/pci/mmconfig-shared.c
@@ -375,7 +375,7 @@ static acpi_status __init check_mcfg_resource(struct acpi_resource *res,
                 if (!fixmem32)
                         return AE_OK;
                 if ((mcfg_res->start >= fixmem32->address) &&
-                    (mcfg_res->end <= (fixmem32->address +
+                    (mcfg_res->end < (fixmem32->address +
                                       fixmem32->address_length))) {
                         mcfg_res->flags = 1;
                         return AE_CTRL_TERMINATE;
@@ -392,7 +392,7 @@ static acpi_status __init check_mcfg_resource(struct acpi_resource *res,
                 return AE_OK;
 
         if ((mcfg_res->start >= address.minimum) &&
-            (mcfg_res->end <= (address.minimum + address.address_length))) {
+            (mcfg_res->end < (address.minimum + address.address_length))) {
                 mcfg_res->flags = 1;
                 return AE_CTRL_TERMINATE;
         }
@@ -418,7 +418,7 @@ static int __init is_acpi_reserved(u64 start, u64 end, unsigned not_used)
         struct resource mcfg_res;
 
         mcfg_res.start = start;
-        mcfg_res.end = end;
+        mcfg_res.end = end - 1;
         mcfg_res.flags = 0;
 
         acpi_get_devices("PNP0C01", find_mboard_resource, &mcfg_res, NULL);
diff --git a/arch/x86/vdso/vdso32-setup.c b/arch/x86/vdso/vdso32-setup.c
index 1241f118ab56..58bc00f68b12 100644
--- a/arch/x86/vdso/vdso32-setup.c
+++ b/arch/x86/vdso/vdso32-setup.c
@@ -338,6 +338,8 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
                 }
         }
 
+        current->mm->context.vdso = (void *)addr;
+
         if (compat_uses_vma || !compat) {
                 /*
                  * MAYWRITE to allow gdb to COW and set breakpoints
@@ -358,11 +360,13 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
                         goto up_fail;
         }
 
-        current->mm->context.vdso = (void *)addr;
         current_thread_info()->sysenter_return =
                 VDSO32_SYMBOL(addr, SYSENTER_RETURN);
 
   up_fail:
+        if (ret)
+                current->mm->context.vdso = NULL;
+
         up_write(&mm->mmap_sem);
 
         return ret;
diff --git a/arch/x86/vdso/vma.c b/arch/x86/vdso/vma.c
index 7133cdf9098b..21e1aeb9f3ea 100644
--- a/arch/x86/vdso/vma.c
+++ b/arch/x86/vdso/vma.c
@@ -8,6 +8,7 @@
 #include <linux/sched.h>
 #include <linux/init.h>
 #include <linux/random.h>
+#include <linux/elf.h>
 #include <asm/vsyscall.h>
 #include <asm/vgtod.h>
 #include <asm/proto.h>
@@ -115,15 +116,18 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
                 goto up_fail;
         }
 
+        current->mm->context.vdso = (void *)addr;
+
         ret = install_special_mapping(mm, addr, vdso_size,
                                       VM_READ|VM_EXEC|
                                       VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC|
                                       VM_ALWAYSDUMP,
                                       vdso_pages);
-        if (ret)
+        if (ret) {
+                current->mm->context.vdso = NULL;
                 goto up_fail;
+        }
 
-        current->mm->context.vdso = (void *)addr;
 up_fail:
         up_write(&mm->mmap_sem);
         return ret;
diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
index f09e8c36ee80..0a1700a2be9c 100644
--- a/arch/x86/xen/enlighten.c
+++ b/arch/x86/xen/enlighten.c
@@ -20,6 +20,7 @@
 #include <linux/delay.h>
 #include <linux/start_kernel.h>
 #include <linux/sched.h>
+#include <linux/kprobes.h>
 #include <linux/bootmem.h>
 #include <linux/module.h>
 #include <linux/mm.h>
@@ -44,6 +45,7 @@
 #include <asm/processor.h>
 #include <asm/proto.h>
 #include <asm/msr-index.h>
+#include <asm/traps.h>
 #include <asm/setup.h>
 #include <asm/desc.h>
 #include <asm/pgtable.h>
@@ -240,10 +242,10 @@ static unsigned long xen_get_debugreg(int reg)
         return HYPERVISOR_get_debugreg(reg);
 }
 
-void xen_leave_lazy(void)
+static void xen_end_context_switch(struct task_struct *next)
 {
-        paravirt_leave_lazy(paravirt_get_lazy_mode());
         xen_mc_flush();
+        paravirt_end_context_switch(next);
 }
 
 static unsigned long xen_store_tr(void)
@@ -428,11 +430,44 @@ static void xen_write_ldt_entry(struct desc_struct *dt, int entrynum,
 static int cvt_gate_to_trap(int vector, const gate_desc *val,
                             struct trap_info *info)
 {
+        unsigned long addr;
+
         if (val->type != GATE_TRAP && val->type != GATE_INTERRUPT)
                 return 0;
 
         info->vector = vector;
-        info->address = gate_offset(*val);
+
+        addr = gate_offset(*val);
+#ifdef CONFIG_X86_64
+        /*
+         * Look for known traps using IST, and substitute them
+         * appropriately.  The debugger ones are the only ones we care
+         * about.  Xen will handle faults like double_fault and
+         * machine_check, so we should never see them.  Warn if
+         * there's an unexpected IST-using fault handler.
+         */
+        if (addr == (unsigned long)debug)
+                addr = (unsigned long)xen_debug;
+        else if (addr == (unsigned long)int3)
+                addr = (unsigned long)xen_int3;
+        else if (addr == (unsigned long)stack_segment)
+                addr = (unsigned long)xen_stack_segment;
+        else if (addr == (unsigned long)double_fault ||
+                 addr == (unsigned long)nmi) {
+                /* Don't need to handle these */
+                return 0;
+#ifdef CONFIG_X86_MCE
+        } else if (addr == (unsigned long)machine_check) {
+                return 0;
+#endif
+        } else {
+                /* Some other trap using IST? */
+                if (WARN_ON(val->ist != 0))
+                        return 0;
+        }
+#endif  /* CONFIG_X86_64 */
+        info->address = addr;
+
         info->cs = gate_segment(*val);
         info->flags = val->dpl;
         /* interrupt gates clear IF */
@@ -623,10 +658,26 @@ static void xen_clts(void)
         xen_mc_issue(PARAVIRT_LAZY_CPU);
 }
 
+static DEFINE_PER_CPU(unsigned long, xen_cr0_value);
+
+static unsigned long xen_read_cr0(void)
+{
+        unsigned long cr0 = percpu_read(xen_cr0_value);
+
+        if (unlikely(cr0 == 0)) {
+                cr0 = native_read_cr0();
+                percpu_write(xen_cr0_value, cr0);
+        }
+
+        return cr0;
+}
+
 static void xen_write_cr0(unsigned long cr0)
 {
         struct multicall_space mcs;
 
+        percpu_write(xen_cr0_value, cr0);
+
         /* Only pay attention to cr0.TS; everything else is
            ignored. */
         mcs = xen_mc_entry(0);
@@ -812,7 +863,7 @@ static const struct pv_cpu_ops xen_cpu_ops __initdata = {
 
         .clts = xen_clts,
 
-        .read_cr0 = native_read_cr0,
+        .read_cr0 = xen_read_cr0,
         .write_cr0 = xen_write_cr0,
 
         .read_cr4 = native_read_cr4,
@@ -860,10 +911,8 @@ static const struct pv_cpu_ops xen_cpu_ops __initdata = {
         /* Xen takes care of %gs when switching to usermode for us */
         .swapgs = paravirt_nop,
 
-        .lazy_mode = {
-                .enter = paravirt_enter_lazy_cpu,
-                .leave = xen_leave_lazy,
-        },
+        .start_context_switch = paravirt_start_context_switch,
+        .end_context_switch = xen_end_context_switch,
 };
 
 static const struct pv_apic_ops xen_apic_ops __initdata = {
diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
index fba55b1a4021..4ceb28581652 100644
--- a/arch/x86/xen/mmu.c
+++ b/arch/x86/xen/mmu.c
@@ -452,10 +452,6 @@ void set_pte_mfn(unsigned long vaddr, unsigned long mfn, pgprot_t flags)
 void xen_set_pte_at(struct mm_struct *mm, unsigned long addr,
                     pte_t *ptep, pte_t pteval)
 {
-        /* updates to init_mm may be done without lock */
-        if (mm == &init_mm)
-                preempt_disable();
-
         ADD_STATS(set_pte_at, 1);
 //      ADD_STATS(set_pte_at_pinned, xen_page_pinned(ptep));
         ADD_STATS(set_pte_at_current, mm == current->mm);
@@ -476,9 +472,7 @@ void xen_set_pte_at(struct mm_struct *mm, unsigned long addr,
         }
         xen_set_pte(ptep, pteval);
 
-out:
-        if (mm == &init_mm)
-                preempt_enable();
+out:    return;
 }
 
 pte_t xen_ptep_modify_prot_start(struct mm_struct *mm,
@@ -1152,10 +1146,8 @@ static void drop_other_mm_ref(void *info)
 
         /* If this cpu still has a stale cr3 reference, then make sure
            it has been flushed. */
-        if (percpu_read(xen_current_cr3) == __pa(mm->pgd)) {
+        if (percpu_read(xen_current_cr3) == __pa(mm->pgd))
                 load_cr3(swapper_pg_dir);
-                arch_flush_lazy_cpu_mode();
-        }
 }
 
 static void xen_drop_mm_ref(struct mm_struct *mm)
@@ -1168,7 +1160,6 @@ static void xen_drop_mm_ref(struct mm_struct *mm)
                         load_cr3(swapper_pg_dir);
                 else
                         leave_mm(smp_processor_id());
-                arch_flush_lazy_cpu_mode();
         }
 
         /* Get the "official" set of cpus referring to our pagetable. */
@@ -1876,6 +1867,14 @@ __init void xen_post_allocator_init(void)
         xen_mark_init_mm_pinned();
 }
 
+static void xen_leave_lazy_mmu(void)
+{
+        preempt_disable();
+        xen_mc_flush();
+        paravirt_leave_lazy_mmu();
+        preempt_enable();
+}
+
 const struct pv_mmu_ops xen_mmu_ops __initdata = {
         .pagetable_setup_start = xen_pagetable_setup_start,
         .pagetable_setup_done = xen_pagetable_setup_done,
@@ -1949,7 +1948,7 @@ const struct pv_mmu_ops xen_mmu_ops __initdata = {
 
         .lazy_mode = {
                 .enter = paravirt_enter_lazy_mmu,
-                .leave = xen_leave_lazy,
+                .leave = xen_leave_lazy_mmu,
         },
 
         .set_fixmap = xen_set_fixmap,
diff --git a/arch/x86/xen/setup.c b/arch/x86/xen/setup.c
index 15c6c68db6a2..ad0047f47cd4 100644
--- a/arch/x86/xen/setup.c
+++ b/arch/x86/xen/setup.c
@@ -61,9 +61,9 @@ char * __init xen_memory_setup(void)
          *  - xen_start_info
          * See comment above "struct start_info" in <xen/interface/xen.h>
          */
-        e820_add_region(__pa(xen_start_info->mfn_list),
-                        xen_start_info->pt_base - xen_start_info->mfn_list,
-                        E820_RESERVED);
+        reserve_early(__pa(xen_start_info->mfn_list),
+                      __pa(xen_start_info->pt_base),
+                        "XEN START INFO");
 
         sanitize_e820_map(e820.map, ARRAY_SIZE(e820.map), &e820.nr_map);
 
diff --git a/arch/x86/xen/xen-ops.h b/arch/x86/xen/xen-ops.h
index ca6596b05d53..22494fd4c9b5 100644
--- a/arch/x86/xen/xen-ops.h
+++ b/arch/x86/xen/xen-ops.h
@@ -30,7 +30,6 @@ pgd_t *xen_setup_kernel_pagetable(pgd_t *pgd, unsigned long max_pfn);
 void xen_ident_map_ISA(void);
 void xen_reserve_top(void);
 
-void xen_leave_lazy(void);
 void xen_post_allocator_init(void);
 
 char * __init xen_memory_setup(void);