191 files changed, 6030 insertions, 1424 deletions

diff --git a/arch/alpha/kernel/time.c b/arch/alpha/kernel/time.c
index 0f1d8493cfc..c1f3e7cb82a 100644
--- a/arch/alpha/kernel/time.c
+++ b/arch/alpha/kernel/time.c
@@ -506,7 +506,7 @@ set_rtc_mmss(unsigned long nowtime)
                 CMOS_WRITE(real_seconds,RTC_SECONDS);
                 CMOS_WRITE(real_minutes,RTC_MINUTES);
         } else {
-                printk(KERN_WARNING
+                printk_once(KERN_NOTICE
                        "set_rtc_mmss: can't update from %d to %d\n",
                        cmos_minutes, real_minutes);
                 retval = -1;
diff --git a/arch/arm/common/it8152.c b/arch/arm/common/it8152.c
index 42ff90b46df..665ebf7e62a 100644
--- a/arch/arm/common/it8152.c
+++ b/arch/arm/common/it8152.c
@@ -236,7 +236,7 @@ static struct resource it8152_mem = {
 
 /*
  * The following functions are needed for DMA bouncing.
- * ITE8152 chip can addrees up to 64MByte, so all the devices
+ * ITE8152 chip can address up to 64MByte, so all the devices
  * connected to ITE8152 (PCI and USB) should have limited DMA window
  */
 
diff --git a/arch/arm/common/vic.c b/arch/arm/common/vic.c
index ba65f6eedca..cb660bc54d7 100644
--- a/arch/arm/common/vic.c
+++ b/arch/arm/common/vic.c
@@ -70,7 +70,7 @@ static inline struct vic_device *to_vic(struct sys_device *sys)
  * vic_init2 - common initialisation code
  * @base: Base of the VIC.
  *
- * Common initialisation code for registeration
+ * Common initialisation code for registration
  * and resume.
 */
 static void vic_init2(void __iomem *base)
diff --git a/arch/arm/mach-at91/board-ecbat91.c b/arch/arm/mach-at91/board-ecbat91.c
index 7b58c948a95..de2fd04e7c8 100644
--- a/arch/arm/mach-at91/board-ecbat91.c
+++ b/arch/arm/mach-at91/board-ecbat91.c
@@ -128,17 +128,17 @@ static struct spi_board_info __initdata ecb_at91spi_devices[] = {
                 .platform_data  = &my_flash0_platform,
 #endif
         },
-        {       /* User accessable spi - cs1 (250KHz) */
+        {       /* User accessible spi - cs1 (250KHz) */
                 .modalias       = "spi-cs1",
                 .chip_select    = 1,
                 .max_speed_hz   = 250 * 1000,
         },
-        {       /* User accessable spi - cs2 (1MHz) */
+        {       /* User accessible spi - cs2 (1MHz) */
                 .modalias       = "spi-cs2",
                 .chip_select    = 2,
                 .max_speed_hz   = 1 * 1000 * 1000,
         },
-        {       /* User accessable spi - cs3 (10MHz) */
+        {       /* User accessible spi - cs3 (10MHz) */
                 .modalias       = "spi-cs3",
                 .chip_select    = 3,
                 .max_speed_hz   = 10 * 1000 * 1000,
diff --git a/arch/arm/mach-at91/pm.c b/arch/arm/mach-at91/pm.c
index dafbacc25eb..ea53f4d9b28 100644
--- a/arch/arm/mach-at91/pm.c
+++ b/arch/arm/mach-at91/pm.c
@@ -301,7 +301,7 @@ static void at91_pm_end(void)
 }
 
 
-static struct platform_suspend_ops at91_pm_ops ={
+static const struct platform_suspend_ops at91_pm_ops = {
         .valid  = at91_pm_valid_state,
         .begin  = at91_pm_begin,
         .enter  = at91_pm_enter,
diff --git a/arch/arm/mach-bcmring/csp/chipc/chipcHw.c b/arch/arm/mach-bcmring/csp/chipc/chipcHw.c
index b3a61d860c6..96273ff3495 100644
--- a/arch/arm/mach-bcmring/csp/chipc/chipcHw.c
+++ b/arch/arm/mach-bcmring/csp/chipc/chipcHw.c
@@ -757,7 +757,7 @@ static int chipcHw_divide(int num, int denom)
                 t = t << 1;
         }
 
-        /* Intialize the result */
+        /* Initialize the result */
         r = 0;
 
         do {
diff --git a/arch/arm/mach-bcmring/csp/dmac/dmacHw.c b/arch/arm/mach-bcmring/csp/dmac/dmacHw.c
index 7b9bac2d79a..6b9be2e98e5 100644
--- a/arch/arm/mach-bcmring/csp/dmac/dmacHw.c
+++ b/arch/arm/mach-bcmring/csp/dmac/dmacHw.c
@@ -893,7 +893,7 @@ int dmacHw_setDataDescriptor(dmacHw_CONFIG_t *pConfig,	/*   [ IN ] Configuration
 */
 /****************************************************************************/
 uint32_t dmacHw_getDmaControllerAttribute(dmacHw_HANDLE_t handle,       /*  [ IN ]  DMA Channel handle */
-                                          dmacHw_CONTROLLER_ATTRIB_e attr       /*  [ IN ]  DMA Controler attribute of type  dmacHw_CONTROLLER_ATTRIB_e */
+                                          dmacHw_CONTROLLER_ATTRIB_e attr       /*  [ IN ]  DMA Controller attribute of type  dmacHw_CONTROLLER_ATTRIB_e */
     ) {
         dmacHw_CBLK_t *pCblk = dmacHw_HANDLE_TO_CBLK(handle);
 
diff --git a/arch/arm/mach-bcmring/csp/dmac/dmacHw_extra.c b/arch/arm/mach-bcmring/csp/dmac/dmacHw_extra.c
index ff7b436d093..77f84b40dda 100644
--- a/arch/arm/mach-bcmring/csp/dmac/dmacHw_extra.c
+++ b/arch/arm/mach-bcmring/csp/dmac/dmacHw_extra.c
@@ -316,7 +316,7 @@ static void DisplayDescRing(void *pDescriptor,	/*   [ IN ] Descriptor buffer */
 /**
 *  @brief   Check if DMA channel is the flow controller
 *
-*  @return  1 : If DMA is a flow controler
+*  @return  1 : If DMA is a flow controller
 *           0 : Peripheral is the flow controller
 *
 *  @note
diff --git a/arch/arm/mach-bcmring/csp/tmr/tmrHw.c b/arch/arm/mach-bcmring/csp/tmr/tmrHw.c
index 5c1c9a0e5ed..16225e43f3c 100644
--- a/arch/arm/mach-bcmring/csp/tmr/tmrHw.c
+++ b/arch/arm/mach-bcmring/csp/tmr/tmrHw.c
@@ -558,7 +558,7 @@ static int tmrHw_divide(int num, int denom)
                 t = t << 1;
         }
 
-        /* Intialize the result */
+        /* Initialize the result */
         r = 0;
 
         do {
diff --git a/arch/arm/mach-bcmring/dma.c b/arch/arm/mach-bcmring/dma.c
index 77eb35c89cd..8d1baf3f468 100644
--- a/arch/arm/mach-bcmring/dma.c
+++ b/arch/arm/mach-bcmring/dma.c
@@ -671,7 +671,7 @@ static int ConfigChannel(DMA_Handle_t handle)
 
 /****************************************************************************/
 /**
-*   Intializes all of the data structures associated with the DMA.
+*   Initializes all of the data structures associated with the DMA.
 *   @return
 *       >= 0    - Initialization was successfull.
 *
diff --git a/arch/arm/mach-bcmring/include/csp/dmacHw.h b/arch/arm/mach-bcmring/include/csp/dmacHw.h
index 5d510130a25..6c8da2b9fc1 100644
--- a/arch/arm/mach-bcmring/include/csp/dmacHw.h
+++ b/arch/arm/mach-bcmring/include/csp/dmacHw.h
@@ -590,7 +590,7 @@ void dmacHw_printDebugInfo(dmacHw_HANDLE_t handle,	/*  [ IN ] DMA Channel handle
 */
 /****************************************************************************/
 uint32_t dmacHw_getDmaControllerAttribute(dmacHw_HANDLE_t handle,       /*  [ IN ]  DMA Channel handle  */
-                                          dmacHw_CONTROLLER_ATTRIB_e attr       /*  [ IN ]  DMA Controler attribute of type  dmacHw_CONTROLLER_ATTRIB_e */
+                                          dmacHw_CONTROLLER_ATTRIB_e attr       /*  [ IN ]  DMA Controller attribute of type  dmacHw_CONTROLLER_ATTRIB_e */
     );
 
 #endif /* _DMACHW_H */
diff --git a/arch/arm/mach-bcmring/include/csp/tmrHw.h b/arch/arm/mach-bcmring/include/csp/tmrHw.h
index f1236d00cb9..2cbb530db8e 100644
--- a/arch/arm/mach-bcmring/include/csp/tmrHw.h
+++ b/arch/arm/mach-bcmring/include/csp/tmrHw.h
@@ -76,7 +76,7 @@ tmrHw_RATE_t tmrHw_setPeriodicTimerRate(tmrHw_ID_t timerId,	/*  [ IN ] Timer Id
 *           certain time interval
 *
 *  This function initializes a periodic timer to generate timer interrupt
-*  after every time interval in milisecond
+*  after every time interval in millisecond
 *
 *  @return   On success: Effective interval set in mili-second
 *            On failure: 0
@@ -93,7 +93,7 @@ tmrHw_INTERVAL_t tmrHw_setPeriodicTimerInterval(tmrHw_ID_t timerId,	/*  [ IN ] T
 *           after certain time interval
 *
 *  This function initializes a periodic timer to generate a single ticks after
-*  certain time interval in milisecond
+*  certain time interval in millisecond
 *
 *  @return   On success: Effective interval set in mili-second
 *            On failure: 0
diff --git a/arch/arm/mach-bcmring/include/mach/csp/dmacHw_priv.h b/arch/arm/mach-bcmring/include/mach/csp/dmacHw_priv.h
index cbf334d1c76..d67e2f8c22d 100644
--- a/arch/arm/mach-bcmring/include/mach/csp/dmacHw_priv.h
+++ b/arch/arm/mach-bcmring/include/mach/csp/dmacHw_priv.h
@@ -28,7 +28,7 @@
 
 /* Data type for DMA Link List Item */
 typedef struct {
-        uint32_t sar;           /* Source Adress Register.
+        uint32_t sar;           /* Source Address Register.
                                    Address must be aligned to CTLx.SRC_TR_WIDTH.             */
         uint32_t dar;           /* Destination Address Register.
                                    Address must be aligned to CTLx.DST_TR_WIDTH.             */
diff --git a/arch/arm/mach-bcmring/include/mach/csp/dmacHw_reg.h b/arch/arm/mach-bcmring/include/mach/csp/dmacHw_reg.h
index 891cea87e33..f1ecf96f2da 100644
--- a/arch/arm/mach-bcmring/include/mach/csp/dmacHw_reg.h
+++ b/arch/arm/mach-bcmring/include/mach/csp/dmacHw_reg.h
@@ -35,7 +35,7 @@ typedef struct {
 
 /* Data type representing DMA channel registers */
 typedef struct {
-        dmacHw_REG64_t ChannelSar;      /*  Source Adress Register. 64 bits (upper 32 bits are reserved)
+        dmacHw_REG64_t ChannelSar;      /*  Source Address Register. 64 bits (upper 32 bits are reserved)
                                            Address must be aligned to CTLx.SRC_TR_WIDTH.
                                          */
         dmacHw_REG64_t ChannelDar;      /*  Destination Address Register.64 bits (upper 32 bits are reserved)
diff --git a/arch/arm/mach-davinci/pm.c b/arch/arm/mach-davinci/pm.c
index fab953b43de..1bd73a04be2 100644
--- a/arch/arm/mach-davinci/pm.c
+++ b/arch/arm/mach-davinci/pm.c
@@ -110,7 +110,7 @@ static int davinci_pm_enter(suspend_state_t state)
         return ret;
 }
 
-static struct platform_suspend_ops davinci_pm_ops = {
+static const struct platform_suspend_ops davinci_pm_ops = {
         .enter          = davinci_pm_enter,
         .valid          = suspend_valid_only_mem,
 };
diff --git a/arch/arm/mach-gemini/include/mach/hardware.h b/arch/arm/mach-gemini/include/mach/hardware.h
index 213a4fcfeb1..8c950e1d06b 100644
--- a/arch/arm/mach-gemini/include/mach/hardware.h
+++ b/arch/arm/mach-gemini/include/mach/hardware.h
@@ -33,7 +33,7 @@
 #define GEMINI_LPC_HOST_BASE    0x47000000
 #define GEMINI_LPC_IO_BASE      0x47800000
 #define GEMINI_INTERRUPT_BASE   0x48000000
-/* TODO: Different interrupt controlers when SMP
+/* TODO: Different interrupt controllers when SMP
  * #define GEMINI_INTERRUPT0_BASE       0x48000000
  * #define GEMINI_INTERRUPT1_BASE       0x49000000
  */
diff --git a/arch/arm/mach-imx/mach-pcm038.c b/arch/arm/mach-imx/mach-pcm038.c
index f667a262dfc..505614803bc 100644
--- a/arch/arm/mach-imx/mach-pcm038.c
+++ b/arch/arm/mach-imx/mach-pcm038.c
@@ -254,10 +254,10 @@ static struct regulator_init_data cam_data = {
 
 static struct mc13783_regulator_init_data pcm038_regulators[] = {
         {
-                .id = MC13783_REGU_VCAM,
+                .id = MC13783_REG_VCAM,
                 .init_data = &cam_data,
         }, {
-                .id = MC13783_REGU_VMMC1,
+                .id = MC13783_REG_VMMC1,
                 .init_data = &sdhc1_data,
         },
 };
diff --git a/arch/arm/mach-imx/pm-imx27.c b/arch/arm/mach-imx/pm-imx27.c
index 6bf81ceea13..acf17691d2c 100644
--- a/arch/arm/mach-imx/pm-imx27.c
+++ b/arch/arm/mach-imx/pm-imx27.c
@@ -32,7 +32,7 @@ static int mx27_suspend_enter(suspend_state_t state)
         return 0;
 }
 
-static struct platform_suspend_ops mx27_suspend_ops = {
+static const struct platform_suspend_ops mx27_suspend_ops = {
         .enter = mx27_suspend_enter,
         .valid = suspend_valid_only_mem,
 };
diff --git a/arch/arm/mach-kirkwood/openrd-setup.c b/arch/arm/mach-kirkwood/openrd-setup.c
index c9d77fad10a..cfcca4174e2 100644
--- a/arch/arm/mach-kirkwood/openrd-setup.c
+++ b/arch/arm/mach-kirkwood/openrd-setup.c
@@ -171,7 +171,7 @@ static void __init openrd_init(void)
 
         kirkwood_i2c_init();
 
-        if (machine_is_openrd_client()) {
+        if (machine_is_openrd_client() || machine_is_openrd_ultimate()) {
                 i2c_register_board_info(0, i2c_board_info,
                         ARRAY_SIZE(i2c_board_info));
                 kirkwood_audio_init();
diff --git a/arch/arm/mach-lpc32xx/pm.c b/arch/arm/mach-lpc32xx/pm.c
index a6e2aed9a49..e76d41bb705 100644
--- a/arch/arm/mach-lpc32xx/pm.c
+++ b/arch/arm/mach-lpc32xx/pm.c
@@ -123,7 +123,7 @@ static int lpc32xx_pm_enter(suspend_state_t state)
         return 0;
 }
 
-static struct platform_suspend_ops lpc32xx_pm_ops = {
+static const struct platform_suspend_ops lpc32xx_pm_ops = {
         .valid  = suspend_valid_only_mem,
         .enter  = lpc32xx_pm_enter,
 };
diff --git a/arch/arm/mach-msm/include/mach/entry-macro-qgic.S b/arch/arm/mach-msm/include/mach/entry-macro-qgic.S
index 4dc99aa65d0..12467157afb 100644
--- a/arch/arm/mach-msm/include/mach/entry-macro-qgic.S
+++ b/arch/arm/mach-msm/include/mach/entry-macro-qgic.S
@@ -26,7 +26,7 @@
          * The interrupt numbering scheme is defined in the
          * interrupt controller spec.  To wit:
          *
-         * Migrated the code from ARM MP port to be more consistant
+         * Migrated the code from ARM MP port to be more consistent
          * with interrupt processing , the following still holds true
          * however, all interrupts are treated the same regardless of
          * if they are local IPI or PPI
diff --git a/arch/arm/mach-msm/io.c b/arch/arm/mach-msm/io.c
index 800f327a7ec..1260007a9dd 100644
--- a/arch/arm/mach-msm/io.c
+++ b/arch/arm/mach-msm/io.c
@@ -154,7 +154,7 @@ __msm_ioremap(unsigned long phys_addr, size_t size, unsigned int mtype)
 {
         if (mtype == MT_DEVICE) {
                 /* The peripherals in the 88000000 - D0000000 range
-                 * are only accessable by type MT_DEVICE_NONSHARED.
+                 * are only accessible by type MT_DEVICE_NONSHARED.
                  * Adjust mtype as necessary to make this "just work."
                  */
                 if ((phys_addr >= 0x88000000) && (phys_addr < 0xD0000000))
diff --git a/arch/arm/mach-mx3/mach-mx31_3ds.c b/arch/arm/mach-mx3/mach-mx31_3ds.c
index 4e516b49a90..899a969e92f 100644
--- a/arch/arm/mach-mx3/mach-mx31_3ds.c
+++ b/arch/arm/mach-mx3/mach-mx31_3ds.c
@@ -140,10 +140,10 @@ static struct regulator_init_data gpo_init = {
 
 static struct mc13783_regulator_init_data mx31_3ds_regulators[] = {
         {
-                .id = MC13783_REGU_PWGT1SPI, /* Power Gate for ARM core. */
+                .id = MC13783_REG_PWGT1SPI, /* Power Gate for ARM core. */
                 .init_data = &pwgtx_init,
         }, {
-                .id = MC13783_REGU_PWGT2SPI, /* Power Gate for L2 Cache. */
+                .id = MC13783_REG_PWGT2SPI, /* Power Gate for L2 Cache. */
                 .init_data = &pwgtx_init,
         }, {
 
diff --git a/arch/arm/mach-mx3/mach-mx31moboard.c b/arch/arm/mach-mx3/mach-mx31moboard.c
index 203d21a510a..1aa8d65fccb 100644
--- a/arch/arm/mach-mx3/mach-mx31moboard.c
+++ b/arch/arm/mach-mx3/mach-mx31moboard.c
@@ -216,11 +216,11 @@ static struct regulator_init_data cam_vreg_data = {
 
 static struct mc13783_regulator_init_data moboard_regulators[] = {
         {
-                .id = MC13783_REGU_VMMC1,
+                .id = MC13783_REG_VMMC1,
                 .init_data = &sdhc_vreg_data,
         },
         {
-                .id = MC13783_REGU_VCAM,
+                .id = MC13783_REG_VCAM,
                 .init_data = &cam_vreg_data,
         },
 };
diff --git a/arch/arm/mach-omap1/pm.c b/arch/arm/mach-omap1/pm.c
index 0cca23a8517..98ba9784aa1 100644
--- a/arch/arm/mach-omap1/pm.c
+++ b/arch/arm/mach-omap1/pm.c
@@ -647,7 +647,7 @@ static struct irqaction omap_wakeup_irq = {
 
 
 
-static struct platform_suspend_ops omap_pm_ops ={
+static const struct platform_suspend_ops omap_pm_ops = {
         .prepare        = omap_pm_prepare,
         .enter          = omap_pm_enter,
         .finish         = omap_pm_finish,
diff --git a/arch/arm/mach-omap2/Makefile b/arch/arm/mach-omap2/Makefile
index cd7332f50b2..1c0c2b02d87 100644
--- a/arch/arm/mach-omap2/Makefile
+++ b/arch/arm/mach-omap2/Makefile
@@ -187,16 +187,19 @@ obj-$(CONFIG_MACH_NOKIA_RX51)		+= board-rx51.o \
                                            hsmmc.o
 obj-$(CONFIG_MACH_OMAP_ZOOM2)           += board-zoom.o \
                                            board-zoom-peripherals.o \
+                                           board-zoom-display.o \
                                            board-flash.o \
                                            hsmmc.o \
                                            board-zoom-debugboard.o
 obj-$(CONFIG_MACH_OMAP_ZOOM3)           += board-zoom.o \
                                            board-zoom-peripherals.o \
+                                           board-zoom-display.o \
                                            board-flash.o \
                                            hsmmc.o \
                                            board-zoom-debugboard.o
 obj-$(CONFIG_MACH_OMAP_3630SDP)         += board-3630sdp.o \
                                            board-zoom-peripherals.o \
+                                           board-zoom-display.o \
                                            board-flash.o \
                                            hsmmc.o
 obj-$(CONFIG_MACH_CM_T35)               += board-cm-t35.o \
diff --git a/arch/arm/mach-omap2/board-3430sdp.c b/arch/arm/mach-omap2/board-3430sdp.c
index 3b39ef1a680..d4e41ef86aa 100644
--- a/arch/arm/mach-omap2/board-3430sdp.c
+++ b/arch/arm/mach-omap2/board-3430sdp.c
@@ -38,6 +38,7 @@
 #include <plat/dma.h>
 #include <plat/gpmc.h>
 #include <plat/display.h>
+#include <plat/panel-generic-dpi.h>
 
 #include <plat/gpmc-smc91x.h>
 
@@ -270,13 +271,18 @@ static struct omap_dss_device sdp3430_lcd_device = {
         .platform_disable       = sdp3430_panel_disable_lcd,
 };
 
+static struct panel_generic_dpi_data dvi_panel = {
+        .name                   = "generic",
+        .platform_enable        = sdp3430_panel_enable_dvi,
+        .platform_disable       = sdp3430_panel_disable_dvi,
+};
+
 static struct omap_dss_device sdp3430_dvi_device = {
         .name                   = "dvi",
-        .driver_name            = "generic_panel",
         .type                   = OMAP_DISPLAY_TYPE_DPI,
+        .driver_name            = "generic_dpi_panel",
+        .data                   = &dvi_panel,
         .phy.dpi.data_lines     = 24,
-        .platform_enable        = sdp3430_panel_enable_dvi,
-        .platform_disable       = sdp3430_panel_disable_dvi,
 };
 
 static struct omap_dss_device sdp3430_tv_device = {
diff --git a/arch/arm/mach-omap2/board-3630sdp.c b/arch/arm/mach-omap2/board-3630sdp.c
index 5d41dbe059a..62645640f5e 100644
--- a/arch/arm/mach-omap2/board-3630sdp.c
+++ b/arch/arm/mach-omap2/board-3630sdp.c
@@ -207,6 +207,7 @@ static void __init omap_sdp_init(void)
 {
         omap3_mux_init(board_mux, OMAP_PACKAGE_CBP);
         zoom_peripherals_init();
+        zoom_display_init();
         board_smc91x_init();
         board_flash_init(sdp_flash_partitions, chip_sel_sdp);
         enable_board_wakeup_source();
diff --git a/arch/arm/mach-omap2/board-am3517evm.c b/arch/arm/mach-omap2/board-am3517evm.c
index bc156264802..10d60b7743c 100644
--- a/arch/arm/mach-omap2/board-am3517evm.c
+++ b/arch/arm/mach-omap2/board-am3517evm.c
@@ -35,6 +35,7 @@
 #include <plat/common.h>
 #include <plat/usb.h>
 #include <plat/display.h>
+#include <plat/panel-generic-dpi.h>
 
 #include "mux.h"
 #include "control.h"
@@ -303,13 +304,18 @@ static void am3517_evm_panel_disable_lcd(struct omap_dss_device *dssdev)
         lcd_enabled = 0;
 }
 
+static struct panel_generic_dpi_data lcd_panel = {
+        .name                   = "sharp_lq",
+        .platform_enable        = am3517_evm_panel_enable_lcd,
+        .platform_disable       = am3517_evm_panel_disable_lcd,
+};
+
 static struct omap_dss_device am3517_evm_lcd_device = {
         .type                   = OMAP_DISPLAY_TYPE_DPI,
         .name                   = "lcd",
-        .driver_name            = "sharp_lq_panel",
+        .driver_name            = "generic_dpi_panel",
+        .data                   = &lcd_panel,
         .phy.dpi.data_lines     = 16,
-        .platform_enable        = am3517_evm_panel_enable_lcd,
-        .platform_disable       = am3517_evm_panel_disable_lcd,
 };
 
 static int am3517_evm_panel_enable_tv(struct omap_dss_device *dssdev)
@@ -346,13 +352,18 @@ static void am3517_evm_panel_disable_dvi(struct omap_dss_device *dssdev)
         dvi_enabled = 0;
 }
 
+static struct panel_generic_dpi_data dvi_panel = {
+        .name                   = "generic",
+        .platform_enable        = am3517_evm_panel_enable_dvi,
+        .platform_disable       = am3517_evm_panel_disable_dvi,
+};
+
 static struct omap_dss_device am3517_evm_dvi_device = {
         .type                   = OMAP_DISPLAY_TYPE_DPI,
         .name                   = "dvi",
-        .driver_name            = "generic_panel",
+        .driver_name            = "generic_dpi_panel",
+        .data                   = &dvi_panel,
         .phy.dpi.data_lines     = 24,
-        .platform_enable        = am3517_evm_panel_enable_dvi,
-        .platform_disable       = am3517_evm_panel_disable_dvi,
 };
 
 static struct omap_dss_device *am3517_evm_dss_devices[] = {
diff --git a/arch/arm/mach-omap2/board-cm-t35.c b/arch/arm/mach-omap2/board-cm-t35.c
index 486a3de5f40..dac14161066 100644
--- a/arch/arm/mach-omap2/board-cm-t35.c
+++ b/arch/arm/mach-omap2/board-cm-t35.c
@@ -46,6 +46,7 @@
 #include <plat/gpmc.h>
 #include <plat/usb.h>
 #include <plat/display.h>
+#include <plat/panel-generic-dpi.h>
 #include <plat/mcspi.h>
 
 #include <mach/hardware.h>
@@ -351,22 +352,32 @@ static void cm_t35_panel_disable_tv(struct omap_dss_device *dssdev)
 {
 }
 
+static struct panel_generic_dpi_data lcd_panel = {
+        .name                   = "toppoly_tdo35s",
+        .platform_enable        = cm_t35_panel_enable_lcd,
+        .platform_disable       = cm_t35_panel_disable_lcd,
+};
+
 static struct omap_dss_device cm_t35_lcd_device = {
         .name                   = "lcd",
-        .driver_name            = "toppoly_tdo35s_panel",
         .type                   = OMAP_DISPLAY_TYPE_DPI,
+        .driver_name            = "generic_dpi_panel",
+        .data                   = &lcd_panel,
         .phy.dpi.data_lines     = 18,
-        .platform_enable        = cm_t35_panel_enable_lcd,
-        .platform_disable       = cm_t35_panel_disable_lcd,
+};
+
+static struct panel_generic_dpi_data dvi_panel = {
+        .name                   = "generic",
+        .platform_enable        = cm_t35_panel_enable_dvi,
+        .platform_disable       = cm_t35_panel_disable_dvi,
 };
 
 static struct omap_dss_device cm_t35_dvi_device = {
         .name                   = "dvi",
-        .driver_name            = "generic_panel",
         .type                   = OMAP_DISPLAY_TYPE_DPI,
+        .driver_name            = "generic_dpi_panel",
+        .data                   = &dvi_panel,
         .phy.dpi.data_lines     = 24,
-        .platform_enable        = cm_t35_panel_enable_dvi,
-        .platform_disable       = cm_t35_panel_disable_dvi,
 };
 
 static struct omap_dss_device cm_t35_tv_device = {
diff --git a/arch/arm/mach-omap2/board-devkit8000.c b/arch/arm/mach-omap2/board-devkit8000.c
index 451e7ff08b1..00bb1fc5e01 100644
--- a/arch/arm/mach-omap2/board-devkit8000.c
+++ b/arch/arm/mach-omap2/board-devkit8000.c
@@ -46,6 +46,7 @@
 #include <plat/nand.h>
 #include <plat/usb.h>
 #include <plat/display.h>
+#include <plat/panel-generic-dpi.h>
 
 #include <plat/mcspi.h>
 #include <linux/input/matrix_keypad.h>
@@ -149,23 +150,32 @@ static struct regulator_consumer_supply devkit8000_vmmc1_supply =
 static struct regulator_consumer_supply devkit8000_vio_supply =
         REGULATOR_SUPPLY("vcc", "spi2.0");
 
+static struct panel_generic_dpi_data lcd_panel = {
+        .name                   = "generic",
+        .platform_enable        = devkit8000_panel_enable_lcd,
+        .platform_disable       = devkit8000_panel_disable_lcd,
+};
+
 static struct omap_dss_device devkit8000_lcd_device = {
         .name                   = "lcd",
-        .driver_name            = "generic_panel",
         .type                   = OMAP_DISPLAY_TYPE_DPI,
+        .driver_name            = "generic_dpi_panel",
+        .data                   = &lcd_panel,
         .phy.dpi.data_lines     = 24,
-        .reset_gpio             = -EINVAL, /* will be replaced */
-        .platform_enable        = devkit8000_panel_enable_lcd,
-        .platform_disable       = devkit8000_panel_disable_lcd,
 };
+
+static struct panel_generic_dpi_data dvi_panel = {
+        .name                   = "generic",
+        .platform_enable        = devkit8000_panel_enable_dvi,
+        .platform_disable       = devkit8000_panel_disable_dvi,
+};
+
 static struct omap_dss_device devkit8000_dvi_device = {
         .name                   = "dvi",
-        .driver_name            = "generic_panel",
         .type                   = OMAP_DISPLAY_TYPE_DPI,
+        .driver_name            = "generic_dpi_panel",
+        .data                   = &dvi_panel,
         .phy.dpi.data_lines     = 24,
-        .reset_gpio             = -EINVAL, /* will be replaced */
-        .platform_enable        = devkit8000_panel_enable_dvi,
-        .platform_disable       = devkit8000_panel_disable_dvi,
 };
 
 static struct omap_dss_device devkit8000_tv_device = {
diff --git a/arch/arm/mach-omap2/board-igep0020.c b/arch/arm/mach-omap2/board-igep0020.c
index 0afa3011db0..ebaa230e67e 100644
--- a/arch/arm/mach-omap2/board-igep0020.c
+++ b/arch/arm/mach-omap2/board-igep0020.c
@@ -31,6 +31,7 @@
 #include <plat/gpmc.h>
 #include <plat/usb.h>
 #include <plat/display.h>
+#include <plat/panel-generic-dpi.h>
 #include <plat/onenand.h>
 
 #include "mux.h"
@@ -459,13 +460,18 @@ static void igep2_disable_dvi(struct omap_dss_device *dssdev)
         gpio_direction_output(IGEP2_GPIO_DVI_PUP, 0);
 }
 
+static struct panel_generic_dpi_data dvi_panel = {
+        .name                   = "generic",
+        .platform_enable        = igep2_enable_dvi,
+        .platform_disable       = igep2_disable_dvi,
+};
+
 static struct omap_dss_device igep2_dvi_device = {
         .type                   = OMAP_DISPLAY_TYPE_DPI,
         .name                   = "dvi",
-        .driver_name            = "generic_panel",
+        .driver_name            = "generic_dpi_panel",
+        .data                   = &dvi_panel,
         .phy.dpi.data_lines     = 24,
-        .platform_enable        = igep2_enable_dvi,
-        .platform_disable       = igep2_disable_dvi,
 };
 
 static struct omap_dss_device *igep2_dss_devices[] = {
diff --git a/arch/arm/mach-omap2/board-omap3beagle.c b/arch/arm/mach-omap2/board-omap3beagle.c
index 6c127605942..a4fe8e1ee1b 100644
--- a/arch/arm/mach-omap2/board-omap3beagle.c
+++ b/arch/arm/mach-omap2/board-omap3beagle.c
@@ -41,6 +41,7 @@
 #include <plat/board.h>
 #include <plat/common.h>
 #include <plat/display.h>
+#include <plat/panel-generic-dpi.h>
 #include <plat/gpmc.h>
 #include <plat/nand.h>
 #include <plat/usb.h>
@@ -194,14 +195,19 @@ static void beagle_disable_dvi(struct omap_dss_device *dssdev)
                 gpio_set_value(dssdev->reset_gpio, 0);
 }
 
+static struct panel_generic_dpi_data dvi_panel = {
+        .name = "generic",
+        .platform_enable = beagle_enable_dvi,
+        .platform_disable = beagle_disable_dvi,
+};
+
 static struct omap_dss_device beagle_dvi_device = {
         .type = OMAP_DISPLAY_TYPE_DPI,
         .name = "dvi",
-        .driver_name = "generic_panel",
+        .driver_name = "generic_dpi_panel",
+        .data = &dvi_panel,
         .phy.dpi.data_lines = 24,
         .reset_gpio = 170,
-        .platform_enable = beagle_enable_dvi,
-        .platform_disable = beagle_disable_dvi,
 };
 
 static struct omap_dss_device beagle_tv_device = {
diff --git a/arch/arm/mach-omap2/board-omap3evm.c b/arch/arm/mach-omap2/board-omap3evm.c
index 3de8d9b8ec7..323c3809ce3 100644
--- a/arch/arm/mach-omap2/board-omap3evm.c
+++ b/arch/arm/mach-omap2/board-omap3evm.c
@@ -43,6 +43,7 @@
 #include <plat/common.h>
 #include <plat/mcspi.h>
 #include <plat/display.h>
+#include <plat/panel-generic-dpi.h>
 
 #include "mux.h"
 #include "sdram-micron-mt46h32m32lf-6.h"
@@ -301,13 +302,18 @@ static void omap3_evm_disable_dvi(struct omap_dss_device *dssdev)
         dvi_enabled = 0;
 }
 
+static struct panel_generic_dpi_data dvi_panel = {
+        .name                   = "generic",
+        .platform_enable        = omap3_evm_enable_dvi,
+        .platform_disable       = omap3_evm_disable_dvi,
+};
+
 static struct omap_dss_device omap3_evm_dvi_device = {
         .name                   = "dvi",
-        .driver_name            = "generic_panel",
         .type                   = OMAP_DISPLAY_TYPE_DPI,
+        .driver_name            = "generic_dpi_panel",
+        .data                   = &dvi_panel,
         .phy.dpi.data_lines     = 24,
-        .platform_enable        = omap3_evm_enable_dvi,
-        .platform_disable       = omap3_evm_disable_dvi,
 };
 
 static struct omap_dss_device *omap3_evm_dss_devices[] = {
diff --git a/arch/arm/mach-omap2/board-omap3stalker.c b/arch/arm/mach-omap2/board-omap3stalker.c
index 9df9d936760..2a2dad447e8 100644
--- a/arch/arm/mach-omap2/board-omap3stalker.c
+++ b/arch/arm/mach-omap2/board-omap3stalker.c
@@ -40,6 +40,7 @@
 #include <plat/nand.h>
 #include <plat/usb.h>
 #include <plat/display.h>
+#include <plat/panel-generic-dpi.h>
 
 #include <plat/mcspi.h>
 #include <linux/input/matrix_keypad.h>
@@ -160,13 +161,18 @@ static void omap3_stalker_disable_lcd(struct omap_dss_device *dssdev)
         lcd_enabled = 0;
 }
 
+static struct panel_generic_dpi_data lcd_panel = {
+        .name                   = "generic",
+        .platform_enable        = omap3_stalker_enable_lcd,
+        .platform_disable       = omap3_stalker_disable_lcd,
+};
+
 static struct omap_dss_device omap3_stalker_lcd_device = {
         .name                   = "lcd",
-        .driver_name            = "generic_panel",
+        .driver_name            = "generic_dpi_panel",
+        .data                   = &lcd_panel,
         .phy.dpi.data_lines     = 24,
         .type                   = OMAP_DISPLAY_TYPE_DPI,
-        .platform_enable        = omap3_stalker_enable_lcd,
-        .platform_disable       = omap3_stalker_disable_lcd,
 };
 
 static int omap3_stalker_enable_tv(struct omap_dss_device *dssdev)
@@ -208,13 +214,18 @@ static void omap3_stalker_disable_dvi(struct omap_dss_device *dssdev)
         dvi_enabled = 0;
 }
 
+static struct panel_generic_dpi_data dvi_panel = {
+        .name                   = "generic",
+        .platform_enable        = omap3_stalker_enable_dvi,
+        .platform_disable       = omap3_stalker_disable_dvi,
+};
+
 static struct omap_dss_device omap3_stalker_dvi_device = {
         .name                   = "dvi",
-        .driver_name            = "generic_panel",
         .type                   = OMAP_DISPLAY_TYPE_DPI,
+        .driver_name            = "generic_dpi_panel",
+        .data                   = &dvi_panel,
         .phy.dpi.data_lines     = 24,
-        .platform_enable        = omap3_stalker_enable_dvi,
-        .platform_disable       = omap3_stalker_disable_dvi,
 };
 
 static struct omap_dss_device *omap3_stalker_dss_devices[] = {
diff --git a/arch/arm/mach-omap2/board-zoom-display.c b/arch/arm/mach-omap2/board-zoom-display.c
new file mode 100644
index 00000000000..6bcd43657ae
--- /dev/null
+++ b/arch/arm/mach-omap2/board-zoom-display.c
@@ -0,0 +1,168 @@
+/*
+ * Copyright (C) 2010 Texas Instruments Inc.
+ *
+ * Modified from mach-omap2/board-zoom-peripherals.c
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/kernel.h>
+#include <linux/init.h>
+#include <linux/platform_device.h>
+#include <linux/gpio.h>
+#include <linux/i2c/twl.h>
+#include <linux/spi/spi.h>
+#include <plat/mcspi.h>
+#include <plat/display.h>
+
+#define LCD_PANEL_RESET_GPIO_PROD       96
+#define LCD_PANEL_RESET_GPIO_PILOT      55
+#define LCD_PANEL_QVGA_GPIO             56
+
+static void zoom_lcd_panel_init(void)
+{
+        int ret;
+        unsigned char lcd_panel_reset_gpio;
+
+        lcd_panel_reset_gpio = (omap_rev() > OMAP3430_REV_ES3_0) ?
+                        LCD_PANEL_RESET_GPIO_PROD :
+                        LCD_PANEL_RESET_GPIO_PILOT;
+
+        ret = gpio_request(lcd_panel_reset_gpio, "lcd reset");
+        if (ret) {
+                pr_err("Failed to get LCD reset GPIO (gpio%d).\n",
+                        lcd_panel_reset_gpio);
+                return;
+        }
+        gpio_direction_output(lcd_panel_reset_gpio, 1);
+
+        ret = gpio_request(LCD_PANEL_QVGA_GPIO, "lcd qvga");
+        if (ret) {
+                pr_err("Failed to get LCD_PANEL_QVGA_GPIO (gpio%d).\n",
+                        LCD_PANEL_QVGA_GPIO);
+                goto err0;
+        }
+        gpio_direction_output(LCD_PANEL_QVGA_GPIO, 1);
+
+        return;
+err0:
+        gpio_free(lcd_panel_reset_gpio);
+}
+
+static int zoom_panel_enable_lcd(struct omap_dss_device *dssdev)
+{
+        return 0;
+}
+
+static void zoom_panel_disable_lcd(struct omap_dss_device *dssdev)
+{
+}
+
+/*
+ * PWMA/B register offsets (TWL4030_MODULE_PWMA)
+ */
+#define TWL_INTBR_PMBR1 0xD
+#define TWL_INTBR_GPBR1 0xC
+#define TWL_LED_PWMON   0x0
+#define TWL_LED_PWMOFF  0x1
+
+static int zoom_set_bl_intensity(struct omap_dss_device *dssdev, int level)
+{
+        unsigned char c;
+        u8 mux_pwm, enb_pwm;
+
+        if (level > 100)
+                return -1;
+
+        twl_i2c_read_u8(TWL4030_MODULE_INTBR, &mux_pwm, TWL_INTBR_PMBR1);
+        twl_i2c_read_u8(TWL4030_MODULE_INTBR, &enb_pwm, TWL_INTBR_GPBR1);
+
+        if (level == 0) {
+                /* disable pwm1 output and clock */
+                enb_pwm = enb_pwm & 0xF5;
+                /* change pwm1 pin to gpio pin */
+                mux_pwm = mux_pwm & 0xCF;
+                twl_i2c_write_u8(TWL4030_MODULE_INTBR,
+                                        enb_pwm, TWL_INTBR_GPBR1);
+                twl_i2c_write_u8(TWL4030_MODULE_INTBR,
+                                        mux_pwm, TWL_INTBR_PMBR1);
+                return 0;
+        }
+
+        if (!((enb_pwm & 0xA) && (mux_pwm & 0x30))) {
+                /* change gpio pin to pwm1 pin */
+                mux_pwm = mux_pwm | 0x30;
+                /* enable pwm1 output and clock*/
+                enb_pwm = enb_pwm | 0x0A;
+                twl_i2c_write_u8(TWL4030_MODULE_INTBR,
+                                        mux_pwm, TWL_INTBR_PMBR1);
+                twl_i2c_write_u8(TWL4030_MODULE_INTBR,
+                                        enb_pwm, TWL_INTBR_GPBR1);
+        }
+
+        c = ((50 * (100 - level)) / 100) + 1;
+        twl_i2c_write_u8(TWL4030_MODULE_PWM1, 0x7F, TWL_LED_PWMOFF);
+        twl_i2c_write_u8(TWL4030_MODULE_PWM1, c, TWL_LED_PWMON);
+
+        return 0;
+}
+
+static struct omap_dss_device zoom_lcd_device = {
+        .name                   = "lcd",
+        .driver_name            = "NEC_8048_panel",
+        .type                   = OMAP_DISPLAY_TYPE_DPI,
+        .phy.dpi.data_lines     = 24,
+        .platform_enable        = zoom_panel_enable_lcd,
+        .platform_disable       = zoom_panel_disable_lcd,
+        .max_backlight_level    = 100,
+        .set_backlight          = zoom_set_bl_intensity,
+};
+
+static struct omap_dss_device *zoom_dss_devices[] = {
+        &zoom_lcd_device,
+};
+
+static struct omap_dss_board_info zoom_dss_data = {
+        .num_devices            = ARRAY_SIZE(zoom_dss_devices),
+        .devices                = zoom_dss_devices,
+        .default_device         = &zoom_lcd_device,
+};
+
+static struct platform_device zoom_dss_device = {
+        .name                           = "omapdss",
+        .id                             = -1,
+        .dev                            = {
+                .platform_data          = &zoom_dss_data,
+        },
+};
+
+static struct omap2_mcspi_device_config dss_lcd_mcspi_config = {
+        .turbo_mode             = 1,
+        .single_channel = 1,  /* 0: slave, 1: master */
+};
+
+static struct spi_board_info nec_8048_spi_board_info[] __initdata = {
+        [0] = {
+                .modalias               = "nec_8048_spi",
+                .bus_num                = 1,
+                .chip_select            = 2,
+                .max_speed_hz           = 375000,
+                .controller_data        = &dss_lcd_mcspi_config,
+        },
+};
+
+static struct platform_device *zoom_display_devices[] __initdata = {
+        &zoom_dss_device,
+};
+
+void __init zoom_display_init(void)
+{
+        platform_add_devices(zoom_display_devices,
+                                ARRAY_SIZE(zoom_display_devices));
+        spi_register_board_info(nec_8048_spi_board_info,
+                                ARRAY_SIZE(nec_8048_spi_board_info));
+        zoom_lcd_panel_init();
+}
+
diff --git a/arch/arm/mach-omap2/board-zoom-peripherals.c b/arch/arm/mach-omap2/board-zoom-peripherals.c
index 3fbd0edd712..14d95afa3f0 100644
--- a/arch/arm/mach-omap2/board-zoom-peripherals.c
+++ b/arch/arm/mach-omap2/board-zoom-peripherals.c
@@ -35,6 +35,8 @@
 #define OMAP_ZOOM_WLAN_PMENA_GPIO       (101)
 #define OMAP_ZOOM_WLAN_IRQ_GPIO         (162)
 
+#define LCD_PANEL_ENABLE_GPIO           (7 + OMAP_MAX_GPIO_LINES)
+
 /* Zoom2 has Qwerty keyboard*/
 static uint32_t board_keymap[] = {
         KEY(0, 0, KEY_E),
@@ -224,9 +226,43 @@ static struct omap2_hsmmc_info mmc[] = {
         {}      /* Terminator */
 };
 
+static struct regulator_consumer_supply zoom_vpll2_supply =
+        REGULATOR_SUPPLY("vdds_dsi", "omapdss");
+
+static struct regulator_consumer_supply zoom_vdda_dac_supply =
+        REGULATOR_SUPPLY("vdda_dac", "omapdss");
+
+static struct regulator_init_data zoom_vpll2 = {
+        .constraints = {
+                .min_uV                 = 1800000,
+                .max_uV                 = 1800000,
+                .valid_modes_mask       = REGULATOR_MODE_NORMAL
+                                        | REGULATOR_MODE_STANDBY,
+                .valid_ops_mask         = REGULATOR_CHANGE_MODE
+                                        | REGULATOR_CHANGE_STATUS,
+        },
+        .num_consumer_supplies          = 1,
+        .consumer_supplies              = &zoom_vpll2_supply,
+};
+
+static struct regulator_init_data zoom_vdac = {
+        .constraints = {
+                .min_uV                 = 1800000,
+                .max_uV                 = 1800000,
+                .valid_modes_mask       = REGULATOR_MODE_NORMAL
+                                        | REGULATOR_MODE_STANDBY,
+                .valid_ops_mask         = REGULATOR_CHANGE_MODE
+                                        | REGULATOR_CHANGE_STATUS,
+        },
+        .num_consumer_supplies          = 1,
+        .consumer_supplies              = &zoom_vdda_dac_supply,
+};
+
 static int zoom_twl_gpio_setup(struct device *dev,
                 unsigned gpio, unsigned ngpio)
 {
+        int ret;
+
         /* gpio + 0 is "mmc0_cd" (input/IRQ) */
         mmc[0].gpio_cd = gpio + 0;
         omap2_hsmmc_init(mmc);
@@ -238,7 +274,15 @@ static int zoom_twl_gpio_setup(struct device *dev,
         zoom_vsim_supply.dev = mmc[0].dev;
         zoom_vmmc2_supply.dev = mmc[1].dev;
 
-        return 0;
+        ret = gpio_request(LCD_PANEL_ENABLE_GPIO, "lcd enable");
+        if (ret) {
+                pr_err("Failed to get LCD_PANEL_ENABLE_GPIO (gpio%d).\n",
+                                LCD_PANEL_ENABLE_GPIO);
+                return ret;
+        }
+        gpio_direction_output(LCD_PANEL_ENABLE_GPIO, 0);
+
+        return ret;
 }
 
 /* EXTMUTE callback function */
@@ -301,7 +345,8 @@ static struct twl4030_platform_data zoom_twldata = {
         .vmmc1          = &zoom_vmmc1,
         .vmmc2          = &zoom_vmmc2,
         .vsim           = &zoom_vsim,
-
+        .vpll2          = &zoom_vpll2,
+        .vdac           = &zoom_vdac,
 };
 
 static struct i2c_board_info __initdata zoom_i2c_boardinfo[] = {
diff --git a/arch/arm/mach-omap2/board-zoom.c b/arch/arm/mach-omap2/board-zoom.c
index e041c537ea3..e26754c24ee 100644
--- a/arch/arm/mach-omap2/board-zoom.c
+++ b/arch/arm/mach-omap2/board-zoom.c
@@ -130,6 +130,7 @@ static void __init omap_zoom_init(void)
                         ARRAY_SIZE(zoom_nand_partitions), ZOOM_NAND_CS);
         zoom_debugboard_init();
         zoom_peripherals_init();
+        zoom_display_init();
 }
 
 MACHINE_START(OMAP_ZOOM2, "OMAP Zoom2 board")
diff --git a/arch/arm/mach-omap2/cpuidle34xx.c b/arch/arm/mach-omap2/cpuidle34xx.c
index f3e043fe5eb..11b89e9687f 100644
--- a/arch/arm/mach-omap2/cpuidle34xx.c
+++ b/arch/arm/mach-omap2/cpuidle34xx.c
@@ -252,7 +252,7 @@ static int omap3_enter_idle_bm(struct cpuidle_device *dev,
          * FIXME: we currently manage device-specific idle states
          *        for PER and CORE in combination with CPU-specific
          *        idle states.  This is wrong, and device-specific
-         *        idle managment needs to be separated out into 
+         *        idle management needs to be separated out into 
          *        its own code.
          */
 
diff --git a/arch/arm/mach-omap2/include/mach/board-zoom.h b/arch/arm/mach-omap2/include/mach/board-zoom.h
index f93ca3928c3..d20bd9c1a10 100644
--- a/arch/arm/mach-omap2/include/mach/board-zoom.h
+++ b/arch/arm/mach-omap2/include/mach/board-zoom.h
@@ -1,9 +1,12 @@
 /*
  * Defines for zoom boards
  */
+#include <plat/display.h>
+
 #define ZOOM_NAND_CS    0
 
 extern int __init zoom_debugboard_init(void);
 extern void __init zoom_peripherals_init(void);
+extern void __init zoom_display_init(void);
 
 #define ZOOM2_HEADSET_EXTMUTE_GPIO      153
diff --git a/arch/arm/mach-omap2/pm24xx.c b/arch/arm/mach-omap2/pm24xx.c
index dac2d1d9987..9e5dc8ed51e 100644
--- a/arch/arm/mach-omap2/pm24xx.c
+++ b/arch/arm/mach-omap2/pm24xx.c
@@ -350,7 +350,7 @@ static void omap2_pm_end(void)
         enable_hlt();
 }
 
-static struct platform_suspend_ops omap_pm_ops = {
+static const struct platform_suspend_ops omap_pm_ops = {
         .begin          = omap2_pm_begin,
         .enter          = omap2_pm_enter,
         .end            = omap2_pm_end,
diff --git a/arch/arm/mach-omap2/pm34xx.c b/arch/arm/mach-omap2/pm34xx.c
index 5b323f28da2..8cbbeade4b8 100644
--- a/arch/arm/mach-omap2/pm34xx.c
+++ b/arch/arm/mach-omap2/pm34xx.c
@@ -605,7 +605,7 @@ static void omap3_pm_end(void)
         return;
 }
 
-static struct platform_suspend_ops omap_pm_ops = {
+static const struct platform_suspend_ops omap_pm_ops = {
         .begin          = omap3_pm_begin,
         .end            = omap3_pm_end,
         .enter          = omap3_pm_enter,
diff --git a/arch/arm/mach-omap2/pm44xx.c b/arch/arm/mach-omap2/pm44xx.c
index e9f4862c4de..76cfff2db51 100644
--- a/arch/arm/mach-omap2/pm44xx.c
+++ b/arch/arm/mach-omap2/pm44xx.c
@@ -65,7 +65,7 @@ static void omap4_pm_end(void)
         return;
 }
 
-static struct platform_suspend_ops omap_pm_ops = {
+static const struct platform_suspend_ops omap_pm_ops = {
         .begin          = omap4_pm_begin,
         .end            = omap4_pm_end,
         .enter          = omap4_pm_enter,
diff --git a/arch/arm/mach-omap2/serial.c b/arch/arm/mach-omap2/serial.c
index c64578853a8..302da7403a1 100644
--- a/arch/arm/mach-omap2/serial.c
+++ b/arch/arm/mach-omap2/serial.c
@@ -852,7 +852,7 @@ void __init omap_serial_init_port(struct omap_board_data *bdata)
 }
 
 /**
- * omap_serial_init() - intialize all supported serial ports
+ * omap_serial_init() - initialize all supported serial ports
  *
  * Initializes all available UARTs as serial ports. Platforms
  * can call this function when they want to have default behaviour
diff --git a/arch/arm/mach-pnx4008/pm.c b/arch/arm/mach-pnx4008/pm.c
index ee3c29c57ae..f3e60a049f9 100644
--- a/arch/arm/mach-pnx4008/pm.c
+++ b/arch/arm/mach-pnx4008/pm.c
@@ -119,7 +119,7 @@ static int pnx4008_pm_valid(suspend_state_t state)
                (state == PM_SUSPEND_MEM);
 }
 
-static struct platform_suspend_ops pnx4008_pm_ops = {
+static const struct platform_suspend_ops pnx4008_pm_ops = {
         .enter = pnx4008_pm_enter,
         .valid = pnx4008_pm_valid,
 };
diff --git a/arch/arm/mach-pxa/mxm8x10.c b/arch/arm/mach-pxa/mxm8x10.c
index 462167ac05f..cdf7f41e2bb 100644
--- a/arch/arm/mach-pxa/mxm8x10.c
+++ b/arch/arm/mach-pxa/mxm8x10.c
@@ -337,7 +337,7 @@ void __init mxm_8x10_mmc_init(void)
 }
 #endif
 
-/* USB Open Host Controler Interface */
+/* USB Open Host Controller Interface */
 static struct pxaohci_platform_data mxm_8x10_ohci_platform_data = {
         .port_mode = PMM_NPS_MODE,
         .flags = ENABLE_PORT_ALL
diff --git a/arch/arm/mach-pxa/pm.c b/arch/arm/mach-pxa/pm.c
index 166c15f6291..978e1b28954 100644
--- a/arch/arm/mach-pxa/pm.c
+++ b/arch/arm/mach-pxa/pm.c
@@ -96,7 +96,7 @@ void pxa_pm_finish(void)
                 pxa_cpu_pm_fns->finish();
 }
 
-static struct platform_suspend_ops pxa_pm_ops = {
+static const struct platform_suspend_ops pxa_pm_ops = {
         .valid          = pxa_pm_valid,
         .enter          = pxa_pm_enter,
         .prepare        = pxa_pm_prepare,
diff --git a/arch/arm/mach-pxa/sharpsl_pm.c b/arch/arm/mach-pxa/sharpsl_pm.c
index e68d46d415f..785880f67b6 100644
--- a/arch/arm/mach-pxa/sharpsl_pm.c
+++ b/arch/arm/mach-pxa/sharpsl_pm.c
@@ -869,7 +869,7 @@ static void sharpsl_apm_get_power_status(struct apm_power_info *info)
 }
 
 #ifdef CONFIG_PM
-static struct platform_suspend_ops sharpsl_pm_ops = {
+static const struct platform_suspend_ops sharpsl_pm_ops = {
         .prepare        = pxa_pm_prepare,
         .finish         = pxa_pm_finish,
         .enter          = corgi_pxa_pm_enter,
diff --git a/arch/arm/mach-s3c2410/mach-h1940.c b/arch/arm/mach-s3c2410/mach-h1940.c
index d7ada8c7e41..1a81fe12ccd 100644
--- a/arch/arm/mach-s3c2410/mach-h1940.c
+++ b/arch/arm/mach-s3c2410/mach-h1940.c
@@ -387,7 +387,7 @@ static struct platform_device *h1940_devices[] __initdata = {
         &s3c_device_wdt,
         &s3c_device_i2c0,
         &s3c_device_iis,
-        &s3c_device_pcm,
+        &samsung_asoc_dma,
         &s3c_device_usbgadget,
         &h1940_device_leds,
         &h1940_device_bluetooth,
diff --git a/arch/arm/mach-s3c2440/mach-rx1950.c b/arch/arm/mach-s3c2440/mach-rx1950.c
index e0622bbb6df..eab6ae50683 100644
--- a/arch/arm/mach-s3c2440/mach-rx1950.c
+++ b/arch/arm/mach-s3c2440/mach-rx1950.c
@@ -692,7 +692,7 @@ static struct platform_device *rx1950_devices[] __initdata = {
         &s3c_device_wdt,
         &s3c_device_i2c0,
         &s3c_device_iis,
-        &s3c_device_pcm,
+        &samsung_asoc_dma,
         &s3c_device_usbgadget,
         &s3c_device_rtc,
         &s3c_device_nand,
diff --git a/arch/arm/mach-s3c64xx/clock.c b/arch/arm/mach-s3c64xx/clock.c
index 7e03f0ae2fc..1c98d2ff2ed 100644
--- a/arch/arm/mach-s3c64xx/clock.c
+++ b/arch/arm/mach-s3c64xx/clock.c
@@ -695,7 +695,7 @@ static struct clksrc_clk clksrcs[] = {
         }, {
                 .clk    = {
                         .name           = "audio-bus",
-                        .id             = -1,  /* There's only one IISv4 port */
+                        .id             = 2,
                         .ctrlbit        = S3C6410_CLKCON_SCLK_AUDIO2,
                         .enable         = s3c64xx_sclk_ctrl,
                 },
diff --git a/arch/arm/mach-s3c64xx/dev-audio.c b/arch/arm/mach-s3c64xx/dev-audio.c
index 76426a32c01..cad67022fa2 100644
--- a/arch/arm/mach-s3c64xx/dev-audio.c
+++ b/arch/arm/mach-s3c64xx/dev-audio.c
@@ -22,7 +22,12 @@
 #include <plat/audio.h>
 #include <plat/gpio-cfg.h>
 
-static int s3c64xx_i2sv3_cfg_gpio(struct platform_device *pdev)
+static const char *rclksrc[] = {
+        [0] = "iis",
+        [1] = "audio-bus",
+};
+
+static int s3c64xx_i2s_cfg_gpio(struct platform_device *pdev)
 {
         unsigned int base;
 
@@ -33,6 +38,12 @@ static int s3c64xx_i2sv3_cfg_gpio(struct platform_device *pdev)
         case 1:
                 base = S3C64XX_GPE(0);
                 break;
+        case 2:
+                s3c_gpio_cfgpin(S3C64XX_GPC(4), S3C_GPIO_SFN(5));
+                s3c_gpio_cfgpin(S3C64XX_GPC(5), S3C_GPIO_SFN(5));
+                s3c_gpio_cfgpin(S3C64XX_GPC(7), S3C_GPIO_SFN(5));
+                s3c_gpio_cfgpin_range(S3C64XX_GPH(6), 4, S3C_GPIO_SFN(5));
+                return 0;
         default:
                 printk(KERN_DEBUG "Invalid I2S Controller number: %d\n",
                         pdev->id);
@@ -44,16 +55,6 @@ static int s3c64xx_i2sv3_cfg_gpio(struct platform_device *pdev)
         return 0;
 }
 
-static int s3c64xx_i2sv4_cfg_gpio(struct platform_device *pdev)
-{
-        s3c_gpio_cfgpin(S3C64XX_GPC(4), S3C_GPIO_SFN(5));
-        s3c_gpio_cfgpin(S3C64XX_GPC(5), S3C_GPIO_SFN(5));
-        s3c_gpio_cfgpin(S3C64XX_GPC(7), S3C_GPIO_SFN(5));
-        s3c_gpio_cfgpin_range(S3C64XX_GPH(6), 4, S3C_GPIO_SFN(5));
-
-        return 0;
-}
-
 static struct resource s3c64xx_iis0_resource[] = {
         [0] = {
                 .start = S3C64XX_PA_IIS0,
@@ -72,17 +73,22 @@ static struct resource s3c64xx_iis0_resource[] = {
         },
 };
 
-static struct s3c_audio_pdata s3c_i2s0_pdata = {
-        .cfg_gpio = s3c64xx_i2sv3_cfg_gpio,
+static struct s3c_audio_pdata i2sv3_pdata = {
+        .cfg_gpio = s3c64xx_i2s_cfg_gpio,
+        .type = {
+                .i2s = {
+                        .src_clk = rclksrc,
+                },
+        },
 };
 
 struct platform_device s3c64xx_device_iis0 = {
-        .name             = "s3c64xx-iis",
+        .name             = "samsung-i2s",
         .id               = 0,
         .num_resources    = ARRAY_SIZE(s3c64xx_iis0_resource),
         .resource         = s3c64xx_iis0_resource,
         .dev = {
-                .platform_data = &s3c_i2s0_pdata,
+                .platform_data = &i2sv3_pdata,
         },
 };
 EXPORT_SYMBOL(s3c64xx_device_iis0);
@@ -105,17 +111,13 @@ static struct resource s3c64xx_iis1_resource[] = {
         },
 };
 
-static struct s3c_audio_pdata s3c_i2s1_pdata = {
-        .cfg_gpio = s3c64xx_i2sv3_cfg_gpio,
-};
-
 struct platform_device s3c64xx_device_iis1 = {
-        .name             = "s3c64xx-iis",
+        .name             = "samsung-i2s",
         .id               = 1,
         .num_resources    = ARRAY_SIZE(s3c64xx_iis1_resource),
         .resource         = s3c64xx_iis1_resource,
         .dev = {
-                .platform_data = &s3c_i2s1_pdata,
+                .platform_data = &i2sv3_pdata,
         },
 };
 EXPORT_SYMBOL(s3c64xx_device_iis1);
@@ -138,17 +140,23 @@ static struct resource s3c64xx_iisv4_resource[] = {
         },
 };
 
-static struct s3c_audio_pdata s3c_i2sv4_pdata = {
-        .cfg_gpio = s3c64xx_i2sv4_cfg_gpio,
+static struct s3c_audio_pdata i2sv4_pdata = {
+        .cfg_gpio = s3c64xx_i2s_cfg_gpio,
+        .type = {
+                .i2s = {
+                        .quirks = QUIRK_PRI_6CHAN,
+                        .src_clk = rclksrc,
+                },
+        },
 };
 
 struct platform_device s3c64xx_device_iisv4 = {
-        .name             = "s3c64xx-iis-v4",
-        .id               = -1,
+        .name = "samsung-i2s",
+        .id = 2,
         .num_resources    = ARRAY_SIZE(s3c64xx_iisv4_resource),
         .resource         = s3c64xx_iisv4_resource,
         .dev = {
-                .platform_data = &s3c_i2sv4_pdata,
+                .platform_data = &i2sv4_pdata,
         },
 };
 EXPORT_SYMBOL(s3c64xx_device_iisv4);
@@ -288,7 +296,7 @@ static struct s3c_audio_pdata s3c_ac97_pdata;
 static u64 s3c64xx_ac97_dmamask = DMA_BIT_MASK(32);
 
 struct platform_device s3c64xx_device_ac97 = {
-        .name             = "s3c-ac97",
+        .name             = "samsung-ac97",
         .id               = -1,
         .num_resources    = ARRAY_SIZE(s3c64xx_ac97_resource),
         .resource         = s3c64xx_ac97_resource,
@@ -307,16 +315,3 @@ void __init s3c64xx_ac97_setup_gpio(int num)
         else
                 s3c_ac97_pdata.cfg_gpio = s3c64xx_ac97_cfg_gpe;
 }
-
-static u64 s3c_device_audio_dmamask = 0xffffffffUL;
-
-struct platform_device s3c_device_pcm = {
-        .name             = "s3c24xx-pcm-audio",
-        .id               = -1,
-        .dev              = {
-                .dma_mask = &s3c_device_audio_dmamask,
-                .coherent_dma_mask = 0xffffffffUL
-        }
-};
-EXPORT_SYMBOL(s3c_device_pcm);
-
diff --git a/arch/arm/mach-s3c64xx/dma.c b/arch/arm/mach-s3c64xx/dma.c
index e7d03ab41d8..372ea685545 100644
--- a/arch/arm/mach-s3c64xx/dma.c
+++ b/arch/arm/mach-s3c64xx/dma.c
@@ -740,7 +740,7 @@ static int __init s3c64xx_dma_init(void)
         /* Set all DMA configuration to be DMA, not SDMA */
         writel(0xffffff, S3C_SYSREG(0x110));
 
-        /* Register standard DMA controlers */
+        /* Register standard DMA controllers */
         s3c64xx_dma_init1(0, DMACH_UART0, IRQ_DMA0, 0x75000000);
         s3c64xx_dma_init1(8, DMACH_PCM1_TX, IRQ_DMA1, 0x75100000);
 
diff --git a/arch/arm/mach-s3c64xx/mach-smdk6410.c b/arch/arm/mach-s3c64xx/mach-smdk6410.c
index 77488facfe4..e85192a86fb 100644
--- a/arch/arm/mach-s3c64xx/mach-smdk6410.c
+++ b/arch/arm/mach-s3c64xx/mach-smdk6410.c
@@ -283,7 +283,7 @@ static struct platform_device *smdk6410_devices[] __initdata = {
         &s3c_device_fb,
         &s3c_device_ohci,
         &s3c_device_usb_hsotg,
-        &s3c_device_pcm,
+        &samsung_asoc_dma,
         &s3c64xx_device_iisv4,
         &samsung_device_keypad,
 
diff --git a/arch/arm/mach-s5p6442/dev-audio.c b/arch/arm/mach-s5p6442/dev-audio.c
index 3462197ff35..8719dc41fe3 100644
--- a/arch/arm/mach-s5p6442/dev-audio.c
+++ b/arch/arm/mach-s5p6442/dev-audio.c
@@ -29,7 +29,7 @@ static int s5p6442_cfg_i2s(struct platform_device *pdev)
                 base = S5P6442_GPC1(0);
                 break;
 
-        case -1:
+        case 0:
                 base = S5P6442_GPC0(0);
                 break;
 
@@ -42,8 +42,19 @@ static int s5p6442_cfg_i2s(struct platform_device *pdev)
         return 0;
 }
 
-static struct s3c_audio_pdata s3c_i2s_pdata = {
+static const char *rclksrc_v35[] = {
+        [0] = "busclk",
+        [1] = "i2sclk",
+};
+
+static struct s3c_audio_pdata i2sv35_pdata = {
         .cfg_gpio = s5p6442_cfg_i2s,
+        .type = {
+                .i2s = {
+                        .quirks = QUIRK_SEC_DAI | QUIRK_NEED_RSTCLR,
+                        .src_clk = rclksrc_v35,
+                },
+        },
 };
 
 static struct resource s5p6442_iis0_resource[] = {
@@ -62,15 +73,34 @@ static struct resource s5p6442_iis0_resource[] = {
                 .end   = DMACH_I2S0_RX,
                 .flags = IORESOURCE_DMA,
         },
+        [3] = {
+                .start = DMACH_I2S0S_TX,
+                .end = DMACH_I2S0S_TX,
+                .flags = IORESOURCE_DMA,
+        },
 };
 
 struct platform_device s5p6442_device_iis0 = {
-        .name             = "s3c64xx-iis-v4",
-        .id               = -1,
+        .name = "samsung-i2s",
+        .id = 0,
         .num_resources    = ARRAY_SIZE(s5p6442_iis0_resource),
         .resource         = s5p6442_iis0_resource,
         .dev = {
-                .platform_data = &s3c_i2s_pdata,
+                .platform_data = &i2sv35_pdata,
+        },
+};
+
+static const char *rclksrc_v3[] = {
+        [0] = "iis",
+        [1] = "sclk_audio",
+};
+
+static struct s3c_audio_pdata i2sv3_pdata = {
+        .cfg_gpio = s5p6442_cfg_i2s,
+        .type = {
+                .i2s = {
+                        .src_clk = rclksrc_v3,
+                },
         },
 };
 
@@ -93,12 +123,12 @@ static struct resource s5p6442_iis1_resource[] = {
 };
 
 struct platform_device s5p6442_device_iis1 = {
-        .name             = "s3c64xx-iis",
+        .name             = "samsung-i2s",
         .id               = 1,
         .num_resources    = ARRAY_SIZE(s5p6442_iis1_resource),
         .resource         = s5p6442_iis1_resource,
         .dev = {
-                .platform_data = &s3c_i2s_pdata,
+                .platform_data = &i2sv3_pdata,
         },
 };
 
diff --git a/arch/arm/mach-s5p64x0/clock-s5p6440.c b/arch/arm/mach-s5p64x0/clock-s5p6440.c
index e4883dc1c8d..409c5fc3670 100644
--- a/arch/arm/mach-s5p64x0/clock-s5p6440.c
+++ b/arch/arm/mach-s5p64x0/clock-s5p6440.c
@@ -261,7 +261,7 @@ static struct clk init_clocks_disable[] = {
                 .enable         = s5p64x0_pclk_ctrl,
                 .ctrlbit        = (1 << 25),
         }, {
-                .name           = "i2s_v40",
+                .name           = "iis",
                 .id             = 0,
                 .parent         = &clk_pclk_low.clk,
                 .enable         = s5p64x0_pclk_ctrl,
diff --git a/arch/arm/mach-s5p64x0/clock-s5p6450.c b/arch/arm/mach-s5p64x0/clock-s5p6450.c
index 7dbf3c968f5..7fc6abd3591 100644
--- a/arch/arm/mach-s5p64x0/clock-s5p6450.c
+++ b/arch/arm/mach-s5p64x0/clock-s5p6450.c
@@ -256,7 +256,7 @@ static struct clk init_clocks_disable[] = {
                 .ctrlbit        = (1 << 22),
         }, {
                 .name           = "iis",
-                .id             = -1,
+                .id             = 0,
                 .parent         = &clk_pclk_low.clk,
                 .enable         = s5p64x0_pclk_ctrl,
                 .ctrlbit        = (1 << 26),
diff --git a/arch/arm/mach-s5p64x0/dev-audio.c b/arch/arm/mach-s5p64x0/dev-audio.c
index 396bacc0a39..14f89e73b8d 100644
--- a/arch/arm/mach-s5p64x0/dev-audio.c
+++ b/arch/arm/mach-s5p64x0/dev-audio.c
@@ -19,15 +19,19 @@
 #include <mach/dma.h>
 #include <mach/irqs.h>
 
-static int s5p6440_cfg_i2s(struct platform_device *pdev)
+static const char *rclksrc[] = {
+        [0] = "iis",
+        [1] = "sclk_audio2",
+};
+
+static int s5p64x0_cfg_i2s(struct platform_device *pdev)
 {
         /* configure GPIO for i2s port */
         switch (pdev->id) {
-        case -1:
+        case 0:
                 s3c_gpio_cfgpin_range(S5P6440_GPR(4), 5, S3C_GPIO_SFN(5));
                 s3c_gpio_cfgpin_range(S5P6440_GPR(13), 2, S3C_GPIO_SFN(5));
                 break;
-
         default:
                 printk(KERN_ERR "Invalid Device %d\n", pdev->id);
                 return -EINVAL;
@@ -36,31 +40,14 @@ static int s5p6440_cfg_i2s(struct platform_device *pdev)
         return 0;
 }
 
-static int s5p6450_cfg_i2s(struct platform_device *pdev)
-{
-        /* configure GPIO for i2s port */
-        switch (pdev->id) {
-        case -1:
-                s3c_gpio_cfgpin(S5P6450_GPB(4), S3C_GPIO_SFN(5));
-                s3c_gpio_cfgpin_range(S5P6450_GPR(4), 5, S3C_GPIO_SFN(5));
-                s3c_gpio_cfgpin_range(S5P6450_GPR(13), 2, S3C_GPIO_SFN(5));
-
-                break;
-
-        default:
-                printk(KERN_ERR "Invalid Device %d\n", pdev->id);
-                return -EINVAL;
-        }
-
-        return 0;
-}
-
-static struct s3c_audio_pdata s5p6440_i2s_pdata = {
-        .cfg_gpio = s5p6440_cfg_i2s,
-};
-
-static struct s3c_audio_pdata s5p6450_i2s_pdata = {
-        .cfg_gpio = s5p6450_cfg_i2s,
+static struct s3c_audio_pdata s5p64x0_i2s_pdata = {
+        .cfg_gpio = s5p64x0_cfg_i2s,
+        .type = {
+                .i2s = {
+                        .quirks = QUIRK_PRI_6CHAN,
+                        .src_clk = rclksrc,
+                },
+        },
 };
 
 static struct resource s5p64x0_iis0_resource[] = {
@@ -82,22 +69,22 @@ static struct resource s5p64x0_iis0_resource[] = {
 };
 
 struct platform_device s5p6440_device_iis = {
-        .name           = "s3c64xx-iis-v4",
-        .id             = -1,
+        .name           = "samsung-i2s",
+        .id             = 0,
         .num_resources  = ARRAY_SIZE(s5p64x0_iis0_resource),
         .resource       = s5p64x0_iis0_resource,
         .dev = {
-                .platform_data = &s5p6440_i2s_pdata,
+                .platform_data = &s5p64x0_i2s_pdata,
         },
 };
 
 struct platform_device s5p6450_device_iis0 = {
-        .name           = "s3c64xx-iis-v4",
-        .id             = -1,
+        .name           = "samsung-i2s",
+        .id             = 0,
         .num_resources  = ARRAY_SIZE(s5p64x0_iis0_resource),
         .resource       = s5p64x0_iis0_resource,
         .dev = {
-                .platform_data = &s5p6450_i2s_pdata,
+                .platform_data = &s5p64x0_i2s_pdata,
         },
 };
 
diff --git a/arch/arm/mach-s5pc100/dev-audio.c b/arch/arm/mach-s5pc100/dev-audio.c
index 564e195ec49..ab2d27172cb 100644
--- a/arch/arm/mach-s5pc100/dev-audio.c
+++ b/arch/arm/mach-s5pc100/dev-audio.c
@@ -23,17 +23,14 @@ static int s5pc100_cfg_i2s(struct platform_device *pdev)
 {
         /* configure GPIO for i2s port */
         switch (pdev->id) {
+        case 0: /* Dedicated pins */
+                break;
         case 1:
                 s3c_gpio_cfgpin_range(S5PC100_GPC(0), 5, S3C_GPIO_SFN(2));
                 break;
-
         case 2:
                 s3c_gpio_cfgpin_range(S5PC100_GPG3(0), 5, S3C_GPIO_SFN(4));
                 break;
-
-        case -1: /* Dedicated pins */
-                break;
-
         default:
                 printk(KERN_ERR "Invalid Device %d\n", pdev->id);
                 return -EINVAL;
@@ -42,8 +39,20 @@ static int s5pc100_cfg_i2s(struct platform_device *pdev)
         return 0;
 }
 
-static struct s3c_audio_pdata s3c_i2s_pdata = {
+static const char *rclksrc_v5[] = {
+        [0] = "iis",
+        [1] = "i2sclkd2",
+};
+
+static struct s3c_audio_pdata i2sv5_pdata = {
         .cfg_gpio = s5pc100_cfg_i2s,
+        .type = {
+                .i2s = {
+                        .quirks = QUIRK_PRI_6CHAN | QUIRK_SEC_DAI
+                                         | QUIRK_NEED_RSTCLR,
+                        .src_clk = rclksrc_v5,
+                },
+        },
 };
 
 static struct resource s5pc100_iis0_resource[] = {
@@ -62,15 +71,34 @@ static struct resource s5pc100_iis0_resource[] = {
                 .end   = DMACH_I2S0_RX,
                 .flags = IORESOURCE_DMA,
         },
+        [3] = {
+                .start = DMACH_I2S0S_TX,
+                .end = DMACH_I2S0S_TX,
+                .flags = IORESOURCE_DMA,
+        },
 };
 
 struct platform_device s5pc100_device_iis0 = {
-        .name             = "s3c64xx-iis-v4",
-        .id               = -1,
+        .name = "samsung-i2s",
+        .id = 0,
         .num_resources    = ARRAY_SIZE(s5pc100_iis0_resource),
         .resource         = s5pc100_iis0_resource,
         .dev = {
-                .platform_data = &s3c_i2s_pdata,
+                .platform_data = &i2sv5_pdata,
+        },
+};
+
+static const char *rclksrc_v3[] = {
+        [0] = "iis",
+        [1] = "sclk_audio",
+};
+
+static struct s3c_audio_pdata i2sv3_pdata = {
+        .cfg_gpio = s5pc100_cfg_i2s,
+        .type = {
+                .i2s = {
+                        .src_clk = rclksrc_v3,
+                },
         },
 };
 
@@ -93,12 +121,12 @@ static struct resource s5pc100_iis1_resource[] = {
 };
 
 struct platform_device s5pc100_device_iis1 = {
-        .name             = "s3c64xx-iis",
+        .name             = "samsung-i2s",
         .id               = 1,
         .num_resources    = ARRAY_SIZE(s5pc100_iis1_resource),
         .resource         = s5pc100_iis1_resource,
         .dev = {
-                .platform_data = &s3c_i2s_pdata,
+                .platform_data = &i2sv3_pdata,
         },
 };
 
@@ -121,12 +149,12 @@ static struct resource s5pc100_iis2_resource[] = {
 };
 
 struct platform_device s5pc100_device_iis2 = {
-        .name             = "s3c64xx-iis",
+        .name             = "samsung-i2s",
         .id               = 2,
         .num_resources    = ARRAY_SIZE(s5pc100_iis2_resource),
         .resource         = s5pc100_iis2_resource,
         .dev = {
-                .platform_data = &s3c_i2s_pdata,
+                .platform_data = &i2sv3_pdata,
         },
 };
 
@@ -253,7 +281,7 @@ static struct s3c_audio_pdata s3c_ac97_pdata = {
 static u64 s5pc100_ac97_dmamask = DMA_BIT_MASK(32);
 
 struct platform_device s5pc100_device_ac97 = {
-        .name             = "s3c-ac97",
+        .name             = "samsung-ac97",
         .id               = -1,
         .num_resources    = ARRAY_SIZE(s5pc100_ac97_resource),
         .resource         = s5pc100_ac97_resource,
diff --git a/arch/arm/mach-s5pc100/mach-smdkc100.c b/arch/arm/mach-s5pc100/mach-smdkc100.c
index 18b405d514d..dd192a27524 100644
--- a/arch/arm/mach-s5pc100/mach-smdkc100.c
+++ b/arch/arm/mach-s5pc100/mach-smdkc100.c
@@ -96,6 +96,7 @@ static struct s3c2410_uartcfg smdkc100_uartcfgs[] __initdata = {
 
 /* I2C0 */
 static struct i2c_board_info i2c_devs0[] __initdata = {
+        {I2C_BOARD_INFO("wm8580", 0x1b),},
 };
 
 /* I2C1 */
@@ -190,6 +191,7 @@ static struct platform_device *smdkc100_devices[] __initdata = {
         &s3c_device_ts,
         &s3c_device_wdt,
         &smdkc100_lcd_powerdev,
+        &samsung_asoc_dma,
         &s5pc100_device_iis0,
         &samsung_device_keypad,
         &s5pc100_device_ac97,
diff --git a/arch/arm/mach-s5pv210/clock.c b/arch/arm/mach-s5pv210/clock.c
index 019c3a69b0e..b774ff1805d 100644
--- a/arch/arm/mach-s5pv210/clock.c
+++ b/arch/arm/mach-s5pv210/clock.c
@@ -467,20 +467,20 @@ static struct clk init_clocks_disable[] = {
                 .enable         = s5pv210_clk_ip3_ctrl,
                 .ctrlbit        = (1<<21),
         }, {
-                .name           = "i2s_v50",
+                .name           = "iis",
                 .id             = 0,
                 .parent         = &clk_p,
                 .enable         = s5pv210_clk_ip3_ctrl,
                 .ctrlbit        = (1<<4),
         }, {
-                .name           = "i2s_v32",
-                .id             = 0,
+                .name           = "iis",
+                .id             = 1,
                 .parent         = &clk_p,
                 .enable         = s5pv210_clk_ip3_ctrl,
                 .ctrlbit        = (1 << 5),
         }, {
-                .name           = "i2s_v32",
-                .id             = 1,
+                .name           = "iis",
+                .id             = 2,
                 .parent         = &clk_p,
                 .enable         = s5pv210_clk_ip3_ctrl,
                 .ctrlbit        = (1 << 6),
diff --git a/arch/arm/mach-s5pv210/dev-audio.c b/arch/arm/mach-s5pv210/dev-audio.c
index 1303fcb12b5..8d58f192624 100644
--- a/arch/arm/mach-s5pv210/dev-audio.c
+++ b/arch/arm/mach-s5pv210/dev-audio.c
@@ -19,22 +19,24 @@
 #include <mach/dma.h>
 #include <mach/irqs.h>
 
+static const char *rclksrc[] = {
+        [0] = "busclk",
+        [1] = "i2sclk",
+};
+
 static int s5pv210_cfg_i2s(struct platform_device *pdev)
 {
         /* configure GPIO for i2s port */
         switch (pdev->id) {
+        case 0:
+                s3c_gpio_cfgpin_range(S5PV210_GPI(0), 7, S3C_GPIO_SFN(2));
+                break;
         case 1:
                 s3c_gpio_cfgpin_range(S5PV210_GPC0(0), 5, S3C_GPIO_SFN(2));
                 break;
-
         case 2:
                 s3c_gpio_cfgpin_range(S5PV210_GPC1(0), 5, S3C_GPIO_SFN(4));
                 break;
-
-        case -1:
-                s3c_gpio_cfgpin_range(S5PV210_GPI(0), 7, S3C_GPIO_SFN(2));
-                break;
-
         default:
                 printk(KERN_ERR "Invalid Device %d\n", pdev->id);
                 return -EINVAL;
@@ -43,8 +45,15 @@ static int s5pv210_cfg_i2s(struct platform_device *pdev)
         return 0;
 }
 
-static struct s3c_audio_pdata s3c_i2s_pdata = {
+static struct s3c_audio_pdata i2sv5_pdata = {
         .cfg_gpio = s5pv210_cfg_i2s,
+        .type = {
+                .i2s = {
+                        .quirks = QUIRK_PRI_6CHAN | QUIRK_SEC_DAI
+                                         | QUIRK_NEED_RSTCLR,
+                        .src_clk = rclksrc,
+                },
+        },
 };
 
 static struct resource s5pv210_iis0_resource[] = {
@@ -63,15 +72,34 @@ static struct resource s5pv210_iis0_resource[] = {
                 .end   = DMACH_I2S0_RX,
                 .flags = IORESOURCE_DMA,
         },
+        [3] = {
+                .start = DMACH_I2S0S_TX,
+                .end = DMACH_I2S0S_TX,
+                .flags = IORESOURCE_DMA,
+        },
 };
 
 struct platform_device s5pv210_device_iis0 = {
-        .name             = "s3c64xx-iis-v4",
-        .id               = -1,
+        .name = "samsung-i2s",
+        .id = 0,
         .num_resources    = ARRAY_SIZE(s5pv210_iis0_resource),
         .resource         = s5pv210_iis0_resource,
         .dev = {
-                .platform_data = &s3c_i2s_pdata,
+                .platform_data = &i2sv5_pdata,
+        },
+};
+
+static const char *rclksrc_v3[] = {
+        [0] = "iis",
+        [1] = "audio-bus",
+};
+
+static struct s3c_audio_pdata i2sv3_pdata = {
+        .cfg_gpio = s5pv210_cfg_i2s,
+        .type = {
+                .i2s = {
+                        .src_clk = rclksrc_v3,
+                },
         },
 };
 
@@ -94,12 +122,12 @@ static struct resource s5pv210_iis1_resource[] = {
 };
 
 struct platform_device s5pv210_device_iis1 = {
-        .name             = "s3c64xx-iis",
+        .name             = "samsung-i2s",
         .id               = 1,
         .num_resources    = ARRAY_SIZE(s5pv210_iis1_resource),
         .resource         = s5pv210_iis1_resource,
         .dev = {
-                .platform_data = &s3c_i2s_pdata,
+                .platform_data = &i2sv3_pdata,
         },
 };
 
@@ -122,12 +150,12 @@ static struct resource s5pv210_iis2_resource[] = {
 };
 
 struct platform_device s5pv210_device_iis2 = {
-        .name             = "s3c64xx-iis",
+        .name             = "samsung-i2s",
         .id               = 2,
         .num_resources    = ARRAY_SIZE(s5pv210_iis2_resource),
         .resource         = s5pv210_iis2_resource,
         .dev = {
-                .platform_data = &s3c_i2s_pdata,
+                .platform_data = &i2sv3_pdata,
         },
 };
 
@@ -283,7 +311,7 @@ static struct s3c_audio_pdata s3c_ac97_pdata = {
 static u64 s5pv210_ac97_dmamask = DMA_BIT_MASK(32);
 
 struct platform_device s5pv210_device_ac97 = {
-        .name             = "s3c-ac97",
+        .name             = "samsung-ac97",
         .id               = -1,
         .num_resources    = ARRAY_SIZE(s5pv210_ac97_resource),
         .resource         = s5pv210_ac97_resource,
diff --git a/arch/arm/mach-s5pv310/Kconfig b/arch/arm/mach-s5pv310/Kconfig
index 1150b360f38..d64efe0d4c9 100644
--- a/arch/arm/mach-s5pv310/Kconfig
+++ b/arch/arm/mach-s5pv310/Kconfig
@@ -11,6 +11,7 @@ if ARCH_S5PV310
 
 config CPU_S5PV310
         bool
+        select S3C_PL330_DMA
         help
           Enable S5PV310 CPU support
 
diff --git a/arch/arm/mach-s5pv310/Makefile b/arch/arm/mach-s5pv310/Makefile
index 84afc64e7c0..61e3cb65426 100644
--- a/arch/arm/mach-s5pv310/Makefile
+++ b/arch/arm/mach-s5pv310/Makefile
@@ -13,7 +13,7 @@ obj-				:=
 # Core support for S5PV310 system
 
 obj-$(CONFIG_CPU_S5PV310)       += cpu.o init.o clock.o irq-combiner.o
-obj-$(CONFIG_CPU_S5PV310)       += setup-i2c0.o time.o gpiolib.o irq-eint.o
+obj-$(CONFIG_CPU_S5PV310)       += setup-i2c0.o time.o gpiolib.o irq-eint.o dma.o
 
 obj-$(CONFIG_SMP)               += platsmp.o headsmp.o
 obj-$(CONFIG_LOCAL_TIMERS)      += localtimer.o
@@ -27,6 +27,7 @@ obj-$(CONFIG_MACH_UNIVERSAL_C210)	+= mach-universal_c210.o
 
 # device support
 
+obj-y += dev-audio.o
 obj-$(CONFIG_S5PV310_SETUP_I2C1)        += setup-i2c1.o
 obj-$(CONFIG_S5PV310_SETUP_I2C2)        += setup-i2c2.o
 obj-$(CONFIG_S5PV310_SETUP_I2C3)        += setup-i2c3.o
diff --git a/arch/arm/mach-s5pv310/dev-audio.c b/arch/arm/mach-s5pv310/dev-audio.c
new file mode 100644
index 00000000000..a1964242f0f
--- /dev/null
+++ b/arch/arm/mach-s5pv310/dev-audio.c
@@ -0,0 +1,364 @@
+/* linux/arch/arm/mach-s5pv310/dev-audio.c
+ *
+ * Copyright (c) 2010 Samsung Electronics Co. Ltd
+ *      Jaswinder Singh <jassi.brar@samsung.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/platform_device.h>
+#include <linux/dma-mapping.h>
+#include <linux/gpio.h>
+
+#include <plat/gpio-cfg.h>
+#include <plat/audio.h>
+
+#include <mach/map.h>
+#include <mach/dma.h>
+#include <mach/irqs.h>
+
+static const char *rclksrc[] = {
+        [0] = "busclk",
+        [1] = "i2sclk",
+};
+
+static int s5pv310_cfg_i2s(struct platform_device *pdev)
+{
+        /* configure GPIO for i2s port */
+        switch (pdev->id) {
+        case 0:
+                s3c_gpio_cfgpin_range(S5PV310_GPZ(0), 7, S3C_GPIO_SFN(2));
+                break;
+        case 1:
+                s3c_gpio_cfgpin_range(S5PV310_GPC0(0), 5, S3C_GPIO_SFN(2));
+                break;
+        case 2:
+                s3c_gpio_cfgpin_range(S5PV310_GPC1(0), 5, S3C_GPIO_SFN(4));
+                break;
+        default:
+                printk(KERN_ERR "Invalid Device %d\n", pdev->id);
+                return -EINVAL;
+        }
+
+        return 0;
+}
+
+static struct s3c_audio_pdata i2sv5_pdata = {
+        .cfg_gpio = s5pv310_cfg_i2s,
+        .type = {
+                .i2s = {
+                        .quirks = QUIRK_PRI_6CHAN | QUIRK_SEC_DAI
+                                         | QUIRK_NEED_RSTCLR,
+                        .src_clk = rclksrc,
+                },
+        },
+};
+
+static struct resource s5pv310_i2s0_resource[] = {
+        [0] = {
+                .start  = S5PV310_PA_I2S0,
+                .end    = S5PV310_PA_I2S0 + 0x100 - 1,
+                .flags  = IORESOURCE_MEM,
+        },
+        [1] = {
+                .start  = DMACH_I2S0_TX,
+                .end    = DMACH_I2S0_TX,
+                .flags  = IORESOURCE_DMA,
+        },
+        [2] = {
+                .start  = DMACH_I2S0_RX,
+                .end    = DMACH_I2S0_RX,
+                .flags  = IORESOURCE_DMA,
+        },
+        [3] = {
+                .start  = DMACH_I2S0S_TX,
+                .end    = DMACH_I2S0S_TX,
+                .flags  = IORESOURCE_DMA,
+        },
+};
+
+struct platform_device s5pv310_device_i2s0 = {
+        .name = "samsung-i2s",
+        .id = 0,
+        .num_resources = ARRAY_SIZE(s5pv310_i2s0_resource),
+        .resource = s5pv310_i2s0_resource,
+        .dev = {
+                .platform_data = &i2sv5_pdata,
+        },
+};
+
+static const char *rclksrc_v3[] = {
+        [0] = "sclk_i2s",
+        [1] = "no_such_clock",
+};
+
+static struct s3c_audio_pdata i2sv3_pdata = {
+        .cfg_gpio = s5pv310_cfg_i2s,
+        .type = {
+                .i2s = {
+                        .quirks = QUIRK_NO_MUXPSR,
+                        .src_clk = rclksrc_v3,
+                },
+        },
+};
+
+static struct resource s5pv310_i2s1_resource[] = {
+        [0] = {
+                .start  = S5PV310_PA_I2S1,
+                .end    = S5PV310_PA_I2S1 + 0x100 - 1,
+                .flags  = IORESOURCE_MEM,
+        },
+        [1] = {
+                .start  = DMACH_I2S1_TX,
+                .end    = DMACH_I2S1_TX,
+                .flags  = IORESOURCE_DMA,
+        },
+        [2] = {
+                .start  = DMACH_I2S1_RX,
+                .end    = DMACH_I2S1_RX,
+                .flags  = IORESOURCE_DMA,
+        },
+};
+
+struct platform_device s5pv310_device_i2s1 = {
+        .name = "samsung-i2s",
+        .id = 1,
+        .num_resources = ARRAY_SIZE(s5pv310_i2s1_resource),
+        .resource = s5pv310_i2s1_resource,
+        .dev = {
+                .platform_data = &i2sv3_pdata,
+        },
+};
+
+static struct resource s5pv310_i2s2_resource[] = {
+        [0] = {
+                .start  = S5PV310_PA_I2S2,
+                .end    = S5PV310_PA_I2S2 + 0x100 - 1,
+                .flags  = IORESOURCE_MEM,
+        },
+        [1] = {
+                .start  = DMACH_I2S2_TX,
+                .end    = DMACH_I2S2_TX,
+                .flags  = IORESOURCE_DMA,
+        },
+        [2] = {
+                .start  = DMACH_I2S2_RX,
+                .end    = DMACH_I2S2_RX,
+                .flags  = IORESOURCE_DMA,
+        },
+};
+
+struct platform_device s5pv310_device_i2s2 = {
+        .name = "samsung-i2s",
+        .id = 2,
+        .num_resources = ARRAY_SIZE(s5pv310_i2s2_resource),
+        .resource = s5pv310_i2s2_resource,
+        .dev = {
+                .platform_data = &i2sv3_pdata,
+        },
+};
+
+/* PCM Controller platform_devices */
+
+static int s5pv310_pcm_cfg_gpio(struct platform_device *pdev)
+{
+        switch (pdev->id) {
+        case 0:
+                s3c_gpio_cfgpin_range(S5PV310_GPZ(0), 5, S3C_GPIO_SFN(3));
+                break;
+        case 1:
+                s3c_gpio_cfgpin_range(S5PV310_GPC0(0), 5, S3C_GPIO_SFN(3));
+                break;
+        case 2:
+                s3c_gpio_cfgpin_range(S5PV310_GPC1(0), 5, S3C_GPIO_SFN(3));
+                break;
+        default:
+                printk(KERN_DEBUG "Invalid PCM Controller number!");
+                return -EINVAL;
+        }
+
+        return 0;
+}
+
+static struct s3c_audio_pdata s3c_pcm_pdata = {
+        .cfg_gpio = s5pv310_pcm_cfg_gpio,
+};
+
+static struct resource s5pv310_pcm0_resource[] = {
+        [0] = {
+                .start  = S5PV310_PA_PCM0,
+                .end    = S5PV310_PA_PCM0 + 0x100 - 1,
+                .flags  = IORESOURCE_MEM,
+        },
+        [1] = {
+                .start  = DMACH_PCM0_TX,
+                .end    = DMACH_PCM0_TX,
+                .flags  = IORESOURCE_DMA,
+        },
+        [2] = {
+                .start  = DMACH_PCM0_RX,
+                .end    = DMACH_PCM0_RX,
+                .flags  = IORESOURCE_DMA,
+        },
+};
+
+struct platform_device s5pv310_device_pcm0 = {
+        .name = "samsung-pcm",
+        .id = 0,
+        .num_resources = ARRAY_SIZE(s5pv310_pcm0_resource),
+        .resource = s5pv310_pcm0_resource,
+        .dev = {
+                .platform_data = &s3c_pcm_pdata,
+        },
+};
+
+static struct resource s5pv310_pcm1_resource[] = {
+        [0] = {
+                .start  = S5PV310_PA_PCM1,
+                .end    = S5PV310_PA_PCM1 + 0x100 - 1,
+                .flags  = IORESOURCE_MEM,
+        },
+        [1] = {
+                .start  = DMACH_PCM1_TX,
+                .end    = DMACH_PCM1_TX,
+                .flags  = IORESOURCE_DMA,
+        },
+        [2] = {
+                .start  = DMACH_PCM1_RX,
+                .end    = DMACH_PCM1_RX,
+                .flags  = IORESOURCE_DMA,
+        },
+};
+
+struct platform_device s5pv310_device_pcm1 = {
+        .name = "samsung-pcm",
+        .id = 1,
+        .num_resources = ARRAY_SIZE(s5pv310_pcm1_resource),
+        .resource = s5pv310_pcm1_resource,
+        .dev = {
+                .platform_data = &s3c_pcm_pdata,
+        },
+};
+
+static struct resource s5pv310_pcm2_resource[] = {
+        [0] = {
+                .start  = S5PV310_PA_PCM2,
+                .end    = S5PV310_PA_PCM2 + 0x100 - 1,
+                .flags  = IORESOURCE_MEM,
+        },
+        [1] = {
+                .start  = DMACH_PCM2_TX,
+                .end    = DMACH_PCM2_TX,
+                .flags  = IORESOURCE_DMA,
+        },
+        [2] = {
+                .start  = DMACH_PCM2_RX,
+                .end    = DMACH_PCM2_RX,
+                .flags  = IORESOURCE_DMA,
+        },
+};
+
+struct platform_device s5pv310_device_pcm2 = {
+        .name = "samsung-pcm",
+        .id = 2,
+        .num_resources = ARRAY_SIZE(s5pv310_pcm2_resource),
+        .resource = s5pv310_pcm2_resource,
+        .dev = {
+                .platform_data = &s3c_pcm_pdata,
+        },
+};
+
+/* AC97 Controller platform devices */
+
+static int s5pv310_ac97_cfg_gpio(struct platform_device *pdev)
+{
+        return s3c_gpio_cfgpin_range(S5PV310_GPC0(0), 5, S3C_GPIO_SFN(4));
+}
+
+static struct resource s5pv310_ac97_resource[] = {
+        [0] = {
+                .start  = S5PV310_PA_AC97,
+                .end    = S5PV310_PA_AC97 + 0x100 - 1,
+                .flags  = IORESOURCE_MEM,
+        },
+        [1] = {
+                .start  = DMACH_AC97_PCMOUT,
+                .end    = DMACH_AC97_PCMOUT,
+                .flags  = IORESOURCE_DMA,
+        },
+        [2] = {
+                .start  = DMACH_AC97_PCMIN,
+                .end    = DMACH_AC97_PCMIN,
+                .flags  = IORESOURCE_DMA,
+        },
+        [3] = {
+                .start  = DMACH_AC97_MICIN,
+                .end    = DMACH_AC97_MICIN,
+                .flags  = IORESOURCE_DMA,
+        },
+        [4] = {
+                .start  = IRQ_AC97,
+                .end    = IRQ_AC97,
+                .flags  = IORESOURCE_IRQ,
+        },
+};
+
+static struct s3c_audio_pdata s3c_ac97_pdata = {
+        .cfg_gpio = s5pv310_ac97_cfg_gpio,
+};
+
+static u64 s5pv310_ac97_dmamask = DMA_BIT_MASK(32);
+
+struct platform_device s5pv310_device_ac97 = {
+        .name = "samsung-ac97",
+        .id = -1,
+        .num_resources = ARRAY_SIZE(s5pv310_ac97_resource),
+        .resource = s5pv310_ac97_resource,
+        .dev = {
+                .platform_data = &s3c_ac97_pdata,
+                .dma_mask = &s5pv310_ac97_dmamask,
+                .coherent_dma_mask = DMA_BIT_MASK(32),
+        },
+};
+
+/* S/PDIF Controller platform_device */
+
+static int s5pv310_spdif_cfg_gpio(struct platform_device *pdev)
+{
+        s3c_gpio_cfgpin_range(S5PV310_GPC1(0), 2, S3C_GPIO_SFN(3));
+
+        return 0;
+}
+
+static struct resource s5pv310_spdif_resource[] = {
+        [0] = {
+                .start  = S5PV310_PA_SPDIF,
+                .end    = S5PV310_PA_SPDIF + 0x100 - 1,
+                .flags  = IORESOURCE_MEM,
+        },
+        [1] = {
+                .start  = DMACH_SPDIF,
+                .end    = DMACH_SPDIF,
+                .flags  = IORESOURCE_DMA,
+        },
+};
+
+static struct s3c_audio_pdata samsung_spdif_pdata = {
+        .cfg_gpio = s5pv310_spdif_cfg_gpio,
+};
+
+static u64 s5pv310_spdif_dmamask = DMA_BIT_MASK(32);
+
+struct platform_device s5pv310_device_spdif = {
+        .name = "samsung-spdif",
+        .id = -1,
+        .num_resources = ARRAY_SIZE(s5pv310_spdif_resource),
+        .resource = s5pv310_spdif_resource,
+        .dev = {
+                .platform_data = &samsung_spdif_pdata,
+                .dma_mask = &s5pv310_spdif_dmamask,
+                .coherent_dma_mask = DMA_BIT_MASK(32),
+        },
+};
diff --git a/arch/arm/mach-s5pv310/dma.c b/arch/arm/mach-s5pv310/dma.c
new file mode 100644
index 00000000000..20066c7c9e5
--- /dev/null
+++ b/arch/arm/mach-s5pv310/dma.c
@@ -0,0 +1,168 @@
+/*
+ * Copyright (C) 2010 Samsung Electronics Co. Ltd.
+ *      Jaswinder Singh <jassi.brar@samsung.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+#include <linux/platform_device.h>
+#include <linux/dma-mapping.h>
+
+#include <plat/devs.h>
+#include <plat/irqs.h>
+
+#include <mach/map.h>
+#include <mach/irqs.h>
+
+#include <plat/s3c-pl330-pdata.h>
+
+static u64 dma_dmamask = DMA_BIT_MASK(32);
+
+static struct resource s5pv310_pdma0_resource[] = {
+        [0] = {
+                .start  = S5PV310_PA_PDMA0,
+                .end    = S5PV310_PA_PDMA0 + SZ_4K,
+                .flags  = IORESOURCE_MEM,
+        },
+        [1] = {
+                .start  = IRQ_PDMA0,
+                .end    = IRQ_PDMA0,
+                .flags  = IORESOURCE_IRQ,
+        },
+};
+
+static struct s3c_pl330_platdata s5pv310_pdma0_pdata = {
+        .peri = {
+                [0] = DMACH_PCM0_RX,
+                [1] = DMACH_PCM0_TX,
+                [2] = DMACH_PCM2_RX,
+                [3] = DMACH_PCM2_TX,
+                [4] = DMACH_MSM_REQ0,
+                [5] = DMACH_MSM_REQ2,
+                [6] = DMACH_SPI0_RX,
+                [7] = DMACH_SPI0_TX,
+                [8] = DMACH_SPI2_RX,
+                [9] = DMACH_SPI2_TX,
+                [10] = DMACH_I2S0S_TX,
+                [11] = DMACH_I2S0_RX,
+                [12] = DMACH_I2S0_TX,
+                [13] = DMACH_I2S2_RX,
+                [14] = DMACH_I2S2_TX,
+                [15] = DMACH_UART0_RX,
+                [16] = DMACH_UART0_TX,
+                [17] = DMACH_UART2_RX,
+                [18] = DMACH_UART2_TX,
+                [19] = DMACH_UART4_RX,
+                [20] = DMACH_UART4_TX,
+                [21] = DMACH_SLIMBUS0_RX,
+                [22] = DMACH_SLIMBUS0_TX,
+                [23] = DMACH_SLIMBUS2_RX,
+                [24] = DMACH_SLIMBUS2_TX,
+                [25] = DMACH_SLIMBUS4_RX,
+                [26] = DMACH_SLIMBUS4_TX,
+                [27] = DMACH_AC97_MICIN,
+                [28] = DMACH_AC97_PCMIN,
+                [29] = DMACH_AC97_PCMOUT,
+                [30] = DMACH_MAX,
+                [31] = DMACH_MAX,
+        },
+};
+
+static struct platform_device s5pv310_device_pdma0 = {
+        .name           = "s3c-pl330",
+        .id             = 0,
+        .num_resources  = ARRAY_SIZE(s5pv310_pdma0_resource),
+        .resource       = s5pv310_pdma0_resource,
+        .dev            = {
+                .dma_mask = &dma_dmamask,
+                .coherent_dma_mask = DMA_BIT_MASK(32),
+                .platform_data = &s5pv310_pdma0_pdata,
+        },
+};
+
+static struct resource s5pv310_pdma1_resource[] = {
+        [0] = {
+                .start  = S5PV310_PA_PDMA1,
+                .end    = S5PV310_PA_PDMA1 + SZ_4K,
+                .flags  = IORESOURCE_MEM,
+        },
+        [1] = {
+                .start  = IRQ_PDMA1,
+                .end    = IRQ_PDMA1,
+                .flags  = IORESOURCE_IRQ,
+        },
+};
+
+static struct s3c_pl330_platdata s5pv310_pdma1_pdata = {
+        .peri = {
+                [0] = DMACH_PCM0_RX,
+                [1] = DMACH_PCM0_TX,
+                [2] = DMACH_PCM1_RX,
+                [3] = DMACH_PCM1_TX,
+                [4] = DMACH_MSM_REQ1,
+                [5] = DMACH_MSM_REQ3,
+                [6] = DMACH_SPI1_RX,
+                [7] = DMACH_SPI1_TX,
+                [8] = DMACH_I2S0S_TX,
+                [9] = DMACH_I2S0_RX,
+                [10] = DMACH_I2S0_TX,
+                [11] = DMACH_I2S1_RX,
+                [12] = DMACH_I2S1_TX,
+                [13] = DMACH_UART0_RX,
+                [14] = DMACH_UART0_TX,
+                [15] = DMACH_UART1_RX,
+                [16] = DMACH_UART1_TX,
+                [17] = DMACH_UART3_RX,
+                [18] = DMACH_UART3_TX,
+                [19] = DMACH_SLIMBUS1_RX,
+                [20] = DMACH_SLIMBUS1_TX,
+                [21] = DMACH_SLIMBUS3_RX,
+                [22] = DMACH_SLIMBUS3_TX,
+                [23] = DMACH_SLIMBUS5_RX,
+                [24] = DMACH_SLIMBUS5_TX,
+                [25] = DMACH_SLIMBUS0AUX_RX,
+                [26] = DMACH_SLIMBUS0AUX_TX,
+                [27] = DMACH_SPDIF,
+                [28] = DMACH_MAX,
+                [29] = DMACH_MAX,
+                [30] = DMACH_MAX,
+                [31] = DMACH_MAX,
+        },
+};
+
+static struct platform_device s5pv310_device_pdma1 = {
+        .name           = "s3c-pl330",
+        .id             = 1,
+        .num_resources  = ARRAY_SIZE(s5pv310_pdma1_resource),
+        .resource       = s5pv310_pdma1_resource,
+        .dev            = {
+                .dma_mask = &dma_dmamask,
+                .coherent_dma_mask = DMA_BIT_MASK(32),
+                .platform_data = &s5pv310_pdma1_pdata,
+        },
+};
+
+static struct platform_device *s5pv310_dmacs[] __initdata = {
+        &s5pv310_device_pdma0,
+        &s5pv310_device_pdma1,
+};
+
+static int __init s5pv310_dma_init(void)
+{
+        platform_add_devices(s5pv310_dmacs, ARRAY_SIZE(s5pv310_dmacs));
+
+        return 0;
+}
+arch_initcall(s5pv310_dma_init);
diff --git a/arch/arm/mach-s5pv310/include/mach/dma.h b/arch/arm/mach-s5pv310/include/mach/dma.h
new file mode 100644
index 00000000000..81209eb1409
--- /dev/null
+++ b/arch/arm/mach-s5pv310/include/mach/dma.h
@@ -0,0 +1,26 @@
+/*
+ * Copyright (C) 2010 Samsung Electronics Co. Ltd.
+ *      Jaswinder Singh <jassi.brar@samsung.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+#ifndef __MACH_DMA_H
+#define __MACH_DMA_H
+
+/* This platform uses the common S3C DMA API driver for PL330 */
+#include <plat/s3c-dma-pl330.h>
+
+#endif /* __MACH_DMA_H */
diff --git a/arch/arm/mach-s5pv310/include/mach/irqs.h b/arch/arm/mach-s5pv310/include/mach/irqs.h
index 99e7dad8a85..3c05c58b539 100644
--- a/arch/arm/mach-s5pv310/include/mach/irqs.h
+++ b/arch/arm/mach-s5pv310/include/mach/irqs.h
@@ -54,6 +54,9 @@
 #define COMBINER_GROUP(x)       ((x) * MAX_IRQ_IN_COMBINER + IRQ_SPI(64))
 #define COMBINER_IRQ(x, y)      (COMBINER_GROUP(x) + y)
 
+#define IRQ_PDMA0               COMBINER_IRQ(21, 0)
+#define IRQ_PDMA1               COMBINER_IRQ(21, 1)
+
 #define IRQ_TIMER0_VIC          COMBINER_IRQ(22, 0)
 #define IRQ_TIMER1_VIC          COMBINER_IRQ(22, 1)
 #define IRQ_TIMER2_VIC          COMBINER_IRQ(22, 2)
diff --git a/arch/arm/mach-s5pv310/include/mach/map.h b/arch/arm/mach-s5pv310/include/mach/map.h
index 7acf4e77e92..53994467605 100644
--- a/arch/arm/mach-s5pv310/include/mach/map.h
+++ b/arch/arm/mach-s5pv310/include/mach/map.h
@@ -52,6 +52,11 @@
 #define S5PV310_PA_GIC_DIST             (0x10501000)
 #define S5PV310_PA_L2CC                 (0x10502000)
 
+/* DMA */
+#define S5PV310_PA_MDMA         0x10810000
+#define S5PV310_PA_PDMA0        0x12680000
+#define S5PV310_PA_PDMA1        0x12690000
+
 #define S5PV310_PA_GPIO1                (0x11400000)
 #define S5PV310_PA_GPIO2                (0x11000000)
 #define S5PV310_PA_GPIO3                (0x03860000)
@@ -60,6 +65,22 @@
 
 #define S5PV310_PA_SROMC                (0x12570000)
 
+/* S/PDIF */
+#define S5PV310_PA_SPDIF        0xE1100000
+
+/* I2S */
+#define S5PV310_PA_I2S0         0x03830000
+#define S5PV310_PA_I2S1         0xE3100000
+#define S5PV310_PA_I2S2         0xE2A00000
+
+/* PCM */
+#define S5PV310_PA_PCM0         0x03840000
+#define S5PV310_PA_PCM1         0x13980000
+#define S5PV310_PA_PCM2         0x13990000
+
+/* AC97 */
+#define S5PV310_PA_AC97         0x139A0000
+
 #define S5PV310_PA_UART                 (0x13800000)
 
 #define S5P_PA_UART(x)                  (S5PV310_PA_UART + ((x) * S3C_UART_OFFSET))
diff --git a/arch/arm/mach-sa1100/pm.c b/arch/arm/mach-sa1100/pm.c
index c83fdc80edf..ab9fc4470d3 100644
--- a/arch/arm/mach-sa1100/pm.c
+++ b/arch/arm/mach-sa1100/pm.c
@@ -120,7 +120,7 @@ unsigned long sleep_phys_sp(void *sp)
         return virt_to_phys(sp);
 }
 
-static struct platform_suspend_ops sa11x0_pm_ops = {
+static const struct platform_suspend_ops sa11x0_pm_ops = {
         .enter          = sa11x0_pm_enter,
         .valid          = suspend_valid_only_mem,
 };
diff --git a/arch/arm/mach-shmobile/board-ap4evb.c b/arch/arm/mach-shmobile/board-ap4evb.c
index cd79d7c1ba0..5b9937c910f 100644
--- a/arch/arm/mach-shmobile/board-ap4evb.c
+++ b/arch/arm/mach-shmobile/board-ap4evb.c
@@ -711,6 +711,10 @@ static struct platform_device fsi_device = {
         },
 };
 
+static struct platform_device fsi_ak4643_device = {
+        .name           = "sh_fsi2_a_ak4643",
+};
+
 static struct sh_mobile_lcdc_info sh_mobile_lcdc1_info = {
         .clock_source = LCDC_CLK_EXTERNAL,
         .ch[0] = {
@@ -933,6 +937,7 @@ static struct platform_device *ap4evb_devices[] __initdata = {
         &sdhi1_device,
         &usb1_host_device,
         &fsi_device,
+        &fsi_ak4643_device,
         &sh_mmcif_device,
         &lcdc1_device,
         &lcdc_device,
diff --git a/arch/arm/mach-spear3xx/spear300.c b/arch/arm/mach-spear3xx/spear300.c
index 3560f8c1e72..5aa2d54ebfa 100644
--- a/arch/arm/mach-spear3xx/spear300.c
+++ b/arch/arm/mach-spear3xx/spear300.c
@@ -371,7 +371,7 @@ struct pmx_driver pmx_driver = {
 };
 
 /* Add spear300 specific devices here */
-/* arm gpio1 device registeration */
+/* arm gpio1 device registration */
 static struct pl061_platform_data gpio1_plat_data = {
         .gpio_base      = 8,
         .irq_base       = SPEAR_GPIO1_INT_BASE,
@@ -451,7 +451,7 @@ void __init spear300_init(void)
         /* call spear3xx family common init function */
         spear3xx_init();
 
-        /* shared irq registeration */
+        /* shared irq registration */
         shirq_ras1.regs.base =
                 ioremap(SPEAR300_TELECOM_BASE, SPEAR300_TELECOM_REG_SIZE);
         if (shirq_ras1.regs.base) {
diff --git a/arch/arm/mach-spear3xx/spear310.c b/arch/arm/mach-spear3xx/spear310.c
index 96a1ab824ba..53b41b52d7e 100644
--- a/arch/arm/mach-spear3xx/spear310.c
+++ b/arch/arm/mach-spear3xx/spear310.c
@@ -266,7 +266,7 @@ void __init spear310_init(void)
         /* call spear3xx family common init function */
         spear3xx_init();
 
-        /* shared irq registeration */
+        /* shared irq registration */
         base = ioremap(SPEAR310_SOC_CONFIG_BASE, SPEAR310_SOC_CONFIG_SIZE);
         if (base) {
                 /* shirq 1 */
diff --git a/arch/arm/mach-spear3xx/spear320.c b/arch/arm/mach-spear3xx/spear320.c
index 6a121954936..88b465284c3 100644
--- a/arch/arm/mach-spear3xx/spear320.c
+++ b/arch/arm/mach-spear3xx/spear320.c
@@ -519,7 +519,7 @@ void __init spear320_init(void)
         /* call spear3xx family common init function */
         spear3xx_init();
 
-        /* shared irq registeration */
+        /* shared irq registration */
         base = ioremap(SPEAR320_SOC_CONFIG_BASE, SPEAR320_SOC_CONFIG_SIZE);
         if (base) {
                 /* shirq 1 */
diff --git a/arch/arm/mach-spear3xx/spear3xx.c b/arch/arm/mach-spear3xx/spear3xx.c
index e87313aeae2..52f553c8c46 100644
--- a/arch/arm/mach-spear3xx/spear3xx.c
+++ b/arch/arm/mach-spear3xx/spear3xx.c
@@ -22,7 +22,7 @@
 #include <mach/spear.h>
 
 /* Add spear3xx machines common devices here */
-/* gpio device registeration */
+/* gpio device registration */
 static struct pl061_platform_data gpio_plat_data = {
         .gpio_base      = 0,
         .irq_base       = SPEAR_GPIO_INT_BASE,
@@ -41,7 +41,7 @@ struct amba_device gpio_device = {
         .irq = {IRQ_BASIC_GPIO, NO_IRQ},
 };
 
-/* uart device registeration */
+/* uart device registration */
 struct amba_device uart_device = {
         .dev = {
                 .init_name = "uart",
@@ -543,6 +543,6 @@ void spear_pmx_init(struct pmx_driver *pmx_driver, uint base, uint size)
 
 pmx_fail:
         if (ret)
-                printk(KERN_ERR "padmux: registeration failed. err no: %d\n",
+                printk(KERN_ERR "padmux: registration failed. err no: %d\n",
                                 ret);
 }
diff --git a/arch/arm/mach-spear6xx/spear6xx.c b/arch/arm/mach-spear6xx/spear6xx.c
index baf6bcc3169..f2fe14e8471 100644
--- a/arch/arm/mach-spear6xx/spear6xx.c
+++ b/arch/arm/mach-spear6xx/spear6xx.c
@@ -23,7 +23,7 @@
 #include <mach/spear.h>
 
 /* Add spear6xx machines common devices here */
-/* uart device registeration */
+/* uart device registration */
 struct amba_device uart_device[] = {
         {
                 .dev = {
@@ -50,7 +50,7 @@ struct amba_device uart_device[] = {
         }
 };
 
-/* gpio device registeration */
+/* gpio device registration */
 static struct pl061_platform_data gpio_plat_data[] = {
         {
                 .gpio_base      = 0,
diff --git a/arch/arm/mach-u300/Kconfig b/arch/arm/mach-u300/Kconfig
index 801b21e7f67..32a7b0f7e9f 100644
--- a/arch/arm/mach-u300/Kconfig
+++ b/arch/arm/mach-u300/Kconfig
@@ -64,7 +64,7 @@ config MACH_U300_DUAL_RAM
         bool "Dual RAM"
         help
                 Select this if you want support for Dual RAM phones.
-                This is two RAM memorys on different EMIFs.
+                This is two RAM memories on different EMIFs.
 endchoice
 
 config U300_DEBUG
diff --git a/arch/arm/mach-u300/include/mach/coh901318.h b/arch/arm/mach-u300/include/mach/coh901318.h
index 193da2df732..6193aaa4779 100644
--- a/arch/arm/mach-u300/include/mach/coh901318.h
+++ b/arch/arm/mach-u300/include/mach/coh901318.h
@@ -24,7 +24,7 @@
  * @src_addr: transfer source address
  * @dst_addr: transfer destination address
  * @link_addr:  physical address to next lli
- * @virt_link_addr: virtual addres of next lli (only used by pool_free)
+ * @virt_link_addr: virtual address of next lli (only used by pool_free)
  * @phy_this: physical address of current lli (only used by pool_free)
  */
 struct coh901318_lli {
@@ -90,7 +90,7 @@ struct powersave {
  * struct coh901318_platform - platform arch structure
  * @chans_slave: specifying dma slave channels
  * @chans_memcpy: specifying dma memcpy channels
- * @access_memory_state: requesting DMA memeory access (on / off)
+ * @access_memory_state: requesting DMA memory access (on / off)
  * @chan_conf: dma channel configurations
  * @max_channels: max number of dma chanenls
  */
diff --git a/arch/arm/mach-ux500/board-mop500-regulators.c b/arch/arm/mach-ux500/board-mop500-regulators.c
index 1187f1fc2e5..533967c2d09 100644
--- a/arch/arm/mach-ux500/board-mop500-regulators.c
+++ b/arch/arm/mach-ux500/board-mop500-regulators.c
@@ -3,99 +3,94 @@
  *
  * License Terms: GNU General Public License v2
  *
- * Author: Sundar Iyer <sundar.iyer@stericsson.com>
+ * Authors: Sundar Iyer <sundar.iyer@stericsson.com>
+ *          Bengt Jonsson <bengt.g.jonsson@stericsson.com>
  *
  * MOP500 board specific initialization for regulators
  */
 #include <linux/kernel.h>
 #include <linux/regulator/machine.h>
+#include <linux/regulator/ab8500.h>
 
-/* supplies to the display/camera */
-static struct regulator_init_data ab8500_vaux1_regulator = {
-        .constraints = {
-                .name = "V-DISPLAY",
-                .min_uV = 2500000,
-                .max_uV = 2900000,
-                .valid_ops_mask = REGULATOR_CHANGE_VOLTAGE|
-                                        REGULATOR_CHANGE_STATUS,
+/* AB8500 regulators */
+struct regulator_init_data ab8500_regulators[AB8500_NUM_REGULATORS] = {
+        /* supplies to the display/camera */
+        [AB8500_LDO_AUX1] = {
+                .constraints = {
+                        .name = "V-DISPLAY",
+                        .min_uV = 2500000,
+                        .max_uV = 2900000,
+                        .valid_ops_mask = REGULATOR_CHANGE_VOLTAGE |
+                                          REGULATOR_CHANGE_STATUS,
+                },
         },
-};
-
-/* supplies to the on-board eMMC */
-static struct regulator_init_data ab8500_vaux2_regulator = {
-        .constraints = {
-                .name = "V-eMMC1",
-                .min_uV = 1100000,
-                .max_uV = 3300000,
-                .valid_ops_mask = REGULATOR_CHANGE_VOLTAGE|
-                                        REGULATOR_CHANGE_STATUS,
+        /* supplies to the on-board eMMC */
+        [AB8500_LDO_AUX2] = {
+                .constraints = {
+                        .name = "V-eMMC1",
+                        .min_uV = 1100000,
+                        .max_uV = 3300000,
+                        .valid_ops_mask = REGULATOR_CHANGE_VOLTAGE |
+                                          REGULATOR_CHANGE_STATUS,
+                },
         },
-};
-
-/* supply for VAUX3, supplies to SDcard slots */
-static struct regulator_init_data ab8500_vaux3_regulator = {
-        .constraints = {
-                .name = "V-MMC-SD",
-                .min_uV = 1100000,
-                .max_uV = 3300000,
-                .valid_ops_mask = REGULATOR_CHANGE_VOLTAGE|
-                                        REGULATOR_CHANGE_STATUS,
+        /* supply for VAUX3, supplies to SDcard slots */
+        [AB8500_LDO_AUX3] = {
+                .constraints = {
+                        .name = "V-MMC-SD",
+                        .min_uV = 1100000,
+                        .max_uV = 3300000,
+                        .valid_ops_mask = REGULATOR_CHANGE_VOLTAGE |
+                                          REGULATOR_CHANGE_STATUS,
+                },
         },
-};
-
-/* supply for tvout, gpadc, TVOUT LDO */
-static struct regulator_init_data ab8500_vtvout_init = {
-        .constraints = {
-                .name = "V-TVOUT",
-                .valid_ops_mask = REGULATOR_CHANGE_STATUS,
+        /* supply for tvout, gpadc, TVOUT LDO */
+        [AB8500_LDO_TVOUT] = {
+                .constraints = {
+                        .name = "V-TVOUT",
+                        .valid_ops_mask = REGULATOR_CHANGE_STATUS,
+                },
         },
-};
-
-/* supply for ab8500-vaudio, VAUDIO LDO */
-static struct regulator_init_data ab8500_vaudio_init = {
-        .constraints = {
-                .name = "V-AUD",
-                .valid_ops_mask = REGULATOR_CHANGE_STATUS,
+        /* supply for ab8500-vaudio, VAUDIO LDO */
+        [AB8500_LDO_AUDIO] = {
+                .constraints = {
+                        .name = "V-AUD",
+                        .valid_ops_mask = REGULATOR_CHANGE_STATUS,
+                },
         },
-};
-
-/* supply for v-anamic1 VAMic1-LDO */
-static struct regulator_init_data ab8500_vamic1_init = {
-        .constraints = {
-                .name = "V-AMIC1",
-                .valid_ops_mask = REGULATOR_CHANGE_STATUS,
+        /* supply for v-anamic1 VAMic1-LDO */
+        [AB8500_LDO_ANAMIC1] = {
+                .constraints = {
+                        .name = "V-AMIC1",
+                        .valid_ops_mask = REGULATOR_CHANGE_STATUS,
+                },
         },
-};
-
-/* supply for v-amic2, VAMIC2 LDO, reuse constants for AMIC1 */
-static struct regulator_init_data ab8500_vamic2_init = {
-        .constraints = {
-                .name = "V-AMIC2",
-                .valid_ops_mask = REGULATOR_CHANGE_STATUS,
+        /* supply for v-amic2, VAMIC2 LDO, reuse constants for AMIC1 */
+        [AB8500_LDO_ANAMIC2] = {
+                .constraints = {
+                        .name = "V-AMIC2",
+                        .valid_ops_mask = REGULATOR_CHANGE_STATUS,
+                },
         },
-};
-
-/* supply for v-dmic, VDMIC LDO */
-static struct regulator_init_data ab8500_vdmic_init = {
-        .constraints = {
-                .name = "V-DMIC",
-                .valid_ops_mask = REGULATOR_CHANGE_STATUS,
+        /* supply for v-dmic, VDMIC LDO */
+        [AB8500_LDO_DMIC] = {
+                .constraints = {
+                        .name = "V-DMIC",
+                        .valid_ops_mask = REGULATOR_CHANGE_STATUS,
+                },
         },
-};
-
-/* supply for v-intcore12, VINTCORE12 LDO */
-static struct regulator_init_data ab8500_vintcore_init = {
-        .constraints = {
-                .name = "V-INTCORE",
-                .valid_ops_mask = REGULATOR_CHANGE_STATUS,
+        /* supply for v-intcore12, VINTCORE12 LDO */
+        [AB8500_LDO_INTCORE] = {
+                .constraints = {
+                        .name = "V-INTCORE",
+                        .valid_ops_mask = REGULATOR_CHANGE_STATUS,
+                },
         },
-};
-
-/* supply for U8500 CSI/DSI, VANA LDO */
-static struct regulator_init_data ab8500_vana_init = {
-        .constraints = {
-                .name = "V-CSI/DSI",
-                .valid_ops_mask = REGULATOR_CHANGE_STATUS,
+        /* supply for U8500 CSI/DSI, VANA LDO */
+        [AB8500_LDO_ANA] = {
+                .constraints = {
+                        .name = "V-CSI/DSI",
+                        .valid_ops_mask = REGULATOR_CHANGE_STATUS,
+                },
         },
 };
-
diff --git a/arch/arm/mach-ux500/board-mop500-regulators.h b/arch/arm/mach-ux500/board-mop500-regulators.h
new file mode 100644
index 00000000000..2675fae5253
--- /dev/null
+++ b/arch/arm/mach-ux500/board-mop500-regulators.h
@@ -0,0 +1,19 @@
+/*
+ * Copyright (C) ST-Ericsson SA 2010
+ *
+ * License Terms: GNU General Public License v2
+ *
+ * Author: Bengt Jonsson <bengt.g.jonsson@stericsson.com> for ST-Ericsson
+ *
+ * MOP500 board specific initialization for regulators
+ */
+
+#ifndef __BOARD_MOP500_REGULATORS_H
+#define __BOARD_MOP500_REGULATORS_H
+
+#include <linux/regulator/machine.h>
+#include <linux/regulator/ab8500.h>
+
+extern struct regulator_init_data ab8500_regulators[AB8500_NUM_REGULATORS];
+
+#endif
diff --git a/arch/arm/mach-ux500/board-mop500.c b/arch/arm/mach-ux500/board-mop500.c
index a1c9ea1a66d..a393f57ed2a 100644
--- a/arch/arm/mach-ux500/board-mop500.c
+++ b/arch/arm/mach-ux500/board-mop500.c
@@ -35,6 +35,7 @@
 #include "devices-db8500.h"
 #include "pins-db8500.h"
 #include "board-mop500.h"
+#include "board-mop500-regulators.h"
 
 static pin_cfg_t mop500_pins[] = {
         /* SSP0 */
@@ -80,6 +81,8 @@ static pin_cfg_t mop500_pins[] = {
 
 static struct ab8500_platform_data ab8500_platdata = {
         .irq_base       = MOP500_AB8500_IRQ_BASE,
+        .regulator      = ab8500_regulators,
+        .num_regulator  = ARRAY_SIZE(ab8500_regulators),
 };
 
 static struct resource ab8500_resources[] = {
diff --git a/arch/arm/mm/flush.c b/arch/arm/mm/flush.c
index c29f2839f1d..2b269c95552 100644
--- a/arch/arm/mm/flush.c
+++ b/arch/arm/mm/flush.c
@@ -18,7 +18,6 @@
 #include <asm/smp_plat.h>
 #include <asm/system.h>
 #include <asm/tlbflush.h>
-#include <asm/smp_plat.h>
 
 #include "mm.h"
 
diff --git a/arch/arm/plat-mxc/include/mach/irqs.h b/arch/arm/plat-mxc/include/mach/irqs.h
index 58a49cc8379..ba65c9231a7 100644
--- a/arch/arm/plat-mxc/include/mach/irqs.h
+++ b/arch/arm/plat-mxc/include/mach/irqs.h
@@ -70,7 +70,7 @@ extern int imx_irq_set_priority(unsigned char irq, unsigned char prio);
 
 /* all normal IRQs can be FIQs */
 #define FIQ_START       0
-/* switch betwean IRQ and FIQ */
+/* switch between IRQ and FIQ */
 extern int mxc_set_irq_fiq(unsigned int irq, unsigned int type);
 
 #endif /* __ASM_ARCH_MXC_IRQS_H__ */
diff --git a/arch/arm/plat-omap/include/plat/display.h b/arch/arm/plat-omap/include/plat/display.h
index c915a661f1f..537f4e449f5 100644
--- a/arch/arm/plat-omap/include/plat/display.h
+++ b/arch/arm/plat-omap/include/plat/display.h
@@ -42,6 +42,10 @@
 #define DISPC_IRQ_SYNC_LOST             (1 << 14)
 #define DISPC_IRQ_SYNC_LOST_DIGIT       (1 << 15)
 #define DISPC_IRQ_WAKEUP                (1 << 16)
+#define DISPC_IRQ_SYNC_LOST2            (1 << 17)
+#define DISPC_IRQ_VSYNC2                (1 << 18)
+#define DISPC_IRQ_ACBIAS_COUNT_STAT2    (1 << 21)
+#define DISPC_IRQ_FRAMEDONE2            (1 << 22)
 
 struct omap_dss_device;
 struct omap_overlay_manager;
@@ -64,6 +68,7 @@ enum omap_plane {
 enum omap_channel {
         OMAP_DSS_CHANNEL_LCD    = 0,
         OMAP_DSS_CHANNEL_DIGIT  = 1,
+        OMAP_DSS_CHANNEL_LCD2   = 2,
 };
 
 enum omap_color_mode {
@@ -142,6 +147,7 @@ enum omap_dss_display_state {
 enum omap_dss_overlay_managers {
         OMAP_DSS_OVL_MGR_LCD,
         OMAP_DSS_OVL_MGR_TV,
+        OMAP_DSS_OVL_MGR_LCD2,
 };
 
 enum omap_dss_rotation_type {
@@ -268,6 +274,7 @@ struct omap_overlay_info {
         u16 out_width;  /* if 0, out_width == width */
         u16 out_height; /* if 0, out_height == height */
         u8 global_alpha;
+        u8 pre_mult_alpha;
 };
 
 struct omap_overlay {
@@ -351,6 +358,8 @@ struct omap_dss_device {
 
         enum omap_display_type type;
 
+        enum omap_channel channel;
+
         union {
                 struct {
                         u8 data_lines;
diff --git a/arch/arm/plat-omap/include/plat/omap_hwmod.h b/arch/arm/plat-omap/include/plat/omap_hwmod.h
index 6864a997f2c..1eee85a8abb 100644
--- a/arch/arm/plat-omap/include/plat/omap_hwmod.h
+++ b/arch/arm/plat-omap/include/plat/omap_hwmod.h
@@ -351,7 +351,7 @@ struct omap_hwmod_omap2_prcm {
 /**
  * struct omap_hwmod_omap4_prcm - OMAP4-specific PRCM data
  * @clkctrl_reg: PRCM address of the clock control register
- * @rstctrl_reg: adress of the XXX_RSTCTRL register located in the PRM
+ * @rstctrl_reg: address of the XXX_RSTCTRL register located in the PRM
  * @submodule_wkdep_bit: bit shift of the WKDEP range
  */
 struct omap_hwmod_omap4_prcm {
diff --git a/arch/arm/plat-omap/include/plat/panel-generic-dpi.h b/arch/arm/plat-omap/include/plat/panel-generic-dpi.h
new file mode 100644
index 00000000000..790619734bc
--- /dev/null
+++ b/arch/arm/plat-omap/include/plat/panel-generic-dpi.h
@@ -0,0 +1,37 @@
+/*
+ * Header for generic DPI panel driver
+ *
+ * Copyright (C) 2010 Canonical Ltd.
+ * Author: Bryan Wu <bryan.wu@canonical.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __ARCH_ARM_PLAT_OMAP_PANEL_GENERIC_DPI_H
+#define __ARCH_ARM_PLAT_OMAP_PANEL_GENERIC_DPI_H
+
+#include "display.h"
+
+/**
+ * struct panel_generic_dpi_data - panel driver configuration data
+ * @name: panel name
+ * @platform_enable: platform specific panel enable function
+ * @platform_disable: platform specific panel disable function
+ */
+struct panel_generic_dpi_data {
+        const char *name;
+        int (*platform_enable)(struct omap_dss_device *dssdev);
+        void (*platform_disable)(struct omap_dss_device *dssdev);
+};
+
+#endif /* __ARCH_ARM_PLAT_OMAP_PANEL_GENERIC_DPI_H */
diff --git a/arch/arm/plat-s3c24xx/devs.c b/arch/arm/plat-s3c24xx/devs.c
index 2f91057a0c0..8a42bc48dbf 100644
--- a/arch/arm/plat-s3c24xx/devs.c
+++ b/arch/arm/plat-s3c24xx/devs.c
@@ -259,21 +259,6 @@ struct platform_device s3c_device_iis = {
 
 EXPORT_SYMBOL(s3c_device_iis);
 
-/* ASoC PCM DMA */
-
-static u64 s3c_device_audio_dmamask = 0xffffffffUL;
-
-struct platform_device s3c_device_pcm = {
-        .name             = "s3c24xx-pcm-audio",
-        .id               = -1,
-        .dev              = {
-                .dma_mask = &s3c_device_audio_dmamask,
-                .coherent_dma_mask = 0xffffffffUL
-        }
-};
-
-EXPORT_SYMBOL(s3c_device_pcm);
-
 /* RTC */
 
 static struct resource s3c_rtc_resource[] = {
@@ -496,8 +481,10 @@ static struct resource s3c_ac97_resource[] = {
         },
 };
 
+static u64 s3c_device_audio_dmamask = 0xffffffffUL;
+
 struct platform_device s3c_device_ac97 = {
-        .name             = "s3c-ac97",
+        .name             = "samsung-ac97",
         .id               = -1,
         .num_resources    = ARRAY_SIZE(s3c_ac97_resource),
         .resource         = s3c_ac97_resource,
diff --git a/arch/arm/plat-samsung/Makefile b/arch/arm/plat-samsung/Makefile
index afcce474af8..19d8a16c306 100644
--- a/arch/arm/plat-samsung/Makefile
+++ b/arch/arm/plat-samsung/Makefile
@@ -17,6 +17,7 @@ obj-y				+= clock.o
 obj-y                           += pwm-clock.o
 obj-y                           += gpio.o
 obj-y                           += gpio-config.o
+obj-y                           += dev-asocdma.o
 
 obj-$(CONFIG_SAMSUNG_GPIOLIB_4BIT)      += gpiolib.o
 obj-$(CONFIG_SAMSUNG_CLKSRC)    += clock-clksrc.o
diff --git a/arch/arm/plat-samsung/dev-asocdma.c b/arch/arm/plat-samsung/dev-asocdma.c
new file mode 100644
index 00000000000..a068c4f42d5
--- /dev/null
+++ b/arch/arm/plat-samsung/dev-asocdma.c
@@ -0,0 +1,25 @@
+/* linux/arch/arm/plat-samsung/dev-asocdma.c
+ *
+ * Copyright (c) 2010 Samsung Electronics Co. Ltd
+ *      Jaswinder Singh <jassi.brar@samsung.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/platform_device.h>
+#include <linux/dma-mapping.h>
+#include <plat/devs.h>
+
+static u64 audio_dmamask = DMA_BIT_MASK(32);
+
+struct platform_device samsung_asoc_dma = {
+        .name             = "samsung-audio",
+        .id               = -1,
+        .dev              = {
+                .dma_mask = &audio_dmamask,
+                .coherent_dma_mask = DMA_BIT_MASK(32),
+        }
+};
+EXPORT_SYMBOL(samsung_asoc_dma);
diff --git a/arch/arm/plat-samsung/include/plat/audio.h b/arch/arm/plat-samsung/include/plat/audio.h
index 7712ff6336f..a0826ed2f9f 100644
--- a/arch/arm/plat-samsung/include/plat/audio.h
+++ b/arch/arm/plat-samsung/include/plat/audio.h
@@ -25,10 +25,34 @@ extern void s3c64xx_ac97_setup_gpio(int);
 #define S5PC100_SPDIF_GPG3 1
 extern void s5pc100_spdif_setup_gpio(int);
 
+struct samsung_i2s {
+/* If the Primary DAI has 5.1 Channels */
+#define QUIRK_PRI_6CHAN         (1 << 0)
+/* If the I2S block has a Stereo Overlay Channel */
+#define QUIRK_SEC_DAI           (1 << 1)
+/*
+ * If the I2S block has no internal prescalar or MUX (I2SMOD[10] bit)
+ * The Machine driver must provide suitably set clock to the I2S block.
+ */
+#define QUIRK_NO_MUXPSR         (1 << 2)
+#define QUIRK_NEED_RSTCLR       (1 << 3)
+        /* Quirks of the I2S controller */
+        u32 quirks;
+
+        /*
+         * Array of clock names that can be used to generate I2S signals.
+         * Also corresponds to clocks of I2SMOD[10]
+         */
+        const char **src_clk;
+};
+
 /**
  * struct s3c_audio_pdata - common platform data for audio device drivers
  * @cfg_gpio: Callback function to setup mux'ed pins in I2S/PCM/AC97 mode
  */
 struct s3c_audio_pdata {
         int (*cfg_gpio)(struct platform_device *);
+        union {
+                struct samsung_i2s i2s;
+        } type;
 };
diff --git a/arch/arm/plat-samsung/include/plat/devs.h b/arch/arm/plat-samsung/include/plat/devs.h
index 2d82a6cb144..e9e3b6e3ec7 100644
--- a/arch/arm/plat-samsung/include/plat/devs.h
+++ b/arch/arm/plat-samsung/include/plat/devs.h
@@ -32,7 +32,7 @@ extern struct platform_device s3c64xx_device_iisv4;
 extern struct platform_device s3c64xx_device_spi0;
 extern struct platform_device s3c64xx_device_spi1;
 
-extern struct platform_device s3c_device_pcm;
+extern struct platform_device samsung_asoc_dma;
 
 extern struct platform_device s3c64xx_device_pcm0;
 extern struct platform_device s3c64xx_device_pcm1;
@@ -96,6 +96,15 @@ extern struct platform_device s5pv210_device_iis1;
 extern struct platform_device s5pv210_device_iis2;
 extern struct platform_device s5pv210_device_spdif;
 
+extern struct platform_device s5pv310_device_ac97;
+extern struct platform_device s5pv310_device_pcm0;
+extern struct platform_device s5pv310_device_pcm1;
+extern struct platform_device s5pv310_device_pcm2;
+extern struct platform_device s5pv310_device_i2s0;
+extern struct platform_device s5pv310_device_i2s1;
+extern struct platform_device s5pv310_device_i2s2;
+extern struct platform_device s5pv310_device_spdif;
+
 extern struct platform_device s5p6442_device_pcm0;
 extern struct platform_device s5p6442_device_pcm1;
 extern struct platform_device s5p6442_device_iis0;
diff --git a/arch/arm/plat-samsung/pm.c b/arch/arm/plat-samsung/pm.c
index 27cfca59769..5bf3f2f09e7 100644
--- a/arch/arm/plat-samsung/pm.c
+++ b/arch/arm/plat-samsung/pm.c
@@ -355,7 +355,7 @@ static void s3c_pm_finish(void)
         s3c_pm_check_cleanup();
 }
 
-static struct platform_suspend_ops s3c_pm_ops = {
+static const struct platform_suspend_ops s3c_pm_ops = {
         .enter          = s3c_pm_enter,
         .prepare        = s3c_pm_prepare,
         .finish         = s3c_pm_finish,
diff --git a/arch/avr32/mach-at32ap/pm.c b/arch/avr32/mach-at32ap/pm.c
index f021edfeaab..32d680eb6f4 100644
--- a/arch/avr32/mach-at32ap/pm.c
+++ b/arch/avr32/mach-at32ap/pm.c
@@ -176,7 +176,7 @@ out:
         return 0;
 }
 
-static struct platform_suspend_ops avr32_pm_ops = {
+static const struct platform_suspend_ops avr32_pm_ops = {
         .valid  = avr32_pm_valid_state,
         .enter  = avr32_pm_enter,
 };
diff --git a/arch/blackfin/mach-bf537/include/mach/defBF534.h b/arch/blackfin/mach-bf537/include/mach/defBF534.h
index 32529a03b26..725bb35f3aa 100644
--- a/arch/blackfin/mach-bf537/include/mach/defBF534.h
+++ b/arch/blackfin/mach-bf537/include/mach/defBF534.h
@@ -1418,7 +1418,7 @@
 #define SADD_LEN        0x0002  /* Slave Address Length                                                 */
 #define STDVAL          0x0004  /* Slave Transmit Data Valid                                    */
 #define NAK                     0x0008  /* NAK/ACK* Generated At Conclusion Of Transfer */
-#define GEN                     0x0010  /* General Call Adrress Matching Enabled                */
+#define GEN                     0x0010  /* General Call Address Matching Enabled                */
 
 /* TWI_SLAVE_STAT Masks                                                                                                                 */
 #define SDIR            0x0001  /* Slave Transfer Direction (Transmit/Receive*) */
diff --git a/arch/blackfin/mach-common/pm.c b/arch/blackfin/mach-common/pm.c
index 42fa87e8375..3c648a077e7 100644
--- a/arch/blackfin/mach-common/pm.c
+++ b/arch/blackfin/mach-common/pm.c
@@ -223,7 +223,7 @@ static int bfin_pm_enter(suspend_state_t state)
         return 0;
 }
 
-struct platform_suspend_ops bfin_pm_ops = {
+static const struct platform_suspend_ops bfin_pm_ops = {
         .enter = bfin_pm_enter,
         .valid  = bfin_pm_valid,
 };
diff --git a/arch/cris/arch-v32/lib/nand_init.S b/arch/cris/arch-v32/lib/nand_init.S
index e705f5cce96..d671fed451c 100644
--- a/arch/cris/arch-v32/lib/nand_init.S
+++ b/arch/cris/arch-v32/lib/nand_init.S
@@ -139,7 +139,7 @@ copy_nand_to_ram:
         lsrq    8, $r4
         move.b  $r4, [$r1]      ; Row address
         lsrq    8, $r4
-        move.b  $r4, [$r1]      ; Row adddress
+        move.b  $r4, [$r1]      ; Row address
         moveq   20, $r4
 2:      bne     2b
         subq    1, $r4
diff --git a/arch/cris/include/asm/etraxgpio.h b/arch/cris/include/asm/etraxgpio.h
index d474818a537..461c089db76 100644
--- a/arch/cris/include/asm/etraxgpio.h
+++ b/arch/cris/include/asm/etraxgpio.h
@@ -1,5 +1,5 @@
 /*
- * The following devices are accessable using this driver using
+ * The following devices are accessible using this driver using
  * GPIO_MAJOR (120) and a couple of minor numbers.
  *
  * For ETRAX 100LX (CONFIG_ETRAX_ARCH_V10):
diff --git a/arch/cris/kernel/time.c b/arch/cris/kernel/time.c
index b5096430ce1..4e73092e85c 100644
--- a/arch/cris/kernel/time.c
+++ b/arch/cris/kernel/time.c
@@ -86,7 +86,7 @@ int set_rtc_mmss(unsigned long nowtime)
                 CMOS_WRITE(real_seconds,RTC_SECONDS);
                 CMOS_WRITE(real_minutes,RTC_MINUTES);
         } else {
-                printk(KERN_WARNING
+                printk_once(KERN_NOTICE
                        "set_rtc_mmss: can't update from %d to %d\n",
                        cmos_minutes, real_minutes);
                 retval = -1;
diff --git a/arch/h8300/Kconfig.debug b/arch/h8300/Kconfig.debug
index ee671c3f2c7..e8d1b236ad8 100644
--- a/arch/h8300/Kconfig.debug
+++ b/arch/h8300/Kconfig.debug
@@ -48,7 +48,7 @@ config DEFAULT_CMDLINE
           builtin kernel commandline enabled.
 
 config KERNEL_COMMAND
-        string "Buildin commmand string"
+        string "Buildin command string"
         depends on DEFAULT_CMDLINE
         help
           builtin kernel commandline strings.
diff --git a/arch/ia64/include/asm/kvm_host.h b/arch/ia64/include/asm/kvm_host.h
index 2f229e5de49..2689ee54a1c 100644
--- a/arch/ia64/include/asm/kvm_host.h
+++ b/arch/ia64/include/asm/kvm_host.h
@@ -590,6 +590,10 @@ int kvm_emulate_halt(struct kvm_vcpu *vcpu);
 int kvm_pal_emul(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run);
 void kvm_sal_emul(struct kvm_vcpu *vcpu);
 
+#define __KVM_HAVE_ARCH_VM_ALLOC 1
+struct kvm *kvm_arch_alloc_vm(void);
+void kvm_arch_free_vm(struct kvm *kvm);
+
 #endif /* __ASSEMBLY__*/
 
 #endif
diff --git a/arch/ia64/kernel/perfmon.c b/arch/ia64/kernel/perfmon.c
index d92d5b5161f..ac76da099a6 100644
--- a/arch/ia64/kernel/perfmon.c
+++ b/arch/ia64/kernel/perfmon.c
@@ -617,11 +617,14 @@ pfm_get_unmapped_area(struct file *file, unsigned long addr, unsigned long len,
         return get_unmapped_area(file, addr, len, pgoff, flags);
 }
 
+/* forward declaration */
+static static const struct dentry_operations pfmfs_dentry_operations;
 
 static struct dentry *
 pfmfs_mount(struct file_system_type *fs_type, int flags, const char *dev_name, void *data)
 {
-        return mount_pseudo(fs_type, "pfm:", NULL, PFMFS_MAGIC);
+        return mount_pseudo(fs_type, "pfm:", NULL, &pfmfs_dentry_operations,
+                        PFMFS_MAGIC);
 }
 
 static struct file_system_type pfm_fs_type = {
@@ -2232,7 +2235,6 @@ pfm_alloc_file(pfm_context_t *ctx)
         }
         path.mnt = mntget(pfmfs_mnt);
 
-        d_set_d_op(path.dentry, &pfmfs_dentry_operations);
         d_add(path.dentry, inode);
 
         file = alloc_file(&path, FMODE_READ, &pfm_file_ops);
diff --git a/arch/ia64/kvm/kvm-ia64.c b/arch/ia64/kvm/kvm-ia64.c
index f56a6316e13..70d224d4264 100644
--- a/arch/ia64/kvm/kvm-ia64.c
+++ b/arch/ia64/kvm/kvm-ia64.c
@@ -749,7 +749,7 @@ out:
         return r;
 }
 
-static struct kvm *kvm_alloc_kvm(void)
+struct kvm *kvm_arch_alloc_vm(void)
 {
 
         struct kvm *kvm;
@@ -760,7 +760,7 @@ static struct kvm *kvm_alloc_kvm(void)
         vm_base = __get_free_pages(GFP_KERNEL, get_order(KVM_VM_DATA_SIZE));
 
         if (!vm_base)
-                return ERR_PTR(-ENOMEM);
+                return NULL;
 
         memset((void *)vm_base, 0, KVM_VM_DATA_SIZE);
         kvm = (struct kvm *)(vm_base +
@@ -806,10 +806,12 @@ static void kvm_build_io_pmt(struct kvm *kvm)
 #define GUEST_PHYSICAL_RR4      0x2739
 #define VMM_INIT_RR             0x1660
 
-static void kvm_init_vm(struct kvm *kvm)
+int kvm_arch_init_vm(struct kvm *kvm)
 {
         BUG_ON(!kvm);
 
+        kvm->arch.is_sn2 = ia64_platform_is("sn2");
+
         kvm->arch.metaphysical_rr0 = GUEST_PHYSICAL_RR0;
         kvm->arch.metaphysical_rr4 = GUEST_PHYSICAL_RR4;
         kvm->arch.vmm_init_rr = VMM_INIT_RR;
@@ -823,21 +825,8 @@ static void kvm_init_vm(struct kvm *kvm)
 
         /* Reserve bit 0 of irq_sources_bitmap for userspace irq source */
         set_bit(KVM_USERSPACE_IRQ_SOURCE_ID, &kvm->arch.irq_sources_bitmap);
-}
-
-struct  kvm *kvm_arch_create_vm(void)
-{
-        struct kvm *kvm = kvm_alloc_kvm();
-
-        if (IS_ERR(kvm))
-                return ERR_PTR(-ENOMEM);
-
-        kvm->arch.is_sn2 = ia64_platform_is("sn2");
-
-        kvm_init_vm(kvm);
-
-        return kvm;
 
+        return 0;
 }
 
 static int kvm_vm_ioctl_get_irqchip(struct kvm *kvm,
@@ -962,7 +951,9 @@ long kvm_arch_vm_ioctl(struct file *filp,
                         goto out;
                 r = kvm_setup_default_irq_routing(kvm);
                 if (r) {
+                        mutex_lock(&kvm->slots_lock);
                         kvm_ioapic_destroy(kvm);
+                        mutex_unlock(&kvm->slots_lock);
                         goto out;
                 }
                 break;
@@ -1357,7 +1348,7 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
         return -EINVAL;
 }
 
-static void free_kvm(struct kvm *kvm)
+void kvm_arch_free_vm(struct kvm *kvm)
 {
         unsigned long vm_base = kvm->arch.vm_base;
 
@@ -1399,9 +1390,6 @@ void kvm_arch_destroy_vm(struct kvm *kvm)
 #endif
         kfree(kvm->arch.vioapic);
         kvm_release_vm_pages(kvm);
-        kvm_free_physmem(kvm);
-        cleanup_srcu_struct(&kvm->srcu);
-        free_kvm(kvm);
 }
 
 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
diff --git a/arch/ia64/kvm/mmio.c b/arch/ia64/kvm/mmio.c
index fb8f9f59a1e..f1e17d3d6cd 100644
--- a/arch/ia64/kvm/mmio.c
+++ b/arch/ia64/kvm/mmio.c
@@ -130,7 +130,7 @@ static void mmio_access(struct kvm_vcpu *vcpu, u64 src_pa, u64 *dest,
 
         local_irq_save(psr);
 
-        /*Intercept the acces for PIB range*/
+        /*Intercept the access for PIB range*/
         if (iot == GPFN_PIB) {
                 if (!dir)
                         lsapic_write(vcpu, src_pa, s, *dest);
diff --git a/arch/m68k/ifpsp060/src/fpsp.S b/arch/m68k/ifpsp060/src/fpsp.S
index 73613b5f1ee..26e85e2b7a5 100644
--- a/arch/m68k/ifpsp060/src/fpsp.S
+++ b/arch/m68k/ifpsp060/src/fpsp.S
@@ -3881,7 +3881,7 @@ _fpsp_fline:
 # FP Unimplemented Instruction stack frame and jump to that entry
 # point.
 #
-# but, if the FPU is disabled, then we need to jump to the FPU diabled
+# but, if the FPU is disabled, then we need to jump to the FPU disabled
 # entry point.
         movc            %pcr,%d0
         btst            &0x1,%d0
diff --git a/arch/m68k/include/asm/m548xgpt.h b/arch/m68k/include/asm/m548xgpt.h
index c8ef158a1c4..33b2eef90f0 100644
--- a/arch/m68k/include/asm/m548xgpt.h
+++ b/arch/m68k/include/asm/m548xgpt.h
@@ -59,11 +59,13 @@
 #define MCF_GPT_GMS_GPIO_INPUT     (0x00000000)
 #define MCF_GPT_GMS_GPIO_OUTLO     (0x00000020)
 #define MCF_GPT_GMS_GPIO_OUTHI     (0x00000030)
+#define MCF_GPT_GMS_GPIO_MASK      (0x00000030)
 #define MCF_GPT_GMS_TMS_DISABLE    (0x00000000)
 #define MCF_GPT_GMS_TMS_INCAPT     (0x00000001)
 #define MCF_GPT_GMS_TMS_OUTCAPT    (0x00000002)
 #define MCF_GPT_GMS_TMS_PWM        (0x00000003)
 #define MCF_GPT_GMS_TMS_GPIO       (0x00000004)
+#define MCF_GPT_GMS_TMS_MASK       (0x00000007)
 
 /* Bit definitions and macros for MCF_GPT_GCIR */
 #define MCF_GPT_GCIR_CNT(x)        (((x)&0x0000FFFF)<<0)
diff --git a/arch/m68k/mac/psc.c b/arch/m68k/mac/psc.c
index ba6ccab6401..a4c3eb60706 100644
--- a/arch/m68k/mac/psc.c
+++ b/arch/m68k/mac/psc.c
@@ -88,7 +88,7 @@ void __init psc_init(void)
 
         /*
          * The PSC is always at the same spot, but using psc
-         * keeps things consisant with the psc_xxxx functions.
+         * keeps things consistent with the psc_xxxx functions.
          */
 
         psc = (void *) PSC_BASE;
diff --git a/arch/mips/alchemy/common/power.c b/arch/mips/alchemy/common/power.c
index e5916a516e5..647e518c90b 100644
--- a/arch/mips/alchemy/common/power.c
+++ b/arch/mips/alchemy/common/power.c
@@ -130,7 +130,7 @@ static void restore_core_regs(void)
         au_writel(sleep_usb[1], USBD_ENABLE);
         au_sync();
 #else
-        /* enable accces to OTG memory */
+        /* enable access to OTG memory */
         au_writel(au_readl(USB_MSR_BASE + 4) | (1 << 6), USB_MSR_BASE + 4);
         au_sync();
 
diff --git a/arch/mips/alchemy/devboards/pm.c b/arch/mips/alchemy/devboards/pm.c
index 4bbd3133e45..acaf91b5e46 100644
--- a/arch/mips/alchemy/devboards/pm.c
+++ b/arch/mips/alchemy/devboards/pm.c
@@ -110,7 +110,7 @@ static void db1x_pm_end(void)
 
 }
 
-static struct platform_suspend_ops db1x_pm_ops = {
+static const struct platform_suspend_ops db1x_pm_ops = {
         .valid          = suspend_valid_only_mem,
         .begin          = db1x_pm_begin,
         .enter          = db1x_pm_enter,
diff --git a/arch/mips/dec/time.c b/arch/mips/dec/time.c
index 02f505f23c3..ea57f39e673 100644
--- a/arch/mips/dec/time.c
+++ b/arch/mips/dec/time.c
@@ -104,7 +104,7 @@ int rtc_mips_set_mmss(unsigned long nowtime)
                 CMOS_WRITE(real_seconds, RTC_SECONDS);
                 CMOS_WRITE(real_minutes, RTC_MINUTES);
         } else {
-                printk(KERN_WARNING
+                printk_once(KERN_NOTICE
                        "set_rtc_mmss: can't update from %d to %d\n",
                        cmos_minutes, real_minutes);
                 retval = -1;
diff --git a/arch/mips/include/asm/mach-powertv/ioremap.h b/arch/mips/include/asm/mach-powertv/ioremap.h
index 076f2eeaa57..c86ef094ec3 100644
--- a/arch/mips/include/asm/mach-powertv/ioremap.h
+++ b/arch/mips/include/asm/mach-powertv/ioremap.h
@@ -88,7 +88,7 @@ static inline dma_addr_t _dma_to_phys_offset_raw(dma_addr_t dma)
 }
 
 /* These are not portable and should not be used in drivers. Drivers should
- * be using ioremap() and friends to map physical addreses to virtual
+ * be using ioremap() and friends to map physical addresses to virtual
  * addresses and dma_map*() and friends to map virtual addresses into DMA
  * addresses and back.
  */
diff --git a/arch/mips/include/asm/mc146818-time.h b/arch/mips/include/asm/mc146818-time.h
index 199b45733a9..4a08dbe37db 100644
--- a/arch/mips/include/asm/mc146818-time.h
+++ b/arch/mips/include/asm/mc146818-time.h
@@ -66,7 +66,7 @@ static inline int mc146818_set_rtc_mmss(unsigned long nowtime)
                 CMOS_WRITE(real_seconds, RTC_SECONDS);
                 CMOS_WRITE(real_minutes, RTC_MINUTES);
         } else {
-                printk(KERN_WARNING
+                printk_once(KERN_NOTICE
                        "set_rtc_mmss: can't update from %d to %d\n",
                        cmos_minutes, real_minutes);
                 retval = -1;
diff --git a/arch/mips/jz4740/board-qi_lb60.c b/arch/mips/jz4740/board-qi_lb60.c
index 5c0a3575877..2c0e107966a 100644
--- a/arch/mips/jz4740/board-qi_lb60.c
+++ b/arch/mips/jz4740/board-qi_lb60.c
@@ -65,7 +65,7 @@ static struct nand_ecclayout qi_lb60_ecclayout_1gb = {
 
 /* Early prototypes of the QI LB60 had only 1GB of NAND.
  * In order to support these devices aswell the partition and ecc layout is
- * initalized depending on the NAND size */
+ * initialized depending on the NAND size */
 static struct mtd_partition qi_lb60_partitions_1gb[] = {
         {
                 .name = "NAND BOOT partition",
@@ -464,7 +464,7 @@ static int __init qi_lb60_board_setup(void)
         board_gpio_setup();
 
         if (qi_lb60_init_platform_devices())
-                panic("Failed to initalize platform devices\n");
+                panic("Failed to initialize platform devices\n");
 
         return 0;
 }
diff --git a/arch/mips/jz4740/gpio.c b/arch/mips/jz4740/gpio.c
index 38f60f35156..88e6aeda5bf 100644
--- a/arch/mips/jz4740/gpio.c
+++ b/arch/mips/jz4740/gpio.c
@@ -546,7 +546,7 @@ static int __init jz4740_gpio_init(void)
         for (i = 0; i < ARRAY_SIZE(jz4740_gpio_chips); ++i)
                 jz4740_gpio_chip_init(&jz4740_gpio_chips[i], i);
 
-        printk(KERN_INFO "JZ4740 GPIO initalized\n");
+        printk(KERN_INFO "JZ4740 GPIO initialized\n");
 
         return 0;
 }
diff --git a/arch/mips/jz4740/pm.c b/arch/mips/jz4740/pm.c
index a9994585424..902d5b50124 100644
--- a/arch/mips/jz4740/pm.c
+++ b/arch/mips/jz4740/pm.c
@@ -42,7 +42,7 @@ static int jz4740_pm_enter(suspend_state_t state)
         return 0;
 }
 
-static struct platform_suspend_ops jz4740_pm_ops = {
+static const struct platform_suspend_ops jz4740_pm_ops = {
         .valid          = suspend_valid_only_mem,
         .enter          = jz4740_pm_enter,
 };
diff --git a/arch/mips/loongson/common/pm.c b/arch/mips/loongson/common/pm.c
index 6c1fd900171..f55e07aee07 100644
--- a/arch/mips/loongson/common/pm.c
+++ b/arch/mips/loongson/common/pm.c
@@ -147,7 +147,7 @@ static int loongson_pm_valid_state(suspend_state_t state)
         }
 }
 
-static struct platform_suspend_ops loongson_pm_ops = {
+static const struct platform_suspend_ops loongson_pm_ops = {
         .valid  = loongson_pm_valid_state,
         .enter  = loongson_pm_enter,
 };
diff --git a/arch/mips/mti-malta/malta-memory.c b/arch/mips/mti-malta/malta-memory.c
index b27419c8491..a96d281f922 100644
--- a/arch/mips/mti-malta/malta-memory.c
+++ b/arch/mips/mti-malta/malta-memory.c
@@ -43,7 +43,7 @@ static struct prom_pmemblock mdesc[PROM_MAX_PMEMBLOCKS];
 static char *mtypes[3] = {
         "Dont use memory",
         "YAMON PROM memory",
-        "Free memmory",
+        "Free memory",
 };
 #endif
 
diff --git a/arch/mips/pci/pcie-octeon.c b/arch/mips/pci/pcie-octeon.c
index 385f035b24e..0583c463e5f 100644
--- a/arch/mips/pci/pcie-octeon.c
+++ b/arch/mips/pci/pcie-octeon.c
@@ -900,7 +900,7 @@ static int cvmx_pcie_rc_initialize(int pcie_port)
         mem_access_subid.s.ror = 0;
         /* Disable Relaxed Ordering for Writes. */
         mem_access_subid.s.row = 0;
-        /* PCIe Adddress Bits <63:34>. */
+        /* PCIe Address Bits <63:34>. */
         mem_access_subid.s.ba = 0;
 
         /*
diff --git a/arch/mips/powertv/memory.c b/arch/mips/powertv/memory.c
index 73880ad29bc..fb3d29660c4 100644
--- a/arch/mips/powertv/memory.c
+++ b/arch/mips/powertv/memory.c
@@ -57,7 +57,7 @@
 unsigned long ptv_memsize;
 
 /*
- * struct low_mem_reserved - Items in low memmory that are reserved
+ * struct low_mem_reserved - Items in low memory that are reserved
  * @start:      Physical address of item
  * @size:       Size, in bytes, of this item
  * @is_aliased: True if this is RAM aliased from another location. If false,
diff --git a/arch/mips/txx9/generic/pci.c b/arch/mips/txx9/generic/pci.c
index 9a0be810caf..96e69a00ffc 100644
--- a/arch/mips/txx9/generic/pci.c
+++ b/arch/mips/txx9/generic/pci.c
@@ -107,7 +107,7 @@ int txx9_pci_mem_high __initdata;
 
 /*
  * allocate pci_controller and resources.
- * mem_base, io_base: physical addresss.  0 for auto assignment.
+ * mem_base, io_base: physical address.  0 for auto assignment.
  * mem_size and io_size means max size on auto assignment.
  * pcic must be &txx9_primary_pcic or NULL.
  */
diff --git a/arch/mn10300/Kconfig b/arch/mn10300/Kconfig
index 41ba38513c8..8ed41cf2b08 100644
--- a/arch/mn10300/Kconfig
+++ b/arch/mn10300/Kconfig
@@ -203,6 +203,7 @@ endmenu
 config SMP
         bool "Symmetric multi-processing support"
         default y
+        select USE_GENERIC_SMP_HELPERS
         depends on MN10300_PROC_MN2WS0038 || MN10300_PROC_MN2WS0050
         ---help---
           This enables support for systems with more than one CPU. If you have
@@ -226,11 +227,6 @@ config NR_CPUS
         depends on SMP
         default "2"
 
-config USE_GENERIC_SMP_HELPERS
-        bool
-        depends on SMP
-        default y
-
 source "kernel/Kconfig.preempt"
 
 config MN10300_CURRENT_IN_E2
diff --git a/arch/mn10300/kernel/rtc.c b/arch/mn10300/kernel/rtc.c
index e9e20f9a4dd..48d7058b329 100644
--- a/arch/mn10300/kernel/rtc.c
+++ b/arch/mn10300/kernel/rtc.c
@@ -89,7 +89,7 @@ static int set_rtc_mmss(unsigned long nowtime)
                 CMOS_WRITE(real_seconds, RTC_SECONDS);
                 CMOS_WRITE(real_minutes, RTC_MINUTES);
         } else {
-                printk(KERN_WARNING
+                printk_once(KERN_NOTICE
                        "set_rtc_mmss: can't update from %d to %d\n",
                        cmos_minutes, real_minutes);
                 retval = -1;
diff --git a/arch/powerpc/include/asm/8xx_immap.h b/arch/powerpc/include/asm/8xx_immap.h
index 4b0e1520600..6b6dc20b0be 100644
--- a/arch/powerpc/include/asm/8xx_immap.h
+++ b/arch/powerpc/include/asm/8xx_immap.h
@@ -93,7 +93,7 @@ typedef struct	mem_ctlr {
 } memctl8xx_t;
 
 /*-----------------------------------------------------------------------
- * BR - Memory Controler: Base Register                                 16-9
+ * BR - Memory Controller: Base Register                                        16-9
  */
 #define BR_BA_MSK       0xffff8000      /* Base Address Mask                    */
 #define BR_AT_MSK       0x00007000      /* Address Type Mask                    */
@@ -110,7 +110,7 @@ typedef struct	mem_ctlr {
 #define BR_V            0x00000001      /* Bank Valid                           */
 
 /*-----------------------------------------------------------------------
- * OR - Memory Controler: Option Register                               16-11
+ * OR - Memory Controller: Option Register                              16-11
  */
 #define OR_AM_MSK       0xffff8000      /* Address Mask Mask                    */
 #define OR_ATM_MSK      0x00007000      /* Address Type Mask Mask               */
diff --git a/arch/powerpc/kvm/book3s.c b/arch/powerpc/kvm/book3s.c
index e316847c08c..badc983031b 100644
--- a/arch/powerpc/kvm/book3s.c
+++ b/arch/powerpc/kvm/book3s.c
@@ -1307,12 +1307,10 @@ struct kvm_vcpu *kvmppc_core_vcpu_create(struct kvm *kvm, unsigned int id)
         int err = -ENOMEM;
         unsigned long p;
 
-        vcpu_book3s = vmalloc(sizeof(struct kvmppc_vcpu_book3s));
+        vcpu_book3s = vzalloc(sizeof(struct kvmppc_vcpu_book3s));
         if (!vcpu_book3s)
                 goto out;
 
-        memset(vcpu_book3s, 0, sizeof(struct kvmppc_vcpu_book3s));
-
         vcpu_book3s->shadow_vcpu = (struct kvmppc_book3s_shadow_vcpu *)
                 kzalloc(sizeof(*vcpu_book3s->shadow_vcpu), GFP_KERNEL);
         if (!vcpu_book3s->shadow_vcpu)
diff --git a/arch/powerpc/kvm/powerpc.c b/arch/powerpc/kvm/powerpc.c
index 38f756f2505..99758460efd 100644
--- a/arch/powerpc/kvm/powerpc.c
+++ b/arch/powerpc/kvm/powerpc.c
@@ -145,18 +145,12 @@ void kvm_arch_check_processor_compat(void *rtn)
         *(int *)rtn = kvmppc_core_check_processor_compat();
 }
 
-struct kvm *kvm_arch_create_vm(void)
+int kvm_arch_init_vm(struct kvm *kvm)
 {
-        struct kvm *kvm;
-
-        kvm = kzalloc(sizeof(struct kvm), GFP_KERNEL);
-        if (!kvm)
-                return ERR_PTR(-ENOMEM);
-
-        return kvm;
+        return 0;
 }
 
-static void kvmppc_free_vcpus(struct kvm *kvm)
+void kvm_arch_destroy_vm(struct kvm *kvm)
 {
         unsigned int i;
         struct kvm_vcpu *vcpu;
@@ -176,14 +170,6 @@ void kvm_arch_sync_events(struct kvm *kvm)
 {
 }
 
-void kvm_arch_destroy_vm(struct kvm *kvm)
-{
-        kvmppc_free_vcpus(kvm);
-        kvm_free_physmem(kvm);
-        cleanup_srcu_struct(&kvm->srcu);
-        kfree(kvm);
-}
-
 int kvm_dev_ioctl_check_extension(long ext)
 {
         int r;
diff --git a/arch/powerpc/oprofile/op_model_cell.c b/arch/powerpc/oprofile/op_model_cell.c
index 7fd90d02d8c..c4d2b716756 100644
--- a/arch/powerpc/oprofile/op_model_cell.c
+++ b/arch/powerpc/oprofile/op_model_cell.c
@@ -1469,7 +1469,7 @@ static int cell_global_start(struct op_counter_config *ctr)
  * The pm_interval register is setup to write the SPU PC value into the
  * trace buffer at the maximum rate possible.  The trace buffer is configured
  * to store the PCs, wrapping when it is full.  The performance counter is
- * intialized to the max hardware count minus the number of events, N, between
+ * initialized to the max hardware count minus the number of events, N, between
  * samples.  Once the N events have occured, a HW counter overflow occurs
  * causing the generation of a HW counter interrupt which also stops the
  * writing of the SPU PC values to the trace buffer.  Hence the last PC
diff --git a/arch/powerpc/platforms/52xx/lite5200_pm.c b/arch/powerpc/platforms/52xx/lite5200_pm.c
index 80234e5921f..eda0fc2a391 100644
--- a/arch/powerpc/platforms/52xx/lite5200_pm.c
+++ b/arch/powerpc/platforms/52xx/lite5200_pm.c
@@ -232,7 +232,7 @@ static void lite5200_pm_end(void)
         lite5200_pm_target_state = PM_SUSPEND_ON;
 }
 
-static struct platform_suspend_ops lite5200_pm_ops = {
+static const struct platform_suspend_ops lite5200_pm_ops = {
         .valid          = lite5200_pm_valid,
         .begin          = lite5200_pm_begin,
         .prepare        = lite5200_pm_prepare,
diff --git a/arch/powerpc/platforms/52xx/mpc52xx_pm.c b/arch/powerpc/platforms/52xx/mpc52xx_pm.c
index 568cef63627..8310e8b5b57 100644
--- a/arch/powerpc/platforms/52xx/mpc52xx_pm.c
+++ b/arch/powerpc/platforms/52xx/mpc52xx_pm.c
@@ -186,7 +186,7 @@ void mpc52xx_pm_finish(void)
         iounmap(mbar);
 }
 
-static struct platform_suspend_ops mpc52xx_pm_ops = {
+static const struct platform_suspend_ops mpc52xx_pm_ops = {
         .valid          = mpc52xx_pm_valid,
         .prepare        = mpc52xx_pm_prepare,
         .enter          = mpc52xx_pm_enter,
diff --git a/arch/powerpc/platforms/83xx/suspend-asm.S b/arch/powerpc/platforms/83xx/suspend-asm.S
index 1930543c98d..3d1ecd21177 100644
--- a/arch/powerpc/platforms/83xx/suspend-asm.S
+++ b/arch/powerpc/platforms/83xx/suspend-asm.S
@@ -231,7 +231,7 @@ _GLOBAL(mpc83xx_enter_deep_sleep)
         ori     r4, r4, 0x002a
         mtspr   SPRN_DBAT0L, r4
         lis     r8, TMP_VIRT_IMMR@h
-        ori     r4, r8, 0x001e  /* 1 MByte accessable from Kernel Space only */
+        ori     r4, r8, 0x001e  /* 1 MByte accessible from Kernel Space only */
         mtspr   SPRN_DBAT0U, r4
         isync
 
@@ -241,7 +241,7 @@ _GLOBAL(mpc83xx_enter_deep_sleep)
         ori     r4, r4, 0x002a
         mtspr   SPRN_DBAT1L, r4
         lis     r9, (TMP_VIRT_IMMR + 0x01000000)@h
-        ori     r4, r9, 0x001e  /* 1 MByte accessable from Kernel Space only */
+        ori     r4, r9, 0x001e  /* 1 MByte accessible from Kernel Space only */
         mtspr   SPRN_DBAT1U, r4
         isync
 
@@ -253,7 +253,7 @@ _GLOBAL(mpc83xx_enter_deep_sleep)
         li      r4, 0x0002
         mtspr   SPRN_DBAT2L, r4
         lis     r4, KERNELBASE@h
-        ori     r4, r4, 0x001e  /* 1 MByte accessable from Kernel Space only */
+        ori     r4, r4, 0x001e  /* 1 MByte accessible from Kernel Space only */
         mtspr   SPRN_DBAT2U, r4
         isync
 
diff --git a/arch/powerpc/platforms/83xx/suspend.c b/arch/powerpc/platforms/83xx/suspend.c
index 75ae77f1af6..fd4f2f2f19e 100644
--- a/arch/powerpc/platforms/83xx/suspend.c
+++ b/arch/powerpc/platforms/83xx/suspend.c
@@ -311,7 +311,7 @@ static int mpc83xx_is_pci_agent(void)
         return ret;
 }
 
-static struct platform_suspend_ops mpc83xx_suspend_ops = {
+static const struct platform_suspend_ops mpc83xx_suspend_ops = {
         .valid = mpc83xx_suspend_valid,
         .begin = mpc83xx_suspend_begin,
         .enter = mpc83xx_suspend_enter,
diff --git a/arch/powerpc/platforms/85xx/mpc85xx_mds.c b/arch/powerpc/platforms/85xx/mpc85xx_mds.c
index aa34cac4eb5..747d1ee661f 100644
--- a/arch/powerpc/platforms/85xx/mpc85xx_mds.c
+++ b/arch/powerpc/platforms/85xx/mpc85xx_mds.c
@@ -309,7 +309,7 @@ static void __init mpc85xx_mds_qe_init(void)
                         /* P1021 has pins muxed for QE and other functions. To
                          * enable QE UEC mode, we need to set bit QE0 for UCC1
                          * in Eth mode, QE0 and QE3 for UCC5 in Eth mode, QE9
-                         * and QE12 for QE MII management singals in PMUXCR
+                         * and QE12 for QE MII management signals in PMUXCR
                          * register.
                          */
                                 setbits32(pmuxcr, MPC85xx_PMUXCR_QE0 |
diff --git a/arch/powerpc/platforms/ps3/device-init.c b/arch/powerpc/platforms/ps3/device-init.c
index b341018326d..6c4b5837fc8 100644
--- a/arch/powerpc/platforms/ps3/device-init.c
+++ b/arch/powerpc/platforms/ps3/device-init.c
@@ -566,10 +566,10 @@ static int ps3_setup_dynamic_device(const struct ps3_repository_device *repo)
         case PS3_DEV_TYPE_STOR_DISK:
                 result = ps3_setup_storage_dev(repo, PS3_MATCH_ID_STOR_DISK);
 
-                /* Some devices are not accessable from the Other OS lpar. */
+                /* Some devices are not accessible from the Other OS lpar. */
                 if (result == -ENODEV) {
                         result = 0;
-                        pr_debug("%s:%u: not accessable\n", __func__,
+                        pr_debug("%s:%u: not accessible\n", __func__,
                                  __LINE__);
                 }
 
diff --git a/arch/powerpc/platforms/ps3/interrupt.c b/arch/powerpc/platforms/ps3/interrupt.c
index 59d9712d736..92290ff4761 100644
--- a/arch/powerpc/platforms/ps3/interrupt.c
+++ b/arch/powerpc/platforms/ps3/interrupt.c
@@ -44,7 +44,7 @@
  * @lock:
  * @ipi_debug_brk_mask:
  *
- * The HV mantains per SMT thread mappings of HV outlet to HV plug on
+ * The HV maintains per SMT thread mappings of HV outlet to HV plug on
  * behalf of the guest.  These mappings are implemented as 256 bit guest
  * supplied bitmaps indexed by plug number.  The addresses of the bitmaps
  * are registered with the HV through lv1_configure_irq_state_bitmap().
diff --git a/arch/powerpc/platforms/pseries/hvCall_inst.c b/arch/powerpc/platforms/pseries/hvCall_inst.c
index e19ff021e71..f106662f438 100644
--- a/arch/powerpc/platforms/pseries/hvCall_inst.c
+++ b/arch/powerpc/platforms/pseries/hvCall_inst.c
@@ -55,7 +55,7 @@ static void hc_stop(struct seq_file *m, void *p)
 static int hc_show(struct seq_file *m, void *p)
 {
         unsigned long h_num = (unsigned long)p;
-        struct hcall_stats *hs = (struct hcall_stats *)m->private;
+        struct hcall_stats *hs = m->private;
 
         if (hs[h_num].num_calls) {
                 if (cpu_has_feature(CPU_FTR_PURR))
diff --git a/arch/powerpc/platforms/pseries/suspend.c b/arch/powerpc/platforms/pseries/suspend.c
index ed72098bb4e..a8ca289ff26 100644
--- a/arch/powerpc/platforms/pseries/suspend.c
+++ b/arch/powerpc/platforms/pseries/suspend.c
@@ -153,7 +153,7 @@ static struct sysdev_class suspend_sysdev_class = {
         .name = "power",
 };
 
-static struct platform_suspend_ops pseries_suspend_ops = {
+static const struct platform_suspend_ops pseries_suspend_ops = {
         .valid          = suspend_valid_only_mem,
         .begin          = pseries_suspend_begin,
         .prepare_late   = pseries_prepare_late,
diff --git a/arch/powerpc/sysdev/fsl_pmc.c b/arch/powerpc/sysdev/fsl_pmc.c
index 44de8559c97..e9381bfefb2 100644
--- a/arch/powerpc/sysdev/fsl_pmc.c
+++ b/arch/powerpc/sysdev/fsl_pmc.c
@@ -53,7 +53,7 @@ static int pmc_suspend_valid(suspend_state_t state)
         return 1;
 }
 
-static struct platform_suspend_ops pmc_suspend_ops = {
+static const struct platform_suspend_ops pmc_suspend_ops = {
         .valid = pmc_suspend_valid,
         .enter = pmc_suspend_enter,
 };
diff --git a/arch/powerpc/sysdev/fsl_rio.c b/arch/powerpc/sysdev/fsl_rio.c
index 9725369d432..9f99bef2ade 100644
--- a/arch/powerpc/sysdev/fsl_rio.c
+++ b/arch/powerpc/sysdev/fsl_rio.c
@@ -973,7 +973,6 @@ fsl_rio_dbell_handler(int irq, void *dev_instance)
         if (dsr & DOORBELL_DSR_QFI) {
                 pr_info("RIO: doorbell queue full\n");
                 out_be32(&priv->msg_regs->dsr, DOORBELL_DSR_QFI);
-                goto out;
         }
 
         /* XXX Need to check/dispatch until queue empty */
diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
index 985d825494f..bade533ba28 100644
--- a/arch/s390/kvm/kvm-s390.c
+++ b/arch/s390/kvm/kvm-s390.c
@@ -164,24 +164,18 @@ long kvm_arch_vm_ioctl(struct file *filp,
         return r;
 }
 
-struct kvm *kvm_arch_create_vm(void)
+int kvm_arch_init_vm(struct kvm *kvm)
 {
-        struct kvm *kvm;
         int rc;
         char debug_name[16];
 
         rc = s390_enable_sie();
         if (rc)
-                goto out_nokvm;
-
-        rc = -ENOMEM;
-        kvm = kzalloc(sizeof(struct kvm), GFP_KERNEL);
-        if (!kvm)
-                goto out_nokvm;
+                goto out_err;
 
         kvm->arch.sca = (struct sca_block *) get_zeroed_page(GFP_KERNEL);
         if (!kvm->arch.sca)
-                goto out_nosca;
+                goto out_err;
 
         sprintf(debug_name, "kvm-%u", current->pid);
 
@@ -195,13 +189,11 @@ struct kvm *kvm_arch_create_vm(void)
         debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
         VM_EVENT(kvm, 3, "%s", "vm created");
 
-        return kvm;
+        return 0;
 out_nodbf:
         free_page((unsigned long)(kvm->arch.sca));
-out_nosca:
-        kfree(kvm);
-out_nokvm:
-        return ERR_PTR(rc);
+out_err:
+        return rc;
 }
 
 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
@@ -240,11 +232,8 @@ void kvm_arch_sync_events(struct kvm *kvm)
 void kvm_arch_destroy_vm(struct kvm *kvm)
 {
         kvm_free_vcpus(kvm);
-        kvm_free_physmem(kvm);
         free_page((unsigned long)(kvm->arch.sca));
         debug_unregister(kvm->arch.dbf);
-        cleanup_srcu_struct(&kvm->srcu);
-        kfree(kvm);
 }
 
 /* Section: vcpu related */
diff --git a/arch/sh/boards/mach-hp6xx/pm.c b/arch/sh/boards/mach-hp6xx/pm.c
index 4499a3749d4..adc9b4bba82 100644
--- a/arch/sh/boards/mach-hp6xx/pm.c
+++ b/arch/sh/boards/mach-hp6xx/pm.c
@@ -143,7 +143,7 @@ static int hp6x0_pm_enter(suspend_state_t state)
         return 0;
 }
 
-static struct platform_suspend_ops hp6x0_pm_ops = {
+static const struct platform_suspend_ops hp6x0_pm_ops = {
         .enter          = hp6x0_pm_enter,
         .valid          = suspend_valid_only_mem,
 };
diff --git a/arch/sh/boards/mach-se/7724/setup.c b/arch/sh/boards/mach-se/7724/setup.c
index 6dc7407f2cd..527679394a2 100644
--- a/arch/sh/boards/mach-se/7724/setup.c
+++ b/arch/sh/boards/mach-se/7724/setup.c
@@ -319,6 +319,10 @@ static struct platform_device fsi_device = {
         },
 };
 
+static struct platform_device fsi_ak4642_device = {
+        .name           = "sh_fsi_a_ak4642",
+};
+
 /* KEYSC in SoC (Needs SW33-2 set to ON) */
 static struct sh_keysc_info keysc_info = {
         .mode = SH_KEYSC_MODE_1,
@@ -593,6 +597,7 @@ static struct platform_device *ms7724se_devices[] __initdata = {
         &sh7724_usb0_host_device,
         &sh7724_usb1_gadget_device,
         &fsi_device,
+        &fsi_ak4642_device,
         &sdhi0_cn7_device,
         &sdhi1_cn8_device,
         &irda_device,
diff --git a/arch/sh/boards/mach-sh03/rtc.c b/arch/sh/boards/mach-sh03/rtc.c
index 1b200990500..f83ac7995d0 100644
--- a/arch/sh/boards/mach-sh03/rtc.c
+++ b/arch/sh/boards/mach-sh03/rtc.c
@@ -108,7 +108,7 @@ static int set_rtc_mmss(unsigned long nowtime)
                 __raw_writeb(real_minutes % 10, RTC_MIN1);
                 __raw_writeb(real_minutes / 10, RTC_MIN10);
         } else {
-                printk(KERN_WARNING
+                printk_once(KERN_NOTICE
                        "set_rtc_mmss: can't update from %d to %d\n",
                        cmos_minutes, real_minutes);
                 retval = -1;
diff --git a/arch/sh/kernel/cpu/shmobile/pm.c b/arch/sh/kernel/cpu/shmobile/pm.c
index e5596871270..a6f95ae4aae 100644
--- a/arch/sh/kernel/cpu/shmobile/pm.c
+++ b/arch/sh/kernel/cpu/shmobile/pm.c
@@ -141,7 +141,7 @@ static int sh_pm_enter(suspend_state_t state)
         return 0;
 }
 
-static struct platform_suspend_ops sh_pm_ops = {
+static const struct platform_suspend_ops sh_pm_ops = {
         .enter          = sh_pm_enter,
         .valid          = suspend_valid_only_mem,
 };
diff --git a/arch/sh/mm/cache-sh5.c b/arch/sh/mm/cache-sh5.c
index eb4cc4ec795..d1bffbcd9d5 100644
--- a/arch/sh/mm/cache-sh5.c
+++ b/arch/sh/mm/cache-sh5.c
@@ -568,7 +568,7 @@ static void sh5_flush_dcache_page(void *page)
 }
 
 /*
- * Flush the range [start,end] of kernel virtual adddress space from
+ * Flush the range [start,end] of kernel virtual address space from
  * the I-cache.  The corresponding range must be purged from the
  * D-cache also because the SH-5 doesn't have cache snooping between
  * the caches.  The addresses will be visible through the superpage
diff --git a/arch/sparc/kernel/traps_64.c b/arch/sparc/kernel/traps_64.c
index 42ad2ba8501..1e9770936c3 100644
--- a/arch/sparc/kernel/traps_64.c
+++ b/arch/sparc/kernel/traps_64.c
@@ -622,7 +622,7 @@ static const char CHAFSR_PERR_msg[] =
 static const char CHAFSR_IERR_msg[] =
         "Internal processor error";
 static const char CHAFSR_ISAP_msg[] =
-        "System request parity error on incoming addresss";
+        "System request parity error on incoming address";
 static const char CHAFSR_UCU_msg[] =
         "Uncorrectable E-cache ECC error for ifetch/data";
 static const char CHAFSR_UCC_msg[] =
diff --git a/arch/um/Kconfig.um b/arch/um/Kconfig.um
index 50d6aa20c35..f8d1d0d47fe 100644
--- a/arch/um/Kconfig.um
+++ b/arch/um/Kconfig.um
@@ -131,7 +131,7 @@ config NR_CPUS
 
 config HIGHMEM
         bool "Highmem support (EXPERIMENTAL)"
-        depends on !64BIT && EXPERIMENTAL
+        depends on !64BIT && BROKEN
         default n
         help
           This was used to allow UML to run with big amounts of memory.
diff --git a/arch/um/drivers/line.c b/arch/um/drivers/line.c
index 1664cce7b0a..050e4ddbbb6 100644
--- a/arch/um/drivers/line.c
+++ b/arch/um/drivers/line.c
@@ -821,12 +821,12 @@ void register_winch_irq(int fd, int tty_fd, int pid, struct tty_struct *tty,
 
 static void unregister_winch(struct tty_struct *tty)
 {
-        struct list_head *ele;
+        struct list_head *ele, *next;
         struct winch *winch;
 
         spin_lock(&winch_handler_lock);
 
-        list_for_each(ele, &winch_handlers) {
+        list_for_each_safe(ele, next, &winch_handlers) {
                 winch = list_entry(ele, struct winch, list);
                 if (winch->tty == tty) {
                         free_winch(winch, 1);
diff --git a/arch/um/drivers/mmapper_kern.c b/arch/um/drivers/mmapper_kern.c
index 8501e7d0015..7e0619c2c2c 100644
--- a/arch/um/drivers/mmapper_kern.c
+++ b/arch/um/drivers/mmapper_kern.c
@@ -37,13 +37,7 @@ static ssize_t mmapper_write(struct file *file, const char __user *buf,
         if (*ppos > mmapper_size)
                 return -EINVAL;
 
-        if (count > mmapper_size - *ppos)
-                count = mmapper_size - *ppos;
-
-        if (copy_from_user(&v_buf[*ppos], buf, count))
-                return -EFAULT;
-
-        return count;
+        return simple_write_to_buffer(v_buf, mmapper_size, ppos, buf, count);
 }
 
 static long mmapper_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
@@ -137,3 +131,4 @@ module_exit(mmapper_exit);
 
 MODULE_AUTHOR("Greg Lonnon <glonnon@ridgerun.com>");
 MODULE_DESCRIPTION("DSPLinux simulator mmapper driver");
+MODULE_LICENSE("GPL");
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index b6fccb07123..36ed2e2c896 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -51,6 +51,7 @@ config X86
         select HAVE_KERNEL_GZIP
         select HAVE_KERNEL_BZIP2
         select HAVE_KERNEL_LZMA
+        select HAVE_KERNEL_XZ
         select HAVE_KERNEL_LZO
         select HAVE_HW_BREAKPOINT
         select HAVE_MIXED_BREAKPOINTS_REGS
@@ -65,6 +66,7 @@ config X86
         select HAVE_SPARSE_IRQ
         select GENERIC_IRQ_PROBE
         select GENERIC_PENDING_IRQ if SMP
+        select USE_GENERIC_SMP_HELPERS if SMP
 
 config INSTRUCTION_DECODER
         def_bool (KPROBES || PERF_EVENTS)
@@ -203,10 +205,6 @@ config HAVE_INTEL_TXT
         def_bool y
         depends on EXPERIMENTAL && DMAR && ACPI
 
-config USE_GENERIC_SMP_HELPERS
-        def_bool y
-        depends on SMP
-
 config X86_32_SMP
         def_bool y
         depends on X86_32 && SMP
@@ -2078,11 +2076,17 @@ config OLPC_OPENFIRMWARE
         bool "Support for OLPC's Open Firmware"
         depends on !X86_64 && !X86_PAE
         default n
+        select OF
         help
           This option adds support for the implementation of Open Firmware
           that is used on the OLPC XO-1 Children's Machine.
           If unsure, say N here.
 
+config OLPC_OPENFIRMWARE_DT
+        bool
+        default y if OLPC_OPENFIRMWARE && PROC_DEVICETREE
+        select OF_PROMTREE
+
 endif # X86_32
 
 config AMD_NB
diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile
index 0c229551eea..09664efb9ce 100644
--- a/arch/x86/boot/compressed/Makefile
+++ b/arch/x86/boot/compressed/Makefile
@@ -4,7 +4,7 @@
 # create a compressed vmlinux image from the original vmlinux
 #
 
-targets := vmlinux.lds vmlinux vmlinux.bin vmlinux.bin.gz vmlinux.bin.bz2 vmlinux.bin.lzma vmlinux.bin.lzo head_$(BITS).o misc.o string.o cmdline.o early_serial_console.o piggy.o
+targets := vmlinux.lds vmlinux vmlinux.bin vmlinux.bin.gz vmlinux.bin.bz2 vmlinux.bin.lzma vmlinux.bin.xz vmlinux.bin.lzo head_$(BITS).o misc.o string.o cmdline.o early_serial_console.o piggy.o
 
 KBUILD_CFLAGS := -m$(BITS) -D__KERNEL__ $(LINUX_INCLUDE) -O2
 KBUILD_CFLAGS += -fno-strict-aliasing -fPIC
@@ -49,12 +49,15 @@ $(obj)/vmlinux.bin.bz2: $(vmlinux.bin.all-y) FORCE
         $(call if_changed,bzip2)
 $(obj)/vmlinux.bin.lzma: $(vmlinux.bin.all-y) FORCE
         $(call if_changed,lzma)
+$(obj)/vmlinux.bin.xz: $(vmlinux.bin.all-y) FORCE
+        $(call if_changed,xzkern)
 $(obj)/vmlinux.bin.lzo: $(vmlinux.bin.all-y) FORCE
         $(call if_changed,lzo)
 
 suffix-$(CONFIG_KERNEL_GZIP)    := gz
 suffix-$(CONFIG_KERNEL_BZIP2)   := bz2
 suffix-$(CONFIG_KERNEL_LZMA)    := lzma
+suffix-$(CONFIG_KERNEL_XZ)      := xz
 suffix-$(CONFIG_KERNEL_LZO)     := lzo
 
 quiet_cmd_mkpiggy = MKPIGGY $@
diff --git a/arch/x86/boot/compressed/misc.c b/arch/x86/boot/compressed/misc.c
index 325c05294fc..3a19d04cebe 100644
--- a/arch/x86/boot/compressed/misc.c
+++ b/arch/x86/boot/compressed/misc.c
@@ -139,6 +139,10 @@ static int lines, cols;
 #include "../../../../lib/decompress_unlzma.c"
 #endif
 
+#ifdef CONFIG_KERNEL_XZ
+#include "../../../../lib/decompress_unxz.c"
+#endif
+
 #ifdef CONFIG_KERNEL_LZO
 #include "../../../../lib/decompress_unlzo.c"
 #endif
diff --git a/arch/x86/boot/compressed/mkpiggy.c b/arch/x86/boot/compressed/mkpiggy.c
index 5c228129d17..646aa78ba5f 100644
--- a/arch/x86/boot/compressed/mkpiggy.c
+++ b/arch/x86/boot/compressed/mkpiggy.c
@@ -74,7 +74,7 @@ int main(int argc, char *argv[])
 
         offs = (olen > ilen) ? olen - ilen : 0;
         offs += olen >> 12;     /* Add 8 bytes for each 32K block */
-        offs += 32*1024 + 18;   /* Add 32K + 18 bytes slack */
+        offs += 64*1024 + 128;  /* Add 64K + 128 bytes slack */
         offs = (offs+4095) & ~4095; /* Round to a 4K boundary */
 
         printf(".section \".rodata..compressed\",\"a\",@progbits\n");
diff --git a/arch/x86/crypto/aesni-intel_asm.S b/arch/x86/crypto/aesni-intel_asm.S
index ff16756a51c..8fe2a4966b7 100644
--- a/arch/x86/crypto/aesni-intel_asm.S
+++ b/arch/x86/crypto/aesni-intel_asm.S
@@ -9,6 +9,20 @@
  *            Vinodh Gopal <vinodh.gopal@intel.com>
  *            Kahraman Akdemir
  *
+ * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
+ * interface for 64-bit kernels.
+ *    Authors: Erdinc Ozturk (erdinc.ozturk@intel.com)
+ *             Aidan O'Mahony (aidan.o.mahony@intel.com)
+ *             Adrian Hoban <adrian.hoban@intel.com>
+ *             James Guilford (james.guilford@intel.com)
+ *             Gabriele Paoloni <gabriele.paoloni@intel.com>
+ *             Tadeusz Struk (tadeusz.struk@intel.com)
+ *             Wajdi Feghali (wajdi.k.feghali@intel.com)
+ *    Copyright (c) 2010, Intel Corporation.
+ *
+ * Ported x86_64 version to x86:
+ *    Author: Mathias Krause <minipli@googlemail.com>
+ *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
  * the Free Software Foundation; either version 2 of the License, or
@@ -18,8 +32,62 @@
 #include <linux/linkage.h>
 #include <asm/inst.h>
 
+#ifdef __x86_64__
+.data
+POLY:   .octa 0xC2000000000000000000000000000001
+TWOONE: .octa 0x00000001000000000000000000000001
+
+# order of these constants should not change.
+# more specifically, ALL_F should follow SHIFT_MASK,
+# and ZERO should follow ALL_F
+
+SHUF_MASK:  .octa 0x000102030405060708090A0B0C0D0E0F
+MASK1:      .octa 0x0000000000000000ffffffffffffffff
+MASK2:      .octa 0xffffffffffffffff0000000000000000
+SHIFT_MASK: .octa 0x0f0e0d0c0b0a09080706050403020100
+ALL_F:      .octa 0xffffffffffffffffffffffffffffffff
+ZERO:       .octa 0x00000000000000000000000000000000
+ONE:        .octa 0x00000000000000000000000000000001
+F_MIN_MASK: .octa 0xf1f2f3f4f5f6f7f8f9fafbfcfdfeff0
+dec:        .octa 0x1
+enc:        .octa 0x2
+
+
 .text
 
+
+#define STACK_OFFSET    8*3
+#define HashKey         16*0    // store HashKey <<1 mod poly here
+#define HashKey_2       16*1    // store HashKey^2 <<1 mod poly here
+#define HashKey_3       16*2    // store HashKey^3 <<1 mod poly here
+#define HashKey_4       16*3    // store HashKey^4 <<1 mod poly here
+#define HashKey_k       16*4    // store XOR of High 64 bits and Low 64
+                                // bits of  HashKey <<1 mod poly here
+                                //(for Karatsuba purposes)
+#define HashKey_2_k     16*5    // store XOR of High 64 bits and Low 64
+                                // bits of  HashKey^2 <<1 mod poly here
+                                // (for Karatsuba purposes)
+#define HashKey_3_k     16*6    // store XOR of High 64 bits and Low 64
+                                // bits of  HashKey^3 <<1 mod poly here
+                                // (for Karatsuba purposes)
+#define HashKey_4_k     16*7    // store XOR of High 64 bits and Low 64
+                                // bits of  HashKey^4 <<1 mod poly here
+                                // (for Karatsuba purposes)
+#define VARIABLE_OFFSET 16*8
+
+#define arg1 rdi
+#define arg2 rsi
+#define arg3 rdx
+#define arg4 rcx
+#define arg5 r8
+#define arg6 r9
+#define arg7 STACK_OFFSET+8(%r14)
+#define arg8 STACK_OFFSET+16(%r14)
+#define arg9 STACK_OFFSET+24(%r14)
+#define arg10 STACK_OFFSET+32(%r14)
+#endif
+
+
 #define STATE1  %xmm0
 #define STATE2  %xmm4
 #define STATE3  %xmm5
@@ -32,12 +100,16 @@
 #define IN      IN1
 #define KEY     %xmm2
 #define IV      %xmm3
+
 #define BSWAP_MASK %xmm10
 #define CTR     %xmm11
 #define INC     %xmm12
 
+#ifdef __x86_64__
+#define AREG    %rax
 #define KEYP    %rdi
 #define OUTP    %rsi
+#define UKEYP   OUTP
 #define INP     %rdx
 #define LEN     %rcx
 #define IVP     %r8
@@ -46,6 +118,1588 @@
 #define TKEYP   T1
 #define T2      %r11
 #define TCTR_LOW T2
+#else
+#define AREG    %eax
+#define KEYP    %edi
+#define OUTP    AREG
+#define UKEYP   OUTP
+#define INP     %edx
+#define LEN     %esi
+#define IVP     %ebp
+#define KLEN    %ebx
+#define T1      %ecx
+#define TKEYP   T1
+#endif
+
+
+#ifdef __x86_64__
+/* GHASH_MUL MACRO to implement: Data*HashKey mod (128,127,126,121,0)
+*
+*
+* Input: A and B (128-bits each, bit-reflected)
+* Output: C = A*B*x mod poly, (i.e. >>1 )
+* To compute GH = GH*HashKey mod poly, give HK = HashKey<<1 mod poly as input
+* GH = GH * HK * x mod poly which is equivalent to GH*HashKey mod poly.
+*
+*/
+.macro GHASH_MUL GH HK TMP1 TMP2 TMP3 TMP4 TMP5
+        movdqa    \GH, \TMP1
+        pshufd    $78, \GH, \TMP2
+        pshufd    $78, \HK, \TMP3
+        pxor      \GH, \TMP2            # TMP2 = a1+a0
+        pxor      \HK, \TMP3            # TMP3 = b1+b0
+        PCLMULQDQ 0x11, \HK, \TMP1     # TMP1 = a1*b1
+        PCLMULQDQ 0x00, \HK, \GH       # GH = a0*b0
+        PCLMULQDQ 0x00, \TMP3, \TMP2   # TMP2 = (a0+a1)*(b1+b0)
+        pxor      \GH, \TMP2
+        pxor      \TMP1, \TMP2          # TMP2 = (a0*b0)+(a1*b0)
+        movdqa    \TMP2, \TMP3
+        pslldq    $8, \TMP3             # left shift TMP3 2 DWs
+        psrldq    $8, \TMP2             # right shift TMP2 2 DWs
+        pxor      \TMP3, \GH
+        pxor      \TMP2, \TMP1          # TMP2:GH holds the result of GH*HK
+
+        # first phase of the reduction
+
+        movdqa    \GH, \TMP2
+        movdqa    \GH, \TMP3
+        movdqa    \GH, \TMP4            # copy GH into TMP2,TMP3 and TMP4
+                                        # in in order to perform
+                                        # independent shifts
+        pslld     $31, \TMP2            # packed right shift <<31
+        pslld     $30, \TMP3            # packed right shift <<30
+        pslld     $25, \TMP4            # packed right shift <<25
+        pxor      \TMP3, \TMP2          # xor the shifted versions
+        pxor      \TMP4, \TMP2
+        movdqa    \TMP2, \TMP5
+        psrldq    $4, \TMP5             # right shift TMP5 1 DW
+        pslldq    $12, \TMP2            # left shift TMP2 3 DWs
+        pxor      \TMP2, \GH
+
+        # second phase of the reduction
+
+        movdqa    \GH,\TMP2             # copy GH into TMP2,TMP3 and TMP4
+                                        # in in order to perform
+                                        # independent shifts
+        movdqa    \GH,\TMP3
+        movdqa    \GH,\TMP4
+        psrld     $1,\TMP2              # packed left shift >>1
+        psrld     $2,\TMP3              # packed left shift >>2
+        psrld     $7,\TMP4              # packed left shift >>7
+        pxor      \TMP3,\TMP2           # xor the shifted versions
+        pxor      \TMP4,\TMP2
+        pxor      \TMP5, \TMP2
+        pxor      \TMP2, \GH
+        pxor      \TMP1, \GH            # result is in TMP1
+.endm
+
+/*
+* if a = number of total plaintext bytes
+* b = floor(a/16)
+* num_initial_blocks = b mod 4
+* encrypt the initial num_initial_blocks blocks and apply ghash on
+* the ciphertext
+* %r10, %r11, %r12, %rax, %xmm5, %xmm6, %xmm7, %xmm8, %xmm9 registers
+* are clobbered
+* arg1, %arg2, %arg3, %r14 are used as a pointer only, not modified
+*/
+
+
+.macro INITIAL_BLOCKS_DEC num_initial_blocks TMP1 TMP2 TMP3 TMP4 TMP5 XMM0 XMM1 \
+XMM2 XMM3 XMM4 XMMDst TMP6 TMP7 i i_seq operation
+        mov        arg7, %r10           # %r10 = AAD
+        mov        arg8, %r12           # %r12 = aadLen
+        mov        %r12, %r11
+        pxor       %xmm\i, %xmm\i
+_get_AAD_loop\num_initial_blocks\operation:
+        movd       (%r10), \TMP1
+        pslldq     $12, \TMP1
+        psrldq     $4, %xmm\i
+        pxor       \TMP1, %xmm\i
+        add        $4, %r10
+        sub        $4, %r12
+        jne        _get_AAD_loop\num_initial_blocks\operation
+        cmp        $16, %r11
+        je         _get_AAD_loop2_done\num_initial_blocks\operation
+        mov        $16, %r12
+_get_AAD_loop2\num_initial_blocks\operation:
+        psrldq     $4, %xmm\i
+        sub        $4, %r12
+        cmp        %r11, %r12
+        jne        _get_AAD_loop2\num_initial_blocks\operation
+_get_AAD_loop2_done\num_initial_blocks\operation:
+        movdqa     SHUF_MASK(%rip), %xmm14
+        PSHUFB_XMM   %xmm14, %xmm\i # byte-reflect the AAD data
+
+        xor        %r11, %r11 # initialise the data pointer offset as zero
+
+        # start AES for num_initial_blocks blocks
+
+        mov        %arg5, %rax                      # %rax = *Y0
+        movdqu     (%rax), \XMM0                    # XMM0 = Y0
+        movdqa     SHUF_MASK(%rip), %xmm14
+        PSHUFB_XMM   %xmm14, \XMM0
+
+.if (\i == 5) || (\i == 6) || (\i == 7)
+.irpc index, \i_seq
+        paddd      ONE(%rip), \XMM0                 # INCR Y0
+        movdqa     \XMM0, %xmm\index
+        movdqa     SHUF_MASK(%rip), %xmm14
+        PSHUFB_XMM   %xmm14, %xmm\index      # perform a 16 byte swap
+
+.endr
+.irpc index, \i_seq
+        pxor       16*0(%arg1), %xmm\index
+.endr
+.irpc index, \i_seq
+        movaps 0x10(%rdi), \TMP1
+        AESENC     \TMP1, %xmm\index          # Round 1
+.endr
+.irpc index, \i_seq
+        movaps 0x20(%arg1), \TMP1
+        AESENC     \TMP1, %xmm\index          # Round 2
+.endr
+.irpc index, \i_seq
+        movaps 0x30(%arg1), \TMP1
+        AESENC     \TMP1, %xmm\index          # Round 2
+.endr
+.irpc index, \i_seq
+        movaps 0x40(%arg1), \TMP1
+        AESENC     \TMP1, %xmm\index          # Round 2
+.endr
+.irpc index, \i_seq
+        movaps 0x50(%arg1), \TMP1
+        AESENC     \TMP1, %xmm\index          # Round 2
+.endr
+.irpc index, \i_seq
+        movaps 0x60(%arg1), \TMP1
+        AESENC     \TMP1, %xmm\index          # Round 2
+.endr
+.irpc index, \i_seq
+        movaps 0x70(%arg1), \TMP1
+        AESENC     \TMP1, %xmm\index          # Round 2
+.endr
+.irpc index, \i_seq
+        movaps 0x80(%arg1), \TMP1
+        AESENC     \TMP1, %xmm\index          # Round 2
+.endr
+.irpc index, \i_seq
+        movaps 0x90(%arg1), \TMP1
+        AESENC     \TMP1, %xmm\index          # Round 2
+.endr
+.irpc index, \i_seq
+        movaps 0xa0(%arg1), \TMP1
+        AESENCLAST \TMP1, %xmm\index         # Round 10
+.endr
+.irpc index, \i_seq
+        movdqu     (%arg3 , %r11, 1), \TMP1
+        pxor       \TMP1, %xmm\index
+        movdqu     %xmm\index, (%arg2 , %r11, 1)
+        # write back plaintext/ciphertext for num_initial_blocks
+        add        $16, %r11
+
+        movdqa     \TMP1, %xmm\index
+        movdqa     SHUF_MASK(%rip), %xmm14
+        PSHUFB_XMM         %xmm14, %xmm\index
+
+                # prepare plaintext/ciphertext for GHASH computation
+.endr
+.endif
+        GHASH_MUL  %xmm\i, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
+        # apply GHASH on num_initial_blocks blocks
+
+.if \i == 5
+        pxor       %xmm5, %xmm6
+        GHASH_MUL  %xmm6, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
+        pxor       %xmm6, %xmm7
+        GHASH_MUL  %xmm7, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
+        pxor       %xmm7, %xmm8
+        GHASH_MUL  %xmm8, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
+.elseif \i == 6
+        pxor       %xmm6, %xmm7
+        GHASH_MUL  %xmm7, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
+        pxor       %xmm7, %xmm8
+        GHASH_MUL  %xmm8, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
+.elseif \i == 7
+        pxor       %xmm7, %xmm8
+        GHASH_MUL  %xmm8, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
+.endif
+        cmp        $64, %r13
+        jl      _initial_blocks_done\num_initial_blocks\operation
+        # no need for precomputed values
+/*
+*
+* Precomputations for HashKey parallel with encryption of first 4 blocks.
+* Haskey_i_k holds XORed values of the low and high parts of the Haskey_i
+*/
+        paddd      ONE(%rip), \XMM0              # INCR Y0
+        movdqa     \XMM0, \XMM1
+        movdqa     SHUF_MASK(%rip), %xmm14
+        PSHUFB_XMM  %xmm14, \XMM1        # perform a 16 byte swap
+
+        paddd      ONE(%rip), \XMM0              # INCR Y0
+        movdqa     \XMM0, \XMM2
+        movdqa     SHUF_MASK(%rip), %xmm14
+        PSHUFB_XMM  %xmm14, \XMM2        # perform a 16 byte swap
+
+        paddd      ONE(%rip), \XMM0              # INCR Y0
+        movdqa     \XMM0, \XMM3
+        movdqa     SHUF_MASK(%rip), %xmm14
+        PSHUFB_XMM %xmm14, \XMM3        # perform a 16 byte swap
+
+        paddd      ONE(%rip), \XMM0              # INCR Y0
+        movdqa     \XMM0, \XMM4
+        movdqa     SHUF_MASK(%rip), %xmm14
+        PSHUFB_XMM %xmm14, \XMM4        # perform a 16 byte swap
+
+        pxor       16*0(%arg1), \XMM1
+        pxor       16*0(%arg1), \XMM2
+        pxor       16*0(%arg1), \XMM3
+        pxor       16*0(%arg1), \XMM4
+        movdqa     \TMP3, \TMP5
+        pshufd     $78, \TMP3, \TMP1
+        pxor       \TMP3, \TMP1
+        movdqa     \TMP1, HashKey_k(%rsp)
+        GHASH_MUL  \TMP5, \TMP3, \TMP1, \TMP2, \TMP4, \TMP6, \TMP7
+# TMP5 = HashKey^2<<1 (mod poly)
+        movdqa     \TMP5, HashKey_2(%rsp)
+# HashKey_2 = HashKey^2<<1 (mod poly)
+        pshufd     $78, \TMP5, \TMP1
+        pxor       \TMP5, \TMP1
+        movdqa     \TMP1, HashKey_2_k(%rsp)
+.irpc index, 1234 # do 4 rounds
+        movaps 0x10*\index(%arg1), \TMP1
+        AESENC     \TMP1, \XMM1
+        AESENC     \TMP1, \XMM2
+        AESENC     \TMP1, \XMM3
+        AESENC     \TMP1, \XMM4
+.endr
+        GHASH_MUL  \TMP5, \TMP3, \TMP1, \TMP2, \TMP4, \TMP6, \TMP7
+# TMP5 = HashKey^3<<1 (mod poly)
+        movdqa     \TMP5, HashKey_3(%rsp)
+        pshufd     $78, \TMP5, \TMP1
+        pxor       \TMP5, \TMP1
+        movdqa     \TMP1, HashKey_3_k(%rsp)
+.irpc index, 56789 # do next 5 rounds
+        movaps 0x10*\index(%arg1), \TMP1
+        AESENC     \TMP1, \XMM1
+        AESENC     \TMP1, \XMM2
+        AESENC     \TMP1, \XMM3
+        AESENC     \TMP1, \XMM4
+.endr
+        GHASH_MUL  \TMP5, \TMP3, \TMP1, \TMP2, \TMP4, \TMP6, \TMP7
+# TMP5 = HashKey^3<<1 (mod poly)
+        movdqa     \TMP5, HashKey_4(%rsp)
+        pshufd     $78, \TMP5, \TMP1
+        pxor       \TMP5, \TMP1
+        movdqa     \TMP1, HashKey_4_k(%rsp)
+        movaps 0xa0(%arg1), \TMP2
+        AESENCLAST \TMP2, \XMM1
+        AESENCLAST \TMP2, \XMM2
+        AESENCLAST \TMP2, \XMM3
+        AESENCLAST \TMP2, \XMM4
+        movdqu     16*0(%arg3 , %r11 , 1), \TMP1
+        pxor       \TMP1, \XMM1
+        movdqu     \XMM1, 16*0(%arg2 , %r11 , 1)
+        movdqa     \TMP1, \XMM1
+        movdqu     16*1(%arg3 , %r11 , 1), \TMP1
+        pxor       \TMP1, \XMM2
+        movdqu     \XMM2, 16*1(%arg2 , %r11 , 1)
+        movdqa     \TMP1, \XMM2
+        movdqu     16*2(%arg3 , %r11 , 1), \TMP1
+        pxor       \TMP1, \XMM3
+        movdqu     \XMM3, 16*2(%arg2 , %r11 , 1)
+        movdqa     \TMP1, \XMM3
+        movdqu     16*3(%arg3 , %r11 , 1), \TMP1
+        pxor       \TMP1, \XMM4
+        movdqu     \XMM4, 16*3(%arg2 , %r11 , 1)
+        movdqa     \TMP1, \XMM4
+        add        $64, %r11
+        movdqa     SHUF_MASK(%rip), %xmm14
+        PSHUFB_XMM %xmm14, \XMM1 # perform a 16 byte swap
+        pxor       \XMMDst, \XMM1
+# combine GHASHed value with the corresponding ciphertext
+        movdqa     SHUF_MASK(%rip), %xmm14
+        PSHUFB_XMM %xmm14, \XMM2 # perform a 16 byte swap
+        movdqa     SHUF_MASK(%rip), %xmm14
+        PSHUFB_XMM %xmm14, \XMM3 # perform a 16 byte swap
+        movdqa     SHUF_MASK(%rip), %xmm14
+        PSHUFB_XMM %xmm14, \XMM4 # perform a 16 byte swap
+
+_initial_blocks_done\num_initial_blocks\operation:
+
+.endm
+
+
+/*
+* if a = number of total plaintext bytes
+* b = floor(a/16)
+* num_initial_blocks = b mod 4
+* encrypt the initial num_initial_blocks blocks and apply ghash on
+* the ciphertext
+* %r10, %r11, %r12, %rax, %xmm5, %xmm6, %xmm7, %xmm8, %xmm9 registers
+* are clobbered
+* arg1, %arg2, %arg3, %r14 are used as a pointer only, not modified
+*/
+
+
+.macro INITIAL_BLOCKS_ENC num_initial_blocks TMP1 TMP2 TMP3 TMP4 TMP5 XMM0 XMM1 \
+XMM2 XMM3 XMM4 XMMDst TMP6 TMP7 i i_seq operation
+        mov        arg7, %r10           # %r10 = AAD
+        mov        arg8, %r12           # %r12 = aadLen
+        mov        %r12, %r11
+        pxor       %xmm\i, %xmm\i
+_get_AAD_loop\num_initial_blocks\operation:
+        movd       (%r10), \TMP1
+        pslldq     $12, \TMP1
+        psrldq     $4, %xmm\i
+        pxor       \TMP1, %xmm\i
+        add        $4, %r10
+        sub        $4, %r12
+        jne        _get_AAD_loop\num_initial_blocks\operation
+        cmp        $16, %r11
+        je         _get_AAD_loop2_done\num_initial_blocks\operation
+        mov        $16, %r12
+_get_AAD_loop2\num_initial_blocks\operation:
+        psrldq     $4, %xmm\i
+        sub        $4, %r12
+        cmp        %r11, %r12
+        jne        _get_AAD_loop2\num_initial_blocks\operation
+_get_AAD_loop2_done\num_initial_blocks\operation:
+        movdqa     SHUF_MASK(%rip), %xmm14
+        PSHUFB_XMM   %xmm14, %xmm\i # byte-reflect the AAD data
+
+        xor        %r11, %r11 # initialise the data pointer offset as zero
+
+        # start AES for num_initial_blocks blocks
+
+        mov        %arg5, %rax                      # %rax = *Y0
+        movdqu     (%rax), \XMM0                    # XMM0 = Y0
+        movdqa     SHUF_MASK(%rip), %xmm14
+        PSHUFB_XMM   %xmm14, \XMM0
+
+.if (\i == 5) || (\i == 6) || (\i == 7)
+.irpc index, \i_seq
+        paddd      ONE(%rip), \XMM0                 # INCR Y0
+        movdqa     \XMM0, %xmm\index
+        movdqa     SHUF_MASK(%rip), %xmm14
+        PSHUFB_XMM   %xmm14, %xmm\index      # perform a 16 byte swap
+
+.endr
+.irpc index, \i_seq
+        pxor       16*0(%arg1), %xmm\index
+.endr
+.irpc index, \i_seq
+        movaps 0x10(%rdi), \TMP1
+        AESENC     \TMP1, %xmm\index          # Round 1
+.endr
+.irpc index, \i_seq
+        movaps 0x20(%arg1), \TMP1
+        AESENC     \TMP1, %xmm\index          # Round 2
+.endr
+.irpc index, \i_seq
+        movaps 0x30(%arg1), \TMP1
+        AESENC     \TMP1, %xmm\index          # Round 2
+.endr
+.irpc index, \i_seq
+        movaps 0x40(%arg1), \TMP1
+        AESENC     \TMP1, %xmm\index          # Round 2
+.endr
+.irpc index, \i_seq
+        movaps 0x50(%arg1), \TMP1
+        AESENC     \TMP1, %xmm\index          # Round 2
+.endr
+.irpc index, \i_seq
+        movaps 0x60(%arg1), \TMP1
+        AESENC     \TMP1, %xmm\index          # Round 2
+.endr
+.irpc index, \i_seq
+        movaps 0x70(%arg1), \TMP1
+        AESENC     \TMP1, %xmm\index          # Round 2
+.endr
+.irpc index, \i_seq
+        movaps 0x80(%arg1), \TMP1
+        AESENC     \TMP1, %xmm\index          # Round 2
+.endr
+.irpc index, \i_seq
+        movaps 0x90(%arg1), \TMP1
+        AESENC     \TMP1, %xmm\index          # Round 2
+.endr
+.irpc index, \i_seq
+        movaps 0xa0(%arg1), \TMP1
+        AESENCLAST \TMP1, %xmm\index         # Round 10
+.endr
+.irpc index, \i_seq
+        movdqu     (%arg3 , %r11, 1), \TMP1
+        pxor       \TMP1, %xmm\index
+        movdqu     %xmm\index, (%arg2 , %r11, 1)
+        # write back plaintext/ciphertext for num_initial_blocks
+        add        $16, %r11
+
+        movdqa     SHUF_MASK(%rip), %xmm14
+        PSHUFB_XMM         %xmm14, %xmm\index
+
+                # prepare plaintext/ciphertext for GHASH computation
+.endr
+.endif
+        GHASH_MUL  %xmm\i, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
+        # apply GHASH on num_initial_blocks blocks
+
+.if \i == 5
+        pxor       %xmm5, %xmm6
+        GHASH_MUL  %xmm6, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
+        pxor       %xmm6, %xmm7
+        GHASH_MUL  %xmm7, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
+        pxor       %xmm7, %xmm8
+        GHASH_MUL  %xmm8, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
+.elseif \i == 6
+        pxor       %xmm6, %xmm7
+        GHASH_MUL  %xmm7, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
+        pxor       %xmm7, %xmm8
+        GHASH_MUL  %xmm8, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
+.elseif \i == 7
+        pxor       %xmm7, %xmm8
+        GHASH_MUL  %xmm8, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
+.endif
+        cmp        $64, %r13
+        jl      _initial_blocks_done\num_initial_blocks\operation
+        # no need for precomputed values
+/*
+*
+* Precomputations for HashKey parallel with encryption of first 4 blocks.
+* Haskey_i_k holds XORed values of the low and high parts of the Haskey_i
+*/
+        paddd      ONE(%rip), \XMM0              # INCR Y0
+        movdqa     \XMM0, \XMM1
+        movdqa     SHUF_MASK(%rip), %xmm14
+        PSHUFB_XMM  %xmm14, \XMM1        # perform a 16 byte swap
+
+        paddd      ONE(%rip), \XMM0              # INCR Y0
+        movdqa     \XMM0, \XMM2
+        movdqa     SHUF_MASK(%rip), %xmm14
+        PSHUFB_XMM  %xmm14, \XMM2        # perform a 16 byte swap
+
+        paddd      ONE(%rip), \XMM0              # INCR Y0
+        movdqa     \XMM0, \XMM3
+        movdqa     SHUF_MASK(%rip), %xmm14
+        PSHUFB_XMM %xmm14, \XMM3        # perform a 16 byte swap
+
+        paddd      ONE(%rip), \XMM0              # INCR Y0
+        movdqa     \XMM0, \XMM4
+        movdqa     SHUF_MASK(%rip), %xmm14
+        PSHUFB_XMM %xmm14, \XMM4        # perform a 16 byte swap
+
+        pxor       16*0(%arg1), \XMM1
+        pxor       16*0(%arg1), \XMM2
+        pxor       16*0(%arg1), \XMM3
+        pxor       16*0(%arg1), \XMM4
+        movdqa     \TMP3, \TMP5
+        pshufd     $78, \TMP3, \TMP1
+        pxor       \TMP3, \TMP1
+        movdqa     \TMP1, HashKey_k(%rsp)
+        GHASH_MUL  \TMP5, \TMP3, \TMP1, \TMP2, \TMP4, \TMP6, \TMP7
+# TMP5 = HashKey^2<<1 (mod poly)
+        movdqa     \TMP5, HashKey_2(%rsp)
+# HashKey_2 = HashKey^2<<1 (mod poly)
+        pshufd     $78, \TMP5, \TMP1
+        pxor       \TMP5, \TMP1
+        movdqa     \TMP1, HashKey_2_k(%rsp)
+.irpc index, 1234 # do 4 rounds
+        movaps 0x10*\index(%arg1), \TMP1
+        AESENC     \TMP1, \XMM1
+        AESENC     \TMP1, \XMM2
+        AESENC     \TMP1, \XMM3
+        AESENC     \TMP1, \XMM4
+.endr
+        GHASH_MUL  \TMP5, \TMP3, \TMP1, \TMP2, \TMP4, \TMP6, \TMP7
+# TMP5 = HashKey^3<<1 (mod poly)
+        movdqa     \TMP5, HashKey_3(%rsp)
+        pshufd     $78, \TMP5, \TMP1
+        pxor       \TMP5, \TMP1
+        movdqa     \TMP1, HashKey_3_k(%rsp)
+.irpc index, 56789 # do next 5 rounds
+        movaps 0x10*\index(%arg1), \TMP1
+        AESENC     \TMP1, \XMM1
+        AESENC     \TMP1, \XMM2
+        AESENC     \TMP1, \XMM3
+        AESENC     \TMP1, \XMM4
+.endr
+        GHASH_MUL  \TMP5, \TMP3, \TMP1, \TMP2, \TMP4, \TMP6, \TMP7
+# TMP5 = HashKey^3<<1 (mod poly)
+        movdqa     \TMP5, HashKey_4(%rsp)
+        pshufd     $78, \TMP5, \TMP1
+        pxor       \TMP5, \TMP1
+        movdqa     \TMP1, HashKey_4_k(%rsp)
+        movaps 0xa0(%arg1), \TMP2
+        AESENCLAST \TMP2, \XMM1
+        AESENCLAST \TMP2, \XMM2
+        AESENCLAST \TMP2, \XMM3
+        AESENCLAST \TMP2, \XMM4
+        movdqu     16*0(%arg3 , %r11 , 1), \TMP1
+        pxor       \TMP1, \XMM1
+        movdqu     16*1(%arg3 , %r11 , 1), \TMP1
+        pxor       \TMP1, \XMM2
+        movdqu     16*2(%arg3 , %r11 , 1), \TMP1
+        pxor       \TMP1, \XMM3
+        movdqu     16*3(%arg3 , %r11 , 1), \TMP1
+        pxor       \TMP1, \XMM4
+        movdqu     \XMM1, 16*0(%arg2 , %r11 , 1)
+        movdqu     \XMM2, 16*1(%arg2 , %r11 , 1)
+        movdqu     \XMM3, 16*2(%arg2 , %r11 , 1)
+        movdqu     \XMM4, 16*3(%arg2 , %r11 , 1)
+
+        add        $64, %r11
+        movdqa     SHUF_MASK(%rip), %xmm14
+        PSHUFB_XMM %xmm14, \XMM1 # perform a 16 byte swap
+        pxor       \XMMDst, \XMM1
+# combine GHASHed value with the corresponding ciphertext
+        movdqa     SHUF_MASK(%rip), %xmm14
+        PSHUFB_XMM %xmm14, \XMM2 # perform a 16 byte swap
+        movdqa     SHUF_MASK(%rip), %xmm14
+        PSHUFB_XMM %xmm14, \XMM3 # perform a 16 byte swap
+        movdqa     SHUF_MASK(%rip), %xmm14
+        PSHUFB_XMM %xmm14, \XMM4 # perform a 16 byte swap
+
+_initial_blocks_done\num_initial_blocks\operation:
+
+.endm
+
+/*
+* encrypt 4 blocks at a time
+* ghash the 4 previously encrypted ciphertext blocks
+* arg1, %arg2, %arg3 are used as pointers only, not modified
+* %r11 is the data offset value
+*/
+.macro GHASH_4_ENCRYPT_4_PARALLEL_ENC TMP1 TMP2 TMP3 TMP4 TMP5 \
+TMP6 XMM0 XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 operation
+
+        movdqa    \XMM1, \XMM5
+        movdqa    \XMM2, \XMM6
+        movdqa    \XMM3, \XMM7
+        movdqa    \XMM4, \XMM8
+
+        movdqa    SHUF_MASK(%rip), %xmm15
+        # multiply TMP5 * HashKey using karatsuba
+
+        movdqa    \XMM5, \TMP4
+        pshufd    $78, \XMM5, \TMP6
+        pxor      \XMM5, \TMP6
+        paddd     ONE(%rip), \XMM0              # INCR CNT
+        movdqa    HashKey_4(%rsp), \TMP5
+        PCLMULQDQ 0x11, \TMP5, \TMP4           # TMP4 = a1*b1
+        movdqa    \XMM0, \XMM1
+        paddd     ONE(%rip), \XMM0              # INCR CNT
+        movdqa    \XMM0, \XMM2
+        paddd     ONE(%rip), \XMM0              # INCR CNT
+        movdqa    \XMM0, \XMM3
+        paddd     ONE(%rip), \XMM0              # INCR CNT
+        movdqa    \XMM0, \XMM4
+        PSHUFB_XMM %xmm15, \XMM1        # perform a 16 byte swap
+        PCLMULQDQ 0x00, \TMP5, \XMM5           # XMM5 = a0*b0
+        PSHUFB_XMM %xmm15, \XMM2        # perform a 16 byte swap
+        PSHUFB_XMM %xmm15, \XMM3        # perform a 16 byte swap
+        PSHUFB_XMM %xmm15, \XMM4        # perform a 16 byte swap
+
+        pxor      (%arg1), \XMM1
+        pxor      (%arg1), \XMM2
+        pxor      (%arg1), \XMM3
+        pxor      (%arg1), \XMM4
+        movdqa    HashKey_4_k(%rsp), \TMP5
+        PCLMULQDQ 0x00, \TMP5, \TMP6           # TMP6 = (a1+a0)*(b1+b0)
+        movaps 0x10(%arg1), \TMP1
+        AESENC    \TMP1, \XMM1              # Round 1
+        AESENC    \TMP1, \XMM2
+        AESENC    \TMP1, \XMM3
+        AESENC    \TMP1, \XMM4
+        movaps 0x20(%arg1), \TMP1
+        AESENC    \TMP1, \XMM1              # Round 2
+        AESENC    \TMP1, \XMM2
+        AESENC    \TMP1, \XMM3
+        AESENC    \TMP1, \XMM4
+        movdqa    \XMM6, \TMP1
+        pshufd    $78, \XMM6, \TMP2
+        pxor      \XMM6, \TMP2
+        movdqa    HashKey_3(%rsp), \TMP5
+        PCLMULQDQ 0x11, \TMP5, \TMP1           # TMP1 = a1 * b1
+        movaps 0x30(%arg1), \TMP3
+        AESENC    \TMP3, \XMM1              # Round 3
+        AESENC    \TMP3, \XMM2
+        AESENC    \TMP3, \XMM3
+        AESENC    \TMP3, \XMM4
+        PCLMULQDQ 0x00, \TMP5, \XMM6           # XMM6 = a0*b0
+        movaps 0x40(%arg1), \TMP3
+        AESENC    \TMP3, \XMM1              # Round 4
+        AESENC    \TMP3, \XMM2
+        AESENC    \TMP3, \XMM3
+        AESENC    \TMP3, \XMM4
+        movdqa    HashKey_3_k(%rsp), \TMP5
+        PCLMULQDQ 0x00, \TMP5, \TMP2           # TMP2 = (a1+a0)*(b1+b0)
+        movaps 0x50(%arg1), \TMP3
+        AESENC    \TMP3, \XMM1              # Round 5
+        AESENC    \TMP3, \XMM2
+        AESENC    \TMP3, \XMM3
+        AESENC    \TMP3, \XMM4
+        pxor      \TMP1, \TMP4
+# accumulate the results in TMP4:XMM5, TMP6 holds the middle part
+        pxor      \XMM6, \XMM5
+        pxor      \TMP2, \TMP6
+        movdqa    \XMM7, \TMP1
+        pshufd    $78, \XMM7, \TMP2
+        pxor      \XMM7, \TMP2
+        movdqa    HashKey_2(%rsp ), \TMP5
+
+        # Multiply TMP5 * HashKey using karatsuba
+
+        PCLMULQDQ 0x11, \TMP5, \TMP1           # TMP1 = a1*b1
+        movaps 0x60(%arg1), \TMP3
+        AESENC    \TMP3, \XMM1              # Round 6
+        AESENC    \TMP3, \XMM2
+        AESENC    \TMP3, \XMM3
+        AESENC    \TMP3, \XMM4
+        PCLMULQDQ 0x00, \TMP5, \XMM7           # XMM7 = a0*b0
+        movaps 0x70(%arg1), \TMP3
+        AESENC    \TMP3, \XMM1             # Round 7
+        AESENC    \TMP3, \XMM2
+        AESENC    \TMP3, \XMM3
+        AESENC    \TMP3, \XMM4
+        movdqa    HashKey_2_k(%rsp), \TMP5
+        PCLMULQDQ 0x00, \TMP5, \TMP2           # TMP2 = (a1+a0)*(b1+b0)
+        movaps 0x80(%arg1), \TMP3
+        AESENC    \TMP3, \XMM1             # Round 8
+        AESENC    \TMP3, \XMM2
+        AESENC    \TMP3, \XMM3
+        AESENC    \TMP3, \XMM4
+        pxor      \TMP1, \TMP4
+# accumulate the results in TMP4:XMM5, TMP6 holds the middle part
+        pxor      \XMM7, \XMM5
+        pxor      \TMP2, \TMP6
+
+        # Multiply XMM8 * HashKey
+        # XMM8 and TMP5 hold the values for the two operands
+
+        movdqa    \XMM8, \TMP1
+        pshufd    $78, \XMM8, \TMP2
+        pxor      \XMM8, \TMP2
+        movdqa    HashKey(%rsp), \TMP5
+        PCLMULQDQ 0x11, \TMP5, \TMP1          # TMP1 = a1*b1
+        movaps 0x90(%arg1), \TMP3
+        AESENC    \TMP3, \XMM1            # Round 9
+        AESENC    \TMP3, \XMM2
+        AESENC    \TMP3, \XMM3
+        AESENC    \TMP3, \XMM4
+        PCLMULQDQ 0x00, \TMP5, \XMM8          # XMM8 = a0*b0
+        movaps 0xa0(%arg1), \TMP3
+        AESENCLAST \TMP3, \XMM1           # Round 10
+        AESENCLAST \TMP3, \XMM2
+        AESENCLAST \TMP3, \XMM3
+        AESENCLAST \TMP3, \XMM4
+        movdqa    HashKey_k(%rsp), \TMP5
+        PCLMULQDQ 0x00, \TMP5, \TMP2          # TMP2 = (a1+a0)*(b1+b0)
+        movdqu    (%arg3,%r11,1), \TMP3
+        pxor      \TMP3, \XMM1                 # Ciphertext/Plaintext XOR EK
+        movdqu    16(%arg3,%r11,1), \TMP3
+        pxor      \TMP3, \XMM2                 # Ciphertext/Plaintext XOR EK
+        movdqu    32(%arg3,%r11,1), \TMP3
+        pxor      \TMP3, \XMM3                 # Ciphertext/Plaintext XOR EK
+        movdqu    48(%arg3,%r11,1), \TMP3
+        pxor      \TMP3, \XMM4                 # Ciphertext/Plaintext XOR EK
+        movdqu    \XMM1, (%arg2,%r11,1)        # Write to the ciphertext buffer
+        movdqu    \XMM2, 16(%arg2,%r11,1)      # Write to the ciphertext buffer
+        movdqu    \XMM3, 32(%arg2,%r11,1)      # Write to the ciphertext buffer
+        movdqu    \XMM4, 48(%arg2,%r11,1)      # Write to the ciphertext buffer
+        PSHUFB_XMM %xmm15, \XMM1        # perform a 16 byte swap
+        PSHUFB_XMM %xmm15, \XMM2        # perform a 16 byte swap
+        PSHUFB_XMM %xmm15, \XMM3        # perform a 16 byte swap
+        PSHUFB_XMM %xmm15, \XMM4        # perform a 16 byte swap
+
+        pxor      \TMP4, \TMP1
+        pxor      \XMM8, \XMM5
+        pxor      \TMP6, \TMP2
+        pxor      \TMP1, \TMP2
+        pxor      \XMM5, \TMP2
+        movdqa    \TMP2, \TMP3
+        pslldq    $8, \TMP3                    # left shift TMP3 2 DWs
+        psrldq    $8, \TMP2                    # right shift TMP2 2 DWs
+        pxor      \TMP3, \XMM5
+        pxor      \TMP2, \TMP1    # accumulate the results in TMP1:XMM5
+
+        # first phase of reduction
+
+        movdqa    \XMM5, \TMP2
+        movdqa    \XMM5, \TMP3
+        movdqa    \XMM5, \TMP4
+# move XMM5 into TMP2, TMP3, TMP4 in order to perform shifts independently
+        pslld     $31, \TMP2                   # packed right shift << 31
+        pslld     $30, \TMP3                   # packed right shift << 30
+        pslld     $25, \TMP4                   # packed right shift << 25
+        pxor      \TMP3, \TMP2                 # xor the shifted versions
+        pxor      \TMP4, \TMP2
+        movdqa    \TMP2, \TMP5
+        psrldq    $4, \TMP5                    # right shift T5 1 DW
+        pslldq    $12, \TMP2                   # left shift T2 3 DWs
+        pxor      \TMP2, \XMM5
+
+        # second phase of reduction
+
+        movdqa    \XMM5,\TMP2 # make 3 copies of XMM5 into TMP2, TMP3, TMP4
+        movdqa    \XMM5,\TMP3
+        movdqa    \XMM5,\TMP4
+        psrld     $1, \TMP2                    # packed left shift >>1
+        psrld     $2, \TMP3                    # packed left shift >>2
+        psrld     $7, \TMP4                    # packed left shift >>7
+        pxor      \TMP3,\TMP2                  # xor the shifted versions
+        pxor      \TMP4,\TMP2
+        pxor      \TMP5, \TMP2
+        pxor      \TMP2, \XMM5
+        pxor      \TMP1, \XMM5                 # result is in TMP1
+
+        pxor      \XMM5, \XMM1
+.endm
+
+/*
+* decrypt 4 blocks at a time
+* ghash the 4 previously decrypted ciphertext blocks
+* arg1, %arg2, %arg3 are used as pointers only, not modified
+* %r11 is the data offset value
+*/
+.macro GHASH_4_ENCRYPT_4_PARALLEL_DEC TMP1 TMP2 TMP3 TMP4 TMP5 \
+TMP6 XMM0 XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 operation
+
+        movdqa    \XMM1, \XMM5
+        movdqa    \XMM2, \XMM6
+        movdqa    \XMM3, \XMM7
+        movdqa    \XMM4, \XMM8
+
+        movdqa    SHUF_MASK(%rip), %xmm15
+        # multiply TMP5 * HashKey using karatsuba
+
+        movdqa    \XMM5, \TMP4
+        pshufd    $78, \XMM5, \TMP6
+        pxor      \XMM5, \TMP6
+        paddd     ONE(%rip), \XMM0              # INCR CNT
+        movdqa    HashKey_4(%rsp), \TMP5
+        PCLMULQDQ 0x11, \TMP5, \TMP4           # TMP4 = a1*b1
+        movdqa    \XMM0, \XMM1
+        paddd     ONE(%rip), \XMM0              # INCR CNT
+        movdqa    \XMM0, \XMM2
+        paddd     ONE(%rip), \XMM0              # INCR CNT
+        movdqa    \XMM0, \XMM3
+        paddd     ONE(%rip), \XMM0              # INCR CNT
+        movdqa    \XMM0, \XMM4
+        PSHUFB_XMM %xmm15, \XMM1        # perform a 16 byte swap
+        PCLMULQDQ 0x00, \TMP5, \XMM5           # XMM5 = a0*b0
+        PSHUFB_XMM %xmm15, \XMM2        # perform a 16 byte swap
+        PSHUFB_XMM %xmm15, \XMM3        # perform a 16 byte swap
+        PSHUFB_XMM %xmm15, \XMM4        # perform a 16 byte swap
+
+        pxor      (%arg1), \XMM1
+        pxor      (%arg1), \XMM2
+        pxor      (%arg1), \XMM3
+        pxor      (%arg1), \XMM4
+        movdqa    HashKey_4_k(%rsp), \TMP5
+        PCLMULQDQ 0x00, \TMP5, \TMP6           # TMP6 = (a1+a0)*(b1+b0)
+        movaps 0x10(%arg1), \TMP1
+        AESENC    \TMP1, \XMM1              # Round 1
+        AESENC    \TMP1, \XMM2
+        AESENC    \TMP1, \XMM3
+        AESENC    \TMP1, \XMM4
+        movaps 0x20(%arg1), \TMP1
+        AESENC    \TMP1, \XMM1              # Round 2
+        AESENC    \TMP1, \XMM2
+        AESENC    \TMP1, \XMM3
+        AESENC    \TMP1, \XMM4
+        movdqa    \XMM6, \TMP1
+        pshufd    $78, \XMM6, \TMP2
+        pxor      \XMM6, \TMP2
+        movdqa    HashKey_3(%rsp), \TMP5
+        PCLMULQDQ 0x11, \TMP5, \TMP1           # TMP1 = a1 * b1
+        movaps 0x30(%arg1), \TMP3
+        AESENC    \TMP3, \XMM1              # Round 3
+        AESENC    \TMP3, \XMM2
+        AESENC    \TMP3, \XMM3
+        AESENC    \TMP3, \XMM4
+        PCLMULQDQ 0x00, \TMP5, \XMM6           # XMM6 = a0*b0
+        movaps 0x40(%arg1), \TMP3
+        AESENC    \TMP3, \XMM1              # Round 4
+        AESENC    \TMP3, \XMM2
+        AESENC    \TMP3, \XMM3
+        AESENC    \TMP3, \XMM4
+        movdqa    HashKey_3_k(%rsp), \TMP5
+        PCLMULQDQ 0x00, \TMP5, \TMP2           # TMP2 = (a1+a0)*(b1+b0)
+        movaps 0x50(%arg1), \TMP3
+        AESENC    \TMP3, \XMM1              # Round 5
+        AESENC    \TMP3, \XMM2
+        AESENC    \TMP3, \XMM3
+        AESENC    \TMP3, \XMM4
+        pxor      \TMP1, \TMP4
+# accumulate the results in TMP4:XMM5, TMP6 holds the middle part
+        pxor      \XMM6, \XMM5
+        pxor      \TMP2, \TMP6
+        movdqa    \XMM7, \TMP1
+        pshufd    $78, \XMM7, \TMP2
+        pxor      \XMM7, \TMP2
+        movdqa    HashKey_2(%rsp ), \TMP5
+
+        # Multiply TMP5 * HashKey using karatsuba
+
+        PCLMULQDQ 0x11, \TMP5, \TMP1           # TMP1 = a1*b1
+        movaps 0x60(%arg1), \TMP3
+        AESENC    \TMP3, \XMM1              # Round 6
+        AESENC    \TMP3, \XMM2
+        AESENC    \TMP3, \XMM3
+        AESENC    \TMP3, \XMM4
+        PCLMULQDQ 0x00, \TMP5, \XMM7           # XMM7 = a0*b0
+        movaps 0x70(%arg1), \TMP3
+        AESENC    \TMP3, \XMM1             # Round 7
+        AESENC    \TMP3, \XMM2
+        AESENC    \TMP3, \XMM3
+        AESENC    \TMP3, \XMM4
+        movdqa    HashKey_2_k(%rsp), \TMP5
+        PCLMULQDQ 0x00, \TMP5, \TMP2           # TMP2 = (a1+a0)*(b1+b0)
+        movaps 0x80(%arg1), \TMP3
+        AESENC    \TMP3, \XMM1             # Round 8
+        AESENC    \TMP3, \XMM2
+        AESENC    \TMP3, \XMM3
+        AESENC    \TMP3, \XMM4
+        pxor      \TMP1, \TMP4
+# accumulate the results in TMP4:XMM5, TMP6 holds the middle part
+        pxor      \XMM7, \XMM5
+        pxor      \TMP2, \TMP6
+
+        # Multiply XMM8 * HashKey
+        # XMM8 and TMP5 hold the values for the two operands
+
+        movdqa    \XMM8, \TMP1
+        pshufd    $78, \XMM8, \TMP2
+        pxor      \XMM8, \TMP2
+        movdqa    HashKey(%rsp), \TMP5
+        PCLMULQDQ 0x11, \TMP5, \TMP1          # TMP1 = a1*b1
+        movaps 0x90(%arg1), \TMP3
+        AESENC    \TMP3, \XMM1            # Round 9
+        AESENC    \TMP3, \XMM2
+        AESENC    \TMP3, \XMM3
+        AESENC    \TMP3, \XMM4
+        PCLMULQDQ 0x00, \TMP5, \XMM8          # XMM8 = a0*b0
+        movaps 0xa0(%arg1), \TMP3
+        AESENCLAST \TMP3, \XMM1           # Round 10
+        AESENCLAST \TMP3, \XMM2
+        AESENCLAST \TMP3, \XMM3
+        AESENCLAST \TMP3, \XMM4
+        movdqa    HashKey_k(%rsp), \TMP5
+        PCLMULQDQ 0x00, \TMP5, \TMP2          # TMP2 = (a1+a0)*(b1+b0)
+        movdqu    (%arg3,%r11,1), \TMP3
+        pxor      \TMP3, \XMM1                 # Ciphertext/Plaintext XOR EK
+        movdqu    \XMM1, (%arg2,%r11,1)        # Write to plaintext buffer
+        movdqa    \TMP3, \XMM1
+        movdqu    16(%arg3,%r11,1), \TMP3
+        pxor      \TMP3, \XMM2                 # Ciphertext/Plaintext XOR EK
+        movdqu    \XMM2, 16(%arg2,%r11,1)      # Write to plaintext buffer
+        movdqa    \TMP3, \XMM2
+        movdqu    32(%arg3,%r11,1), \TMP3
+        pxor      \TMP3, \XMM3                 # Ciphertext/Plaintext XOR EK
+        movdqu    \XMM3, 32(%arg2,%r11,1)      # Write to plaintext buffer
+        movdqa    \TMP3, \XMM3
+        movdqu    48(%arg3,%r11,1), \TMP3
+        pxor      \TMP3, \XMM4                 # Ciphertext/Plaintext XOR EK
+        movdqu    \XMM4, 48(%arg2,%r11,1)      # Write to plaintext buffer
+        movdqa    \TMP3, \XMM4
+        PSHUFB_XMM %xmm15, \XMM1        # perform a 16 byte swap
+        PSHUFB_XMM %xmm15, \XMM2        # perform a 16 byte swap
+        PSHUFB_XMM %xmm15, \XMM3        # perform a 16 byte swap
+        PSHUFB_XMM %xmm15, \XMM4        # perform a 16 byte swap
+
+        pxor      \TMP4, \TMP1
+        pxor      \XMM8, \XMM5
+        pxor      \TMP6, \TMP2
+        pxor      \TMP1, \TMP2
+        pxor      \XMM5, \TMP2
+        movdqa    \TMP2, \TMP3
+        pslldq    $8, \TMP3                    # left shift TMP3 2 DWs
+        psrldq    $8, \TMP2                    # right shift TMP2 2 DWs
+        pxor      \TMP3, \XMM5
+        pxor      \TMP2, \TMP1    # accumulate the results in TMP1:XMM5
+
+        # first phase of reduction
+
+        movdqa    \XMM5, \TMP2
+        movdqa    \XMM5, \TMP3
+        movdqa    \XMM5, \TMP4
+# move XMM5 into TMP2, TMP3, TMP4 in order to perform shifts independently
+        pslld     $31, \TMP2                   # packed right shift << 31
+        pslld     $30, \TMP3                   # packed right shift << 30
+        pslld     $25, \TMP4                   # packed right shift << 25
+        pxor      \TMP3, \TMP2                 # xor the shifted versions
+        pxor      \TMP4, \TMP2
+        movdqa    \TMP2, \TMP5
+        psrldq    $4, \TMP5                    # right shift T5 1 DW
+        pslldq    $12, \TMP2                   # left shift T2 3 DWs
+        pxor      \TMP2, \XMM5
+
+        # second phase of reduction
+
+        movdqa    \XMM5,\TMP2 # make 3 copies of XMM5 into TMP2, TMP3, TMP4
+        movdqa    \XMM5,\TMP3
+        movdqa    \XMM5,\TMP4
+        psrld     $1, \TMP2                    # packed left shift >>1
+        psrld     $2, \TMP3                    # packed left shift >>2
+        psrld     $7, \TMP4                    # packed left shift >>7
+        pxor      \TMP3,\TMP2                  # xor the shifted versions
+        pxor      \TMP4,\TMP2
+        pxor      \TMP5, \TMP2
+        pxor      \TMP2, \XMM5
+        pxor      \TMP1, \XMM5                 # result is in TMP1
+
+        pxor      \XMM5, \XMM1
+.endm
+
+/* GHASH the last 4 ciphertext blocks. */
+.macro  GHASH_LAST_4 TMP1 TMP2 TMP3 TMP4 TMP5 TMP6 \
+TMP7 XMM1 XMM2 XMM3 XMM4 XMMDst
+
+        # Multiply TMP6 * HashKey (using Karatsuba)
+
+        movdqa    \XMM1, \TMP6
+        pshufd    $78, \XMM1, \TMP2
+        pxor      \XMM1, \TMP2
+        movdqa    HashKey_4(%rsp), \TMP5
+        PCLMULQDQ 0x11, \TMP5, \TMP6       # TMP6 = a1*b1
+        PCLMULQDQ 0x00, \TMP5, \XMM1       # XMM1 = a0*b0
+        movdqa    HashKey_4_k(%rsp), \TMP4
+        PCLMULQDQ 0x00, \TMP4, \TMP2       # TMP2 = (a1+a0)*(b1+b0)
+        movdqa    \XMM1, \XMMDst
+        movdqa    \TMP2, \XMM1              # result in TMP6, XMMDst, XMM1
+
+        # Multiply TMP1 * HashKey (using Karatsuba)
+
+        movdqa    \XMM2, \TMP1
+        pshufd    $78, \XMM2, \TMP2
+        pxor      \XMM2, \TMP2
+        movdqa    HashKey_3(%rsp), \TMP5
+        PCLMULQDQ 0x11, \TMP5, \TMP1       # TMP1 = a1*b1
+        PCLMULQDQ 0x00, \TMP5, \XMM2       # XMM2 = a0*b0
+        movdqa    HashKey_3_k(%rsp), \TMP4
+        PCLMULQDQ 0x00, \TMP4, \TMP2       # TMP2 = (a1+a0)*(b1+b0)
+        pxor      \TMP1, \TMP6
+        pxor      \XMM2, \XMMDst
+        pxor      \TMP2, \XMM1
+# results accumulated in TMP6, XMMDst, XMM1
+
+        # Multiply TMP1 * HashKey (using Karatsuba)
+
+        movdqa    \XMM3, \TMP1
+        pshufd    $78, \XMM3, \TMP2
+        pxor      \XMM3, \TMP2
+        movdqa    HashKey_2(%rsp), \TMP5
+        PCLMULQDQ 0x11, \TMP5, \TMP1       # TMP1 = a1*b1
+        PCLMULQDQ 0x00, \TMP5, \XMM3       # XMM3 = a0*b0
+        movdqa    HashKey_2_k(%rsp), \TMP4
+        PCLMULQDQ 0x00, \TMP4, \TMP2       # TMP2 = (a1+a0)*(b1+b0)
+        pxor      \TMP1, \TMP6
+        pxor      \XMM3, \XMMDst
+        pxor      \TMP2, \XMM1   # results accumulated in TMP6, XMMDst, XMM1
+
+        # Multiply TMP1 * HashKey (using Karatsuba)
+        movdqa    \XMM4, \TMP1
+        pshufd    $78, \XMM4, \TMP2
+        pxor      \XMM4, \TMP2
+        movdqa    HashKey(%rsp), \TMP5
+        PCLMULQDQ 0x11, \TMP5, \TMP1        # TMP1 = a1*b1
+        PCLMULQDQ 0x00, \TMP5, \XMM4       # XMM4 = a0*b0
+        movdqa    HashKey_k(%rsp), \TMP4
+        PCLMULQDQ 0x00, \TMP4, \TMP2       # TMP2 = (a1+a0)*(b1+b0)
+        pxor      \TMP1, \TMP6
+        pxor      \XMM4, \XMMDst
+        pxor      \XMM1, \TMP2
+        pxor      \TMP6, \TMP2
+        pxor      \XMMDst, \TMP2
+        # middle section of the temp results combined as in karatsuba algorithm
+        movdqa    \TMP2, \TMP4
+        pslldq    $8, \TMP4                 # left shift TMP4 2 DWs
+        psrldq    $8, \TMP2                 # right shift TMP2 2 DWs
+        pxor      \TMP4, \XMMDst
+        pxor      \TMP2, \TMP6
+# TMP6:XMMDst holds the result of the accumulated carry-less multiplications
+        # first phase of the reduction
+        movdqa    \XMMDst, \TMP2
+        movdqa    \XMMDst, \TMP3
+        movdqa    \XMMDst, \TMP4
+# move XMMDst into TMP2, TMP3, TMP4 in order to perform 3 shifts independently
+        pslld     $31, \TMP2                # packed right shifting << 31
+        pslld     $30, \TMP3                # packed right shifting << 30
+        pslld     $25, \TMP4                # packed right shifting << 25
+        pxor      \TMP3, \TMP2              # xor the shifted versions
+        pxor      \TMP4, \TMP2
+        movdqa    \TMP2, \TMP7
+        psrldq    $4, \TMP7                 # right shift TMP7 1 DW
+        pslldq    $12, \TMP2                # left shift TMP2 3 DWs
+        pxor      \TMP2, \XMMDst
+
+        # second phase of the reduction
+        movdqa    \XMMDst, \TMP2
+        # make 3 copies of XMMDst for doing 3 shift operations
+        movdqa    \XMMDst, \TMP3
+        movdqa    \XMMDst, \TMP4
+        psrld     $1, \TMP2                 # packed left shift >> 1
+        psrld     $2, \TMP3                 # packed left shift >> 2
+        psrld     $7, \TMP4                 # packed left shift >> 7
+        pxor      \TMP3, \TMP2              # xor the shifted versions
+        pxor      \TMP4, \TMP2
+        pxor      \TMP7, \TMP2
+        pxor      \TMP2, \XMMDst
+        pxor      \TMP6, \XMMDst            # reduced result is in XMMDst
+.endm
+
+/* Encryption of a single block done*/
+.macro ENCRYPT_SINGLE_BLOCK XMM0 TMP1
+
+        pxor    (%arg1), \XMM0
+        movaps 16(%arg1), \TMP1
+        AESENC  \TMP1, \XMM0
+        movaps 32(%arg1), \TMP1
+        AESENC  \TMP1, \XMM0
+        movaps 48(%arg1), \TMP1
+        AESENC  \TMP1, \XMM0
+        movaps 64(%arg1), \TMP1
+        AESENC  \TMP1, \XMM0
+        movaps 80(%arg1), \TMP1
+        AESENC  \TMP1, \XMM0
+        movaps 96(%arg1), \TMP1
+        AESENC  \TMP1, \XMM0
+        movaps 112(%arg1), \TMP1
+        AESENC  \TMP1, \XMM0
+        movaps 128(%arg1), \TMP1
+        AESENC  \TMP1, \XMM0
+        movaps 144(%arg1), \TMP1
+        AESENC  \TMP1, \XMM0
+        movaps 160(%arg1), \TMP1
+        AESENCLAST      \TMP1, \XMM0
+.endm
+
+
+/*****************************************************************************
+* void aesni_gcm_dec(void *aes_ctx,    // AES Key schedule. Starts on a 16 byte boundary.
+*                   u8 *out,           // Plaintext output. Encrypt in-place is allowed.
+*                   const u8 *in,      // Ciphertext input
+*                   u64 plaintext_len, // Length of data in bytes for decryption.
+*                   u8 *iv,            // Pre-counter block j0: 4 byte salt (from Security Association)
+*                                      // concatenated with 8 byte Initialisation Vector (from IPSec ESP Payload)
+*                                      // concatenated with 0x00000001. 16-byte aligned pointer.
+*                   u8 *hash_subkey,   // H, the Hash sub key input. Data starts on a 16-byte boundary.
+*                   const u8 *aad,     // Additional Authentication Data (AAD)
+*                   u64 aad_len,       // Length of AAD in bytes. With RFC4106 this is going to be 8 or 12 bytes
+*                   u8  *auth_tag,     // Authenticated Tag output. The driver will compare this to the
+*                                      // given authentication tag and only return the plaintext if they match.
+*                   u64 auth_tag_len); // Authenticated Tag Length in bytes. Valid values are 16
+*                                      // (most likely), 12 or 8.
+*
+* Assumptions:
+*
+* keys:
+*       keys are pre-expanded and aligned to 16 bytes. we are using the first
+*       set of 11 keys in the data structure void *aes_ctx
+*
+* iv:
+*       0                   1                   2                   3
+*       0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+*       |                             Salt  (From the SA)               |
+*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+*       |                     Initialization Vector                     |
+*       |         (This is the sequence number from IPSec header)       |
+*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+*       |                              0x1                              |
+*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+*
+*
+*
+* AAD:
+*       AAD padded to 128 bits with 0
+*       for example, assume AAD is a u32 vector
+*
+*       if AAD is 8 bytes:
+*       AAD[3] = {A0, A1};
+*       padded AAD in xmm register = {A1 A0 0 0}
+*
+*       0                   1                   2                   3
+*       0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+*       |                               SPI (A1)                        |
+*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+*       |                     32-bit Sequence Number (A0)               |
+*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+*       |                              0x0                              |
+*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+*
+*                                       AAD Format with 32-bit Sequence Number
+*
+*       if AAD is 12 bytes:
+*       AAD[3] = {A0, A1, A2};
+*       padded AAD in xmm register = {A2 A1 A0 0}
+*
+*       0                   1                   2                   3
+*       0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+*       0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+*       |                               SPI (A2)                        |
+*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+*       |                 64-bit Extended Sequence Number {A1,A0}       |
+*       |                                                               |
+*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+*       |                              0x0                              |
+*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+*
+*                        AAD Format with 64-bit Extended Sequence Number
+*
+* aadLen:
+*       from the definition of the spec, aadLen can only be 8 or 12 bytes.
+*       The code supports 16 too but for other sizes, the code will fail.
+*
+* TLen:
+*       from the definition of the spec, TLen can only be 8, 12 or 16 bytes.
+*       For other sizes, the code will fail.
+*
+* poly = x^128 + x^127 + x^126 + x^121 + 1
+*
+*****************************************************************************/
+
+ENTRY(aesni_gcm_dec)
+        push    %r12
+        push    %r13
+        push    %r14
+        mov     %rsp, %r14
+/*
+* states of %xmm registers %xmm6:%xmm15 not saved
+* all %xmm registers are clobbered
+*/
+        sub     $VARIABLE_OFFSET, %rsp
+        and     $~63, %rsp                        # align rsp to 64 bytes
+        mov     %arg6, %r12
+        movdqu  (%r12), %xmm13                    # %xmm13 = HashKey
+        movdqa  SHUF_MASK(%rip), %xmm2
+        PSHUFB_XMM %xmm2, %xmm13
+
+
+# Precompute HashKey<<1 (mod poly) from the hash key (required for GHASH)
+
+        movdqa  %xmm13, %xmm2
+        psllq   $1, %xmm13
+        psrlq   $63, %xmm2
+        movdqa  %xmm2, %xmm1
+        pslldq  $8, %xmm2
+        psrldq  $8, %xmm1
+        por     %xmm2, %xmm13
+
+        # Reduction
+
+        pshufd  $0x24, %xmm1, %xmm2
+        pcmpeqd TWOONE(%rip), %xmm2
+        pand    POLY(%rip), %xmm2
+        pxor    %xmm2, %xmm13     # %xmm13 holds the HashKey<<1 (mod poly)
+
+
+        # Decrypt first few blocks
+
+        movdqa %xmm13, HashKey(%rsp)           # store HashKey<<1 (mod poly)
+        mov %arg4, %r13    # save the number of bytes of plaintext/ciphertext
+        and $-16, %r13                      # %r13 = %r13 - (%r13 mod 16)
+        mov %r13, %r12
+        and $(3<<4), %r12
+        jz _initial_num_blocks_is_0_decrypt
+        cmp $(2<<4), %r12
+        jb _initial_num_blocks_is_1_decrypt
+        je _initial_num_blocks_is_2_decrypt
+_initial_num_blocks_is_3_decrypt:
+        INITIAL_BLOCKS_DEC 3, %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \
+%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 5, 678, dec
+        sub     $48, %r13
+        jmp     _initial_blocks_decrypted
+_initial_num_blocks_is_2_decrypt:
+        INITIAL_BLOCKS_DEC      2, %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \
+%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 6, 78, dec
+        sub     $32, %r13
+        jmp     _initial_blocks_decrypted
+_initial_num_blocks_is_1_decrypt:
+        INITIAL_BLOCKS_DEC      1, %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \
+%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 7, 8, dec
+        sub     $16, %r13
+        jmp     _initial_blocks_decrypted
+_initial_num_blocks_is_0_decrypt:
+        INITIAL_BLOCKS_DEC      0, %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \
+%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 8, 0, dec
+_initial_blocks_decrypted:
+        cmp     $0, %r13
+        je      _zero_cipher_left_decrypt
+        sub     $64, %r13
+        je      _four_cipher_left_decrypt
+_decrypt_by_4:
+        GHASH_4_ENCRYPT_4_PARALLEL_DEC  %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, \
+%xmm14, %xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, dec
+        add     $64, %r11
+        sub     $64, %r13
+        jne     _decrypt_by_4
+_four_cipher_left_decrypt:
+        GHASH_LAST_4    %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, \
+%xmm15, %xmm1, %xmm2, %xmm3, %xmm4, %xmm8
+_zero_cipher_left_decrypt:
+        mov     %arg4, %r13
+        and     $15, %r13                               # %r13 = arg4 (mod 16)
+        je      _multiple_of_16_bytes_decrypt
+
+        # Handle the last <16 byte block seperately
+
+        paddd ONE(%rip), %xmm0         # increment CNT to get Yn
+        movdqa SHUF_MASK(%rip), %xmm10
+        PSHUFB_XMM %xmm10, %xmm0
+
+        ENCRYPT_SINGLE_BLOCK  %xmm0, %xmm1    # E(K, Yn)
+        sub $16, %r11
+        add %r13, %r11
+        movdqu (%arg3,%r11,1), %xmm1   # recieve the last <16 byte block
+        lea SHIFT_MASK+16(%rip), %r12
+        sub %r13, %r12
+# adjust the shuffle mask pointer to be able to shift 16-%r13 bytes
+# (%r13 is the number of bytes in plaintext mod 16)
+        movdqu (%r12), %xmm2           # get the appropriate shuffle mask
+        PSHUFB_XMM %xmm2, %xmm1            # right shift 16-%r13 butes
+
+        movdqa  %xmm1, %xmm2
+        pxor %xmm1, %xmm0            # Ciphertext XOR E(K, Yn)
+        movdqu ALL_F-SHIFT_MASK(%r12), %xmm1
+        # get the appropriate mask to mask out top 16-%r13 bytes of %xmm0
+        pand %xmm1, %xmm0            # mask out top 16-%r13 bytes of %xmm0
+        pand    %xmm1, %xmm2
+        movdqa SHUF_MASK(%rip), %xmm10
+        PSHUFB_XMM %xmm10 ,%xmm2
+
+        pxor %xmm2, %xmm8
+        GHASH_MUL %xmm8, %xmm13, %xmm9, %xmm10, %xmm11, %xmm5, %xmm6
+                  # GHASH computation for the last <16 byte block
+        sub %r13, %r11
+        add $16, %r11
+
+        # output %r13 bytes
+        MOVQ_R64_XMM    %xmm0, %rax
+        cmp     $8, %r13
+        jle     _less_than_8_bytes_left_decrypt
+        mov     %rax, (%arg2 , %r11, 1)
+        add     $8, %r11
+        psrldq  $8, %xmm0
+        MOVQ_R64_XMM    %xmm0, %rax
+        sub     $8, %r13
+_less_than_8_bytes_left_decrypt:
+        mov     %al,  (%arg2, %r11, 1)
+        add     $1, %r11
+        shr     $8, %rax
+        sub     $1, %r13
+        jne     _less_than_8_bytes_left_decrypt
+_multiple_of_16_bytes_decrypt:
+        mov     arg8, %r12                # %r13 = aadLen (number of bytes)
+        shl     $3, %r12                  # convert into number of bits
+        movd    %r12d, %xmm15             # len(A) in %xmm15
+        shl     $3, %arg4                 # len(C) in bits (*128)
+        MOVQ_R64_XMM    %arg4, %xmm1
+        pslldq  $8, %xmm15                # %xmm15 = len(A)||0x0000000000000000
+        pxor    %xmm1, %xmm15             # %xmm15 = len(A)||len(C)
+        pxor    %xmm15, %xmm8
+        GHASH_MUL       %xmm8, %xmm13, %xmm9, %xmm10, %xmm11, %xmm5, %xmm6
+                 # final GHASH computation
+        movdqa SHUF_MASK(%rip), %xmm10
+        PSHUFB_XMM %xmm10, %xmm8
+
+        mov     %arg5, %rax               # %rax = *Y0
+        movdqu  (%rax), %xmm0             # %xmm0 = Y0
+        ENCRYPT_SINGLE_BLOCK    %xmm0,  %xmm1     # E(K, Y0)
+        pxor    %xmm8, %xmm0
+_return_T_decrypt:
+        mov     arg9, %r10                # %r10 = authTag
+        mov     arg10, %r11               # %r11 = auth_tag_len
+        cmp     $16, %r11
+        je      _T_16_decrypt
+        cmp     $12, %r11
+        je      _T_12_decrypt
+_T_8_decrypt:
+        MOVQ_R64_XMM    %xmm0, %rax
+        mov     %rax, (%r10)
+        jmp     _return_T_done_decrypt
+_T_12_decrypt:
+        MOVQ_R64_XMM    %xmm0, %rax
+        mov     %rax, (%r10)
+        psrldq  $8, %xmm0
+        movd    %xmm0, %eax
+        mov     %eax, 8(%r10)
+        jmp     _return_T_done_decrypt
+_T_16_decrypt:
+        movdqu  %xmm0, (%r10)
+_return_T_done_decrypt:
+        mov     %r14, %rsp
+        pop     %r14
+        pop     %r13
+        pop     %r12
+        ret
+
+
+/*****************************************************************************
+* void aesni_gcm_enc(void *aes_ctx,      // AES Key schedule. Starts on a 16 byte boundary.
+*                    u8 *out,            // Ciphertext output. Encrypt in-place is allowed.
+*                    const u8 *in,       // Plaintext input
+*                    u64 plaintext_len,  // Length of data in bytes for encryption.
+*                    u8 *iv,             // Pre-counter block j0: 4 byte salt (from Security Association)
+*                                        // concatenated with 8 byte Initialisation Vector (from IPSec ESP Payload)
+*                                        // concatenated with 0x00000001. 16-byte aligned pointer.
+*                    u8 *hash_subkey,    // H, the Hash sub key input. Data starts on a 16-byte boundary.
+*                    const u8 *aad,      // Additional Authentication Data (AAD)
+*                    u64 aad_len,        // Length of AAD in bytes. With RFC4106 this is going to be 8 or 12 bytes
+*                    u8 *auth_tag,       // Authenticated Tag output.
+*                    u64 auth_tag_len);  // Authenticated Tag Length in bytes. Valid values are 16 (most likely),
+*                                        // 12 or 8.
+*
+* Assumptions:
+*
+* keys:
+*       keys are pre-expanded and aligned to 16 bytes. we are using the
+*       first set of 11 keys in the data structure void *aes_ctx
+*
+*
+* iv:
+*       0                   1                   2                   3
+*       0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+*       |                             Salt  (From the SA)               |
+*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+*       |                     Initialization Vector                     |
+*       |         (This is the sequence number from IPSec header)       |
+*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+*       |                              0x1                              |
+*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+*
+*
+*
+* AAD:
+*       AAD padded to 128 bits with 0
+*       for example, assume AAD is a u32 vector
+*
+*       if AAD is 8 bytes:
+*       AAD[3] = {A0, A1};
+*       padded AAD in xmm register = {A1 A0 0 0}
+*
+*       0                   1                   2                   3
+*       0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+*       |                               SPI (A1)                        |
+*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+*       |                     32-bit Sequence Number (A0)               |
+*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+*       |                              0x0                              |
+*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+*
+*                                 AAD Format with 32-bit Sequence Number
+*
+*       if AAD is 12 bytes:
+*       AAD[3] = {A0, A1, A2};
+*       padded AAD in xmm register = {A2 A1 A0 0}
+*
+*       0                   1                   2                   3
+*       0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+*       |                               SPI (A2)                        |
+*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+*       |                 64-bit Extended Sequence Number {A1,A0}       |
+*       |                                                               |
+*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+*       |                              0x0                              |
+*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+*
+*                         AAD Format with 64-bit Extended Sequence Number
+*
+* aadLen:
+*       from the definition of the spec, aadLen can only be 8 or 12 bytes.
+*       The code supports 16 too but for other sizes, the code will fail.
+*
+* TLen:
+*       from the definition of the spec, TLen can only be 8, 12 or 16 bytes.
+*       For other sizes, the code will fail.
+*
+* poly = x^128 + x^127 + x^126 + x^121 + 1
+***************************************************************************/
+ENTRY(aesni_gcm_enc)
+        push    %r12
+        push    %r13
+        push    %r14
+        mov     %rsp, %r14
+#
+# states of %xmm registers %xmm6:%xmm15 not saved
+# all %xmm registers are clobbered
+#
+        sub     $VARIABLE_OFFSET, %rsp
+        and     $~63, %rsp
+        mov     %arg6, %r12
+        movdqu  (%r12), %xmm13
+        movdqa  SHUF_MASK(%rip), %xmm2
+        PSHUFB_XMM %xmm2, %xmm13
+
+
+# precompute HashKey<<1 mod poly from the HashKey (required for GHASH)
+
+        movdqa  %xmm13, %xmm2
+        psllq   $1, %xmm13
+        psrlq   $63, %xmm2
+        movdqa  %xmm2, %xmm1
+        pslldq  $8, %xmm2
+        psrldq  $8, %xmm1
+        por     %xmm2, %xmm13
+
+        # reduce HashKey<<1
+
+        pshufd  $0x24, %xmm1, %xmm2
+        pcmpeqd TWOONE(%rip), %xmm2
+        pand    POLY(%rip), %xmm2
+        pxor    %xmm2, %xmm13
+        movdqa  %xmm13, HashKey(%rsp)
+        mov     %arg4, %r13            # %xmm13 holds HashKey<<1 (mod poly)
+        and     $-16, %r13
+        mov     %r13, %r12
+
+        # Encrypt first few blocks
+
+        and     $(3<<4), %r12
+        jz      _initial_num_blocks_is_0_encrypt
+        cmp     $(2<<4), %r12
+        jb      _initial_num_blocks_is_1_encrypt
+        je      _initial_num_blocks_is_2_encrypt
+_initial_num_blocks_is_3_encrypt:
+        INITIAL_BLOCKS_ENC      3, %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \
+%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 5, 678, enc
+        sub     $48, %r13
+        jmp     _initial_blocks_encrypted
+_initial_num_blocks_is_2_encrypt:
+        INITIAL_BLOCKS_ENC      2, %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \
+%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 6, 78, enc
+        sub     $32, %r13
+        jmp     _initial_blocks_encrypted
+_initial_num_blocks_is_1_encrypt:
+        INITIAL_BLOCKS_ENC      1, %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \
+%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 7, 8, enc
+        sub     $16, %r13
+        jmp     _initial_blocks_encrypted
+_initial_num_blocks_is_0_encrypt:
+        INITIAL_BLOCKS_ENC      0, %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \
+%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 8, 0, enc
+_initial_blocks_encrypted:
+
+        # Main loop - Encrypt remaining blocks
+
+        cmp     $0, %r13
+        je      _zero_cipher_left_encrypt
+        sub     $64, %r13
+        je      _four_cipher_left_encrypt
+_encrypt_by_4_encrypt:
+        GHASH_4_ENCRYPT_4_PARALLEL_ENC  %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, \
+%xmm14, %xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, enc
+        add     $64, %r11
+        sub     $64, %r13
+        jne     _encrypt_by_4_encrypt
+_four_cipher_left_encrypt:
+        GHASH_LAST_4    %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, \
+%xmm15, %xmm1, %xmm2, %xmm3, %xmm4, %xmm8
+_zero_cipher_left_encrypt:
+        mov     %arg4, %r13
+        and     $15, %r13                       # %r13 = arg4 (mod 16)
+        je      _multiple_of_16_bytes_encrypt
+
+         # Handle the last <16 Byte block seperately
+        paddd ONE(%rip), %xmm0                # INCR CNT to get Yn
+        movdqa SHUF_MASK(%rip), %xmm10
+        PSHUFB_XMM %xmm10, %xmm0
+
+        ENCRYPT_SINGLE_BLOCK    %xmm0, %xmm1        # Encrypt(K, Yn)
+        sub $16, %r11
+        add %r13, %r11
+        movdqu (%arg3,%r11,1), %xmm1     # receive the last <16 byte blocks
+        lea SHIFT_MASK+16(%rip), %r12
+        sub %r13, %r12
+        # adjust the shuffle mask pointer to be able to shift 16-r13 bytes
+        # (%r13 is the number of bytes in plaintext mod 16)
+        movdqu  (%r12), %xmm2           # get the appropriate shuffle mask
+        PSHUFB_XMM      %xmm2, %xmm1            # shift right 16-r13 byte
+        pxor    %xmm1, %xmm0            # Plaintext XOR Encrypt(K, Yn)
+        movdqu  ALL_F-SHIFT_MASK(%r12), %xmm1
+        # get the appropriate mask to mask out top 16-r13 bytes of xmm0
+        pand    %xmm1, %xmm0            # mask out top 16-r13 bytes of xmm0
+        movdqa SHUF_MASK(%rip), %xmm10
+        PSHUFB_XMM %xmm10,%xmm0
+
+        pxor    %xmm0, %xmm8
+        GHASH_MUL %xmm8, %xmm13, %xmm9, %xmm10, %xmm11, %xmm5, %xmm6
+        # GHASH computation for the last <16 byte block
+        sub     %r13, %r11
+        add     $16, %r11
+        PSHUFB_XMM %xmm10, %xmm1
+
+        # shuffle xmm0 back to output as ciphertext
+
+        # Output %r13 bytes
+        MOVQ_R64_XMM %xmm0, %rax
+        cmp $8, %r13
+        jle _less_than_8_bytes_left_encrypt
+        mov %rax, (%arg2 , %r11, 1)
+        add $8, %r11
+        psrldq $8, %xmm0
+        MOVQ_R64_XMM %xmm0, %rax
+        sub $8, %r13
+_less_than_8_bytes_left_encrypt:
+        mov %al,  (%arg2, %r11, 1)
+        add $1, %r11
+        shr $8, %rax
+        sub $1, %r13
+        jne _less_than_8_bytes_left_encrypt
+_multiple_of_16_bytes_encrypt:
+        mov     arg8, %r12    # %r12 = addLen (number of bytes)
+        shl     $3, %r12
+        movd    %r12d, %xmm15       # len(A) in %xmm15
+        shl     $3, %arg4               # len(C) in bits (*128)
+        MOVQ_R64_XMM    %arg4, %xmm1
+        pslldq  $8, %xmm15          # %xmm15 = len(A)||0x0000000000000000
+        pxor    %xmm1, %xmm15       # %xmm15 = len(A)||len(C)
+        pxor    %xmm15, %xmm8
+        GHASH_MUL       %xmm8, %xmm13, %xmm9, %xmm10, %xmm11, %xmm5, %xmm6
+        # final GHASH computation
+        movdqa SHUF_MASK(%rip), %xmm10
+        PSHUFB_XMM %xmm10, %xmm8         # perform a 16 byte swap
+
+        mov     %arg5, %rax                    # %rax  = *Y0
+        movdqu  (%rax), %xmm0                  # %xmm0 = Y0
+        ENCRYPT_SINGLE_BLOCK    %xmm0, %xmm15         # Encrypt(K, Y0)
+        pxor    %xmm8, %xmm0
+_return_T_encrypt:
+        mov     arg9, %r10                     # %r10 = authTag
+        mov     arg10, %r11                    # %r11 = auth_tag_len
+        cmp     $16, %r11
+        je      _T_16_encrypt
+        cmp     $12, %r11
+        je      _T_12_encrypt
+_T_8_encrypt:
+        MOVQ_R64_XMM    %xmm0, %rax
+        mov     %rax, (%r10)
+        jmp     _return_T_done_encrypt
+_T_12_encrypt:
+        MOVQ_R64_XMM    %xmm0, %rax
+        mov     %rax, (%r10)
+        psrldq  $8, %xmm0
+        movd    %xmm0, %eax
+        mov     %eax, 8(%r10)
+        jmp     _return_T_done_encrypt
+_T_16_encrypt:
+        movdqu  %xmm0, (%r10)
+_return_T_done_encrypt:
+        mov     %r14, %rsp
+        pop     %r14
+        pop     %r13
+        pop     %r12
+        ret
+
+#endif
+
 
 _key_expansion_128:
 _key_expansion_256a:
@@ -55,10 +1709,11 @@ _key_expansion_256a:
         shufps $0b10001100, %xmm0, %xmm4
         pxor %xmm4, %xmm0
         pxor %xmm1, %xmm0
-        movaps %xmm0, (%rcx)
-        add $0x10, %rcx
+        movaps %xmm0, (TKEYP)
+        add $0x10, TKEYP
         ret
 
+.align 4
 _key_expansion_192a:
         pshufd $0b01010101, %xmm1, %xmm1
         shufps $0b00010000, %xmm0, %xmm4
@@ -76,12 +1731,13 @@ _key_expansion_192a:
 
         movaps %xmm0, %xmm1
         shufps $0b01000100, %xmm0, %xmm6
-        movaps %xmm6, (%rcx)
+        movaps %xmm6, (TKEYP)
         shufps $0b01001110, %xmm2, %xmm1
-        movaps %xmm1, 16(%rcx)
-        add $0x20, %rcx
+        movaps %xmm1, 0x10(TKEYP)
+        add $0x20, TKEYP
         ret
 
+.align 4
 _key_expansion_192b:
         pshufd $0b01010101, %xmm1, %xmm1
         shufps $0b00010000, %xmm0, %xmm4
@@ -96,10 +1752,11 @@ _key_expansion_192b:
         pxor %xmm3, %xmm2
         pxor %xmm5, %xmm2
 
-        movaps %xmm0, (%rcx)
-        add $0x10, %rcx
+        movaps %xmm0, (TKEYP)
+        add $0x10, TKEYP
         ret
 
+.align 4
 _key_expansion_256b:
         pshufd $0b10101010, %xmm1, %xmm1
         shufps $0b00010000, %xmm2, %xmm4
@@ -107,8 +1764,8 @@ _key_expansion_256b:
         shufps $0b10001100, %xmm2, %xmm4
         pxor %xmm4, %xmm2
         pxor %xmm1, %xmm2
-        movaps %xmm2, (%rcx)
-        add $0x10, %rcx
+        movaps %xmm2, (TKEYP)
+        add $0x10, TKEYP
         ret
 
 /*
@@ -116,17 +1773,23 @@ _key_expansion_256b:
  *                   unsigned int key_len)
  */
 ENTRY(aesni_set_key)
-        movups (%rsi), %xmm0            # user key (first 16 bytes)
-        movaps %xmm0, (%rdi)
-        lea 0x10(%rdi), %rcx            # key addr
-        movl %edx, 480(%rdi)
+#ifndef __x86_64__
+        pushl KEYP
+        movl 8(%esp), KEYP              # ctx
+        movl 12(%esp), UKEYP            # in_key
+        movl 16(%esp), %edx             # key_len
+#endif
+        movups (UKEYP), %xmm0           # user key (first 16 bytes)
+        movaps %xmm0, (KEYP)
+        lea 0x10(KEYP), TKEYP           # key addr
+        movl %edx, 480(KEYP)
         pxor %xmm4, %xmm4               # xmm4 is assumed 0 in _key_expansion_x
         cmp $24, %dl
         jb .Lenc_key128
         je .Lenc_key192
-        movups 0x10(%rsi), %xmm2        # other user key
-        movaps %xmm2, (%rcx)
-        add $0x10, %rcx
+        movups 0x10(UKEYP), %xmm2       # other user key
+        movaps %xmm2, (TKEYP)
+        add $0x10, TKEYP
         AESKEYGENASSIST 0x1 %xmm2 %xmm1         # round 1
         call _key_expansion_256a
         AESKEYGENASSIST 0x1 %xmm0 %xmm1
@@ -155,7 +1818,7 @@ ENTRY(aesni_set_key)
         call _key_expansion_256a
         jmp .Ldec_key
 .Lenc_key192:
-        movq 0x10(%rsi), %xmm2          # other user key
+        movq 0x10(UKEYP), %xmm2         # other user key
         AESKEYGENASSIST 0x1 %xmm2 %xmm1         # round 1
         call _key_expansion_192a
         AESKEYGENASSIST 0x2 %xmm2 %xmm1         # round 2
@@ -195,33 +1858,47 @@ ENTRY(aesni_set_key)
         AESKEYGENASSIST 0x36 %xmm0 %xmm1        # round 10
         call _key_expansion_128
 .Ldec_key:
-        sub $0x10, %rcx
-        movaps (%rdi), %xmm0
-        movaps (%rcx), %xmm1
-        movaps %xmm0, 240(%rcx)
-        movaps %xmm1, 240(%rdi)
-        add $0x10, %rdi
-        lea 240-16(%rcx), %rsi
+        sub $0x10, TKEYP
+        movaps (KEYP), %xmm0
+        movaps (TKEYP), %xmm1
+        movaps %xmm0, 240(TKEYP)
+        movaps %xmm1, 240(KEYP)
+        add $0x10, KEYP
+        lea 240-16(TKEYP), UKEYP
 .align 4
 .Ldec_key_loop:
-        movaps (%rdi), %xmm0
+        movaps (KEYP), %xmm0
         AESIMC %xmm0 %xmm1
-        movaps %xmm1, (%rsi)
-        add $0x10, %rdi
-        sub $0x10, %rsi
-        cmp %rcx, %rdi
+        movaps %xmm1, (UKEYP)
+        add $0x10, KEYP
+        sub $0x10, UKEYP
+        cmp TKEYP, KEYP
         jb .Ldec_key_loop
-        xor %rax, %rax
+        xor AREG, AREG
+#ifndef __x86_64__
+        popl KEYP
+#endif
         ret
 
 /*
  * void aesni_enc(struct crypto_aes_ctx *ctx, u8 *dst, const u8 *src)
  */
 ENTRY(aesni_enc)
+#ifndef __x86_64__
+        pushl KEYP
+        pushl KLEN
+        movl 12(%esp), KEYP
+        movl 16(%esp), OUTP
+        movl 20(%esp), INP
+#endif
         movl 480(KEYP), KLEN            # key length
         movups (INP), STATE             # input
         call _aesni_enc1
         movups STATE, (OUTP)            # output
+#ifndef __x86_64__
+        popl KLEN
+        popl KEYP
+#endif
         ret
 
 /*
@@ -236,6 +1913,7 @@ ENTRY(aesni_enc)
  *      KEY
  *      TKEYP (T1)
  */
+.align 4
 _aesni_enc1:
         movaps (KEYP), KEY              # key
         mov KEYP, TKEYP
@@ -298,6 +1976,7 @@ _aesni_enc1:
  *      KEY
  *      TKEYP (T1)
  */
+.align 4
 _aesni_enc4:
         movaps (KEYP), KEY              # key
         mov KEYP, TKEYP
@@ -391,11 +2070,22 @@ _aesni_enc4:
  * void aesni_dec (struct crypto_aes_ctx *ctx, u8 *dst, const u8 *src)
  */
 ENTRY(aesni_dec)
+#ifndef __x86_64__
+        pushl KEYP
+        pushl KLEN
+        movl 12(%esp), KEYP
+        movl 16(%esp), OUTP
+        movl 20(%esp), INP
+#endif
         mov 480(KEYP), KLEN             # key length
         add $240, KEYP
         movups (INP), STATE             # input
         call _aesni_dec1
         movups STATE, (OUTP)            #output
+#ifndef __x86_64__
+        popl KLEN
+        popl KEYP
+#endif
         ret
 
 /*
@@ -410,6 +2100,7 @@ ENTRY(aesni_dec)
  *      KEY
  *      TKEYP (T1)
  */
+.align 4
 _aesni_dec1:
         movaps (KEYP), KEY              # key
         mov KEYP, TKEYP
@@ -472,6 +2163,7 @@ _aesni_dec1:
  *      KEY
  *      TKEYP (T1)
  */
+.align 4
 _aesni_dec4:
         movaps (KEYP), KEY              # key
         mov KEYP, TKEYP
@@ -566,6 +2258,15 @@ _aesni_dec4:
  *                    size_t len)
  */
 ENTRY(aesni_ecb_enc)
+#ifndef __x86_64__
+        pushl LEN
+        pushl KEYP
+        pushl KLEN
+        movl 16(%esp), KEYP
+        movl 20(%esp), OUTP
+        movl 24(%esp), INP
+        movl 28(%esp), LEN
+#endif
         test LEN, LEN           # check length
         jz .Lecb_enc_ret
         mov 480(KEYP), KLEN
@@ -602,6 +2303,11 @@ ENTRY(aesni_ecb_enc)
         cmp $16, LEN
         jge .Lecb_enc_loop1
 .Lecb_enc_ret:
+#ifndef __x86_64__
+        popl KLEN
+        popl KEYP
+        popl LEN
+#endif
         ret
 
 /*
@@ -609,6 +2315,15 @@ ENTRY(aesni_ecb_enc)
  *                    size_t len);
  */
 ENTRY(aesni_ecb_dec)
+#ifndef __x86_64__
+        pushl LEN
+        pushl KEYP
+        pushl KLEN
+        movl 16(%esp), KEYP
+        movl 20(%esp), OUTP
+        movl 24(%esp), INP
+        movl 28(%esp), LEN
+#endif
         test LEN, LEN
         jz .Lecb_dec_ret
         mov 480(KEYP), KLEN
@@ -646,6 +2361,11 @@ ENTRY(aesni_ecb_dec)
         cmp $16, LEN
         jge .Lecb_dec_loop1
 .Lecb_dec_ret:
+#ifndef __x86_64__
+        popl KLEN
+        popl KEYP
+        popl LEN
+#endif
         ret
 
 /*
@@ -653,6 +2373,17 @@ ENTRY(aesni_ecb_dec)
  *                    size_t len, u8 *iv)
  */
 ENTRY(aesni_cbc_enc)
+#ifndef __x86_64__
+        pushl IVP
+        pushl LEN
+        pushl KEYP
+        pushl KLEN
+        movl 20(%esp), KEYP
+        movl 24(%esp), OUTP
+        movl 28(%esp), INP
+        movl 32(%esp), LEN
+        movl 36(%esp), IVP
+#endif
         cmp $16, LEN
         jb .Lcbc_enc_ret
         mov 480(KEYP), KLEN
@@ -670,6 +2401,12 @@ ENTRY(aesni_cbc_enc)
         jge .Lcbc_enc_loop
         movups STATE, (IVP)
 .Lcbc_enc_ret:
+#ifndef __x86_64__
+        popl KLEN
+        popl KEYP
+        popl LEN
+        popl IVP
+#endif
         ret
 
 /*
@@ -677,6 +2414,17 @@ ENTRY(aesni_cbc_enc)
  *                    size_t len, u8 *iv)
  */
 ENTRY(aesni_cbc_dec)
+#ifndef __x86_64__
+        pushl IVP
+        pushl LEN
+        pushl KEYP
+        pushl KLEN
+        movl 20(%esp), KEYP
+        movl 24(%esp), OUTP
+        movl 28(%esp), INP
+        movl 32(%esp), LEN
+        movl 36(%esp), IVP
+#endif
         cmp $16, LEN
         jb .Lcbc_dec_just_ret
         mov 480(KEYP), KLEN
@@ -690,16 +2438,30 @@ ENTRY(aesni_cbc_dec)
         movaps IN1, STATE1
         movups 0x10(INP), IN2
         movaps IN2, STATE2
+#ifdef __x86_64__
         movups 0x20(INP), IN3
         movaps IN3, STATE3
         movups 0x30(INP), IN4
         movaps IN4, STATE4
+#else
+        movups 0x20(INP), IN1
+        movaps IN1, STATE3
+        movups 0x30(INP), IN2
+        movaps IN2, STATE4
+#endif
         call _aesni_dec4
         pxor IV, STATE1
+#ifdef __x86_64__
         pxor IN1, STATE2
         pxor IN2, STATE3
         pxor IN3, STATE4
         movaps IN4, IV
+#else
+        pxor (INP), STATE2
+        pxor 0x10(INP), STATE3
+        pxor IN1, STATE4
+        movaps IN2, IV
+#endif
         movups STATE1, (OUTP)
         movups STATE2, 0x10(OUTP)
         movups STATE3, 0x20(OUTP)
@@ -727,8 +2489,15 @@ ENTRY(aesni_cbc_dec)
 .Lcbc_dec_ret:
         movups IV, (IVP)
 .Lcbc_dec_just_ret:
+#ifndef __x86_64__
+        popl KLEN
+        popl KEYP
+        popl LEN
+        popl IVP
+#endif
         ret
 
+#ifdef __x86_64__
 .align 16
 .Lbswap_mask:
         .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
@@ -744,6 +2513,7 @@ ENTRY(aesni_cbc_dec)
  *      INC:    == 1, in little endian
  *      BSWAP_MASK == endian swapping mask
  */
+.align 4
 _aesni_inc_init:
         movaps .Lbswap_mask, BSWAP_MASK
         movaps IV, CTR
@@ -768,6 +2538,7 @@ _aesni_inc_init:
  *      CTR:    == output IV, in little endian
  *      TCTR_LOW: == lower qword of CTR
  */
+.align 4
 _aesni_inc:
         paddq INC, CTR
         add $1, TCTR_LOW
@@ -839,3 +2610,4 @@ ENTRY(aesni_ctr_enc)
         movups IV, (IVP)
 .Lctr_enc_just_ret:
         ret
+#endif
diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c
index 2cb3dcc4490..e1e60c7d581 100644
--- a/arch/x86/crypto/aesni-intel_glue.c
+++ b/arch/x86/crypto/aesni-intel_glue.c
@@ -5,6 +5,14 @@
  * Copyright (C) 2008, Intel Corp.
  *    Author: Huang Ying <ying.huang@intel.com>
  *
+ * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
+ * interface for 64-bit kernels.
+ *    Authors: Adrian Hoban <adrian.hoban@intel.com>
+ *             Gabriele Paoloni <gabriele.paoloni@intel.com>
+ *             Tadeusz Struk (tadeusz.struk@intel.com)
+ *             Aidan O'Mahony (aidan.o.mahony@intel.com)
+ *    Copyright (c) 2010, Intel Corporation.
+ *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
  * the Free Software Foundation; either version 2 of the License, or
@@ -21,6 +29,10 @@
 #include <crypto/ctr.h>
 #include <asm/i387.h>
 #include <asm/aes.h>
+#include <crypto/scatterwalk.h>
+#include <crypto/internal/aead.h>
+#include <linux/workqueue.h>
+#include <linux/spinlock.h>
 
 #if defined(CONFIG_CRYPTO_CTR) || defined(CONFIG_CRYPTO_CTR_MODULE)
 #define HAS_CTR
@@ -42,8 +54,31 @@ struct async_aes_ctx {
         struct cryptd_ablkcipher *cryptd_tfm;
 };
 
-#define AESNI_ALIGN     16
+/* This data is stored at the end of the crypto_tfm struct.
+ * It's a type of per "session" data storage location.
+ * This needs to be 16 byte aligned.
+ */
+struct aesni_rfc4106_gcm_ctx {
+        u8 hash_subkey[16];
+        struct crypto_aes_ctx aes_key_expanded;
+        u8 nonce[4];
+        struct cryptd_aead *cryptd_tfm;
+};
+
+struct aesni_gcm_set_hash_subkey_result {
+        int err;
+        struct completion completion;
+};
+
+struct aesni_hash_subkey_req_data {
+        u8 iv[16];
+        struct aesni_gcm_set_hash_subkey_result result;
+        struct scatterlist sg;
+};
+
+#define AESNI_ALIGN     (16)
 #define AES_BLOCK_MASK  (~(AES_BLOCK_SIZE-1))
+#define RFC4106_HASH_SUBKEY_SIZE 16
 
 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
                              unsigned int key_len);
@@ -59,9 +94,62 @@ asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
                               const u8 *in, unsigned int len, u8 *iv);
 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
                               const u8 *in, unsigned int len, u8 *iv);
+#ifdef CONFIG_X86_64
 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out,
                               const u8 *in, unsigned int len, u8 *iv);
 
+/* asmlinkage void aesni_gcm_enc()
+ * void *ctx,  AES Key schedule. Starts on a 16 byte boundary.
+ * u8 *out, Ciphertext output. Encrypt in-place is allowed.
+ * const u8 *in, Plaintext input
+ * unsigned long plaintext_len, Length of data in bytes for encryption.
+ * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association)
+ *         concatenated with 8 byte Initialisation Vector (from IPSec ESP
+ *         Payload) concatenated with 0x00000001. 16-byte aligned pointer.
+ * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
+ * const u8 *aad, Additional Authentication Data (AAD)
+ * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this
+ *          is going to be 8 or 12 bytes
+ * u8 *auth_tag, Authenticated Tag output.
+ * unsigned long auth_tag_len), Authenticated Tag Length in bytes.
+ *          Valid values are 16 (most likely), 12 or 8.
+ */
+asmlinkage void aesni_gcm_enc(void *ctx, u8 *out,
+                        const u8 *in, unsigned long plaintext_len, u8 *iv,
+                        u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
+                        u8 *auth_tag, unsigned long auth_tag_len);
+
+/* asmlinkage void aesni_gcm_dec()
+ * void *ctx, AES Key schedule. Starts on a 16 byte boundary.
+ * u8 *out, Plaintext output. Decrypt in-place is allowed.
+ * const u8 *in, Ciphertext input
+ * unsigned long ciphertext_len, Length of data in bytes for decryption.
+ * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association)
+ *         concatenated with 8 byte Initialisation Vector (from IPSec ESP
+ *         Payload) concatenated with 0x00000001. 16-byte aligned pointer.
+ * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
+ * const u8 *aad, Additional Authentication Data (AAD)
+ * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this is going
+ * to be 8 or 12 bytes
+ * u8 *auth_tag, Authenticated Tag output.
+ * unsigned long auth_tag_len) Authenticated Tag Length in bytes.
+ * Valid values are 16 (most likely), 12 or 8.
+ */
+asmlinkage void aesni_gcm_dec(void *ctx, u8 *out,
+                        const u8 *in, unsigned long ciphertext_len, u8 *iv,
+                        u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
+                        u8 *auth_tag, unsigned long auth_tag_len);
+
+static inline struct
+aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm)
+{
+        return
+                (struct aesni_rfc4106_gcm_ctx *)
+                PTR_ALIGN((u8 *)
+                crypto_tfm_ctx(crypto_aead_tfm(tfm)), AESNI_ALIGN);
+}
+#endif
+
 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx)
 {
         unsigned long addr = (unsigned long)raw_ctx;
@@ -324,6 +412,7 @@ static struct crypto_alg blk_cbc_alg = {
         },
 };
 
+#ifdef CONFIG_X86_64
 static void ctr_crypt_final(struct crypto_aes_ctx *ctx,
                             struct blkcipher_walk *walk)
 {
@@ -389,6 +478,7 @@ static struct crypto_alg blk_ctr_alg = {
                 },
         },
 };
+#endif
 
 static int ablk_set_key(struct crypto_ablkcipher *tfm, const u8 *key,
                         unsigned int key_len)
@@ -536,6 +626,7 @@ static struct crypto_alg ablk_cbc_alg = {
         },
 };
 
+#ifdef CONFIG_X86_64
 static int ablk_ctr_init(struct crypto_tfm *tfm)
 {
         struct cryptd_ablkcipher *cryptd_tfm;
@@ -612,6 +703,7 @@ static struct crypto_alg ablk_rfc3686_ctr_alg = {
         },
 };
 #endif
+#endif
 
 #ifdef HAS_LRW
 static int ablk_lrw_init(struct crypto_tfm *tfm)
@@ -730,6 +822,424 @@ static struct crypto_alg ablk_xts_alg = {
 };
 #endif
 
+#ifdef CONFIG_X86_64
+static int rfc4106_init(struct crypto_tfm *tfm)
+{
+        struct cryptd_aead *cryptd_tfm;
+        struct aesni_rfc4106_gcm_ctx *ctx = (struct aesni_rfc4106_gcm_ctx *)
+                PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN);
+        cryptd_tfm = cryptd_alloc_aead("__driver-gcm-aes-aesni", 0, 0);
+        if (IS_ERR(cryptd_tfm))
+                return PTR_ERR(cryptd_tfm);
+        ctx->cryptd_tfm = cryptd_tfm;
+        tfm->crt_aead.reqsize = sizeof(struct aead_request)
+                + crypto_aead_reqsize(&cryptd_tfm->base);
+        return 0;
+}
+
+static void rfc4106_exit(struct crypto_tfm *tfm)
+{
+        struct aesni_rfc4106_gcm_ctx *ctx =
+                (struct aesni_rfc4106_gcm_ctx *)
+                PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN);
+        if (!IS_ERR(ctx->cryptd_tfm))
+                cryptd_free_aead(ctx->cryptd_tfm);
+        return;
+}
+
+static void
+rfc4106_set_hash_subkey_done(struct crypto_async_request *req, int err)
+{
+        struct aesni_gcm_set_hash_subkey_result *result = req->data;
+
+        if (err == -EINPROGRESS)
+                return;
+        result->err = err;
+        complete(&result->completion);
+}
+
+static int
+rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len)
+{
+        struct crypto_ablkcipher *ctr_tfm;
+        struct ablkcipher_request *req;
+        int ret = -EINVAL;
+        struct aesni_hash_subkey_req_data *req_data;
+
+        ctr_tfm = crypto_alloc_ablkcipher("ctr(aes)", 0, 0);
+        if (IS_ERR(ctr_tfm))
+                return PTR_ERR(ctr_tfm);
+
+        crypto_ablkcipher_clear_flags(ctr_tfm, ~0);
+
+        ret = crypto_ablkcipher_setkey(ctr_tfm, key, key_len);
+        if (ret) {
+                crypto_free_ablkcipher(ctr_tfm);
+                return ret;
+        }
+
+        req = ablkcipher_request_alloc(ctr_tfm, GFP_KERNEL);
+        if (!req) {
+                crypto_free_ablkcipher(ctr_tfm);
+                return -EINVAL;
+        }
+
+        req_data = kmalloc(sizeof(*req_data), GFP_KERNEL);
+        if (!req_data) {
+                crypto_free_ablkcipher(ctr_tfm);
+                return -ENOMEM;
+        }
+        memset(req_data->iv, 0, sizeof(req_data->iv));
+
+        /* Clear the data in the hash sub key container to zero.*/
+        /* We want to cipher all zeros to create the hash sub key. */
+        memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE);
+
+        init_completion(&req_data->result.completion);
+        sg_init_one(&req_data->sg, hash_subkey, RFC4106_HASH_SUBKEY_SIZE);
+        ablkcipher_request_set_tfm(req, ctr_tfm);
+        ablkcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP |
+                                        CRYPTO_TFM_REQ_MAY_BACKLOG,
+                                        rfc4106_set_hash_subkey_done,
+                                        &req_data->result);
+
+        ablkcipher_request_set_crypt(req, &req_data->sg,
+                &req_data->sg, RFC4106_HASH_SUBKEY_SIZE, req_data->iv);
+
+        ret = crypto_ablkcipher_encrypt(req);
+        if (ret == -EINPROGRESS || ret == -EBUSY) {
+                ret = wait_for_completion_interruptible
+                        (&req_data->result.completion);
+                if (!ret)
+                        ret = req_data->result.err;
+        }
+        ablkcipher_request_free(req);
+        kfree(req_data);
+        crypto_free_ablkcipher(ctr_tfm);
+        return ret;
+}
+
+static int rfc4106_set_key(struct crypto_aead *parent, const u8 *key,
+                                                   unsigned int key_len)
+{
+        int ret = 0;
+        struct crypto_tfm *tfm = crypto_aead_tfm(parent);
+        struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent);
+        u8 *new_key_mem = NULL;
+
+        if (key_len < 4) {
+                crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
+                return -EINVAL;
+        }
+        /*Account for 4 byte nonce at the end.*/
+        key_len -= 4;
+        if (key_len != AES_KEYSIZE_128) {
+                crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
+                return -EINVAL;
+        }
+
+        memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce));
+        /*This must be on a 16 byte boundary!*/
+        if ((unsigned long)(&(ctx->aes_key_expanded.key_enc[0])) % AESNI_ALIGN)
+                return -EINVAL;
+
+        if ((unsigned long)key % AESNI_ALIGN) {
+                /*key is not aligned: use an auxuliar aligned pointer*/
+                new_key_mem = kmalloc(key_len+AESNI_ALIGN, GFP_KERNEL);
+                if (!new_key_mem)
+                        return -ENOMEM;
+
+                new_key_mem = PTR_ALIGN(new_key_mem, AESNI_ALIGN);
+                memcpy(new_key_mem, key, key_len);
+                key = new_key_mem;
+        }
+
+        if (!irq_fpu_usable())
+                ret = crypto_aes_expand_key(&(ctx->aes_key_expanded),
+                key, key_len);
+        else {
+                kernel_fpu_begin();
+                ret = aesni_set_key(&(ctx->aes_key_expanded), key, key_len);
+                kernel_fpu_end();
+        }
+        /*This must be on a 16 byte boundary!*/
+        if ((unsigned long)(&(ctx->hash_subkey[0])) % AESNI_ALIGN) {
+                ret = -EINVAL;
+                goto exit;
+        }
+        ret = rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
+exit:
+        kfree(new_key_mem);
+        return ret;
+}
+
+/* This is the Integrity Check Value (aka the authentication tag length and can
+ * be 8, 12 or 16 bytes long. */
+static int rfc4106_set_authsize(struct crypto_aead *parent,
+                                unsigned int authsize)
+{
+        struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent);
+        struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
+
+        switch (authsize) {
+        case 8:
+        case 12:
+        case 16:
+                break;
+        default:
+                return -EINVAL;
+        }
+        crypto_aead_crt(parent)->authsize = authsize;
+        crypto_aead_crt(cryptd_child)->authsize = authsize;
+        return 0;
+}
+
+static int rfc4106_encrypt(struct aead_request *req)
+{
+        int ret;
+        struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+        struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
+        struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
+
+        if (!irq_fpu_usable()) {
+                struct aead_request *cryptd_req =
+                        (struct aead_request *) aead_request_ctx(req);
+                memcpy(cryptd_req, req, sizeof(*req));
+                aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
+                return crypto_aead_encrypt(cryptd_req);
+        } else {
+                kernel_fpu_begin();
+                ret = cryptd_child->base.crt_aead.encrypt(req);
+                kernel_fpu_end();
+                return ret;
+        }
+}
+
+static int rfc4106_decrypt(struct aead_request *req)
+{
+        int ret;
+        struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+        struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
+        struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
+
+        if (!irq_fpu_usable()) {
+                struct aead_request *cryptd_req =
+                        (struct aead_request *) aead_request_ctx(req);
+                memcpy(cryptd_req, req, sizeof(*req));
+                aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
+                return crypto_aead_decrypt(cryptd_req);
+        } else {
+                kernel_fpu_begin();
+                ret = cryptd_child->base.crt_aead.decrypt(req);
+                kernel_fpu_end();
+                return ret;
+        }
+}
+
+static struct crypto_alg rfc4106_alg = {
+        .cra_name = "rfc4106(gcm(aes))",
+        .cra_driver_name = "rfc4106-gcm-aesni",
+        .cra_priority = 400,
+        .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
+        .cra_blocksize = 1,
+        .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx) + AESNI_ALIGN,
+        .cra_alignmask = 0,
+        .cra_type = &crypto_nivaead_type,
+        .cra_module = THIS_MODULE,
+        .cra_list = LIST_HEAD_INIT(rfc4106_alg.cra_list),
+        .cra_init = rfc4106_init,
+        .cra_exit = rfc4106_exit,
+        .cra_u = {
+                .aead = {
+                        .setkey = rfc4106_set_key,
+                        .setauthsize = rfc4106_set_authsize,
+                        .encrypt = rfc4106_encrypt,
+                        .decrypt = rfc4106_decrypt,
+                        .geniv = "seqiv",
+                        .ivsize = 8,
+                        .maxauthsize = 16,
+                },
+        },
+};
+
+static int __driver_rfc4106_encrypt(struct aead_request *req)
+{
+        u8 one_entry_in_sg = 0;
+        u8 *src, *dst, *assoc;
+        __be32 counter = cpu_to_be32(1);
+        struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+        struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
+        void *aes_ctx = &(ctx->aes_key_expanded);
+        unsigned long auth_tag_len = crypto_aead_authsize(tfm);
+        u8 iv_tab[16+AESNI_ALIGN];
+        u8* iv = (u8 *) PTR_ALIGN((u8 *)iv_tab, AESNI_ALIGN);
+        struct scatter_walk src_sg_walk;
+        struct scatter_walk assoc_sg_walk;
+        struct scatter_walk dst_sg_walk;
+        unsigned int i;
+
+        /* Assuming we are supporting rfc4106 64-bit extended */
+        /* sequence numbers We need to have the AAD length equal */
+        /* to 8 or 12 bytes */
+        if (unlikely(req->assoclen != 8 && req->assoclen != 12))
+                return -EINVAL;
+        /* IV below built */
+        for (i = 0; i < 4; i++)
+                *(iv+i) = ctx->nonce[i];
+        for (i = 0; i < 8; i++)
+                *(iv+4+i) = req->iv[i];
+        *((__be32 *)(iv+12)) = counter;
+
+        if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) {
+                one_entry_in_sg = 1;
+                scatterwalk_start(&src_sg_walk, req->src);
+                scatterwalk_start(&assoc_sg_walk, req->assoc);
+                src = scatterwalk_map(&src_sg_walk, 0);
+                assoc = scatterwalk_map(&assoc_sg_walk, 0);
+                dst = src;
+                if (unlikely(req->src != req->dst)) {
+                        scatterwalk_start(&dst_sg_walk, req->dst);
+                        dst = scatterwalk_map(&dst_sg_walk, 0);
+                }
+
+        } else {
+                /* Allocate memory for src, dst, assoc */
+                src = kmalloc(req->cryptlen + auth_tag_len + req->assoclen,
+                        GFP_ATOMIC);
+                if (unlikely(!src))
+                        return -ENOMEM;
+                assoc = (src + req->cryptlen + auth_tag_len);
+                scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0);
+                scatterwalk_map_and_copy(assoc, req->assoc, 0,
+                                        req->assoclen, 0);
+                dst = src;
+        }
+
+        aesni_gcm_enc(aes_ctx, dst, src, (unsigned long)req->cryptlen, iv,
+                ctx->hash_subkey, assoc, (unsigned long)req->assoclen, dst
+                + ((unsigned long)req->cryptlen), auth_tag_len);
+
+        /* The authTag (aka the Integrity Check Value) needs to be written
+         * back to the packet. */
+        if (one_entry_in_sg) {
+                if (unlikely(req->src != req->dst)) {
+                        scatterwalk_unmap(dst, 0);
+                        scatterwalk_done(&dst_sg_walk, 0, 0);
+                }
+                scatterwalk_unmap(src, 0);
+                scatterwalk_unmap(assoc, 0);
+                scatterwalk_done(&src_sg_walk, 0, 0);
+                scatterwalk_done(&assoc_sg_walk, 0, 0);
+        } else {
+                scatterwalk_map_and_copy(dst, req->dst, 0,
+                        req->cryptlen + auth_tag_len, 1);
+                kfree(src);
+        }
+        return 0;
+}
+
+static int __driver_rfc4106_decrypt(struct aead_request *req)
+{
+        u8 one_entry_in_sg = 0;
+        u8 *src, *dst, *assoc;
+        unsigned long tempCipherLen = 0;
+        __be32 counter = cpu_to_be32(1);
+        int retval = 0;
+        struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+        struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
+        void *aes_ctx = &(ctx->aes_key_expanded);
+        unsigned long auth_tag_len = crypto_aead_authsize(tfm);
+        u8 iv_and_authTag[32+AESNI_ALIGN];
+        u8 *iv = (u8 *) PTR_ALIGN((u8 *)iv_and_authTag, AESNI_ALIGN);
+        u8 *authTag = iv + 16;
+        struct scatter_walk src_sg_walk;
+        struct scatter_walk assoc_sg_walk;
+        struct scatter_walk dst_sg_walk;
+        unsigned int i;
+
+        if (unlikely((req->cryptlen < auth_tag_len) ||
+                (req->assoclen != 8 && req->assoclen != 12)))
+                return -EINVAL;
+        /* Assuming we are supporting rfc4106 64-bit extended */
+        /* sequence numbers We need to have the AAD length */
+        /* equal to 8 or 12 bytes */
+
+        tempCipherLen = (unsigned long)(req->cryptlen - auth_tag_len);
+        /* IV below built */
+        for (i = 0; i < 4; i++)
+                *(iv+i) = ctx->nonce[i];
+        for (i = 0; i < 8; i++)
+                *(iv+4+i) = req->iv[i];
+        *((__be32 *)(iv+12)) = counter;
+
+        if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) {
+                one_entry_in_sg = 1;
+                scatterwalk_start(&src_sg_walk, req->src);
+                scatterwalk_start(&assoc_sg_walk, req->assoc);
+                src = scatterwalk_map(&src_sg_walk, 0);
+                assoc = scatterwalk_map(&assoc_sg_walk, 0);
+                dst = src;
+                if (unlikely(req->src != req->dst)) {
+                        scatterwalk_start(&dst_sg_walk, req->dst);
+                        dst = scatterwalk_map(&dst_sg_walk, 0);
+                }
+
+        } else {
+                /* Allocate memory for src, dst, assoc */
+                src = kmalloc(req->cryptlen + req->assoclen, GFP_ATOMIC);
+                if (!src)
+                        return -ENOMEM;
+                assoc = (src + req->cryptlen + auth_tag_len);
+                scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0);
+                scatterwalk_map_and_copy(assoc, req->assoc, 0,
+                        req->assoclen, 0);
+                dst = src;
+        }
+
+        aesni_gcm_dec(aes_ctx, dst, src, tempCipherLen, iv,
+                ctx->hash_subkey, assoc, (unsigned long)req->assoclen,
+                authTag, auth_tag_len);
+
+        /* Compare generated tag with passed in tag. */
+        retval = memcmp(src + tempCipherLen, authTag, auth_tag_len) ?
+                -EBADMSG : 0;
+
+        if (one_entry_in_sg) {
+                if (unlikely(req->src != req->dst)) {
+                        scatterwalk_unmap(dst, 0);
+                        scatterwalk_done(&dst_sg_walk, 0, 0);
+                }
+                scatterwalk_unmap(src, 0);
+                scatterwalk_unmap(assoc, 0);
+                scatterwalk_done(&src_sg_walk, 0, 0);
+                scatterwalk_done(&assoc_sg_walk, 0, 0);
+        } else {
+                scatterwalk_map_and_copy(dst, req->dst, 0, req->cryptlen, 1);
+                kfree(src);
+        }
+        return retval;
+}
+
+static struct crypto_alg __rfc4106_alg = {
+        .cra_name               = "__gcm-aes-aesni",
+        .cra_driver_name        = "__driver-gcm-aes-aesni",
+        .cra_priority           = 0,
+        .cra_flags              = CRYPTO_ALG_TYPE_AEAD,
+        .cra_blocksize          = 1,
+        .cra_ctxsize    = sizeof(struct aesni_rfc4106_gcm_ctx) + AESNI_ALIGN,
+        .cra_alignmask          = 0,
+        .cra_type               = &crypto_aead_type,
+        .cra_module             = THIS_MODULE,
+        .cra_list               = LIST_HEAD_INIT(__rfc4106_alg.cra_list),
+        .cra_u = {
+                .aead = {
+                        .encrypt        = __driver_rfc4106_encrypt,
+                        .decrypt        = __driver_rfc4106_decrypt,
+                },
+        },
+};
+#endif
+
 static int __init aesni_init(void)
 {
         int err;
@@ -738,6 +1248,7 @@ static int __init aesni_init(void)
                 printk(KERN_INFO "Intel AES-NI instructions are not detected.\n");
                 return -ENODEV;
         }
+
         if ((err = crypto_register_alg(&aesni_alg)))
                 goto aes_err;
         if ((err = crypto_register_alg(&__aesni_alg)))
@@ -746,18 +1257,24 @@ static int __init aesni_init(void)
                 goto blk_ecb_err;
         if ((err = crypto_register_alg(&blk_cbc_alg)))
                 goto blk_cbc_err;
-        if ((err = crypto_register_alg(&blk_ctr_alg)))
-                goto blk_ctr_err;
         if ((err = crypto_register_alg(&ablk_ecb_alg)))
                 goto ablk_ecb_err;
         if ((err = crypto_register_alg(&ablk_cbc_alg)))
                 goto ablk_cbc_err;
+#ifdef CONFIG_X86_64
+        if ((err = crypto_register_alg(&blk_ctr_alg)))
+                goto blk_ctr_err;
         if ((err = crypto_register_alg(&ablk_ctr_alg)))
                 goto ablk_ctr_err;
+        if ((err = crypto_register_alg(&__rfc4106_alg)))
+                goto __aead_gcm_err;
+        if ((err = crypto_register_alg(&rfc4106_alg)))
+                goto aead_gcm_err;
 #ifdef HAS_CTR
         if ((err = crypto_register_alg(&ablk_rfc3686_ctr_alg)))
                 goto ablk_rfc3686_ctr_err;
 #endif
+#endif
 #ifdef HAS_LRW
         if ((err = crypto_register_alg(&ablk_lrw_alg)))
                 goto ablk_lrw_err;
@@ -770,7 +1287,6 @@ static int __init aesni_init(void)
         if ((err = crypto_register_alg(&ablk_xts_alg)))
                 goto ablk_xts_err;
 #endif
-
         return err;
 
 #ifdef HAS_XTS
@@ -784,18 +1300,24 @@ ablk_pcbc_err:
         crypto_unregister_alg(&ablk_lrw_alg);
 ablk_lrw_err:
 #endif
+#ifdef CONFIG_X86_64
 #ifdef HAS_CTR
         crypto_unregister_alg(&ablk_rfc3686_ctr_alg);
 ablk_rfc3686_ctr_err:
 #endif
+        crypto_unregister_alg(&rfc4106_alg);
+aead_gcm_err:
+        crypto_unregister_alg(&__rfc4106_alg);
+__aead_gcm_err:
         crypto_unregister_alg(&ablk_ctr_alg);
 ablk_ctr_err:
+        crypto_unregister_alg(&blk_ctr_alg);
+blk_ctr_err:
+#endif
         crypto_unregister_alg(&ablk_cbc_alg);
 ablk_cbc_err:
         crypto_unregister_alg(&ablk_ecb_alg);
 ablk_ecb_err:
-        crypto_unregister_alg(&blk_ctr_alg);
-blk_ctr_err:
         crypto_unregister_alg(&blk_cbc_alg);
 blk_cbc_err:
         crypto_unregister_alg(&blk_ecb_alg);
@@ -818,13 +1340,17 @@ static void __exit aesni_exit(void)
 #ifdef HAS_LRW
         crypto_unregister_alg(&ablk_lrw_alg);
 #endif
+#ifdef CONFIG_X86_64
 #ifdef HAS_CTR
         crypto_unregister_alg(&ablk_rfc3686_ctr_alg);
 #endif
+        crypto_unregister_alg(&rfc4106_alg);
+        crypto_unregister_alg(&__rfc4106_alg);
         crypto_unregister_alg(&ablk_ctr_alg);
+        crypto_unregister_alg(&blk_ctr_alg);
+#endif
         crypto_unregister_alg(&ablk_cbc_alg);
         crypto_unregister_alg(&ablk_ecb_alg);
-        crypto_unregister_alg(&blk_ctr_alg);
         crypto_unregister_alg(&blk_cbc_alg);
         crypto_unregister_alg(&blk_ecb_alg);
         crypto_unregister_alg(&__aesni_alg);
diff --git a/arch/x86/include/asm/boot.h b/arch/x86/include/asm/boot.h
index 3b62ab56c7a..5e1a2eef3e7 100644
--- a/arch/x86/include/asm/boot.h
+++ b/arch/x86/include/asm/boot.h
@@ -32,11 +32,7 @@
 #define BOOT_HEAP_SIZE             0x400000
 #else /* !CONFIG_KERNEL_BZIP2 */
 
-#ifdef CONFIG_X86_64
-#define BOOT_HEAP_SIZE  0x7000
-#else
-#define BOOT_HEAP_SIZE  0x4000
-#endif
+#define BOOT_HEAP_SIZE  0x8000
 
 #endif /* !CONFIG_KERNEL_BZIP2 */
 
diff --git a/arch/x86/include/asm/irq.h b/arch/x86/include/asm/irq.h
index ba870bb6dd8..c704b38c57a 100644
--- a/arch/x86/include/asm/irq.h
+++ b/arch/x86/include/asm/irq.h
@@ -10,6 +10,9 @@
 #include <asm/apicdef.h>
 #include <asm/irq_vectors.h>
 
+/* Even though we don't support this, supply it to appease OF */
+static inline void irq_dispose_mapping(unsigned int virq) { }
+
 static inline int irq_canonicalize(int irq)
 {
         return ((irq == 2) ? 9 : irq);
diff --git a/arch/x86/include/asm/kvm_emulate.h b/arch/x86/include/asm/kvm_emulate.h
index b36c6b3fe14..8e37deb1eb3 100644
--- a/arch/x86/include/asm/kvm_emulate.h
+++ b/arch/x86/include/asm/kvm_emulate.h
@@ -15,6 +15,14 @@
 
 struct x86_emulate_ctxt;
 
+struct x86_exception {
+        u8 vector;
+        bool error_code_valid;
+        u16 error_code;
+        bool nested_page_fault;
+        u64 address; /* cr2 or nested page fault gpa */
+};
+
 /*
  * x86_emulate_ops:
  *
@@ -64,7 +72,8 @@ struct x86_emulate_ops {
          *  @bytes: [IN ] Number of bytes to read from memory.
          */
         int (*read_std)(unsigned long addr, void *val,
-                        unsigned int bytes, struct kvm_vcpu *vcpu, u32 *error);
+                        unsigned int bytes, struct kvm_vcpu *vcpu,
+                        struct x86_exception *fault);
 
         /*
          * write_std: Write bytes of standard (non-emulated/special) memory.
@@ -74,7 +83,8 @@ struct x86_emulate_ops {
          *  @bytes: [IN ] Number of bytes to write to memory.
          */
         int (*write_std)(unsigned long addr, void *val,
-                         unsigned int bytes, struct kvm_vcpu *vcpu, u32 *error);
+                         unsigned int bytes, struct kvm_vcpu *vcpu,
+                         struct x86_exception *fault);
         /*
          * fetch: Read bytes of standard (non-emulated/special) memory.
          *        Used for instruction fetch.
@@ -83,7 +93,8 @@ struct x86_emulate_ops {
          *  @bytes: [IN ] Number of bytes to read from memory.
          */
         int (*fetch)(unsigned long addr, void *val,
-                        unsigned int bytes, struct kvm_vcpu *vcpu, u32 *error);
+                     unsigned int bytes, struct kvm_vcpu *vcpu,
+                     struct x86_exception *fault);
 
         /*
          * read_emulated: Read bytes from emulated/special memory area.
@@ -94,7 +105,7 @@ struct x86_emulate_ops {
         int (*read_emulated)(unsigned long addr,
                              void *val,
                              unsigned int bytes,
-                             unsigned int *error,
+                             struct x86_exception *fault,
                              struct kvm_vcpu *vcpu);
 
         /*
@@ -107,7 +118,7 @@ struct x86_emulate_ops {
         int (*write_emulated)(unsigned long addr,
                               const void *val,
                               unsigned int bytes,
-                              unsigned int *error,
+                              struct x86_exception *fault,
                               struct kvm_vcpu *vcpu);
 
         /*
@@ -122,7 +133,7 @@ struct x86_emulate_ops {
                                 const void *old,
                                 const void *new,
                                 unsigned int bytes,
-                                unsigned int *error,
+                                struct x86_exception *fault,
                                 struct kvm_vcpu *vcpu);
 
         int (*pio_in_emulated)(int size, unsigned short port, void *val,
@@ -159,7 +170,10 @@ struct operand {
         };
         union {
                 unsigned long *reg;
-                unsigned long mem;
+                struct segmented_address {
+                        ulong ea;
+                        unsigned seg;
+                } mem;
         } addr;
         union {
                 unsigned long val;
@@ -226,9 +240,8 @@ struct x86_emulate_ctxt {
 
         bool perm_ok; /* do not check permissions if true */
 
-        int exception; /* exception that happens during emulation or -1 */
-        u32 error_code; /* error code for exception */
-        bool error_code_valid;
+        bool have_exception;
+        struct x86_exception exception;
 
         /* decode cache */
         struct decode_cache decode;
@@ -252,7 +265,7 @@ struct x86_emulate_ctxt {
 #define X86EMUL_MODE_HOST X86EMUL_MODE_PROT64
 #endif
 
-int x86_decode_insn(struct x86_emulate_ctxt *ctxt);
+int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len);
 #define EMULATION_FAILED -1
 #define EMULATION_OK 0
 #define EMULATION_RESTART 1
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index f702f82aa1e..aa75f21a9fb 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -83,11 +83,14 @@
 #define KVM_NR_FIXED_MTRR_REGION 88
 #define KVM_NR_VAR_MTRR 8
 
+#define ASYNC_PF_PER_VCPU 64
+
 extern spinlock_t kvm_lock;
 extern struct list_head vm_list;
 
 struct kvm_vcpu;
 struct kvm;
+struct kvm_async_pf;
 
 enum kvm_reg {
         VCPU_REGS_RAX = 0,
@@ -114,6 +117,7 @@ enum kvm_reg {
 
 enum kvm_reg_ex {
         VCPU_EXREG_PDPTR = NR_VCPU_REGS,
+        VCPU_EXREG_CR3,
 };
 
 enum {
@@ -238,16 +242,18 @@ struct kvm_mmu {
         void (*new_cr3)(struct kvm_vcpu *vcpu);
         void (*set_cr3)(struct kvm_vcpu *vcpu, unsigned long root);
         unsigned long (*get_cr3)(struct kvm_vcpu *vcpu);
-        int (*page_fault)(struct kvm_vcpu *vcpu, gva_t gva, u32 err);
-        void (*inject_page_fault)(struct kvm_vcpu *vcpu);
+        int (*page_fault)(struct kvm_vcpu *vcpu, gva_t gva, u32 err,
+                          bool prefault);
+        void (*inject_page_fault)(struct kvm_vcpu *vcpu,
+                                  struct x86_exception *fault);
         void (*free)(struct kvm_vcpu *vcpu);
         gpa_t (*gva_to_gpa)(struct kvm_vcpu *vcpu, gva_t gva, u32 access,
-                            u32 *error);
+                            struct x86_exception *exception);
         gpa_t (*translate_gpa)(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access);
         void (*prefetch_page)(struct kvm_vcpu *vcpu,
                               struct kvm_mmu_page *page);
         int (*sync_page)(struct kvm_vcpu *vcpu,
-                         struct kvm_mmu_page *sp, bool clear_unsync);
+                         struct kvm_mmu_page *sp);
         void (*invlpg)(struct kvm_vcpu *vcpu, gva_t gva);
         hpa_t root_hpa;
         int root_level;
@@ -315,16 +321,6 @@ struct kvm_vcpu_arch {
          */
         struct kvm_mmu *walk_mmu;
 
-        /*
-         * This struct is filled with the necessary information to propagate a
-         * page fault into the guest
-         */
-        struct {
-                u64      address;
-                unsigned error_code;
-                bool     nested;
-        } fault;
-
         /* only needed in kvm_pv_mmu_op() path, but it's hot so
          * put it here to avoid allocation */
         struct kvm_pv_mmu_op_buffer mmu_op_buffer;
@@ -412,6 +408,15 @@ struct kvm_vcpu_arch {
         u64 hv_vapic;
 
         cpumask_var_t wbinvd_dirty_mask;
+
+        struct {
+                bool halted;
+                gfn_t gfns[roundup_pow_of_two(ASYNC_PF_PER_VCPU)];
+                struct gfn_to_hva_cache data;
+                u64 msr_val;
+                u32 id;
+                bool send_user_only;
+        } apf;
 };
 
 struct kvm_arch {
@@ -456,6 +461,10 @@ struct kvm_arch {
         /* fields used by HYPER-V emulation */
         u64 hv_guest_os_id;
         u64 hv_hypercall;
+
+        #ifdef CONFIG_KVM_MMU_AUDIT
+        int audit_point;
+        #endif
 };
 
 struct kvm_vm_stat {
@@ -529,6 +538,7 @@ struct kvm_x86_ops {
                             struct kvm_segment *var, int seg);
         void (*get_cs_db_l_bits)(struct kvm_vcpu *vcpu, int *db, int *l);
         void (*decache_cr0_guest_bits)(struct kvm_vcpu *vcpu);
+        void (*decache_cr3)(struct kvm_vcpu *vcpu);
         void (*decache_cr4_guest_bits)(struct kvm_vcpu *vcpu);
         void (*set_cr0)(struct kvm_vcpu *vcpu, unsigned long cr0);
         void (*set_cr3)(struct kvm_vcpu *vcpu, unsigned long cr3);
@@ -582,9 +592,17 @@ struct kvm_x86_ops {
 
         void (*write_tsc_offset)(struct kvm_vcpu *vcpu, u64 offset);
 
+        void (*get_exit_info)(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2);
         const struct trace_print_flags *exit_reasons_str;
 };
 
+struct kvm_arch_async_pf {
+        u32 token;
+        gfn_t gfn;
+        unsigned long cr3;
+        bool direct_map;
+};
+
 extern struct kvm_x86_ops *kvm_x86_ops;
 
 int kvm_mmu_module_init(void);
@@ -594,7 +612,6 @@ void kvm_mmu_destroy(struct kvm_vcpu *vcpu);
 int kvm_mmu_create(struct kvm_vcpu *vcpu);
 int kvm_mmu_setup(struct kvm_vcpu *vcpu);
 void kvm_mmu_set_nonpresent_ptes(u64 trap_pte, u64 notrap_pte);
-void kvm_mmu_set_base_ptes(u64 base_pte);
 void kvm_mmu_set_mask_ptes(u64 user_mask, u64 accessed_mask,
                 u64 dirty_mask, u64 nx_mask, u64 x_mask);
 
@@ -623,8 +640,15 @@ enum emulation_result {
 #define EMULTYPE_NO_DECODE          (1 << 0)
 #define EMULTYPE_TRAP_UD            (1 << 1)
 #define EMULTYPE_SKIP               (1 << 2)
-int emulate_instruction(struct kvm_vcpu *vcpu,
-                        unsigned long cr2, u16 error_code, int emulation_type);
+int x86_emulate_instruction(struct kvm_vcpu *vcpu, unsigned long cr2,
+                            int emulation_type, void *insn, int insn_len);
+
+static inline int emulate_instruction(struct kvm_vcpu *vcpu,
+                        int emulation_type)
+{
+        return x86_emulate_instruction(vcpu, 0, emulation_type, NULL, 0);
+}
+
 void realmode_lgdt(struct kvm_vcpu *vcpu, u16 size, unsigned long address);
 void realmode_lidt(struct kvm_vcpu *vcpu, u16 size, unsigned long address);
 
@@ -650,7 +674,7 @@ int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int reason,
 int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0);
 int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3);
 int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4);
-void kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8);
+int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8);
 int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val);
 int kvm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *val);
 unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu);
@@ -668,11 +692,11 @@ void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr);
 void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code);
 void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr);
 void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code);
-void kvm_inject_page_fault(struct kvm_vcpu *vcpu);
+void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault);
 int kvm_read_guest_page_mmu(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
                             gfn_t gfn, void *data, int offset, int len,
                             u32 access);
-void kvm_propagate_fault(struct kvm_vcpu *vcpu);
+void kvm_propagate_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault);
 bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl);
 
 int kvm_pic_set_irq(void *opaque, int irq, int level);
@@ -690,16 +714,21 @@ void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu);
 int kvm_mmu_load(struct kvm_vcpu *vcpu);
 void kvm_mmu_unload(struct kvm_vcpu *vcpu);
 void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu);
-gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva, u32 *error);
-gpa_t kvm_mmu_gva_to_gpa_fetch(struct kvm_vcpu *vcpu, gva_t gva, u32 *error);
-gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva, u32 *error);
-gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva, u32 *error);
+gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva,
+                              struct x86_exception *exception);
+gpa_t kvm_mmu_gva_to_gpa_fetch(struct kvm_vcpu *vcpu, gva_t gva,
+                               struct x86_exception *exception);
+gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva,
+                               struct x86_exception *exception);
+gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva,
+                                struct x86_exception *exception);
 
 int kvm_emulate_hypercall(struct kvm_vcpu *vcpu);
 
 int kvm_fix_hypercall(struct kvm_vcpu *vcpu);
 
-int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t gva, u32 error_code);
+int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t gva, u32 error_code,
+                       void *insn, int insn_len);
 void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva);
 
 void kvm_enable_tdp(void);
@@ -766,20 +795,25 @@ enum {
 #define HF_VINTR_MASK           (1 << 2)
 #define HF_NMI_MASK             (1 << 3)
 #define HF_IRET_MASK            (1 << 4)
+#define HF_GUEST_MASK           (1 << 5) /* VCPU is in guest-mode */
 
 /*
  * Hardware virtualization extension instructions may fault if a
  * reboot turns off virtualization while processes are running.
  * Trap the fault and ignore the instruction if that happens.
  */
-asmlinkage void kvm_handle_fault_on_reboot(void);
+asmlinkage void kvm_spurious_fault(void);
+extern bool kvm_rebooting;
 
 #define __kvm_handle_fault_on_reboot(insn) \
         "666: " insn "\n\t" \
+        "668: \n\t"                           \
         ".pushsection .fixup, \"ax\" \n" \
         "667: \n\t" \
+        "cmpb $0, kvm_rebooting \n\t"         \
+        "jne 668b \n\t"                       \
         __ASM_SIZE(push) " $666b \n\t"        \
-        "jmp kvm_handle_fault_on_reboot \n\t" \
+        "call kvm_spurious_fault \n\t"        \
         ".popsection \n\t" \
         ".pushsection __ex_table, \"a\" \n\t" \
         _ASM_PTR " 666b, 667b \n\t" \
@@ -799,4 +833,15 @@ void kvm_set_shared_msr(unsigned index, u64 val, u64 mask);
 
 bool kvm_is_linear_rip(struct kvm_vcpu *vcpu, unsigned long linear_rip);
 
+void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
+                                     struct kvm_async_pf *work);
+void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
+                                 struct kvm_async_pf *work);
+void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
+                               struct kvm_async_pf *work);
+bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu);
+extern bool kvm_find_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn);
+
+void kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err);
+
 #endif /* _ASM_X86_KVM_HOST_H */
diff --git a/arch/x86/include/asm/kvm_para.h b/arch/x86/include/asm/kvm_para.h
index 7b562b6184b..a427bf77a93 100644
--- a/arch/x86/include/asm/kvm_para.h
+++ b/arch/x86/include/asm/kvm_para.h
@@ -20,6 +20,7 @@
  * are available. The use of 0x11 and 0x12 is deprecated
  */
 #define KVM_FEATURE_CLOCKSOURCE2        3
+#define KVM_FEATURE_ASYNC_PF            4
 
 /* The last 8 bits are used to indicate how to interpret the flags field
  * in pvclock structure. If no bits are set, all flags are ignored.
@@ -32,9 +33,13 @@
 /* Custom MSRs falls in the range 0x4b564d00-0x4b564dff */
 #define MSR_KVM_WALL_CLOCK_NEW  0x4b564d00
 #define MSR_KVM_SYSTEM_TIME_NEW 0x4b564d01
+#define MSR_KVM_ASYNC_PF_EN 0x4b564d02
 
 #define KVM_MAX_MMU_OP_BATCH           32
 
+#define KVM_ASYNC_PF_ENABLED                    (1 << 0)
+#define KVM_ASYNC_PF_SEND_ALWAYS                (1 << 1)
+
 /* Operations for KVM_HC_MMU_OP */
 #define KVM_MMU_OP_WRITE_PTE            1
 #define KVM_MMU_OP_FLUSH_TLB            2
@@ -61,10 +66,20 @@ struct kvm_mmu_op_release_pt {
         __u64 pt_phys;
 };
 
+#define KVM_PV_REASON_PAGE_NOT_PRESENT 1
+#define KVM_PV_REASON_PAGE_READY 2
+
+struct kvm_vcpu_pv_apf_data {
+        __u32 reason;
+        __u8 pad[60];
+        __u32 enabled;
+};
+
 #ifdef __KERNEL__
 #include <asm/processor.h>
 
 extern void kvmclock_init(void);
+extern int kvm_register_clock(char *txt);
 
 
 /* This instruction is vmcall.  On non-VT architectures, it will generate a
@@ -160,8 +175,17 @@ static inline unsigned int kvm_arch_para_features(void)
 
 #ifdef CONFIG_KVM_GUEST
 void __init kvm_guest_init(void);
+void kvm_async_pf_task_wait(u32 token);
+void kvm_async_pf_task_wake(u32 token);
+u32 kvm_read_and_reset_pf_reason(void);
 #else
 #define kvm_guest_init() do { } while (0)
+#define kvm_async_pf_task_wait(T) do {} while(0)
+#define kvm_async_pf_task_wake(T) do {} while(0)
+static inline u32 kvm_read_and_reset_pf_reason(void)
+{
+        return 0;
+}
 #endif
 
 #endif /* __KERNEL__ */
diff --git a/arch/x86/include/asm/olpc.h b/arch/x86/include/asm/olpc.h
index 42a978c0c1b..f482010350f 100644
--- a/arch/x86/include/asm/olpc.h
+++ b/arch/x86/include/asm/olpc.h
@@ -107,10 +107,14 @@ extern int olpc_ec_mask_unset(uint8_t bits);
 /* GPIO assignments */
 
 #define OLPC_GPIO_MIC_AC        1
-#define OLPC_GPIO_DCON_IRQ      geode_gpio(7)
+#define OLPC_GPIO_DCON_STAT0    5
+#define OLPC_GPIO_DCON_STAT1    6
+#define OLPC_GPIO_DCON_IRQ      7
 #define OLPC_GPIO_THRM_ALRM     geode_gpio(10)
-#define OLPC_GPIO_SMB_CLK       geode_gpio(14)
-#define OLPC_GPIO_SMB_DATA      geode_gpio(15)
+#define OLPC_GPIO_DCON_LOAD    11
+#define OLPC_GPIO_DCON_BLANK   12
+#define OLPC_GPIO_SMB_CLK      14
+#define OLPC_GPIO_SMB_DATA     15
 #define OLPC_GPIO_WORKAUX       geode_gpio(24)
 #define OLPC_GPIO_LID           geode_gpio(26)
 #define OLPC_GPIO_ECSCI         geode_gpio(27)
diff --git a/arch/x86/include/asm/olpc_ofw.h b/arch/x86/include/asm/olpc_ofw.h
index 2a8478140bb..641988efe06 100644
--- a/arch/x86/include/asm/olpc_ofw.h
+++ b/arch/x86/include/asm/olpc_ofw.h
@@ -8,6 +8,8 @@
 
 #ifdef CONFIG_OLPC_OPENFIRMWARE
 
+extern bool olpc_ofw_is_installed(void);
+
 /* run an OFW command by calling into the firmware */
 #define olpc_ofw(name, args, res) \
         __olpc_ofw((name), ARRAY_SIZE(args), args, ARRAY_SIZE(res), res)
@@ -26,10 +28,17 @@ extern bool olpc_ofw_present(void);
 
 #else /* !CONFIG_OLPC_OPENFIRMWARE */
 
+static inline bool olpc_ofw_is_installed(void) { return false; }
 static inline void olpc_ofw_detect(void) { }
 static inline void setup_olpc_ofw_pgd(void) { }
 static inline bool olpc_ofw_present(void) { return false; }
 
 #endif /* !CONFIG_OLPC_OPENFIRMWARE */
 
+#ifdef CONFIG_OLPC_OPENFIRMWARE_DT
+extern void olpc_dt_build_devicetree(void);
+#else
+static inline void olpc_dt_build_devicetree(void) { }
+#endif /* CONFIG_OLPC_OPENFIRMWARE_DT */
+
 #endif /* _ASM_X86_OLPC_OFW_H */
diff --git a/arch/x86/include/asm/pgalloc.h b/arch/x86/include/asm/pgalloc.h
index 271de94c381..b4389a468fb 100644
--- a/arch/x86/include/asm/pgalloc.h
+++ b/arch/x86/include/asm/pgalloc.h
@@ -92,7 +92,7 @@ static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
 extern void ___pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd);
 
 static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd,
-                                  unsigned long adddress)
+                                  unsigned long address)
 {
         ___pmd_free_tlb(tlb, pmd);
 }
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index c6efecf85a6..53fd1d5a1fe 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -901,7 +901,7 @@ extern unsigned long thread_saved_pc(struct task_struct *tsk);
 /*
  * The below -8 is to reserve 8 bytes on top of the ring0 stack.
  * This is necessary to guarantee that the entire "struct pt_regs"
- * is accessable even if the CPU haven't stored the SS/ESP registers
+ * is accessible even if the CPU haven't stored the SS/ESP registers
  * on the stack (interrupt gate does not save these registers
  * when switching to the same priv ring).
  * Therefore beware: accessing the ss/esp fields of the
diff --git a/arch/x86/include/asm/prom.h b/arch/x86/include/asm/prom.h
new file mode 100644
index 00000000000..b4ec95f0751
--- /dev/null
+++ b/arch/x86/include/asm/prom.h
@@ -0,0 +1 @@
+/* dummy prom.h; here to make linux/of.h's #includes happy */
diff --git a/arch/x86/include/asm/svm.h b/arch/x86/include/asm/svm.h
index 0e831059ac5..f2b83bc7d78 100644
--- a/arch/x86/include/asm/svm.h
+++ b/arch/x86/include/asm/svm.h
@@ -47,14 +47,13 @@ enum {
         INTERCEPT_MONITOR,
         INTERCEPT_MWAIT,
         INTERCEPT_MWAIT_COND,
+        INTERCEPT_XSETBV,
 };
 
 
 struct __attribute__ ((__packed__)) vmcb_control_area {
-        u16 intercept_cr_read;
-        u16 intercept_cr_write;
-        u16 intercept_dr_read;
-        u16 intercept_dr_write;
+        u32 intercept_cr;
+        u32 intercept_dr;
         u32 intercept_exceptions;
         u64 intercept;
         u8 reserved_1[42];
@@ -81,14 +80,19 @@ struct __attribute__ ((__packed__)) vmcb_control_area {
         u32 event_inj_err;
         u64 nested_cr3;
         u64 lbr_ctl;
-        u64 reserved_5;
+        u32 clean;
+        u32 reserved_5;
         u64 next_rip;
-        u8 reserved_6[816];
+        u8 insn_len;
+        u8 insn_bytes[15];
+        u8 reserved_6[800];
 };
 
 
 #define TLB_CONTROL_DO_NOTHING 0
 #define TLB_CONTROL_FLUSH_ALL_ASID 1
+#define TLB_CONTROL_FLUSH_ASID 3
+#define TLB_CONTROL_FLUSH_ASID_LOCAL 7
 
 #define V_TPR_MASK 0x0f
 
@@ -204,19 +208,31 @@ struct __attribute__ ((__packed__)) vmcb {
 #define SVM_SELECTOR_READ_MASK SVM_SELECTOR_WRITE_MASK
 #define SVM_SELECTOR_CODE_MASK (1 << 3)
 
-#define INTERCEPT_CR0_MASK 1
-#define INTERCEPT_CR3_MASK (1 << 3)
-#define INTERCEPT_CR4_MASK (1 << 4)
-#define INTERCEPT_CR8_MASK (1 << 8)
-
-#define INTERCEPT_DR0_MASK 1
-#define INTERCEPT_DR1_MASK (1 << 1)
-#define INTERCEPT_DR2_MASK (1 << 2)
-#define INTERCEPT_DR3_MASK (1 << 3)
-#define INTERCEPT_DR4_MASK (1 << 4)
-#define INTERCEPT_DR5_MASK (1 << 5)
-#define INTERCEPT_DR6_MASK (1 << 6)
-#define INTERCEPT_DR7_MASK (1 << 7)
+#define INTERCEPT_CR0_READ      0
+#define INTERCEPT_CR3_READ      3
+#define INTERCEPT_CR4_READ      4
+#define INTERCEPT_CR8_READ      8
+#define INTERCEPT_CR0_WRITE     (16 + 0)
+#define INTERCEPT_CR3_WRITE     (16 + 3)
+#define INTERCEPT_CR4_WRITE     (16 + 4)
+#define INTERCEPT_CR8_WRITE     (16 + 8)
+
+#define INTERCEPT_DR0_READ      0
+#define INTERCEPT_DR1_READ      1
+#define INTERCEPT_DR2_READ      2
+#define INTERCEPT_DR3_READ      3
+#define INTERCEPT_DR4_READ      4
+#define INTERCEPT_DR5_READ      5
+#define INTERCEPT_DR6_READ      6
+#define INTERCEPT_DR7_READ      7
+#define INTERCEPT_DR0_WRITE     (16 + 0)
+#define INTERCEPT_DR1_WRITE     (16 + 1)
+#define INTERCEPT_DR2_WRITE     (16 + 2)
+#define INTERCEPT_DR3_WRITE     (16 + 3)
+#define INTERCEPT_DR4_WRITE     (16 + 4)
+#define INTERCEPT_DR5_WRITE     (16 + 5)
+#define INTERCEPT_DR6_WRITE     (16 + 6)
+#define INTERCEPT_DR7_WRITE     (16 + 7)
 
 #define SVM_EVTINJ_VEC_MASK 0xff
 
@@ -246,6 +262,8 @@ struct __attribute__ ((__packed__)) vmcb {
 #define SVM_EXITINFOSHIFT_TS_REASON_JMP 38
 #define SVM_EXITINFOSHIFT_TS_HAS_ERROR_CODE 44
 
+#define SVM_EXITINFO_REG_MASK 0x0F
+
 #define SVM_EXIT_READ_CR0       0x000
 #define SVM_EXIT_READ_CR3       0x003
 #define SVM_EXIT_READ_CR4       0x004
@@ -316,6 +334,7 @@ struct __attribute__ ((__packed__)) vmcb {
 #define SVM_EXIT_MONITOR        0x08a
 #define SVM_EXIT_MWAIT          0x08b
 #define SVM_EXIT_MWAIT_COND     0x08c
+#define SVM_EXIT_XSETBV         0x08d
 #define SVM_EXIT_NPF            0x400
 
 #define SVM_EXIT_ERR            -1
diff --git a/arch/x86/include/asm/traps.h b/arch/x86/include/asm/traps.h
index f66cda56781..0310da67307 100644
--- a/arch/x86/include/asm/traps.h
+++ b/arch/x86/include/asm/traps.h
@@ -30,6 +30,7 @@ asmlinkage void segment_not_present(void);
 asmlinkage void stack_segment(void);
 asmlinkage void general_protection(void);
 asmlinkage void page_fault(void);
+asmlinkage void async_page_fault(void);
 asmlinkage void spurious_interrupt_bug(void);
 asmlinkage void coprocessor_error(void);
 asmlinkage void alignment_check(void);
diff --git a/arch/x86/include/asm/vmx.h b/arch/x86/include/asm/vmx.h
index 9f0cbd987d5..84471b81046 100644
--- a/arch/x86/include/asm/vmx.h
+++ b/arch/x86/include/asm/vmx.h
@@ -66,15 +66,23 @@
 #define PIN_BASED_NMI_EXITING                   0x00000008
 #define PIN_BASED_VIRTUAL_NMIS                  0x00000020
 
+#define VM_EXIT_SAVE_DEBUG_CONTROLS             0x00000002
 #define VM_EXIT_HOST_ADDR_SPACE_SIZE            0x00000200
+#define VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL      0x00001000
 #define VM_EXIT_ACK_INTR_ON_EXIT                0x00008000
 #define VM_EXIT_SAVE_IA32_PAT                   0x00040000
 #define VM_EXIT_LOAD_IA32_PAT                   0x00080000
+#define VM_EXIT_SAVE_IA32_EFER                  0x00100000
+#define VM_EXIT_LOAD_IA32_EFER                  0x00200000
+#define VM_EXIT_SAVE_VMX_PREEMPTION_TIMER       0x00400000
 
+#define VM_ENTRY_LOAD_DEBUG_CONTROLS            0x00000002
 #define VM_ENTRY_IA32E_MODE                     0x00000200
 #define VM_ENTRY_SMM                            0x00000400
 #define VM_ENTRY_DEACT_DUAL_MONITOR             0x00000800
+#define VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL     0x00002000
 #define VM_ENTRY_LOAD_IA32_PAT                  0x00004000
+#define VM_ENTRY_LOAD_IA32_EFER                 0x00008000
 
 /* VMCS Encodings */
 enum vmcs_field {
@@ -239,6 +247,7 @@ enum vmcs_field {
 #define EXIT_REASON_TASK_SWITCH         9
 #define EXIT_REASON_CPUID               10
 #define EXIT_REASON_HLT                 12
+#define EXIT_REASON_INVD                13
 #define EXIT_REASON_INVLPG              14
 #define EXIT_REASON_RDPMC               15
 #define EXIT_REASON_RDTSC               16
@@ -296,6 +305,12 @@ enum vmcs_field {
 #define GUEST_INTR_STATE_SMI            0x00000004
 #define GUEST_INTR_STATE_NMI            0x00000008
 
+/* GUEST_ACTIVITY_STATE flags */
+#define GUEST_ACTIVITY_ACTIVE           0
+#define GUEST_ACTIVITY_HLT              1
+#define GUEST_ACTIVITY_SHUTDOWN         2
+#define GUEST_ACTIVITY_WAIT_SIPI        3
+
 /*
  * Exit Qualifications for MOV for Control Register Access
  */
diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c
index d2fdb0826df..57ca7778722 100644
--- a/arch/x86/kernel/amd_iommu.c
+++ b/arch/x86/kernel/amd_iommu.c
@@ -1086,7 +1086,7 @@ static int alloc_new_range(struct dma_ops_domain *dma_dom,
 
         dma_dom->aperture_size += APERTURE_RANGE_SIZE;
 
-        /* Intialize the exclusion range if necessary */
+        /* Initialize the exclusion range if necessary */
         for_each_iommu(iommu) {
                 if (iommu->exclusion_start &&
                     iommu->exclusion_start >= dma_dom->aperture[index]->offset
@@ -1353,7 +1353,7 @@ static void dma_ops_domain_free(struct dma_ops_domain *dom)
 
 /*
  * Allocates a new protection domain usable for the dma_ops functions.
- * It also intializes the page table and the address allocator data
+ * It also initializes the page table and the address allocator data
  * structures required for the dma_ops interface
  */
 static struct dma_ops_domain *dma_ops_domain_alloc(void)
diff --git a/arch/x86/kernel/entry_32.S b/arch/x86/kernel/entry_32.S
index 591e6010427..c8b4efad7eb 100644
--- a/arch/x86/kernel/entry_32.S
+++ b/arch/x86/kernel/entry_32.S
@@ -1406,6 +1406,16 @@ ENTRY(general_protection)
         CFI_ENDPROC
 END(general_protection)
 
+#ifdef CONFIG_KVM_GUEST
+ENTRY(async_page_fault)
+        RING0_EC_FRAME
+        pushl $do_async_page_fault
+        CFI_ADJUST_CFA_OFFSET 4
+        jmp error_code
+        CFI_ENDPROC
+END(apf_page_fault)
+#endif
+
 /*
  * End of kprobes section
  */
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index d3b895f375d..aed1ffbeb0c 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -1329,6 +1329,9 @@ errorentry xen_stack_segment do_stack_segment
 #endif
 errorentry general_protection do_general_protection
 errorentry page_fault do_page_fault
+#ifdef CONFIG_KVM_GUEST
+errorentry async_page_fault do_async_page_fault
+#endif
 #ifdef CONFIG_X86_MCE
 paranoidzeroentry machine_check *machine_check_vector(%rip)
 #endif
diff --git a/arch/x86/kernel/head_32.S b/arch/x86/kernel/head_32.S
index 9f54b209c37..fc293dc8dc3 100644
--- a/arch/x86/kernel/head_32.S
+++ b/arch/x86/kernel/head_32.S
@@ -126,7 +126,7 @@ ENTRY(startup_32)
         movsl
         movl pa(boot_params) + NEW_CL_POINTER,%esi
         andl %esi,%esi
-        jz 1f                   # No comand line
+        jz 1f                   # No command line
         movl $pa(boot_command_line),%edi
         movl $(COMMAND_LINE_SIZE/4),%ecx
         rep
diff --git a/arch/x86/kernel/i387.c b/arch/x86/kernel/i387.c
index 58bb239a2fd..e60c38cc0ee 100644
--- a/arch/x86/kernel/i387.c
+++ b/arch/x86/kernel/i387.c
@@ -169,6 +169,7 @@ int init_fpu(struct task_struct *tsk)
         set_stopped_child_used_math(tsk);
         return 0;
 }
+EXPORT_SYMBOL_GPL(init_fpu);
 
 /*
  * The xstateregs_active() routine is the same as the fpregs_active() routine,
diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c
index 3a43caa3beb..52945da52a9 100644
--- a/arch/x86/kernel/irq.c
+++ b/arch/x86/kernel/irq.c
@@ -4,6 +4,7 @@
 #include <linux/cpu.h>
 #include <linux/interrupt.h>
 #include <linux/kernel_stat.h>
+#include <linux/of.h>
 #include <linux/seq_file.h>
 #include <linux/smp.h>
 #include <linux/ftrace.h>
@@ -275,6 +276,15 @@ void smp_x86_platform_ipi(struct pt_regs *regs)
 
 EXPORT_SYMBOL_GPL(vector_used_by_percpu_irq);
 
+#ifdef CONFIG_OF
+unsigned int irq_create_of_mapping(struct device_node *controller,
+                const u32 *intspec, unsigned int intsize)
+{
+        return intspec[0];
+}
+EXPORT_SYMBOL_GPL(irq_create_of_mapping);
+#endif
+
 #ifdef CONFIG_HOTPLUG_CPU
 /* A cpu has been removed from cpu_online_mask.  Reset irq affinities. */
 void fixup_irqs(void)
diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c
index 63b0ec8d3d4..8dc44662394 100644
--- a/arch/x86/kernel/kvm.c
+++ b/arch/x86/kernel/kvm.c
@@ -27,16 +27,37 @@
 #include <linux/mm.h>
 #include <linux/highmem.h>
 #include <linux/hardirq.h>
+#include <linux/notifier.h>
+#include <linux/reboot.h>
+#include <linux/hash.h>
+#include <linux/sched.h>
+#include <linux/slab.h>
+#include <linux/kprobes.h>
 #include <asm/timer.h>
+#include <asm/cpu.h>
+#include <asm/traps.h>
+#include <asm/desc.h>
+#include <asm/tlbflush.h>
 
 #define MMU_QUEUE_SIZE 1024
 
+static int kvmapf = 1;
+
+static int parse_no_kvmapf(char *arg)
+{
+        kvmapf = 0;
+        return 0;
+}
+
+early_param("no-kvmapf", parse_no_kvmapf);
+
 struct kvm_para_state {
         u8 mmu_queue[MMU_QUEUE_SIZE];
         int mmu_queue_len;
 };
 
 static DEFINE_PER_CPU(struct kvm_para_state, para_state);
+static DEFINE_PER_CPU(struct kvm_vcpu_pv_apf_data, apf_reason) __aligned(64);
 
 static struct kvm_para_state *kvm_para_state(void)
 {
@@ -50,6 +71,195 @@ static void kvm_io_delay(void)
 {
 }
 
+#define KVM_TASK_SLEEP_HASHBITS 8
+#define KVM_TASK_SLEEP_HASHSIZE (1<<KVM_TASK_SLEEP_HASHBITS)
+
+struct kvm_task_sleep_node {
+        struct hlist_node link;
+        wait_queue_head_t wq;
+        u32 token;
+        int cpu;
+        bool halted;
+        struct mm_struct *mm;
+};
+
+static struct kvm_task_sleep_head {
+        spinlock_t lock;
+        struct hlist_head list;
+} async_pf_sleepers[KVM_TASK_SLEEP_HASHSIZE];
+
+static struct kvm_task_sleep_node *_find_apf_task(struct kvm_task_sleep_head *b,
+                                                  u32 token)
+{
+        struct hlist_node *p;
+
+        hlist_for_each(p, &b->list) {
+                struct kvm_task_sleep_node *n =
+                        hlist_entry(p, typeof(*n), link);
+                if (n->token == token)
+                        return n;
+        }
+
+        return NULL;
+}
+
+void kvm_async_pf_task_wait(u32 token)
+{
+        u32 key = hash_32(token, KVM_TASK_SLEEP_HASHBITS);
+        struct kvm_task_sleep_head *b = &async_pf_sleepers[key];
+        struct kvm_task_sleep_node n, *e;
+        DEFINE_WAIT(wait);
+        int cpu, idle;
+
+        cpu = get_cpu();
+        idle = idle_cpu(cpu);
+        put_cpu();
+
+        spin_lock(&b->lock);
+        e = _find_apf_task(b, token);
+        if (e) {
+                /* dummy entry exist -> wake up was delivered ahead of PF */
+                hlist_del(&e->link);
+                kfree(e);
+                spin_unlock(&b->lock);
+                return;
+        }
+
+        n.token = token;
+        n.cpu = smp_processor_id();
+        n.mm = current->active_mm;
+        n.halted = idle || preempt_count() > 1;
+        atomic_inc(&n.mm->mm_count);
+        init_waitqueue_head(&n.wq);
+        hlist_add_head(&n.link, &b->list);
+        spin_unlock(&b->lock);
+
+        for (;;) {
+                if (!n.halted)
+                        prepare_to_wait(&n.wq, &wait, TASK_UNINTERRUPTIBLE);
+                if (hlist_unhashed(&n.link))
+                        break;
+
+                if (!n.halted) {
+                        local_irq_enable();
+                        schedule();
+                        local_irq_disable();
+                } else {
+                        /*
+                         * We cannot reschedule. So halt.
+                         */
+                        native_safe_halt();
+                        local_irq_disable();
+                }
+        }
+        if (!n.halted)
+                finish_wait(&n.wq, &wait);
+
+        return;
+}
+EXPORT_SYMBOL_GPL(kvm_async_pf_task_wait);
+
+static void apf_task_wake_one(struct kvm_task_sleep_node *n)
+{
+        hlist_del_init(&n->link);
+        if (!n->mm)
+                return;
+        mmdrop(n->mm);
+        if (n->halted)
+                smp_send_reschedule(n->cpu);
+        else if (waitqueue_active(&n->wq))
+                wake_up(&n->wq);
+}
+
+static void apf_task_wake_all(void)
+{
+        int i;
+
+        for (i = 0; i < KVM_TASK_SLEEP_HASHSIZE; i++) {
+                struct hlist_node *p, *next;
+                struct kvm_task_sleep_head *b = &async_pf_sleepers[i];
+                spin_lock(&b->lock);
+                hlist_for_each_safe(p, next, &b->list) {
+                        struct kvm_task_sleep_node *n =
+                                hlist_entry(p, typeof(*n), link);
+                        if (n->cpu == smp_processor_id())
+                                apf_task_wake_one(n);
+                }
+                spin_unlock(&b->lock);
+        }
+}
+
+void kvm_async_pf_task_wake(u32 token)
+{
+        u32 key = hash_32(token, KVM_TASK_SLEEP_HASHBITS);
+        struct kvm_task_sleep_head *b = &async_pf_sleepers[key];
+        struct kvm_task_sleep_node *n;
+
+        if (token == ~0) {
+                apf_task_wake_all();
+                return;
+        }
+
+again:
+        spin_lock(&b->lock);
+        n = _find_apf_task(b, token);
+        if (!n) {
+                /*
+                 * async PF was not yet handled.
+                 * Add dummy entry for the token.
+                 */
+                n = kmalloc(sizeof(*n), GFP_ATOMIC);
+                if (!n) {
+                        /*
+                         * Allocation failed! Busy wait while other cpu
+                         * handles async PF.
+                         */
+                        spin_unlock(&b->lock);
+                        cpu_relax();
+                        goto again;
+                }
+                n->token = token;
+                n->cpu = smp_processor_id();
+                n->mm = NULL;
+                init_waitqueue_head(&n->wq);
+                hlist_add_head(&n->link, &b->list);
+        } else
+                apf_task_wake_one(n);
+        spin_unlock(&b->lock);
+        return;
+}
+EXPORT_SYMBOL_GPL(kvm_async_pf_task_wake);
+
+u32 kvm_read_and_reset_pf_reason(void)
+{
+        u32 reason = 0;
+
+        if (__get_cpu_var(apf_reason).enabled) {
+                reason = __get_cpu_var(apf_reason).reason;
+                __get_cpu_var(apf_reason).reason = 0;
+        }
+
+        return reason;
+}
+EXPORT_SYMBOL_GPL(kvm_read_and_reset_pf_reason);
+
+dotraplinkage void __kprobes
+do_async_page_fault(struct pt_regs *regs, unsigned long error_code)
+{
+        switch (kvm_read_and_reset_pf_reason()) {
+        default:
+                do_page_fault(regs, error_code);
+                break;
+        case KVM_PV_REASON_PAGE_NOT_PRESENT:
+                /* page is swapped out by the host. */
+                kvm_async_pf_task_wait((u32)read_cr2());
+                break;
+        case KVM_PV_REASON_PAGE_READY:
+                kvm_async_pf_task_wake((u32)read_cr2());
+                break;
+        }
+}
+
 static void kvm_mmu_op(void *buffer, unsigned len)
 {
         int r;
@@ -231,10 +441,117 @@ static void __init paravirt_ops_setup(void)
 #endif
 }
 
+void __cpuinit kvm_guest_cpu_init(void)
+{
+        if (!kvm_para_available())
+                return;
+
+        if (kvm_para_has_feature(KVM_FEATURE_ASYNC_PF) && kvmapf) {
+                u64 pa = __pa(&__get_cpu_var(apf_reason));
+
+#ifdef CONFIG_PREEMPT
+                pa |= KVM_ASYNC_PF_SEND_ALWAYS;
+#endif
+                wrmsrl(MSR_KVM_ASYNC_PF_EN, pa | KVM_ASYNC_PF_ENABLED);
+                __get_cpu_var(apf_reason).enabled = 1;
+                printk(KERN_INFO"KVM setup async PF for cpu %d\n",
+                       smp_processor_id());
+        }
+}
+
+static void kvm_pv_disable_apf(void *unused)
+{
+        if (!__get_cpu_var(apf_reason).enabled)
+                return;
+
+        wrmsrl(MSR_KVM_ASYNC_PF_EN, 0);
+        __get_cpu_var(apf_reason).enabled = 0;
+
+        printk(KERN_INFO"Unregister pv shared memory for cpu %d\n",
+               smp_processor_id());
+}
+
+static int kvm_pv_reboot_notify(struct notifier_block *nb,
+                                unsigned long code, void *unused)
+{
+        if (code == SYS_RESTART)
+                on_each_cpu(kvm_pv_disable_apf, NULL, 1);
+        return NOTIFY_DONE;
+}
+
+static struct notifier_block kvm_pv_reboot_nb = {
+        .notifier_call = kvm_pv_reboot_notify,
+};
+
+#ifdef CONFIG_SMP
+static void __init kvm_smp_prepare_boot_cpu(void)
+{
+#ifdef CONFIG_KVM_CLOCK
+        WARN_ON(kvm_register_clock("primary cpu clock"));
+#endif
+        kvm_guest_cpu_init();
+        native_smp_prepare_boot_cpu();
+}
+
+static void kvm_guest_cpu_online(void *dummy)
+{
+        kvm_guest_cpu_init();
+}
+
+static void kvm_guest_cpu_offline(void *dummy)
+{
+        kvm_pv_disable_apf(NULL);
+        apf_task_wake_all();
+}
+
+static int __cpuinit kvm_cpu_notify(struct notifier_block *self,
+                                    unsigned long action, void *hcpu)
+{
+        int cpu = (unsigned long)hcpu;
+        switch (action) {
+        case CPU_ONLINE:
+        case CPU_DOWN_FAILED:
+        case CPU_ONLINE_FROZEN:
+                smp_call_function_single(cpu, kvm_guest_cpu_online, NULL, 0);
+                break;
+        case CPU_DOWN_PREPARE:
+        case CPU_DOWN_PREPARE_FROZEN:
+                smp_call_function_single(cpu, kvm_guest_cpu_offline, NULL, 1);
+                break;
+        default:
+                break;
+        }
+        return NOTIFY_OK;
+}
+
+static struct notifier_block __cpuinitdata kvm_cpu_notifier = {
+        .notifier_call  = kvm_cpu_notify,
+};
+#endif
+
+static void __init kvm_apf_trap_init(void)
+{
+        set_intr_gate(14, &async_page_fault);
+}
+
 void __init kvm_guest_init(void)
 {
+        int i;
+
         if (!kvm_para_available())
                 return;
 
         paravirt_ops_setup();
+        register_reboot_notifier(&kvm_pv_reboot_nb);
+        for (i = 0; i < KVM_TASK_SLEEP_HASHSIZE; i++)
+                spin_lock_init(&async_pf_sleepers[i].lock);
+        if (kvm_para_has_feature(KVM_FEATURE_ASYNC_PF))
+                x86_init.irqs.trap_init = kvm_apf_trap_init;
+
+#ifdef CONFIG_SMP
+        smp_ops.smp_prepare_boot_cpu = kvm_smp_prepare_boot_cpu;
+        register_cpu_notifier(&kvm_cpu_notifier);
+#else
+        kvm_guest_cpu_init();
+#endif
 }
diff --git a/arch/x86/kernel/kvmclock.c b/arch/x86/kernel/kvmclock.c
index ca43ce31a19..f98d3eafe07 100644
--- a/arch/x86/kernel/kvmclock.c
+++ b/arch/x86/kernel/kvmclock.c
@@ -125,7 +125,7 @@ static struct clocksource kvm_clock = {
         .flags = CLOCK_SOURCE_IS_CONTINUOUS,
 };
 
-static int kvm_register_clock(char *txt)
+int kvm_register_clock(char *txt)
 {
         int cpu = smp_processor_id();
         int low, high, ret;
@@ -152,14 +152,6 @@ static void __cpuinit kvm_setup_secondary_clock(void)
 }
 #endif
 
-#ifdef CONFIG_SMP
-static void __init kvm_smp_prepare_boot_cpu(void)
-{
-        WARN_ON(kvm_register_clock("primary cpu clock"));
-        native_smp_prepare_boot_cpu();
-}
-#endif
-
 /*
  * After the clock is registered, the host will keep writing to the
  * registered memory location. If the guest happens to shutdown, this memory
@@ -206,9 +198,6 @@ void __init kvmclock_init(void)
         x86_cpuinit.setup_percpu_clockev =
                 kvm_setup_secondary_clock;
 #endif
-#ifdef CONFIG_SMP
-        smp_ops.smp_prepare_boot_cpu = kvm_smp_prepare_boot_cpu;
-#endif
         machine_ops.shutdown  = kvm_shutdown;
 #ifdef CONFIG_KEXEC
         machine_ops.crash_shutdown  = kvm_crash_shutdown;
diff --git a/arch/x86/kernel/rtc.c b/arch/x86/kernel/rtc.c
index 1cfbbfc3ae2..6f39cab052d 100644
--- a/arch/x86/kernel/rtc.c
+++ b/arch/x86/kernel/rtc.c
@@ -76,7 +76,7 @@ int mach_set_rtc_mmss(unsigned long nowtime)
                 CMOS_WRITE(real_seconds, RTC_SECONDS);
                 CMOS_WRITE(real_minutes, RTC_MINUTES);
         } else {
-                printk(KERN_WARNING
+                printk_once(KERN_NOTICE
                        "set_rtc_mmss: can't update from %d to %d\n",
                        cmos_minutes, real_minutes);
                 retval = -1;
diff --git a/arch/x86/kvm/Kconfig b/arch/x86/kvm/Kconfig
index ddc131ff438..50f63648ce1 100644
--- a/arch/x86/kvm/Kconfig
+++ b/arch/x86/kvm/Kconfig
@@ -28,6 +28,7 @@ config KVM
         select HAVE_KVM_IRQCHIP
         select HAVE_KVM_EVENTFD
         select KVM_APIC_ARCHITECTURE
+        select KVM_ASYNC_PF
         select USER_RETURN_NOTIFIER
         select KVM_MMIO
         ---help---
diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile
index 31a7035c4bd..f15501f431c 100644
--- a/arch/x86/kvm/Makefile
+++ b/arch/x86/kvm/Makefile
@@ -1,5 +1,5 @@
 
-EXTRA_CFLAGS += -Ivirt/kvm -Iarch/x86/kvm
+ccflags-y += -Ivirt/kvm -Iarch/x86/kvm
 
 CFLAGS_x86.o := -I.
 CFLAGS_svm.o := -I.
@@ -9,6 +9,7 @@ kvm-y			+= $(addprefix ../../../virt/kvm/, kvm_main.o ioapic.o \
                                 coalesced_mmio.o irq_comm.o eventfd.o \
                                 assigned-dev.o)
 kvm-$(CONFIG_IOMMU_API) += $(addprefix ../../../virt/kvm/, iommu.o)
+kvm-$(CONFIG_KVM_ASYNC_PF)      += $(addprefix ../../../virt/kvm/, async_pf.o)
 
 kvm-y                   += x86.o mmu.o emulate.o i8259.o irq.o lapic.o \
                            i8254.o timer.o
diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c
index 38b6e8dafaf..caf966781d2 100644
--- a/arch/x86/kvm/emulate.c
+++ b/arch/x86/kvm/emulate.c
@@ -20,16 +20,8 @@
  * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
  */
 
-#ifndef __KERNEL__
-#include <stdio.h>
-#include <stdint.h>
-#include <public/xen.h>
-#define DPRINTF(_f, _a ...) printf(_f , ## _a)
-#else
 #include <linux/kvm_host.h>
 #include "kvm_cache_regs.h"
-#define DPRINTF(x...) do {} while (0)
-#endif
 #include <linux/module.h>
 #include <asm/kvm_emulate.h>
 
@@ -418,9 +410,9 @@ address_mask(struct decode_cache *c, unsigned long reg)
 }
 
 static inline unsigned long
-register_address(struct decode_cache *c, unsigned long base, unsigned long reg)
+register_address(struct decode_cache *c, unsigned long reg)
 {
-        return base + address_mask(c, reg);
+        return address_mask(c, reg);
 }
 
 static inline void
@@ -452,60 +444,55 @@ static unsigned long seg_base(struct x86_emulate_ctxt *ctxt,
         return ops->get_cached_segment_base(seg, ctxt->vcpu);
 }
 
-static unsigned long seg_override_base(struct x86_emulate_ctxt *ctxt,
-                                       struct x86_emulate_ops *ops,
-                                       struct decode_cache *c)
+static unsigned seg_override(struct x86_emulate_ctxt *ctxt,
+                             struct x86_emulate_ops *ops,
+                             struct decode_cache *c)
 {
         if (!c->has_seg_override)
                 return 0;
 
-        return seg_base(ctxt, ops, c->seg_override);
+        return c->seg_override;
 }
 
-static unsigned long es_base(struct x86_emulate_ctxt *ctxt,
-                             struct x86_emulate_ops *ops)
+static ulong linear(struct x86_emulate_ctxt *ctxt,
+                    struct segmented_address addr)
 {
-        return seg_base(ctxt, ops, VCPU_SREG_ES);
-}
-
-static unsigned long ss_base(struct x86_emulate_ctxt *ctxt,
-                             struct x86_emulate_ops *ops)
-{
-        return seg_base(ctxt, ops, VCPU_SREG_SS);
-}
+        struct decode_cache *c = &ctxt->decode;
+        ulong la;
 
-static void emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
-                                      u32 error, bool valid)
-{
-        ctxt->exception = vec;
-        ctxt->error_code = error;
-        ctxt->error_code_valid = valid;
+        la = seg_base(ctxt, ctxt->ops, addr.seg) + addr.ea;
+        if (c->ad_bytes != 8)
+                la &= (u32)-1;
+        return la;
 }
 
-static void emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
+static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
+                             u32 error, bool valid)
 {
-        emulate_exception(ctxt, GP_VECTOR, err, true);
+        ctxt->exception.vector = vec;
+        ctxt->exception.error_code = error;
+        ctxt->exception.error_code_valid = valid;
+        return X86EMUL_PROPAGATE_FAULT;
 }
 
-static void emulate_pf(struct x86_emulate_ctxt *ctxt)
+static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
 {
-        emulate_exception(ctxt, PF_VECTOR, 0, true);
+        return emulate_exception(ctxt, GP_VECTOR, err, true);
 }
 
-static void emulate_ud(struct x86_emulate_ctxt *ctxt)
+static int emulate_ud(struct x86_emulate_ctxt *ctxt)
 {
-        emulate_exception(ctxt, UD_VECTOR, 0, false);
+        return emulate_exception(ctxt, UD_VECTOR, 0, false);
 }
 
-static void emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
+static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
 {
-        emulate_exception(ctxt, TS_VECTOR, err, true);
+        return emulate_exception(ctxt, TS_VECTOR, err, true);
 }
 
 static int emulate_de(struct x86_emulate_ctxt *ctxt)
 {
-        emulate_exception(ctxt, DE_VECTOR, 0, false);
-        return X86EMUL_PROPAGATE_FAULT;
+        return emulate_exception(ctxt, DE_VECTOR, 0, false);
 }
 
 static int do_fetch_insn_byte(struct x86_emulate_ctxt *ctxt,
@@ -520,7 +507,7 @@ static int do_fetch_insn_byte(struct x86_emulate_ctxt *ctxt,
                 cur_size = fc->end - fc->start;
                 size = min(15UL - cur_size, PAGE_SIZE - offset_in_page(eip));
                 rc = ops->fetch(ctxt->cs_base + eip, fc->data + cur_size,
-                                size, ctxt->vcpu, NULL);
+                                size, ctxt->vcpu, &ctxt->exception);
                 if (rc != X86EMUL_CONTINUE)
                         return rc;
                 fc->end += size;
@@ -564,7 +551,7 @@ static void *decode_register(u8 modrm_reg, unsigned long *regs,
 
 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
                            struct x86_emulate_ops *ops,
-                           ulong addr,
+                           struct segmented_address addr,
                            u16 *size, unsigned long *address, int op_bytes)
 {
         int rc;
@@ -572,10 +559,13 @@ static int read_descriptor(struct x86_emulate_ctxt *ctxt,
         if (op_bytes == 2)
                 op_bytes = 3;
         *address = 0;
-        rc = ops->read_std(addr, (unsigned long *)size, 2, ctxt->vcpu, NULL);
+        rc = ops->read_std(linear(ctxt, addr), (unsigned long *)size, 2,
+                           ctxt->vcpu, &ctxt->exception);
         if (rc != X86EMUL_CONTINUE)
                 return rc;
-        rc = ops->read_std(addr + 2, address, op_bytes, ctxt->vcpu, NULL);
+        addr.ea += 2;
+        rc = ops->read_std(linear(ctxt, addr), address, op_bytes,
+                           ctxt->vcpu, &ctxt->exception);
         return rc;
 }
 
@@ -768,7 +758,7 @@ static int decode_modrm(struct x86_emulate_ctxt *ctxt,
                         break;
                 }
         }
-        op->addr.mem = modrm_ea;
+        op->addr.mem.ea = modrm_ea;
 done:
         return rc;
 }
@@ -783,13 +773,13 @@ static int decode_abs(struct x86_emulate_ctxt *ctxt,
         op->type = OP_MEM;
         switch (c->ad_bytes) {
         case 2:
-                op->addr.mem = insn_fetch(u16, 2, c->eip);
+                op->addr.mem.ea = insn_fetch(u16, 2, c->eip);
                 break;
         case 4:
-                op->addr.mem = insn_fetch(u32, 4, c->eip);
+                op->addr.mem.ea = insn_fetch(u32, 4, c->eip);
                 break;
         case 8:
-                op->addr.mem = insn_fetch(u64, 8, c->eip);
+                op->addr.mem.ea = insn_fetch(u64, 8, c->eip);
                 break;
         }
 done:
@@ -808,7 +798,7 @@ static void fetch_bit_operand(struct decode_cache *c)
                 else if (c->src.bytes == 4)
                         sv = (s32)c->src.val & (s32)mask;
 
-                c->dst.addr.mem += (sv >> 3);
+                c->dst.addr.mem.ea += (sv >> 3);
         }
 
         /* only subword offset */
@@ -821,7 +811,6 @@ static int read_emulated(struct x86_emulate_ctxt *ctxt,
 {
         int rc;
         struct read_cache *mc = &ctxt->decode.mem_read;
-        u32 err;
 
         while (size) {
                 int n = min(size, 8u);
@@ -829,10 +818,8 @@ static int read_emulated(struct x86_emulate_ctxt *ctxt,
                 if (mc->pos < mc->end)
                         goto read_cached;
 
-                rc = ops->read_emulated(addr, mc->data + mc->end, n, &err,
-                                        ctxt->vcpu);
-                if (rc == X86EMUL_PROPAGATE_FAULT)
-                        emulate_pf(ctxt);
+                rc = ops->read_emulated(addr, mc->data + mc->end, n,
+                                        &ctxt->exception, ctxt->vcpu);
                 if (rc != X86EMUL_CONTINUE)
                         return rc;
                 mc->end += n;
@@ -907,19 +894,15 @@ static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
         struct desc_ptr dt;
         u16 index = selector >> 3;
         int ret;
-        u32 err;
         ulong addr;
 
         get_descriptor_table_ptr(ctxt, ops, selector, &dt);
 
-        if (dt.size < index * 8 + 7) {
-                emulate_gp(ctxt, selector & 0xfffc);
-                return X86EMUL_PROPAGATE_FAULT;
-        }
+        if (dt.size < index * 8 + 7)
+                return emulate_gp(ctxt, selector & 0xfffc);
         addr = dt.address + index * 8;
-        ret = ops->read_std(addr, desc, sizeof *desc, ctxt->vcpu,  &err);
-        if (ret == X86EMUL_PROPAGATE_FAULT)
-                emulate_pf(ctxt);
+        ret = ops->read_std(addr, desc, sizeof *desc, ctxt->vcpu,
+                            &ctxt->exception);
 
        return ret;
 }
@@ -931,21 +914,17 @@ static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
 {
         struct desc_ptr dt;
         u16 index = selector >> 3;
-        u32 err;
         ulong addr;
         int ret;
 
         get_descriptor_table_ptr(ctxt, ops, selector, &dt);
 
-        if (dt.size < index * 8 + 7) {
-                emulate_gp(ctxt, selector & 0xfffc);
-                return X86EMUL_PROPAGATE_FAULT;
-        }
+        if (dt.size < index * 8 + 7)
+                return emulate_gp(ctxt, selector & 0xfffc);
 
         addr = dt.address + index * 8;
-        ret = ops->write_std(addr, desc, sizeof *desc, ctxt->vcpu, &err);
-        if (ret == X86EMUL_PROPAGATE_FAULT)
-                emulate_pf(ctxt);
+        ret = ops->write_std(addr, desc, sizeof *desc, ctxt->vcpu,
+                             &ctxt->exception);
 
         return ret;
 }
@@ -1092,7 +1071,6 @@ static inline int writeback(struct x86_emulate_ctxt *ctxt,
 {
         int rc;
         struct decode_cache *c = &ctxt->decode;
-        u32 err;
 
         switch (c->dst.type) {
         case OP_REG:
@@ -1101,21 +1079,19 @@ static inline int writeback(struct x86_emulate_ctxt *ctxt,
         case OP_MEM:
                 if (c->lock_prefix)
                         rc = ops->cmpxchg_emulated(
-                                        c->dst.addr.mem,
+                                        linear(ctxt, c->dst.addr.mem),
                                         &c->dst.orig_val,
                                         &c->dst.val,
                                         c->dst.bytes,
-                                        &err,
+                                        &ctxt->exception,
                                         ctxt->vcpu);
                 else
                         rc = ops->write_emulated(
-                                        c->dst.addr.mem,
+                                        linear(ctxt, c->dst.addr.mem),
                                         &c->dst.val,
                                         c->dst.bytes,
-                                        &err,
+                                        &ctxt->exception,
                                         ctxt->vcpu);
-                if (rc == X86EMUL_PROPAGATE_FAULT)
-                        emulate_pf(ctxt);
                 if (rc != X86EMUL_CONTINUE)
                         return rc;
                 break;
@@ -1137,8 +1113,8 @@ static inline void emulate_push(struct x86_emulate_ctxt *ctxt,
         c->dst.bytes = c->op_bytes;
         c->dst.val = c->src.val;
         register_address_increment(c, &c->regs[VCPU_REGS_RSP], -c->op_bytes);
-        c->dst.addr.mem = register_address(c, ss_base(ctxt, ops),
-                                           c->regs[VCPU_REGS_RSP]);
+        c->dst.addr.mem.ea = register_address(c, c->regs[VCPU_REGS_RSP]);
+        c->dst.addr.mem.seg = VCPU_SREG_SS;
 }
 
 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
@@ -1147,10 +1123,11 @@ static int emulate_pop(struct x86_emulate_ctxt *ctxt,
 {
         struct decode_cache *c = &ctxt->decode;
         int rc;
+        struct segmented_address addr;
 
-        rc = read_emulated(ctxt, ops, register_address(c, ss_base(ctxt, ops),
-                                                       c->regs[VCPU_REGS_RSP]),
-                           dest, len);
+        addr.ea = register_address(c, c->regs[VCPU_REGS_RSP]);
+        addr.seg = VCPU_SREG_SS;
+        rc = read_emulated(ctxt, ops, linear(ctxt, addr), dest, len);
         if (rc != X86EMUL_CONTINUE)
                 return rc;
 
@@ -1184,10 +1161,8 @@ static int emulate_popf(struct x86_emulate_ctxt *ctxt,
                         change_mask |= EFLG_IF;
                 break;
         case X86EMUL_MODE_VM86:
-                if (iopl < 3) {
-                        emulate_gp(ctxt, 0);
-                        return X86EMUL_PROPAGATE_FAULT;
-                }
+                if (iopl < 3)
+                        return emulate_gp(ctxt, 0);
                 change_mask |= EFLG_IF;
                 break;
         default: /* real mode */
@@ -1198,9 +1173,6 @@ static int emulate_popf(struct x86_emulate_ctxt *ctxt,
         *(unsigned long *)dest =
                 (ctxt->eflags & ~change_mask) | (val & change_mask);
 
-        if (rc == X86EMUL_PROPAGATE_FAULT)
-                emulate_pf(ctxt);
-
         return rc;
 }
 
@@ -1287,7 +1259,6 @@ int emulate_int_real(struct x86_emulate_ctxt *ctxt,
         gva_t cs_addr;
         gva_t eip_addr;
         u16 cs, eip;
-        u32 err;
 
         /* TODO: Add limit checks */
         c->src.val = ctxt->eflags;
@@ -1317,11 +1288,11 @@ int emulate_int_real(struct x86_emulate_ctxt *ctxt,
         eip_addr = dt.address + (irq << 2);
         cs_addr = dt.address + (irq << 2) + 2;
 
-        rc = ops->read_std(cs_addr, &cs, 2, ctxt->vcpu, &err);
+        rc = ops->read_std(cs_addr, &cs, 2, ctxt->vcpu, &ctxt->exception);
         if (rc != X86EMUL_CONTINUE)
                 return rc;
 
-        rc = ops->read_std(eip_addr, &eip, 2, ctxt->vcpu, &err);
+        rc = ops->read_std(eip_addr, &eip, 2, ctxt->vcpu, &ctxt->exception);
         if (rc != X86EMUL_CONTINUE)
                 return rc;
 
@@ -1370,10 +1341,8 @@ static int emulate_iret_real(struct x86_emulate_ctxt *ctxt,
         if (rc != X86EMUL_CONTINUE)
                 return rc;
 
-        if (temp_eip & ~0xffff) {
-                emulate_gp(ctxt, 0);
-                return X86EMUL_PROPAGATE_FAULT;
-        }
+        if (temp_eip & ~0xffff)
+                return emulate_gp(ctxt, 0);
 
         rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
 
@@ -1624,10 +1593,8 @@ emulate_syscall(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
 
         /* syscall is not available in real mode */
         if (ctxt->mode == X86EMUL_MODE_REAL ||
-            ctxt->mode == X86EMUL_MODE_VM86) {
-                emulate_ud(ctxt);
-                return X86EMUL_PROPAGATE_FAULT;
-        }
+            ctxt->mode == X86EMUL_MODE_VM86)
+                return emulate_ud(ctxt);
 
         setup_syscalls_segments(ctxt, ops, &cs, &ss);
 
@@ -1678,34 +1645,26 @@ emulate_sysenter(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
         u16 cs_sel, ss_sel;
 
         /* inject #GP if in real mode */
-        if (ctxt->mode == X86EMUL_MODE_REAL) {
-                emulate_gp(ctxt, 0);
-                return X86EMUL_PROPAGATE_FAULT;
-        }
+        if (ctxt->mode == X86EMUL_MODE_REAL)
+                return emulate_gp(ctxt, 0);
 
         /* XXX sysenter/sysexit have not been tested in 64bit mode.
         * Therefore, we inject an #UD.
         */
-        if (ctxt->mode == X86EMUL_MODE_PROT64) {
-                emulate_ud(ctxt);
-                return X86EMUL_PROPAGATE_FAULT;
-        }
+        if (ctxt->mode == X86EMUL_MODE_PROT64)
+                return emulate_ud(ctxt);
 
         setup_syscalls_segments(ctxt, ops, &cs, &ss);
 
         ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
         switch (ctxt->mode) {
         case X86EMUL_MODE_PROT32:
-                if ((msr_data & 0xfffc) == 0x0) {
-                        emulate_gp(ctxt, 0);
-                        return X86EMUL_PROPAGATE_FAULT;
-                }
+                if ((msr_data & 0xfffc) == 0x0)
+                        return emulate_gp(ctxt, 0);
                 break;
         case X86EMUL_MODE_PROT64:
-                if (msr_data == 0x0) {
-                        emulate_gp(ctxt, 0);
-                        return X86EMUL_PROPAGATE_FAULT;
-                }
+                if (msr_data == 0x0)
+                        return emulate_gp(ctxt, 0);
                 break;
         }
 
@@ -1745,10 +1704,8 @@ emulate_sysexit(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
 
         /* inject #GP if in real mode or Virtual 8086 mode */
         if (ctxt->mode == X86EMUL_MODE_REAL ||
-            ctxt->mode == X86EMUL_MODE_VM86) {
-                emulate_gp(ctxt, 0);
-                return X86EMUL_PROPAGATE_FAULT;
-        }
+            ctxt->mode == X86EMUL_MODE_VM86)
+                return emulate_gp(ctxt, 0);
 
         setup_syscalls_segments(ctxt, ops, &cs, &ss);
 
@@ -1763,18 +1720,14 @@ emulate_sysexit(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
         switch (usermode) {
         case X86EMUL_MODE_PROT32:
                 cs_sel = (u16)(msr_data + 16);
-                if ((msr_data & 0xfffc) == 0x0) {
-                        emulate_gp(ctxt, 0);
-                        return X86EMUL_PROPAGATE_FAULT;
-                }
+                if ((msr_data & 0xfffc) == 0x0)
+                        return emulate_gp(ctxt, 0);
                 ss_sel = (u16)(msr_data + 24);
                 break;
         case X86EMUL_MODE_PROT64:
                 cs_sel = (u16)(msr_data + 32);
-                if (msr_data == 0x0) {
-                        emulate_gp(ctxt, 0);
-                        return X86EMUL_PROPAGATE_FAULT;
-                }
+                if (msr_data == 0x0)
+                        return emulate_gp(ctxt, 0);
                 ss_sel = cs_sel + 8;
                 cs.d = 0;
                 cs.l = 1;
@@ -1934,33 +1887,27 @@ static int task_switch_16(struct x86_emulate_ctxt *ctxt,
 {
         struct tss_segment_16 tss_seg;
         int ret;
-        u32 err, new_tss_base = get_desc_base(new_desc);
+        u32 new_tss_base = get_desc_base(new_desc);
 
         ret = ops->read_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
-                            &err);
-        if (ret == X86EMUL_PROPAGATE_FAULT) {
+                            &ctxt->exception);
+        if (ret != X86EMUL_CONTINUE)
                 /* FIXME: need to provide precise fault address */
-                emulate_pf(ctxt);
                 return ret;
-        }
 
         save_state_to_tss16(ctxt, ops, &tss_seg);
 
         ret = ops->write_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
-                             &err);
-        if (ret == X86EMUL_PROPAGATE_FAULT) {
+                             &ctxt->exception);
+        if (ret != X86EMUL_CONTINUE)
                 /* FIXME: need to provide precise fault address */
-                emulate_pf(ctxt);
                 return ret;
-        }
 
         ret = ops->read_std(new_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
-                            &err);
-        if (ret == X86EMUL_PROPAGATE_FAULT) {
+                            &ctxt->exception);
+        if (ret != X86EMUL_CONTINUE)
                 /* FIXME: need to provide precise fault address */
-                emulate_pf(ctxt);
                 return ret;
-        }
 
         if (old_tss_sel != 0xffff) {
                 tss_seg.prev_task_link = old_tss_sel;
@@ -1968,12 +1915,10 @@ static int task_switch_16(struct x86_emulate_ctxt *ctxt,
                 ret = ops->write_std(new_tss_base,
                                      &tss_seg.prev_task_link,
                                      sizeof tss_seg.prev_task_link,
-                                     ctxt->vcpu, &err);
-                if (ret == X86EMUL_PROPAGATE_FAULT) {
+                                     ctxt->vcpu, &ctxt->exception);
+                if (ret != X86EMUL_CONTINUE)
                         /* FIXME: need to provide precise fault address */
-                        emulate_pf(ctxt);
                         return ret;
-                }
         }
 
         return load_state_from_tss16(ctxt, ops, &tss_seg);
@@ -2013,10 +1958,8 @@ static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
         struct decode_cache *c = &ctxt->decode;
         int ret;
 
-        if (ops->set_cr(3, tss->cr3, ctxt->vcpu)) {
-                emulate_gp(ctxt, 0);
-                return X86EMUL_PROPAGATE_FAULT;
-        }
+        if (ops->set_cr(3, tss->cr3, ctxt->vcpu))
+                return emulate_gp(ctxt, 0);
         c->eip = tss->eip;
         ctxt->eflags = tss->eflags | 2;
         c->regs[VCPU_REGS_RAX] = tss->eax;
@@ -2076,33 +2019,27 @@ static int task_switch_32(struct x86_emulate_ctxt *ctxt,
 {
         struct tss_segment_32 tss_seg;
         int ret;
-        u32 err, new_tss_base = get_desc_base(new_desc);
+        u32 new_tss_base = get_desc_base(new_desc);
 
         ret = ops->read_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
-                            &err);
-        if (ret == X86EMUL_PROPAGATE_FAULT) {
+                            &ctxt->exception);
+        if (ret != X86EMUL_CONTINUE)
                 /* FIXME: need to provide precise fault address */
-                emulate_pf(ctxt);
                 return ret;
-        }
 
         save_state_to_tss32(ctxt, ops, &tss_seg);
 
         ret = ops->write_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
-                             &err);
-        if (ret == X86EMUL_PROPAGATE_FAULT) {
+                             &ctxt->exception);
+        if (ret != X86EMUL_CONTINUE)
                 /* FIXME: need to provide precise fault address */
-                emulate_pf(ctxt);
                 return ret;
-        }
 
         ret = ops->read_std(new_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
-                            &err);
-        if (ret == X86EMUL_PROPAGATE_FAULT) {
+                            &ctxt->exception);
+        if (ret != X86EMUL_CONTINUE)
                 /* FIXME: need to provide precise fault address */
-                emulate_pf(ctxt);
                 return ret;
-        }
 
         if (old_tss_sel != 0xffff) {
                 tss_seg.prev_task_link = old_tss_sel;
@@ -2110,12 +2047,10 @@ static int task_switch_32(struct x86_emulate_ctxt *ctxt,
                 ret = ops->write_std(new_tss_base,
                                      &tss_seg.prev_task_link,
                                      sizeof tss_seg.prev_task_link,
-                                     ctxt->vcpu, &err);
-                if (ret == X86EMUL_PROPAGATE_FAULT) {
+                                     ctxt->vcpu, &ctxt->exception);
+                if (ret != X86EMUL_CONTINUE)
                         /* FIXME: need to provide precise fault address */
-                        emulate_pf(ctxt);
                         return ret;
-                }
         }
 
         return load_state_from_tss32(ctxt, ops, &tss_seg);
@@ -2146,10 +2081,8 @@ static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
 
         if (reason != TASK_SWITCH_IRET) {
                 if ((tss_selector & 3) > next_tss_desc.dpl ||
-                    ops->cpl(ctxt->vcpu) > next_tss_desc.dpl) {
-                        emulate_gp(ctxt, 0);
-                        return X86EMUL_PROPAGATE_FAULT;
-                }
+                    ops->cpl(ctxt->vcpu) > next_tss_desc.dpl)
+                        return emulate_gp(ctxt, 0);
         }
 
         desc_limit = desc_limit_scaled(&next_tss_desc);
@@ -2231,14 +2164,15 @@ int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
         return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0;
 }
 
-static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned long base,
+static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned seg,
                             int reg, struct operand *op)
 {
         struct decode_cache *c = &ctxt->decode;
         int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;
 
         register_address_increment(c, &c->regs[reg], df * op->bytes);
-        op->addr.mem = register_address(c,  base, c->regs[reg]);
+        op->addr.mem.ea = register_address(c, c->regs[reg]);
+        op->addr.mem.seg = seg;
 }
 
 static int em_push(struct x86_emulate_ctxt *ctxt)
@@ -2369,10 +2303,8 @@ static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
         struct decode_cache *c = &ctxt->decode;
         u64 tsc = 0;
 
-        if (cpl > 0 && (ctxt->ops->get_cr(4, ctxt->vcpu) & X86_CR4_TSD)) {
-                emulate_gp(ctxt, 0);
-                return X86EMUL_PROPAGATE_FAULT;
-        }
+        if (cpl > 0 && (ctxt->ops->get_cr(4, ctxt->vcpu) & X86_CR4_TSD))
+                return emulate_gp(ctxt, 0);
         ctxt->ops->get_msr(ctxt->vcpu, MSR_IA32_TSC, &tsc);
         c->regs[VCPU_REGS_RAX] = (u32)tsc;
         c->regs[VCPU_REGS_RDX] = tsc >> 32;
@@ -2647,7 +2579,7 @@ static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
 
         op->type = OP_IMM;
         op->bytes = size;
-        op->addr.mem = c->eip;
+        op->addr.mem.ea = c->eip;
         /* NB. Immediates are sign-extended as necessary. */
         switch (op->bytes) {
         case 1:
@@ -2678,7 +2610,7 @@ done:
 }
 
 int
-x86_decode_insn(struct x86_emulate_ctxt *ctxt)
+x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
 {
         struct x86_emulate_ops *ops = ctxt->ops;
         struct decode_cache *c = &ctxt->decode;
@@ -2689,7 +2621,10 @@ x86_decode_insn(struct x86_emulate_ctxt *ctxt)
         struct operand memop = { .type = OP_NONE };
 
         c->eip = ctxt->eip;
-        c->fetch.start = c->fetch.end = c->eip;
+        c->fetch.start = c->eip;
+        c->fetch.end = c->fetch.start + insn_len;
+        if (insn_len > 0)
+                memcpy(c->fetch.data, insn, insn_len);
         ctxt->cs_base = seg_base(ctxt, ops, VCPU_SREG_CS);
 
         switch (mode) {
@@ -2803,10 +2738,8 @@ done_prefixes:
         c->execute = opcode.u.execute;
 
         /* Unrecognised? */
-        if (c->d == 0 || (c->d & Undefined)) {
-                DPRINTF("Cannot emulate %02x\n", c->b);
+        if (c->d == 0 || (c->d & Undefined))
                 return -1;
-        }
 
         if (mode == X86EMUL_MODE_PROT64 && (c->d & Stack))
                 c->op_bytes = 8;
@@ -2831,14 +2764,13 @@ done_prefixes:
         if (!c->has_seg_override)
                 set_seg_override(c, VCPU_SREG_DS);
 
-        if (memop.type == OP_MEM && !(!c->twobyte && c->b == 0x8d))
-                memop.addr.mem += seg_override_base(ctxt, ops, c);
+        memop.addr.mem.seg = seg_override(ctxt, ops, c);
 
         if (memop.type == OP_MEM && c->ad_bytes != 8)
-                memop.addr.mem = (u32)memop.addr.mem;
+                memop.addr.mem.ea = (u32)memop.addr.mem.ea;
 
         if (memop.type == OP_MEM && c->rip_relative)
-                memop.addr.mem += c->eip;
+                memop.addr.mem.ea += c->eip;
 
         /*
          * Decode and fetch the source operand: register, memory
@@ -2890,14 +2822,14 @@ done_prefixes:
         case SrcSI:
                 c->src.type = OP_MEM;
                 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
-                c->src.addr.mem =
-                        register_address(c,  seg_override_base(ctxt, ops, c),
-                                         c->regs[VCPU_REGS_RSI]);
+                c->src.addr.mem.ea =
+                        register_address(c, c->regs[VCPU_REGS_RSI]);
+                c->src.addr.mem.seg = seg_override(ctxt, ops, c),
                 c->src.val = 0;
                 break;
         case SrcImmFAddr:
                 c->src.type = OP_IMM;
-                c->src.addr.mem = c->eip;
+                c->src.addr.mem.ea = c->eip;
                 c->src.bytes = c->op_bytes + 2;
                 insn_fetch_arr(c->src.valptr, c->src.bytes, c->eip);
                 break;
@@ -2944,7 +2876,7 @@ done_prefixes:
                 break;
         case DstImmUByte:
                 c->dst.type = OP_IMM;
-                c->dst.addr.mem = c->eip;
+                c->dst.addr.mem.ea = c->eip;
                 c->dst.bytes = 1;
                 c->dst.val = insn_fetch(u8, 1, c->eip);
                 break;
@@ -2969,9 +2901,9 @@ done_prefixes:
         case DstDI:
                 c->dst.type = OP_MEM;
                 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
-                c->dst.addr.mem =
-                        register_address(c, es_base(ctxt, ops),
-                                         c->regs[VCPU_REGS_RDI]);
+                c->dst.addr.mem.ea =
+                        register_address(c, c->regs[VCPU_REGS_RDI]);
+                c->dst.addr.mem.seg = VCPU_SREG_ES;
                 c->dst.val = 0;
                 break;
         case ImplicitOps:
@@ -3020,24 +2952,24 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
         ctxt->decode.mem_read.pos = 0;
 
         if (ctxt->mode == X86EMUL_MODE_PROT64 && (c->d & No64)) {
-                emulate_ud(ctxt);
+                rc = emulate_ud(ctxt);
                 goto done;
         }
 
         /* LOCK prefix is allowed only with some instructions */
         if (c->lock_prefix && (!(c->d & Lock) || c->dst.type != OP_MEM)) {
-                emulate_ud(ctxt);
+                rc = emulate_ud(ctxt);
                 goto done;
         }
 
         if ((c->d & SrcMask) == SrcMemFAddr && c->src.type != OP_MEM) {
-                emulate_ud(ctxt);
+                rc = emulate_ud(ctxt);
                 goto done;
         }
 
         /* Privileged instruction can be executed only in CPL=0 */
         if ((c->d & Priv) && ops->cpl(ctxt->vcpu)) {
-                emulate_gp(ctxt, 0);
+                rc = emulate_gp(ctxt, 0);
                 goto done;
         }
 
@@ -3050,7 +2982,7 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
         }
 
         if ((c->src.type == OP_MEM) && !(c->d & NoAccess)) {
-                rc = read_emulated(ctxt, ops, c->src.addr.mem,
+                rc = read_emulated(ctxt, ops, linear(ctxt, c->src.addr.mem),
                                         c->src.valptr, c->src.bytes);
                 if (rc != X86EMUL_CONTINUE)
                         goto done;
@@ -3058,7 +2990,7 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
         }
 
         if (c->src2.type == OP_MEM) {
-                rc = read_emulated(ctxt, ops, c->src2.addr.mem,
+                rc = read_emulated(ctxt, ops, linear(ctxt, c->src2.addr.mem),
                                         &c->src2.val, c->src2.bytes);
                 if (rc != X86EMUL_CONTINUE)
                         goto done;
@@ -3070,7 +3002,7 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
 
         if ((c->dst.type == OP_MEM) && !(c->d & Mov)) {
                 /* optimisation - avoid slow emulated read if Mov */
-                rc = read_emulated(ctxt, ops, c->dst.addr.mem,
+                rc = read_emulated(ctxt, ops, linear(ctxt, c->dst.addr.mem),
                                    &c->dst.val, c->dst.bytes);
                 if (rc != X86EMUL_CONTINUE)
                         goto done;
@@ -3215,13 +3147,13 @@ special_insn:
                 break;
         case 0x8c:  /* mov r/m, sreg */
                 if (c->modrm_reg > VCPU_SREG_GS) {
-                        emulate_ud(ctxt);
+                        rc = emulate_ud(ctxt);
                         goto done;
                 }
                 c->dst.val = ops->get_segment_selector(c->modrm_reg, ctxt->vcpu);
                 break;
         case 0x8d: /* lea r16/r32, m */
-                c->dst.val = c->src.addr.mem;
+                c->dst.val = c->src.addr.mem.ea;
                 break;
         case 0x8e: { /* mov seg, r/m16 */
                 uint16_t sel;
@@ -3230,7 +3162,7 @@ special_insn:
 
                 if (c->modrm_reg == VCPU_SREG_CS ||
                     c->modrm_reg > VCPU_SREG_GS) {
-                        emulate_ud(ctxt);
+                        rc = emulate_ud(ctxt);
                         goto done;
                 }
 
@@ -3268,7 +3200,6 @@ special_insn:
                 break;
         case 0xa6 ... 0xa7:     /* cmps */
                 c->dst.type = OP_NONE; /* Disable writeback. */
-                DPRINTF("cmps: mem1=0x%p mem2=0x%p\n", c->src.addr.mem, c->dst.addr.mem);
                 goto cmp;
         case 0xa8 ... 0xa9:     /* test ax, imm */
                 goto test;
@@ -3363,7 +3294,7 @@ special_insn:
         do_io_in:
                 c->dst.bytes = min(c->dst.bytes, 4u);
                 if (!emulator_io_permited(ctxt, ops, c->src.val, c->dst.bytes)) {
-                        emulate_gp(ctxt, 0);
+                        rc = emulate_gp(ctxt, 0);
                         goto done;
                 }
                 if (!pio_in_emulated(ctxt, ops, c->dst.bytes, c->src.val,
@@ -3377,7 +3308,7 @@ special_insn:
                 c->src.bytes = min(c->src.bytes, 4u);
                 if (!emulator_io_permited(ctxt, ops, c->dst.val,
                                           c->src.bytes)) {
-                        emulate_gp(ctxt, 0);
+                        rc = emulate_gp(ctxt, 0);
                         goto done;
                 }
                 ops->pio_out_emulated(c->src.bytes, c->dst.val,
@@ -3402,14 +3333,14 @@ special_insn:
                 break;
         case 0xfa: /* cli */
                 if (emulator_bad_iopl(ctxt, ops)) {
-                        emulate_gp(ctxt, 0);
+                        rc = emulate_gp(ctxt, 0);
                         goto done;
                 } else
                         ctxt->eflags &= ~X86_EFLAGS_IF;
                 break;
         case 0xfb: /* sti */
                 if (emulator_bad_iopl(ctxt, ops)) {
-                        emulate_gp(ctxt, 0);
+                        rc = emulate_gp(ctxt, 0);
                         goto done;
                 } else {
                         ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
@@ -3449,11 +3380,11 @@ writeback:
         c->dst.type = saved_dst_type;
 
         if ((c->d & SrcMask) == SrcSI)
-                string_addr_inc(ctxt, seg_override_base(ctxt, ops, c),
+                string_addr_inc(ctxt, seg_override(ctxt, ops, c),
                                 VCPU_REGS_RSI, &c->src);
 
         if ((c->d & DstMask) == DstDI)
-                string_addr_inc(ctxt, es_base(ctxt, ops), VCPU_REGS_RDI,
+                string_addr_inc(ctxt, VCPU_SREG_ES, VCPU_REGS_RDI,
                                 &c->dst);
 
         if (c->rep_prefix && (c->d & String)) {
@@ -3482,6 +3413,8 @@ writeback:
         ctxt->eip = c->eip;
 
 done:
+        if (rc == X86EMUL_PROPAGATE_FAULT)
+                ctxt->have_exception = true;
         return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
 
 twobyte_insn:
@@ -3544,9 +3477,11 @@ twobyte_insn:
                         break;
                 case 5: /* not defined */
                         emulate_ud(ctxt);
+                        rc = X86EMUL_PROPAGATE_FAULT;
                         goto done;
                 case 7: /* invlpg*/
-                        emulate_invlpg(ctxt->vcpu, c->src.addr.mem);
+                        emulate_invlpg(ctxt->vcpu,
+                                       linear(ctxt, c->src.addr.mem));
                         /* Disable writeback. */
                         c->dst.type = OP_NONE;
                         break;
@@ -3573,6 +3508,7 @@ twobyte_insn:
                 case 5 ... 7:
                 case 9 ... 15:
                         emulate_ud(ctxt);
+                        rc = X86EMUL_PROPAGATE_FAULT;
                         goto done;
                 }
                 c->dst.val = ops->get_cr(c->modrm_reg, ctxt->vcpu);
@@ -3581,6 +3517,7 @@ twobyte_insn:
                 if ((ops->get_cr(4, ctxt->vcpu) & X86_CR4_DE) &&
                     (c->modrm_reg == 4 || c->modrm_reg == 5)) {
                         emulate_ud(ctxt);
+                        rc = X86EMUL_PROPAGATE_FAULT;
                         goto done;
                 }
                 ops->get_dr(c->modrm_reg, &c->dst.val, ctxt->vcpu);
@@ -3588,6 +3525,7 @@ twobyte_insn:
         case 0x22: /* mov reg, cr */
                 if (ops->set_cr(c->modrm_reg, c->src.val, ctxt->vcpu)) {
                         emulate_gp(ctxt, 0);
+                        rc = X86EMUL_PROPAGATE_FAULT;
                         goto done;
                 }
                 c->dst.type = OP_NONE;
@@ -3596,6 +3534,7 @@ twobyte_insn:
                 if ((ops->get_cr(4, ctxt->vcpu) & X86_CR4_DE) &&
                     (c->modrm_reg == 4 || c->modrm_reg == 5)) {
                         emulate_ud(ctxt);
+                        rc = X86EMUL_PROPAGATE_FAULT;
                         goto done;
                 }
 
@@ -3604,6 +3543,7 @@ twobyte_insn:
                                  ~0ULL : ~0U), ctxt->vcpu) < 0) {
                         /* #UD condition is already handled by the code above */
                         emulate_gp(ctxt, 0);
+                        rc = X86EMUL_PROPAGATE_FAULT;
                         goto done;
                 }
 
@@ -3615,6 +3555,7 @@ twobyte_insn:
                         | ((u64)c->regs[VCPU_REGS_RDX] << 32);
                 if (ops->set_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], msr_data)) {
                         emulate_gp(ctxt, 0);
+                        rc = X86EMUL_PROPAGATE_FAULT;
                         goto done;
                 }
                 rc = X86EMUL_CONTINUE;
@@ -3623,6 +3564,7 @@ twobyte_insn:
                 /* rdmsr */
                 if (ops->get_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], &msr_data)) {
                         emulate_gp(ctxt, 0);
+                        rc = X86EMUL_PROPAGATE_FAULT;
                         goto done;
                 } else {
                         c->regs[VCPU_REGS_RAX] = (u32)msr_data;
@@ -3785,6 +3727,5 @@ twobyte_insn:
         goto writeback;
 
 cannot_emulate:
-        DPRINTF("Cannot emulate %02x\n", c->b);
         return -1;
 }
diff --git a/arch/x86/kvm/kvm_cache_regs.h b/arch/x86/kvm/kvm_cache_regs.h
index 975bb45329a..3377d53fcd3 100644
--- a/arch/x86/kvm/kvm_cache_regs.h
+++ b/arch/x86/kvm/kvm_cache_regs.h
@@ -73,6 +73,13 @@ static inline ulong kvm_read_cr4_bits(struct kvm_vcpu *vcpu, ulong mask)
         return vcpu->arch.cr4 & mask;
 }
 
+static inline ulong kvm_read_cr3(struct kvm_vcpu *vcpu)
+{
+        if (!test_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail))
+                kvm_x86_ops->decache_cr3(vcpu);
+        return vcpu->arch.cr3;
+}
+
 static inline ulong kvm_read_cr4(struct kvm_vcpu *vcpu)
 {
         return kvm_read_cr4_bits(vcpu, ~0UL);
@@ -84,4 +91,19 @@ static inline u64 kvm_read_edx_eax(struct kvm_vcpu *vcpu)
                 | ((u64)(kvm_register_read(vcpu, VCPU_REGS_RDX) & -1u) << 32);
 }
 
+static inline void enter_guest_mode(struct kvm_vcpu *vcpu)
+{
+        vcpu->arch.hflags |= HF_GUEST_MASK;
+}
+
+static inline void leave_guest_mode(struct kvm_vcpu *vcpu)
+{
+        vcpu->arch.hflags &= ~HF_GUEST_MASK;
+}
+
+static inline bool is_guest_mode(struct kvm_vcpu *vcpu)
+{
+        return vcpu->arch.hflags & HF_GUEST_MASK;
+}
+
 #endif
diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c
index 413f8973a85..93cf9d0d365 100644
--- a/arch/x86/kvm/lapic.c
+++ b/arch/x86/kvm/lapic.c
@@ -277,7 +277,8 @@ static void apic_update_ppr(struct kvm_lapic *apic)
 
         if (old_ppr != ppr) {
                 apic_set_reg(apic, APIC_PROCPRI, ppr);
-                kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
+                if (ppr < old_ppr)
+                        kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
         }
 }
 
diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c
index fbb04aee830..9cafbb49981 100644
--- a/arch/x86/kvm/mmu.c
+++ b/arch/x86/kvm/mmu.c
@@ -18,9 +18,11 @@
  *
  */
 
+#include "irq.h"
 #include "mmu.h"
 #include "x86.h"
 #include "kvm_cache_regs.h"
+#include "x86.h"
 
 #include <linux/kvm_host.h>
 #include <linux/types.h>
@@ -194,7 +196,6 @@ static struct percpu_counter kvm_total_used_mmu_pages;
 
 static u64 __read_mostly shadow_trap_nonpresent_pte;
 static u64 __read_mostly shadow_notrap_nonpresent_pte;
-static u64 __read_mostly shadow_base_present_pte;
 static u64 __read_mostly shadow_nx_mask;
 static u64 __read_mostly shadow_x_mask; /* mutual exclusive with nx_mask */
 static u64 __read_mostly shadow_user_mask;
@@ -213,12 +214,6 @@ void kvm_mmu_set_nonpresent_ptes(u64 trap_pte, u64 notrap_pte)
 }
 EXPORT_SYMBOL_GPL(kvm_mmu_set_nonpresent_ptes);
 
-void kvm_mmu_set_base_ptes(u64 base_pte)
-{
-        shadow_base_present_pte = base_pte;
-}
-EXPORT_SYMBOL_GPL(kvm_mmu_set_base_ptes);
-
 void kvm_mmu_set_mask_ptes(u64 user_mask, u64 accessed_mask,
                 u64 dirty_mask, u64 nx_mask, u64 x_mask)
 {
@@ -482,46 +477,46 @@ static void kvm_mmu_page_set_gfn(struct kvm_mmu_page *sp, int index, gfn_t gfn)
 }
 
 /*
- * Return the pointer to the largepage write count for a given
- * gfn, handling slots that are not large page aligned.
+ * Return the pointer to the large page information for a given gfn,
+ * handling slots that are not large page aligned.
  */
-static int *slot_largepage_idx(gfn_t gfn,
-                               struct kvm_memory_slot *slot,
-                               int level)
+static struct kvm_lpage_info *lpage_info_slot(gfn_t gfn,
+                                              struct kvm_memory_slot *slot,
+                                              int level)
 {
         unsigned long idx;
 
         idx = (gfn >> KVM_HPAGE_GFN_SHIFT(level)) -
               (slot->base_gfn >> KVM_HPAGE_GFN_SHIFT(level));
-        return &slot->lpage_info[level - 2][idx].write_count;
+        return &slot->lpage_info[level - 2][idx];
 }
 
 static void account_shadowed(struct kvm *kvm, gfn_t gfn)
 {
         struct kvm_memory_slot *slot;
-        int *write_count;
+        struct kvm_lpage_info *linfo;
         int i;
 
         slot = gfn_to_memslot(kvm, gfn);
         for (i = PT_DIRECTORY_LEVEL;
              i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
-                write_count   = slot_largepage_idx(gfn, slot, i);
-                *write_count += 1;
+                linfo = lpage_info_slot(gfn, slot, i);
+                linfo->write_count += 1;
         }
 }
 
 static void unaccount_shadowed(struct kvm *kvm, gfn_t gfn)
 {
         struct kvm_memory_slot *slot;
-        int *write_count;
+        struct kvm_lpage_info *linfo;
         int i;
 
         slot = gfn_to_memslot(kvm, gfn);
         for (i = PT_DIRECTORY_LEVEL;
              i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
-                write_count   = slot_largepage_idx(gfn, slot, i);
-                *write_count -= 1;
-                WARN_ON(*write_count < 0);
+                linfo = lpage_info_slot(gfn, slot, i);
+                linfo->write_count -= 1;
+                WARN_ON(linfo->write_count < 0);
         }
 }
 
@@ -530,12 +525,12 @@ static int has_wrprotected_page(struct kvm *kvm,
                                 int level)
 {
         struct kvm_memory_slot *slot;
-        int *largepage_idx;
+        struct kvm_lpage_info *linfo;
 
         slot = gfn_to_memslot(kvm, gfn);
         if (slot) {
-                largepage_idx = slot_largepage_idx(gfn, slot, level);
-                return *largepage_idx;
+                linfo = lpage_info_slot(gfn, slot, level);
+                return linfo->write_count;
         }
 
         return 1;
@@ -590,16 +585,15 @@ static int mapping_level(struct kvm_vcpu *vcpu, gfn_t large_gfn)
 static unsigned long *gfn_to_rmap(struct kvm *kvm, gfn_t gfn, int level)
 {
         struct kvm_memory_slot *slot;
-        unsigned long idx;
+        struct kvm_lpage_info *linfo;
 
         slot = gfn_to_memslot(kvm, gfn);
         if (likely(level == PT_PAGE_TABLE_LEVEL))
                 return &slot->rmap[gfn - slot->base_gfn];
 
-        idx = (gfn >> KVM_HPAGE_GFN_SHIFT(level)) -
-                (slot->base_gfn >> KVM_HPAGE_GFN_SHIFT(level));
+        linfo = lpage_info_slot(gfn, slot, level);
 
-        return &slot->lpage_info[level - 2][idx].rmap_pde;
+        return &linfo->rmap_pde;
 }
 
 /*
@@ -887,19 +881,16 @@ static int kvm_handle_hva(struct kvm *kvm, unsigned long hva,
                 end = start + (memslot->npages << PAGE_SHIFT);
                 if (hva >= start && hva < end) {
                         gfn_t gfn_offset = (hva - start) >> PAGE_SHIFT;
+                        gfn_t gfn = memslot->base_gfn + gfn_offset;
 
                         ret = handler(kvm, &memslot->rmap[gfn_offset], data);
 
                         for (j = 0; j < KVM_NR_PAGE_SIZES - 1; ++j) {
-                                unsigned long idx;
-                                int sh;
-
-                                sh = KVM_HPAGE_GFN_SHIFT(PT_DIRECTORY_LEVEL+j);
-                                idx = ((memslot->base_gfn+gfn_offset) >> sh) -
-                                        (memslot->base_gfn >> sh);
-                                ret |= handler(kvm,
-                                        &memslot->lpage_info[j][idx].rmap_pde,
-                                        data);
+                                struct kvm_lpage_info *linfo;
+
+                                linfo = lpage_info_slot(gfn, memslot,
+                                                        PT_DIRECTORY_LEVEL + j);
+                                ret |= handler(kvm, &linfo->rmap_pde, data);
                         }
                         trace_kvm_age_page(hva, memslot, ret);
                         retval |= ret;
@@ -1161,7 +1152,7 @@ static void nonpaging_prefetch_page(struct kvm_vcpu *vcpu,
 }
 
 static int nonpaging_sync_page(struct kvm_vcpu *vcpu,
-                               struct kvm_mmu_page *sp, bool clear_unsync)
+                               struct kvm_mmu_page *sp)
 {
         return 1;
 }
@@ -1291,7 +1282,7 @@ static int __kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
         if (clear_unsync)
                 kvm_unlink_unsync_page(vcpu->kvm, sp);
 
-        if (vcpu->arch.mmu.sync_page(vcpu, sp, clear_unsync)) {
+        if (vcpu->arch.mmu.sync_page(vcpu, sp)) {
                 kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list);
                 return 1;
         }
@@ -1332,12 +1323,12 @@ static void kvm_sync_pages(struct kvm_vcpu *vcpu,  gfn_t gfn)
                         continue;
 
                 WARN_ON(s->role.level != PT_PAGE_TABLE_LEVEL);
+                kvm_unlink_unsync_page(vcpu->kvm, s);
                 if ((s->role.cr4_pae != !!is_pae(vcpu)) ||
-                        (vcpu->arch.mmu.sync_page(vcpu, s, true))) {
+                        (vcpu->arch.mmu.sync_page(vcpu, s))) {
                         kvm_mmu_prepare_zap_page(vcpu->kvm, s, &invalid_list);
                         continue;
                 }
-                kvm_unlink_unsync_page(vcpu->kvm, s);
                 flush = true;
         }
 
@@ -1963,9 +1954,9 @@ static int set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
                     unsigned pte_access, int user_fault,
                     int write_fault, int dirty, int level,
                     gfn_t gfn, pfn_t pfn, bool speculative,
-                    bool can_unsync, bool reset_host_protection)
+                    bool can_unsync, bool host_writable)
 {
-        u64 spte;
+        u64 spte, entry = *sptep;
         int ret = 0;
 
         /*
@@ -1973,7 +1964,7 @@ static int set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
          * whether the guest actually used the pte (in order to detect
          * demand paging).
          */
-        spte = shadow_base_present_pte;
+        spte = PT_PRESENT_MASK;
         if (!speculative)
                 spte |= shadow_accessed_mask;
         if (!dirty)
@@ -1990,8 +1981,10 @@ static int set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
                 spte |= kvm_x86_ops->get_mt_mask(vcpu, gfn,
                         kvm_is_mmio_pfn(pfn));
 
-        if (reset_host_protection)
+        if (host_writable)
                 spte |= SPTE_HOST_WRITEABLE;
+        else
+                pte_access &= ~ACC_WRITE_MASK;
 
         spte |= (u64)pfn << PAGE_SHIFT;
 
@@ -2036,6 +2029,14 @@ static int set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
 
 set_pte:
         update_spte(sptep, spte);
+        /*
+         * If we overwrite a writable spte with a read-only one we
+         * should flush remote TLBs. Otherwise rmap_write_protect
+         * will find a read-only spte, even though the writable spte
+         * might be cached on a CPU's TLB.
+         */
+        if (is_writable_pte(entry) && !is_writable_pte(*sptep))
+                kvm_flush_remote_tlbs(vcpu->kvm);
 done:
         return ret;
 }
@@ -2045,7 +2046,7 @@ static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
                          int user_fault, int write_fault, int dirty,
                          int *ptwrite, int level, gfn_t gfn,
                          pfn_t pfn, bool speculative,
-                         bool reset_host_protection)
+                         bool host_writable)
 {
         int was_rmapped = 0;
         int rmap_count;
@@ -2080,7 +2081,7 @@ static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
 
         if (set_spte(vcpu, sptep, pte_access, user_fault, write_fault,
                       dirty, level, gfn, pfn, speculative, true,
-                      reset_host_protection)) {
+                      host_writable)) {
                 if (write_fault)
                         *ptwrite = 1;
                 kvm_mmu_flush_tlb(vcpu);
@@ -2211,7 +2212,8 @@ static void direct_pte_prefetch(struct kvm_vcpu *vcpu, u64 *sptep)
 }
 
 static int __direct_map(struct kvm_vcpu *vcpu, gpa_t v, int write,
-                        int level, gfn_t gfn, pfn_t pfn)
+                        int map_writable, int level, gfn_t gfn, pfn_t pfn,
+                        bool prefault)
 {
         struct kvm_shadow_walk_iterator iterator;
         struct kvm_mmu_page *sp;
@@ -2220,9 +2222,11 @@ static int __direct_map(struct kvm_vcpu *vcpu, gpa_t v, int write,
 
         for_each_shadow_entry(vcpu, (u64)gfn << PAGE_SHIFT, iterator) {
                 if (iterator.level == level) {
-                        mmu_set_spte(vcpu, iterator.sptep, ACC_ALL, ACC_ALL,
+                        unsigned pte_access = ACC_ALL;
+
+                        mmu_set_spte(vcpu, iterator.sptep, ACC_ALL, pte_access,
                                      0, write, 1, &pt_write,
-                                     level, gfn, pfn, false, true);
+                                     level, gfn, pfn, prefault, map_writable);
                         direct_pte_prefetch(vcpu, iterator.sptep);
                         ++vcpu->stat.pf_fixed;
                         break;
@@ -2277,12 +2281,17 @@ static int kvm_handle_bad_page(struct kvm *kvm, gfn_t gfn, pfn_t pfn)
         return 1;
 }
 
-static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, int write, gfn_t gfn)
+static bool try_async_pf(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn,
+                         gva_t gva, pfn_t *pfn, bool write, bool *writable);
+
+static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, int write, gfn_t gfn,
+                         bool prefault)
 {
         int r;
         int level;
         pfn_t pfn;
         unsigned long mmu_seq;
+        bool map_writable;
 
         level = mapping_level(vcpu, gfn);
 
@@ -2297,7 +2306,9 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, int write, gfn_t gfn)
 
         mmu_seq = vcpu->kvm->mmu_notifier_seq;
         smp_rmb();
-        pfn = gfn_to_pfn(vcpu->kvm, gfn);
+
+        if (try_async_pf(vcpu, prefault, gfn, v, &pfn, write, &map_writable))
+                return 0;
 
         /* mmio */
         if (is_error_pfn(pfn))
@@ -2307,7 +2318,8 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, int write, gfn_t gfn)
         if (mmu_notifier_retry(vcpu, mmu_seq))
                 goto out_unlock;
         kvm_mmu_free_some_pages(vcpu);
-        r = __direct_map(vcpu, v, write, level, gfn, pfn);
+        r = __direct_map(vcpu, v, write, map_writable, level, gfn, pfn,
+                         prefault);
         spin_unlock(&vcpu->kvm->mmu_lock);
 
 
@@ -2530,6 +2542,7 @@ static void mmu_sync_roots(struct kvm_vcpu *vcpu)
                 hpa_t root = vcpu->arch.mmu.root_hpa;
                 sp = page_header(root);
                 mmu_sync_children(vcpu, sp);
+                trace_kvm_mmu_audit(vcpu, AUDIT_POST_SYNC);
                 return;
         }
         for (i = 0; i < 4; ++i) {
@@ -2552,23 +2565,24 @@ void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu)
 }
 
 static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr,
-                                  u32 access, u32 *error)
+                                  u32 access, struct x86_exception *exception)
 {
-        if (error)
-                *error = 0;
+        if (exception)
+                exception->error_code = 0;
         return vaddr;
 }
 
 static gpa_t nonpaging_gva_to_gpa_nested(struct kvm_vcpu *vcpu, gva_t vaddr,
-                                         u32 access, u32 *error)
+                                         u32 access,
+                                         struct x86_exception *exception)
 {
-        if (error)
-                *error = 0;
+        if (exception)
+                exception->error_code = 0;
         return vcpu->arch.nested_mmu.translate_gpa(vcpu, vaddr, access);
 }
 
 static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva,
-                                u32 error_code)
+                                u32 error_code, bool prefault)
 {
         gfn_t gfn;
         int r;
@@ -2584,17 +2598,67 @@ static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva,
         gfn = gva >> PAGE_SHIFT;
 
         return nonpaging_map(vcpu, gva & PAGE_MASK,
-                             error_code & PFERR_WRITE_MASK, gfn);
+                             error_code & PFERR_WRITE_MASK, gfn, prefault);
+}
+
+static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu, gva_t gva, gfn_t gfn)
+{
+        struct kvm_arch_async_pf arch;
+
+        arch.token = (vcpu->arch.apf.id++ << 12) | vcpu->vcpu_id;
+        arch.gfn = gfn;
+        arch.direct_map = vcpu->arch.mmu.direct_map;
+        arch.cr3 = vcpu->arch.mmu.get_cr3(vcpu);
+
+        return kvm_setup_async_pf(vcpu, gva, gfn, &arch);
 }
 
-static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa,
-                                u32 error_code)
+static bool can_do_async_pf(struct kvm_vcpu *vcpu)
+{
+        if (unlikely(!irqchip_in_kernel(vcpu->kvm) ||
+                     kvm_event_needs_reinjection(vcpu)))
+                return false;
+
+        return kvm_x86_ops->interrupt_allowed(vcpu);
+}
+
+static bool try_async_pf(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn,
+                         gva_t gva, pfn_t *pfn, bool write, bool *writable)
+{
+        bool async;
+
+        *pfn = gfn_to_pfn_async(vcpu->kvm, gfn, &async, write, writable);
+
+        if (!async)
+                return false; /* *pfn has correct page already */
+
+        put_page(pfn_to_page(*pfn));
+
+        if (!prefault && can_do_async_pf(vcpu)) {
+                trace_kvm_try_async_get_page(gva, gfn);
+                if (kvm_find_async_pf_gfn(vcpu, gfn)) {
+                        trace_kvm_async_pf_doublefault(gva, gfn);
+                        kvm_make_request(KVM_REQ_APF_HALT, vcpu);
+                        return true;
+                } else if (kvm_arch_setup_async_pf(vcpu, gva, gfn))
+                        return true;
+        }
+
+        *pfn = gfn_to_pfn_prot(vcpu->kvm, gfn, write, writable);
+
+        return false;
+}
+
+static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa, u32 error_code,
+                          bool prefault)
 {
         pfn_t pfn;
         int r;
         int level;
         gfn_t gfn = gpa >> PAGE_SHIFT;
         unsigned long mmu_seq;
+        int write = error_code & PFERR_WRITE_MASK;
+        bool map_writable;
 
         ASSERT(vcpu);
         ASSERT(VALID_PAGE(vcpu->arch.mmu.root_hpa));
@@ -2609,15 +2673,19 @@ static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa,
 
         mmu_seq = vcpu->kvm->mmu_notifier_seq;
         smp_rmb();
-        pfn = gfn_to_pfn(vcpu->kvm, gfn);
+
+        if (try_async_pf(vcpu, prefault, gfn, gpa, &pfn, write, &map_writable))
+                return 0;
+
+        /* mmio */
         if (is_error_pfn(pfn))
                 return kvm_handle_bad_page(vcpu->kvm, gfn, pfn);
         spin_lock(&vcpu->kvm->mmu_lock);
         if (mmu_notifier_retry(vcpu, mmu_seq))
                 goto out_unlock;
         kvm_mmu_free_some_pages(vcpu);
-        r = __direct_map(vcpu, gpa, error_code & PFERR_WRITE_MASK,
-                         level, gfn, pfn);
+        r = __direct_map(vcpu, gpa, write, map_writable,
+                         level, gfn, pfn, prefault);
         spin_unlock(&vcpu->kvm->mmu_lock);
 
         return r;
@@ -2659,18 +2727,19 @@ void kvm_mmu_flush_tlb(struct kvm_vcpu *vcpu)
 
 static void paging_new_cr3(struct kvm_vcpu *vcpu)
 {
-        pgprintk("%s: cr3 %lx\n", __func__, vcpu->arch.cr3);
+        pgprintk("%s: cr3 %lx\n", __func__, kvm_read_cr3(vcpu));
         mmu_free_roots(vcpu);
 }
 
 static unsigned long get_cr3(struct kvm_vcpu *vcpu)
 {
-        return vcpu->arch.cr3;
+        return kvm_read_cr3(vcpu);
 }
 
-static void inject_page_fault(struct kvm_vcpu *vcpu)
+static void inject_page_fault(struct kvm_vcpu *vcpu,
+                              struct x86_exception *fault)
 {
-        vcpu->arch.mmu.inject_page_fault(vcpu);
+        vcpu->arch.mmu.inject_page_fault(vcpu, fault);
 }
 
 static void paging_free(struct kvm_vcpu *vcpu)
@@ -2816,6 +2885,7 @@ static int init_kvm_tdp_mmu(struct kvm_vcpu *vcpu)
 {
         struct kvm_mmu *context = vcpu->arch.walk_mmu;
 
+        context->base_role.word = 0;
         context->new_cr3 = nonpaging_new_cr3;
         context->page_fault = tdp_page_fault;
         context->free = nonpaging_free;
@@ -3008,9 +3078,6 @@ static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu,
                 return;
         }
 
-        if (is_rsvd_bits_set(&vcpu->arch.mmu, *(u64 *)new, PT_PAGE_TABLE_LEVEL))
-                return;
-
         ++vcpu->kvm->stat.mmu_pte_updated;
         if (!sp->role.cr4_pae)
                 paging32_update_pte(vcpu, sp, spte, new);
@@ -3264,12 +3331,13 @@ void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu)
         }
 }
 
-int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code)
+int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code,
+                       void *insn, int insn_len)
 {
         int r;
         enum emulation_result er;
 
-        r = vcpu->arch.mmu.page_fault(vcpu, cr2, error_code);
+        r = vcpu->arch.mmu.page_fault(vcpu, cr2, error_code, false);
         if (r < 0)
                 goto out;
 
@@ -3282,7 +3350,7 @@ int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code)
         if (r)
                 goto out;
 
-        er = emulate_instruction(vcpu, cr2, error_code, 0);
+        er = x86_emulate_instruction(vcpu, cr2, 0, insn, insn_len);
 
         switch (er) {
         case EMULATE_DONE:
@@ -3377,11 +3445,14 @@ void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot)
                 if (!test_bit(slot, sp->slot_bitmap))
                         continue;
 
+                if (sp->role.level != PT_PAGE_TABLE_LEVEL)
+                        continue;
+
                 pt = sp->spt;
                 for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
                         /* avoid RMW */
                         if (is_writable_pte(pt[i]))
-                                pt[i] &= ~PT_WRITABLE_MASK;
+                                update_spte(&pt[i], pt[i] & ~PT_WRITABLE_MASK);
         }
         kvm_flush_remote_tlbs(kvm);
 }
@@ -3463,13 +3534,6 @@ static void mmu_destroy_caches(void)
                 kmem_cache_destroy(mmu_page_header_cache);
 }
 
-void kvm_mmu_module_exit(void)
-{
-        mmu_destroy_caches();
-        percpu_counter_destroy(&kvm_total_used_mmu_pages);
-        unregister_shrinker(&mmu_shrinker);
-}
-
 int kvm_mmu_module_init(void)
 {
         pte_chain_cache = kmem_cache_create("kvm_pte_chain",
@@ -3566,7 +3630,7 @@ static int kvm_pv_mmu_write(struct kvm_vcpu *vcpu,
 
 static int kvm_pv_mmu_flush_tlb(struct kvm_vcpu *vcpu)
 {
-        (void)kvm_set_cr3(vcpu, vcpu->arch.cr3);
+        (void)kvm_set_cr3(vcpu, kvm_read_cr3(vcpu));
         return 1;
 }
 
@@ -3662,12 +3726,6 @@ int kvm_mmu_get_spte_hierarchy(struct kvm_vcpu *vcpu, u64 addr, u64 sptes[4])
 }
 EXPORT_SYMBOL_GPL(kvm_mmu_get_spte_hierarchy);
 
-#ifdef CONFIG_KVM_MMU_AUDIT
-#include "mmu_audit.c"
-#else
-static void mmu_audit_disable(void) { }
-#endif
-
 void kvm_mmu_destroy(struct kvm_vcpu *vcpu)
 {
         ASSERT(vcpu);
@@ -3675,5 +3733,18 @@ void kvm_mmu_destroy(struct kvm_vcpu *vcpu)
         destroy_kvm_mmu(vcpu);
         free_mmu_pages(vcpu);
         mmu_free_memory_caches(vcpu);
+}
+
+#ifdef CONFIG_KVM_MMU_AUDIT
+#include "mmu_audit.c"
+#else
+static void mmu_audit_disable(void) { }
+#endif
+
+void kvm_mmu_module_exit(void)
+{
+        mmu_destroy_caches();
+        percpu_counter_destroy(&kvm_total_used_mmu_pages);
+        unregister_shrinker(&mmu_shrinker);
         mmu_audit_disable();
 }
diff --git a/arch/x86/kvm/mmu_audit.c b/arch/x86/kvm/mmu_audit.c
index ba2bcdde622..5f6223b8bcf 100644
--- a/arch/x86/kvm/mmu_audit.c
+++ b/arch/x86/kvm/mmu_audit.c
@@ -19,11 +19,9 @@
 
 #include <linux/ratelimit.h>
 
-static int audit_point;
-
-#define audit_printk(fmt, args...)              \
+#define audit_printk(kvm, fmt, args...)         \
         printk(KERN_ERR "audit: (%s) error: "   \
-                fmt, audit_point_name[audit_point], ##args)
+                fmt, audit_point_name[kvm->arch.audit_point], ##args)
 
 typedef void (*inspect_spte_fn) (struct kvm_vcpu *vcpu, u64 *sptep, int level);
 
@@ -97,18 +95,21 @@ static void audit_mappings(struct kvm_vcpu *vcpu, u64 *sptep, int level)
 
         if (sp->unsync) {
                 if (level != PT_PAGE_TABLE_LEVEL) {
-                        audit_printk("unsync sp: %p level = %d\n", sp, level);
+                        audit_printk(vcpu->kvm, "unsync sp: %p "
+                                     "level = %d\n", sp, level);
                         return;
                 }
 
                 if (*sptep == shadow_notrap_nonpresent_pte) {
-                        audit_printk("notrap spte in unsync sp: %p\n", sp);
+                        audit_printk(vcpu->kvm, "notrap spte in unsync "
+                                     "sp: %p\n", sp);
                         return;
                 }
         }
 
         if (sp->role.direct && *sptep == shadow_notrap_nonpresent_pte) {
-                audit_printk("notrap spte in direct sp: %p\n", sp);
+                audit_printk(vcpu->kvm, "notrap spte in direct sp: %p\n",
+                             sp);
                 return;
         }
 
@@ -125,8 +126,9 @@ static void audit_mappings(struct kvm_vcpu *vcpu, u64 *sptep, int level)
 
         hpa =  pfn << PAGE_SHIFT;
         if ((*sptep & PT64_BASE_ADDR_MASK) != hpa)
-                audit_printk("levels %d pfn %llx hpa %llx ent %llxn",
-                                   vcpu->arch.mmu.root_level, pfn, hpa, *sptep);
+                audit_printk(vcpu->kvm, "levels %d pfn %llx hpa %llx "
+                             "ent %llxn", vcpu->arch.mmu.root_level, pfn,
+                             hpa, *sptep);
 }
 
 static void inspect_spte_has_rmap(struct kvm *kvm, u64 *sptep)
@@ -142,8 +144,8 @@ static void inspect_spte_has_rmap(struct kvm *kvm, u64 *sptep)
         if (!gfn_to_memslot(kvm, gfn)) {
                 if (!printk_ratelimit())
                         return;
-                audit_printk("no memslot for gfn %llx\n", gfn);
-                audit_printk("index %ld of sp (gfn=%llx)\n",
+                audit_printk(kvm, "no memslot for gfn %llx\n", gfn);
+                audit_printk(kvm, "index %ld of sp (gfn=%llx)\n",
                        (long int)(sptep - rev_sp->spt), rev_sp->gfn);
                 dump_stack();
                 return;
@@ -153,7 +155,8 @@ static void inspect_spte_has_rmap(struct kvm *kvm, u64 *sptep)
         if (!*rmapp) {
                 if (!printk_ratelimit())
                         return;
-                audit_printk("no rmap for writable spte %llx\n", *sptep);
+                audit_printk(kvm, "no rmap for writable spte %llx\n",
+                             *sptep);
                 dump_stack();
         }
 }
@@ -168,8 +171,9 @@ static void audit_spte_after_sync(struct kvm_vcpu *vcpu, u64 *sptep, int level)
 {
         struct kvm_mmu_page *sp = page_header(__pa(sptep));
 
-        if (audit_point == AUDIT_POST_SYNC && sp->unsync)
-                audit_printk("meet unsync sp(%p) after sync root.\n", sp);
+        if (vcpu->kvm->arch.audit_point == AUDIT_POST_SYNC && sp->unsync)
+                audit_printk(vcpu->kvm, "meet unsync sp(%p) after sync "
+                             "root.\n", sp);
 }
 
 static void check_mappings_rmap(struct kvm *kvm, struct kvm_mmu_page *sp)
@@ -202,8 +206,9 @@ static void audit_write_protection(struct kvm *kvm, struct kvm_mmu_page *sp)
         spte = rmap_next(kvm, rmapp, NULL);
         while (spte) {
                 if (is_writable_pte(*spte))
-                        audit_printk("shadow page has writable mappings: gfn "
-                                     "%llx role %x\n", sp->gfn, sp->role.word);
+                        audit_printk(kvm, "shadow page has writable "
+                                     "mappings: gfn %llx role %x\n",
+                                     sp->gfn, sp->role.word);
                 spte = rmap_next(kvm, rmapp, spte);
         }
 }
@@ -238,7 +243,7 @@ static void kvm_mmu_audit(void *ignore, struct kvm_vcpu *vcpu, int point)
         if (!__ratelimit(&ratelimit_state))
                 return;
 
-        audit_point = point;
+        vcpu->kvm->arch.audit_point = point;
         audit_all_active_sps(vcpu->kvm);
         audit_vcpu_spte(vcpu);
 }
diff --git a/arch/x86/kvm/paging_tmpl.h b/arch/x86/kvm/paging_tmpl.h
index cd7a833a3b5..53210f1e94c 100644
--- a/arch/x86/kvm/paging_tmpl.h
+++ b/arch/x86/kvm/paging_tmpl.h
@@ -72,7 +72,7 @@ struct guest_walker {
         unsigned pt_access;
         unsigned pte_access;
         gfn_t gfn;
-        u32 error_code;
+        struct x86_exception fault;
 };
 
 static gfn_t gpte_to_gfn_lvl(pt_element_t gpte, int lvl)
@@ -266,21 +266,23 @@ walk:
         return 1;
 
 error:
-        walker->error_code = 0;
+        walker->fault.vector = PF_VECTOR;
+        walker->fault.error_code_valid = true;
+        walker->fault.error_code = 0;
         if (present)
-                walker->error_code |= PFERR_PRESENT_MASK;
+                walker->fault.error_code |= PFERR_PRESENT_MASK;
 
-        walker->error_code |= write_fault | user_fault;
+        walker->fault.error_code |= write_fault | user_fault;
 
         if (fetch_fault && mmu->nx)
-                walker->error_code |= PFERR_FETCH_MASK;
+                walker->fault.error_code |= PFERR_FETCH_MASK;
         if (rsvd_fault)
-                walker->error_code |= PFERR_RSVD_MASK;
+                walker->fault.error_code |= PFERR_RSVD_MASK;
 
-        vcpu->arch.fault.address    = addr;
-        vcpu->arch.fault.error_code = walker->error_code;
+        walker->fault.address = addr;
+        walker->fault.nested_page_fault = mmu != vcpu->arch.walk_mmu;
 
-        trace_kvm_mmu_walker_error(walker->error_code);
+        trace_kvm_mmu_walker_error(walker->fault.error_code);
         return 0;
 }
 
@@ -299,25 +301,42 @@ static int FNAME(walk_addr_nested)(struct guest_walker *walker,
                                         addr, access);
 }
 
+static bool FNAME(prefetch_invalid_gpte)(struct kvm_vcpu *vcpu,
+                                    struct kvm_mmu_page *sp, u64 *spte,
+                                    pt_element_t gpte)
+{
+        u64 nonpresent = shadow_trap_nonpresent_pte;
+
+        if (is_rsvd_bits_set(&vcpu->arch.mmu, gpte, PT_PAGE_TABLE_LEVEL))
+                goto no_present;
+
+        if (!is_present_gpte(gpte)) {
+                if (!sp->unsync)
+                        nonpresent = shadow_notrap_nonpresent_pte;
+                goto no_present;
+        }
+
+        if (!(gpte & PT_ACCESSED_MASK))
+                goto no_present;
+
+        return false;
+
+no_present:
+        drop_spte(vcpu->kvm, spte, nonpresent);
+        return true;
+}
+
 static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
                               u64 *spte, const void *pte)
 {
         pt_element_t gpte;
         unsigned pte_access;
         pfn_t pfn;
-        u64 new_spte;
 
         gpte = *(const pt_element_t *)pte;
-        if (~gpte & (PT_PRESENT_MASK | PT_ACCESSED_MASK)) {
-                if (!is_present_gpte(gpte)) {
-                        if (sp->unsync)
-                                new_spte = shadow_trap_nonpresent_pte;
-                        else
-                                new_spte = shadow_notrap_nonpresent_pte;
-                        __set_spte(spte, new_spte);
-                }
+        if (FNAME(prefetch_invalid_gpte)(vcpu, sp, spte, gpte))
                 return;
-        }
+
         pgprintk("%s: gpte %llx spte %p\n", __func__, (u64)gpte, spte);
         pte_access = sp->role.access & FNAME(gpte_access)(vcpu, gpte);
         if (gpte_to_gfn(gpte) != vcpu->arch.update_pte.gfn)
@@ -329,7 +348,7 @@ static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
                 return;
         kvm_get_pfn(pfn);
         /*
-         * we call mmu_set_spte() with reset_host_protection = true beacuse that
+         * we call mmu_set_spte() with host_writable = true beacuse that
          * vcpu->arch.update_pte.pfn was fetched from get_user_pages(write = 1).
          */
         mmu_set_spte(vcpu, spte, sp->role.access, pte_access, 0, 0,
@@ -364,7 +383,6 @@ static void FNAME(pte_prefetch)(struct kvm_vcpu *vcpu, struct guest_walker *gw,
                                 u64 *sptep)
 {
         struct kvm_mmu_page *sp;
-        struct kvm_mmu *mmu = &vcpu->arch.mmu;
         pt_element_t *gptep = gw->prefetch_ptes;
         u64 *spte;
         int i;
@@ -395,14 +413,7 @@ static void FNAME(pte_prefetch)(struct kvm_vcpu *vcpu, struct guest_walker *gw,
 
                 gpte = gptep[i];
 
-                if (!is_present_gpte(gpte) ||
-                      is_rsvd_bits_set(mmu, gpte, PT_PAGE_TABLE_LEVEL)) {
-                        if (!sp->unsync)
-                                __set_spte(spte, shadow_notrap_nonpresent_pte);
-                        continue;
-                }
-
-                if (!(gpte & PT_ACCESSED_MASK))
+                if (FNAME(prefetch_invalid_gpte)(vcpu, sp, spte, gpte))
                         continue;
 
                 pte_access = sp->role.access & FNAME(gpte_access)(vcpu, gpte);
@@ -427,7 +438,8 @@ static void FNAME(pte_prefetch)(struct kvm_vcpu *vcpu, struct guest_walker *gw,
 static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr,
                          struct guest_walker *gw,
                          int user_fault, int write_fault, int hlevel,
-                         int *ptwrite, pfn_t pfn)
+                         int *ptwrite, pfn_t pfn, bool map_writable,
+                         bool prefault)
 {
         unsigned access = gw->pt_access;
         struct kvm_mmu_page *sp = NULL;
@@ -501,7 +513,7 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr,
 
         mmu_set_spte(vcpu, it.sptep, access, gw->pte_access & access,
                      user_fault, write_fault, dirty, ptwrite, it.level,
-                     gw->gfn, pfn, false, true);
+                     gw->gfn, pfn, prefault, map_writable);
         FNAME(pte_prefetch)(vcpu, gw, it.sptep);
 
         return it.sptep;
@@ -527,8 +539,8 @@ out_gpte_changed:
  *  Returns: 1 if we need to emulate the instruction, 0 otherwise, or
  *           a negative value on error.
  */
-static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr,
-                               u32 error_code)
+static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr, u32 error_code,
+                             bool prefault)
 {
         int write_fault = error_code & PFERR_WRITE_MASK;
         int user_fault = error_code & PFERR_USER_MASK;
@@ -539,6 +551,7 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr,
         pfn_t pfn;
         int level = PT_PAGE_TABLE_LEVEL;
         unsigned long mmu_seq;
+        bool map_writable;
 
         pgprintk("%s: addr %lx err %x\n", __func__, addr, error_code);
 
@@ -556,8 +569,11 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr,
          */
         if (!r) {
                 pgprintk("%s: guest page fault\n", __func__);
-                inject_page_fault(vcpu);
-                vcpu->arch.last_pt_write_count = 0; /* reset fork detector */
+                if (!prefault) {
+                        inject_page_fault(vcpu, &walker.fault);
+                        /* reset fork detector */
+                        vcpu->arch.last_pt_write_count = 0;
+                }
                 return 0;
         }
 
@@ -568,7 +584,10 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr,
 
         mmu_seq = vcpu->kvm->mmu_notifier_seq;
         smp_rmb();
-        pfn = gfn_to_pfn(vcpu->kvm, walker.gfn);
+
+        if (try_async_pf(vcpu, prefault, walker.gfn, addr, &pfn, write_fault,
+                         &map_writable))
+                return 0;
 
         /* mmio */
         if (is_error_pfn(pfn))
@@ -581,7 +600,7 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr,
         trace_kvm_mmu_audit(vcpu, AUDIT_PRE_PAGE_FAULT);
         kvm_mmu_free_some_pages(vcpu);
         sptep = FNAME(fetch)(vcpu, addr, &walker, user_fault, write_fault,
-                             level, &write_pt, pfn);
+                             level, &write_pt, pfn, map_writable, prefault);
         (void)sptep;
         pgprintk("%s: shadow pte %p %llx ptwrite %d\n", __func__,
                  sptep, *sptep, write_pt);
@@ -661,7 +680,7 @@ static void FNAME(invlpg)(struct kvm_vcpu *vcpu, gva_t gva)
 }
 
 static gpa_t FNAME(gva_to_gpa)(struct kvm_vcpu *vcpu, gva_t vaddr, u32 access,
-                               u32 *error)
+                               struct x86_exception *exception)
 {
         struct guest_walker walker;
         gpa_t gpa = UNMAPPED_GVA;
@@ -672,14 +691,15 @@ static gpa_t FNAME(gva_to_gpa)(struct kvm_vcpu *vcpu, gva_t vaddr, u32 access,
         if (r) {
                 gpa = gfn_to_gpa(walker.gfn);
                 gpa |= vaddr & ~PAGE_MASK;
-        } else if (error)
-                *error = walker.error_code;
+        } else if (exception)
+                *exception = walker.fault;
 
         return gpa;
 }
 
 static gpa_t FNAME(gva_to_gpa_nested)(struct kvm_vcpu *vcpu, gva_t vaddr,
-                                      u32 access, u32 *error)
+                                      u32 access,
+                                      struct x86_exception *exception)
 {
         struct guest_walker walker;
         gpa_t gpa = UNMAPPED_GVA;
@@ -690,8 +710,8 @@ static gpa_t FNAME(gva_to_gpa_nested)(struct kvm_vcpu *vcpu, gva_t vaddr,
         if (r) {
                 gpa = gfn_to_gpa(walker.gfn);
                 gpa |= vaddr & ~PAGE_MASK;
-        } else if (error)
-                *error = walker.error_code;
+        } else if (exception)
+                *exception = walker.fault;
 
         return gpa;
 }
@@ -730,12 +750,19 @@ static void FNAME(prefetch_page)(struct kvm_vcpu *vcpu,
  * Using the cached information from sp->gfns is safe because:
  * - The spte has a reference to the struct page, so the pfn for a given gfn
  *   can't change unless all sptes pointing to it are nuked first.
+ *
+ * Note:
+ *   We should flush all tlbs if spte is dropped even though guest is
+ *   responsible for it. Since if we don't, kvm_mmu_notifier_invalidate_page
+ *   and kvm_mmu_notifier_invalidate_range_start detect the mapping page isn't
+ *   used by guest then tlbs are not flushed, so guest is allowed to access the
+ *   freed pages.
+ *   And we increase kvm->tlbs_dirty to delay tlbs flush in this case.
  */
-static int FNAME(sync_page)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
-                            bool clear_unsync)
+static int FNAME(sync_page)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
 {
         int i, offset, nr_present;
-        bool reset_host_protection;
+        bool host_writable;
         gpa_t first_pte_gpa;
 
         offset = nr_present = 0;
@@ -764,31 +791,27 @@ static int FNAME(sync_page)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
                         return -EINVAL;
 
                 gfn = gpte_to_gfn(gpte);
-                if (is_rsvd_bits_set(&vcpu->arch.mmu, gpte, PT_PAGE_TABLE_LEVEL)
-                      || gfn != sp->gfns[i] || !is_present_gpte(gpte)
-                      || !(gpte & PT_ACCESSED_MASK)) {
-                        u64 nonpresent;
 
-                        if (is_present_gpte(gpte) || !clear_unsync)
-                                nonpresent = shadow_trap_nonpresent_pte;
-                        else
-                                nonpresent = shadow_notrap_nonpresent_pte;
-                        drop_spte(vcpu->kvm, &sp->spt[i], nonpresent);
+                if (FNAME(prefetch_invalid_gpte)(vcpu, sp, &sp->spt[i], gpte)) {
+                        vcpu->kvm->tlbs_dirty++;
+                        continue;
+                }
+
+                if (gfn != sp->gfns[i]) {
+                        drop_spte(vcpu->kvm, &sp->spt[i],
+                                      shadow_trap_nonpresent_pte);
+                        vcpu->kvm->tlbs_dirty++;
                         continue;
                 }
 
                 nr_present++;
                 pte_access = sp->role.access & FNAME(gpte_access)(vcpu, gpte);
-                if (!(sp->spt[i] & SPTE_HOST_WRITEABLE)) {
-                        pte_access &= ~ACC_WRITE_MASK;
-                        reset_host_protection = 0;
-                } else {
-                        reset_host_protection = 1;
-                }
+                host_writable = sp->spt[i] & SPTE_HOST_WRITEABLE;
+
                 set_spte(vcpu, &sp->spt[i], pte_access, 0, 0,
                          is_dirty_gpte(gpte), PT_PAGE_TABLE_LEVEL, gfn,
                          spte_to_pfn(sp->spt[i]), true, false,
-                         reset_host_protection);
+                         host_writable);
         }
 
         return !nr_present;
diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c
index b81a9b7c2ca..25bd1bc5aad 100644
--- a/arch/x86/kvm/svm.c
+++ b/arch/x86/kvm/svm.c
@@ -31,6 +31,7 @@
 
 #include <asm/tlbflush.h>
 #include <asm/desc.h>
+#include <asm/kvm_para.h>
 
 #include <asm/virtext.h>
 #include "trace.h"
@@ -50,6 +51,10 @@ MODULE_LICENSE("GPL");
 #define SVM_FEATURE_LBRV           (1 <<  1)
 #define SVM_FEATURE_SVML           (1 <<  2)
 #define SVM_FEATURE_NRIP           (1 <<  3)
+#define SVM_FEATURE_TSC_RATE       (1 <<  4)
+#define SVM_FEATURE_VMCB_CLEAN     (1 <<  5)
+#define SVM_FEATURE_FLUSH_ASID     (1 <<  6)
+#define SVM_FEATURE_DECODE_ASSIST  (1 <<  7)
 #define SVM_FEATURE_PAUSE_FILTER   (1 << 10)
 
 #define NESTED_EXIT_HOST        0       /* Exit handled on host level */
@@ -97,10 +102,8 @@ struct nested_state {
         unsigned long vmexit_rax;
 
         /* cache for intercepts of the guest */
-        u16 intercept_cr_read;
-        u16 intercept_cr_write;
-        u16 intercept_dr_read;
-        u16 intercept_dr_write;
+        u32 intercept_cr;
+        u32 intercept_dr;
         u32 intercept_exceptions;
         u64 intercept;
 
@@ -123,7 +126,12 @@ struct vcpu_svm {
         u64 next_rip;
 
         u64 host_user_msrs[NR_HOST_SAVE_USER_MSRS];
-        u64 host_gs_base;
+        struct {
+                u16 fs;
+                u16 gs;
+                u16 ldt;
+                u64 gs_base;
+        } host;
 
         u32 *msrpm;
 
@@ -133,6 +141,7 @@ struct vcpu_svm {
 
         unsigned int3_injected;
         unsigned long int3_rip;
+        u32 apf_reason;
 };
 
 #define MSR_INVALID                     0xffffffffU
@@ -180,14 +189,151 @@ static int nested_svm_vmexit(struct vcpu_svm *svm);
 static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr,
                                       bool has_error_code, u32 error_code);
 
+enum {
+        VMCB_INTERCEPTS, /* Intercept vectors, TSC offset,
+                            pause filter count */
+        VMCB_PERM_MAP,   /* IOPM Base and MSRPM Base */
+        VMCB_ASID,       /* ASID */
+        VMCB_INTR,       /* int_ctl, int_vector */
+        VMCB_NPT,        /* npt_en, nCR3, gPAT */
+        VMCB_CR,         /* CR0, CR3, CR4, EFER */
+        VMCB_DR,         /* DR6, DR7 */
+        VMCB_DT,         /* GDT, IDT */
+        VMCB_SEG,        /* CS, DS, SS, ES, CPL */
+        VMCB_CR2,        /* CR2 only */
+        VMCB_LBR,        /* DBGCTL, BR_FROM, BR_TO, LAST_EX_FROM, LAST_EX_TO */
+        VMCB_DIRTY_MAX,
+};
+
+/* TPR and CR2 are always written before VMRUN */
+#define VMCB_ALWAYS_DIRTY_MASK  ((1U << VMCB_INTR) | (1U << VMCB_CR2))
+
+static inline void mark_all_dirty(struct vmcb *vmcb)
+{
+        vmcb->control.clean = 0;
+}
+
+static inline void mark_all_clean(struct vmcb *vmcb)
+{
+        vmcb->control.clean = ((1 << VMCB_DIRTY_MAX) - 1)
+                               & ~VMCB_ALWAYS_DIRTY_MASK;
+}
+
+static inline void mark_dirty(struct vmcb *vmcb, int bit)
+{
+        vmcb->control.clean &= ~(1 << bit);
+}
+
 static inline struct vcpu_svm *to_svm(struct kvm_vcpu *vcpu)
 {
         return container_of(vcpu, struct vcpu_svm, vcpu);
 }
 
-static inline bool is_nested(struct vcpu_svm *svm)
+static void recalc_intercepts(struct vcpu_svm *svm)
+{
+        struct vmcb_control_area *c, *h;
+        struct nested_state *g;
+
+        mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
+
+        if (!is_guest_mode(&svm->vcpu))
+                return;
+
+        c = &svm->vmcb->control;
+        h = &svm->nested.hsave->control;
+        g = &svm->nested;
+
+        c->intercept_cr = h->intercept_cr | g->intercept_cr;
+        c->intercept_dr = h->intercept_dr | g->intercept_dr;
+        c->intercept_exceptions = h->intercept_exceptions | g->intercept_exceptions;
+        c->intercept = h->intercept | g->intercept;
+}
+
+static inline struct vmcb *get_host_vmcb(struct vcpu_svm *svm)
+{
+        if (is_guest_mode(&svm->vcpu))
+                return svm->nested.hsave;
+        else
+                return svm->vmcb;
+}
+
+static inline void set_cr_intercept(struct vcpu_svm *svm, int bit)
+{
+        struct vmcb *vmcb = get_host_vmcb(svm);
+
+        vmcb->control.intercept_cr |= (1U << bit);
+
+        recalc_intercepts(svm);
+}
+
+static inline void clr_cr_intercept(struct vcpu_svm *svm, int bit)
+{
+        struct vmcb *vmcb = get_host_vmcb(svm);
+
+        vmcb->control.intercept_cr &= ~(1U << bit);
+
+        recalc_intercepts(svm);
+}
+
+static inline bool is_cr_intercept(struct vcpu_svm *svm, int bit)
+{
+        struct vmcb *vmcb = get_host_vmcb(svm);
+
+        return vmcb->control.intercept_cr & (1U << bit);
+}
+
+static inline void set_dr_intercept(struct vcpu_svm *svm, int bit)
+{
+        struct vmcb *vmcb = get_host_vmcb(svm);
+
+        vmcb->control.intercept_dr |= (1U << bit);
+
+        recalc_intercepts(svm);
+}
+
+static inline void clr_dr_intercept(struct vcpu_svm *svm, int bit)
+{
+        struct vmcb *vmcb = get_host_vmcb(svm);
+
+        vmcb->control.intercept_dr &= ~(1U << bit);
+
+        recalc_intercepts(svm);
+}
+
+static inline void set_exception_intercept(struct vcpu_svm *svm, int bit)
+{
+        struct vmcb *vmcb = get_host_vmcb(svm);
+
+        vmcb->control.intercept_exceptions |= (1U << bit);
+
+        recalc_intercepts(svm);
+}
+
+static inline void clr_exception_intercept(struct vcpu_svm *svm, int bit)
 {
-        return svm->nested.vmcb;
+        struct vmcb *vmcb = get_host_vmcb(svm);
+
+        vmcb->control.intercept_exceptions &= ~(1U << bit);
+
+        recalc_intercepts(svm);
+}
+
+static inline void set_intercept(struct vcpu_svm *svm, int bit)
+{
+        struct vmcb *vmcb = get_host_vmcb(svm);
+
+        vmcb->control.intercept |= (1ULL << bit);
+
+        recalc_intercepts(svm);
+}
+
+static inline void clr_intercept(struct vcpu_svm *svm, int bit)
+{
+        struct vmcb *vmcb = get_host_vmcb(svm);
+
+        vmcb->control.intercept &= ~(1ULL << bit);
+
+        recalc_intercepts(svm);
 }
 
 static inline void enable_gif(struct vcpu_svm *svm)
@@ -264,11 +410,6 @@ static u32 svm_msrpm_offset(u32 msr)
 
 #define MAX_INST_SIZE 15
 
-static inline u32 svm_has(u32 feat)
-{
-        return svm_features & feat;
-}
-
 static inline void clgi(void)
 {
         asm volatile (__ex(SVM_CLGI));
@@ -284,16 +425,6 @@ static inline void invlpga(unsigned long addr, u32 asid)
         asm volatile (__ex(SVM_INVLPGA) : : "a"(addr), "c"(asid));
 }
 
-static inline void force_new_asid(struct kvm_vcpu *vcpu)
-{
-        to_svm(vcpu)->asid_generation--;
-}
-
-static inline void flush_guest_tlb(struct kvm_vcpu *vcpu)
-{
-        force_new_asid(vcpu);
-}
-
 static int get_npt_level(void)
 {
 #ifdef CONFIG_X86_64
@@ -310,6 +441,7 @@ static void svm_set_efer(struct kvm_vcpu *vcpu, u64 efer)
                 efer &= ~EFER_LME;
 
         to_svm(vcpu)->vmcb->save.efer = efer | EFER_SVME;
+        mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR);
 }
 
 static int is_external_interrupt(u32 info)
@@ -347,7 +479,7 @@ static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
                 svm->next_rip = svm->vmcb->control.next_rip;
 
         if (!svm->next_rip) {
-                if (emulate_instruction(vcpu, 0, 0, EMULTYPE_SKIP) !=
+                if (emulate_instruction(vcpu, EMULTYPE_SKIP) !=
                                 EMULATE_DONE)
                         printk(KERN_DEBUG "%s: NOP\n", __func__);
                 return;
@@ -374,7 +506,7 @@ static void svm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr,
             nested_svm_check_exception(svm, nr, has_error_code, error_code))
                 return;
 
-        if (nr == BP_VECTOR && !svm_has(SVM_FEATURE_NRIP)) {
+        if (nr == BP_VECTOR && !static_cpu_has(X86_FEATURE_NRIPS)) {
                 unsigned long rip, old_rip = kvm_rip_read(&svm->vcpu);
 
                 /*
@@ -670,7 +802,7 @@ static __init int svm_hardware_setup(void)
 
         svm_features = cpuid_edx(SVM_CPUID_FUNC);
 
-        if (!svm_has(SVM_FEATURE_NPT))
+        if (!boot_cpu_has(X86_FEATURE_NPT))
                 npt_enabled = false;
 
         if (npt_enabled && !npt) {
@@ -725,13 +857,15 @@ static void svm_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)
         struct vcpu_svm *svm = to_svm(vcpu);
         u64 g_tsc_offset = 0;
 
-        if (is_nested(svm)) {
+        if (is_guest_mode(vcpu)) {
                 g_tsc_offset = svm->vmcb->control.tsc_offset -
                                svm->nested.hsave->control.tsc_offset;
                 svm->nested.hsave->control.tsc_offset = offset;
         }
 
         svm->vmcb->control.tsc_offset = offset + g_tsc_offset;
+
+        mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
 }
 
 static void svm_adjust_tsc_offset(struct kvm_vcpu *vcpu, s64 adjustment)
@@ -739,8 +873,9 @@ static void svm_adjust_tsc_offset(struct kvm_vcpu *vcpu, s64 adjustment)
         struct vcpu_svm *svm = to_svm(vcpu);
 
         svm->vmcb->control.tsc_offset += adjustment;
-        if (is_nested(svm))
+        if (is_guest_mode(vcpu))
                 svm->nested.hsave->control.tsc_offset += adjustment;
+        mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
 }
 
 static void init_vmcb(struct vcpu_svm *svm)
@@ -749,62 +884,62 @@ static void init_vmcb(struct vcpu_svm *svm)
         struct vmcb_save_area *save = &svm->vmcb->save;
 
         svm->vcpu.fpu_active = 1;
+        svm->vcpu.arch.hflags = 0;
 
-        control->intercept_cr_read =    INTERCEPT_CR0_MASK |
-                                        INTERCEPT_CR3_MASK |
-                                        INTERCEPT_CR4_MASK;
-
-        control->intercept_cr_write =   INTERCEPT_CR0_MASK |
-                                        INTERCEPT_CR3_MASK |
-                                        INTERCEPT_CR4_MASK |
-                                        INTERCEPT_CR8_MASK;
-
-        control->intercept_dr_read =    INTERCEPT_DR0_MASK |
-                                        INTERCEPT_DR1_MASK |
-                                        INTERCEPT_DR2_MASK |
-                                        INTERCEPT_DR3_MASK |
-                                        INTERCEPT_DR4_MASK |
-                                        INTERCEPT_DR5_MASK |
-                                        INTERCEPT_DR6_MASK |
-                                        INTERCEPT_DR7_MASK;
-
-        control->intercept_dr_write =   INTERCEPT_DR0_MASK |
-                                        INTERCEPT_DR1_MASK |
-                                        INTERCEPT_DR2_MASK |
-                                        INTERCEPT_DR3_MASK |
-                                        INTERCEPT_DR4_MASK |
-                                        INTERCEPT_DR5_MASK |
-                                        INTERCEPT_DR6_MASK |
-                                        INTERCEPT_DR7_MASK;
-
-        control->intercept_exceptions = (1 << PF_VECTOR) |
-                                        (1 << UD_VECTOR) |
-                                        (1 << MC_VECTOR);
-
-
-        control->intercept =    (1ULL << INTERCEPT_INTR) |
-                                (1ULL << INTERCEPT_NMI) |
-                                (1ULL << INTERCEPT_SMI) |
-                                (1ULL << INTERCEPT_SELECTIVE_CR0) |
-                                (1ULL << INTERCEPT_CPUID) |
-                                (1ULL << INTERCEPT_INVD) |
-                                (1ULL << INTERCEPT_HLT) |
-                                (1ULL << INTERCEPT_INVLPG) |
-                                (1ULL << INTERCEPT_INVLPGA) |
-                                (1ULL << INTERCEPT_IOIO_PROT) |
-                                (1ULL << INTERCEPT_MSR_PROT) |
-                                (1ULL << INTERCEPT_TASK_SWITCH) |
-                                (1ULL << INTERCEPT_SHUTDOWN) |
-                                (1ULL << INTERCEPT_VMRUN) |
-                                (1ULL << INTERCEPT_VMMCALL) |
-                                (1ULL << INTERCEPT_VMLOAD) |
-                                (1ULL << INTERCEPT_VMSAVE) |
-                                (1ULL << INTERCEPT_STGI) |
-                                (1ULL << INTERCEPT_CLGI) |
-                                (1ULL << INTERCEPT_SKINIT) |
-                                (1ULL << INTERCEPT_WBINVD) |
-                                (1ULL << INTERCEPT_MONITOR) |
-                                (1ULL << INTERCEPT_MWAIT);
+        set_cr_intercept(svm, INTERCEPT_CR0_READ);
+        set_cr_intercept(svm, INTERCEPT_CR3_READ);
+        set_cr_intercept(svm, INTERCEPT_CR4_READ);
+        set_cr_intercept(svm, INTERCEPT_CR0_WRITE);
+        set_cr_intercept(svm, INTERCEPT_CR3_WRITE);
+        set_cr_intercept(svm, INTERCEPT_CR4_WRITE);
+        set_cr_intercept(svm, INTERCEPT_CR8_WRITE);
+
+        set_dr_intercept(svm, INTERCEPT_DR0_READ);
+        set_dr_intercept(svm, INTERCEPT_DR1_READ);
+        set_dr_intercept(svm, INTERCEPT_DR2_READ);
+        set_dr_intercept(svm, INTERCEPT_DR3_READ);
+        set_dr_intercept(svm, INTERCEPT_DR4_READ);
+        set_dr_intercept(svm, INTERCEPT_DR5_READ);
+        set_dr_intercept(svm, INTERCEPT_DR6_READ);
+        set_dr_intercept(svm, INTERCEPT_DR7_READ);
+
+        set_dr_intercept(svm, INTERCEPT_DR0_WRITE);
+        set_dr_intercept(svm, INTERCEPT_DR1_WRITE);
+        set_dr_intercept(svm, INTERCEPT_DR2_WRITE);
+        set_dr_intercept(svm, INTERCEPT_DR3_WRITE);
+        set_dr_intercept(svm, INTERCEPT_DR4_WRITE);
+        set_dr_intercept(svm, INTERCEPT_DR5_WRITE);
+        set_dr_intercept(svm, INTERCEPT_DR6_WRITE);
+        set_dr_intercept(svm, INTERCEPT_DR7_WRITE);
+
+        set_exception_intercept(svm, PF_VECTOR);
+        set_exception_intercept(svm, UD_VECTOR);
+        set_exception_intercept(svm, MC_VECTOR);
+
+        set_intercept(svm, INTERCEPT_INTR);
+        set_intercept(svm, INTERCEPT_NMI);
+        set_intercept(svm, INTERCEPT_SMI);
+        set_intercept(svm, INTERCEPT_SELECTIVE_CR0);
+        set_intercept(svm, INTERCEPT_CPUID);
+        set_intercept(svm, INTERCEPT_INVD);
+        set_intercept(svm, INTERCEPT_HLT);
+        set_intercept(svm, INTERCEPT_INVLPG);
+        set_intercept(svm, INTERCEPT_INVLPGA);
+        set_intercept(svm, INTERCEPT_IOIO_PROT);
+        set_intercept(svm, INTERCEPT_MSR_PROT);
+        set_intercept(svm, INTERCEPT_TASK_SWITCH);
+        set_intercept(svm, INTERCEPT_SHUTDOWN);
+        set_intercept(svm, INTERCEPT_VMRUN);
+        set_intercept(svm, INTERCEPT_VMMCALL);
+        set_intercept(svm, INTERCEPT_VMLOAD);
+        set_intercept(svm, INTERCEPT_VMSAVE);
+        set_intercept(svm, INTERCEPT_STGI);
+        set_intercept(svm, INTERCEPT_CLGI);
+        set_intercept(svm, INTERCEPT_SKINIT);
+        set_intercept(svm, INTERCEPT_WBINVD);
+        set_intercept(svm, INTERCEPT_MONITOR);
+        set_intercept(svm, INTERCEPT_MWAIT);
+        set_intercept(svm, INTERCEPT_XSETBV);
 
         control->iopm_base_pa = iopm_base;
         control->msrpm_base_pa = __pa(svm->msrpm);
@@ -855,25 +990,27 @@ static void init_vmcb(struct vcpu_svm *svm)
         if (npt_enabled) {
                 /* Setup VMCB for Nested Paging */
                 control->nested_ctl = 1;
-                control->intercept &= ~((1ULL << INTERCEPT_TASK_SWITCH) |
-                                        (1ULL << INTERCEPT_INVLPG));
-                control->intercept_exceptions &= ~(1 << PF_VECTOR);
-                control->intercept_cr_read &= ~INTERCEPT_CR3_MASK;
-                control->intercept_cr_write &= ~INTERCEPT_CR3_MASK;
+                clr_intercept(svm, INTERCEPT_TASK_SWITCH);
+                clr_intercept(svm, INTERCEPT_INVLPG);
+                clr_exception_intercept(svm, PF_VECTOR);
+                clr_cr_intercept(svm, INTERCEPT_CR3_READ);
+                clr_cr_intercept(svm, INTERCEPT_CR3_WRITE);
                 save->g_pat = 0x0007040600070406ULL;
                 save->cr3 = 0;
                 save->cr4 = 0;
         }
-        force_new_asid(&svm->vcpu);
+        svm->asid_generation = 0;
 
         svm->nested.vmcb = 0;
         svm->vcpu.arch.hflags = 0;
 
-        if (svm_has(SVM_FEATURE_PAUSE_FILTER)) {
+        if (boot_cpu_has(X86_FEATURE_PAUSEFILTER)) {
                 control->pause_filter_count = 3000;
-                control->intercept |= (1ULL << INTERCEPT_PAUSE);
+                set_intercept(svm, INTERCEPT_PAUSE);
         }
 
+        mark_all_dirty(svm->vmcb);
+
         enable_gif(svm);
 }
 
@@ -990,8 +1127,16 @@ static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 
         if (unlikely(cpu != vcpu->cpu)) {
                 svm->asid_generation = 0;
+                mark_all_dirty(svm->vmcb);
         }
 
+#ifdef CONFIG_X86_64
+        rdmsrl(MSR_GS_BASE, to_svm(vcpu)->host.gs_base);
+#endif
+        savesegment(fs, svm->host.fs);
+        savesegment(gs, svm->host.gs);
+        svm->host.ldt = kvm_read_ldt();
+
         for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
                 rdmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
 }
@@ -1002,6 +1147,14 @@ static void svm_vcpu_put(struct kvm_vcpu *vcpu)
         int i;
 
         ++vcpu->stat.host_state_reload;
+        kvm_load_ldt(svm->host.ldt);
+#ifdef CONFIG_X86_64
+        loadsegment(fs, svm->host.fs);
+        load_gs_index(svm->host.gs);
+        wrmsrl(MSR_KERNEL_GS_BASE, current->thread.gs);
+#else
+        loadsegment(gs, svm->host.gs);
+#endif
         for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
                 wrmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
 }
@@ -1021,7 +1174,7 @@ static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
         switch (reg) {
         case VCPU_EXREG_PDPTR:
                 BUG_ON(!npt_enabled);
-                load_pdptrs(vcpu, vcpu->arch.walk_mmu, vcpu->arch.cr3);
+                load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu));
                 break;
         default:
                 BUG();
@@ -1030,12 +1183,12 @@ static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
 
 static void svm_set_vintr(struct vcpu_svm *svm)
 {
-        svm->vmcb->control.intercept |= 1ULL << INTERCEPT_VINTR;
+        set_intercept(svm, INTERCEPT_VINTR);
 }
 
 static void svm_clear_vintr(struct vcpu_svm *svm)
 {
-        svm->vmcb->control.intercept &= ~(1ULL << INTERCEPT_VINTR);
+        clr_intercept(svm, INTERCEPT_VINTR);
 }
 
 static struct vmcb_seg *svm_seg(struct kvm_vcpu *vcpu, int seg)
@@ -1150,6 +1303,7 @@ static void svm_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
 
         svm->vmcb->save.idtr.limit = dt->size;
         svm->vmcb->save.idtr.base = dt->address ;
+        mark_dirty(svm->vmcb, VMCB_DT);
 }
 
 static void svm_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
@@ -1166,19 +1320,23 @@ static void svm_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
 
         svm->vmcb->save.gdtr.limit = dt->size;
         svm->vmcb->save.gdtr.base = dt->address ;
+        mark_dirty(svm->vmcb, VMCB_DT);
 }
 
 static void svm_decache_cr0_guest_bits(struct kvm_vcpu *vcpu)
 {
 }
 
+static void svm_decache_cr3(struct kvm_vcpu *vcpu)
+{
+}
+
 static void svm_decache_cr4_guest_bits(struct kvm_vcpu *vcpu)
 {
 }
 
 static void update_cr0_intercept(struct vcpu_svm *svm)
 {
-        struct vmcb *vmcb = svm->vmcb;
         ulong gcr0 = svm->vcpu.arch.cr0;
         u64 *hcr0 = &svm->vmcb->save.cr0;
 
@@ -1188,27 +1346,14 @@ static void update_cr0_intercept(struct vcpu_svm *svm)
                 *hcr0 = (*hcr0 & ~SVM_CR0_SELECTIVE_MASK)
                         | (gcr0 & SVM_CR0_SELECTIVE_MASK);
 
+        mark_dirty(svm->vmcb, VMCB_CR);
 
         if (gcr0 == *hcr0 && svm->vcpu.fpu_active) {
-                vmcb->control.intercept_cr_read &= ~INTERCEPT_CR0_MASK;
-                vmcb->control.intercept_cr_write &= ~INTERCEPT_CR0_MASK;
-                if (is_nested(svm)) {
-                        struct vmcb *hsave = svm->nested.hsave;
-
-                        hsave->control.intercept_cr_read  &= ~INTERCEPT_CR0_MASK;
-                        hsave->control.intercept_cr_write &= ~INTERCEPT_CR0_MASK;
-                        vmcb->control.intercept_cr_read  |= svm->nested.intercept_cr_read;
-                        vmcb->control.intercept_cr_write |= svm->nested.intercept_cr_write;
-                }
+                clr_cr_intercept(svm, INTERCEPT_CR0_READ);
+                clr_cr_intercept(svm, INTERCEPT_CR0_WRITE);
         } else {
-                svm->vmcb->control.intercept_cr_read |= INTERCEPT_CR0_MASK;
-                svm->vmcb->control.intercept_cr_write |= INTERCEPT_CR0_MASK;
-                if (is_nested(svm)) {
-                        struct vmcb *hsave = svm->nested.hsave;
-
-                        hsave->control.intercept_cr_read |= INTERCEPT_CR0_MASK;
-                        hsave->control.intercept_cr_write |= INTERCEPT_CR0_MASK;
-                }
+                set_cr_intercept(svm, INTERCEPT_CR0_READ);
+                set_cr_intercept(svm, INTERCEPT_CR0_WRITE);
         }
 }
 
@@ -1216,7 +1361,7 @@ static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
 {
         struct vcpu_svm *svm = to_svm(vcpu);
 
-        if (is_nested(svm)) {
+        if (is_guest_mode(vcpu)) {
                 /*
                  * We are here because we run in nested mode, the host kvm
                  * intercepts cr0 writes but the l1 hypervisor does not.
@@ -1268,6 +1413,7 @@ static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
          */
         cr0 &= ~(X86_CR0_CD | X86_CR0_NW);
         svm->vmcb->save.cr0 = cr0;
+        mark_dirty(svm->vmcb, VMCB_CR);
         update_cr0_intercept(svm);
 }
 
@@ -1277,13 +1423,14 @@ static void svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
         unsigned long old_cr4 = to_svm(vcpu)->vmcb->save.cr4;
 
         if (npt_enabled && ((old_cr4 ^ cr4) & X86_CR4_PGE))
-                force_new_asid(vcpu);
+                svm_flush_tlb(vcpu);
 
         vcpu->arch.cr4 = cr4;
         if (!npt_enabled)
                 cr4 |= X86_CR4_PAE;
         cr4 |= host_cr4_mce;
         to_svm(vcpu)->vmcb->save.cr4 = cr4;
+        mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR);
 }
 
 static void svm_set_segment(struct kvm_vcpu *vcpu,
@@ -1312,26 +1459,25 @@ static void svm_set_segment(struct kvm_vcpu *vcpu,
                         = (svm->vmcb->save.cs.attrib
                            >> SVM_SELECTOR_DPL_SHIFT) & 3;
 
+        mark_dirty(svm->vmcb, VMCB_SEG);
 }
 
 static void update_db_intercept(struct kvm_vcpu *vcpu)
 {
         struct vcpu_svm *svm = to_svm(vcpu);
 
-        svm->vmcb->control.intercept_exceptions &=
-                ~((1 << DB_VECTOR) | (1 << BP_VECTOR));
+        clr_exception_intercept(svm, DB_VECTOR);
+        clr_exception_intercept(svm, BP_VECTOR);
 
         if (svm->nmi_singlestep)
-                svm->vmcb->control.intercept_exceptions |= (1 << DB_VECTOR);
+                set_exception_intercept(svm, DB_VECTOR);
 
         if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) {
                 if (vcpu->guest_debug &
                     (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))
-                        svm->vmcb->control.intercept_exceptions |=
-                                1 << DB_VECTOR;
+                        set_exception_intercept(svm, DB_VECTOR);
                 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
-                        svm->vmcb->control.intercept_exceptions |=
-                                1 << BP_VECTOR;
+                        set_exception_intercept(svm, BP_VECTOR);
         } else
                 vcpu->guest_debug = 0;
 }
@@ -1345,21 +1491,9 @@ static void svm_guest_debug(struct kvm_vcpu *vcpu, struct kvm_guest_debug *dbg)
         else
                 svm->vmcb->save.dr7 = vcpu->arch.dr7;
 
-        update_db_intercept(vcpu);
-}
-
-static void load_host_msrs(struct kvm_vcpu *vcpu)
-{
-#ifdef CONFIG_X86_64
-        wrmsrl(MSR_GS_BASE, to_svm(vcpu)->host_gs_base);
-#endif
-}
+        mark_dirty(svm->vmcb, VMCB_DR);
 
-static void save_host_msrs(struct kvm_vcpu *vcpu)
-{
-#ifdef CONFIG_X86_64
-        rdmsrl(MSR_GS_BASE, to_svm(vcpu)->host_gs_base);
-#endif
+        update_db_intercept(vcpu);
 }
 
 static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd)
@@ -1372,6 +1506,8 @@ static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd)
 
         svm->asid_generation = sd->asid_generation;
         svm->vmcb->control.asid = sd->next_asid++;
+
+        mark_dirty(svm->vmcb, VMCB_ASID);
 }
 
 static void svm_set_dr7(struct kvm_vcpu *vcpu, unsigned long value)
@@ -1379,20 +1515,40 @@ static void svm_set_dr7(struct kvm_vcpu *vcpu, unsigned long value)
         struct vcpu_svm *svm = to_svm(vcpu);
 
         svm->vmcb->save.dr7 = value;
+        mark_dirty(svm->vmcb, VMCB_DR);
 }
 
 static int pf_interception(struct vcpu_svm *svm)
 {
-        u64 fault_address;
+        u64 fault_address = svm->vmcb->control.exit_info_2;
         u32 error_code;
+        int r = 1;
 
-        fault_address  = svm->vmcb->control.exit_info_2;
-        error_code = svm->vmcb->control.exit_info_1;
-
-        trace_kvm_page_fault(fault_address, error_code);
-        if (!npt_enabled && kvm_event_needs_reinjection(&svm->vcpu))
-                kvm_mmu_unprotect_page_virt(&svm->vcpu, fault_address);
-        return kvm_mmu_page_fault(&svm->vcpu, fault_address, error_code);
+        switch (svm->apf_reason) {
+        default:
+                error_code = svm->vmcb->control.exit_info_1;
+
+                trace_kvm_page_fault(fault_address, error_code);
+                if (!npt_enabled && kvm_event_needs_reinjection(&svm->vcpu))
+                        kvm_mmu_unprotect_page_virt(&svm->vcpu, fault_address);
+                r = kvm_mmu_page_fault(&svm->vcpu, fault_address, error_code,
+                        svm->vmcb->control.insn_bytes,
+                        svm->vmcb->control.insn_len);
+                break;
+        case KVM_PV_REASON_PAGE_NOT_PRESENT:
+                svm->apf_reason = 0;
+                local_irq_disable();
+                kvm_async_pf_task_wait(fault_address);
+                local_irq_enable();
+                break;
+        case KVM_PV_REASON_PAGE_READY:
+                svm->apf_reason = 0;
+                local_irq_disable();
+                kvm_async_pf_task_wake(fault_address);
+                local_irq_enable();
+                break;
+        }
+        return r;
 }
 
 static int db_interception(struct vcpu_svm *svm)
@@ -1440,7 +1596,7 @@ static int ud_interception(struct vcpu_svm *svm)
 {
         int er;
 
-        er = emulate_instruction(&svm->vcpu, 0, 0, EMULTYPE_TRAP_UD);
+        er = emulate_instruction(&svm->vcpu, EMULTYPE_TRAP_UD);
         if (er != EMULATE_DONE)
                 kvm_queue_exception(&svm->vcpu, UD_VECTOR);
         return 1;
@@ -1449,21 +1605,8 @@ static int ud_interception(struct vcpu_svm *svm)
 static void svm_fpu_activate(struct kvm_vcpu *vcpu)
 {
         struct vcpu_svm *svm = to_svm(vcpu);
-        u32 excp;
-
-        if (is_nested(svm)) {
-                u32 h_excp, n_excp;
-
-                h_excp  = svm->nested.hsave->control.intercept_exceptions;
-                n_excp  = svm->nested.intercept_exceptions;
-                h_excp &= ~(1 << NM_VECTOR);
-                excp    = h_excp | n_excp;
-        } else {
-                excp  = svm->vmcb->control.intercept_exceptions;
-                excp &= ~(1 << NM_VECTOR);
-        }
 
-        svm->vmcb->control.intercept_exceptions = excp;
+        clr_exception_intercept(svm, NM_VECTOR);
 
         svm->vcpu.fpu_active = 1;
         update_cr0_intercept(svm);
@@ -1570,7 +1713,7 @@ static int io_interception(struct vcpu_svm *svm)
         string = (io_info & SVM_IOIO_STR_MASK) != 0;
         in = (io_info & SVM_IOIO_TYPE_MASK) != 0;
         if (string || in)
-                return emulate_instruction(vcpu, 0, 0, 0) == EMULATE_DONE;
+                return emulate_instruction(vcpu, 0) == EMULATE_DONE;
 
         port = io_info >> 16;
         size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT;
@@ -1624,17 +1767,19 @@ static void nested_svm_set_tdp_cr3(struct kvm_vcpu *vcpu,
         struct vcpu_svm *svm = to_svm(vcpu);
 
         svm->vmcb->control.nested_cr3 = root;
-        force_new_asid(vcpu);
+        mark_dirty(svm->vmcb, VMCB_NPT);
+        svm_flush_tlb(vcpu);
 }
 
-static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu)
+static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu,
+                                       struct x86_exception *fault)
 {
         struct vcpu_svm *svm = to_svm(vcpu);
 
         svm->vmcb->control.exit_code = SVM_EXIT_NPF;
         svm->vmcb->control.exit_code_hi = 0;
-        svm->vmcb->control.exit_info_1 = vcpu->arch.fault.error_code;
-        svm->vmcb->control.exit_info_2 = vcpu->arch.fault.address;
+        svm->vmcb->control.exit_info_1 = fault->error_code;
+        svm->vmcb->control.exit_info_2 = fault->address;
 
         nested_svm_vmexit(svm);
 }
@@ -1680,7 +1825,7 @@ static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr,
 {
         int vmexit;
 
-        if (!is_nested(svm))
+        if (!is_guest_mode(&svm->vcpu))
                 return 0;
 
         svm->vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + nr;
@@ -1698,7 +1843,7 @@ static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr,
 /* This function returns true if it is save to enable the irq window */
 static inline bool nested_svm_intr(struct vcpu_svm *svm)
 {
-        if (!is_nested(svm))
+        if (!is_guest_mode(&svm->vcpu))
                 return true;
 
         if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK))
@@ -1737,7 +1882,7 @@ static inline bool nested_svm_intr(struct vcpu_svm *svm)
 /* This function returns true if it is save to enable the nmi window */
 static inline bool nested_svm_nmi(struct vcpu_svm *svm)
 {
-        if (!is_nested(svm))
+        if (!is_guest_mode(&svm->vcpu))
                 return true;
 
         if (!(svm->nested.intercept & (1ULL << INTERCEPT_NMI)))
@@ -1836,8 +1981,8 @@ static int nested_svm_exit_special(struct vcpu_svm *svm)
                         return NESTED_EXIT_HOST;
                 break;
         case SVM_EXIT_EXCP_BASE + PF_VECTOR:
-                /* When we're shadowing, trap PFs */
-                if (!npt_enabled)
+                /* When we're shadowing, trap PFs, but not async PF */
+                if (!npt_enabled && svm->apf_reason == 0)
                         return NESTED_EXIT_HOST;
                 break;
         case SVM_EXIT_EXCP_BASE + NM_VECTOR:
@@ -1865,27 +2010,15 @@ static int nested_svm_intercept(struct vcpu_svm *svm)
         case SVM_EXIT_IOIO:
                 vmexit = nested_svm_intercept_ioio(svm);
                 break;
-        case SVM_EXIT_READ_CR0 ... SVM_EXIT_READ_CR8: {
-                u32 cr_bits = 1 << (exit_code - SVM_EXIT_READ_CR0);
-                if (svm->nested.intercept_cr_read & cr_bits)
+        case SVM_EXIT_READ_CR0 ... SVM_EXIT_WRITE_CR8: {
+                u32 bit = 1U << (exit_code - SVM_EXIT_READ_CR0);
+                if (svm->nested.intercept_cr & bit)
                         vmexit = NESTED_EXIT_DONE;
                 break;
         }
-        case SVM_EXIT_WRITE_CR0 ... SVM_EXIT_WRITE_CR8: {
-                u32 cr_bits = 1 << (exit_code - SVM_EXIT_WRITE_CR0);
-                if (svm->nested.intercept_cr_write & cr_bits)
-                        vmexit = NESTED_EXIT_DONE;
-                break;
-        }
-        case SVM_EXIT_READ_DR0 ... SVM_EXIT_READ_DR7: {
-                u32 dr_bits = 1 << (exit_code - SVM_EXIT_READ_DR0);
-                if (svm->nested.intercept_dr_read & dr_bits)
-                        vmexit = NESTED_EXIT_DONE;
-                break;
-        }
-        case SVM_EXIT_WRITE_DR0 ... SVM_EXIT_WRITE_DR7: {
-                u32 dr_bits = 1 << (exit_code - SVM_EXIT_WRITE_DR0);
-                if (svm->nested.intercept_dr_write & dr_bits)
+        case SVM_EXIT_READ_DR0 ... SVM_EXIT_WRITE_DR7: {
+                u32 bit = 1U << (exit_code - SVM_EXIT_READ_DR0);
+                if (svm->nested.intercept_dr & bit)
                         vmexit = NESTED_EXIT_DONE;
                 break;
         }
@@ -1893,6 +2026,10 @@ static int nested_svm_intercept(struct vcpu_svm *svm)
                 u32 excp_bits = 1 << (exit_code - SVM_EXIT_EXCP_BASE);
                 if (svm->nested.intercept_exceptions & excp_bits)
                         vmexit = NESTED_EXIT_DONE;
+                /* async page fault always cause vmexit */
+                else if ((exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR) &&
+                         svm->apf_reason != 0)
+                        vmexit = NESTED_EXIT_DONE;
                 break;
         }
         case SVM_EXIT_ERR: {
@@ -1926,10 +2063,8 @@ static inline void copy_vmcb_control_area(struct vmcb *dst_vmcb, struct vmcb *fr
         struct vmcb_control_area *dst  = &dst_vmcb->control;
         struct vmcb_control_area *from = &from_vmcb->control;
 
-        dst->intercept_cr_read    = from->intercept_cr_read;
-        dst->intercept_cr_write   = from->intercept_cr_write;
-        dst->intercept_dr_read    = from->intercept_dr_read;
-        dst->intercept_dr_write   = from->intercept_dr_write;
+        dst->intercept_cr         = from->intercept_cr;
+        dst->intercept_dr         = from->intercept_dr;
         dst->intercept_exceptions = from->intercept_exceptions;
         dst->intercept            = from->intercept;
         dst->iopm_base_pa         = from->iopm_base_pa;
@@ -1970,7 +2105,8 @@ static int nested_svm_vmexit(struct vcpu_svm *svm)
         if (!nested_vmcb)
                 return 1;
 
-        /* Exit nested SVM mode */
+        /* Exit Guest-Mode */
+        leave_guest_mode(&svm->vcpu);
         svm->nested.vmcb = 0;
 
         /* Give the current vmcb to the guest */
@@ -1984,7 +2120,7 @@ static int nested_svm_vmexit(struct vcpu_svm *svm)
         nested_vmcb->save.idtr   = vmcb->save.idtr;
         nested_vmcb->save.efer   = svm->vcpu.arch.efer;
         nested_vmcb->save.cr0    = kvm_read_cr0(&svm->vcpu);
-        nested_vmcb->save.cr3    = svm->vcpu.arch.cr3;
+        nested_vmcb->save.cr3    = kvm_read_cr3(&svm->vcpu);
         nested_vmcb->save.cr2    = vmcb->save.cr2;
         nested_vmcb->save.cr4    = svm->vcpu.arch.cr4;
         nested_vmcb->save.rflags = vmcb->save.rflags;
@@ -2061,6 +2197,8 @@ static int nested_svm_vmexit(struct vcpu_svm *svm)
         svm->vmcb->save.cpl = 0;
         svm->vmcb->control.exit_int_info = 0;
 
+        mark_all_dirty(svm->vmcb);
+
         nested_svm_unmap(page);
 
         nested_svm_uninit_mmu_context(&svm->vcpu);
@@ -2148,8 +2286,8 @@ static bool nested_svm_vmrun(struct vcpu_svm *svm)
                                nested_vmcb->control.event_inj,
                                nested_vmcb->control.nested_ctl);
 
-        trace_kvm_nested_intercepts(nested_vmcb->control.intercept_cr_read,
-                                    nested_vmcb->control.intercept_cr_write,
+        trace_kvm_nested_intercepts(nested_vmcb->control.intercept_cr & 0xffff,
+                                    nested_vmcb->control.intercept_cr >> 16,
                                     nested_vmcb->control.intercept_exceptions,
                                     nested_vmcb->control.intercept);
 
@@ -2177,7 +2315,7 @@ static bool nested_svm_vmrun(struct vcpu_svm *svm)
         if (npt_enabled)
                 hsave->save.cr3    = vmcb->save.cr3;
         else
-                hsave->save.cr3    = svm->vcpu.arch.cr3;
+                hsave->save.cr3    = kvm_read_cr3(&svm->vcpu);
 
         copy_vmcb_control_area(hsave, vmcb);
 
@@ -2229,14 +2367,12 @@ static bool nested_svm_vmrun(struct vcpu_svm *svm)
         svm->nested.vmcb_iopm  = nested_vmcb->control.iopm_base_pa  & ~0x0fffULL;
 
         /* cache intercepts */
-        svm->nested.intercept_cr_read    = nested_vmcb->control.intercept_cr_read;
-        svm->nested.intercept_cr_write   = nested_vmcb->control.intercept_cr_write;
-        svm->nested.intercept_dr_read    = nested_vmcb->control.intercept_dr_read;
-        svm->nested.intercept_dr_write   = nested_vmcb->control.intercept_dr_write;
+        svm->nested.intercept_cr         = nested_vmcb->control.intercept_cr;
+        svm->nested.intercept_dr         = nested_vmcb->control.intercept_dr;
         svm->nested.intercept_exceptions = nested_vmcb->control.intercept_exceptions;
         svm->nested.intercept            = nested_vmcb->control.intercept;
 
-        force_new_asid(&svm->vcpu);
+        svm_flush_tlb(&svm->vcpu);
         svm->vmcb->control.int_ctl = nested_vmcb->control.int_ctl | V_INTR_MASKING_MASK;
         if (nested_vmcb->control.int_ctl & V_INTR_MASKING_MASK)
                 svm->vcpu.arch.hflags |= HF_VINTR_MASK;
@@ -2245,29 +2381,12 @@ static bool nested_svm_vmrun(struct vcpu_svm *svm)
 
         if (svm->vcpu.arch.hflags & HF_VINTR_MASK) {
                 /* We only want the cr8 intercept bits of the guest */
-                svm->vmcb->control.intercept_cr_read &= ~INTERCEPT_CR8_MASK;
-                svm->vmcb->control.intercept_cr_write &= ~INTERCEPT_CR8_MASK;
+                clr_cr_intercept(svm, INTERCEPT_CR8_READ);
+                clr_cr_intercept(svm, INTERCEPT_CR8_WRITE);
         }
 
         /* We don't want to see VMMCALLs from a nested guest */
-        svm->vmcb->control.intercept &= ~(1ULL << INTERCEPT_VMMCALL);
-
-        /*
-         * We don't want a nested guest to be more powerful than the guest, so
-         * all intercepts are ORed
-         */
-        svm->vmcb->control.intercept_cr_read |=
-                nested_vmcb->control.intercept_cr_read;
-        svm->vmcb->control.intercept_cr_write |=
-                nested_vmcb->control.intercept_cr_write;
-        svm->vmcb->control.intercept_dr_read |=
-                nested_vmcb->control.intercept_dr_read;
-        svm->vmcb->control.intercept_dr_write |=
-                nested_vmcb->control.intercept_dr_write;
-        svm->vmcb->control.intercept_exceptions |=
-                nested_vmcb->control.intercept_exceptions;
-
-        svm->vmcb->control.intercept |= nested_vmcb->control.intercept;
+        clr_intercept(svm, INTERCEPT_VMMCALL);
 
         svm->vmcb->control.lbr_ctl = nested_vmcb->control.lbr_ctl;
         svm->vmcb->control.int_vector = nested_vmcb->control.int_vector;
@@ -2278,11 +2397,21 @@ static bool nested_svm_vmrun(struct vcpu_svm *svm)
 
         nested_svm_unmap(page);
 
-        /* nested_vmcb is our indicator if nested SVM is activated */
+        /* Enter Guest-Mode */
+        enter_guest_mode(&svm->vcpu);
+
+        /*
+         * Merge guest and host intercepts - must be called  with vcpu in
+         * guest-mode to take affect here
+         */
+        recalc_intercepts(svm);
+
         svm->nested.vmcb = vmcb_gpa;
 
         enable_gif(svm);
 
+        mark_all_dirty(svm->vmcb);
+
         return true;
 }
 
@@ -2400,6 +2529,8 @@ static int clgi_interception(struct vcpu_svm *svm)
         svm_clear_vintr(svm);
         svm->vmcb->control.int_ctl &= ~V_IRQ_MASK;
 
+        mark_dirty(svm->vmcb, VMCB_INTR);
+
         return 1;
 }
 
@@ -2426,6 +2557,19 @@ static int skinit_interception(struct vcpu_svm *svm)
         return 1;
 }
 
+static int xsetbv_interception(struct vcpu_svm *svm)
+{
+        u64 new_bv = kvm_read_edx_eax(&svm->vcpu);
+        u32 index = kvm_register_read(&svm->vcpu, VCPU_REGS_RCX);
+
+        if (kvm_set_xcr(&svm->vcpu, index, new_bv) == 0) {
+                svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
+                skip_emulated_instruction(&svm->vcpu);
+        }
+
+        return 1;
+}
+
 static int invalid_op_interception(struct vcpu_svm *svm)
 {
         kvm_queue_exception(&svm->vcpu, UD_VECTOR);
@@ -2507,19 +2651,92 @@ static int cpuid_interception(struct vcpu_svm *svm)
 static int iret_interception(struct vcpu_svm *svm)
 {
         ++svm->vcpu.stat.nmi_window_exits;
-        svm->vmcb->control.intercept &= ~(1ULL << INTERCEPT_IRET);
+        clr_intercept(svm, INTERCEPT_IRET);
         svm->vcpu.arch.hflags |= HF_IRET_MASK;
         return 1;
 }
 
 static int invlpg_interception(struct vcpu_svm *svm)
 {
-        return emulate_instruction(&svm->vcpu, 0, 0, 0) == EMULATE_DONE;
+        if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
+                return emulate_instruction(&svm->vcpu, 0) == EMULATE_DONE;
+
+        kvm_mmu_invlpg(&svm->vcpu, svm->vmcb->control.exit_info_1);
+        skip_emulated_instruction(&svm->vcpu);
+        return 1;
 }
 
 static int emulate_on_interception(struct vcpu_svm *svm)
 {
-        return emulate_instruction(&svm->vcpu, 0, 0, 0) == EMULATE_DONE;
+        return emulate_instruction(&svm->vcpu, 0) == EMULATE_DONE;
+}
+
+#define CR_VALID (1ULL << 63)
+
+static int cr_interception(struct vcpu_svm *svm)
+{
+        int reg, cr;
+        unsigned long val;
+        int err;
+
+        if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
+                return emulate_on_interception(svm);
+
+        if (unlikely((svm->vmcb->control.exit_info_1 & CR_VALID) == 0))
+                return emulate_on_interception(svm);
+
+        reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
+        cr = svm->vmcb->control.exit_code - SVM_EXIT_READ_CR0;
+
+        err = 0;
+        if (cr >= 16) { /* mov to cr */
+                cr -= 16;
+                val = kvm_register_read(&svm->vcpu, reg);
+                switch (cr) {
+                case 0:
+                        err = kvm_set_cr0(&svm->vcpu, val);
+                        break;
+                case 3:
+                        err = kvm_set_cr3(&svm->vcpu, val);
+                        break;
+                case 4:
+                        err = kvm_set_cr4(&svm->vcpu, val);
+                        break;
+                case 8:
+                        err = kvm_set_cr8(&svm->vcpu, val);
+                        break;
+                default:
+                        WARN(1, "unhandled write to CR%d", cr);
+                        kvm_queue_exception(&svm->vcpu, UD_VECTOR);
+                        return 1;
+                }
+        } else { /* mov from cr */
+                switch (cr) {
+                case 0:
+                        val = kvm_read_cr0(&svm->vcpu);
+                        break;
+                case 2:
+                        val = svm->vcpu.arch.cr2;
+                        break;
+                case 3:
+                        val = kvm_read_cr3(&svm->vcpu);
+                        break;
+                case 4:
+                        val = kvm_read_cr4(&svm->vcpu);
+                        break;
+                case 8:
+                        val = kvm_get_cr8(&svm->vcpu);
+                        break;
+                default:
+                        WARN(1, "unhandled read from CR%d", cr);
+                        kvm_queue_exception(&svm->vcpu, UD_VECTOR);
+                        return 1;
+                }
+                kvm_register_write(&svm->vcpu, reg, val);
+        }
+        kvm_complete_insn_gp(&svm->vcpu, err);
+
+        return 1;
 }
 
 static int cr0_write_interception(struct vcpu_svm *svm)
@@ -2527,7 +2744,7 @@ static int cr0_write_interception(struct vcpu_svm *svm)
         struct kvm_vcpu *vcpu = &svm->vcpu;
         int r;
 
-        r = emulate_instruction(&svm->vcpu, 0, 0, 0);
+        r = cr_interception(svm);
 
         if (svm->nested.vmexit_rip) {
                 kvm_register_write(vcpu, VCPU_REGS_RIP, svm->nested.vmexit_rip);
@@ -2536,22 +2753,47 @@ static int cr0_write_interception(struct vcpu_svm *svm)
                 svm->nested.vmexit_rip = 0;
         }
 
-        return r == EMULATE_DONE;
+        return r;
+}
+
+static int dr_interception(struct vcpu_svm *svm)
+{
+        int reg, dr;
+        unsigned long val;
+        int err;
+
+        if (!boot_cpu_has(X86_FEATURE_DECODEASSISTS))
+                return emulate_on_interception(svm);
+
+        reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
+        dr = svm->vmcb->control.exit_code - SVM_EXIT_READ_DR0;
+
+        if (dr >= 16) { /* mov to DRn */
+                val = kvm_register_read(&svm->vcpu, reg);
+                kvm_set_dr(&svm->vcpu, dr - 16, val);
+        } else {
+                err = kvm_get_dr(&svm->vcpu, dr, &val);
+                if (!err)
+                        kvm_register_write(&svm->vcpu, reg, val);
+        }
+
+        return 1;
 }
 
 static int cr8_write_interception(struct vcpu_svm *svm)
 {
         struct kvm_run *kvm_run = svm->vcpu.run;
+        int r;
 
         u8 cr8_prev = kvm_get_cr8(&svm->vcpu);
         /* instruction emulation calls kvm_set_cr8() */
-        emulate_instruction(&svm->vcpu, 0, 0, 0);
+        r = cr_interception(svm);
         if (irqchip_in_kernel(svm->vcpu.kvm)) {
-                svm->vmcb->control.intercept_cr_write &= ~INTERCEPT_CR8_MASK;
-                return 1;
+                clr_cr_intercept(svm, INTERCEPT_CR8_WRITE);
+                return r;
         }
         if (cr8_prev <= kvm_get_cr8(&svm->vcpu))
-                return 1;
+                return r;
         kvm_run->exit_reason = KVM_EXIT_SET_TPR;
         return 0;
 }
@@ -2562,14 +2804,9 @@ static int svm_get_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 *data)
 
         switch (ecx) {
         case MSR_IA32_TSC: {
-                u64 tsc_offset;
+                struct vmcb *vmcb = get_host_vmcb(svm);
 
-                if (is_nested(svm))
-                        tsc_offset = svm->nested.hsave->control.tsc_offset;
-                else
-                        tsc_offset = svm->vmcb->control.tsc_offset;
-
-                *data = tsc_offset + native_read_tsc();
+                *data = vmcb->control.tsc_offset + native_read_tsc();
                 break;
         }
         case MSR_STAR:
@@ -2714,7 +2951,7 @@ static int svm_set_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 data)
                 svm->vmcb->save.sysenter_esp = data;
                 break;
         case MSR_IA32_DEBUGCTLMSR:
-                if (!svm_has(SVM_FEATURE_LBRV)) {
+                if (!boot_cpu_has(X86_FEATURE_LBRV)) {
                         pr_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTL 0x%llx, nop\n",
                                         __func__, data);
                         break;
@@ -2723,6 +2960,7 @@ static int svm_set_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 data)
                         return 1;
 
                 svm->vmcb->save.dbgctl = data;
+                mark_dirty(svm->vmcb, VMCB_LBR);
                 if (data & (1ULL<<0))
                         svm_enable_lbrv(svm);
                 else
@@ -2775,6 +3013,7 @@ static int interrupt_window_interception(struct vcpu_svm *svm)
         kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
         svm_clear_vintr(svm);
         svm->vmcb->control.int_ctl &= ~V_IRQ_MASK;
+        mark_dirty(svm->vmcb, VMCB_INTR);
         /*
          * If the user space waits to inject interrupts, exit as soon as
          * possible
@@ -2797,31 +3036,31 @@ static int pause_interception(struct vcpu_svm *svm)
 }
 
 static int (*svm_exit_handlers[])(struct vcpu_svm *svm) = {
-        [SVM_EXIT_READ_CR0]                     = emulate_on_interception,
-        [SVM_EXIT_READ_CR3]                     = emulate_on_interception,
-        [SVM_EXIT_READ_CR4]                     = emulate_on_interception,
-        [SVM_EXIT_READ_CR8]                     = emulate_on_interception,
+        [SVM_EXIT_READ_CR0]                     = cr_interception,
+        [SVM_EXIT_READ_CR3]                     = cr_interception,
+        [SVM_EXIT_READ_CR4]                     = cr_interception,
+        [SVM_EXIT_READ_CR8]                     = cr_interception,
         [SVM_EXIT_CR0_SEL_WRITE]                = emulate_on_interception,
         [SVM_EXIT_WRITE_CR0]                    = cr0_write_interception,
-        [SVM_EXIT_WRITE_CR3]                    = emulate_on_interception,
-        [SVM_EXIT_WRITE_CR4]                    = emulate_on_interception,
+        [SVM_EXIT_WRITE_CR3]                    = cr_interception,
+        [SVM_EXIT_WRITE_CR4]                    = cr_interception,
         [SVM_EXIT_WRITE_CR8]                    = cr8_write_interception,
-        [SVM_EXIT_READ_DR0]                     = emulate_on_interception,
-        [SVM_EXIT_READ_DR1]                     = emulate_on_interception,
-        [SVM_EXIT_READ_DR2]                     = emulate_on_interception,
-        [SVM_EXIT_READ_DR3]                     = emulate_on_interception,
-        [SVM_EXIT_READ_DR4]                     = emulate_on_interception,
-        [SVM_EXIT_READ_DR5]                     = emulate_on_interception,
-        [SVM_EXIT_READ_DR6]                     = emulate_on_interception,
-        [SVM_EXIT_READ_DR7]                     = emulate_on_interception,
-        [SVM_EXIT_WRITE_DR0]                    = emulate_on_interception,
-        [SVM_EXIT_WRITE_DR1]                    = emulate_on_interception,
-        [SVM_EXIT_WRITE_DR2]                    = emulate_on_interception,
-        [SVM_EXIT_WRITE_DR3]                    = emulate_on_interception,
-        [SVM_EXIT_WRITE_DR4]                    = emulate_on_interception,
-        [SVM_EXIT_WRITE_DR5]                    = emulate_on_interception,
-        [SVM_EXIT_WRITE_DR6]                    = emulate_on_interception,
-        [SVM_EXIT_WRITE_DR7]                    = emulate_on_interception,
+        [SVM_EXIT_READ_DR0]                     = dr_interception,
+        [SVM_EXIT_READ_DR1]                     = dr_interception,
+        [SVM_EXIT_READ_DR2]                     = dr_interception,
+        [SVM_EXIT_READ_DR3]                     = dr_interception,
+        [SVM_EXIT_READ_DR4]                     = dr_interception,
+        [SVM_EXIT_READ_DR5]                     = dr_interception,
+        [SVM_EXIT_READ_DR6]                     = dr_interception,
+        [SVM_EXIT_READ_DR7]                     = dr_interception,
+        [SVM_EXIT_WRITE_DR0]                    = dr_interception,
+        [SVM_EXIT_WRITE_DR1]                    = dr_interception,
+        [SVM_EXIT_WRITE_DR2]                    = dr_interception,
+        [SVM_EXIT_WRITE_DR3]                    = dr_interception,
+        [SVM_EXIT_WRITE_DR4]                    = dr_interception,
+        [SVM_EXIT_WRITE_DR5]                    = dr_interception,
+        [SVM_EXIT_WRITE_DR6]                    = dr_interception,
+        [SVM_EXIT_WRITE_DR7]                    = dr_interception,
         [SVM_EXIT_EXCP_BASE + DB_VECTOR]        = db_interception,
         [SVM_EXIT_EXCP_BASE + BP_VECTOR]        = bp_interception,
         [SVM_EXIT_EXCP_BASE + UD_VECTOR]        = ud_interception,
@@ -2854,6 +3093,7 @@ static int (*svm_exit_handlers[])(struct vcpu_svm *svm) = {
         [SVM_EXIT_WBINVD]                       = emulate_on_interception,
         [SVM_EXIT_MONITOR]                      = invalid_op_interception,
         [SVM_EXIT_MWAIT]                        = invalid_op_interception,
+        [SVM_EXIT_XSETBV]                       = xsetbv_interception,
         [SVM_EXIT_NPF]                          = pf_interception,
 };
 
@@ -2864,10 +3104,10 @@ void dump_vmcb(struct kvm_vcpu *vcpu)
         struct vmcb_save_area *save = &svm->vmcb->save;
 
         pr_err("VMCB Control Area:\n");
-        pr_err("cr_read:            %04x\n", control->intercept_cr_read);
-        pr_err("cr_write:           %04x\n", control->intercept_cr_write);
-        pr_err("dr_read:            %04x\n", control->intercept_dr_read);
-        pr_err("dr_write:           %04x\n", control->intercept_dr_write);
+        pr_err("cr_read:            %04x\n", control->intercept_cr & 0xffff);
+        pr_err("cr_write:           %04x\n", control->intercept_cr >> 16);
+        pr_err("dr_read:            %04x\n", control->intercept_dr & 0xffff);
+        pr_err("dr_write:           %04x\n", control->intercept_dr >> 16);
         pr_err("exceptions:         %08x\n", control->intercept_exceptions);
         pr_err("intercepts:         %016llx\n", control->intercept);
         pr_err("pause filter count: %d\n", control->pause_filter_count);
@@ -2950,15 +3190,23 @@ void dump_vmcb(struct kvm_vcpu *vcpu)
 
 }
 
+static void svm_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2)
+{
+        struct vmcb_control_area *control = &to_svm(vcpu)->vmcb->control;
+
+        *info1 = control->exit_info_1;
+        *info2 = control->exit_info_2;
+}
+
 static int handle_exit(struct kvm_vcpu *vcpu)
 {
         struct vcpu_svm *svm = to_svm(vcpu);
         struct kvm_run *kvm_run = vcpu->run;
         u32 exit_code = svm->vmcb->control.exit_code;
 
-        trace_kvm_exit(exit_code, vcpu);
+        trace_kvm_exit(exit_code, vcpu, KVM_ISA_SVM);
 
-        if (!(svm->vmcb->control.intercept_cr_write & INTERCEPT_CR0_MASK))
+        if (!is_cr_intercept(svm, INTERCEPT_CR0_WRITE))
                 vcpu->arch.cr0 = svm->vmcb->save.cr0;
         if (npt_enabled)
                 vcpu->arch.cr3 = svm->vmcb->save.cr3;
@@ -2970,7 +3218,7 @@ static int handle_exit(struct kvm_vcpu *vcpu)
                 return 1;
         }
 
-        if (is_nested(svm)) {
+        if (is_guest_mode(vcpu)) {
                 int vmexit;
 
                 trace_kvm_nested_vmexit(svm->vmcb->save.rip, exit_code,
@@ -3033,7 +3281,6 @@ static void pre_svm_run(struct vcpu_svm *svm)
 
         struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
 
-        svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING;
         /* FIXME: handle wraparound of asid_generation */
         if (svm->asid_generation != sd->asid_generation)
                 new_asid(svm, sd);
@@ -3045,7 +3292,7 @@ static void svm_inject_nmi(struct kvm_vcpu *vcpu)
 
         svm->vmcb->control.event_inj = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI;
         vcpu->arch.hflags |= HF_NMI_MASK;
-        svm->vmcb->control.intercept |= (1ULL << INTERCEPT_IRET);
+        set_intercept(svm, INTERCEPT_IRET);
         ++vcpu->stat.nmi_injections;
 }
 
@@ -3058,6 +3305,7 @@ static inline void svm_inject_irq(struct vcpu_svm *svm, int irq)
         control->int_ctl &= ~V_INTR_PRIO_MASK;
         control->int_ctl |= V_IRQ_MASK |
                 ((/*control->int_vector >> 4*/ 0xf) << V_INTR_PRIO_SHIFT);
+        mark_dirty(svm->vmcb, VMCB_INTR);
 }
 
 static void svm_set_irq(struct kvm_vcpu *vcpu)
@@ -3077,14 +3325,14 @@ static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
 {
         struct vcpu_svm *svm = to_svm(vcpu);
 
-        if (is_nested(svm) && (vcpu->arch.hflags & HF_VINTR_MASK))
+        if (is_guest_mode(vcpu) && (vcpu->arch.hflags & HF_VINTR_MASK))
                 return;
 
         if (irr == -1)
                 return;
 
         if (tpr >= irr)
-                svm->vmcb->control.intercept_cr_write |= INTERCEPT_CR8_MASK;
+                set_cr_intercept(svm, INTERCEPT_CR8_WRITE);
 }
 
 static int svm_nmi_allowed(struct kvm_vcpu *vcpu)
@@ -3112,10 +3360,10 @@ static void svm_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked)
 
         if (masked) {
                 svm->vcpu.arch.hflags |= HF_NMI_MASK;
-                svm->vmcb->control.intercept |= (1ULL << INTERCEPT_IRET);
+                set_intercept(svm, INTERCEPT_IRET);
         } else {
                 svm->vcpu.arch.hflags &= ~HF_NMI_MASK;
-                svm->vmcb->control.intercept &= ~(1ULL << INTERCEPT_IRET);
+                clr_intercept(svm, INTERCEPT_IRET);
         }
 }
 
@@ -3131,7 +3379,7 @@ static int svm_interrupt_allowed(struct kvm_vcpu *vcpu)
 
         ret = !!(vmcb->save.rflags & X86_EFLAGS_IF);
 
-        if (is_nested(svm))
+        if (is_guest_mode(vcpu))
                 return ret && !(svm->vcpu.arch.hflags & HF_VINTR_MASK);
 
         return ret;
@@ -3177,7 +3425,12 @@ static int svm_set_tss_addr(struct kvm *kvm, unsigned int addr)
 
 static void svm_flush_tlb(struct kvm_vcpu *vcpu)
 {
-        force_new_asid(vcpu);
+        struct vcpu_svm *svm = to_svm(vcpu);
+
+        if (static_cpu_has(X86_FEATURE_FLUSHBYASID))
+                svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID;
+        else
+                svm->asid_generation--;
 }
 
 static void svm_prepare_guest_switch(struct kvm_vcpu *vcpu)
@@ -3188,10 +3441,10 @@ static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu)
 {
         struct vcpu_svm *svm = to_svm(vcpu);
 
-        if (is_nested(svm) && (vcpu->arch.hflags & HF_VINTR_MASK))
+        if (is_guest_mode(vcpu) && (vcpu->arch.hflags & HF_VINTR_MASK))
                 return;
 
-        if (!(svm->vmcb->control.intercept_cr_write & INTERCEPT_CR8_MASK)) {
+        if (!is_cr_intercept(svm, INTERCEPT_CR8_WRITE)) {
                 int cr8 = svm->vmcb->control.int_ctl & V_TPR_MASK;
                 kvm_set_cr8(vcpu, cr8);
         }
@@ -3202,7 +3455,7 @@ static inline void sync_lapic_to_cr8(struct kvm_vcpu *vcpu)
         struct vcpu_svm *svm = to_svm(vcpu);
         u64 cr8;
 
-        if (is_nested(svm) && (vcpu->arch.hflags & HF_VINTR_MASK))
+        if (is_guest_mode(vcpu) && (vcpu->arch.hflags & HF_VINTR_MASK))
                 return;
 
         cr8 = kvm_get_cr8(vcpu);
@@ -3289,9 +3542,6 @@ static void svm_cancel_injection(struct kvm_vcpu *vcpu)
 static void svm_vcpu_run(struct kvm_vcpu *vcpu)
 {
         struct vcpu_svm *svm = to_svm(vcpu);
-        u16 fs_selector;
-        u16 gs_selector;
-        u16 ldt_selector;
 
         svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
         svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
@@ -3308,10 +3558,6 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu)
 
         sync_lapic_to_cr8(vcpu);
 
-        save_host_msrs(vcpu);
-        savesegment(fs, fs_selector);
-        savesegment(gs, gs_selector);
-        ldt_selector = kvm_read_ldt();
         svm->vmcb->save.cr2 = vcpu->arch.cr2;
 
         clgi();
@@ -3389,19 +3635,10 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu)
 #endif
                 );
 
-        vcpu->arch.cr2 = svm->vmcb->save.cr2;
-        vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax;
-        vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp;
-        vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip;
-
-        load_host_msrs(vcpu);
-        kvm_load_ldt(ldt_selector);
-        loadsegment(fs, fs_selector);
 #ifdef CONFIG_X86_64
-        load_gs_index(gs_selector);
-        wrmsrl(MSR_KERNEL_GS_BASE, current->thread.gs);
+        wrmsrl(MSR_GS_BASE, svm->host.gs_base);
 #else
-        loadsegment(gs, gs_selector);
+        loadsegment(fs, svm->host.fs);
 #endif
 
         reload_tss(vcpu);
@@ -3410,10 +3647,21 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu)
 
         stgi();
 
+        vcpu->arch.cr2 = svm->vmcb->save.cr2;
+        vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax;
+        vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp;
+        vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip;
+
         sync_cr8_to_lapic(vcpu);
 
         svm->next_rip = 0;
 
+        svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING;
+
+        /* if exit due to PF check for async PF */
+        if (svm->vmcb->control.exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR)
+                svm->apf_reason = kvm_read_and_reset_pf_reason();
+
         if (npt_enabled) {
                 vcpu->arch.regs_avail &= ~(1 << VCPU_EXREG_PDPTR);
                 vcpu->arch.regs_dirty &= ~(1 << VCPU_EXREG_PDPTR);
@@ -3426,6 +3674,8 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu)
         if (unlikely(svm->vmcb->control.exit_code ==
                      SVM_EXIT_EXCP_BASE + MC_VECTOR))
                 svm_handle_mce(svm);
+
+        mark_all_clean(svm->vmcb);
 }
 
 #undef R
@@ -3435,7 +3685,8 @@ static void svm_set_cr3(struct kvm_vcpu *vcpu, unsigned long root)
         struct vcpu_svm *svm = to_svm(vcpu);
 
         svm->vmcb->save.cr3 = root;
-        force_new_asid(vcpu);
+        mark_dirty(svm->vmcb, VMCB_CR);
+        svm_flush_tlb(vcpu);
 }
 
 static void set_tdp_cr3(struct kvm_vcpu *vcpu, unsigned long root)
@@ -3443,11 +3694,13 @@ static void set_tdp_cr3(struct kvm_vcpu *vcpu, unsigned long root)
         struct vcpu_svm *svm = to_svm(vcpu);
 
         svm->vmcb->control.nested_cr3 = root;
+        mark_dirty(svm->vmcb, VMCB_NPT);
 
         /* Also sync guest cr3 here in case we live migrate */
-        svm->vmcb->save.cr3 = vcpu->arch.cr3;
+        svm->vmcb->save.cr3 = kvm_read_cr3(vcpu);
+        mark_dirty(svm->vmcb, VMCB_CR);
 
-        force_new_asid(vcpu);
+        svm_flush_tlb(vcpu);
 }
 
 static int is_disabled(void)
@@ -3494,10 +3747,6 @@ static void svm_cpuid_update(struct kvm_vcpu *vcpu)
 static void svm_set_supported_cpuid(u32 func, struct kvm_cpuid_entry2 *entry)
 {
         switch (func) {
-        case 0x00000001:
-                /* Mask out xsave bit as long as it is not supported by SVM */
-                entry->ecx &= ~(bit(X86_FEATURE_XSAVE));
-                break;
         case 0x80000001:
                 if (nested)
                         entry->ecx |= (1 << 2); /* Set SVM bit */
@@ -3511,7 +3760,7 @@ static void svm_set_supported_cpuid(u32 func, struct kvm_cpuid_entry2 *entry)
                                    additional features */
 
                 /* Support next_rip if host supports it */
-                if (svm_has(SVM_FEATURE_NRIP))
+                if (boot_cpu_has(X86_FEATURE_NRIPS))
                         entry->edx |= SVM_FEATURE_NRIP;
 
                 /* Support NPT for the guest if enabled */
@@ -3571,6 +3820,7 @@ static const struct trace_print_flags svm_exit_reasons_str[] = {
         { SVM_EXIT_WBINVD,                      "wbinvd" },
         { SVM_EXIT_MONITOR,                     "monitor" },
         { SVM_EXIT_MWAIT,                       "mwait" },
+        { SVM_EXIT_XSETBV,                      "xsetbv" },
         { SVM_EXIT_NPF,                         "npf" },
         { -1, NULL }
 };
@@ -3594,9 +3844,7 @@ static void svm_fpu_deactivate(struct kvm_vcpu *vcpu)
 {
         struct vcpu_svm *svm = to_svm(vcpu);
 
-        svm->vmcb->control.intercept_exceptions |= 1 << NM_VECTOR;
-        if (is_nested(svm))
-                svm->nested.hsave->control.intercept_exceptions |= 1 << NM_VECTOR;
+        set_exception_intercept(svm, NM_VECTOR);
         update_cr0_intercept(svm);
 }
 
@@ -3627,6 +3875,7 @@ static struct kvm_x86_ops svm_x86_ops = {
         .get_cpl = svm_get_cpl,
         .get_cs_db_l_bits = kvm_get_cs_db_l_bits,
         .decache_cr0_guest_bits = svm_decache_cr0_guest_bits,
+        .decache_cr3 = svm_decache_cr3,
         .decache_cr4_guest_bits = svm_decache_cr4_guest_bits,
         .set_cr0 = svm_set_cr0,
         .set_cr3 = svm_set_cr3,
@@ -3667,7 +3916,9 @@ static struct kvm_x86_ops svm_x86_ops = {
         .get_tdp_level = get_npt_level,
         .get_mt_mask = svm_get_mt_mask,
 
+        .get_exit_info = svm_get_exit_info,
         .exit_reasons_str = svm_exit_reasons_str,
+
         .get_lpage_level = svm_get_lpage_level,
 
         .cpuid_update = svm_cpuid_update,
diff --git a/arch/x86/kvm/trace.h b/arch/x86/kvm/trace.h
index a6544b8e7c0..1357d7cf4ec 100644
--- a/arch/x86/kvm/trace.h
+++ b/arch/x86/kvm/trace.h
@@ -178,27 +178,36 @@ TRACE_EVENT(kvm_apic,
 #define trace_kvm_apic_read(reg, val)           trace_kvm_apic(0, reg, val)
 #define trace_kvm_apic_write(reg, val)          trace_kvm_apic(1, reg, val)
 
+#define KVM_ISA_VMX   1
+#define KVM_ISA_SVM   2
+
 /*
  * Tracepoint for kvm guest exit:
  */
 TRACE_EVENT(kvm_exit,
-        TP_PROTO(unsigned int exit_reason, struct kvm_vcpu *vcpu),
-        TP_ARGS(exit_reason, vcpu),
+        TP_PROTO(unsigned int exit_reason, struct kvm_vcpu *vcpu, u32 isa),
+        TP_ARGS(exit_reason, vcpu, isa),
 
         TP_STRUCT__entry(
                 __field(        unsigned int,   exit_reason     )
                 __field(        unsigned long,  guest_rip       )
+                __field(        u32,            isa             )
+                __field(        u64,            info1           )
+                __field(        u64,            info2           )
         ),
 
         TP_fast_assign(
                 __entry->exit_reason    = exit_reason;
                 __entry->guest_rip      = kvm_rip_read(vcpu);
+                __entry->isa            = isa;
+                kvm_x86_ops->get_exit_info(vcpu, &__entry->info1,
+                                           &__entry->info2);
         ),
 
-        TP_printk("reason %s rip 0x%lx",
+        TP_printk("reason %s rip 0x%lx info %llx %llx",
                  ftrace_print_symbols_seq(p, __entry->exit_reason,
                                           kvm_x86_ops->exit_reasons_str),
-                 __entry->guest_rip)
+                 __entry->guest_rip, __entry->info1, __entry->info2)
 );
 
 /*
diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index 81fcbe9515c..bf89ec2cfb8 100644
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -69,6 +69,9 @@ module_param(emulate_invalid_guest_state, bool, S_IRUGO);
 static int __read_mostly vmm_exclusive = 1;
 module_param(vmm_exclusive, bool, S_IRUGO);
 
+static int __read_mostly yield_on_hlt = 1;
+module_param(yield_on_hlt, bool, S_IRUGO);
+
 #define KVM_GUEST_CR0_MASK_UNRESTRICTED_GUEST                           \
         (X86_CR0_WP | X86_CR0_NE | X86_CR0_NW | X86_CR0_CD)
 #define KVM_GUEST_CR0_MASK                                              \
@@ -177,6 +180,7 @@ static int init_rmode(struct kvm *kvm);
 static u64 construct_eptp(unsigned long root_hpa);
 static void kvm_cpu_vmxon(u64 addr);
 static void kvm_cpu_vmxoff(void);
+static void vmx_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3);
 
 static DEFINE_PER_CPU(struct vmcs *, vmxarea);
 static DEFINE_PER_CPU(struct vmcs *, current_vmcs);
@@ -188,6 +192,8 @@ static unsigned long *vmx_io_bitmap_b;
 static unsigned long *vmx_msr_bitmap_legacy;
 static unsigned long *vmx_msr_bitmap_longmode;
 
+static bool cpu_has_load_ia32_efer;
+
 static DECLARE_BITMAP(vmx_vpid_bitmap, VMX_NR_VPIDS);
 static DEFINE_SPINLOCK(vmx_vpid_lock);
 
@@ -472,7 +478,7 @@ static void vmcs_clear(struct vmcs *vmcs)
         u8 error;
 
         asm volatile (__ex(ASM_VMX_VMCLEAR_RAX) "; setna %0"
-                      : "=g"(error) : "a"(&phys_addr), "m"(phys_addr)
+                      : "=qm"(error) : "a"(&phys_addr), "m"(phys_addr)
                       : "cc", "memory");
         if (error)
                 printk(KERN_ERR "kvm: vmclear fail: %p/%llx\n",
@@ -485,7 +491,7 @@ static void vmcs_load(struct vmcs *vmcs)
         u8 error;
 
         asm volatile (__ex(ASM_VMX_VMPTRLD_RAX) "; setna %0"
-                        : "=g"(error) : "a"(&phys_addr), "m"(phys_addr)
+                        : "=qm"(error) : "a"(&phys_addr), "m"(phys_addr)
                         : "cc", "memory");
         if (error)
                 printk(KERN_ERR "kvm: vmptrld %p/%llx fail\n",
@@ -565,10 +571,10 @@ static inline void ept_sync_individual_addr(u64 eptp, gpa_t gpa)
 
 static unsigned long vmcs_readl(unsigned long field)
 {
-        unsigned long value;
+        unsigned long value = 0;
 
         asm volatile (__ex(ASM_VMX_VMREAD_RDX_RAX)
-                      : "=a"(value) : "d"(field) : "cc");
+                      : "+a"(value) : "d"(field) : "cc");
         return value;
 }
 
@@ -661,6 +667,12 @@ static void clear_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr)
         unsigned i;
         struct msr_autoload *m = &vmx->msr_autoload;
 
+        if (msr == MSR_EFER && cpu_has_load_ia32_efer) {
+                vmcs_clear_bits(VM_ENTRY_CONTROLS, VM_ENTRY_LOAD_IA32_EFER);
+                vmcs_clear_bits(VM_EXIT_CONTROLS, VM_EXIT_LOAD_IA32_EFER);
+                return;
+        }
+
         for (i = 0; i < m->nr; ++i)
                 if (m->guest[i].index == msr)
                         break;
@@ -680,6 +692,14 @@ static void add_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr,
         unsigned i;
         struct msr_autoload *m = &vmx->msr_autoload;
 
+        if (msr == MSR_EFER && cpu_has_load_ia32_efer) {
+                vmcs_write64(GUEST_IA32_EFER, guest_val);
+                vmcs_write64(HOST_IA32_EFER, host_val);
+                vmcs_set_bits(VM_ENTRY_CONTROLS, VM_ENTRY_LOAD_IA32_EFER);
+                vmcs_set_bits(VM_EXIT_CONTROLS, VM_EXIT_LOAD_IA32_EFER);
+                return;
+        }
+
         for (i = 0; i < m->nr; ++i)
                 if (m->guest[i].index == msr)
                         break;
@@ -1009,6 +1029,17 @@ static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
         vmx_set_interrupt_shadow(vcpu, 0);
 }
 
+static void vmx_clear_hlt(struct kvm_vcpu *vcpu)
+{
+        /* Ensure that we clear the HLT state in the VMCS.  We don't need to
+         * explicitly skip the instruction because if the HLT state is set, then
+         * the instruction is already executing and RIP has already been
+         * advanced. */
+        if (!yield_on_hlt &&
+            vmcs_read32(GUEST_ACTIVITY_STATE) == GUEST_ACTIVITY_HLT)
+                vmcs_write32(GUEST_ACTIVITY_STATE, GUEST_ACTIVITY_ACTIVE);
+}
+
 static void vmx_queue_exception(struct kvm_vcpu *vcpu, unsigned nr,
                                 bool has_error_code, u32 error_code,
                                 bool reinject)
@@ -1035,6 +1066,7 @@ static void vmx_queue_exception(struct kvm_vcpu *vcpu, unsigned nr,
                 intr_info |= INTR_TYPE_HARD_EXCEPTION;
 
         vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr_info);
+        vmx_clear_hlt(vcpu);
 }
 
 static bool vmx_rdtscp_supported(void)
@@ -1305,8 +1337,11 @@ static __init int vmx_disabled_by_bios(void)
                         && tboot_enabled())
                         return 1;
                 if (!(msr & FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX)
-                        && !tboot_enabled())
+                        && !tboot_enabled()) {
+                        printk(KERN_WARNING "kvm: disable TXT in the BIOS or "
+                                " activate TXT before enabling KVM\n");
                         return 1;
+                }
         }
 
         return 0;
@@ -1400,6 +1435,14 @@ static __init int adjust_vmx_controls(u32 ctl_min, u32 ctl_opt,
         return 0;
 }
 
+static __init bool allow_1_setting(u32 msr, u32 ctl)
+{
+        u32 vmx_msr_low, vmx_msr_high;
+
+        rdmsr(msr, vmx_msr_low, vmx_msr_high);
+        return vmx_msr_high & ctl;
+}
+
 static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf)
 {
         u32 vmx_msr_low, vmx_msr_high;
@@ -1416,7 +1459,7 @@ static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf)
                                 &_pin_based_exec_control) < 0)
                 return -EIO;
 
-        min = CPU_BASED_HLT_EXITING |
+        min =
 #ifdef CONFIG_X86_64
               CPU_BASED_CR8_LOAD_EXITING |
               CPU_BASED_CR8_STORE_EXITING |
@@ -1429,6 +1472,10 @@ static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf)
               CPU_BASED_MWAIT_EXITING |
               CPU_BASED_MONITOR_EXITING |
               CPU_BASED_INVLPG_EXITING;
+
+        if (yield_on_hlt)
+                min |= CPU_BASED_HLT_EXITING;
+
         opt = CPU_BASED_TPR_SHADOW |
               CPU_BASED_USE_MSR_BITMAPS |
               CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
@@ -1510,6 +1557,12 @@ static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf)
         vmcs_conf->vmexit_ctrl         = _vmexit_control;
         vmcs_conf->vmentry_ctrl        = _vmentry_control;
 
+        cpu_has_load_ia32_efer =
+                allow_1_setting(MSR_IA32_VMX_ENTRY_CTLS,
+                                VM_ENTRY_LOAD_IA32_EFER)
+                && allow_1_setting(MSR_IA32_VMX_EXIT_CTLS,
+                                   VM_EXIT_LOAD_IA32_EFER);
+
         return 0;
 }
 
@@ -1683,9 +1736,13 @@ static void fix_rmode_seg(int seg, struct kvm_save_segment *save)
         save->limit = vmcs_read32(sf->limit);
         save->ar = vmcs_read32(sf->ar_bytes);
         vmcs_write16(sf->selector, save->base >> 4);
-        vmcs_write32(sf->base, save->base & 0xfffff);
+        vmcs_write32(sf->base, save->base & 0xffff0);
         vmcs_write32(sf->limit, 0xffff);
         vmcs_write32(sf->ar_bytes, 0xf3);
+        if (save->base & 0xf)
+                printk_once(KERN_WARNING "kvm: segment base is not paragraph"
+                            " aligned when entering protected mode (seg=%d)",
+                            seg);
 }
 
 static void enter_rmode(struct kvm_vcpu *vcpu)
@@ -1814,6 +1871,13 @@ static void vmx_decache_cr0_guest_bits(struct kvm_vcpu *vcpu)
         vcpu->arch.cr0 |= vmcs_readl(GUEST_CR0) & cr0_guest_owned_bits;
 }
 
+static void vmx_decache_cr3(struct kvm_vcpu *vcpu)
+{
+        if (enable_ept && is_paging(vcpu))
+                vcpu->arch.cr3 = vmcs_readl(GUEST_CR3);
+        __set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail);
+}
+
 static void vmx_decache_cr4_guest_bits(struct kvm_vcpu *vcpu)
 {
         ulong cr4_guest_owned_bits = vcpu->arch.cr4_guest_owned_bits;
@@ -1857,6 +1921,7 @@ static void ept_update_paging_mode_cr0(unsigned long *hw_cr0,
                                         unsigned long cr0,
                                         struct kvm_vcpu *vcpu)
 {
+        vmx_decache_cr3(vcpu);
         if (!(cr0 & X86_CR0_PG)) {
                 /* From paging/starting to nonpaging */
                 vmcs_write32(CPU_BASED_VM_EXEC_CONTROL,
@@ -1937,7 +2002,7 @@ static void vmx_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
         if (enable_ept) {
                 eptp = construct_eptp(cr3);
                 vmcs_write64(EPT_POINTER, eptp);
-                guest_cr3 = is_paging(vcpu) ? vcpu->arch.cr3 :
+                guest_cr3 = is_paging(vcpu) ? kvm_read_cr3(vcpu) :
                         vcpu->kvm->arch.ept_identity_map_addr;
                 ept_load_pdptrs(vcpu);
         }
@@ -2725,7 +2790,7 @@ static int vmx_vcpu_reset(struct kvm_vcpu *vcpu)
         vmcs_writel(GUEST_IDTR_BASE, 0);
         vmcs_write32(GUEST_IDTR_LIMIT, 0xffff);
 
-        vmcs_write32(GUEST_ACTIVITY_STATE, 0);
+        vmcs_write32(GUEST_ACTIVITY_STATE, GUEST_ACTIVITY_ACTIVE);
         vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 0);
         vmcs_write32(GUEST_PENDING_DBG_EXCEPTIONS, 0);
 
@@ -2787,6 +2852,10 @@ static void enable_nmi_window(struct kvm_vcpu *vcpu)
                 return;
         }
 
+        if (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & GUEST_INTR_STATE_STI) {
+                enable_irq_window(vcpu);
+                return;
+        }
         cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
         cpu_based_vm_exec_control |= CPU_BASED_VIRTUAL_NMI_PENDING;
         vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
@@ -2814,6 +2883,7 @@ static void vmx_inject_irq(struct kvm_vcpu *vcpu)
         } else
                 intr |= INTR_TYPE_EXT_INTR;
         vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr);
+        vmx_clear_hlt(vcpu);
 }
 
 static void vmx_inject_nmi(struct kvm_vcpu *vcpu)
@@ -2841,6 +2911,7 @@ static void vmx_inject_nmi(struct kvm_vcpu *vcpu)
         }
         vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
                         INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK | NMI_VECTOR);
+        vmx_clear_hlt(vcpu);
 }
 
 static int vmx_nmi_allowed(struct kvm_vcpu *vcpu)
@@ -2849,7 +2920,8 @@ static int vmx_nmi_allowed(struct kvm_vcpu *vcpu)
                 return 0;
 
         return  !(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) &
-                        (GUEST_INTR_STATE_MOV_SS | GUEST_INTR_STATE_NMI));
+                  (GUEST_INTR_STATE_MOV_SS | GUEST_INTR_STATE_STI
+                   | GUEST_INTR_STATE_NMI));
 }
 
 static bool vmx_get_nmi_mask(struct kvm_vcpu *vcpu)
@@ -2910,7 +2982,7 @@ static int handle_rmode_exception(struct kvm_vcpu *vcpu,
          * Cause the #SS fault with 0 error code in VM86 mode.
          */
         if (((vec == GP_VECTOR) || (vec == SS_VECTOR)) && err_code == 0)
-                if (emulate_instruction(vcpu, 0, 0, 0) == EMULATE_DONE)
+                if (emulate_instruction(vcpu, 0) == EMULATE_DONE)
                         return 1;
         /*
          * Forward all other exceptions that are valid in real mode.
@@ -3007,7 +3079,7 @@ static int handle_exception(struct kvm_vcpu *vcpu)
         }
 
         if (is_invalid_opcode(intr_info)) {
-                er = emulate_instruction(vcpu, 0, 0, EMULTYPE_TRAP_UD);
+                er = emulate_instruction(vcpu, EMULTYPE_TRAP_UD);
                 if (er != EMULATE_DONE)
                         kvm_queue_exception(vcpu, UD_VECTOR);
                 return 1;
@@ -3026,7 +3098,7 @@ static int handle_exception(struct kvm_vcpu *vcpu)
 
                 if (kvm_event_needs_reinjection(vcpu))
                         kvm_mmu_unprotect_page_virt(vcpu, cr2);
-                return kvm_mmu_page_fault(vcpu, cr2, error_code);
+                return kvm_mmu_page_fault(vcpu, cr2, error_code, NULL, 0);
         }
 
         if (vmx->rmode.vm86_active &&
@@ -3098,7 +3170,7 @@ static int handle_io(struct kvm_vcpu *vcpu)
         ++vcpu->stat.io_exits;
 
         if (string || in)
-                return emulate_instruction(vcpu, 0, 0, 0) == EMULATE_DONE;
+                return emulate_instruction(vcpu, 0) == EMULATE_DONE;
 
         port = exit_qualification >> 16;
         size = (exit_qualification & 7) + 1;
@@ -3118,14 +3190,6 @@ vmx_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
         hypercall[2] = 0xc1;
 }
 
-static void complete_insn_gp(struct kvm_vcpu *vcpu, int err)
-{
-        if (err)
-                kvm_inject_gp(vcpu, 0);
-        else
-                skip_emulated_instruction(vcpu);
-}
-
 static int handle_cr(struct kvm_vcpu *vcpu)
 {
         unsigned long exit_qualification, val;
@@ -3143,21 +3207,21 @@ static int handle_cr(struct kvm_vcpu *vcpu)
                 switch (cr) {
                 case 0:
                         err = kvm_set_cr0(vcpu, val);
-                        complete_insn_gp(vcpu, err);
+                        kvm_complete_insn_gp(vcpu, err);
                         return 1;
                 case 3:
                         err = kvm_set_cr3(vcpu, val);
-                        complete_insn_gp(vcpu, err);
+                        kvm_complete_insn_gp(vcpu, err);
                         return 1;
                 case 4:
                         err = kvm_set_cr4(vcpu, val);
-                        complete_insn_gp(vcpu, err);
+                        kvm_complete_insn_gp(vcpu, err);
                         return 1;
                 case 8: {
                                 u8 cr8_prev = kvm_get_cr8(vcpu);
                                 u8 cr8 = kvm_register_read(vcpu, reg);
-                                kvm_set_cr8(vcpu, cr8);
-                                skip_emulated_instruction(vcpu);
+                                err = kvm_set_cr8(vcpu, cr8);
+                                kvm_complete_insn_gp(vcpu, err);
                                 if (irqchip_in_kernel(vcpu->kvm))
                                         return 1;
                                 if (cr8_prev <= cr8)
@@ -3176,8 +3240,9 @@ static int handle_cr(struct kvm_vcpu *vcpu)
         case 1: /*mov from cr*/
                 switch (cr) {
                 case 3:
-                        kvm_register_write(vcpu, reg, vcpu->arch.cr3);
-                        trace_kvm_cr_read(cr, vcpu->arch.cr3);
+                        val = kvm_read_cr3(vcpu);
+                        kvm_register_write(vcpu, reg, val);
+                        trace_kvm_cr_read(cr, val);
                         skip_emulated_instruction(vcpu);
                         return 1;
                 case 8:
@@ -3349,6 +3414,11 @@ static int handle_vmx_insn(struct kvm_vcpu *vcpu)
         return 1;
 }
 
+static int handle_invd(struct kvm_vcpu *vcpu)
+{
+        return emulate_instruction(vcpu, 0) == EMULATE_DONE;
+}
+
 static int handle_invlpg(struct kvm_vcpu *vcpu)
 {
         unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
@@ -3377,7 +3447,7 @@ static int handle_xsetbv(struct kvm_vcpu *vcpu)
 
 static int handle_apic_access(struct kvm_vcpu *vcpu)
 {
-        return emulate_instruction(vcpu, 0, 0, 0) == EMULATE_DONE;
+        return emulate_instruction(vcpu, 0) == EMULATE_DONE;
 }
 
 static int handle_task_switch(struct kvm_vcpu *vcpu)
@@ -3476,7 +3546,7 @@ static int handle_ept_violation(struct kvm_vcpu *vcpu)
 
         gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS);
         trace_kvm_page_fault(gpa, exit_qualification);
-        return kvm_mmu_page_fault(vcpu, gpa & PAGE_MASK, 0);
+        return kvm_mmu_page_fault(vcpu, gpa, exit_qualification & 0x3, NULL, 0);
 }
 
 static u64 ept_rsvd_mask(u64 spte, int level)
@@ -3592,7 +3662,7 @@ static int handle_invalid_guest_state(struct kvm_vcpu *vcpu)
                     && (kvm_get_rflags(&vmx->vcpu) & X86_EFLAGS_IF))
                         return handle_interrupt_window(&vmx->vcpu);
 
-                err = emulate_instruction(vcpu, 0, 0, 0);
+                err = emulate_instruction(vcpu, 0);
 
                 if (err == EMULATE_DO_MMIO) {
                         ret = 0;
@@ -3649,6 +3719,7 @@ static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = {
         [EXIT_REASON_MSR_WRITE]               = handle_wrmsr,
         [EXIT_REASON_PENDING_INTERRUPT]       = handle_interrupt_window,
         [EXIT_REASON_HLT]                     = handle_halt,
+        [EXIT_REASON_INVD]                    = handle_invd,
         [EXIT_REASON_INVLPG]                  = handle_invlpg,
         [EXIT_REASON_VMCALL]                  = handle_vmcall,
         [EXIT_REASON_VMCLEAR]                 = handle_vmx_insn,
@@ -3676,6 +3747,12 @@ static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = {
 static const int kvm_vmx_max_exit_handlers =
         ARRAY_SIZE(kvm_vmx_exit_handlers);
 
+static void vmx_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2)
+{
+        *info1 = vmcs_readl(EXIT_QUALIFICATION);
+        *info2 = vmcs_read32(VM_EXIT_INTR_INFO);
+}
+
 /*
  * The guest has exited.  See if we can fix it or if we need userspace
  * assistance.
@@ -3686,17 +3763,12 @@ static int vmx_handle_exit(struct kvm_vcpu *vcpu)
         u32 exit_reason = vmx->exit_reason;
         u32 vectoring_info = vmx->idt_vectoring_info;
 
-        trace_kvm_exit(exit_reason, vcpu);
+        trace_kvm_exit(exit_reason, vcpu, KVM_ISA_VMX);
 
         /* If guest state is invalid, start emulating */
         if (vmx->emulation_required && emulate_invalid_guest_state)
                 return handle_invalid_guest_state(vcpu);
 
-        /* Access CR3 don't cause VMExit in paging mode, so we need
-         * to sync with guest real CR3. */
-        if (enable_ept && is_paging(vcpu))
-                vcpu->arch.cr3 = vmcs_readl(GUEST_CR3);
-
         if (exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY) {
                 vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY;
                 vcpu->run->fail_entry.hardware_entry_failure_reason
@@ -4013,7 +4085,8 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu)
               );
 
         vcpu->arch.regs_avail = ~((1 << VCPU_REGS_RIP) | (1 << VCPU_REGS_RSP)
-                                  | (1 << VCPU_EXREG_PDPTR));
+                                  | (1 << VCPU_EXREG_PDPTR)
+                                  | (1 << VCPU_EXREG_CR3));
         vcpu->arch.regs_dirty = 0;
 
         vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD);
@@ -4280,6 +4353,7 @@ static struct kvm_x86_ops vmx_x86_ops = {
         .get_cpl = vmx_get_cpl,
         .get_cs_db_l_bits = vmx_get_cs_db_l_bits,
         .decache_cr0_guest_bits = vmx_decache_cr0_guest_bits,
+        .decache_cr3 = vmx_decache_cr3,
         .decache_cr4_guest_bits = vmx_decache_cr4_guest_bits,
         .set_cr0 = vmx_set_cr0,
         .set_cr3 = vmx_set_cr3,
@@ -4320,7 +4394,9 @@ static struct kvm_x86_ops vmx_x86_ops = {
         .get_tdp_level = get_ept_level,
         .get_mt_mask = vmx_get_mt_mask,
 
+        .get_exit_info = vmx_get_exit_info,
         .exit_reasons_str = vmx_exit_reasons_str,
+
         .get_lpage_level = vmx_get_lpage_level,
 
         .cpuid_update = vmx_cpuid_update,
@@ -4396,8 +4472,6 @@ static int __init vmx_init(void)
 
         if (enable_ept) {
                 bypass_guest_pf = 0;
-                kvm_mmu_set_base_ptes(VMX_EPT_READABLE_MASK |
-                        VMX_EPT_WRITABLE_MASK);
                 kvm_mmu_set_mask_ptes(0ull, 0ull, 0ull, 0ull,
                                 VMX_EPT_EXECUTABLE_MASK);
                 kvm_enable_tdp();
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 46a368cb651..bcc0efce85b 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -43,6 +43,7 @@
 #include <linux/slab.h>
 #include <linux/perf_event.h>
 #include <linux/uaccess.h>
+#include <linux/hash.h>
 #include <trace/events/kvm.h>
 
 #define CREATE_TRACE_POINTS
@@ -155,6 +156,13 @@ struct kvm_stats_debugfs_item debugfs_entries[] = {
 
 u64 __read_mostly host_xcr0;
 
+static inline void kvm_async_pf_hash_reset(struct kvm_vcpu *vcpu)
+{
+        int i;
+        for (i = 0; i < roundup_pow_of_two(ASYNC_PF_PER_VCPU); i++)
+                vcpu->arch.apf.gfns[i] = ~0;
+}
+
 static void kvm_on_user_return(struct user_return_notifier *urn)
 {
         unsigned slot;
@@ -326,23 +334,28 @@ void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr)
 }
 EXPORT_SYMBOL_GPL(kvm_requeue_exception);
 
-void kvm_inject_page_fault(struct kvm_vcpu *vcpu)
+void kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err)
 {
-        unsigned error_code = vcpu->arch.fault.error_code;
+        if (err)
+                kvm_inject_gp(vcpu, 0);
+        else
+                kvm_x86_ops->skip_emulated_instruction(vcpu);
+}
+EXPORT_SYMBOL_GPL(kvm_complete_insn_gp);
 
+void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault)
+{
         ++vcpu->stat.pf_guest;
-        vcpu->arch.cr2 = vcpu->arch.fault.address;
-        kvm_queue_exception_e(vcpu, PF_VECTOR, error_code);
+        vcpu->arch.cr2 = fault->address;
+        kvm_queue_exception_e(vcpu, PF_VECTOR, fault->error_code);
 }
 
-void kvm_propagate_fault(struct kvm_vcpu *vcpu)
+void kvm_propagate_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault)
 {
-        if (mmu_is_nested(vcpu) && !vcpu->arch.fault.nested)
-                vcpu->arch.nested_mmu.inject_page_fault(vcpu);
+        if (mmu_is_nested(vcpu) && !fault->nested_page_fault)
+                vcpu->arch.nested_mmu.inject_page_fault(vcpu, fault);
         else
-                vcpu->arch.mmu.inject_page_fault(vcpu);
-
-        vcpu->arch.fault.nested = false;
+                vcpu->arch.mmu.inject_page_fault(vcpu, fault);
 }
 
 void kvm_inject_nmi(struct kvm_vcpu *vcpu)
@@ -460,8 +473,8 @@ static bool pdptrs_changed(struct kvm_vcpu *vcpu)
                       (unsigned long *)&vcpu->arch.regs_avail))
                 return true;
 
-        gfn = (vcpu->arch.cr3 & ~31u) >> PAGE_SHIFT;
-        offset = (vcpu->arch.cr3 & ~31u) & (PAGE_SIZE - 1);
+        gfn = (kvm_read_cr3(vcpu) & ~31u) >> PAGE_SHIFT;
+        offset = (kvm_read_cr3(vcpu) & ~31u) & (PAGE_SIZE - 1);
         r = kvm_read_nested_guest_page(vcpu, gfn, pdpte, offset, sizeof(pdpte),
                                        PFERR_USER_MASK | PFERR_WRITE_MASK);
         if (r < 0)
@@ -506,12 +519,15 @@ int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
                 } else
 #endif
                 if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->arch.walk_mmu,
-                                                 vcpu->arch.cr3))
+                                                 kvm_read_cr3(vcpu)))
                         return 1;
         }
 
         kvm_x86_ops->set_cr0(vcpu, cr0);
 
+        if ((cr0 ^ old_cr0) & X86_CR0_PG)
+                kvm_clear_async_pf_completion_queue(vcpu);
+
         if ((cr0 ^ old_cr0) & update_bits)
                 kvm_mmu_reset_context(vcpu);
         return 0;
@@ -595,7 +611,8 @@ int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
                         return 1;
         } else if (is_paging(vcpu) && (cr4 & X86_CR4_PAE)
                    && ((cr4 ^ old_cr4) & pdptr_bits)
-                   && !load_pdptrs(vcpu, vcpu->arch.walk_mmu, vcpu->arch.cr3))
+                   && !load_pdptrs(vcpu, vcpu->arch.walk_mmu,
+                                   kvm_read_cr3(vcpu)))
                 return 1;
 
         if (cr4 & X86_CR4_VMXE)
@@ -615,7 +632,7 @@ EXPORT_SYMBOL_GPL(kvm_set_cr4);
 
 int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
 {
-        if (cr3 == vcpu->arch.cr3 && !pdptrs_changed(vcpu)) {
+        if (cr3 == kvm_read_cr3(vcpu) && !pdptrs_changed(vcpu)) {
                 kvm_mmu_sync_roots(vcpu);
                 kvm_mmu_flush_tlb(vcpu);
                 return 0;
@@ -650,12 +667,13 @@ int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
         if (unlikely(!gfn_to_memslot(vcpu->kvm, cr3 >> PAGE_SHIFT)))
                 return 1;
         vcpu->arch.cr3 = cr3;
+        __set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail);
         vcpu->arch.mmu.new_cr3(vcpu);
         return 0;
 }
 EXPORT_SYMBOL_GPL(kvm_set_cr3);
 
-int __kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
+int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
 {
         if (cr8 & CR8_RESERVED_BITS)
                 return 1;
@@ -665,12 +683,6 @@ int __kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
                 vcpu->arch.cr8 = cr8;
         return 0;
 }
-
-void kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
-{
-        if (__kvm_set_cr8(vcpu, cr8))
-                kvm_inject_gp(vcpu, 0);
-}
 EXPORT_SYMBOL_GPL(kvm_set_cr8);
 
 unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu)
@@ -775,12 +787,12 @@ EXPORT_SYMBOL_GPL(kvm_get_dr);
  * kvm-specific. Those are put in the beginning of the list.
  */
 
-#define KVM_SAVE_MSRS_BEGIN     7
+#define KVM_SAVE_MSRS_BEGIN     8
 static u32 msrs_to_save[] = {
         MSR_KVM_SYSTEM_TIME, MSR_KVM_WALL_CLOCK,
         MSR_KVM_SYSTEM_TIME_NEW, MSR_KVM_WALL_CLOCK_NEW,
         HV_X64_MSR_GUEST_OS_ID, HV_X64_MSR_HYPERCALL,
-        HV_X64_MSR_APIC_ASSIST_PAGE,
+        HV_X64_MSR_APIC_ASSIST_PAGE, MSR_KVM_ASYNC_PF_EN,
         MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
         MSR_STAR,
 #ifdef CONFIG_X86_64
@@ -830,7 +842,6 @@ static int set_efer(struct kvm_vcpu *vcpu, u64 efer)
         kvm_x86_ops->set_efer(vcpu, efer);
 
         vcpu->arch.mmu.base_role.nxe = (efer & EFER_NX) && !tdp_enabled;
-        kvm_mmu_reset_context(vcpu);
 
         /* Update reserved bits */
         if ((efer ^ old_efer) & EFER_NX)
@@ -1418,6 +1429,30 @@ static int set_msr_hyperv(struct kvm_vcpu *vcpu, u32 msr, u64 data)
         return 0;
 }
 
+static int kvm_pv_enable_async_pf(struct kvm_vcpu *vcpu, u64 data)
+{
+        gpa_t gpa = data & ~0x3f;
+
+        /* Bits 2:5 are resrved, Should be zero */
+        if (data & 0x3c)
+                return 1;
+
+        vcpu->arch.apf.msr_val = data;
+
+        if (!(data & KVM_ASYNC_PF_ENABLED)) {
+                kvm_clear_async_pf_completion_queue(vcpu);
+                kvm_async_pf_hash_reset(vcpu);
+                return 0;
+        }
+
+        if (kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.apf.data, gpa))
+                return 1;
+
+        vcpu->arch.apf.send_user_only = !(data & KVM_ASYNC_PF_SEND_ALWAYS);
+        kvm_async_pf_wakeup_all(vcpu);
+        return 0;
+}
+
 int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data)
 {
         switch (msr) {
@@ -1499,6 +1534,10 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data)
                 }
                 break;
         }
+        case MSR_KVM_ASYNC_PF_EN:
+                if (kvm_pv_enable_async_pf(vcpu, data))
+                        return 1;
+                break;
         case MSR_IA32_MCG_CTL:
         case MSR_IA32_MCG_STATUS:
         case MSR_IA32_MC0_CTL ... MSR_IA32_MC0_CTL + 4 * KVM_MAX_MCE_BANKS - 1:
@@ -1775,6 +1814,9 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
         case MSR_KVM_SYSTEM_TIME_NEW:
                 data = vcpu->arch.time;
                 break;
+        case MSR_KVM_ASYNC_PF_EN:
+                data = vcpu->arch.apf.msr_val;
+                break;
         case MSR_IA32_P5_MC_ADDR:
         case MSR_IA32_P5_MC_TYPE:
         case MSR_IA32_MCG_CAP:
@@ -1904,6 +1946,7 @@ int kvm_dev_ioctl_check_extension(long ext)
         case KVM_CAP_NOP_IO_DELAY:
         case KVM_CAP_MP_STATE:
         case KVM_CAP_SYNC_MMU:
+        case KVM_CAP_USER_NMI:
         case KVM_CAP_REINJECT_CONTROL:
         case KVM_CAP_IRQ_INJECT_STATUS:
         case KVM_CAP_ASSIGN_DEV_IRQ:
@@ -1922,6 +1965,7 @@ int kvm_dev_ioctl_check_extension(long ext)
         case KVM_CAP_DEBUGREGS:
         case KVM_CAP_X86_ROBUST_SINGLESTEP:
         case KVM_CAP_XSAVE:
+        case KVM_CAP_ASYNC_PF:
                 r = 1;
                 break;
         case KVM_CAP_COALESCED_MMIO:
@@ -2185,6 +2229,11 @@ out:
         return r;
 }
 
+static void cpuid_mask(u32 *word, int wordnum)
+{
+        *word &= boot_cpu_data.x86_capability[wordnum];
+}
+
 static void do_cpuid_1_ent(struct kvm_cpuid_entry2 *entry, u32 function,
                            u32 index)
 {
@@ -2259,7 +2308,9 @@ static void do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function,
                 break;
         case 1:
                 entry->edx &= kvm_supported_word0_x86_features;
+                cpuid_mask(&entry->edx, 0);
                 entry->ecx &= kvm_supported_word4_x86_features;
+                cpuid_mask(&entry->ecx, 4);
                 /* we support x2apic emulation even if host does not support
                  * it since we emulate x2apic in software */
                 entry->ecx |= F(X2APIC);
@@ -2350,7 +2401,9 @@ static void do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function,
                 break;
         case 0x80000001:
                 entry->edx &= kvm_supported_word1_x86_features;
+                cpuid_mask(&entry->edx, 1);
                 entry->ecx &= kvm_supported_word6_x86_features;
+                cpuid_mask(&entry->ecx, 6);
                 break;
         }
 
@@ -3169,20 +3222,18 @@ int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
                 struct kvm_memslots *slots, *old_slots;
                 unsigned long *dirty_bitmap;
 
-                r = -ENOMEM;
-                dirty_bitmap = vmalloc(n);
-                if (!dirty_bitmap)
-                        goto out;
+                dirty_bitmap = memslot->dirty_bitmap_head;
+                if (memslot->dirty_bitmap == dirty_bitmap)
+                        dirty_bitmap += n / sizeof(long);
                 memset(dirty_bitmap, 0, n);
 
                 r = -ENOMEM;
                 slots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL);
-                if (!slots) {
-                        vfree(dirty_bitmap);
+                if (!slots)
                         goto out;
-                }
                 memcpy(slots, kvm->memslots, sizeof(struct kvm_memslots));
                 slots->memslots[log->slot].dirty_bitmap = dirty_bitmap;
+                slots->generation++;
 
                 old_slots = kvm->memslots;
                 rcu_assign_pointer(kvm->memslots, slots);
@@ -3195,11 +3246,8 @@ int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
                 spin_unlock(&kvm->mmu_lock);
 
                 r = -EFAULT;
-                if (copy_to_user(log->dirty_bitmap, dirty_bitmap, n)) {
-                        vfree(dirty_bitmap);
+                if (copy_to_user(log->dirty_bitmap, dirty_bitmap, n))
                         goto out;
-                }
-                vfree(dirty_bitmap);
         } else {
                 r = -EFAULT;
                 if (clear_user(log->dirty_bitmap, n))
@@ -3266,8 +3314,10 @@ long kvm_arch_vm_ioctl(struct file *filp,
                 if (vpic) {
                         r = kvm_ioapic_init(kvm);
                         if (r) {
+                                mutex_lock(&kvm->slots_lock);
                                 kvm_io_bus_unregister_dev(kvm, KVM_PIO_BUS,
                                                           &vpic->dev);
+                                mutex_unlock(&kvm->slots_lock);
                                 kfree(vpic);
                                 goto create_irqchip_unlock;
                         }
@@ -3278,10 +3328,12 @@ long kvm_arch_vm_ioctl(struct file *filp,
                 smp_wmb();
                 r = kvm_setup_default_irq_routing(kvm);
                 if (r) {
+                        mutex_lock(&kvm->slots_lock);
                         mutex_lock(&kvm->irq_lock);
                         kvm_ioapic_destroy(kvm);
                         kvm_destroy_pic(kvm);
                         mutex_unlock(&kvm->irq_lock);
+                        mutex_unlock(&kvm->slots_lock);
                 }
         create_irqchip_unlock:
                 mutex_unlock(&kvm->lock);
@@ -3557,63 +3609,63 @@ static gpa_t translate_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access)
 static gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access)
 {
         gpa_t t_gpa;
-        u32 error;
+        struct x86_exception exception;
 
         BUG_ON(!mmu_is_nested(vcpu));
 
         /* NPT walks are always user-walks */
         access |= PFERR_USER_MASK;
-        t_gpa  = vcpu->arch.mmu.gva_to_gpa(vcpu, gpa, access, &error);
-        if (t_gpa == UNMAPPED_GVA)
-                vcpu->arch.fault.nested = true;
+        t_gpa  = vcpu->arch.mmu.gva_to_gpa(vcpu, gpa, access, &exception);
 
         return t_gpa;
 }
 
-gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva, u32 *error)
+gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva,
+                              struct x86_exception *exception)
 {
         u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
-        return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, error);
+        return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
 }
 
- gpa_t kvm_mmu_gva_to_gpa_fetch(struct kvm_vcpu *vcpu, gva_t gva, u32 *error)
+ gpa_t kvm_mmu_gva_to_gpa_fetch(struct kvm_vcpu *vcpu, gva_t gva,
+                                struct x86_exception *exception)
 {
         u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
         access |= PFERR_FETCH_MASK;
-        return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, error);
+        return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
 }
 
-gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva, u32 *error)
+gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva,
+                               struct x86_exception *exception)
 {
         u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
         access |= PFERR_WRITE_MASK;
-        return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, error);
+        return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
 }
 
 /* uses this to access any guest's mapped memory without checking CPL */
-gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva, u32 *error)
+gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva,
+                                struct x86_exception *exception)
 {
-        return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, 0, error);
+        return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, 0, exception);
 }
 
 static int kvm_read_guest_virt_helper(gva_t addr, void *val, unsigned int bytes,
                                       struct kvm_vcpu *vcpu, u32 access,
-                                      u32 *error)
+                                      struct x86_exception *exception)
 {
         void *data = val;
         int r = X86EMUL_CONTINUE;
 
         while (bytes) {
                 gpa_t gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr, access,
-                                                            error);
+                                                            exception);
                 unsigned offset = addr & (PAGE_SIZE-1);
                 unsigned toread = min(bytes, (unsigned)PAGE_SIZE - offset);
                 int ret;
 
-                if (gpa == UNMAPPED_GVA) {
-                        r = X86EMUL_PROPAGATE_FAULT;
-                        goto out;
-                }
+                if (gpa == UNMAPPED_GVA)
+                        return X86EMUL_PROPAGATE_FAULT;
                 ret = kvm_read_guest(vcpu->kvm, gpa, data, toread);
                 if (ret < 0) {
                         r = X86EMUL_IO_NEEDED;
@@ -3630,31 +3682,35 @@ out:
 
 /* used for instruction fetching */
 static int kvm_fetch_guest_virt(gva_t addr, void *val, unsigned int bytes,
-                                struct kvm_vcpu *vcpu, u32 *error)
+                                struct kvm_vcpu *vcpu,
+                                struct x86_exception *exception)
 {
         u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
         return kvm_read_guest_virt_helper(addr, val, bytes, vcpu,
-                                          access | PFERR_FETCH_MASK, error);
+                                          access | PFERR_FETCH_MASK,
+                                          exception);
 }
 
 static int kvm_read_guest_virt(gva_t addr, void *val, unsigned int bytes,
-                               struct kvm_vcpu *vcpu, u32 *error)
+                               struct kvm_vcpu *vcpu,
+                               struct x86_exception *exception)
 {
         u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
         return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access,
-                                          error);
+                                          exception);
 }
 
 static int kvm_read_guest_virt_system(gva_t addr, void *val, unsigned int bytes,
-                               struct kvm_vcpu *vcpu, u32 *error)
+                                      struct kvm_vcpu *vcpu,
+                                      struct x86_exception *exception)
 {
-        return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, 0, error);
+        return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, 0, exception);
 }
 
 static int kvm_write_guest_virt_system(gva_t addr, void *val,
                                        unsigned int bytes,
                                        struct kvm_vcpu *vcpu,
-                                       u32 *error)
+                                       struct x86_exception *exception)
 {
         void *data = val;
         int r = X86EMUL_CONTINUE;
@@ -3662,15 +3718,13 @@ static int kvm_write_guest_virt_system(gva_t addr, void *val,
         while (bytes) {
                 gpa_t gpa =  vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr,
                                                              PFERR_WRITE_MASK,
-                                                             error);
+                                                             exception);
                 unsigned offset = addr & (PAGE_SIZE-1);
                 unsigned towrite = min(bytes, (unsigned)PAGE_SIZE - offset);
                 int ret;
 
-                if (gpa == UNMAPPED_GVA) {
-                        r = X86EMUL_PROPAGATE_FAULT;
-                        goto out;
-                }
+                if (gpa == UNMAPPED_GVA)
+                        return X86EMUL_PROPAGATE_FAULT;
                 ret = kvm_write_guest(vcpu->kvm, gpa, data, towrite);
                 if (ret < 0) {
                         r = X86EMUL_IO_NEEDED;
@@ -3688,7 +3742,7 @@ out:
 static int emulator_read_emulated(unsigned long addr,
                                   void *val,
                                   unsigned int bytes,
-                                  unsigned int *error_code,
+                                  struct x86_exception *exception,
                                   struct kvm_vcpu *vcpu)
 {
         gpa_t                 gpa;
@@ -3701,7 +3755,7 @@ static int emulator_read_emulated(unsigned long addr,
                 return X86EMUL_CONTINUE;
         }
 
-        gpa = kvm_mmu_gva_to_gpa_read(vcpu, addr, error_code);
+        gpa = kvm_mmu_gva_to_gpa_read(vcpu, addr, exception);
 
         if (gpa == UNMAPPED_GVA)
                 return X86EMUL_PROPAGATE_FAULT;
@@ -3710,8 +3764,8 @@ static int emulator_read_emulated(unsigned long addr,
         if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
                 goto mmio;
 
-        if (kvm_read_guest_virt(addr, val, bytes, vcpu, NULL)
-                                == X86EMUL_CONTINUE)
+        if (kvm_read_guest_virt(addr, val, bytes, vcpu, exception)
+            == X86EMUL_CONTINUE)
                 return X86EMUL_CONTINUE;
 
 mmio:
@@ -3735,7 +3789,7 @@ mmio:
 }
 
 int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
-                          const void *val, int bytes)
+                        const void *val, int bytes)
 {
         int ret;
 
@@ -3749,12 +3803,12 @@ int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
 static int emulator_write_emulated_onepage(unsigned long addr,
                                            const void *val,
                                            unsigned int bytes,
-                                           unsigned int *error_code,
+                                           struct x86_exception *exception,
                                            struct kvm_vcpu *vcpu)
 {
         gpa_t                 gpa;
 
-        gpa = kvm_mmu_gva_to_gpa_write(vcpu, addr, error_code);
+        gpa = kvm_mmu_gva_to_gpa_write(vcpu, addr, exception);
 
         if (gpa == UNMAPPED_GVA)
                 return X86EMUL_PROPAGATE_FAULT;
@@ -3787,7 +3841,7 @@ mmio:
 int emulator_write_emulated(unsigned long addr,
                             const void *val,
                             unsigned int bytes,
-                            unsigned int *error_code,
+                            struct x86_exception *exception,
                             struct kvm_vcpu *vcpu)
 {
         /* Crossing a page boundary? */
@@ -3795,7 +3849,7 @@ int emulator_write_emulated(unsigned long addr,
                 int rc, now;
 
                 now = -addr & ~PAGE_MASK;
-                rc = emulator_write_emulated_onepage(addr, val, now, error_code,
+                rc = emulator_write_emulated_onepage(addr, val, now, exception,
                                                      vcpu);
                 if (rc != X86EMUL_CONTINUE)
                         return rc;
@@ -3803,7 +3857,7 @@ int emulator_write_emulated(unsigned long addr,
                 val += now;
                 bytes -= now;
         }
-        return emulator_write_emulated_onepage(addr, val, bytes, error_code,
+        return emulator_write_emulated_onepage(addr, val, bytes, exception,
                                                vcpu);
 }
 
@@ -3821,7 +3875,7 @@ static int emulator_cmpxchg_emulated(unsigned long addr,
                                      const void *old,
                                      const void *new,
                                      unsigned int bytes,
-                                     unsigned int *error_code,
+                                     struct x86_exception *exception,
                                      struct kvm_vcpu *vcpu)
 {
         gpa_t gpa;
@@ -3879,7 +3933,7 @@ static int emulator_cmpxchg_emulated(unsigned long addr,
 emul_write:
         printk_once(KERN_WARNING "kvm: emulating exchange as write\n");
 
-        return emulator_write_emulated(addr, new, bytes, error_code, vcpu);
+        return emulator_write_emulated(addr, new, bytes, exception, vcpu);
 }
 
 static int kernel_pio(struct kvm_vcpu *vcpu, void *pd)
@@ -3904,7 +3958,7 @@ static int emulator_pio_in_emulated(int size, unsigned short port, void *val,
         if (vcpu->arch.pio.count)
                 goto data_avail;
 
-        trace_kvm_pio(0, port, size, 1);
+        trace_kvm_pio(0, port, size, count);
 
         vcpu->arch.pio.port = port;
         vcpu->arch.pio.in = 1;
@@ -3932,7 +3986,7 @@ static int emulator_pio_out_emulated(int size, unsigned short port,
                               const void *val, unsigned int count,
                               struct kvm_vcpu *vcpu)
 {
-        trace_kvm_pio(1, port, size, 1);
+        trace_kvm_pio(1, port, size, count);
 
         vcpu->arch.pio.port = port;
         vcpu->arch.pio.in = 0;
@@ -3973,13 +4027,15 @@ int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu)
                 return X86EMUL_CONTINUE;
 
         if (kvm_x86_ops->has_wbinvd_exit()) {
-                preempt_disable();
+                int cpu = get_cpu();
+
+                cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);
                 smp_call_function_many(vcpu->arch.wbinvd_dirty_mask,
                                 wbinvd_ipi, NULL, 1);
-                preempt_enable();
+                put_cpu();
                 cpumask_clear(vcpu->arch.wbinvd_dirty_mask);
-        }
-        wbinvd();
+        } else
+                wbinvd();
         return X86EMUL_CONTINUE;
 }
 EXPORT_SYMBOL_GPL(kvm_emulate_wbinvd);
@@ -4019,7 +4075,7 @@ static unsigned long emulator_get_cr(int cr, struct kvm_vcpu *vcpu)
                 value = vcpu->arch.cr2;
                 break;
         case 3:
-                value = vcpu->arch.cr3;
+                value = kvm_read_cr3(vcpu);
                 break;
         case 4:
                 value = kvm_read_cr4(vcpu);
@@ -4053,7 +4109,7 @@ static int emulator_set_cr(int cr, unsigned long val, struct kvm_vcpu *vcpu)
                 res = kvm_set_cr4(vcpu, mk_cr_64(kvm_read_cr4(vcpu), val));
                 break;
         case 8:
-                res = __kvm_set_cr8(vcpu, val & 0xfUL);
+                res = kvm_set_cr8(vcpu, val);
                 break;
         default:
                 vcpu_printf(vcpu, "%s: unexpected cr %u\n", __func__, cr);
@@ -4206,12 +4262,13 @@ static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask)
 static void inject_emulated_exception(struct kvm_vcpu *vcpu)
 {
         struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
-        if (ctxt->exception == PF_VECTOR)
-                kvm_propagate_fault(vcpu);
-        else if (ctxt->error_code_valid)
-                kvm_queue_exception_e(vcpu, ctxt->exception, ctxt->error_code);
+        if (ctxt->exception.vector == PF_VECTOR)
+                kvm_propagate_fault(vcpu, &ctxt->exception);
+        else if (ctxt->exception.error_code_valid)
+                kvm_queue_exception_e(vcpu, ctxt->exception.vector,
+                                      ctxt->exception.error_code);
         else
-                kvm_queue_exception(vcpu, ctxt->exception);
+                kvm_queue_exception(vcpu, ctxt->exception.vector);
 }
 
 static void init_emulate_ctxt(struct kvm_vcpu *vcpu)
@@ -4267,13 +4324,19 @@ EXPORT_SYMBOL_GPL(kvm_inject_realmode_interrupt);
 
 static int handle_emulation_failure(struct kvm_vcpu *vcpu)
 {
+        int r = EMULATE_DONE;
+
         ++vcpu->stat.insn_emulation_fail;
         trace_kvm_emulate_insn_failed(vcpu);
-        vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
-        vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
-        vcpu->run->internal.ndata = 0;
+        if (!is_guest_mode(vcpu)) {
+                vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
+                vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
+                vcpu->run->internal.ndata = 0;
+                r = EMULATE_FAIL;
+        }
         kvm_queue_exception(vcpu, UD_VECTOR);
-        return EMULATE_FAIL;
+
+        return r;
 }
 
 static bool reexecute_instruction(struct kvm_vcpu *vcpu, gva_t gva)
@@ -4302,10 +4365,11 @@ static bool reexecute_instruction(struct kvm_vcpu *vcpu, gva_t gva)
         return false;
 }
 
-int emulate_instruction(struct kvm_vcpu *vcpu,
-                        unsigned long cr2,
-                        u16 error_code,
-                        int emulation_type)
+int x86_emulate_instruction(struct kvm_vcpu *vcpu,
+                            unsigned long cr2,
+                            int emulation_type,
+                            void *insn,
+                            int insn_len)
 {
         int r;
         struct decode_cache *c = &vcpu->arch.emulate_ctxt.decode;
@@ -4323,10 +4387,10 @@ int emulate_instruction(struct kvm_vcpu *vcpu,
         if (!(emulation_type & EMULTYPE_NO_DECODE)) {
                 init_emulate_ctxt(vcpu);
                 vcpu->arch.emulate_ctxt.interruptibility = 0;
-                vcpu->arch.emulate_ctxt.exception = -1;
+                vcpu->arch.emulate_ctxt.have_exception = false;
                 vcpu->arch.emulate_ctxt.perm_ok = false;
 
-                r = x86_decode_insn(&vcpu->arch.emulate_ctxt);
+                r = x86_decode_insn(&vcpu->arch.emulate_ctxt, insn, insn_len);
                 if (r == X86EMUL_PROPAGATE_FAULT)
                         goto done;
 
@@ -4389,7 +4453,7 @@ restart:
         }
 
 done:
-        if (vcpu->arch.emulate_ctxt.exception >= 0) {
+        if (vcpu->arch.emulate_ctxt.have_exception) {
                 inject_emulated_exception(vcpu);
                 r = EMULATE_DONE;
         } else if (vcpu->arch.pio.count) {
@@ -4413,7 +4477,7 @@ done:
 
         return r;
 }
-EXPORT_SYMBOL_GPL(emulate_instruction);
+EXPORT_SYMBOL_GPL(x86_emulate_instruction);
 
 int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size, unsigned short port)
 {
@@ -4653,7 +4717,6 @@ int kvm_arch_init(void *opaque)
 
         kvm_x86_ops = ops;
         kvm_mmu_set_nonpresent_ptes(0ull, 0ull);
-        kvm_mmu_set_base_ptes(PT_PRESENT_MASK);
         kvm_mmu_set_mask_ptes(PT_USER_MASK, PT_ACCESSED_MASK,
                         PT_DIRTY_MASK, PT64_NX_MASK, 0);
 
@@ -5116,6 +5179,12 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
                         vcpu->fpu_active = 0;
                         kvm_x86_ops->fpu_deactivate(vcpu);
                 }
+                if (kvm_check_request(KVM_REQ_APF_HALT, vcpu)) {
+                        /* Page is swapped out. Do synthetic halt */
+                        vcpu->arch.apf.halted = true;
+                        r = 1;
+                        goto out;
+                }
         }
 
         r = kvm_mmu_reload(vcpu);
@@ -5244,7 +5313,8 @@ static int __vcpu_run(struct kvm_vcpu *vcpu)
 
         r = 1;
         while (r > 0) {
-                if (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE)
+                if (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE &&
+                    !vcpu->arch.apf.halted)
                         r = vcpu_enter_guest(vcpu);
                 else {
                         srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
@@ -5257,6 +5327,7 @@ static int __vcpu_run(struct kvm_vcpu *vcpu)
                                         vcpu->arch.mp_state =
                                                 KVM_MP_STATE_RUNNABLE;
                                 case KVM_MP_STATE_RUNNABLE:
+                                        vcpu->arch.apf.halted = false;
                                         break;
                                 case KVM_MP_STATE_SIPI_RECEIVED:
                                 default:
@@ -5278,6 +5349,9 @@ static int __vcpu_run(struct kvm_vcpu *vcpu)
                         vcpu->run->exit_reason = KVM_EXIT_INTR;
                         ++vcpu->stat.request_irq_exits;
                 }
+
+                kvm_check_async_pf_completion(vcpu);
+
                 if (signal_pending(current)) {
                         r = -EINTR;
                         vcpu->run->exit_reason = KVM_EXIT_INTR;
@@ -5302,6 +5376,9 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
         int r;
         sigset_t sigsaved;
 
+        if (!tsk_used_math(current) && init_fpu(current))
+                return -ENOMEM;
+
         if (vcpu->sigset_active)
                 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
 
@@ -5313,8 +5390,12 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
         }
 
         /* re-sync apic's tpr */
-        if (!irqchip_in_kernel(vcpu->kvm))
-                kvm_set_cr8(vcpu, kvm_run->cr8);
+        if (!irqchip_in_kernel(vcpu->kvm)) {
+                if (kvm_set_cr8(vcpu, kvm_run->cr8) != 0) {
+                        r = -EINVAL;
+                        goto out;
+                }
+        }
 
         if (vcpu->arch.pio.count || vcpu->mmio_needed) {
                 if (vcpu->mmio_needed) {
@@ -5323,7 +5404,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
                         vcpu->mmio_needed = 0;
                 }
                 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
-                r = emulate_instruction(vcpu, 0, 0, EMULTYPE_NO_DECODE);
+                r = emulate_instruction(vcpu, EMULTYPE_NO_DECODE);
                 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
                 if (r != EMULATE_DONE) {
                         r = 0;
@@ -5436,7 +5517,7 @@ int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
 
         sregs->cr0 = kvm_read_cr0(vcpu);
         sregs->cr2 = vcpu->arch.cr2;
-        sregs->cr3 = vcpu->arch.cr3;
+        sregs->cr3 = kvm_read_cr3(vcpu);
         sregs->cr4 = kvm_read_cr4(vcpu);
         sregs->cr8 = kvm_get_cr8(vcpu);
         sregs->efer = vcpu->arch.efer;
@@ -5504,8 +5585,9 @@ int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
         kvm_x86_ops->set_gdt(vcpu, &dt);
 
         vcpu->arch.cr2 = sregs->cr2;
-        mmu_reset_needed |= vcpu->arch.cr3 != sregs->cr3;
+        mmu_reset_needed |= kvm_read_cr3(vcpu) != sregs->cr3;
         vcpu->arch.cr3 = sregs->cr3;
+        __set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail);
 
         kvm_set_cr8(vcpu, sregs->cr8);
 
@@ -5522,7 +5604,7 @@ int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
         if (sregs->cr4 & X86_CR4_OSXSAVE)
                 update_cpuid(vcpu);
         if (!is_long_mode(vcpu) && is_pae(vcpu)) {
-                load_pdptrs(vcpu, vcpu->arch.walk_mmu, vcpu->arch.cr3);
+                load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu));
                 mmu_reset_needed = 1;
         }
 
@@ -5773,6 +5855,8 @@ free_vcpu:
 
 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
 {
+        vcpu->arch.apf.msr_val = 0;
+
         vcpu_load(vcpu);
         kvm_mmu_unload(vcpu);
         vcpu_put(vcpu);
@@ -5792,6 +5876,11 @@ int kvm_arch_vcpu_reset(struct kvm_vcpu *vcpu)
         vcpu->arch.dr7 = DR7_FIXED_1;
 
         kvm_make_request(KVM_REQ_EVENT, vcpu);
+        vcpu->arch.apf.msr_val = 0;
+
+        kvm_clear_async_pf_completion_queue(vcpu);
+        kvm_async_pf_hash_reset(vcpu);
+        vcpu->arch.apf.halted = false;
 
         return kvm_x86_ops->vcpu_reset(vcpu);
 }
@@ -5881,6 +5970,8 @@ int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
         if (!zalloc_cpumask_var(&vcpu->arch.wbinvd_dirty_mask, GFP_KERNEL))
                 goto fail_free_mce_banks;
 
+        kvm_async_pf_hash_reset(vcpu);
+
         return 0;
 fail_free_mce_banks:
         kfree(vcpu->arch.mce_banks);
@@ -5906,13 +5997,8 @@ void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu)
         free_page((unsigned long)vcpu->arch.pio_data);
 }
 
-struct  kvm *kvm_arch_create_vm(void)
+int kvm_arch_init_vm(struct kvm *kvm)
 {
-        struct kvm *kvm = kzalloc(sizeof(struct kvm), GFP_KERNEL);
-
-        if (!kvm)
-                return ERR_PTR(-ENOMEM);
-
         INIT_LIST_HEAD(&kvm->arch.active_mmu_pages);
         INIT_LIST_HEAD(&kvm->arch.assigned_dev_head);
 
@@ -5921,7 +6007,7 @@ struct  kvm *kvm_arch_create_vm(void)
 
         spin_lock_init(&kvm->arch.tsc_write_lock);
 
-        return kvm;
+        return 0;
 }
 
 static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu)
@@ -5939,8 +6025,10 @@ static void kvm_free_vcpus(struct kvm *kvm)
         /*
          * Unpin any mmu pages first.
          */
-        kvm_for_each_vcpu(i, vcpu, kvm)
+        kvm_for_each_vcpu(i, vcpu, kvm) {
+                kvm_clear_async_pf_completion_queue(vcpu);
                 kvm_unload_vcpu_mmu(vcpu);
+        }
         kvm_for_each_vcpu(i, vcpu, kvm)
                 kvm_arch_vcpu_free(vcpu);
 
@@ -5964,13 +6052,10 @@ void kvm_arch_destroy_vm(struct kvm *kvm)
         kfree(kvm->arch.vpic);
         kfree(kvm->arch.vioapic);
         kvm_free_vcpus(kvm);
-        kvm_free_physmem(kvm);
         if (kvm->arch.apic_access_page)
                 put_page(kvm->arch.apic_access_page);
         if (kvm->arch.ept_identity_pagetable)
                 put_page(kvm->arch.ept_identity_pagetable);
-        cleanup_srcu_struct(&kvm->srcu);
-        kfree(kvm);
 }
 
 int kvm_arch_prepare_memory_region(struct kvm *kvm,
@@ -6051,7 +6136,9 @@ void kvm_arch_flush_shadow(struct kvm *kvm)
 
 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
 {
-        return vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE
+        return (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE &&
+                !vcpu->arch.apf.halted)
+                || !list_empty_careful(&vcpu->async_pf.done)
                 || vcpu->arch.mp_state == KVM_MP_STATE_SIPI_RECEIVED
                 || vcpu->arch.nmi_pending ||
                 (kvm_arch_interrupt_allowed(vcpu) &&
@@ -6110,6 +6197,147 @@ void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
 }
 EXPORT_SYMBOL_GPL(kvm_set_rflags);
 
+void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, struct kvm_async_pf *work)
+{
+        int r;
+
+        if ((vcpu->arch.mmu.direct_map != work->arch.direct_map) ||
+              is_error_page(work->page))
+                return;
+
+        r = kvm_mmu_reload(vcpu);
+        if (unlikely(r))
+                return;
+
+        if (!vcpu->arch.mmu.direct_map &&
+              work->arch.cr3 != vcpu->arch.mmu.get_cr3(vcpu))
+                return;
+
+        vcpu->arch.mmu.page_fault(vcpu, work->gva, 0, true);
+}
+
+static inline u32 kvm_async_pf_hash_fn(gfn_t gfn)
+{
+        return hash_32(gfn & 0xffffffff, order_base_2(ASYNC_PF_PER_VCPU));
+}
+
+static inline u32 kvm_async_pf_next_probe(u32 key)
+{
+        return (key + 1) & (roundup_pow_of_two(ASYNC_PF_PER_VCPU) - 1);
+}
+
+static void kvm_add_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
+{
+        u32 key = kvm_async_pf_hash_fn(gfn);
+
+        while (vcpu->arch.apf.gfns[key] != ~0)
+                key = kvm_async_pf_next_probe(key);
+
+        vcpu->arch.apf.gfns[key] = gfn;
+}
+
+static u32 kvm_async_pf_gfn_slot(struct kvm_vcpu *vcpu, gfn_t gfn)
+{
+        int i;
+        u32 key = kvm_async_pf_hash_fn(gfn);
+
+        for (i = 0; i < roundup_pow_of_two(ASYNC_PF_PER_VCPU) &&
+                     (vcpu->arch.apf.gfns[key] != gfn &&
+                      vcpu->arch.apf.gfns[key] != ~0); i++)
+                key = kvm_async_pf_next_probe(key);
+
+        return key;
+}
+
+bool kvm_find_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
+{
+        return vcpu->arch.apf.gfns[kvm_async_pf_gfn_slot(vcpu, gfn)] == gfn;
+}
+
+static void kvm_del_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
+{
+        u32 i, j, k;
+
+        i = j = kvm_async_pf_gfn_slot(vcpu, gfn);
+        while (true) {
+                vcpu->arch.apf.gfns[i] = ~0;
+                do {
+                        j = kvm_async_pf_next_probe(j);
+                        if (vcpu->arch.apf.gfns[j] == ~0)
+                                return;
+                        k = kvm_async_pf_hash_fn(vcpu->arch.apf.gfns[j]);
+                        /*
+                         * k lies cyclically in ]i,j]
+                         * |    i.k.j |
+                         * |....j i.k.| or  |.k..j i...|
+                         */
+                } while ((i <= j) ? (i < k && k <= j) : (i < k || k <= j));
+                vcpu->arch.apf.gfns[i] = vcpu->arch.apf.gfns[j];
+                i = j;
+        }
+}
+
+static int apf_put_user(struct kvm_vcpu *vcpu, u32 val)
+{
+
+        return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apf.data, &val,
+                                      sizeof(val));
+}
+
+void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
+                                     struct kvm_async_pf *work)
+{
+        struct x86_exception fault;
+
+        trace_kvm_async_pf_not_present(work->arch.token, work->gva);
+        kvm_add_async_pf_gfn(vcpu, work->arch.gfn);
+
+        if (!(vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED) ||
+            (vcpu->arch.apf.send_user_only &&
+             kvm_x86_ops->get_cpl(vcpu) == 0))
+                kvm_make_request(KVM_REQ_APF_HALT, vcpu);
+        else if (!apf_put_user(vcpu, KVM_PV_REASON_PAGE_NOT_PRESENT)) {
+                fault.vector = PF_VECTOR;
+                fault.error_code_valid = true;
+                fault.error_code = 0;
+                fault.nested_page_fault = false;
+                fault.address = work->arch.token;
+                kvm_inject_page_fault(vcpu, &fault);
+        }
+}
+
+void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
+                                 struct kvm_async_pf *work)
+{
+        struct x86_exception fault;
+
+        trace_kvm_async_pf_ready(work->arch.token, work->gva);
+        if (is_error_page(work->page))
+                work->arch.token = ~0; /* broadcast wakeup */
+        else
+                kvm_del_async_pf_gfn(vcpu, work->arch.gfn);
+
+        if ((vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED) &&
+            !apf_put_user(vcpu, KVM_PV_REASON_PAGE_READY)) {
+                fault.vector = PF_VECTOR;
+                fault.error_code_valid = true;
+                fault.error_code = 0;
+                fault.nested_page_fault = false;
+                fault.address = work->arch.token;
+                kvm_inject_page_fault(vcpu, &fault);
+        }
+        vcpu->arch.apf.halted = false;
+}
+
+bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
+{
+        if (!(vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED))
+                return true;
+        else
+                return !kvm_event_needs_reinjection(vcpu) &&
+                        kvm_x86_ops->interrupt_allowed(vcpu);
+}
+
 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_exit);
 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_inj_virq);
 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_page_fault);
diff --git a/arch/x86/mm/init_32.c b/arch/x86/mm/init_32.c
index f89b5bb4e93..c821074b7f0 100644
--- a/arch/x86/mm/init_32.c
+++ b/arch/x86/mm/init_32.c
@@ -45,6 +45,7 @@
 #include <asm/bugs.h>
 #include <asm/tlb.h>
 #include <asm/tlbflush.h>
+#include <asm/olpc_ofw.h>
 #include <asm/pgalloc.h>
 #include <asm/sections.h>
 #include <asm/paravirt.h>
@@ -715,6 +716,7 @@ void __init paging_init(void)
         /*
          * NOTE: at this point the bootmem allocator is fully available.
          */
+        olpc_dt_build_devicetree();
         sparse_init();
         zone_sizes_init();
 }
diff --git a/arch/x86/platform/mrst/early_printk_mrst.c b/arch/x86/platform/mrst/early_printk_mrst.c
index 65df603622b..25bfdbb5b13 100644
--- a/arch/x86/platform/mrst/early_printk_mrst.c
+++ b/arch/x86/platform/mrst/early_printk_mrst.c
@@ -103,7 +103,7 @@ struct dw_spi_reg {
 static unsigned long mrst_spi_paddr = MRST_REGBASE_SPI0;
 
 static u32 *pclk_spi0;
-/* Always contains an accessable address, start with 0 */
+/* Always contains an accessible address, start with 0 */
 static struct dw_spi_reg *pspi;
 
 static struct kmsg_dumper dw_dumper;
diff --git a/arch/x86/platform/olpc/Makefile b/arch/x86/platform/olpc/Makefile
index c31b8fcb5a8..e797428b163 100644
--- a/arch/x86/platform/olpc/Makefile
+++ b/arch/x86/platform/olpc/Makefile
@@ -1,3 +1,4 @@
 obj-$(CONFIG_OLPC)              += olpc.o
 obj-$(CONFIG_OLPC_XO1)          += olpc-xo1.o
 obj-$(CONFIG_OLPC_OPENFIRMWARE) += olpc_ofw.o
+obj-$(CONFIG_OLPC_OPENFIRMWARE_DT)      += olpc_dt.o
diff --git a/arch/x86/platform/olpc/olpc_dt.c b/arch/x86/platform/olpc/olpc_dt.c
new file mode 100644
index 00000000000..dab87464753
--- /dev/null
+++ b/arch/x86/platform/olpc/olpc_dt.c
@@ -0,0 +1,183 @@
+/*
+ * OLPC-specific OFW device tree support code.
+ *
+ * Paul Mackerras       August 1996.
+ * Copyright (C) 1996-2005 Paul Mackerras.
+ *
+ *  Adapted for 64bit PowerPC by Dave Engebretsen and Peter Bergner.
+ *    {engebret|bergner}@us.ibm.com
+ *
+ *  Adapted for sparc by David S. Miller davem@davemloft.net
+ *  Adapted for x86/OLPC by Andres Salomon <dilinger@queued.net>
+ *
+ *      This program is free software; you can redistribute it and/or
+ *      modify it under the terms of the GNU General Public License
+ *      as published by the Free Software Foundation; either version
+ *      2 of the License, or (at your option) any later version.
+ */
+
+#include <linux/kernel.h>
+#include <linux/bootmem.h>
+#include <linux/of.h>
+#include <linux/of_pdt.h>
+#include <asm/olpc_ofw.h>
+
+static phandle __init olpc_dt_getsibling(phandle node)
+{
+        const void *args[] = { (void *)node };
+        void *res[] = { &node };
+
+        if ((s32)node == -1)
+                return 0;
+
+        if (olpc_ofw("peer", args, res) || (s32)node == -1)
+                return 0;
+
+        return node;
+}
+
+static phandle __init olpc_dt_getchild(phandle node)
+{
+        const void *args[] = { (void *)node };
+        void *res[] = { &node };
+
+        if ((s32)node == -1)
+                return 0;
+
+        if (olpc_ofw("child", args, res) || (s32)node == -1) {
+                pr_err("PROM: %s: fetching child failed!\n", __func__);
+                return 0;
+        }
+
+        return node;
+}
+
+static int __init olpc_dt_getproplen(phandle node, const char *prop)
+{
+        const void *args[] = { (void *)node, prop };
+        int len;
+        void *res[] = { &len };
+
+        if ((s32)node == -1)
+                return -1;
+
+        if (olpc_ofw("getproplen", args, res)) {
+                pr_err("PROM: %s: getproplen failed!\n", __func__);
+                return -1;
+        }
+
+        return len;
+}
+
+static int __init olpc_dt_getproperty(phandle node, const char *prop,
+                char *buf, int bufsize)
+{
+        int plen;
+
+        plen = olpc_dt_getproplen(node, prop);
+        if (plen > bufsize || plen < 1) {
+                return -1;
+        } else {
+                const void *args[] = { (void *)node, prop, buf, (void *)plen };
+                void *res[] = { &plen };
+
+                if (olpc_ofw("getprop", args, res)) {
+                        pr_err("PROM: %s: getprop failed!\n", __func__);
+                        return -1;
+                }
+        }
+
+        return plen;
+}
+
+static int __init olpc_dt_nextprop(phandle node, char *prev, char *buf)
+{
+        const void *args[] = { (void *)node, prev, buf };
+        int success;
+        void *res[] = { &success };
+
+        buf[0] = '\0';
+
+        if ((s32)node == -1)
+                return -1;
+
+        if (olpc_ofw("nextprop", args, res) || success != 1)
+                return -1;
+
+        return 0;
+}
+
+static int __init olpc_dt_pkg2path(phandle node, char *buf,
+                const int buflen, int *len)
+{
+        const void *args[] = { (void *)node, buf, (void *)buflen };
+        void *res[] = { len };
+
+        if ((s32)node == -1)
+                return -1;
+
+        if (olpc_ofw("package-to-path", args, res) || *len < 1)
+                return -1;
+
+        return 0;
+}
+
+static unsigned int prom_early_allocated __initdata;
+
+void * __init prom_early_alloc(unsigned long size)
+{
+        static u8 *mem;
+        static size_t free_mem;
+        void *res;
+
+        if (free_mem < size) {
+                const size_t chunk_size = max(PAGE_SIZE, size);
+
+                /*
+                 * To mimimize the number of allocations, grab at least
+                 * PAGE_SIZE of memory (that's an arbitrary choice that's
+                 * fast enough on the platforms we care about while minimizing
+                 * wasted bootmem) and hand off chunks of it to callers.
+                 */
+                res = alloc_bootmem(chunk_size);
+                if (!res)
+                        return NULL;
+                prom_early_allocated += chunk_size;
+                memset(res, 0, chunk_size);
+                free_mem = chunk_size;
+                mem = res;
+        }
+
+        /* allocate from the local cache */
+        free_mem -= size;
+        res = mem;
+        mem += size;
+        return res;
+}
+
+static struct of_pdt_ops prom_olpc_ops __initdata = {
+        .nextprop = olpc_dt_nextprop,
+        .getproplen = olpc_dt_getproplen,
+        .getproperty = olpc_dt_getproperty,
+        .getchild = olpc_dt_getchild,
+        .getsibling = olpc_dt_getsibling,
+        .pkg2path = olpc_dt_pkg2path,
+};
+
+void __init olpc_dt_build_devicetree(void)
+{
+        phandle root;
+
+        if (!olpc_ofw_is_installed())
+                return;
+
+        root = olpc_dt_getsibling(0);
+        if (!root) {
+                pr_err("PROM: unable to get root node from OFW!\n");
+                return;
+        }
+        of_pdt_build_devicetree(root, &prom_olpc_ops);
+
+        pr_info("PROM DT: Built device tree with %u bytes of memory.\n",
+                        prom_early_allocated);
+}
diff --git a/arch/x86/platform/olpc/olpc_ofw.c b/arch/x86/platform/olpc/olpc_ofw.c
index 78732046437..e7604f62870 100644
--- a/arch/x86/platform/olpc/olpc_ofw.c
+++ b/arch/x86/platform/olpc/olpc_ofw.c
@@ -110,3 +110,8 @@ void __init olpc_ofw_detect(void)
                         (unsigned long)olpc_ofw_cif, (-start) >> 20);
         reserve_top_address(-start);
 }
+
+bool __init olpc_ofw_is_installed(void)
+{
+        return olpc_ofw_cif != NULL;
+}