325 files changed, 8357 insertions, 2237 deletions

diff --git a/arch/alpha/include/asm/mman.h b/arch/alpha/include/asm/mman.h
index 99c56d47879d..72db984f8781 100644
--- a/arch/alpha/include/asm/mman.h
+++ b/arch/alpha/include/asm/mman.h
@@ -53,6 +53,9 @@
 #define MADV_MERGEABLE   12             /* KSM may merge identical pages */
 #define MADV_UNMERGEABLE 13             /* KSM may not merge identical pages */
 
+#define MADV_HUGEPAGE   14              /* Worth backing with hugepages */
+#define MADV_NOHUGEPAGE 15              /* Not worth backing with hugepages */
+
 /* compatibility flags */
 #define MAP_FILE        0
 
diff --git a/arch/alpha/kernel/time.c b/arch/alpha/kernel/time.c
index 0f1d8493cfca..c1f3e7cb82a4 100644
--- a/arch/alpha/kernel/time.c
+++ b/arch/alpha/kernel/time.c
@@ -506,7 +506,7 @@ set_rtc_mmss(unsigned long nowtime)
                 CMOS_WRITE(real_seconds,RTC_SECONDS);
                 CMOS_WRITE(real_minutes,RTC_MINUTES);
         } else {
-                printk(KERN_WARNING
+                printk_once(KERN_NOTICE
                        "set_rtc_mmss: can't update from %d to %d\n",
                        cmos_minutes, real_minutes);
                 retval = -1;
diff --git a/arch/arm/common/it8152.c b/arch/arm/common/it8152.c
index 42ff90b46dfb..665ebf7e62a6 100644
--- a/arch/arm/common/it8152.c
+++ b/arch/arm/common/it8152.c
@@ -236,7 +236,7 @@ static struct resource it8152_mem = {
 
 /*
  * The following functions are needed for DMA bouncing.
- * ITE8152 chip can addrees up to 64MByte, so all the devices
+ * ITE8152 chip can address up to 64MByte, so all the devices
  * connected to ITE8152 (PCI and USB) should have limited DMA window
  */
 
diff --git a/arch/arm/common/vic.c b/arch/arm/common/vic.c
index ba65f6eedca6..cb660bc54d7a 100644
--- a/arch/arm/common/vic.c
+++ b/arch/arm/common/vic.c
@@ -70,7 +70,7 @@ static inline struct vic_device *to_vic(struct sys_device *sys)
  * vic_init2 - common initialisation code
  * @base: Base of the VIC.
  *
- * Common initialisation code for registeration
+ * Common initialisation code for registration
  * and resume.
 */
 static void vic_init2(void __iomem *base)
diff --git a/arch/arm/kernel/module.c b/arch/arm/kernel/module.c
index 0c1bb68ff4a8..2cfe8161b478 100644
--- a/arch/arm/kernel/module.c
+++ b/arch/arm/kernel/module.c
@@ -38,17 +38,9 @@
 #ifdef CONFIG_MMU
 void *module_alloc(unsigned long size)
 {
-        struct vm_struct *area;
-
-        size = PAGE_ALIGN(size);
-        if (!size)
-                return NULL;
-
-        area = __get_vm_area(size, VM_ALLOC, MODULES_VADDR, MODULES_END);
-        if (!area)
-                return NULL;
-
-        return __vmalloc_area(area, GFP_KERNEL, PAGE_KERNEL_EXEC);
+        return __vmalloc_node_range(size, 1, MODULES_VADDR, MODULES_END,
+                                GFP_KERNEL, PAGE_KERNEL_EXEC, -1,
+                                __builtin_return_address(0));
 }
 #else /* CONFIG_MMU */
 void *module_alloc(unsigned long size)
diff --git a/arch/arm/mach-at91/board-ecbat91.c b/arch/arm/mach-at91/board-ecbat91.c
index 7b58c948a957..de2fd04e7c8a 100644
--- a/arch/arm/mach-at91/board-ecbat91.c
+++ b/arch/arm/mach-at91/board-ecbat91.c
@@ -128,17 +128,17 @@ static struct spi_board_info __initdata ecb_at91spi_devices[] = {
                 .platform_data  = &my_flash0_platform,
 #endif
         },
-        {       /* User accessable spi - cs1 (250KHz) */
+        {       /* User accessible spi - cs1 (250KHz) */
                 .modalias       = "spi-cs1",
                 .chip_select    = 1,
                 .max_speed_hz   = 250 * 1000,
         },
-        {       /* User accessable spi - cs2 (1MHz) */
+        {       /* User accessible spi - cs2 (1MHz) */
                 .modalias       = "spi-cs2",
                 .chip_select    = 2,
                 .max_speed_hz   = 1 * 1000 * 1000,
         },
-        {       /* User accessable spi - cs3 (10MHz) */
+        {       /* User accessible spi - cs3 (10MHz) */
                 .modalias       = "spi-cs3",
                 .chip_select    = 3,
                 .max_speed_hz   = 10 * 1000 * 1000,
diff --git a/arch/arm/mach-at91/pm.c b/arch/arm/mach-at91/pm.c
index dafbacc25eb1..ea53f4d9b283 100644
--- a/arch/arm/mach-at91/pm.c
+++ b/arch/arm/mach-at91/pm.c
@@ -301,7 +301,7 @@ static void at91_pm_end(void)
 }
 
 
-static struct platform_suspend_ops at91_pm_ops ={
+static const struct platform_suspend_ops at91_pm_ops = {
         .valid  = at91_pm_valid_state,
         .begin  = at91_pm_begin,
         .enter  = at91_pm_enter,
diff --git a/arch/arm/mach-bcmring/csp/chipc/chipcHw.c b/arch/arm/mach-bcmring/csp/chipc/chipcHw.c
index b3a61d860c65..96273ff34956 100644
--- a/arch/arm/mach-bcmring/csp/chipc/chipcHw.c
+++ b/arch/arm/mach-bcmring/csp/chipc/chipcHw.c
@@ -757,7 +757,7 @@ static int chipcHw_divide(int num, int denom)
                 t = t << 1;
         }
 
-        /* Intialize the result */
+        /* Initialize the result */
         r = 0;
 
         do {
diff --git a/arch/arm/mach-bcmring/csp/dmac/dmacHw.c b/arch/arm/mach-bcmring/csp/dmac/dmacHw.c
index 7b9bac2d79a5..6b9be2e98e51 100644
--- a/arch/arm/mach-bcmring/csp/dmac/dmacHw.c
+++ b/arch/arm/mach-bcmring/csp/dmac/dmacHw.c
@@ -893,7 +893,7 @@ int dmacHw_setDataDescriptor(dmacHw_CONFIG_t *pConfig,	/*   [ IN ] Configuration
 */
 /****************************************************************************/
 uint32_t dmacHw_getDmaControllerAttribute(dmacHw_HANDLE_t handle,       /*  [ IN ]  DMA Channel handle */
-                                          dmacHw_CONTROLLER_ATTRIB_e attr       /*  [ IN ]  DMA Controler attribute of type  dmacHw_CONTROLLER_ATTRIB_e */
+                                          dmacHw_CONTROLLER_ATTRIB_e attr       /*  [ IN ]  DMA Controller attribute of type  dmacHw_CONTROLLER_ATTRIB_e */
     ) {
         dmacHw_CBLK_t *pCblk = dmacHw_HANDLE_TO_CBLK(handle);
 
diff --git a/arch/arm/mach-bcmring/csp/dmac/dmacHw_extra.c b/arch/arm/mach-bcmring/csp/dmac/dmacHw_extra.c
index ff7b436d0935..77f84b40dda9 100644
--- a/arch/arm/mach-bcmring/csp/dmac/dmacHw_extra.c
+++ b/arch/arm/mach-bcmring/csp/dmac/dmacHw_extra.c
@@ -316,7 +316,7 @@ static void DisplayDescRing(void *pDescriptor,	/*   [ IN ] Descriptor buffer */
 /**
 *  @brief   Check if DMA channel is the flow controller
 *
-*  @return  1 : If DMA is a flow controler
+*  @return  1 : If DMA is a flow controller
 *           0 : Peripheral is the flow controller
 *
 *  @note
diff --git a/arch/arm/mach-bcmring/csp/tmr/tmrHw.c b/arch/arm/mach-bcmring/csp/tmr/tmrHw.c
index 5c1c9a0e5ed2..16225e43f3c3 100644
--- a/arch/arm/mach-bcmring/csp/tmr/tmrHw.c
+++ b/arch/arm/mach-bcmring/csp/tmr/tmrHw.c
@@ -558,7 +558,7 @@ static int tmrHw_divide(int num, int denom)
                 t = t << 1;
         }
 
-        /* Intialize the result */
+        /* Initialize the result */
         r = 0;
 
         do {
diff --git a/arch/arm/mach-bcmring/dma.c b/arch/arm/mach-bcmring/dma.c
index 77eb35c89cd0..8d1baf3f4683 100644
--- a/arch/arm/mach-bcmring/dma.c
+++ b/arch/arm/mach-bcmring/dma.c
@@ -671,7 +671,7 @@ static int ConfigChannel(DMA_Handle_t handle)
 
 /****************************************************************************/
 /**
-*   Intializes all of the data structures associated with the DMA.
+*   Initializes all of the data structures associated with the DMA.
 *   @return
 *       >= 0    - Initialization was successfull.
 *
diff --git a/arch/arm/mach-bcmring/include/csp/dmacHw.h b/arch/arm/mach-bcmring/include/csp/dmacHw.h
index 5d510130a25f..6c8da2b9fc1f 100644
--- a/arch/arm/mach-bcmring/include/csp/dmacHw.h
+++ b/arch/arm/mach-bcmring/include/csp/dmacHw.h
@@ -590,7 +590,7 @@ void dmacHw_printDebugInfo(dmacHw_HANDLE_t handle,	/*  [ IN ] DMA Channel handle
 */
 /****************************************************************************/
 uint32_t dmacHw_getDmaControllerAttribute(dmacHw_HANDLE_t handle,       /*  [ IN ]  DMA Channel handle  */
-                                          dmacHw_CONTROLLER_ATTRIB_e attr       /*  [ IN ]  DMA Controler attribute of type  dmacHw_CONTROLLER_ATTRIB_e */
+                                          dmacHw_CONTROLLER_ATTRIB_e attr       /*  [ IN ]  DMA Controller attribute of type  dmacHw_CONTROLLER_ATTRIB_e */
     );
 
 #endif /* _DMACHW_H */
diff --git a/arch/arm/mach-bcmring/include/csp/tmrHw.h b/arch/arm/mach-bcmring/include/csp/tmrHw.h
index f1236d00cb97..2cbb530db8ea 100644
--- a/arch/arm/mach-bcmring/include/csp/tmrHw.h
+++ b/arch/arm/mach-bcmring/include/csp/tmrHw.h
@@ -76,7 +76,7 @@ tmrHw_RATE_t tmrHw_setPeriodicTimerRate(tmrHw_ID_t timerId,	/*  [ IN ] Timer Id
 *           certain time interval
 *
 *  This function initializes a periodic timer to generate timer interrupt
-*  after every time interval in milisecond
+*  after every time interval in millisecond
 *
 *  @return   On success: Effective interval set in mili-second
 *            On failure: 0
@@ -93,7 +93,7 @@ tmrHw_INTERVAL_t tmrHw_setPeriodicTimerInterval(tmrHw_ID_t timerId,	/*  [ IN ] T
 *           after certain time interval
 *
 *  This function initializes a periodic timer to generate a single ticks after
-*  certain time interval in milisecond
+*  certain time interval in millisecond
 *
 *  @return   On success: Effective interval set in mili-second
 *            On failure: 0
diff --git a/arch/arm/mach-bcmring/include/mach/csp/dmacHw_priv.h b/arch/arm/mach-bcmring/include/mach/csp/dmacHw_priv.h
index cbf334d1c761..d67e2f8c22de 100644
--- a/arch/arm/mach-bcmring/include/mach/csp/dmacHw_priv.h
+++ b/arch/arm/mach-bcmring/include/mach/csp/dmacHw_priv.h
@@ -28,7 +28,7 @@
 
 /* Data type for DMA Link List Item */
 typedef struct {
-        uint32_t sar;           /* Source Adress Register.
+        uint32_t sar;           /* Source Address Register.
                                    Address must be aligned to CTLx.SRC_TR_WIDTH.             */
         uint32_t dar;           /* Destination Address Register.
                                    Address must be aligned to CTLx.DST_TR_WIDTH.             */
diff --git a/arch/arm/mach-bcmring/include/mach/csp/dmacHw_reg.h b/arch/arm/mach-bcmring/include/mach/csp/dmacHw_reg.h
index 891cea87e333..f1ecf96f2da5 100644
--- a/arch/arm/mach-bcmring/include/mach/csp/dmacHw_reg.h
+++ b/arch/arm/mach-bcmring/include/mach/csp/dmacHw_reg.h
@@ -35,7 +35,7 @@ typedef struct {
 
 /* Data type representing DMA channel registers */
 typedef struct {
-        dmacHw_REG64_t ChannelSar;      /*  Source Adress Register. 64 bits (upper 32 bits are reserved)
+        dmacHw_REG64_t ChannelSar;      /*  Source Address Register. 64 bits (upper 32 bits are reserved)
                                            Address must be aligned to CTLx.SRC_TR_WIDTH.
                                          */
         dmacHw_REG64_t ChannelDar;      /*  Destination Address Register.64 bits (upper 32 bits are reserved)
diff --git a/arch/arm/mach-davinci/pm.c b/arch/arm/mach-davinci/pm.c
index fab953b43dea..1bd73a04be20 100644
--- a/arch/arm/mach-davinci/pm.c
+++ b/arch/arm/mach-davinci/pm.c
@@ -110,7 +110,7 @@ static int davinci_pm_enter(suspend_state_t state)
         return ret;
 }
 
-static struct platform_suspend_ops davinci_pm_ops = {
+static const struct platform_suspend_ops davinci_pm_ops = {
         .enter          = davinci_pm_enter,
         .valid          = suspend_valid_only_mem,
 };
diff --git a/arch/arm/mach-gemini/include/mach/hardware.h b/arch/arm/mach-gemini/include/mach/hardware.h
index 213a4fcfeb1c..8c950e1d06be 100644
--- a/arch/arm/mach-gemini/include/mach/hardware.h
+++ b/arch/arm/mach-gemini/include/mach/hardware.h
@@ -33,7 +33,7 @@
 #define GEMINI_LPC_HOST_BASE    0x47000000
 #define GEMINI_LPC_IO_BASE      0x47800000
 #define GEMINI_INTERRUPT_BASE   0x48000000
-/* TODO: Different interrupt controlers when SMP
+/* TODO: Different interrupt controllers when SMP
  * #define GEMINI_INTERRUPT0_BASE       0x48000000
  * #define GEMINI_INTERRUPT1_BASE       0x49000000
  */
diff --git a/arch/arm/mach-imx/mach-pcm038.c b/arch/arm/mach-imx/mach-pcm038.c
index f667a262dfc1..505614803bc6 100644
--- a/arch/arm/mach-imx/mach-pcm038.c
+++ b/arch/arm/mach-imx/mach-pcm038.c
@@ -254,10 +254,10 @@ static struct regulator_init_data cam_data = {
 
 static struct mc13783_regulator_init_data pcm038_regulators[] = {
         {
-                .id = MC13783_REGU_VCAM,
+                .id = MC13783_REG_VCAM,
                 .init_data = &cam_data,
         }, {
-                .id = MC13783_REGU_VMMC1,
+                .id = MC13783_REG_VMMC1,
                 .init_data = &sdhc1_data,
         },
 };
diff --git a/arch/arm/mach-imx/pm-imx27.c b/arch/arm/mach-imx/pm-imx27.c
index 6bf81ceea137..acf17691d2cc 100644
--- a/arch/arm/mach-imx/pm-imx27.c
+++ b/arch/arm/mach-imx/pm-imx27.c
@@ -32,7 +32,7 @@ static int mx27_suspend_enter(suspend_state_t state)
         return 0;
 }
 
-static struct platform_suspend_ops mx27_suspend_ops = {
+static const struct platform_suspend_ops mx27_suspend_ops = {
         .enter = mx27_suspend_enter,
         .valid = suspend_valid_only_mem,
 };
diff --git a/arch/arm/mach-kirkwood/openrd-setup.c b/arch/arm/mach-kirkwood/openrd-setup.c
index c9d77fad10ab..cfcca4174e25 100644
--- a/arch/arm/mach-kirkwood/openrd-setup.c
+++ b/arch/arm/mach-kirkwood/openrd-setup.c
@@ -171,7 +171,7 @@ static void __init openrd_init(void)
 
         kirkwood_i2c_init();
 
-        if (machine_is_openrd_client()) {
+        if (machine_is_openrd_client() || machine_is_openrd_ultimate()) {
                 i2c_register_board_info(0, i2c_board_info,
                         ARRAY_SIZE(i2c_board_info));
                 kirkwood_audio_init();
diff --git a/arch/arm/mach-lpc32xx/pm.c b/arch/arm/mach-lpc32xx/pm.c
index a6e2aed9a49f..e76d41bb7056 100644
--- a/arch/arm/mach-lpc32xx/pm.c
+++ b/arch/arm/mach-lpc32xx/pm.c
@@ -123,7 +123,7 @@ static int lpc32xx_pm_enter(suspend_state_t state)
         return 0;
 }
 
-static struct platform_suspend_ops lpc32xx_pm_ops = {
+static const struct platform_suspend_ops lpc32xx_pm_ops = {
         .valid  = suspend_valid_only_mem,
         .enter  = lpc32xx_pm_enter,
 };
diff --git a/arch/arm/mach-msm/include/mach/entry-macro-qgic.S b/arch/arm/mach-msm/include/mach/entry-macro-qgic.S
index 4dc99aa65d07..12467157afb9 100644
--- a/arch/arm/mach-msm/include/mach/entry-macro-qgic.S
+++ b/arch/arm/mach-msm/include/mach/entry-macro-qgic.S
@@ -26,7 +26,7 @@
          * The interrupt numbering scheme is defined in the
          * interrupt controller spec.  To wit:
          *
-         * Migrated the code from ARM MP port to be more consistant
+         * Migrated the code from ARM MP port to be more consistent
          * with interrupt processing , the following still holds true
          * however, all interrupts are treated the same regardless of
          * if they are local IPI or PPI
diff --git a/arch/arm/mach-msm/io.c b/arch/arm/mach-msm/io.c
index 800f327a7ecc..1260007a9dd1 100644
--- a/arch/arm/mach-msm/io.c
+++ b/arch/arm/mach-msm/io.c
@@ -154,7 +154,7 @@ __msm_ioremap(unsigned long phys_addr, size_t size, unsigned int mtype)
 {
         if (mtype == MT_DEVICE) {
                 /* The peripherals in the 88000000 - D0000000 range
-                 * are only accessable by type MT_DEVICE_NONSHARED.
+                 * are only accessible by type MT_DEVICE_NONSHARED.
                  * Adjust mtype as necessary to make this "just work."
                  */
                 if ((phys_addr >= 0x88000000) && (phys_addr < 0xD0000000))
diff --git a/arch/arm/mach-mx3/mach-mx31_3ds.c b/arch/arm/mach-mx3/mach-mx31_3ds.c
index 4e516b49a901..899a969e92fa 100644
--- a/arch/arm/mach-mx3/mach-mx31_3ds.c
+++ b/arch/arm/mach-mx3/mach-mx31_3ds.c
@@ -140,10 +140,10 @@ static struct regulator_init_data gpo_init = {
 
 static struct mc13783_regulator_init_data mx31_3ds_regulators[] = {
         {
-                .id = MC13783_REGU_PWGT1SPI, /* Power Gate for ARM core. */
+                .id = MC13783_REG_PWGT1SPI, /* Power Gate for ARM core. */
                 .init_data = &pwgtx_init,
         }, {
-                .id = MC13783_REGU_PWGT2SPI, /* Power Gate for L2 Cache. */
+                .id = MC13783_REG_PWGT2SPI, /* Power Gate for L2 Cache. */
                 .init_data = &pwgtx_init,
         }, {
 
diff --git a/arch/arm/mach-mx3/mach-mx31moboard.c b/arch/arm/mach-mx3/mach-mx31moboard.c
index 203d21a510aa..1aa8d65fccbb 100644
--- a/arch/arm/mach-mx3/mach-mx31moboard.c
+++ b/arch/arm/mach-mx3/mach-mx31moboard.c
@@ -216,11 +216,11 @@ static struct regulator_init_data cam_vreg_data = {
 
 static struct mc13783_regulator_init_data moboard_regulators[] = {
         {
-                .id = MC13783_REGU_VMMC1,
+                .id = MC13783_REG_VMMC1,
                 .init_data = &sdhc_vreg_data,
         },
         {
-                .id = MC13783_REGU_VCAM,
+                .id = MC13783_REG_VCAM,
                 .init_data = &cam_vreg_data,
         },
 };
diff --git a/arch/arm/mach-omap1/pm.c b/arch/arm/mach-omap1/pm.c
index 0cca23a85175..98ba9784aa15 100644
--- a/arch/arm/mach-omap1/pm.c
+++ b/arch/arm/mach-omap1/pm.c
@@ -647,7 +647,7 @@ static struct irqaction omap_wakeup_irq = {
 
 
 
-static struct platform_suspend_ops omap_pm_ops ={
+static const struct platform_suspend_ops omap_pm_ops = {
         .prepare        = omap_pm_prepare,
         .enter          = omap_pm_enter,
         .finish         = omap_pm_finish,
diff --git a/arch/arm/mach-omap2/Makefile b/arch/arm/mach-omap2/Makefile
index cd7332f50b2d..1c0c2b02d870 100644
--- a/arch/arm/mach-omap2/Makefile
+++ b/arch/arm/mach-omap2/Makefile
@@ -187,16 +187,19 @@ obj-$(CONFIG_MACH_NOKIA_RX51)		+= board-rx51.o \
                                            hsmmc.o
 obj-$(CONFIG_MACH_OMAP_ZOOM2)           += board-zoom.o \
                                            board-zoom-peripherals.o \
+                                           board-zoom-display.o \
                                            board-flash.o \
                                            hsmmc.o \
                                            board-zoom-debugboard.o
 obj-$(CONFIG_MACH_OMAP_ZOOM3)           += board-zoom.o \
                                            board-zoom-peripherals.o \
+                                           board-zoom-display.o \
                                            board-flash.o \
                                            hsmmc.o \
                                            board-zoom-debugboard.o
 obj-$(CONFIG_MACH_OMAP_3630SDP)         += board-3630sdp.o \
                                            board-zoom-peripherals.o \
+                                           board-zoom-display.o \
                                            board-flash.o \
                                            hsmmc.o
 obj-$(CONFIG_MACH_CM_T35)               += board-cm-t35.o \
diff --git a/arch/arm/mach-omap2/board-3430sdp.c b/arch/arm/mach-omap2/board-3430sdp.c
index 3b39ef1a680a..d4e41ef86aa5 100644
--- a/arch/arm/mach-omap2/board-3430sdp.c
+++ b/arch/arm/mach-omap2/board-3430sdp.c
@@ -38,6 +38,7 @@
 #include <plat/dma.h>
 #include <plat/gpmc.h>
 #include <plat/display.h>
+#include <plat/panel-generic-dpi.h>
 
 #include <plat/gpmc-smc91x.h>
 
@@ -270,13 +271,18 @@ static struct omap_dss_device sdp3430_lcd_device = {
         .platform_disable       = sdp3430_panel_disable_lcd,
 };
 
+static struct panel_generic_dpi_data dvi_panel = {
+        .name                   = "generic",
+        .platform_enable        = sdp3430_panel_enable_dvi,
+        .platform_disable       = sdp3430_panel_disable_dvi,
+};
+
 static struct omap_dss_device sdp3430_dvi_device = {
         .name                   = "dvi",
-        .driver_name            = "generic_panel",
         .type                   = OMAP_DISPLAY_TYPE_DPI,
+        .driver_name            = "generic_dpi_panel",
+        .data                   = &dvi_panel,
         .phy.dpi.data_lines     = 24,
-        .platform_enable        = sdp3430_panel_enable_dvi,
-        .platform_disable       = sdp3430_panel_disable_dvi,
 };
 
 static struct omap_dss_device sdp3430_tv_device = {
diff --git a/arch/arm/mach-omap2/board-3630sdp.c b/arch/arm/mach-omap2/board-3630sdp.c
index 5d41dbe059a3..62645640f5e4 100644
--- a/arch/arm/mach-omap2/board-3630sdp.c
+++ b/arch/arm/mach-omap2/board-3630sdp.c
@@ -207,6 +207,7 @@ static void __init omap_sdp_init(void)
 {
         omap3_mux_init(board_mux, OMAP_PACKAGE_CBP);
         zoom_peripherals_init();
+        zoom_display_init();
         board_smc91x_init();
         board_flash_init(sdp_flash_partitions, chip_sel_sdp);
         enable_board_wakeup_source();
diff --git a/arch/arm/mach-omap2/board-am3517evm.c b/arch/arm/mach-omap2/board-am3517evm.c
index bc1562648020..10d60b7743cf 100644
--- a/arch/arm/mach-omap2/board-am3517evm.c
+++ b/arch/arm/mach-omap2/board-am3517evm.c
@@ -35,6 +35,7 @@
 #include <plat/common.h>
 #include <plat/usb.h>
 #include <plat/display.h>
+#include <plat/panel-generic-dpi.h>
 
 #include "mux.h"
 #include "control.h"
@@ -303,13 +304,18 @@ static void am3517_evm_panel_disable_lcd(struct omap_dss_device *dssdev)
         lcd_enabled = 0;
 }
 
+static struct panel_generic_dpi_data lcd_panel = {
+        .name                   = "sharp_lq",
+        .platform_enable        = am3517_evm_panel_enable_lcd,
+        .platform_disable       = am3517_evm_panel_disable_lcd,
+};
+
 static struct omap_dss_device am3517_evm_lcd_device = {
         .type                   = OMAP_DISPLAY_TYPE_DPI,
         .name                   = "lcd",
-        .driver_name            = "sharp_lq_panel",
+        .driver_name            = "generic_dpi_panel",
+        .data                   = &lcd_panel,
         .phy.dpi.data_lines     = 16,
-        .platform_enable        = am3517_evm_panel_enable_lcd,
-        .platform_disable       = am3517_evm_panel_disable_lcd,
 };
 
 static int am3517_evm_panel_enable_tv(struct omap_dss_device *dssdev)
@@ -346,13 +352,18 @@ static void am3517_evm_panel_disable_dvi(struct omap_dss_device *dssdev)
         dvi_enabled = 0;
 }
 
+static struct panel_generic_dpi_data dvi_panel = {
+        .name                   = "generic",
+        .platform_enable        = am3517_evm_panel_enable_dvi,
+        .platform_disable       = am3517_evm_panel_disable_dvi,
+};
+
 static struct omap_dss_device am3517_evm_dvi_device = {
         .type                   = OMAP_DISPLAY_TYPE_DPI,
         .name                   = "dvi",
-        .driver_name            = "generic_panel",
+        .driver_name            = "generic_dpi_panel",
+        .data                   = &dvi_panel,
         .phy.dpi.data_lines     = 24,
-        .platform_enable        = am3517_evm_panel_enable_dvi,
-        .platform_disable       = am3517_evm_panel_disable_dvi,
 };
 
 static struct omap_dss_device *am3517_evm_dss_devices[] = {
diff --git a/arch/arm/mach-omap2/board-cm-t35.c b/arch/arm/mach-omap2/board-cm-t35.c
index 486a3de5f401..dac141610666 100644
--- a/arch/arm/mach-omap2/board-cm-t35.c
+++ b/arch/arm/mach-omap2/board-cm-t35.c
@@ -46,6 +46,7 @@
 #include <plat/gpmc.h>
 #include <plat/usb.h>
 #include <plat/display.h>
+#include <plat/panel-generic-dpi.h>
 #include <plat/mcspi.h>
 
 #include <mach/hardware.h>
@@ -351,22 +352,32 @@ static void cm_t35_panel_disable_tv(struct omap_dss_device *dssdev)
 {
 }
 
+static struct panel_generic_dpi_data lcd_panel = {
+        .name                   = "toppoly_tdo35s",
+        .platform_enable        = cm_t35_panel_enable_lcd,
+        .platform_disable       = cm_t35_panel_disable_lcd,
+};
+
 static struct omap_dss_device cm_t35_lcd_device = {
         .name                   = "lcd",
-        .driver_name            = "toppoly_tdo35s_panel",
         .type                   = OMAP_DISPLAY_TYPE_DPI,
+        .driver_name            = "generic_dpi_panel",
+        .data                   = &lcd_panel,
         .phy.dpi.data_lines     = 18,
-        .platform_enable        = cm_t35_panel_enable_lcd,
-        .platform_disable       = cm_t35_panel_disable_lcd,
+};
+
+static struct panel_generic_dpi_data dvi_panel = {
+        .name                   = "generic",
+        .platform_enable        = cm_t35_panel_enable_dvi,
+        .platform_disable       = cm_t35_panel_disable_dvi,
 };
 
 static struct omap_dss_device cm_t35_dvi_device = {
         .name                   = "dvi",
-        .driver_name            = "generic_panel",
         .type                   = OMAP_DISPLAY_TYPE_DPI,
+        .driver_name            = "generic_dpi_panel",
+        .data                   = &dvi_panel,
         .phy.dpi.data_lines     = 24,
-        .platform_enable        = cm_t35_panel_enable_dvi,
-        .platform_disable       = cm_t35_panel_disable_dvi,
 };
 
 static struct omap_dss_device cm_t35_tv_device = {
diff --git a/arch/arm/mach-omap2/board-devkit8000.c b/arch/arm/mach-omap2/board-devkit8000.c
index 451e7ff08b18..00bb1fc5e017 100644
--- a/arch/arm/mach-omap2/board-devkit8000.c
+++ b/arch/arm/mach-omap2/board-devkit8000.c
@@ -46,6 +46,7 @@
 #include <plat/nand.h>
 #include <plat/usb.h>
 #include <plat/display.h>
+#include <plat/panel-generic-dpi.h>
 
 #include <plat/mcspi.h>
 #include <linux/input/matrix_keypad.h>
@@ -149,23 +150,32 @@ static struct regulator_consumer_supply devkit8000_vmmc1_supply =
 static struct regulator_consumer_supply devkit8000_vio_supply =
         REGULATOR_SUPPLY("vcc", "spi2.0");
 
+static struct panel_generic_dpi_data lcd_panel = {
+        .name                   = "generic",
+        .platform_enable        = devkit8000_panel_enable_lcd,
+        .platform_disable       = devkit8000_panel_disable_lcd,
+};
+
 static struct omap_dss_device devkit8000_lcd_device = {
         .name                   = "lcd",
-        .driver_name            = "generic_panel",
         .type                   = OMAP_DISPLAY_TYPE_DPI,
+        .driver_name            = "generic_dpi_panel",
+        .data                   = &lcd_panel,
         .phy.dpi.data_lines     = 24,
-        .reset_gpio             = -EINVAL, /* will be replaced */
-        .platform_enable        = devkit8000_panel_enable_lcd,
-        .platform_disable       = devkit8000_panel_disable_lcd,
 };
+
+static struct panel_generic_dpi_data dvi_panel = {
+        .name                   = "generic",
+        .platform_enable        = devkit8000_panel_enable_dvi,
+        .platform_disable       = devkit8000_panel_disable_dvi,
+};
+
 static struct omap_dss_device devkit8000_dvi_device = {
         .name                   = "dvi",
-        .driver_name            = "generic_panel",
         .type                   = OMAP_DISPLAY_TYPE_DPI,
+        .driver_name            = "generic_dpi_panel",
+        .data                   = &dvi_panel,
         .phy.dpi.data_lines     = 24,
-        .reset_gpio             = -EINVAL, /* will be replaced */
-        .platform_enable        = devkit8000_panel_enable_dvi,
-        .platform_disable       = devkit8000_panel_disable_dvi,
 };
 
 static struct omap_dss_device devkit8000_tv_device = {
diff --git a/arch/arm/mach-omap2/board-igep0020.c b/arch/arm/mach-omap2/board-igep0020.c
index 0afa3011db0f..ebaa230e67ed 100644
--- a/arch/arm/mach-omap2/board-igep0020.c
+++ b/arch/arm/mach-omap2/board-igep0020.c
@@ -31,6 +31,7 @@
 #include <plat/gpmc.h>
 #include <plat/usb.h>
 #include <plat/display.h>
+#include <plat/panel-generic-dpi.h>
 #include <plat/onenand.h>
 
 #include "mux.h"
@@ -459,13 +460,18 @@ static void igep2_disable_dvi(struct omap_dss_device *dssdev)
         gpio_direction_output(IGEP2_GPIO_DVI_PUP, 0);
 }
 
+static struct panel_generic_dpi_data dvi_panel = {
+        .name                   = "generic",
+        .platform_enable        = igep2_enable_dvi,
+        .platform_disable       = igep2_disable_dvi,
+};
+
 static struct omap_dss_device igep2_dvi_device = {
         .type                   = OMAP_DISPLAY_TYPE_DPI,
         .name                   = "dvi",
-        .driver_name            = "generic_panel",
+        .driver_name            = "generic_dpi_panel",
+        .data                   = &dvi_panel,
         .phy.dpi.data_lines     = 24,
-        .platform_enable        = igep2_enable_dvi,
-        .platform_disable       = igep2_disable_dvi,
 };
 
 static struct omap_dss_device *igep2_dss_devices[] = {
diff --git a/arch/arm/mach-omap2/board-omap3beagle.c b/arch/arm/mach-omap2/board-omap3beagle.c
index 6c127605942f..a4fe8e1ee1bd 100644
--- a/arch/arm/mach-omap2/board-omap3beagle.c
+++ b/arch/arm/mach-omap2/board-omap3beagle.c
@@ -41,6 +41,7 @@
 #include <plat/board.h>
 #include <plat/common.h>
 #include <plat/display.h>
+#include <plat/panel-generic-dpi.h>
 #include <plat/gpmc.h>
 #include <plat/nand.h>
 #include <plat/usb.h>
@@ -194,14 +195,19 @@ static void beagle_disable_dvi(struct omap_dss_device *dssdev)
                 gpio_set_value(dssdev->reset_gpio, 0);
 }
 
+static struct panel_generic_dpi_data dvi_panel = {
+        .name = "generic",
+        .platform_enable = beagle_enable_dvi,
+        .platform_disable = beagle_disable_dvi,
+};
+
 static struct omap_dss_device beagle_dvi_device = {
         .type = OMAP_DISPLAY_TYPE_DPI,
         .name = "dvi",
-        .driver_name = "generic_panel",
+        .driver_name = "generic_dpi_panel",
+        .data = &dvi_panel,
         .phy.dpi.data_lines = 24,
         .reset_gpio = 170,
-        .platform_enable = beagle_enable_dvi,
-        .platform_disable = beagle_disable_dvi,
 };
 
 static struct omap_dss_device beagle_tv_device = {
diff --git a/arch/arm/mach-omap2/board-omap3evm.c b/arch/arm/mach-omap2/board-omap3evm.c
index 3de8d9b8ec76..323c3809ce39 100644
--- a/arch/arm/mach-omap2/board-omap3evm.c
+++ b/arch/arm/mach-omap2/board-omap3evm.c
@@ -43,6 +43,7 @@
 #include <plat/common.h>
 #include <plat/mcspi.h>
 #include <plat/display.h>
+#include <plat/panel-generic-dpi.h>
 
 #include "mux.h"
 #include "sdram-micron-mt46h32m32lf-6.h"
@@ -301,13 +302,18 @@ static void omap3_evm_disable_dvi(struct omap_dss_device *dssdev)
         dvi_enabled = 0;
 }
 
+static struct panel_generic_dpi_data dvi_panel = {
+        .name                   = "generic",
+        .platform_enable        = omap3_evm_enable_dvi,
+        .platform_disable       = omap3_evm_disable_dvi,
+};
+
 static struct omap_dss_device omap3_evm_dvi_device = {
         .name                   = "dvi",
-        .driver_name            = "generic_panel",
         .type                   = OMAP_DISPLAY_TYPE_DPI,
+        .driver_name            = "generic_dpi_panel",
+        .data                   = &dvi_panel,
         .phy.dpi.data_lines     = 24,
-        .platform_enable        = omap3_evm_enable_dvi,
-        .platform_disable       = omap3_evm_disable_dvi,
 };
 
 static struct omap_dss_device *omap3_evm_dss_devices[] = {
diff --git a/arch/arm/mach-omap2/board-omap3stalker.c b/arch/arm/mach-omap2/board-omap3stalker.c
index 9df9d9367608..2a2dad447e86 100644
--- a/arch/arm/mach-omap2/board-omap3stalker.c
+++ b/arch/arm/mach-omap2/board-omap3stalker.c
@@ -40,6 +40,7 @@
 #include <plat/nand.h>
 #include <plat/usb.h>
 #include <plat/display.h>
+#include <plat/panel-generic-dpi.h>
 
 #include <plat/mcspi.h>
 #include <linux/input/matrix_keypad.h>
@@ -160,13 +161,18 @@ static void omap3_stalker_disable_lcd(struct omap_dss_device *dssdev)
         lcd_enabled = 0;
 }
 
+static struct panel_generic_dpi_data lcd_panel = {
+        .name                   = "generic",
+        .platform_enable        = omap3_stalker_enable_lcd,
+        .platform_disable       = omap3_stalker_disable_lcd,
+};
+
 static struct omap_dss_device omap3_stalker_lcd_device = {
         .name                   = "lcd",
-        .driver_name            = "generic_panel",
+        .driver_name            = "generic_dpi_panel",
+        .data                   = &lcd_panel,
         .phy.dpi.data_lines     = 24,
         .type                   = OMAP_DISPLAY_TYPE_DPI,
-        .platform_enable        = omap3_stalker_enable_lcd,
-        .platform_disable       = omap3_stalker_disable_lcd,
 };
 
 static int omap3_stalker_enable_tv(struct omap_dss_device *dssdev)
@@ -208,13 +214,18 @@ static void omap3_stalker_disable_dvi(struct omap_dss_device *dssdev)
         dvi_enabled = 0;
 }
 
+static struct panel_generic_dpi_data dvi_panel = {
+        .name                   = "generic",
+        .platform_enable        = omap3_stalker_enable_dvi,
+        .platform_disable       = omap3_stalker_disable_dvi,
+};
+
 static struct omap_dss_device omap3_stalker_dvi_device = {
         .name                   = "dvi",
-        .driver_name            = "generic_panel",
         .type                   = OMAP_DISPLAY_TYPE_DPI,
+        .driver_name            = "generic_dpi_panel",
+        .data                   = &dvi_panel,
         .phy.dpi.data_lines     = 24,
-        .platform_enable        = omap3_stalker_enable_dvi,
-        .platform_disable       = omap3_stalker_disable_dvi,
 };
 
 static struct omap_dss_device *omap3_stalker_dss_devices[] = {
diff --git a/arch/arm/mach-omap2/board-zoom-display.c b/arch/arm/mach-omap2/board-zoom-display.c
new file mode 100644
index 000000000000..6bcd43657aed
--- /dev/null
+++ b/arch/arm/mach-omap2/board-zoom-display.c
@@ -0,0 +1,168 @@
+/*
+ * Copyright (C) 2010 Texas Instruments Inc.
+ *
+ * Modified from mach-omap2/board-zoom-peripherals.c
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/kernel.h>
+#include <linux/init.h>
+#include <linux/platform_device.h>
+#include <linux/gpio.h>
+#include <linux/i2c/twl.h>
+#include <linux/spi/spi.h>
+#include <plat/mcspi.h>
+#include <plat/display.h>
+
+#define LCD_PANEL_RESET_GPIO_PROD       96
+#define LCD_PANEL_RESET_GPIO_PILOT      55
+#define LCD_PANEL_QVGA_GPIO             56
+
+static void zoom_lcd_panel_init(void)
+{
+        int ret;
+        unsigned char lcd_panel_reset_gpio;
+
+        lcd_panel_reset_gpio = (omap_rev() > OMAP3430_REV_ES3_0) ?
+                        LCD_PANEL_RESET_GPIO_PROD :
+                        LCD_PANEL_RESET_GPIO_PILOT;
+
+        ret = gpio_request(lcd_panel_reset_gpio, "lcd reset");
+        if (ret) {
+                pr_err("Failed to get LCD reset GPIO (gpio%d).\n",
+                        lcd_panel_reset_gpio);
+                return;
+        }
+        gpio_direction_output(lcd_panel_reset_gpio, 1);
+
+        ret = gpio_request(LCD_PANEL_QVGA_GPIO, "lcd qvga");
+        if (ret) {
+                pr_err("Failed to get LCD_PANEL_QVGA_GPIO (gpio%d).\n",
+                        LCD_PANEL_QVGA_GPIO);
+                goto err0;
+        }
+        gpio_direction_output(LCD_PANEL_QVGA_GPIO, 1);
+
+        return;
+err0:
+        gpio_free(lcd_panel_reset_gpio);
+}
+
+static int zoom_panel_enable_lcd(struct omap_dss_device *dssdev)
+{
+        return 0;
+}
+
+static void zoom_panel_disable_lcd(struct omap_dss_device *dssdev)
+{
+}
+
+/*
+ * PWMA/B register offsets (TWL4030_MODULE_PWMA)
+ */
+#define TWL_INTBR_PMBR1 0xD
+#define TWL_INTBR_GPBR1 0xC
+#define TWL_LED_PWMON   0x0
+#define TWL_LED_PWMOFF  0x1
+
+static int zoom_set_bl_intensity(struct omap_dss_device *dssdev, int level)
+{
+        unsigned char c;
+        u8 mux_pwm, enb_pwm;
+
+        if (level > 100)
+                return -1;
+
+        twl_i2c_read_u8(TWL4030_MODULE_INTBR, &mux_pwm, TWL_INTBR_PMBR1);
+        twl_i2c_read_u8(TWL4030_MODULE_INTBR, &enb_pwm, TWL_INTBR_GPBR1);
+
+        if (level == 0) {
+                /* disable pwm1 output and clock */
+                enb_pwm = enb_pwm & 0xF5;
+                /* change pwm1 pin to gpio pin */
+                mux_pwm = mux_pwm & 0xCF;
+                twl_i2c_write_u8(TWL4030_MODULE_INTBR,
+                                        enb_pwm, TWL_INTBR_GPBR1);
+                twl_i2c_write_u8(TWL4030_MODULE_INTBR,
+                                        mux_pwm, TWL_INTBR_PMBR1);
+                return 0;
+        }
+
+        if (!((enb_pwm & 0xA) && (mux_pwm & 0x30))) {
+                /* change gpio pin to pwm1 pin */
+                mux_pwm = mux_pwm | 0x30;
+                /* enable pwm1 output and clock*/
+                enb_pwm = enb_pwm | 0x0A;
+                twl_i2c_write_u8(TWL4030_MODULE_INTBR,
+                                        mux_pwm, TWL_INTBR_PMBR1);
+                twl_i2c_write_u8(TWL4030_MODULE_INTBR,
+                                        enb_pwm, TWL_INTBR_GPBR1);
+        }
+
+        c = ((50 * (100 - level)) / 100) + 1;
+        twl_i2c_write_u8(TWL4030_MODULE_PWM1, 0x7F, TWL_LED_PWMOFF);
+        twl_i2c_write_u8(TWL4030_MODULE_PWM1, c, TWL_LED_PWMON);
+
+        return 0;
+}
+
+static struct omap_dss_device zoom_lcd_device = {
+        .name                   = "lcd",
+        .driver_name            = "NEC_8048_panel",
+        .type                   = OMAP_DISPLAY_TYPE_DPI,
+        .phy.dpi.data_lines     = 24,
+        .platform_enable        = zoom_panel_enable_lcd,
+        .platform_disable       = zoom_panel_disable_lcd,
+        .max_backlight_level    = 100,
+        .set_backlight          = zoom_set_bl_intensity,
+};
+
+static struct omap_dss_device *zoom_dss_devices[] = {
+        &zoom_lcd_device,
+};
+
+static struct omap_dss_board_info zoom_dss_data = {
+        .num_devices            = ARRAY_SIZE(zoom_dss_devices),
+        .devices                = zoom_dss_devices,
+        .default_device         = &zoom_lcd_device,
+};
+
+static struct platform_device zoom_dss_device = {
+        .name                           = "omapdss",
+        .id                             = -1,
+        .dev                            = {
+                .platform_data          = &zoom_dss_data,
+        },
+};
+
+static struct omap2_mcspi_device_config dss_lcd_mcspi_config = {
+        .turbo_mode             = 1,
+        .single_channel = 1,  /* 0: slave, 1: master */
+};
+
+static struct spi_board_info nec_8048_spi_board_info[] __initdata = {
+        [0] = {
+                .modalias               = "nec_8048_spi",
+                .bus_num                = 1,
+                .chip_select            = 2,
+                .max_speed_hz           = 375000,
+                .controller_data        = &dss_lcd_mcspi_config,
+        },
+};
+
+static struct platform_device *zoom_display_devices[] __initdata = {
+        &zoom_dss_device,
+};
+
+void __init zoom_display_init(void)
+{
+        platform_add_devices(zoom_display_devices,
+                                ARRAY_SIZE(zoom_display_devices));
+        spi_register_board_info(nec_8048_spi_board_info,
+                                ARRAY_SIZE(nec_8048_spi_board_info));
+        zoom_lcd_panel_init();
+}
+
diff --git a/arch/arm/mach-omap2/board-zoom-peripherals.c b/arch/arm/mach-omap2/board-zoom-peripherals.c
index 3fbd0edd712e..14d95afa3f0d 100644
--- a/arch/arm/mach-omap2/board-zoom-peripherals.c
+++ b/arch/arm/mach-omap2/board-zoom-peripherals.c
@@ -35,6 +35,8 @@
 #define OMAP_ZOOM_WLAN_PMENA_GPIO       (101)
 #define OMAP_ZOOM_WLAN_IRQ_GPIO         (162)
 
+#define LCD_PANEL_ENABLE_GPIO           (7 + OMAP_MAX_GPIO_LINES)
+
 /* Zoom2 has Qwerty keyboard*/
 static uint32_t board_keymap[] = {
         KEY(0, 0, KEY_E),
@@ -224,9 +226,43 @@ static struct omap2_hsmmc_info mmc[] = {
         {}      /* Terminator */
 };
 
+static struct regulator_consumer_supply zoom_vpll2_supply =
+        REGULATOR_SUPPLY("vdds_dsi", "omapdss");
+
+static struct regulator_consumer_supply zoom_vdda_dac_supply =
+        REGULATOR_SUPPLY("vdda_dac", "omapdss");
+
+static struct regulator_init_data zoom_vpll2 = {
+        .constraints = {
+                .min_uV                 = 1800000,
+                .max_uV                 = 1800000,
+                .valid_modes_mask       = REGULATOR_MODE_NORMAL
+                                        | REGULATOR_MODE_STANDBY,
+                .valid_ops_mask         = REGULATOR_CHANGE_MODE
+                                        | REGULATOR_CHANGE_STATUS,
+        },
+        .num_consumer_supplies          = 1,
+        .consumer_supplies              = &zoom_vpll2_supply,
+};
+
+static struct regulator_init_data zoom_vdac = {
+        .constraints = {
+                .min_uV                 = 1800000,
+                .max_uV                 = 1800000,
+                .valid_modes_mask       = REGULATOR_MODE_NORMAL
+                                        | REGULATOR_MODE_STANDBY,
+                .valid_ops_mask         = REGULATOR_CHANGE_MODE
+                                        | REGULATOR_CHANGE_STATUS,
+        },
+        .num_consumer_supplies          = 1,
+        .consumer_supplies              = &zoom_vdda_dac_supply,
+};
+
 static int zoom_twl_gpio_setup(struct device *dev,
                 unsigned gpio, unsigned ngpio)
 {
+        int ret;
+
         /* gpio + 0 is "mmc0_cd" (input/IRQ) */
         mmc[0].gpio_cd = gpio + 0;
         omap2_hsmmc_init(mmc);
@@ -238,7 +274,15 @@ static int zoom_twl_gpio_setup(struct device *dev,
         zoom_vsim_supply.dev = mmc[0].dev;
         zoom_vmmc2_supply.dev = mmc[1].dev;
 
-        return 0;
+        ret = gpio_request(LCD_PANEL_ENABLE_GPIO, "lcd enable");
+        if (ret) {
+                pr_err("Failed to get LCD_PANEL_ENABLE_GPIO (gpio%d).\n",
+                                LCD_PANEL_ENABLE_GPIO);
+                return ret;
+        }
+        gpio_direction_output(LCD_PANEL_ENABLE_GPIO, 0);
+
+        return ret;
 }
 
 /* EXTMUTE callback function */
@@ -301,7 +345,8 @@ static struct twl4030_platform_data zoom_twldata = {
         .vmmc1          = &zoom_vmmc1,
         .vmmc2          = &zoom_vmmc2,
         .vsim           = &zoom_vsim,
-
+        .vpll2          = &zoom_vpll2,
+        .vdac           = &zoom_vdac,
 };
 
 static struct i2c_board_info __initdata zoom_i2c_boardinfo[] = {
diff --git a/arch/arm/mach-omap2/board-zoom.c b/arch/arm/mach-omap2/board-zoom.c
index e041c537ea37..e26754c24ee8 100644
--- a/arch/arm/mach-omap2/board-zoom.c
+++ b/arch/arm/mach-omap2/board-zoom.c
@@ -130,6 +130,7 @@ static void __init omap_zoom_init(void)
                         ARRAY_SIZE(zoom_nand_partitions), ZOOM_NAND_CS);
         zoom_debugboard_init();
         zoom_peripherals_init();
+        zoom_display_init();
 }
 
 MACHINE_START(OMAP_ZOOM2, "OMAP Zoom2 board")
diff --git a/arch/arm/mach-omap2/cpuidle34xx.c b/arch/arm/mach-omap2/cpuidle34xx.c
index f3e043fe5eb8..f7b22a16f385 100644
--- a/arch/arm/mach-omap2/cpuidle34xx.c
+++ b/arch/arm/mach-omap2/cpuidle34xx.c
@@ -47,6 +47,8 @@
 
 #define OMAP3_STATE_MAX OMAP3_STATE_C7
 
+#define CPUIDLE_FLAG_CHECK_BM   0x10000 /* use omap3_enter_idle_bm() */
+
 struct omap3_processor_cx {
         u8 valid;
         u8 type;
@@ -252,7 +254,7 @@ static int omap3_enter_idle_bm(struct cpuidle_device *dev,
          * FIXME: we currently manage device-specific idle states
          *        for PER and CORE in combination with CPU-specific
          *        idle states.  This is wrong, and device-specific
-         *        idle managment needs to be separated out into 
+         *        idle management needs to be separated out into 
          *        its own code.
          */
 
diff --git a/arch/arm/mach-omap2/include/mach/board-zoom.h b/arch/arm/mach-omap2/include/mach/board-zoom.h
index f93ca3928c3b..d20bd9c1a106 100644
--- a/arch/arm/mach-omap2/include/mach/board-zoom.h
+++ b/arch/arm/mach-omap2/include/mach/board-zoom.h
@@ -1,9 +1,12 @@
 /*
  * Defines for zoom boards
  */
+#include <plat/display.h>
+
 #define ZOOM_NAND_CS    0
 
 extern int __init zoom_debugboard_init(void);
 extern void __init zoom_peripherals_init(void);
+extern void __init zoom_display_init(void);
 
 #define ZOOM2_HEADSET_EXTMUTE_GPIO      153
diff --git a/arch/arm/mach-omap2/pm24xx.c b/arch/arm/mach-omap2/pm24xx.c
index dac2d1d9987d..9e5dc8ed51e9 100644
--- a/arch/arm/mach-omap2/pm24xx.c
+++ b/arch/arm/mach-omap2/pm24xx.c
@@ -350,7 +350,7 @@ static void omap2_pm_end(void)
         enable_hlt();
 }
 
-static struct platform_suspend_ops omap_pm_ops = {
+static const struct platform_suspend_ops omap_pm_ops = {
         .begin          = omap2_pm_begin,
         .enter          = omap2_pm_enter,
         .end            = omap2_pm_end,
diff --git a/arch/arm/mach-omap2/pm34xx.c b/arch/arm/mach-omap2/pm34xx.c
index 5b323f28da2d..8cbbeade4b8a 100644
--- a/arch/arm/mach-omap2/pm34xx.c
+++ b/arch/arm/mach-omap2/pm34xx.c
@@ -605,7 +605,7 @@ static void omap3_pm_end(void)
         return;
 }
 
-static struct platform_suspend_ops omap_pm_ops = {
+static const struct platform_suspend_ops omap_pm_ops = {
         .begin          = omap3_pm_begin,
         .end            = omap3_pm_end,
         .enter          = omap3_pm_enter,
diff --git a/arch/arm/mach-omap2/pm44xx.c b/arch/arm/mach-omap2/pm44xx.c
index e9f4862c4de4..76cfff2db514 100644
--- a/arch/arm/mach-omap2/pm44xx.c
+++ b/arch/arm/mach-omap2/pm44xx.c
@@ -65,7 +65,7 @@ static void omap4_pm_end(void)
         return;
 }
 
-static struct platform_suspend_ops omap_pm_ops = {
+static const struct platform_suspend_ops omap_pm_ops = {
         .begin          = omap4_pm_begin,
         .end            = omap4_pm_end,
         .enter          = omap4_pm_enter,
diff --git a/arch/arm/mach-omap2/serial.c b/arch/arm/mach-omap2/serial.c
index c64578853a8d..302da7403a10 100644
--- a/arch/arm/mach-omap2/serial.c
+++ b/arch/arm/mach-omap2/serial.c
@@ -852,7 +852,7 @@ void __init omap_serial_init_port(struct omap_board_data *bdata)
 }
 
 /**
- * omap_serial_init() - intialize all supported serial ports
+ * omap_serial_init() - initialize all supported serial ports
  *
  * Initializes all available UARTs as serial ports. Platforms
  * can call this function when they want to have default behaviour
diff --git a/arch/arm/mach-pnx4008/pm.c b/arch/arm/mach-pnx4008/pm.c
index ee3c29c57ae3..f3e60a049f98 100644
--- a/arch/arm/mach-pnx4008/pm.c
+++ b/arch/arm/mach-pnx4008/pm.c
@@ -119,7 +119,7 @@ static int pnx4008_pm_valid(suspend_state_t state)
                (state == PM_SUSPEND_MEM);
 }
 
-static struct platform_suspend_ops pnx4008_pm_ops = {
+static const struct platform_suspend_ops pnx4008_pm_ops = {
         .enter = pnx4008_pm_enter,
         .valid = pnx4008_pm_valid,
 };
diff --git a/arch/arm/mach-pxa/mxm8x10.c b/arch/arm/mach-pxa/mxm8x10.c
index 462167ac05f9..cdf7f41e2bb3 100644
--- a/arch/arm/mach-pxa/mxm8x10.c
+++ b/arch/arm/mach-pxa/mxm8x10.c
@@ -337,7 +337,7 @@ void __init mxm_8x10_mmc_init(void)
 }
 #endif
 
-/* USB Open Host Controler Interface */
+/* USB Open Host Controller Interface */
 static struct pxaohci_platform_data mxm_8x10_ohci_platform_data = {
         .port_mode = PMM_NPS_MODE,
         .flags = ENABLE_PORT_ALL
diff --git a/arch/arm/mach-pxa/pm.c b/arch/arm/mach-pxa/pm.c
index 166c15f62916..978e1b289544 100644
--- a/arch/arm/mach-pxa/pm.c
+++ b/arch/arm/mach-pxa/pm.c
@@ -96,7 +96,7 @@ void pxa_pm_finish(void)
                 pxa_cpu_pm_fns->finish();
 }
 
-static struct platform_suspend_ops pxa_pm_ops = {
+static const struct platform_suspend_ops pxa_pm_ops = {
         .valid          = pxa_pm_valid,
         .enter          = pxa_pm_enter,
         .prepare        = pxa_pm_prepare,
diff --git a/arch/arm/mach-pxa/sharpsl_pm.c b/arch/arm/mach-pxa/sharpsl_pm.c
index e68d46d415f3..785880f67b60 100644
--- a/arch/arm/mach-pxa/sharpsl_pm.c
+++ b/arch/arm/mach-pxa/sharpsl_pm.c
@@ -869,7 +869,7 @@ static void sharpsl_apm_get_power_status(struct apm_power_info *info)
 }
 
 #ifdef CONFIG_PM
-static struct platform_suspend_ops sharpsl_pm_ops = {
+static const struct platform_suspend_ops sharpsl_pm_ops = {
         .prepare        = pxa_pm_prepare,
         .finish         = pxa_pm_finish,
         .enter          = corgi_pxa_pm_enter,
diff --git a/arch/arm/mach-s3c2410/mach-h1940.c b/arch/arm/mach-s3c2410/mach-h1940.c
index d7ada8c7e41f..1a81fe12ccd7 100644
--- a/arch/arm/mach-s3c2410/mach-h1940.c
+++ b/arch/arm/mach-s3c2410/mach-h1940.c
@@ -387,7 +387,7 @@ static struct platform_device *h1940_devices[] __initdata = {
         &s3c_device_wdt,
         &s3c_device_i2c0,
         &s3c_device_iis,
-        &s3c_device_pcm,
+        &samsung_asoc_dma,
         &s3c_device_usbgadget,
         &h1940_device_leds,
         &h1940_device_bluetooth,
diff --git a/arch/arm/mach-s3c2440/mach-rx1950.c b/arch/arm/mach-s3c2440/mach-rx1950.c
index e0622bbb6dfa..eab6ae50683c 100644
--- a/arch/arm/mach-s3c2440/mach-rx1950.c
+++ b/arch/arm/mach-s3c2440/mach-rx1950.c
@@ -692,7 +692,7 @@ static struct platform_device *rx1950_devices[] __initdata = {
         &s3c_device_wdt,
         &s3c_device_i2c0,
         &s3c_device_iis,
-        &s3c_device_pcm,
+        &samsung_asoc_dma,
         &s3c_device_usbgadget,
         &s3c_device_rtc,
         &s3c_device_nand,
diff --git a/arch/arm/mach-s3c64xx/clock.c b/arch/arm/mach-s3c64xx/clock.c
index 7e03f0ae2fc8..1c98d2ff2ed6 100644
--- a/arch/arm/mach-s3c64xx/clock.c
+++ b/arch/arm/mach-s3c64xx/clock.c
@@ -695,7 +695,7 @@ static struct clksrc_clk clksrcs[] = {
         }, {
                 .clk    = {
                         .name           = "audio-bus",
-                        .id             = -1,  /* There's only one IISv4 port */
+                        .id             = 2,
                         .ctrlbit        = S3C6410_CLKCON_SCLK_AUDIO2,
                         .enable         = s3c64xx_sclk_ctrl,
                 },
diff --git a/arch/arm/mach-s3c64xx/dev-audio.c b/arch/arm/mach-s3c64xx/dev-audio.c
index 76426a32c013..cad67022fa25 100644
--- a/arch/arm/mach-s3c64xx/dev-audio.c
+++ b/arch/arm/mach-s3c64xx/dev-audio.c
@@ -22,7 +22,12 @@
 #include <plat/audio.h>
 #include <plat/gpio-cfg.h>
 
-static int s3c64xx_i2sv3_cfg_gpio(struct platform_device *pdev)
+static const char *rclksrc[] = {
+        [0] = "iis",
+        [1] = "audio-bus",
+};
+
+static int s3c64xx_i2s_cfg_gpio(struct platform_device *pdev)
 {
         unsigned int base;
 
@@ -33,6 +38,12 @@ static int s3c64xx_i2sv3_cfg_gpio(struct platform_device *pdev)
         case 1:
                 base = S3C64XX_GPE(0);
                 break;
+        case 2:
+                s3c_gpio_cfgpin(S3C64XX_GPC(4), S3C_GPIO_SFN(5));
+                s3c_gpio_cfgpin(S3C64XX_GPC(5), S3C_GPIO_SFN(5));
+                s3c_gpio_cfgpin(S3C64XX_GPC(7), S3C_GPIO_SFN(5));
+                s3c_gpio_cfgpin_range(S3C64XX_GPH(6), 4, S3C_GPIO_SFN(5));
+                return 0;
         default:
                 printk(KERN_DEBUG "Invalid I2S Controller number: %d\n",
                         pdev->id);
@@ -44,16 +55,6 @@ static int s3c64xx_i2sv3_cfg_gpio(struct platform_device *pdev)
         return 0;
 }
 
-static int s3c64xx_i2sv4_cfg_gpio(struct platform_device *pdev)
-{
-        s3c_gpio_cfgpin(S3C64XX_GPC(4), S3C_GPIO_SFN(5));
-        s3c_gpio_cfgpin(S3C64XX_GPC(5), S3C_GPIO_SFN(5));
-        s3c_gpio_cfgpin(S3C64XX_GPC(7), S3C_GPIO_SFN(5));
-        s3c_gpio_cfgpin_range(S3C64XX_GPH(6), 4, S3C_GPIO_SFN(5));
-
-        return 0;
-}
-
 static struct resource s3c64xx_iis0_resource[] = {
         [0] = {
                 .start = S3C64XX_PA_IIS0,
@@ -72,17 +73,22 @@ static struct resource s3c64xx_iis0_resource[] = {
         },
 };
 
-static struct s3c_audio_pdata s3c_i2s0_pdata = {
-        .cfg_gpio = s3c64xx_i2sv3_cfg_gpio,
+static struct s3c_audio_pdata i2sv3_pdata = {
+        .cfg_gpio = s3c64xx_i2s_cfg_gpio,
+        .type = {
+                .i2s = {
+                        .src_clk = rclksrc,
+                },
+        },
 };
 
 struct platform_device s3c64xx_device_iis0 = {
-        .name             = "s3c64xx-iis",
+        .name             = "samsung-i2s",
         .id               = 0,
         .num_resources    = ARRAY_SIZE(s3c64xx_iis0_resource),
         .resource         = s3c64xx_iis0_resource,
         .dev = {
-                .platform_data = &s3c_i2s0_pdata,
+                .platform_data = &i2sv3_pdata,
         },
 };
 EXPORT_SYMBOL(s3c64xx_device_iis0);
@@ -105,17 +111,13 @@ static struct resource s3c64xx_iis1_resource[] = {
         },
 };
 
-static struct s3c_audio_pdata s3c_i2s1_pdata = {
-        .cfg_gpio = s3c64xx_i2sv3_cfg_gpio,
-};
-
 struct platform_device s3c64xx_device_iis1 = {
-        .name             = "s3c64xx-iis",
+        .name             = "samsung-i2s",
         .id               = 1,
         .num_resources    = ARRAY_SIZE(s3c64xx_iis1_resource),
         .resource         = s3c64xx_iis1_resource,
         .dev = {
-                .platform_data = &s3c_i2s1_pdata,
+                .platform_data = &i2sv3_pdata,
         },
 };
 EXPORT_SYMBOL(s3c64xx_device_iis1);
@@ -138,17 +140,23 @@ static struct resource s3c64xx_iisv4_resource[] = {
         },
 };
 
-static struct s3c_audio_pdata s3c_i2sv4_pdata = {
-        .cfg_gpio = s3c64xx_i2sv4_cfg_gpio,
+static struct s3c_audio_pdata i2sv4_pdata = {
+        .cfg_gpio = s3c64xx_i2s_cfg_gpio,
+        .type = {
+                .i2s = {
+                        .quirks = QUIRK_PRI_6CHAN,
+                        .src_clk = rclksrc,
+                },
+        },
 };
 
 struct platform_device s3c64xx_device_iisv4 = {
-        .name             = "s3c64xx-iis-v4",
-        .id               = -1,
+        .name = "samsung-i2s",
+        .id = 2,
         .num_resources    = ARRAY_SIZE(s3c64xx_iisv4_resource),
         .resource         = s3c64xx_iisv4_resource,
         .dev = {
-                .platform_data = &s3c_i2sv4_pdata,
+                .platform_data = &i2sv4_pdata,
         },
 };
 EXPORT_SYMBOL(s3c64xx_device_iisv4);
@@ -288,7 +296,7 @@ static struct s3c_audio_pdata s3c_ac97_pdata;
 static u64 s3c64xx_ac97_dmamask = DMA_BIT_MASK(32);
 
 struct platform_device s3c64xx_device_ac97 = {
-        .name             = "s3c-ac97",
+        .name             = "samsung-ac97",
         .id               = -1,
         .num_resources    = ARRAY_SIZE(s3c64xx_ac97_resource),
         .resource         = s3c64xx_ac97_resource,
@@ -307,16 +315,3 @@ void __init s3c64xx_ac97_setup_gpio(int num)
         else
                 s3c_ac97_pdata.cfg_gpio = s3c64xx_ac97_cfg_gpe;
 }
-
-static u64 s3c_device_audio_dmamask = 0xffffffffUL;
-
-struct platform_device s3c_device_pcm = {
-        .name             = "s3c24xx-pcm-audio",
-        .id               = -1,
-        .dev              = {
-                .dma_mask = &s3c_device_audio_dmamask,
-                .coherent_dma_mask = 0xffffffffUL
-        }
-};
-EXPORT_SYMBOL(s3c_device_pcm);
-
diff --git a/arch/arm/mach-s3c64xx/dma.c b/arch/arm/mach-s3c64xx/dma.c
index e7d03ab41d80..372ea6855454 100644
--- a/arch/arm/mach-s3c64xx/dma.c
+++ b/arch/arm/mach-s3c64xx/dma.c
@@ -740,7 +740,7 @@ static int __init s3c64xx_dma_init(void)
         /* Set all DMA configuration to be DMA, not SDMA */
         writel(0xffffff, S3C_SYSREG(0x110));
 
-        /* Register standard DMA controlers */
+        /* Register standard DMA controllers */
         s3c64xx_dma_init1(0, DMACH_UART0, IRQ_DMA0, 0x75000000);
         s3c64xx_dma_init1(8, DMACH_PCM1_TX, IRQ_DMA1, 0x75100000);
 
diff --git a/arch/arm/mach-s3c64xx/mach-smdk6410.c b/arch/arm/mach-s3c64xx/mach-smdk6410.c
index 77488facfe4c..e85192a86fbe 100644
--- a/arch/arm/mach-s3c64xx/mach-smdk6410.c
+++ b/arch/arm/mach-s3c64xx/mach-smdk6410.c
@@ -283,7 +283,7 @@ static struct platform_device *smdk6410_devices[] __initdata = {
         &s3c_device_fb,
         &s3c_device_ohci,
         &s3c_device_usb_hsotg,
-        &s3c_device_pcm,
+        &samsung_asoc_dma,
         &s3c64xx_device_iisv4,
         &samsung_device_keypad,
 
diff --git a/arch/arm/mach-s5p6442/dev-audio.c b/arch/arm/mach-s5p6442/dev-audio.c
index 3462197ff352..8719dc41fe32 100644
--- a/arch/arm/mach-s5p6442/dev-audio.c
+++ b/arch/arm/mach-s5p6442/dev-audio.c
@@ -29,7 +29,7 @@ static int s5p6442_cfg_i2s(struct platform_device *pdev)
                 base = S5P6442_GPC1(0);
                 break;
 
-        case -1:
+        case 0:
                 base = S5P6442_GPC0(0);
                 break;
 
@@ -42,8 +42,19 @@ static int s5p6442_cfg_i2s(struct platform_device *pdev)
         return 0;
 }
 
-static struct s3c_audio_pdata s3c_i2s_pdata = {
+static const char *rclksrc_v35[] = {
+        [0] = "busclk",
+        [1] = "i2sclk",
+};
+
+static struct s3c_audio_pdata i2sv35_pdata = {
         .cfg_gpio = s5p6442_cfg_i2s,
+        .type = {
+                .i2s = {
+                        .quirks = QUIRK_SEC_DAI | QUIRK_NEED_RSTCLR,
+                        .src_clk = rclksrc_v35,
+                },
+        },
 };
 
 static struct resource s5p6442_iis0_resource[] = {
@@ -62,15 +73,34 @@ static struct resource s5p6442_iis0_resource[] = {
                 .end   = DMACH_I2S0_RX,
                 .flags = IORESOURCE_DMA,
         },
+        [3] = {
+                .start = DMACH_I2S0S_TX,
+                .end = DMACH_I2S0S_TX,
+                .flags = IORESOURCE_DMA,
+        },
 };
 
 struct platform_device s5p6442_device_iis0 = {
-        .name             = "s3c64xx-iis-v4",
-        .id               = -1,
+        .name = "samsung-i2s",
+        .id = 0,
         .num_resources    = ARRAY_SIZE(s5p6442_iis0_resource),
         .resource         = s5p6442_iis0_resource,
         .dev = {
-                .platform_data = &s3c_i2s_pdata,
+                .platform_data = &i2sv35_pdata,
+        },
+};
+
+static const char *rclksrc_v3[] = {
+        [0] = "iis",
+        [1] = "sclk_audio",
+};
+
+static struct s3c_audio_pdata i2sv3_pdata = {
+        .cfg_gpio = s5p6442_cfg_i2s,
+        .type = {
+                .i2s = {
+                        .src_clk = rclksrc_v3,
+                },
         },
 };
 
@@ -93,12 +123,12 @@ static struct resource s5p6442_iis1_resource[] = {
 };
 
 struct platform_device s5p6442_device_iis1 = {
-        .name             = "s3c64xx-iis",
+        .name             = "samsung-i2s",
         .id               = 1,
         .num_resources    = ARRAY_SIZE(s5p6442_iis1_resource),
         .resource         = s5p6442_iis1_resource,
         .dev = {
-                .platform_data = &s3c_i2s_pdata,
+                .platform_data = &i2sv3_pdata,
         },
 };
 
diff --git a/arch/arm/mach-s5p64x0/clock-s5p6440.c b/arch/arm/mach-s5p64x0/clock-s5p6440.c
index e4883dc1c8d7..409c5fc3670d 100644
--- a/arch/arm/mach-s5p64x0/clock-s5p6440.c
+++ b/arch/arm/mach-s5p64x0/clock-s5p6440.c
@@ -261,7 +261,7 @@ static struct clk init_clocks_disable[] = {
                 .enable         = s5p64x0_pclk_ctrl,
                 .ctrlbit        = (1 << 25),
         }, {
-                .name           = "i2s_v40",
+                .name           = "iis",
                 .id             = 0,
                 .parent         = &clk_pclk_low.clk,
                 .enable         = s5p64x0_pclk_ctrl,
diff --git a/arch/arm/mach-s5p64x0/clock-s5p6450.c b/arch/arm/mach-s5p64x0/clock-s5p6450.c
index 7dbf3c968f53..7fc6abd35914 100644
--- a/arch/arm/mach-s5p64x0/clock-s5p6450.c
+++ b/arch/arm/mach-s5p64x0/clock-s5p6450.c
@@ -256,7 +256,7 @@ static struct clk init_clocks_disable[] = {
                 .ctrlbit        = (1 << 22),
         }, {
                 .name           = "iis",
-                .id             = -1,
+                .id             = 0,
                 .parent         = &clk_pclk_low.clk,
                 .enable         = s5p64x0_pclk_ctrl,
                 .ctrlbit        = (1 << 26),
diff --git a/arch/arm/mach-s5p64x0/dev-audio.c b/arch/arm/mach-s5p64x0/dev-audio.c
index 396bacc0a39a..14f89e73b8de 100644
--- a/arch/arm/mach-s5p64x0/dev-audio.c
+++ b/arch/arm/mach-s5p64x0/dev-audio.c
@@ -19,15 +19,19 @@
 #include <mach/dma.h>
 #include <mach/irqs.h>
 
-static int s5p6440_cfg_i2s(struct platform_device *pdev)
+static const char *rclksrc[] = {
+        [0] = "iis",
+        [1] = "sclk_audio2",
+};
+
+static int s5p64x0_cfg_i2s(struct platform_device *pdev)
 {
         /* configure GPIO for i2s port */
         switch (pdev->id) {
-        case -1:
+        case 0:
                 s3c_gpio_cfgpin_range(S5P6440_GPR(4), 5, S3C_GPIO_SFN(5));
                 s3c_gpio_cfgpin_range(S5P6440_GPR(13), 2, S3C_GPIO_SFN(5));
                 break;
-
         default:
                 printk(KERN_ERR "Invalid Device %d\n", pdev->id);
                 return -EINVAL;
@@ -36,31 +40,14 @@ static int s5p6440_cfg_i2s(struct platform_device *pdev)
         return 0;
 }
 
-static int s5p6450_cfg_i2s(struct platform_device *pdev)
-{
-        /* configure GPIO for i2s port */
-        switch (pdev->id) {
-        case -1:
-                s3c_gpio_cfgpin(S5P6450_GPB(4), S3C_GPIO_SFN(5));
-                s3c_gpio_cfgpin_range(S5P6450_GPR(4), 5, S3C_GPIO_SFN(5));
-                s3c_gpio_cfgpin_range(S5P6450_GPR(13), 2, S3C_GPIO_SFN(5));
-
-                break;
-
-        default:
-                printk(KERN_ERR "Invalid Device %d\n", pdev->id);
-                return -EINVAL;
-        }
-
-        return 0;
-}
-
-static struct s3c_audio_pdata s5p6440_i2s_pdata = {
-        .cfg_gpio = s5p6440_cfg_i2s,
-};
-
-static struct s3c_audio_pdata s5p6450_i2s_pdata = {
-        .cfg_gpio = s5p6450_cfg_i2s,
+static struct s3c_audio_pdata s5p64x0_i2s_pdata = {
+        .cfg_gpio = s5p64x0_cfg_i2s,
+        .type = {
+                .i2s = {
+                        .quirks = QUIRK_PRI_6CHAN,
+                        .src_clk = rclksrc,
+                },
+        },
 };
 
 static struct resource s5p64x0_iis0_resource[] = {
@@ -82,22 +69,22 @@ static struct resource s5p64x0_iis0_resource[] = {
 };
 
 struct platform_device s5p6440_device_iis = {
-        .name           = "s3c64xx-iis-v4",
-        .id             = -1,
+        .name           = "samsung-i2s",
+        .id             = 0,
         .num_resources  = ARRAY_SIZE(s5p64x0_iis0_resource),
         .resource       = s5p64x0_iis0_resource,
         .dev = {
-                .platform_data = &s5p6440_i2s_pdata,
+                .platform_data = &s5p64x0_i2s_pdata,
         },
 };
 
 struct platform_device s5p6450_device_iis0 = {
-        .name           = "s3c64xx-iis-v4",
-        .id             = -1,
+        .name           = "samsung-i2s",
+        .id             = 0,
         .num_resources  = ARRAY_SIZE(s5p64x0_iis0_resource),
         .resource       = s5p64x0_iis0_resource,
         .dev = {
-                .platform_data = &s5p6450_i2s_pdata,
+                .platform_data = &s5p64x0_i2s_pdata,
         },
 };
 
diff --git a/arch/arm/mach-s5pc100/dev-audio.c b/arch/arm/mach-s5pc100/dev-audio.c
index 564e195ec493..ab2d27172cbc 100644
--- a/arch/arm/mach-s5pc100/dev-audio.c
+++ b/arch/arm/mach-s5pc100/dev-audio.c
@@ -23,17 +23,14 @@ static int s5pc100_cfg_i2s(struct platform_device *pdev)
 {
         /* configure GPIO for i2s port */
         switch (pdev->id) {
+        case 0: /* Dedicated pins */
+                break;
         case 1:
                 s3c_gpio_cfgpin_range(S5PC100_GPC(0), 5, S3C_GPIO_SFN(2));
                 break;
-
         case 2:
                 s3c_gpio_cfgpin_range(S5PC100_GPG3(0), 5, S3C_GPIO_SFN(4));
                 break;
-
-        case -1: /* Dedicated pins */
-                break;
-
         default:
                 printk(KERN_ERR "Invalid Device %d\n", pdev->id);
                 return -EINVAL;
@@ -42,8 +39,20 @@ static int s5pc100_cfg_i2s(struct platform_device *pdev)
         return 0;
 }
 
-static struct s3c_audio_pdata s3c_i2s_pdata = {
+static const char *rclksrc_v5[] = {
+        [0] = "iis",
+        [1] = "i2sclkd2",
+};
+
+static struct s3c_audio_pdata i2sv5_pdata = {
         .cfg_gpio = s5pc100_cfg_i2s,
+        .type = {
+                .i2s = {
+                        .quirks = QUIRK_PRI_6CHAN | QUIRK_SEC_DAI
+                                         | QUIRK_NEED_RSTCLR,
+                        .src_clk = rclksrc_v5,
+                },
+        },
 };
 
 static struct resource s5pc100_iis0_resource[] = {
@@ -62,15 +71,34 @@ static struct resource s5pc100_iis0_resource[] = {
                 .end   = DMACH_I2S0_RX,
                 .flags = IORESOURCE_DMA,
         },
+        [3] = {
+                .start = DMACH_I2S0S_TX,
+                .end = DMACH_I2S0S_TX,
+                .flags = IORESOURCE_DMA,
+        },
 };
 
 struct platform_device s5pc100_device_iis0 = {
-        .name             = "s3c64xx-iis-v4",
-        .id               = -1,
+        .name = "samsung-i2s",
+        .id = 0,
         .num_resources    = ARRAY_SIZE(s5pc100_iis0_resource),
         .resource         = s5pc100_iis0_resource,
         .dev = {
-                .platform_data = &s3c_i2s_pdata,
+                .platform_data = &i2sv5_pdata,
+        },
+};
+
+static const char *rclksrc_v3[] = {
+        [0] = "iis",
+        [1] = "sclk_audio",
+};
+
+static struct s3c_audio_pdata i2sv3_pdata = {
+        .cfg_gpio = s5pc100_cfg_i2s,
+        .type = {
+                .i2s = {
+                        .src_clk = rclksrc_v3,
+                },
         },
 };
 
@@ -93,12 +121,12 @@ static struct resource s5pc100_iis1_resource[] = {
 };
 
 struct platform_device s5pc100_device_iis1 = {
-        .name             = "s3c64xx-iis",
+        .name             = "samsung-i2s",
         .id               = 1,
         .num_resources    = ARRAY_SIZE(s5pc100_iis1_resource),
         .resource         = s5pc100_iis1_resource,
         .dev = {
-                .platform_data = &s3c_i2s_pdata,
+                .platform_data = &i2sv3_pdata,
         },
 };
 
@@ -121,12 +149,12 @@ static struct resource s5pc100_iis2_resource[] = {
 };
 
 struct platform_device s5pc100_device_iis2 = {
-        .name             = "s3c64xx-iis",
+        .name             = "samsung-i2s",
         .id               = 2,
         .num_resources    = ARRAY_SIZE(s5pc100_iis2_resource),
         .resource         = s5pc100_iis2_resource,
         .dev = {
-                .platform_data = &s3c_i2s_pdata,
+                .platform_data = &i2sv3_pdata,
         },
 };
 
@@ -253,7 +281,7 @@ static struct s3c_audio_pdata s3c_ac97_pdata = {
 static u64 s5pc100_ac97_dmamask = DMA_BIT_MASK(32);
 
 struct platform_device s5pc100_device_ac97 = {
-        .name             = "s3c-ac97",
+        .name             = "samsung-ac97",
         .id               = -1,
         .num_resources    = ARRAY_SIZE(s5pc100_ac97_resource),
         .resource         = s5pc100_ac97_resource,
diff --git a/arch/arm/mach-s5pc100/mach-smdkc100.c b/arch/arm/mach-s5pc100/mach-smdkc100.c
index 18b405d514d6..dd192a27524d 100644
--- a/arch/arm/mach-s5pc100/mach-smdkc100.c
+++ b/arch/arm/mach-s5pc100/mach-smdkc100.c
@@ -96,6 +96,7 @@ static struct s3c2410_uartcfg smdkc100_uartcfgs[] __initdata = {
 
 /* I2C0 */
 static struct i2c_board_info i2c_devs0[] __initdata = {
+        {I2C_BOARD_INFO("wm8580", 0x1b),},
 };
 
 /* I2C1 */
@@ -190,6 +191,7 @@ static struct platform_device *smdkc100_devices[] __initdata = {
         &s3c_device_ts,
         &s3c_device_wdt,
         &smdkc100_lcd_powerdev,
+        &samsung_asoc_dma,
         &s5pc100_device_iis0,
         &samsung_device_keypad,
         &s5pc100_device_ac97,
diff --git a/arch/arm/mach-s5pv210/clock.c b/arch/arm/mach-s5pv210/clock.c
index 019c3a69b0e4..b774ff1805db 100644
--- a/arch/arm/mach-s5pv210/clock.c
+++ b/arch/arm/mach-s5pv210/clock.c
@@ -467,20 +467,20 @@ static struct clk init_clocks_disable[] = {
                 .enable         = s5pv210_clk_ip3_ctrl,
                 .ctrlbit        = (1<<21),
         }, {
-                .name           = "i2s_v50",
+                .name           = "iis",
                 .id             = 0,
                 .parent         = &clk_p,
                 .enable         = s5pv210_clk_ip3_ctrl,
                 .ctrlbit        = (1<<4),
         }, {
-                .name           = "i2s_v32",
-                .id             = 0,
+                .name           = "iis",
+                .id             = 1,
                 .parent         = &clk_p,
                 .enable         = s5pv210_clk_ip3_ctrl,
                 .ctrlbit        = (1 << 5),
         }, {
-                .name           = "i2s_v32",
-                .id             = 1,
+                .name           = "iis",
+                .id             = 2,
                 .parent         = &clk_p,
                 .enable         = s5pv210_clk_ip3_ctrl,
                 .ctrlbit        = (1 << 6),
diff --git a/arch/arm/mach-s5pv210/dev-audio.c b/arch/arm/mach-s5pv210/dev-audio.c
index 1303fcb12b51..8d58f1926241 100644
--- a/arch/arm/mach-s5pv210/dev-audio.c
+++ b/arch/arm/mach-s5pv210/dev-audio.c
@@ -19,22 +19,24 @@
 #include <mach/dma.h>
 #include <mach/irqs.h>
 
+static const char *rclksrc[] = {
+        [0] = "busclk",
+        [1] = "i2sclk",
+};
+
 static int s5pv210_cfg_i2s(struct platform_device *pdev)
 {
         /* configure GPIO for i2s port */
         switch (pdev->id) {
+        case 0:
+                s3c_gpio_cfgpin_range(S5PV210_GPI(0), 7, S3C_GPIO_SFN(2));
+                break;
         case 1:
                 s3c_gpio_cfgpin_range(S5PV210_GPC0(0), 5, S3C_GPIO_SFN(2));
                 break;
-
         case 2:
                 s3c_gpio_cfgpin_range(S5PV210_GPC1(0), 5, S3C_GPIO_SFN(4));
                 break;
-
-        case -1:
-                s3c_gpio_cfgpin_range(S5PV210_GPI(0), 7, S3C_GPIO_SFN(2));
-                break;
-
         default:
                 printk(KERN_ERR "Invalid Device %d\n", pdev->id);
                 return -EINVAL;
@@ -43,8 +45,15 @@ static int s5pv210_cfg_i2s(struct platform_device *pdev)
         return 0;
 }
 
-static struct s3c_audio_pdata s3c_i2s_pdata = {
+static struct s3c_audio_pdata i2sv5_pdata = {
         .cfg_gpio = s5pv210_cfg_i2s,
+        .type = {
+                .i2s = {
+                        .quirks = QUIRK_PRI_6CHAN | QUIRK_SEC_DAI
+                                         | QUIRK_NEED_RSTCLR,
+                        .src_clk = rclksrc,
+                },
+        },
 };
 
 static struct resource s5pv210_iis0_resource[] = {
@@ -63,15 +72,34 @@ static struct resource s5pv210_iis0_resource[] = {
                 .end   = DMACH_I2S0_RX,
                 .flags = IORESOURCE_DMA,
         },
+        [3] = {
+                .start = DMACH_I2S0S_TX,
+                .end = DMACH_I2S0S_TX,
+                .flags = IORESOURCE_DMA,
+        },
 };
 
 struct platform_device s5pv210_device_iis0 = {
-        .name             = "s3c64xx-iis-v4",
-        .id               = -1,
+        .name = "samsung-i2s",
+        .id = 0,
         .num_resources    = ARRAY_SIZE(s5pv210_iis0_resource),
         .resource         = s5pv210_iis0_resource,
         .dev = {
-                .platform_data = &s3c_i2s_pdata,
+                .platform_data = &i2sv5_pdata,
+        },
+};
+
+static const char *rclksrc_v3[] = {
+        [0] = "iis",
+        [1] = "audio-bus",
+};
+
+static struct s3c_audio_pdata i2sv3_pdata = {
+        .cfg_gpio = s5pv210_cfg_i2s,
+        .type = {
+                .i2s = {
+                        .src_clk = rclksrc_v3,
+                },
         },
 };
 
@@ -94,12 +122,12 @@ static struct resource s5pv210_iis1_resource[] = {
 };
 
 struct platform_device s5pv210_device_iis1 = {
-        .name             = "s3c64xx-iis",
+        .name             = "samsung-i2s",
         .id               = 1,
         .num_resources    = ARRAY_SIZE(s5pv210_iis1_resource),
         .resource         = s5pv210_iis1_resource,
         .dev = {
-                .platform_data = &s3c_i2s_pdata,
+                .platform_data = &i2sv3_pdata,
         },
 };
 
@@ -122,12 +150,12 @@ static struct resource s5pv210_iis2_resource[] = {
 };
 
 struct platform_device s5pv210_device_iis2 = {
-        .name             = "s3c64xx-iis",
+        .name             = "samsung-i2s",
         .id               = 2,
         .num_resources    = ARRAY_SIZE(s5pv210_iis2_resource),
         .resource         = s5pv210_iis2_resource,
         .dev = {
-                .platform_data = &s3c_i2s_pdata,
+                .platform_data = &i2sv3_pdata,
         },
 };
 
@@ -283,7 +311,7 @@ static struct s3c_audio_pdata s3c_ac97_pdata = {
 static u64 s5pv210_ac97_dmamask = DMA_BIT_MASK(32);
 
 struct platform_device s5pv210_device_ac97 = {
-        .name             = "s3c-ac97",
+        .name             = "samsung-ac97",
         .id               = -1,
         .num_resources    = ARRAY_SIZE(s5pv210_ac97_resource),
         .resource         = s5pv210_ac97_resource,
diff --git a/arch/arm/mach-s5pv310/Kconfig b/arch/arm/mach-s5pv310/Kconfig
index 1150b360f38c..d64efe0d4c97 100644
--- a/arch/arm/mach-s5pv310/Kconfig
+++ b/arch/arm/mach-s5pv310/Kconfig
@@ -11,6 +11,7 @@ if ARCH_S5PV310
 
 config CPU_S5PV310
         bool
+        select S3C_PL330_DMA
         help
           Enable S5PV310 CPU support
 
diff --git a/arch/arm/mach-s5pv310/Makefile b/arch/arm/mach-s5pv310/Makefile
index 84afc64e7c01..61e3cb654269 100644
--- a/arch/arm/mach-s5pv310/Makefile
+++ b/arch/arm/mach-s5pv310/Makefile
@@ -13,7 +13,7 @@ obj-				:=
 # Core support for S5PV310 system
 
 obj-$(CONFIG_CPU_S5PV310)       += cpu.o init.o clock.o irq-combiner.o
-obj-$(CONFIG_CPU_S5PV310)       += setup-i2c0.o time.o gpiolib.o irq-eint.o
+obj-$(CONFIG_CPU_S5PV310)       += setup-i2c0.o time.o gpiolib.o irq-eint.o dma.o
 
 obj-$(CONFIG_SMP)               += platsmp.o headsmp.o
 obj-$(CONFIG_LOCAL_TIMERS)      += localtimer.o
@@ -27,6 +27,7 @@ obj-$(CONFIG_MACH_UNIVERSAL_C210)	+= mach-universal_c210.o
 
 # device support
 
+obj-y += dev-audio.o
 obj-$(CONFIG_S5PV310_SETUP_I2C1)        += setup-i2c1.o
 obj-$(CONFIG_S5PV310_SETUP_I2C2)        += setup-i2c2.o
 obj-$(CONFIG_S5PV310_SETUP_I2C3)        += setup-i2c3.o
diff --git a/arch/arm/mach-s5pv310/dev-audio.c b/arch/arm/mach-s5pv310/dev-audio.c
new file mode 100644
index 000000000000..a1964242f0fa
--- /dev/null
+++ b/arch/arm/mach-s5pv310/dev-audio.c
@@ -0,0 +1,364 @@
+/* linux/arch/arm/mach-s5pv310/dev-audio.c
+ *
+ * Copyright (c) 2010 Samsung Electronics Co. Ltd
+ *      Jaswinder Singh <jassi.brar@samsung.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/platform_device.h>
+#include <linux/dma-mapping.h>
+#include <linux/gpio.h>
+
+#include <plat/gpio-cfg.h>
+#include <plat/audio.h>
+
+#include <mach/map.h>
+#include <mach/dma.h>
+#include <mach/irqs.h>
+
+static const char *rclksrc[] = {
+        [0] = "busclk",
+        [1] = "i2sclk",
+};
+
+static int s5pv310_cfg_i2s(struct platform_device *pdev)
+{
+        /* configure GPIO for i2s port */
+        switch (pdev->id) {
+        case 0:
+                s3c_gpio_cfgpin_range(S5PV310_GPZ(0), 7, S3C_GPIO_SFN(2));
+                break;
+        case 1:
+                s3c_gpio_cfgpin_range(S5PV310_GPC0(0), 5, S3C_GPIO_SFN(2));
+                break;
+        case 2:
+                s3c_gpio_cfgpin_range(S5PV310_GPC1(0), 5, S3C_GPIO_SFN(4));
+                break;
+        default:
+                printk(KERN_ERR "Invalid Device %d\n", pdev->id);
+                return -EINVAL;
+        }
+
+        return 0;
+}
+
+static struct s3c_audio_pdata i2sv5_pdata = {
+        .cfg_gpio = s5pv310_cfg_i2s,
+        .type = {
+                .i2s = {
+                        .quirks = QUIRK_PRI_6CHAN | QUIRK_SEC_DAI
+                                         | QUIRK_NEED_RSTCLR,
+                        .src_clk = rclksrc,
+                },
+        },
+};
+
+static struct resource s5pv310_i2s0_resource[] = {
+        [0] = {
+                .start  = S5PV310_PA_I2S0,
+                .end    = S5PV310_PA_I2S0 + 0x100 - 1,
+                .flags  = IORESOURCE_MEM,
+        },
+        [1] = {
+                .start  = DMACH_I2S0_TX,
+                .end    = DMACH_I2S0_TX,
+                .flags  = IORESOURCE_DMA,
+        },
+        [2] = {
+                .start  = DMACH_I2S0_RX,
+                .end    = DMACH_I2S0_RX,
+                .flags  = IORESOURCE_DMA,
+        },
+        [3] = {
+                .start  = DMACH_I2S0S_TX,
+                .end    = DMACH_I2S0S_TX,
+                .flags  = IORESOURCE_DMA,
+        },
+};
+
+struct platform_device s5pv310_device_i2s0 = {
+        .name = "samsung-i2s",
+        .id = 0,
+        .num_resources = ARRAY_SIZE(s5pv310_i2s0_resource),
+        .resource = s5pv310_i2s0_resource,
+        .dev = {
+                .platform_data = &i2sv5_pdata,
+        },
+};
+
+static const char *rclksrc_v3[] = {
+        [0] = "sclk_i2s",
+        [1] = "no_such_clock",
+};
+
+static struct s3c_audio_pdata i2sv3_pdata = {
+        .cfg_gpio = s5pv310_cfg_i2s,
+        .type = {
+                .i2s = {
+                        .quirks = QUIRK_NO_MUXPSR,
+                        .src_clk = rclksrc_v3,
+                },
+        },
+};
+
+static struct resource s5pv310_i2s1_resource[] = {
+        [0] = {
+                .start  = S5PV310_PA_I2S1,
+                .end    = S5PV310_PA_I2S1 + 0x100 - 1,
+                .flags  = IORESOURCE_MEM,
+        },
+        [1] = {
+                .start  = DMACH_I2S1_TX,
+                .end    = DMACH_I2S1_TX,
+                .flags  = IORESOURCE_DMA,
+        },
+        [2] = {
+                .start  = DMACH_I2S1_RX,
+                .end    = DMACH_I2S1_RX,
+                .flags  = IORESOURCE_DMA,
+        },
+};
+
+struct platform_device s5pv310_device_i2s1 = {
+        .name = "samsung-i2s",
+        .id = 1,
+        .num_resources = ARRAY_SIZE(s5pv310_i2s1_resource),
+        .resource = s5pv310_i2s1_resource,
+        .dev = {
+                .platform_data = &i2sv3_pdata,
+        },
+};
+
+static struct resource s5pv310_i2s2_resource[] = {
+        [0] = {
+                .start  = S5PV310_PA_I2S2,
+                .end    = S5PV310_PA_I2S2 + 0x100 - 1,
+                .flags  = IORESOURCE_MEM,
+        },
+        [1] = {
+                .start  = DMACH_I2S2_TX,
+                .end    = DMACH_I2S2_TX,
+                .flags  = IORESOURCE_DMA,
+        },
+        [2] = {
+                .start  = DMACH_I2S2_RX,
+                .end    = DMACH_I2S2_RX,
+                .flags  = IORESOURCE_DMA,
+        },
+};
+
+struct platform_device s5pv310_device_i2s2 = {
+        .name = "samsung-i2s",
+        .id = 2,
+        .num_resources = ARRAY_SIZE(s5pv310_i2s2_resource),
+        .resource = s5pv310_i2s2_resource,
+        .dev = {
+                .platform_data = &i2sv3_pdata,
+        },
+};
+
+/* PCM Controller platform_devices */
+
+static int s5pv310_pcm_cfg_gpio(struct platform_device *pdev)
+{
+        switch (pdev->id) {
+        case 0:
+                s3c_gpio_cfgpin_range(S5PV310_GPZ(0), 5, S3C_GPIO_SFN(3));
+                break;
+        case 1:
+                s3c_gpio_cfgpin_range(S5PV310_GPC0(0), 5, S3C_GPIO_SFN(3));
+                break;
+        case 2:
+                s3c_gpio_cfgpin_range(S5PV310_GPC1(0), 5, S3C_GPIO_SFN(3));
+                break;
+        default:
+                printk(KERN_DEBUG "Invalid PCM Controller number!");
+                return -EINVAL;
+        }
+
+        return 0;
+}
+
+static struct s3c_audio_pdata s3c_pcm_pdata = {
+        .cfg_gpio = s5pv310_pcm_cfg_gpio,
+};
+
+static struct resource s5pv310_pcm0_resource[] = {
+        [0] = {
+                .start  = S5PV310_PA_PCM0,
+                .end    = S5PV310_PA_PCM0 + 0x100 - 1,
+                .flags  = IORESOURCE_MEM,
+        },
+        [1] = {
+                .start  = DMACH_PCM0_TX,
+                .end    = DMACH_PCM0_TX,
+                .flags  = IORESOURCE_DMA,
+        },
+        [2] = {
+                .start  = DMACH_PCM0_RX,
+                .end    = DMACH_PCM0_RX,
+                .flags  = IORESOURCE_DMA,
+        },
+};
+
+struct platform_device s5pv310_device_pcm0 = {
+        .name = "samsung-pcm",
+        .id = 0,
+        .num_resources = ARRAY_SIZE(s5pv310_pcm0_resource),
+        .resource = s5pv310_pcm0_resource,
+        .dev = {
+                .platform_data = &s3c_pcm_pdata,
+        },
+};
+
+static struct resource s5pv310_pcm1_resource[] = {
+        [0] = {
+                .start  = S5PV310_PA_PCM1,
+                .end    = S5PV310_PA_PCM1 + 0x100 - 1,
+                .flags  = IORESOURCE_MEM,
+        },
+        [1] = {
+                .start  = DMACH_PCM1_TX,
+                .end    = DMACH_PCM1_TX,
+                .flags  = IORESOURCE_DMA,
+        },
+        [2] = {
+                .start  = DMACH_PCM1_RX,
+                .end    = DMACH_PCM1_RX,
+                .flags  = IORESOURCE_DMA,
+        },
+};
+
+struct platform_device s5pv310_device_pcm1 = {
+        .name = "samsung-pcm",
+        .id = 1,
+        .num_resources = ARRAY_SIZE(s5pv310_pcm1_resource),
+        .resource = s5pv310_pcm1_resource,
+        .dev = {
+                .platform_data = &s3c_pcm_pdata,
+        },
+};
+
+static struct resource s5pv310_pcm2_resource[] = {
+        [0] = {
+                .start  = S5PV310_PA_PCM2,
+                .end    = S5PV310_PA_PCM2 + 0x100 - 1,
+                .flags  = IORESOURCE_MEM,
+        },
+        [1] = {
+                .start  = DMACH_PCM2_TX,
+                .end    = DMACH_PCM2_TX,
+                .flags  = IORESOURCE_DMA,
+        },
+        [2] = {
+                .start  = DMACH_PCM2_RX,
+                .end    = DMACH_PCM2_RX,
+                .flags  = IORESOURCE_DMA,
+        },
+};
+
+struct platform_device s5pv310_device_pcm2 = {
+        .name = "samsung-pcm",
+        .id = 2,
+        .num_resources = ARRAY_SIZE(s5pv310_pcm2_resource),
+        .resource = s5pv310_pcm2_resource,
+        .dev = {
+                .platform_data = &s3c_pcm_pdata,
+        },
+};
+
+/* AC97 Controller platform devices */
+
+static int s5pv310_ac97_cfg_gpio(struct platform_device *pdev)
+{
+        return s3c_gpio_cfgpin_range(S5PV310_GPC0(0), 5, S3C_GPIO_SFN(4));
+}
+
+static struct resource s5pv310_ac97_resource[] = {
+        [0] = {
+                .start  = S5PV310_PA_AC97,
+                .end    = S5PV310_PA_AC97 + 0x100 - 1,
+                .flags  = IORESOURCE_MEM,
+        },
+        [1] = {
+                .start  = DMACH_AC97_PCMOUT,
+                .end    = DMACH_AC97_PCMOUT,
+                .flags  = IORESOURCE_DMA,
+        },
+        [2] = {
+                .start  = DMACH_AC97_PCMIN,
+                .end    = DMACH_AC97_PCMIN,
+                .flags  = IORESOURCE_DMA,
+        },
+        [3] = {
+                .start  = DMACH_AC97_MICIN,
+                .end    = DMACH_AC97_MICIN,
+                .flags  = IORESOURCE_DMA,
+        },
+        [4] = {
+                .start  = IRQ_AC97,
+                .end    = IRQ_AC97,
+                .flags  = IORESOURCE_IRQ,
+        },
+};
+
+static struct s3c_audio_pdata s3c_ac97_pdata = {
+        .cfg_gpio = s5pv310_ac97_cfg_gpio,
+};
+
+static u64 s5pv310_ac97_dmamask = DMA_BIT_MASK(32);
+
+struct platform_device s5pv310_device_ac97 = {
+        .name = "samsung-ac97",
+        .id = -1,
+        .num_resources = ARRAY_SIZE(s5pv310_ac97_resource),
+        .resource = s5pv310_ac97_resource,
+        .dev = {
+                .platform_data = &s3c_ac97_pdata,
+                .dma_mask = &s5pv310_ac97_dmamask,
+                .coherent_dma_mask = DMA_BIT_MASK(32),
+        },
+};
+
+/* S/PDIF Controller platform_device */
+
+static int s5pv310_spdif_cfg_gpio(struct platform_device *pdev)
+{
+        s3c_gpio_cfgpin_range(S5PV310_GPC1(0), 2, S3C_GPIO_SFN(3));
+
+        return 0;
+}
+
+static struct resource s5pv310_spdif_resource[] = {
+        [0] = {
+                .start  = S5PV310_PA_SPDIF,
+                .end    = S5PV310_PA_SPDIF + 0x100 - 1,
+                .flags  = IORESOURCE_MEM,
+        },
+        [1] = {
+                .start  = DMACH_SPDIF,
+                .end    = DMACH_SPDIF,
+                .flags  = IORESOURCE_DMA,
+        },
+};
+
+static struct s3c_audio_pdata samsung_spdif_pdata = {
+        .cfg_gpio = s5pv310_spdif_cfg_gpio,
+};
+
+static u64 s5pv310_spdif_dmamask = DMA_BIT_MASK(32);
+
+struct platform_device s5pv310_device_spdif = {
+        .name = "samsung-spdif",
+        .id = -1,
+        .num_resources = ARRAY_SIZE(s5pv310_spdif_resource),
+        .resource = s5pv310_spdif_resource,
+        .dev = {
+                .platform_data = &samsung_spdif_pdata,
+                .dma_mask = &s5pv310_spdif_dmamask,
+                .coherent_dma_mask = DMA_BIT_MASK(32),
+        },
+};
diff --git a/arch/arm/mach-s5pv310/dma.c b/arch/arm/mach-s5pv310/dma.c
new file mode 100644
index 000000000000..20066c7c9e56
--- /dev/null
+++ b/arch/arm/mach-s5pv310/dma.c
@@ -0,0 +1,168 @@
+/*
+ * Copyright (C) 2010 Samsung Electronics Co. Ltd.
+ *      Jaswinder Singh <jassi.brar@samsung.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+#include <linux/platform_device.h>
+#include <linux/dma-mapping.h>
+
+#include <plat/devs.h>
+#include <plat/irqs.h>
+
+#include <mach/map.h>
+#include <mach/irqs.h>
+
+#include <plat/s3c-pl330-pdata.h>
+
+static u64 dma_dmamask = DMA_BIT_MASK(32);
+
+static struct resource s5pv310_pdma0_resource[] = {
+        [0] = {
+                .start  = S5PV310_PA_PDMA0,
+                .end    = S5PV310_PA_PDMA0 + SZ_4K,
+                .flags  = IORESOURCE_MEM,
+        },
+        [1] = {
+                .start  = IRQ_PDMA0,
+                .end    = IRQ_PDMA0,
+                .flags  = IORESOURCE_IRQ,
+        },
+};
+
+static struct s3c_pl330_platdata s5pv310_pdma0_pdata = {
+        .peri = {
+                [0] = DMACH_PCM0_RX,
+                [1] = DMACH_PCM0_TX,
+                [2] = DMACH_PCM2_RX,
+                [3] = DMACH_PCM2_TX,
+                [4] = DMACH_MSM_REQ0,
+                [5] = DMACH_MSM_REQ2,
+                [6] = DMACH_SPI0_RX,
+                [7] = DMACH_SPI0_TX,
+                [8] = DMACH_SPI2_RX,
+                [9] = DMACH_SPI2_TX,
+                [10] = DMACH_I2S0S_TX,
+                [11] = DMACH_I2S0_RX,
+                [12] = DMACH_I2S0_TX,
+                [13] = DMACH_I2S2_RX,
+                [14] = DMACH_I2S2_TX,
+                [15] = DMACH_UART0_RX,
+                [16] = DMACH_UART0_TX,
+                [17] = DMACH_UART2_RX,
+                [18] = DMACH_UART2_TX,
+                [19] = DMACH_UART4_RX,
+                [20] = DMACH_UART4_TX,
+                [21] = DMACH_SLIMBUS0_RX,
+                [22] = DMACH_SLIMBUS0_TX,
+                [23] = DMACH_SLIMBUS2_RX,
+                [24] = DMACH_SLIMBUS2_TX,
+                [25] = DMACH_SLIMBUS4_RX,
+                [26] = DMACH_SLIMBUS4_TX,
+                [27] = DMACH_AC97_MICIN,
+                [28] = DMACH_AC97_PCMIN,
+                [29] = DMACH_AC97_PCMOUT,
+                [30] = DMACH_MAX,
+                [31] = DMACH_MAX,
+        },
+};
+
+static struct platform_device s5pv310_device_pdma0 = {
+        .name           = "s3c-pl330",
+        .id             = 0,
+        .num_resources  = ARRAY_SIZE(s5pv310_pdma0_resource),
+        .resource       = s5pv310_pdma0_resource,
+        .dev            = {
+                .dma_mask = &dma_dmamask,
+                .coherent_dma_mask = DMA_BIT_MASK(32),
+                .platform_data = &s5pv310_pdma0_pdata,
+        },
+};
+
+static struct resource s5pv310_pdma1_resource[] = {
+        [0] = {
+                .start  = S5PV310_PA_PDMA1,
+                .end    = S5PV310_PA_PDMA1 + SZ_4K,
+                .flags  = IORESOURCE_MEM,
+        },
+        [1] = {
+                .start  = IRQ_PDMA1,
+                .end    = IRQ_PDMA1,
+                .flags  = IORESOURCE_IRQ,
+        },
+};
+
+static struct s3c_pl330_platdata s5pv310_pdma1_pdata = {
+        .peri = {
+                [0] = DMACH_PCM0_RX,
+                [1] = DMACH_PCM0_TX,
+                [2] = DMACH_PCM1_RX,
+                [3] = DMACH_PCM1_TX,
+                [4] = DMACH_MSM_REQ1,
+                [5] = DMACH_MSM_REQ3,
+                [6] = DMACH_SPI1_RX,
+                [7] = DMACH_SPI1_TX,
+                [8] = DMACH_I2S0S_TX,
+                [9] = DMACH_I2S0_RX,
+                [10] = DMACH_I2S0_TX,
+                [11] = DMACH_I2S1_RX,
+                [12] = DMACH_I2S1_TX,
+                [13] = DMACH_UART0_RX,
+                [14] = DMACH_UART0_TX,
+                [15] = DMACH_UART1_RX,
+                [16] = DMACH_UART1_TX,
+                [17] = DMACH_UART3_RX,
+                [18] = DMACH_UART3_TX,
+                [19] = DMACH_SLIMBUS1_RX,
+                [20] = DMACH_SLIMBUS1_TX,
+                [21] = DMACH_SLIMBUS3_RX,
+                [22] = DMACH_SLIMBUS3_TX,
+                [23] = DMACH_SLIMBUS5_RX,
+                [24] = DMACH_SLIMBUS5_TX,
+                [25] = DMACH_SLIMBUS0AUX_RX,
+                [26] = DMACH_SLIMBUS0AUX_TX,
+                [27] = DMACH_SPDIF,
+                [28] = DMACH_MAX,
+                [29] = DMACH_MAX,
+                [30] = DMACH_MAX,
+                [31] = DMACH_MAX,
+        },
+};
+
+static struct platform_device s5pv310_device_pdma1 = {
+        .name           = "s3c-pl330",
+        .id             = 1,
+        .num_resources  = ARRAY_SIZE(s5pv310_pdma1_resource),
+        .resource       = s5pv310_pdma1_resource,
+        .dev            = {
+                .dma_mask = &dma_dmamask,
+                .coherent_dma_mask = DMA_BIT_MASK(32),
+                .platform_data = &s5pv310_pdma1_pdata,
+        },
+};
+
+static struct platform_device *s5pv310_dmacs[] __initdata = {
+        &s5pv310_device_pdma0,
+        &s5pv310_device_pdma1,
+};
+
+static int __init s5pv310_dma_init(void)
+{
+        platform_add_devices(s5pv310_dmacs, ARRAY_SIZE(s5pv310_dmacs));
+
+        return 0;
+}
+arch_initcall(s5pv310_dma_init);
diff --git a/arch/arm/mach-s5pv310/include/mach/dma.h b/arch/arm/mach-s5pv310/include/mach/dma.h
new file mode 100644
index 000000000000..81209eb1409b
--- /dev/null
+++ b/arch/arm/mach-s5pv310/include/mach/dma.h
@@ -0,0 +1,26 @@
+/*
+ * Copyright (C) 2010 Samsung Electronics Co. Ltd.
+ *      Jaswinder Singh <jassi.brar@samsung.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+#ifndef __MACH_DMA_H
+#define __MACH_DMA_H
+
+/* This platform uses the common S3C DMA API driver for PL330 */
+#include <plat/s3c-dma-pl330.h>
+
+#endif /* __MACH_DMA_H */
diff --git a/arch/arm/mach-s5pv310/include/mach/irqs.h b/arch/arm/mach-s5pv310/include/mach/irqs.h
index 99e7dad8a85a..3c05c58b5392 100644
--- a/arch/arm/mach-s5pv310/include/mach/irqs.h
+++ b/arch/arm/mach-s5pv310/include/mach/irqs.h
@@ -54,6 +54,9 @@
 #define COMBINER_GROUP(x)       ((x) * MAX_IRQ_IN_COMBINER + IRQ_SPI(64))
 #define COMBINER_IRQ(x, y)      (COMBINER_GROUP(x) + y)
 
+#define IRQ_PDMA0               COMBINER_IRQ(21, 0)
+#define IRQ_PDMA1               COMBINER_IRQ(21, 1)
+
 #define IRQ_TIMER0_VIC          COMBINER_IRQ(22, 0)
 #define IRQ_TIMER1_VIC          COMBINER_IRQ(22, 1)
 #define IRQ_TIMER2_VIC          COMBINER_IRQ(22, 2)
diff --git a/arch/arm/mach-s5pv310/include/mach/map.h b/arch/arm/mach-s5pv310/include/mach/map.h
index 7acf4e77e92e..53994467605d 100644
--- a/arch/arm/mach-s5pv310/include/mach/map.h
+++ b/arch/arm/mach-s5pv310/include/mach/map.h
@@ -52,6 +52,11 @@
 #define S5PV310_PA_GIC_DIST             (0x10501000)
 #define S5PV310_PA_L2CC                 (0x10502000)
 
+/* DMA */
+#define S5PV310_PA_MDMA         0x10810000
+#define S5PV310_PA_PDMA0        0x12680000
+#define S5PV310_PA_PDMA1        0x12690000
+
 #define S5PV310_PA_GPIO1                (0x11400000)
 #define S5PV310_PA_GPIO2                (0x11000000)
 #define S5PV310_PA_GPIO3                (0x03860000)
@@ -60,6 +65,22 @@
 
 #define S5PV310_PA_SROMC                (0x12570000)
 
+/* S/PDIF */
+#define S5PV310_PA_SPDIF        0xE1100000
+
+/* I2S */
+#define S5PV310_PA_I2S0         0x03830000
+#define S5PV310_PA_I2S1         0xE3100000
+#define S5PV310_PA_I2S2         0xE2A00000
+
+/* PCM */
+#define S5PV310_PA_PCM0         0x03840000
+#define S5PV310_PA_PCM1         0x13980000
+#define S5PV310_PA_PCM2         0x13990000
+
+/* AC97 */
+#define S5PV310_PA_AC97         0x139A0000
+
 #define S5PV310_PA_UART                 (0x13800000)
 
 #define S5P_PA_UART(x)                  (S5PV310_PA_UART + ((x) * S3C_UART_OFFSET))
diff --git a/arch/arm/mach-sa1100/pm.c b/arch/arm/mach-sa1100/pm.c
index c83fdc80edfd..ab9fc4470d36 100644
--- a/arch/arm/mach-sa1100/pm.c
+++ b/arch/arm/mach-sa1100/pm.c
@@ -120,7 +120,7 @@ unsigned long sleep_phys_sp(void *sp)
         return virt_to_phys(sp);
 }
 
-static struct platform_suspend_ops sa11x0_pm_ops = {
+static const struct platform_suspend_ops sa11x0_pm_ops = {
         .enter          = sa11x0_pm_enter,
         .valid          = suspend_valid_only_mem,
 };
diff --git a/arch/arm/mach-shmobile/board-ap4evb.c b/arch/arm/mach-shmobile/board-ap4evb.c
index cd79d7c1ba0d..3cf0951caa2d 100644
--- a/arch/arm/mach-shmobile/board-ap4evb.c
+++ b/arch/arm/mach-shmobile/board-ap4evb.c
@@ -247,10 +247,7 @@ static struct platform_device smc911x_device = {
  */
 static int slot_cn7_get_cd(struct platform_device *pdev)
 {
-        if (gpio_is_valid(GPIO_PORT41))
-                return !gpio_get_value(GPIO_PORT41);
-        else
-                return -ENXIO;
+        return !gpio_get_value(GPIO_PORT41);
 }
 
 /* SH_MMCIF */
@@ -308,6 +305,7 @@ static struct platform_device sh_mmcif_device = {
 static struct sh_mobile_sdhi_info sdhi0_info = {
         .dma_slave_tx   = SHDMA_SLAVE_SDHI0_TX,
         .dma_slave_rx   = SHDMA_SLAVE_SDHI0_RX,
+        .tmio_caps      = MMC_CAP_SDIO_IRQ,
 };
 
 static struct resource sdhi0_resources[] = {
@@ -339,7 +337,7 @@ static struct sh_mobile_sdhi_info sdhi1_info = {
         .dma_slave_rx   = SHDMA_SLAVE_SDHI1_RX,
         .tmio_ocr_mask  = MMC_VDD_165_195,
         .tmio_flags     = TMIO_MMC_WRPROTECT_DISABLE,
-        .tmio_caps      = MMC_CAP_NEEDS_POLL,
+        .tmio_caps      = MMC_CAP_NEEDS_POLL | MMC_CAP_SDIO_IRQ,
         .get_cd         = slot_cn7_get_cd,
 };
 
@@ -711,6 +709,10 @@ static struct platform_device fsi_device = {
         },
 };
 
+static struct platform_device fsi_ak4643_device = {
+        .name           = "sh_fsi2_a_ak4643",
+};
+
 static struct sh_mobile_lcdc_info sh_mobile_lcdc1_info = {
         .clock_source = LCDC_CLK_EXTERNAL,
         .ch[0] = {
@@ -933,6 +935,7 @@ static struct platform_device *ap4evb_devices[] __initdata = {
         &sdhi1_device,
         &usb1_host_device,
         &fsi_device,
+        &fsi_ak4643_device,
         &sh_mmcif_device,
         &lcdc1_device,
         &lcdc_device,
diff --git a/arch/arm/mach-shmobile/board-g4evm.c b/arch/arm/mach-shmobile/board-g4evm.c
index c13f01280b7e..dee3e9231fb9 100644
--- a/arch/arm/mach-shmobile/board-g4evm.c
+++ b/arch/arm/mach-shmobile/board-g4evm.c
@@ -30,6 +30,7 @@
 #include <linux/io.h>
 #include <linux/input.h>
 #include <linux/input/sh_keysc.h>
+#include <linux/mmc/host.h>
 #include <linux/mfd/sh_mobile_sdhi.h>
 #include <linux/gpio.h>
 #include <mach/sh7377.h>
@@ -196,6 +197,10 @@ static struct platform_device keysc_device = {
 };
 
 /* SDHI */
+static struct sh_mobile_sdhi_info sdhi0_info = {
+        .tmio_caps      = MMC_CAP_SDIO_IRQ,
+};
+
 static struct resource sdhi0_resources[] = {
         [0] = {
                 .name   = "SDHI0",
@@ -214,6 +219,13 @@ static struct platform_device sdhi0_device = {
         .num_resources  = ARRAY_SIZE(sdhi0_resources),
         .resource       = sdhi0_resources,
         .id             = 0,
+        .dev    = {
+                .platform_data  = &sdhi0_info,
+        },
+};
+
+static struct sh_mobile_sdhi_info sdhi1_info = {
+        .tmio_caps      = MMC_CAP_NONREMOVABLE | MMC_CAP_SDIO_IRQ,
 };
 
 static struct resource sdhi1_resources[] = {
@@ -234,6 +246,9 @@ static struct platform_device sdhi1_device = {
         .num_resources  = ARRAY_SIZE(sdhi1_resources),
         .resource       = sdhi1_resources,
         .id             = 1,
+        .dev    = {
+                .platform_data  = &sdhi1_info,
+        },
 };
 
 static struct platform_device *g4evm_devices[] __initdata = {
diff --git a/arch/arm/mach-shmobile/board-mackerel.c b/arch/arm/mach-shmobile/board-mackerel.c
index 5bcf5c1e1399..7b15d21f0f68 100644
--- a/arch/arm/mach-shmobile/board-mackerel.c
+++ b/arch/arm/mach-shmobile/board-mackerel.c
@@ -657,17 +657,14 @@ static struct platform_device fsi_ak4643_device = {
  */
 static int slot_cn7_get_cd(struct platform_device *pdev)
 {
-        if (gpio_is_valid(GPIO_PORT41))
-                return !gpio_get_value(GPIO_PORT41);
-        else
-                return -ENXIO;
+        return !gpio_get_value(GPIO_PORT41);
 }
 
 /* SDHI0 */
 static struct sh_mobile_sdhi_info sdhi0_info = {
         .dma_slave_tx   = SHDMA_SLAVE_SDHI0_TX,
         .dma_slave_rx   = SHDMA_SLAVE_SDHI0_RX,
-        .tmio_caps      = MMC_CAP_SD_HIGHSPEED,
+        .tmio_caps      = MMC_CAP_SD_HIGHSPEED | MMC_CAP_SDIO_IRQ,
 };
 
 static struct resource sdhi0_resources[] = {
@@ -700,7 +697,7 @@ static struct sh_mobile_sdhi_info sdhi1_info = {
         .dma_slave_rx   = SHDMA_SLAVE_SDHI1_RX,
         .tmio_ocr_mask  = MMC_VDD_165_195,
         .tmio_flags     = TMIO_MMC_WRPROTECT_DISABLE,
-        .tmio_caps      = MMC_CAP_SD_HIGHSPEED |
+        .tmio_caps      = MMC_CAP_SD_HIGHSPEED | MMC_CAP_SDIO_IRQ |
                           MMC_CAP_NEEDS_POLL,
         .get_cd         = slot_cn7_get_cd,
 };
@@ -729,13 +726,23 @@ static struct platform_device sdhi1_device = {
 };
 #endif
 
+/*
+ * The card detect pin of the top SD/MMC slot (CN23) is active low and is
+ * connected to GPIO SCIFB_SCK of SH7372 (GPIO_PORT162).
+ */
+static int slot_cn23_get_cd(struct platform_device *pdev)
+{
+        return !gpio_get_value(GPIO_PORT162);
+}
+
 /* SDHI2 */
 static struct sh_mobile_sdhi_info sdhi2_info = {
         .dma_slave_tx   = SHDMA_SLAVE_SDHI2_TX,
         .dma_slave_rx   = SHDMA_SLAVE_SDHI2_RX,
         .tmio_flags     = TMIO_MMC_WRPROTECT_DISABLE,
-        .tmio_caps      = MMC_CAP_SD_HIGHSPEED |
+        .tmio_caps      = MMC_CAP_SD_HIGHSPEED | MMC_CAP_SDIO_IRQ |
                           MMC_CAP_NEEDS_POLL,
+        .get_cd         = slot_cn23_get_cd,
 };
 
 static struct resource sdhi2_resources[] = {
@@ -953,6 +960,7 @@ static struct tca6416_keys_platform_data mackerel_tca6416_keys_info = {
 };
 
 /* I2C */
+#define IRQ7 evt2irq(0x02e0)
 #define IRQ9 evt2irq(0x0320)
 
 static struct i2c_board_info i2c0_devices[] = {
@@ -965,6 +973,11 @@ static struct i2c_board_info i2c0_devices[] = {
                 .platform_data = &mackerel_tca6416_keys_info,
                 .irq = IRQ9,
         },
+        /* Touchscreen */
+        {
+                I2C_BOARD_INFO("st1232-ts", 0x55),
+                .irq = IRQ7,
+        },
 };
 
 #define IRQ21 evt2irq(0x32a0)
@@ -1092,6 +1105,10 @@ static void __init mackerel_init(void)
         gpio_request(GPIO_FN_IRQ9_42,   NULL);
         set_irq_type(IRQ9, IRQ_TYPE_LEVEL_HIGH);
 
+        /* enable Touchscreen */
+        gpio_request(GPIO_FN_IRQ7_40,   NULL);
+        set_irq_type(IRQ7, IRQ_TYPE_LEVEL_LOW);
+
         /* enable Accelerometer */
         gpio_request(GPIO_FN_IRQ21,     NULL);
         set_irq_type(IRQ21, IRQ_TYPE_LEVEL_HIGH);
@@ -1127,6 +1144,10 @@ static void __init mackerel_init(void)
         gpio_request(GPIO_FN_SDHID2_1, NULL);
         gpio_request(GPIO_FN_SDHID2_0, NULL);
 
+        /* card detect pin for microSD slot (CN23) */
+        gpio_request(GPIO_PORT162, NULL);
+        gpio_direction_input(GPIO_PORT162);
+
         /* MMCIF */
         gpio_request(GPIO_FN_MMCD0_0, NULL);
         gpio_request(GPIO_FN_MMCD0_1, NULL);
diff --git a/arch/arm/mach-shmobile/include/mach/head-ap4evb.txt b/arch/arm/mach-shmobile/include/mach/head-ap4evb.txt
index e3ebfa73956e..efd3687ba190 100644
--- a/arch/arm/mach-shmobile/include/mach/head-ap4evb.txt
+++ b/arch/arm/mach-shmobile/include/mach/head-ap4evb.txt
@@ -85,3 +85,10 @@ ED 0xE6150004, 0x80331050
 WAIT 1, 0xFE40009C
 
 ED 0xE6150354, 0x00000002
+
+LIST "SCIF0 - Serial port for earlyprintk"
+EB 0xE6053098, 0x11
+EB 0xE6053098, 0xe1
+EW 0xE6C40000, 0x0000
+EB 0xE6C40004, 0x19
+EW 0xE6C40008, 0x3000
diff --git a/arch/arm/mach-shmobile/include/mach/head-mackerel.txt b/arch/arm/mach-shmobile/include/mach/head-mackerel.txt
index e3ebfa73956e..efd3687ba190 100644
--- a/arch/arm/mach-shmobile/include/mach/head-mackerel.txt
+++ b/arch/arm/mach-shmobile/include/mach/head-mackerel.txt
@@ -85,3 +85,10 @@ ED 0xE6150004, 0x80331050
 WAIT 1, 0xFE40009C
 
 ED 0xE6150354, 0x00000002
+
+LIST "SCIF0 - Serial port for earlyprintk"
+EB 0xE6053098, 0x11
+EB 0xE6053098, 0xe1
+EW 0xE6C40000, 0x0000
+EB 0xE6C40004, 0x19
+EW 0xE6C40008, 0x3000
diff --git a/arch/arm/mach-shmobile/intc-sh7367.c b/arch/arm/mach-shmobile/intc-sh7367.c
index 1a20c489b20d..2fe9704d5ea1 100644
--- a/arch/arm/mach-shmobile/intc-sh7367.c
+++ b/arch/arm/mach-shmobile/intc-sh7367.c
@@ -189,10 +189,10 @@ static struct intc_mask_reg intca_mask_registers[] __initdata = {
           { SCIFB, SCIFA5, SCIFA4, MSIOF1,
             0, 0, MSIOF2, 0 } },
         { 0xe694009c, 0xe69400dc, 8, /* IMR7A / IMCR7A */
-          { DISABLED, DISABLED, ENABLED, ENABLED,
+          { DISABLED, ENABLED, ENABLED, ENABLED,
             FLCTL_FLTREQ1I, FLCTL_FLTREQ0I, FLCTL_FLTENDI, FLCTL_FLSTEI } },
         { 0xe69400a0, 0xe69400e0, 8, /* IMR8A / IMCR8A */
-          { DISABLED, DISABLED, ENABLED, ENABLED,
+          { DISABLED, ENABLED, ENABLED, ENABLED,
             TTI20, USBDMAC_USHDMI, SPU, SIU } },
         { 0xe69400a4, 0xe69400e4, 8, /* IMR9A / IMCR9A */
           { CMT1_CMT13, CMT1_CMT12, CMT1_CMT11, CMT1_CMT10,
@@ -207,7 +207,7 @@ static struct intc_mask_reg intca_mask_registers[] __initdata = {
           { 0, 0, TPU0, TPU1,
             TPU2, TPU3, TPU4, 0 } },
         { 0xe69400b4, 0xe69400f4, 8, /* IMR13A / IMCR13A */
-          { DISABLED, DISABLED, ENABLED, ENABLED,
+          { DISABLED, ENABLED, ENABLED, ENABLED,
             MISTY, CMT3, RWDT1, RWDT0 } },
 };
 
diff --git a/arch/arm/mach-shmobile/intc-sh7372.c b/arch/arm/mach-shmobile/intc-sh7372.c
index 30b2f400666a..f78a1ead71a5 100644
--- a/arch/arm/mach-shmobile/intc-sh7372.c
+++ b/arch/arm/mach-shmobile/intc-sh7372.c
@@ -230,10 +230,10 @@ static struct intc_mask_reg intca_mask_registers[] __initdata = {
           { SCIFB, SCIFA5, SCIFA4, MSIOF1,
             0, 0, MSIOF2, 0 } },
         { 0xe694009c, 0xe69400dc, 8, /* IMR7A / IMCR7A */
-          { DISABLED, DISABLED, ENABLED, ENABLED,
+          { DISABLED, ENABLED, ENABLED, ENABLED,
             FLCTL_FLTREQ1I, FLCTL_FLTREQ0I, FLCTL_FLTENDI, FLCTL_FLSTEI } },
         { 0xe69400a0, 0xe69400e0, 8, /* IMR8A / IMCR8A */
-          { 0, DISABLED, ENABLED, ENABLED,
+          { 0, ENABLED, ENABLED, ENABLED,
             TTI20, USBHSDMAC0_USHDMI, 0, 0 } },
         { 0xe69400a4, 0xe69400e4, 8, /* IMR9A / IMCR9A */
           { CMT1_CMT13, CMT1_CMT12, CMT1_CMT11, CMT1_CMT10,
diff --git a/arch/arm/mach-shmobile/intc-sh7377.c b/arch/arm/mach-shmobile/intc-sh7377.c
index 2cdeb8ccd821..dd568382cc9f 100644
--- a/arch/arm/mach-shmobile/intc-sh7377.c
+++ b/arch/arm/mach-shmobile/intc-sh7377.c
@@ -234,10 +234,10 @@ static struct intc_mask_reg intca_mask_registers[] __initdata = {
           { SCIFB, SCIFA5, SCIFA4, MSIOF1,
             0, 0, MSIOF2, 0 } },
         { 0xe694009c, 0xe69400dc, 8, /* IMR7A / IMCR7A */
-          { DISABLED, DISABLED, ENABLED, ENABLED,
+          { DISABLED, ENABLED, ENABLED, ENABLED,
             FLCTL_FLTREQ1I, FLCTL_FLTREQ0I, FLCTL_FLTENDI, FLCTL_FLSTEI } },
         { 0xe69400a0, 0xe69400e0, 8, /* IMR8A / IMCR8A */
-          { DISABLED, DISABLED, ENABLED, ENABLED,
+          { DISABLED, ENABLED, ENABLED, ENABLED,
             TTI20, USBDMAC_USHDMI, 0, MSUG } },
         { 0xe69400a4, 0xe69400e4, 8, /* IMR9A / IMCR9A */
           { CMT1_CMT13, CMT1_CMT12, CMT1_CMT11, CMT1_CMT10,
diff --git a/arch/arm/mach-shmobile/setup-sh7367.c b/arch/arm/mach-shmobile/setup-sh7367.c
index 003008c18360..ce28141662da 100644
--- a/arch/arm/mach-shmobile/setup-sh7367.c
+++ b/arch/arm/mach-shmobile/setup-sh7367.c
@@ -35,6 +35,8 @@
 static struct plat_sci_port scif0_platform_data = {
         .mapbase        = 0xe6c40000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE,
+        .scbrr_algo_id  = SCBRR_ALGO_4,
         .type           = PORT_SCIF,
         .irqs           = { evt2irq(0xc00), evt2irq(0xc00),
                             evt2irq(0xc00), evt2irq(0xc00) },
@@ -52,6 +54,8 @@ static struct platform_device scif0_device = {
 static struct plat_sci_port scif1_platform_data = {
         .mapbase        = 0xe6c50000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE,
+        .scbrr_algo_id  = SCBRR_ALGO_4,
         .type           = PORT_SCIF,
         .irqs           = { evt2irq(0xc20), evt2irq(0xc20),
                             evt2irq(0xc20), evt2irq(0xc20) },
@@ -69,6 +73,8 @@ static struct platform_device scif1_device = {
 static struct plat_sci_port scif2_platform_data = {
         .mapbase        = 0xe6c60000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE,
+        .scbrr_algo_id  = SCBRR_ALGO_4,
         .type           = PORT_SCIF,
         .irqs           = { evt2irq(0xc40), evt2irq(0xc40),
                             evt2irq(0xc40), evt2irq(0xc40) },
@@ -86,6 +92,8 @@ static struct platform_device scif2_device = {
 static struct plat_sci_port scif3_platform_data = {
         .mapbase        = 0xe6c70000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE,
+        .scbrr_algo_id  = SCBRR_ALGO_4,
         .type           = PORT_SCIF,
         .irqs           = { evt2irq(0xc60), evt2irq(0xc60),
                             evt2irq(0xc60), evt2irq(0xc60) },
@@ -103,6 +111,8 @@ static struct platform_device scif3_device = {
 static struct plat_sci_port scif4_platform_data = {
         .mapbase        = 0xe6c80000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE,
+        .scbrr_algo_id  = SCBRR_ALGO_4,
         .type           = PORT_SCIF,
         .irqs           = { evt2irq(0xd20), evt2irq(0xd20),
                             evt2irq(0xd20), evt2irq(0xd20) },
@@ -120,6 +130,8 @@ static struct platform_device scif4_device = {
 static struct plat_sci_port scif5_platform_data = {
         .mapbase        = 0xe6cb0000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE,
+        .scbrr_algo_id  = SCBRR_ALGO_4,
         .type           = PORT_SCIF,
         .irqs           = { evt2irq(0xd40), evt2irq(0xd40),
                             evt2irq(0xd40), evt2irq(0xd40) },
@@ -137,6 +149,8 @@ static struct platform_device scif5_device = {
 static struct plat_sci_port scif6_platform_data = {
         .mapbase        = 0xe6c30000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE,
+        .scbrr_algo_id  = SCBRR_ALGO_4,
         .type           = PORT_SCIF,
         .irqs           = { evt2irq(0xd60), evt2irq(0xd60),
                             evt2irq(0xd60), evt2irq(0xd60) },
diff --git a/arch/arm/mach-shmobile/setup-sh7372.c b/arch/arm/mach-shmobile/setup-sh7372.c
index 2e3e11ee7c43..ff0494f3d00c 100644
--- a/arch/arm/mach-shmobile/setup-sh7372.c
+++ b/arch/arm/mach-shmobile/setup-sh7372.c
@@ -38,6 +38,8 @@
 static struct plat_sci_port scif0_platform_data = {
         .mapbase        = 0xe6c40000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE,
+        .scbrr_algo_id  = SCBRR_ALGO_4,
         .type           = PORT_SCIFA,
         .irqs           = { evt2irq(0x0c00), evt2irq(0x0c00),
                             evt2irq(0x0c00), evt2irq(0x0c00) },
@@ -55,6 +57,8 @@ static struct platform_device scif0_device = {
 static struct plat_sci_port scif1_platform_data = {
         .mapbase        = 0xe6c50000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE,
+        .scbrr_algo_id  = SCBRR_ALGO_4,
         .type           = PORT_SCIFA,
         .irqs           = { evt2irq(0x0c20), evt2irq(0x0c20),
                             evt2irq(0x0c20), evt2irq(0x0c20) },
@@ -72,6 +76,8 @@ static struct platform_device scif1_device = {
 static struct plat_sci_port scif2_platform_data = {
         .mapbase        = 0xe6c60000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE,
+        .scbrr_algo_id  = SCBRR_ALGO_4,
         .type           = PORT_SCIFA,
         .irqs           = { evt2irq(0x0c40), evt2irq(0x0c40),
                             evt2irq(0x0c40), evt2irq(0x0c40) },
@@ -89,6 +95,8 @@ static struct platform_device scif2_device = {
 static struct plat_sci_port scif3_platform_data = {
         .mapbase        = 0xe6c70000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE,
+        .scbrr_algo_id  = SCBRR_ALGO_4,
         .type           = PORT_SCIFA,
         .irqs           = { evt2irq(0x0c60), evt2irq(0x0c60),
                             evt2irq(0x0c60), evt2irq(0x0c60) },
@@ -106,6 +114,8 @@ static struct platform_device scif3_device = {
 static struct plat_sci_port scif4_platform_data = {
         .mapbase        = 0xe6c80000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE,
+        .scbrr_algo_id  = SCBRR_ALGO_4,
         .type           = PORT_SCIFA,
         .irqs           = { evt2irq(0x0d20), evt2irq(0x0d20),
                             evt2irq(0x0d20), evt2irq(0x0d20) },
@@ -123,6 +133,8 @@ static struct platform_device scif4_device = {
 static struct plat_sci_port scif5_platform_data = {
         .mapbase        = 0xe6cb0000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE,
+        .scbrr_algo_id  = SCBRR_ALGO_4,
         .type           = PORT_SCIFA,
         .irqs           = { evt2irq(0x0d40), evt2irq(0x0d40),
                             evt2irq(0x0d40), evt2irq(0x0d40) },
@@ -140,6 +152,8 @@ static struct platform_device scif5_device = {
 static struct plat_sci_port scif6_platform_data = {
         .mapbase        = 0xe6c30000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE,
+        .scbrr_algo_id  = SCBRR_ALGO_4,
         .type           = PORT_SCIFB,
         .irqs           = { evt2irq(0x0d60), evt2irq(0x0d60),
                             evt2irq(0x0d60), evt2irq(0x0d60) },
diff --git a/arch/arm/mach-shmobile/setup-sh7377.c b/arch/arm/mach-shmobile/setup-sh7377.c
index 575dbd6c2f1d..8099b0b8a934 100644
--- a/arch/arm/mach-shmobile/setup-sh7377.c
+++ b/arch/arm/mach-shmobile/setup-sh7377.c
@@ -36,6 +36,8 @@
 static struct plat_sci_port scif0_platform_data = {
         .mapbase        = 0xe6c40000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE,
+        .scbrr_algo_id  = SCBRR_ALGO_4,
         .type           = PORT_SCIF,
         .irqs           = { evt2irq(0xc00), evt2irq(0xc00),
                             evt2irq(0xc00), evt2irq(0xc00) },
@@ -53,6 +55,8 @@ static struct platform_device scif0_device = {
 static struct plat_sci_port scif1_platform_data = {
         .mapbase        = 0xe6c50000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE,
+        .scbrr_algo_id  = SCBRR_ALGO_4,
         .type           = PORT_SCIF,
         .irqs           = { evt2irq(0xc20), evt2irq(0xc20),
                             evt2irq(0xc20), evt2irq(0xc20) },
@@ -70,6 +74,8 @@ static struct platform_device scif1_device = {
 static struct plat_sci_port scif2_platform_data = {
         .mapbase        = 0xe6c60000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE,
+        .scbrr_algo_id  = SCBRR_ALGO_4,
         .type           = PORT_SCIF,
         .irqs           = { evt2irq(0xc40), evt2irq(0xc40),
                             evt2irq(0xc40), evt2irq(0xc40) },
@@ -87,6 +93,8 @@ static struct platform_device scif2_device = {
 static struct plat_sci_port scif3_platform_data = {
         .mapbase        = 0xe6c70000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE,
+        .scbrr_algo_id  = SCBRR_ALGO_4,
         .type           = PORT_SCIF,
         .irqs           = { evt2irq(0xc60), evt2irq(0xc60),
                             evt2irq(0xc60), evt2irq(0xc60) },
@@ -104,6 +112,8 @@ static struct platform_device scif3_device = {
 static struct plat_sci_port scif4_platform_data = {
         .mapbase        = 0xe6c80000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE,
+        .scbrr_algo_id  = SCBRR_ALGO_4,
         .type           = PORT_SCIF,
         .irqs           = { evt2irq(0xd20), evt2irq(0xd20),
                             evt2irq(0xd20), evt2irq(0xd20) },
@@ -121,6 +131,8 @@ static struct platform_device scif4_device = {
 static struct plat_sci_port scif5_platform_data = {
         .mapbase        = 0xe6cb0000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE,
+        .scbrr_algo_id  = SCBRR_ALGO_4,
         .type           = PORT_SCIF,
         .irqs           = { evt2irq(0xd40), evt2irq(0xd40),
                             evt2irq(0xd40), evt2irq(0xd40) },
@@ -138,6 +150,8 @@ static struct platform_device scif5_device = {
 static struct plat_sci_port scif6_platform_data = {
         .mapbase        = 0xe6cc0000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE,
+        .scbrr_algo_id  = SCBRR_ALGO_4,
         .type           = PORT_SCIF,
         .irqs           = { intcs_evt2irq(0x1a80), intcs_evt2irq(0x1a80),
                             intcs_evt2irq(0x1a80), intcs_evt2irq(0x1a80) },
@@ -155,6 +169,8 @@ static struct platform_device scif6_device = {
 static struct plat_sci_port scif7_platform_data = {
         .mapbase        = 0xe6c30000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE,
+        .scbrr_algo_id  = SCBRR_ALGO_4,
         .type           = PORT_SCIF,
         .irqs           = { evt2irq(0xd60), evt2irq(0xd60),
                             evt2irq(0xd60), evt2irq(0xd60) },
diff --git a/arch/arm/mach-shmobile/setup-sh73a0.c b/arch/arm/mach-shmobile/setup-sh73a0.c
index f1eff8b37bd6..685c40a2f5e6 100644
--- a/arch/arm/mach-shmobile/setup-sh73a0.c
+++ b/arch/arm/mach-shmobile/setup-sh73a0.c
@@ -36,6 +36,8 @@
 static struct plat_sci_port scif0_platform_data = {
         .mapbase        = 0xe6c40000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE,
+        .scbrr_algo_id  = SCBRR_ALGO_4,
         .type           = PORT_SCIFA,
         .irqs           = { gic_spi(72), gic_spi(72),
                             gic_spi(72), gic_spi(72) },
@@ -52,6 +54,8 @@ static struct platform_device scif0_device = {
 static struct plat_sci_port scif1_platform_data = {
         .mapbase        = 0xe6c50000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE,
+        .scbrr_algo_id  = SCBRR_ALGO_4,
         .type           = PORT_SCIFA,
         .irqs           = { gic_spi(73), gic_spi(73),
                             gic_spi(73), gic_spi(73) },
@@ -68,6 +72,8 @@ static struct platform_device scif1_device = {
 static struct plat_sci_port scif2_platform_data = {
         .mapbase        = 0xe6c60000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE,
+        .scbrr_algo_id  = SCBRR_ALGO_4,
         .type           = PORT_SCIFA,
         .irqs           = { gic_spi(74), gic_spi(74),
                             gic_spi(74), gic_spi(74) },
@@ -84,6 +90,8 @@ static struct platform_device scif2_device = {
 static struct plat_sci_port scif3_platform_data = {
         .mapbase        = 0xe6c70000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE,
+        .scbrr_algo_id  = SCBRR_ALGO_4,
         .type           = PORT_SCIFA,
         .irqs           = { gic_spi(75), gic_spi(75),
                             gic_spi(75), gic_spi(75) },
@@ -100,6 +108,8 @@ static struct platform_device scif3_device = {
 static struct plat_sci_port scif4_platform_data = {
         .mapbase        = 0xe6c80000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE,
+        .scbrr_algo_id  = SCBRR_ALGO_4,
         .type           = PORT_SCIFA,
         .irqs           = { gic_spi(78), gic_spi(78),
                             gic_spi(78), gic_spi(78) },
@@ -116,6 +126,8 @@ static struct platform_device scif4_device = {
 static struct plat_sci_port scif5_platform_data = {
         .mapbase        = 0xe6cb0000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE,
+        .scbrr_algo_id  = SCBRR_ALGO_4,
         .type           = PORT_SCIFA,
         .irqs           = { gic_spi(79), gic_spi(79),
                             gic_spi(79), gic_spi(79) },
@@ -132,6 +144,8 @@ static struct platform_device scif5_device = {
 static struct plat_sci_port scif6_platform_data = {
         .mapbase        = 0xe6cc0000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE,
+        .scbrr_algo_id  = SCBRR_ALGO_4,
         .type           = PORT_SCIFA,
         .irqs           = { gic_spi(156), gic_spi(156),
                             gic_spi(156), gic_spi(156) },
@@ -148,6 +162,8 @@ static struct platform_device scif6_device = {
 static struct plat_sci_port scif7_platform_data = {
         .mapbase        = 0xe6cd0000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE,
+        .scbrr_algo_id  = SCBRR_ALGO_4,
         .type           = PORT_SCIFA,
         .irqs           = { gic_spi(143), gic_spi(143),
                             gic_spi(143), gic_spi(143) },
@@ -164,6 +180,8 @@ static struct platform_device scif7_device = {
 static struct plat_sci_port scif8_platform_data = {
         .mapbase        = 0xe6c30000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE,
+        .scbrr_algo_id  = SCBRR_ALGO_4,
         .type           = PORT_SCIFB,
         .irqs           = { gic_spi(80), gic_spi(80),
                             gic_spi(80), gic_spi(80) },
diff --git a/arch/arm/mach-spear3xx/spear300.c b/arch/arm/mach-spear3xx/spear300.c
index 3560f8c1e723..5aa2d54ebfaa 100644
--- a/arch/arm/mach-spear3xx/spear300.c
+++ b/arch/arm/mach-spear3xx/spear300.c
@@ -371,7 +371,7 @@ struct pmx_driver pmx_driver = {
 };
 
 /* Add spear300 specific devices here */
-/* arm gpio1 device registeration */
+/* arm gpio1 device registration */
 static struct pl061_platform_data gpio1_plat_data = {
         .gpio_base      = 8,
         .irq_base       = SPEAR_GPIO1_INT_BASE,
@@ -451,7 +451,7 @@ void __init spear300_init(void)
         /* call spear3xx family common init function */
         spear3xx_init();
 
-        /* shared irq registeration */
+        /* shared irq registration */
         shirq_ras1.regs.base =
                 ioremap(SPEAR300_TELECOM_BASE, SPEAR300_TELECOM_REG_SIZE);
         if (shirq_ras1.regs.base) {
diff --git a/arch/arm/mach-spear3xx/spear310.c b/arch/arm/mach-spear3xx/spear310.c
index 96a1ab824bac..53b41b52d7ee 100644
--- a/arch/arm/mach-spear3xx/spear310.c
+++ b/arch/arm/mach-spear3xx/spear310.c
@@ -266,7 +266,7 @@ void __init spear310_init(void)
         /* call spear3xx family common init function */
         spear3xx_init();
 
-        /* shared irq registeration */
+        /* shared irq registration */
         base = ioremap(SPEAR310_SOC_CONFIG_BASE, SPEAR310_SOC_CONFIG_SIZE);
         if (base) {
                 /* shirq 1 */
diff --git a/arch/arm/mach-spear3xx/spear320.c b/arch/arm/mach-spear3xx/spear320.c
index 6a1219549369..88b465284c36 100644
--- a/arch/arm/mach-spear3xx/spear320.c
+++ b/arch/arm/mach-spear3xx/spear320.c
@@ -519,7 +519,7 @@ void __init spear320_init(void)
         /* call spear3xx family common init function */
         spear3xx_init();
 
-        /* shared irq registeration */
+        /* shared irq registration */
         base = ioremap(SPEAR320_SOC_CONFIG_BASE, SPEAR320_SOC_CONFIG_SIZE);
         if (base) {
                 /* shirq 1 */
diff --git a/arch/arm/mach-spear3xx/spear3xx.c b/arch/arm/mach-spear3xx/spear3xx.c
index e87313aeae20..52f553c8c46d 100644
--- a/arch/arm/mach-spear3xx/spear3xx.c
+++ b/arch/arm/mach-spear3xx/spear3xx.c
@@ -22,7 +22,7 @@
 #include <mach/spear.h>
 
 /* Add spear3xx machines common devices here */
-/* gpio device registeration */
+/* gpio device registration */
 static struct pl061_platform_data gpio_plat_data = {
         .gpio_base      = 0,
         .irq_base       = SPEAR_GPIO_INT_BASE,
@@ -41,7 +41,7 @@ struct amba_device gpio_device = {
         .irq = {IRQ_BASIC_GPIO, NO_IRQ},
 };
 
-/* uart device registeration */
+/* uart device registration */
 struct amba_device uart_device = {
         .dev = {
                 .init_name = "uart",
@@ -543,6 +543,6 @@ void spear_pmx_init(struct pmx_driver *pmx_driver, uint base, uint size)
 
 pmx_fail:
         if (ret)
-                printk(KERN_ERR "padmux: registeration failed. err no: %d\n",
+                printk(KERN_ERR "padmux: registration failed. err no: %d\n",
                                 ret);
 }
diff --git a/arch/arm/mach-spear6xx/spear6xx.c b/arch/arm/mach-spear6xx/spear6xx.c
index baf6bcc3169c..f2fe14e8471d 100644
--- a/arch/arm/mach-spear6xx/spear6xx.c
+++ b/arch/arm/mach-spear6xx/spear6xx.c
@@ -23,7 +23,7 @@
 #include <mach/spear.h>
 
 /* Add spear6xx machines common devices here */
-/* uart device registeration */
+/* uart device registration */
 struct amba_device uart_device[] = {
         {
                 .dev = {
@@ -50,7 +50,7 @@ struct amba_device uart_device[] = {
         }
 };
 
-/* gpio device registeration */
+/* gpio device registration */
 static struct pl061_platform_data gpio_plat_data[] = {
         {
                 .gpio_base      = 0,
diff --git a/arch/arm/mach-u300/Kconfig b/arch/arm/mach-u300/Kconfig
index 801b21e7f677..32a7b0f7e9f7 100644
--- a/arch/arm/mach-u300/Kconfig
+++ b/arch/arm/mach-u300/Kconfig
@@ -64,7 +64,7 @@ config MACH_U300_DUAL_RAM
         bool "Dual RAM"
         help
                 Select this if you want support for Dual RAM phones.
-                This is two RAM memorys on different EMIFs.
+                This is two RAM memories on different EMIFs.
 endchoice
 
 config U300_DEBUG
diff --git a/arch/arm/mach-u300/include/mach/coh901318.h b/arch/arm/mach-u300/include/mach/coh901318.h
index 193da2df732c..6193aaa47794 100644
--- a/arch/arm/mach-u300/include/mach/coh901318.h
+++ b/arch/arm/mach-u300/include/mach/coh901318.h
@@ -24,7 +24,7 @@
  * @src_addr: transfer source address
  * @dst_addr: transfer destination address
  * @link_addr:  physical address to next lli
- * @virt_link_addr: virtual addres of next lli (only used by pool_free)
+ * @virt_link_addr: virtual address of next lli (only used by pool_free)
  * @phy_this: physical address of current lli (only used by pool_free)
  */
 struct coh901318_lli {
@@ -90,7 +90,7 @@ struct powersave {
  * struct coh901318_platform - platform arch structure
  * @chans_slave: specifying dma slave channels
  * @chans_memcpy: specifying dma memcpy channels
- * @access_memory_state: requesting DMA memeory access (on / off)
+ * @access_memory_state: requesting DMA memory access (on / off)
  * @chan_conf: dma channel configurations
  * @max_channels: max number of dma chanenls
  */
diff --git a/arch/arm/mach-ux500/board-mop500-regulators.c b/arch/arm/mach-ux500/board-mop500-regulators.c
index 1187f1fc2e53..533967c2d095 100644
--- a/arch/arm/mach-ux500/board-mop500-regulators.c
+++ b/arch/arm/mach-ux500/board-mop500-regulators.c
@@ -3,99 +3,94 @@
  *
  * License Terms: GNU General Public License v2
  *
- * Author: Sundar Iyer <sundar.iyer@stericsson.com>
+ * Authors: Sundar Iyer <sundar.iyer@stericsson.com>
+ *          Bengt Jonsson <bengt.g.jonsson@stericsson.com>
  *
  * MOP500 board specific initialization for regulators
  */
 #include <linux/kernel.h>
 #include <linux/regulator/machine.h>
+#include <linux/regulator/ab8500.h>
 
-/* supplies to the display/camera */
-static struct regulator_init_data ab8500_vaux1_regulator = {
-        .constraints = {
-                .name = "V-DISPLAY",
-                .min_uV = 2500000,
-                .max_uV = 2900000,
-                .valid_ops_mask = REGULATOR_CHANGE_VOLTAGE|
-                                        REGULATOR_CHANGE_STATUS,
+/* AB8500 regulators */
+struct regulator_init_data ab8500_regulators[AB8500_NUM_REGULATORS] = {
+        /* supplies to the display/camera */
+        [AB8500_LDO_AUX1] = {
+                .constraints = {
+                        .name = "V-DISPLAY",
+                        .min_uV = 2500000,
+                        .max_uV = 2900000,
+                        .valid_ops_mask = REGULATOR_CHANGE_VOLTAGE |
+                                          REGULATOR_CHANGE_STATUS,
+                },
         },
-};
-
-/* supplies to the on-board eMMC */
-static struct regulator_init_data ab8500_vaux2_regulator = {
-        .constraints = {
-                .name = "V-eMMC1",
-                .min_uV = 1100000,
-                .max_uV = 3300000,
-                .valid_ops_mask = REGULATOR_CHANGE_VOLTAGE|
-                                        REGULATOR_CHANGE_STATUS,
+        /* supplies to the on-board eMMC */
+        [AB8500_LDO_AUX2] = {
+                .constraints = {
+                        .name = "V-eMMC1",
+                        .min_uV = 1100000,
+                        .max_uV = 3300000,
+                        .valid_ops_mask = REGULATOR_CHANGE_VOLTAGE |
+                                          REGULATOR_CHANGE_STATUS,
+                },
         },
-};
-
-/* supply for VAUX3, supplies to SDcard slots */
-static struct regulator_init_data ab8500_vaux3_regulator = {
-        .constraints = {
-                .name = "V-MMC-SD",
-                .min_uV = 1100000,
-                .max_uV = 3300000,
-                .valid_ops_mask = REGULATOR_CHANGE_VOLTAGE|
-                                        REGULATOR_CHANGE_STATUS,
+        /* supply for VAUX3, supplies to SDcard slots */
+        [AB8500_LDO_AUX3] = {
+                .constraints = {
+                        .name = "V-MMC-SD",
+                        .min_uV = 1100000,
+                        .max_uV = 3300000,
+                        .valid_ops_mask = REGULATOR_CHANGE_VOLTAGE |
+                                          REGULATOR_CHANGE_STATUS,
+                },
         },
-};
-
-/* supply for tvout, gpadc, TVOUT LDO */
-static struct regulator_init_data ab8500_vtvout_init = {
-        .constraints = {
-                .name = "V-TVOUT",
-                .valid_ops_mask = REGULATOR_CHANGE_STATUS,
+        /* supply for tvout, gpadc, TVOUT LDO */
+        [AB8500_LDO_TVOUT] = {
+                .constraints = {
+                        .name = "V-TVOUT",
+                        .valid_ops_mask = REGULATOR_CHANGE_STATUS,
+                },
         },
-};
-
-/* supply for ab8500-vaudio, VAUDIO LDO */
-static struct regulator_init_data ab8500_vaudio_init = {
-        .constraints = {
-                .name = "V-AUD",
-                .valid_ops_mask = REGULATOR_CHANGE_STATUS,
+        /* supply for ab8500-vaudio, VAUDIO LDO */
+        [AB8500_LDO_AUDIO] = {
+                .constraints = {
+                        .name = "V-AUD",
+                        .valid_ops_mask = REGULATOR_CHANGE_STATUS,
+                },
         },
-};
-
-/* supply for v-anamic1 VAMic1-LDO */
-static struct regulator_init_data ab8500_vamic1_init = {
-        .constraints = {
-                .name = "V-AMIC1",
-                .valid_ops_mask = REGULATOR_CHANGE_STATUS,
+        /* supply for v-anamic1 VAMic1-LDO */
+        [AB8500_LDO_ANAMIC1] = {
+                .constraints = {
+                        .name = "V-AMIC1",
+                        .valid_ops_mask = REGULATOR_CHANGE_STATUS,
+                },
         },
-};
-
-/* supply for v-amic2, VAMIC2 LDO, reuse constants for AMIC1 */
-static struct regulator_init_data ab8500_vamic2_init = {
-        .constraints = {
-                .name = "V-AMIC2",
-                .valid_ops_mask = REGULATOR_CHANGE_STATUS,
+        /* supply for v-amic2, VAMIC2 LDO, reuse constants for AMIC1 */
+        [AB8500_LDO_ANAMIC2] = {
+                .constraints = {
+                        .name = "V-AMIC2",
+                        .valid_ops_mask = REGULATOR_CHANGE_STATUS,
+                },
         },
-};
-
-/* supply for v-dmic, VDMIC LDO */
-static struct regulator_init_data ab8500_vdmic_init = {
-        .constraints = {
-                .name = "V-DMIC",
-                .valid_ops_mask = REGULATOR_CHANGE_STATUS,
+        /* supply for v-dmic, VDMIC LDO */
+        [AB8500_LDO_DMIC] = {
+                .constraints = {
+                        .name = "V-DMIC",
+                        .valid_ops_mask = REGULATOR_CHANGE_STATUS,
+                },
         },
-};
-
-/* supply for v-intcore12, VINTCORE12 LDO */
-static struct regulator_init_data ab8500_vintcore_init = {
-        .constraints = {
-                .name = "V-INTCORE",
-                .valid_ops_mask = REGULATOR_CHANGE_STATUS,
+        /* supply for v-intcore12, VINTCORE12 LDO */
+        [AB8500_LDO_INTCORE] = {
+                .constraints = {
+                        .name = "V-INTCORE",
+                        .valid_ops_mask = REGULATOR_CHANGE_STATUS,
+                },
         },
-};
-
-/* supply for U8500 CSI/DSI, VANA LDO */
-static struct regulator_init_data ab8500_vana_init = {
-        .constraints = {
-                .name = "V-CSI/DSI",
-                .valid_ops_mask = REGULATOR_CHANGE_STATUS,
+        /* supply for U8500 CSI/DSI, VANA LDO */
+        [AB8500_LDO_ANA] = {
+                .constraints = {
+                        .name = "V-CSI/DSI",
+                        .valid_ops_mask = REGULATOR_CHANGE_STATUS,
+                },
         },
 };
-
diff --git a/arch/arm/mach-ux500/board-mop500-regulators.h b/arch/arm/mach-ux500/board-mop500-regulators.h
new file mode 100644
index 000000000000..2675fae52537
--- /dev/null
+++ b/arch/arm/mach-ux500/board-mop500-regulators.h
@@ -0,0 +1,19 @@
+/*
+ * Copyright (C) ST-Ericsson SA 2010
+ *
+ * License Terms: GNU General Public License v2
+ *
+ * Author: Bengt Jonsson <bengt.g.jonsson@stericsson.com> for ST-Ericsson
+ *
+ * MOP500 board specific initialization for regulators
+ */
+
+#ifndef __BOARD_MOP500_REGULATORS_H
+#define __BOARD_MOP500_REGULATORS_H
+
+#include <linux/regulator/machine.h>
+#include <linux/regulator/ab8500.h>
+
+extern struct regulator_init_data ab8500_regulators[AB8500_NUM_REGULATORS];
+
+#endif
diff --git a/arch/arm/mach-ux500/board-mop500.c b/arch/arm/mach-ux500/board-mop500.c
index a1c9ea1a66df..a393f57ed2a8 100644
--- a/arch/arm/mach-ux500/board-mop500.c
+++ b/arch/arm/mach-ux500/board-mop500.c
@@ -35,6 +35,7 @@
 #include "devices-db8500.h"
 #include "pins-db8500.h"
 #include "board-mop500.h"
+#include "board-mop500-regulators.h"
 
 static pin_cfg_t mop500_pins[] = {
         /* SSP0 */
@@ -80,6 +81,8 @@ static pin_cfg_t mop500_pins[] = {
 
 static struct ab8500_platform_data ab8500_platdata = {
         .irq_base       = MOP500_AB8500_IRQ_BASE,
+        .regulator      = ab8500_regulators,
+        .num_regulator  = ARRAY_SIZE(ab8500_regulators),
 };
 
 static struct resource ab8500_resources[] = {
diff --git a/arch/arm/mm/flush.c b/arch/arm/mm/flush.c
index c29f2839f1d2..2b269c955524 100644
--- a/arch/arm/mm/flush.c
+++ b/arch/arm/mm/flush.c
@@ -18,7 +18,6 @@
 #include <asm/smp_plat.h>
 #include <asm/system.h>
 #include <asm/tlbflush.h>
-#include <asm/smp_plat.h>
 
 #include "mm.h"
 
diff --git a/arch/arm/mm/pgd.c b/arch/arm/mm/pgd.c
index 93292a18cf77..709244c66fa3 100644
--- a/arch/arm/mm/pgd.c
+++ b/arch/arm/mm/pgd.c
@@ -50,7 +50,7 @@ pgd_t *pgd_alloc(struct mm_struct *mm)
                 if (!new_pmd)
                         goto no_pmd;
 
-                new_pte = pte_alloc_map(mm, new_pmd, 0);
+                new_pte = pte_alloc_map(mm, NULL, new_pmd, 0);
                 if (!new_pte)
                         goto no_pte;
 
diff --git a/arch/arm/plat-mxc/include/mach/irqs.h b/arch/arm/plat-mxc/include/mach/irqs.h
index 58a49cc83797..ba65c9231a78 100644
--- a/arch/arm/plat-mxc/include/mach/irqs.h
+++ b/arch/arm/plat-mxc/include/mach/irqs.h
@@ -70,7 +70,7 @@ extern int imx_irq_set_priority(unsigned char irq, unsigned char prio);
 
 /* all normal IRQs can be FIQs */
 #define FIQ_START       0
-/* switch betwean IRQ and FIQ */
+/* switch between IRQ and FIQ */
 extern int mxc_set_irq_fiq(unsigned int irq, unsigned int type);
 
 #endif /* __ASM_ARCH_MXC_IRQS_H__ */
diff --git a/arch/arm/plat-omap/include/plat/display.h b/arch/arm/plat-omap/include/plat/display.h
index c915a661f1f5..537f4e449f50 100644
--- a/arch/arm/plat-omap/include/plat/display.h
+++ b/arch/arm/plat-omap/include/plat/display.h
@@ -42,6 +42,10 @@
 #define DISPC_IRQ_SYNC_LOST             (1 << 14)
 #define DISPC_IRQ_SYNC_LOST_DIGIT       (1 << 15)
 #define DISPC_IRQ_WAKEUP                (1 << 16)
+#define DISPC_IRQ_SYNC_LOST2            (1 << 17)
+#define DISPC_IRQ_VSYNC2                (1 << 18)
+#define DISPC_IRQ_ACBIAS_COUNT_STAT2    (1 << 21)
+#define DISPC_IRQ_FRAMEDONE2            (1 << 22)
 
 struct omap_dss_device;
 struct omap_overlay_manager;
@@ -64,6 +68,7 @@ enum omap_plane {
 enum omap_channel {
         OMAP_DSS_CHANNEL_LCD    = 0,
         OMAP_DSS_CHANNEL_DIGIT  = 1,
+        OMAP_DSS_CHANNEL_LCD2   = 2,
 };
 
 enum omap_color_mode {
@@ -142,6 +147,7 @@ enum omap_dss_display_state {
 enum omap_dss_overlay_managers {
         OMAP_DSS_OVL_MGR_LCD,
         OMAP_DSS_OVL_MGR_TV,
+        OMAP_DSS_OVL_MGR_LCD2,
 };
 
 enum omap_dss_rotation_type {
@@ -268,6 +274,7 @@ struct omap_overlay_info {
         u16 out_width;  /* if 0, out_width == width */
         u16 out_height; /* if 0, out_height == height */
         u8 global_alpha;
+        u8 pre_mult_alpha;
 };
 
 struct omap_overlay {
@@ -351,6 +358,8 @@ struct omap_dss_device {
 
         enum omap_display_type type;
 
+        enum omap_channel channel;
+
         union {
                 struct {
                         u8 data_lines;
diff --git a/arch/arm/plat-omap/include/plat/omap_hwmod.h b/arch/arm/plat-omap/include/plat/omap_hwmod.h
index 6864a997f2ca..1eee85a8abb3 100644
--- a/arch/arm/plat-omap/include/plat/omap_hwmod.h
+++ b/arch/arm/plat-omap/include/plat/omap_hwmod.h
@@ -351,7 +351,7 @@ struct omap_hwmod_omap2_prcm {
 /**
  * struct omap_hwmod_omap4_prcm - OMAP4-specific PRCM data
  * @clkctrl_reg: PRCM address of the clock control register
- * @rstctrl_reg: adress of the XXX_RSTCTRL register located in the PRM
+ * @rstctrl_reg: address of the XXX_RSTCTRL register located in the PRM
  * @submodule_wkdep_bit: bit shift of the WKDEP range
  */
 struct omap_hwmod_omap4_prcm {
diff --git a/arch/arm/plat-omap/include/plat/panel-generic-dpi.h b/arch/arm/plat-omap/include/plat/panel-generic-dpi.h
new file mode 100644
index 000000000000..790619734bcd
--- /dev/null
+++ b/arch/arm/plat-omap/include/plat/panel-generic-dpi.h
@@ -0,0 +1,37 @@
+/*
+ * Header for generic DPI panel driver
+ *
+ * Copyright (C) 2010 Canonical Ltd.
+ * Author: Bryan Wu <bryan.wu@canonical.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __ARCH_ARM_PLAT_OMAP_PANEL_GENERIC_DPI_H
+#define __ARCH_ARM_PLAT_OMAP_PANEL_GENERIC_DPI_H
+
+#include "display.h"
+
+/**
+ * struct panel_generic_dpi_data - panel driver configuration data
+ * @name: panel name
+ * @platform_enable: platform specific panel enable function
+ * @platform_disable: platform specific panel disable function
+ */
+struct panel_generic_dpi_data {
+        const char *name;
+        int (*platform_enable)(struct omap_dss_device *dssdev);
+        void (*platform_disable)(struct omap_dss_device *dssdev);
+};
+
+#endif /* __ARCH_ARM_PLAT_OMAP_PANEL_GENERIC_DPI_H */
diff --git a/arch/arm/plat-s3c24xx/devs.c b/arch/arm/plat-s3c24xx/devs.c
index 2f91057a0c02..8a42bc48dbf0 100644
--- a/arch/arm/plat-s3c24xx/devs.c
+++ b/arch/arm/plat-s3c24xx/devs.c
@@ -259,21 +259,6 @@ struct platform_device s3c_device_iis = {
 
 EXPORT_SYMBOL(s3c_device_iis);
 
-/* ASoC PCM DMA */
-
-static u64 s3c_device_audio_dmamask = 0xffffffffUL;
-
-struct platform_device s3c_device_pcm = {
-        .name             = "s3c24xx-pcm-audio",
-        .id               = -1,
-        .dev              = {
-                .dma_mask = &s3c_device_audio_dmamask,
-                .coherent_dma_mask = 0xffffffffUL
-        }
-};
-
-EXPORT_SYMBOL(s3c_device_pcm);
-
 /* RTC */
 
 static struct resource s3c_rtc_resource[] = {
@@ -496,8 +481,10 @@ static struct resource s3c_ac97_resource[] = {
         },
 };
 
+static u64 s3c_device_audio_dmamask = 0xffffffffUL;
+
 struct platform_device s3c_device_ac97 = {
-        .name             = "s3c-ac97",
+        .name             = "samsung-ac97",
         .id               = -1,
         .num_resources    = ARRAY_SIZE(s3c_ac97_resource),
         .resource         = s3c_ac97_resource,
diff --git a/arch/arm/plat-samsung/Makefile b/arch/arm/plat-samsung/Makefile
index afcce474af8e..19d8a16c3066 100644
--- a/arch/arm/plat-samsung/Makefile
+++ b/arch/arm/plat-samsung/Makefile
@@ -17,6 +17,7 @@ obj-y				+= clock.o
 obj-y                           += pwm-clock.o
 obj-y                           += gpio.o
 obj-y                           += gpio-config.o
+obj-y                           += dev-asocdma.o
 
 obj-$(CONFIG_SAMSUNG_GPIOLIB_4BIT)      += gpiolib.o
 obj-$(CONFIG_SAMSUNG_CLKSRC)    += clock-clksrc.o
diff --git a/arch/arm/plat-samsung/dev-asocdma.c b/arch/arm/plat-samsung/dev-asocdma.c
new file mode 100644
index 000000000000..a068c4f42d56
--- /dev/null
+++ b/arch/arm/plat-samsung/dev-asocdma.c
@@ -0,0 +1,25 @@
+/* linux/arch/arm/plat-samsung/dev-asocdma.c
+ *
+ * Copyright (c) 2010 Samsung Electronics Co. Ltd
+ *      Jaswinder Singh <jassi.brar@samsung.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/platform_device.h>
+#include <linux/dma-mapping.h>
+#include <plat/devs.h>
+
+static u64 audio_dmamask = DMA_BIT_MASK(32);
+
+struct platform_device samsung_asoc_dma = {
+        .name             = "samsung-audio",
+        .id               = -1,
+        .dev              = {
+                .dma_mask = &audio_dmamask,
+                .coherent_dma_mask = DMA_BIT_MASK(32),
+        }
+};
+EXPORT_SYMBOL(samsung_asoc_dma);
diff --git a/arch/arm/plat-samsung/include/plat/audio.h b/arch/arm/plat-samsung/include/plat/audio.h
index 7712ff6336f4..a0826ed2f9fe 100644
--- a/arch/arm/plat-samsung/include/plat/audio.h
+++ b/arch/arm/plat-samsung/include/plat/audio.h
@@ -25,10 +25,34 @@ extern void s3c64xx_ac97_setup_gpio(int);
 #define S5PC100_SPDIF_GPG3 1
 extern void s5pc100_spdif_setup_gpio(int);
 
+struct samsung_i2s {
+/* If the Primary DAI has 5.1 Channels */
+#define QUIRK_PRI_6CHAN         (1 << 0)
+/* If the I2S block has a Stereo Overlay Channel */
+#define QUIRK_SEC_DAI           (1 << 1)
+/*
+ * If the I2S block has no internal prescalar or MUX (I2SMOD[10] bit)
+ * The Machine driver must provide suitably set clock to the I2S block.
+ */
+#define QUIRK_NO_MUXPSR         (1 << 2)
+#define QUIRK_NEED_RSTCLR       (1 << 3)
+        /* Quirks of the I2S controller */
+        u32 quirks;
+
+        /*
+         * Array of clock names that can be used to generate I2S signals.
+         * Also corresponds to clocks of I2SMOD[10]
+         */
+        const char **src_clk;
+};
+
 /**
  * struct s3c_audio_pdata - common platform data for audio device drivers
  * @cfg_gpio: Callback function to setup mux'ed pins in I2S/PCM/AC97 mode
  */
 struct s3c_audio_pdata {
         int (*cfg_gpio)(struct platform_device *);
+        union {
+                struct samsung_i2s i2s;
+        } type;
 };
diff --git a/arch/arm/plat-samsung/include/plat/devs.h b/arch/arm/plat-samsung/include/plat/devs.h
index 2d82a6cb1444..e9e3b6e3ec74 100644
--- a/arch/arm/plat-samsung/include/plat/devs.h
+++ b/arch/arm/plat-samsung/include/plat/devs.h
@@ -32,7 +32,7 @@ extern struct platform_device s3c64xx_device_iisv4;
 extern struct platform_device s3c64xx_device_spi0;
 extern struct platform_device s3c64xx_device_spi1;
 
-extern struct platform_device s3c_device_pcm;
+extern struct platform_device samsung_asoc_dma;
 
 extern struct platform_device s3c64xx_device_pcm0;
 extern struct platform_device s3c64xx_device_pcm1;
@@ -96,6 +96,15 @@ extern struct platform_device s5pv210_device_iis1;
 extern struct platform_device s5pv210_device_iis2;
 extern struct platform_device s5pv210_device_spdif;
 
+extern struct platform_device s5pv310_device_ac97;
+extern struct platform_device s5pv310_device_pcm0;
+extern struct platform_device s5pv310_device_pcm1;
+extern struct platform_device s5pv310_device_pcm2;
+extern struct platform_device s5pv310_device_i2s0;
+extern struct platform_device s5pv310_device_i2s1;
+extern struct platform_device s5pv310_device_i2s2;
+extern struct platform_device s5pv310_device_spdif;
+
 extern struct platform_device s5p6442_device_pcm0;
 extern struct platform_device s5p6442_device_pcm1;
 extern struct platform_device s5p6442_device_iis0;
diff --git a/arch/arm/plat-samsung/pm.c b/arch/arm/plat-samsung/pm.c
index 27cfca597699..5bf3f2f09e74 100644
--- a/arch/arm/plat-samsung/pm.c
+++ b/arch/arm/plat-samsung/pm.c
@@ -355,7 +355,7 @@ static void s3c_pm_finish(void)
         s3c_pm_check_cleanup();
 }
 
-static struct platform_suspend_ops s3c_pm_ops = {
+static const struct platform_suspend_ops s3c_pm_ops = {
         .enter          = s3c_pm_enter,
         .prepare        = s3c_pm_prepare,
         .finish         = s3c_pm_finish,
diff --git a/arch/avr32/boards/atngw100/setup.c b/arch/avr32/boards/atngw100/setup.c
index 8c6a2440e345..659d119ce712 100644
--- a/arch/avr32/boards/atngw100/setup.c
+++ b/arch/avr32/boards/atngw100/setup.c
@@ -188,7 +188,7 @@ static void __init set_hw_addr(struct platform_device *pdev)
          */
         regs = (void __iomem __force *)res->start;
         pclk = clk_get(&pdev->dev, "pclk");
-        if (!pclk)
+        if (IS_ERR(pclk))
                 return;
 
         clk_enable(pclk);
diff --git a/arch/avr32/boards/atstk1000/atstk1002.c b/arch/avr32/boards/atstk1000/atstk1002.c
index 2adc261c9e3d..6ce30fb2ec94 100644
--- a/arch/avr32/boards/atstk1000/atstk1002.c
+++ b/arch/avr32/boards/atstk1000/atstk1002.c
@@ -203,7 +203,7 @@ static void __init set_hw_addr(struct platform_device *pdev)
          */
         regs = (void __iomem __force *)res->start;
         pclk = clk_get(&pdev->dev, "pclk");
-        if (!pclk)
+        if (IS_ERR(pclk))
                 return;
 
         clk_enable(pclk);
diff --git a/arch/avr32/boards/favr-32/setup.c b/arch/avr32/boards/favr-32/setup.c
index 75f19f47fb2f..86fab77a5a00 100644
--- a/arch/avr32/boards/favr-32/setup.c
+++ b/arch/avr32/boards/favr-32/setup.c
@@ -206,7 +206,7 @@ static void __init set_hw_addr(struct platform_device *pdev)
          */
         regs = (void __iomem __force *)res->start;
         pclk = clk_get(&pdev->dev, "pclk");
-        if (!pclk)
+        if (IS_ERR(pclk))
                 return;
 
         clk_enable(pclk);
diff --git a/arch/avr32/boards/hammerhead/setup.c b/arch/avr32/boards/hammerhead/setup.c
index dd009875a405..da14fbdd4e8e 100644
--- a/arch/avr32/boards/hammerhead/setup.c
+++ b/arch/avr32/boards/hammerhead/setup.c
@@ -150,7 +150,7 @@ static void __init set_hw_addr(struct platform_device *pdev)
         regs = (void __iomem __force *)res->start;
         pclk = clk_get(&pdev->dev, "pclk");
 
-        if (!pclk)
+        if (IS_ERR(pclk))
                 return;
 
         clk_enable(pclk);
diff --git a/arch/avr32/boards/merisc/setup.c b/arch/avr32/boards/merisc/setup.c
index 623b077594fc..e61bc948f959 100644
--- a/arch/avr32/boards/merisc/setup.c
+++ b/arch/avr32/boards/merisc/setup.c
@@ -134,7 +134,7 @@ static void __init set_hw_addr(struct platform_device *pdev)
 
         regs = (void __iomem __force *)res->start;
         pclk = clk_get(&pdev->dev, "pclk");
-        if (!pclk)
+        if (IS_ERR(pclk))
                 return;
 
         clk_enable(pclk);
diff --git a/arch/avr32/boards/mimc200/setup.c b/arch/avr32/boards/mimc200/setup.c
index 523d8e183bef..c4da5cba2dbf 100644
--- a/arch/avr32/boards/mimc200/setup.c
+++ b/arch/avr32/boards/mimc200/setup.c
@@ -162,7 +162,7 @@ static void __init set_hw_addr(struct platform_device *pdev)
          */
         regs = (void __iomem __force *)res->start;
         pclk = clk_get(&pdev->dev, "pclk");
-        if (!pclk)
+        if (IS_ERR(pclk))
                 return;
 
         clk_enable(pclk);
diff --git a/arch/avr32/configs/atngw100_defconfig b/arch/avr32/configs/atngw100_defconfig
index 9854013d2728..6f9ca56de1f6 100644
--- a/arch/avr32/configs/atngw100_defconfig
+++ b/arch/avr32/configs/atngw100_defconfig
@@ -2,20 +2,17 @@ CONFIG_EXPERIMENTAL=y
 # CONFIG_LOCALVERSION_AUTO is not set
 CONFIG_SYSVIPC=y
 CONFIG_POSIX_MQUEUE=y
-CONFIG_BSD_PROCESS_ACCT=y
-CONFIG_BSD_PROCESS_ACCT_V3=y
 CONFIG_LOG_BUF_SHIFT=14
-CONFIG_SYSFS_DEPRECATED_V2=y
+CONFIG_RELAY=y
 CONFIG_BLK_DEV_INITRD=y
 # CONFIG_SYSCTL_SYSCALL is not set
 # CONFIG_BASE_FULL is not set
 # CONFIG_COMPAT_BRK is not set
 CONFIG_PROFILING=y
 CONFIG_OPROFILE=m
-CONFIG_KPROBES=y
+# CONFIG_KPROBES is not set
 CONFIG_MODULES=y
 CONFIG_MODULE_UNLOAD=y
-CONFIG_MODULE_FORCE_UNLOAD=y
 # CONFIG_BLK_DEV_BSG is not set
 # CONFIG_IOSCHED_DEADLINE is not set
 CONFIG_NO_HZ=y
@@ -29,6 +26,7 @@ CONFIG_CPU_FREQ=y
 CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND=y
 CONFIG_CPU_FREQ_GOV_USERSPACE=y
 CONFIG_CPU_FREQ_AT32AP=y
+CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS=y
 CONFIG_NET=y
 CONFIG_PACKET=y
 CONFIG_UNIX=y
@@ -72,8 +70,8 @@ CONFIG_MTD_UBI=y
 CONFIG_BLK_DEV_LOOP=m
 CONFIG_BLK_DEV_NBD=m
 CONFIG_BLK_DEV_RAM=m
+CONFIG_MISC_DEVICES=y
 CONFIG_ATMEL_TCLIB=y
-CONFIG_EEPROM_AT24=m
 CONFIG_NETDEVICES=y
 CONFIG_TUN=m
 CONFIG_NET_ETHERNET=y
@@ -106,6 +104,7 @@ CONFIG_GPIO_SYSFS=y
 CONFIG_WATCHDOG=y
 CONFIG_AT32AP700X_WDT=y
 CONFIG_USB_GADGET=y
+CONFIG_USB_GADGET_VBUS_DRAW=350
 CONFIG_USB_ZERO=m
 CONFIG_USB_ETH=m
 CONFIG_USB_GADGETFS=m
@@ -115,14 +114,12 @@ CONFIG_USB_CDC_COMPOSITE=m
 CONFIG_MMC=y
 CONFIG_MMC_TEST=m
 CONFIG_MMC_ATMELMCI=y
-CONFIG_MMC_SPI=m
 CONFIG_NEW_LEDS=y
 CONFIG_LEDS_CLASS=y
 CONFIG_LEDS_GPIO=y
 CONFIG_LEDS_TRIGGERS=y
 CONFIG_LEDS_TRIGGER_TIMER=y
 CONFIG_LEDS_TRIGGER_HEARTBEAT=y
-CONFIG_LEDS_TRIGGER_DEFAULT_ON=y
 CONFIG_RTC_CLASS=y
 CONFIG_RTC_DRV_AT32AP700X=y
 CONFIG_DMADEVICES=y
@@ -130,21 +127,23 @@ CONFIG_EXT2_FS=y
 CONFIG_EXT3_FS=y
 # CONFIG_EXT3_DEFAULTS_TO_ORDERED is not set
 # CONFIG_EXT3_FS_XATTR is not set
+CONFIG_EXT4_FS=y
+# CONFIG_EXT4_FS_XATTR is not set
 # CONFIG_DNOTIFY is not set
 CONFIG_FUSE_FS=m
 CONFIG_MSDOS_FS=m
 CONFIG_VFAT_FS=m
 CONFIG_FAT_DEFAULT_CODEPAGE=850
+CONFIG_PROC_KCORE=y
 CONFIG_TMPFS=y
-CONFIG_CONFIGFS_FS=m
+CONFIG_CONFIGFS_FS=y
 CONFIG_JFFS2_FS=y
-CONFIG_UFS_FS=y
+CONFIG_UBIFS_FS=y
 CONFIG_NFS_FS=y
 CONFIG_NFS_V3=y
 CONFIG_ROOT_NFS=y
 CONFIG_NFSD=m
 CONFIG_NFSD_V3=y
-CONFIG_SMB_FS=m
 CONFIG_CIFS=m
 CONFIG_NLS_CODEPAGE_437=m
 CONFIG_NLS_CODEPAGE_850=m
@@ -155,5 +154,3 @@ CONFIG_DEBUG_FS=y
 CONFIG_DEBUG_KERNEL=y
 CONFIG_DETECT_HUNG_TASK=y
 CONFIG_FRAME_POINTER=y
-# CONFIG_RCU_CPU_STALL_DETECTOR is not set
-CONFIG_CRYPTO_PCBC=m
diff --git a/arch/avr32/configs/atngw100_evklcd100_defconfig b/arch/avr32/configs/atngw100_evklcd100_defconfig
index 7ceda354597b..7eece0af34c9 100644
--- a/arch/avr32/configs/atngw100_evklcd100_defconfig
+++ b/arch/avr32/configs/atngw100_evklcd100_defconfig
@@ -2,20 +2,17 @@ CONFIG_EXPERIMENTAL=y
 # CONFIG_LOCALVERSION_AUTO is not set
 CONFIG_SYSVIPC=y
 CONFIG_POSIX_MQUEUE=y
-CONFIG_BSD_PROCESS_ACCT=y
-CONFIG_BSD_PROCESS_ACCT_V3=y
 CONFIG_LOG_BUF_SHIFT=14
-CONFIG_SYSFS_DEPRECATED_V2=y
+CONFIG_RELAY=y
 CONFIG_BLK_DEV_INITRD=y
 # CONFIG_SYSCTL_SYSCALL is not set
 # CONFIG_BASE_FULL is not set
 # CONFIG_COMPAT_BRK is not set
 CONFIG_PROFILING=y
 CONFIG_OPROFILE=m
-CONFIG_KPROBES=y
+# CONFIG_KPROBES is not set
 CONFIG_MODULES=y
 CONFIG_MODULE_UNLOAD=y
-CONFIG_MODULE_FORCE_UNLOAD=y
 # CONFIG_BLK_DEV_BSG is not set
 # CONFIG_IOSCHED_DEADLINE is not set
 CONFIG_NO_HZ=y
@@ -31,6 +28,7 @@ CONFIG_CPU_FREQ=y
 CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND=y
 CONFIG_CPU_FREQ_GOV_USERSPACE=y
 CONFIG_CPU_FREQ_AT32AP=y
+CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS=y
 CONFIG_NET=y
 CONFIG_PACKET=y
 CONFIG_UNIX=y
@@ -74,8 +72,10 @@ CONFIG_MTD_UBI=y
 CONFIG_BLK_DEV_LOOP=m
 CONFIG_BLK_DEV_NBD=m
 CONFIG_BLK_DEV_RAM=m
+CONFIG_MISC_DEVICES=y
 CONFIG_ATMEL_TCLIB=y
 CONFIG_NETDEVICES=y
+CONFIG_TUN=m
 CONFIG_NET_ETHERNET=y
 CONFIG_MACB=y
 # CONFIG_NETDEV_1000 is not set
@@ -104,6 +104,7 @@ CONFIG_I2C_GPIO=m
 CONFIG_SPI=y
 CONFIG_SPI_ATMEL=y
 CONFIG_SPI_SPIDEV=m
+CONFIG_GPIO_SYSFS=y
 # CONFIG_HWMON is not set
 CONFIG_WATCHDOG=y
 CONFIG_AT32AP700X_WDT=y
@@ -127,6 +128,7 @@ CONFIG_USB_FILE_STORAGE=m
 CONFIG_USB_G_SERIAL=m
 CONFIG_USB_CDC_COMPOSITE=m
 CONFIG_MMC=y
+CONFIG_MMC_TEST=m
 CONFIG_MMC_ATMELMCI=y
 CONFIG_NEW_LEDS=y
 CONFIG_LEDS_CLASS=y
@@ -141,11 +143,14 @@ CONFIG_EXT2_FS=y
 CONFIG_EXT3_FS=y
 # CONFIG_EXT3_DEFAULTS_TO_ORDERED is not set
 # CONFIG_EXT3_FS_XATTR is not set
+CONFIG_EXT4_FS=y
+# CONFIG_EXT4_FS_XATTR is not set
 # CONFIG_DNOTIFY is not set
 CONFIG_FUSE_FS=m
 CONFIG_MSDOS_FS=m
 CONFIG_VFAT_FS=m
 CONFIG_FAT_DEFAULT_CODEPAGE=850
+CONFIG_PROC_KCORE=y
 CONFIG_TMPFS=y
 CONFIG_CONFIGFS_FS=y
 CONFIG_JFFS2_FS=y
@@ -155,7 +160,6 @@ CONFIG_NFS_V3=y
 CONFIG_ROOT_NFS=y
 CONFIG_NFSD=m
 CONFIG_NFSD_V3=y
-CONFIG_SMB_FS=m
 CONFIG_CIFS=m
 CONFIG_NLS_CODEPAGE_437=m
 CONFIG_NLS_CODEPAGE_850=m
@@ -166,4 +170,3 @@ CONFIG_DEBUG_FS=y
 CONFIG_DEBUG_KERNEL=y
 CONFIG_DETECT_HUNG_TASK=y
 CONFIG_FRAME_POINTER=y
-# CONFIG_RCU_CPU_STALL_DETECTOR is not set
diff --git a/arch/avr32/configs/atngw100_evklcd101_defconfig b/arch/avr32/configs/atngw100_evklcd101_defconfig
index 7bc5b2ce68d5..387eb9d6e423 100644
--- a/arch/avr32/configs/atngw100_evklcd101_defconfig
+++ b/arch/avr32/configs/atngw100_evklcd101_defconfig
@@ -2,20 +2,17 @@ CONFIG_EXPERIMENTAL=y
 # CONFIG_LOCALVERSION_AUTO is not set
 CONFIG_SYSVIPC=y
 CONFIG_POSIX_MQUEUE=y
-CONFIG_BSD_PROCESS_ACCT=y
-CONFIG_BSD_PROCESS_ACCT_V3=y
 CONFIG_LOG_BUF_SHIFT=14
-CONFIG_SYSFS_DEPRECATED_V2=y
+CONFIG_RELAY=y
 CONFIG_BLK_DEV_INITRD=y
 # CONFIG_SYSCTL_SYSCALL is not set
 # CONFIG_BASE_FULL is not set
 # CONFIG_COMPAT_BRK is not set
 CONFIG_PROFILING=y
 CONFIG_OPROFILE=m
-CONFIG_KPROBES=y
+# CONFIG_KPROBES is not set
 CONFIG_MODULES=y
 CONFIG_MODULE_UNLOAD=y
-CONFIG_MODULE_FORCE_UNLOAD=y
 # CONFIG_BLK_DEV_BSG is not set
 # CONFIG_IOSCHED_DEADLINE is not set
 CONFIG_NO_HZ=y
@@ -30,6 +27,7 @@ CONFIG_CPU_FREQ=y
 CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND=y
 CONFIG_CPU_FREQ_GOV_USERSPACE=y
 CONFIG_CPU_FREQ_AT32AP=y
+CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS=y
 CONFIG_NET=y
 CONFIG_PACKET=y
 CONFIG_UNIX=y
@@ -73,8 +71,10 @@ CONFIG_MTD_UBI=y
 CONFIG_BLK_DEV_LOOP=m
 CONFIG_BLK_DEV_NBD=m
 CONFIG_BLK_DEV_RAM=m
+CONFIG_MISC_DEVICES=y
 CONFIG_ATMEL_TCLIB=y
 CONFIG_NETDEVICES=y
+CONFIG_TUN=m
 CONFIG_NET_ETHERNET=y
 CONFIG_MACB=y
 # CONFIG_NETDEV_1000 is not set
@@ -103,6 +103,7 @@ CONFIG_I2C_GPIO=m
 CONFIG_SPI=y
 CONFIG_SPI_ATMEL=y
 CONFIG_SPI_SPIDEV=m
+CONFIG_GPIO_SYSFS=y
 # CONFIG_HWMON is not set
 CONFIG_WATCHDOG=y
 CONFIG_AT32AP700X_WDT=y
@@ -126,6 +127,7 @@ CONFIG_USB_FILE_STORAGE=m
 CONFIG_USB_G_SERIAL=m
 CONFIG_USB_CDC_COMPOSITE=m
 CONFIG_MMC=y
+CONFIG_MMC_TEST=m
 CONFIG_MMC_ATMELMCI=y
 CONFIG_NEW_LEDS=y
 CONFIG_LEDS_CLASS=y
@@ -140,11 +142,14 @@ CONFIG_EXT2_FS=y
 CONFIG_EXT3_FS=y
 # CONFIG_EXT3_DEFAULTS_TO_ORDERED is not set
 # CONFIG_EXT3_FS_XATTR is not set
+CONFIG_EXT4_FS=y
+# CONFIG_EXT4_FS_XATTR is not set
 # CONFIG_DNOTIFY is not set
 CONFIG_FUSE_FS=m
 CONFIG_MSDOS_FS=m
 CONFIG_VFAT_FS=m
 CONFIG_FAT_DEFAULT_CODEPAGE=850
+CONFIG_PROC_KCORE=y
 CONFIG_TMPFS=y
 CONFIG_CONFIGFS_FS=y
 CONFIG_JFFS2_FS=y
@@ -154,7 +159,6 @@ CONFIG_NFS_V3=y
 CONFIG_ROOT_NFS=y
 CONFIG_NFSD=m
 CONFIG_NFSD_V3=y
-CONFIG_SMB_FS=m
 CONFIG_CIFS=m
 CONFIG_NLS_CODEPAGE_437=m
 CONFIG_NLS_CODEPAGE_850=m
@@ -165,4 +169,3 @@ CONFIG_DEBUG_FS=y
 CONFIG_DEBUG_KERNEL=y
 CONFIG_DETECT_HUNG_TASK=y
 CONFIG_FRAME_POINTER=y
-# CONFIG_RCU_CPU_STALL_DETECTOR is not set
diff --git a/arch/avr32/configs/atngw100mkii_defconfig b/arch/avr32/configs/atngw100mkii_defconfig
index 4bd36821d4a2..f0fe237133a9 100644
--- a/arch/avr32/configs/atngw100mkii_defconfig
+++ b/arch/avr32/configs/atngw100mkii_defconfig
@@ -2,20 +2,17 @@ CONFIG_EXPERIMENTAL=y
 # CONFIG_LOCALVERSION_AUTO is not set
 CONFIG_SYSVIPC=y
 CONFIG_POSIX_MQUEUE=y
-CONFIG_BSD_PROCESS_ACCT=y
-CONFIG_BSD_PROCESS_ACCT_V3=y
 CONFIG_LOG_BUF_SHIFT=14
-CONFIG_SYSFS_DEPRECATED_V2=y
+CONFIG_RELAY=y
 CONFIG_BLK_DEV_INITRD=y
 # CONFIG_SYSCTL_SYSCALL is not set
 # CONFIG_BASE_FULL is not set
 # CONFIG_COMPAT_BRK is not set
 CONFIG_PROFILING=y
 CONFIG_OPROFILE=m
-CONFIG_KPROBES=y
+# CONFIG_KPROBES is not set
 CONFIG_MODULES=y
 CONFIG_MODULE_UNLOAD=y
-CONFIG_MODULE_FORCE_UNLOAD=y
 # CONFIG_BLK_DEV_BSG is not set
 # CONFIG_IOSCHED_DEADLINE is not set
 CONFIG_NO_HZ=y
@@ -29,6 +26,7 @@ CONFIG_CPU_FREQ=y
 CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND=y
 CONFIG_CPU_FREQ_GOV_USERSPACE=y
 CONFIG_CPU_FREQ_AT32AP=y
+CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS=y
 CONFIG_NET=y
 CONFIG_PACKET=y
 CONFIG_UNIX=y
@@ -74,6 +72,7 @@ CONFIG_MTD_UBI=y
 CONFIG_BLK_DEV_LOOP=m
 CONFIG_BLK_DEV_NBD=m
 CONFIG_BLK_DEV_RAM=m
+CONFIG_MISC_DEVICES=y
 CONFIG_ATMEL_TCLIB=y
 CONFIG_NETDEVICES=y
 CONFIG_TUN=m
@@ -107,6 +106,7 @@ CONFIG_GPIO_SYSFS=y
 CONFIG_WATCHDOG=y
 CONFIG_AT32AP700X_WDT=y
 CONFIG_USB_GADGET=y
+CONFIG_USB_GADGET_VBUS_DRAW=350
 CONFIG_USB_ZERO=m
 CONFIG_USB_ETH=m
 CONFIG_USB_GADGETFS=m
@@ -116,14 +116,12 @@ CONFIG_USB_CDC_COMPOSITE=m
 CONFIG_MMC=y
 CONFIG_MMC_TEST=m
 CONFIG_MMC_ATMELMCI=y
-CONFIG_MMC_SPI=m
 CONFIG_NEW_LEDS=y
 CONFIG_LEDS_CLASS=y
 CONFIG_LEDS_GPIO=y
 CONFIG_LEDS_TRIGGERS=y
 CONFIG_LEDS_TRIGGER_TIMER=y
 CONFIG_LEDS_TRIGGER_HEARTBEAT=y
-CONFIG_LEDS_TRIGGER_DEFAULT_ON=y
 CONFIG_RTC_CLASS=y
 CONFIG_RTC_DRV_AT32AP700X=y
 CONFIG_DMADEVICES=y
@@ -131,21 +129,23 @@ CONFIG_EXT2_FS=y
 CONFIG_EXT3_FS=y
 # CONFIG_EXT3_DEFAULTS_TO_ORDERED is not set
 # CONFIG_EXT3_FS_XATTR is not set
+CONFIG_EXT4_FS=y
+# CONFIG_EXT4_FS_XATTR is not set
 # CONFIG_DNOTIFY is not set
 CONFIG_FUSE_FS=m
 CONFIG_MSDOS_FS=m
 CONFIG_VFAT_FS=m
 CONFIG_FAT_DEFAULT_CODEPAGE=850
+CONFIG_PROC_KCORE=y
 CONFIG_TMPFS=y
-CONFIG_CONFIGFS_FS=m
+CONFIG_CONFIGFS_FS=y
 CONFIG_JFFS2_FS=y
-CONFIG_UFS_FS=y
+CONFIG_UBIFS_FS=y
 CONFIG_NFS_FS=y
 CONFIG_NFS_V3=y
 CONFIG_ROOT_NFS=y
 CONFIG_NFSD=m
 CONFIG_NFSD_V3=y
-CONFIG_SMB_FS=m
 CONFIG_CIFS=m
 CONFIG_NLS_CODEPAGE_437=m
 CONFIG_NLS_CODEPAGE_850=m
@@ -156,5 +156,3 @@ CONFIG_DEBUG_FS=y
 CONFIG_DEBUG_KERNEL=y
 CONFIG_DETECT_HUNG_TASK=y
 CONFIG_FRAME_POINTER=y
-# CONFIG_RCU_CPU_STALL_DETECTOR is not set
-CONFIG_CRYPTO_PCBC=m
diff --git a/arch/avr32/configs/atngw100mkii_evklcd100_defconfig b/arch/avr32/configs/atngw100mkii_evklcd100_defconfig
index f8437ef3237f..e4a7c1dc8380 100644
--- a/arch/avr32/configs/atngw100mkii_evklcd100_defconfig
+++ b/arch/avr32/configs/atngw100mkii_evklcd100_defconfig
@@ -2,20 +2,17 @@ CONFIG_EXPERIMENTAL=y
 # CONFIG_LOCALVERSION_AUTO is not set
 CONFIG_SYSVIPC=y
 CONFIG_POSIX_MQUEUE=y
-CONFIG_BSD_PROCESS_ACCT=y
-CONFIG_BSD_PROCESS_ACCT_V3=y
 CONFIG_LOG_BUF_SHIFT=14
-CONFIG_SYSFS_DEPRECATED_V2=y
+CONFIG_RELAY=y
 CONFIG_BLK_DEV_INITRD=y
 # CONFIG_SYSCTL_SYSCALL is not set
 # CONFIG_BASE_FULL is not set
 # CONFIG_COMPAT_BRK is not set
 CONFIG_PROFILING=y
 CONFIG_OPROFILE=m
-CONFIG_KPROBES=y
+# CONFIG_KPROBES is not set
 CONFIG_MODULES=y
 CONFIG_MODULE_UNLOAD=y
-CONFIG_MODULE_FORCE_UNLOAD=y
 # CONFIG_BLK_DEV_BSG is not set
 # CONFIG_IOSCHED_DEADLINE is not set
 CONFIG_NO_HZ=y
@@ -32,6 +29,7 @@ CONFIG_CPU_FREQ=y
 CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND=y
 CONFIG_CPU_FREQ_GOV_USERSPACE=y
 CONFIG_CPU_FREQ_AT32AP=y
+CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS=y
 CONFIG_NET=y
 CONFIG_PACKET=y
 CONFIG_UNIX=y
@@ -77,8 +75,10 @@ CONFIG_MTD_UBI=y
 CONFIG_BLK_DEV_LOOP=m
 CONFIG_BLK_DEV_NBD=m
 CONFIG_BLK_DEV_RAM=m
+CONFIG_MISC_DEVICES=y
 CONFIG_ATMEL_TCLIB=y
 CONFIG_NETDEVICES=y
+CONFIG_TUN=m
 CONFIG_NET_ETHERNET=y
 CONFIG_MACB=y
 # CONFIG_NETDEV_1000 is not set
@@ -107,6 +107,7 @@ CONFIG_I2C_GPIO=m
 CONFIG_SPI=y
 CONFIG_SPI_ATMEL=y
 CONFIG_SPI_SPIDEV=m
+CONFIG_GPIO_SYSFS=y
 # CONFIG_HWMON is not set
 CONFIG_WATCHDOG=y
 CONFIG_AT32AP700X_WDT=y
@@ -130,6 +131,7 @@ CONFIG_USB_FILE_STORAGE=m
 CONFIG_USB_G_SERIAL=m
 CONFIG_USB_CDC_COMPOSITE=m
 CONFIG_MMC=y
+CONFIG_MMC_TEST=m
 CONFIG_MMC_ATMELMCI=y
 CONFIG_NEW_LEDS=y
 CONFIG_LEDS_CLASS=y
@@ -144,11 +146,14 @@ CONFIG_EXT2_FS=y
 CONFIG_EXT3_FS=y
 # CONFIG_EXT3_DEFAULTS_TO_ORDERED is not set
 # CONFIG_EXT3_FS_XATTR is not set
+CONFIG_EXT4_FS=y
+# CONFIG_EXT4_FS_XATTR is not set
 # CONFIG_DNOTIFY is not set
 CONFIG_FUSE_FS=m
 CONFIG_MSDOS_FS=m
 CONFIG_VFAT_FS=m
 CONFIG_FAT_DEFAULT_CODEPAGE=850
+CONFIG_PROC_KCORE=y
 CONFIG_TMPFS=y
 CONFIG_CONFIGFS_FS=y
 CONFIG_JFFS2_FS=y
@@ -158,7 +163,6 @@ CONFIG_NFS_V3=y
 CONFIG_ROOT_NFS=y
 CONFIG_NFSD=m
 CONFIG_NFSD_V3=y
-CONFIG_SMB_FS=m
 CONFIG_CIFS=m
 CONFIG_NLS_CODEPAGE_437=m
 CONFIG_NLS_CODEPAGE_850=m
@@ -169,4 +173,3 @@ CONFIG_DEBUG_FS=y
 CONFIG_DEBUG_KERNEL=y
 CONFIG_DETECT_HUNG_TASK=y
 CONFIG_FRAME_POINTER=y
-# CONFIG_RCU_CPU_STALL_DETECTOR is not set
diff --git a/arch/avr32/configs/atngw100mkii_evklcd101_defconfig b/arch/avr32/configs/atngw100mkii_evklcd101_defconfig
index 7f58f996d945..6f37f70c2c37 100644
--- a/arch/avr32/configs/atngw100mkii_evklcd101_defconfig
+++ b/arch/avr32/configs/atngw100mkii_evklcd101_defconfig
@@ -2,20 +2,17 @@ CONFIG_EXPERIMENTAL=y
 # CONFIG_LOCALVERSION_AUTO is not set
 CONFIG_SYSVIPC=y
 CONFIG_POSIX_MQUEUE=y
-CONFIG_BSD_PROCESS_ACCT=y
-CONFIG_BSD_PROCESS_ACCT_V3=y
 CONFIG_LOG_BUF_SHIFT=14
-CONFIG_SYSFS_DEPRECATED_V2=y
+CONFIG_RELAY=y
 CONFIG_BLK_DEV_INITRD=y
 # CONFIG_SYSCTL_SYSCALL is not set
 # CONFIG_BASE_FULL is not set
 # CONFIG_COMPAT_BRK is not set
 CONFIG_PROFILING=y
 CONFIG_OPROFILE=m
-CONFIG_KPROBES=y
+# CONFIG_KPROBES is not set
 CONFIG_MODULES=y
 CONFIG_MODULE_UNLOAD=y
-CONFIG_MODULE_FORCE_UNLOAD=y
 # CONFIG_BLK_DEV_BSG is not set
 # CONFIG_IOSCHED_DEADLINE is not set
 CONFIG_NO_HZ=y
@@ -31,6 +28,7 @@ CONFIG_CPU_FREQ=y
 CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND=y
 CONFIG_CPU_FREQ_GOV_USERSPACE=y
 CONFIG_CPU_FREQ_AT32AP=y
+CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS=y
 CONFIG_NET=y
 CONFIG_PACKET=y
 CONFIG_UNIX=y
@@ -76,8 +74,10 @@ CONFIG_MTD_UBI=y
 CONFIG_BLK_DEV_LOOP=m
 CONFIG_BLK_DEV_NBD=m
 CONFIG_BLK_DEV_RAM=m
+CONFIG_MISC_DEVICES=y
 CONFIG_ATMEL_TCLIB=y
 CONFIG_NETDEVICES=y
+CONFIG_TUN=m
 CONFIG_NET_ETHERNET=y
 CONFIG_MACB=y
 # CONFIG_NETDEV_1000 is not set
@@ -106,6 +106,7 @@ CONFIG_I2C_GPIO=m
 CONFIG_SPI=y
 CONFIG_SPI_ATMEL=y
 CONFIG_SPI_SPIDEV=m
+CONFIG_GPIO_SYSFS=y
 # CONFIG_HWMON is not set
 CONFIG_WATCHDOG=y
 CONFIG_AT32AP700X_WDT=y
@@ -129,6 +130,7 @@ CONFIG_USB_FILE_STORAGE=m
 CONFIG_USB_G_SERIAL=m
 CONFIG_USB_CDC_COMPOSITE=m
 CONFIG_MMC=y
+CONFIG_MMC_TEST=m
 CONFIG_MMC_ATMELMCI=y
 CONFIG_NEW_LEDS=y
 CONFIG_LEDS_CLASS=y
@@ -143,11 +145,14 @@ CONFIG_EXT2_FS=y
 CONFIG_EXT3_FS=y
 # CONFIG_EXT3_DEFAULTS_TO_ORDERED is not set
 # CONFIG_EXT3_FS_XATTR is not set
+CONFIG_EXT4_FS=y
+# CONFIG_EXT4_FS_XATTR is not set
 # CONFIG_DNOTIFY is not set
 CONFIG_FUSE_FS=m
 CONFIG_MSDOS_FS=m
 CONFIG_VFAT_FS=m
 CONFIG_FAT_DEFAULT_CODEPAGE=850
+CONFIG_PROC_KCORE=y
 CONFIG_TMPFS=y
 CONFIG_CONFIGFS_FS=y
 CONFIG_JFFS2_FS=y
@@ -157,7 +162,6 @@ CONFIG_NFS_V3=y
 CONFIG_ROOT_NFS=y
 CONFIG_NFSD=m
 CONFIG_NFSD_V3=y
-CONFIG_SMB_FS=m
 CONFIG_CIFS=m
 CONFIG_NLS_CODEPAGE_437=m
 CONFIG_NLS_CODEPAGE_850=m
@@ -168,4 +172,3 @@ CONFIG_DEBUG_FS=y
 CONFIG_DEBUG_KERNEL=y
 CONFIG_DETECT_HUNG_TASK=y
 CONFIG_FRAME_POINTER=y
-# CONFIG_RCU_CPU_STALL_DETECTOR is not set
diff --git a/arch/avr32/configs/atstk1002_defconfig b/arch/avr32/configs/atstk1002_defconfig
index aec4c43a75da..4fb01f5ab42f 100644
--- a/arch/avr32/configs/atstk1002_defconfig
+++ b/arch/avr32/configs/atstk1002_defconfig
@@ -3,7 +3,6 @@ CONFIG_EXPERIMENTAL=y
 CONFIG_SYSVIPC=y
 CONFIG_POSIX_MQUEUE=y
 CONFIG_LOG_BUF_SHIFT=14
-CONFIG_SYSFS_DEPRECATED_V2=y
 CONFIG_RELAY=y
 CONFIG_BLK_DEV_INITRD=y
 # CONFIG_SYSCTL_SYSCALL is not set
@@ -11,7 +10,7 @@ CONFIG_BLK_DEV_INITRD=y
 # CONFIG_COMPAT_BRK is not set
 CONFIG_PROFILING=y
 CONFIG_OPROFILE=m
-CONFIG_KPROBES=y
+# CONFIG_KPROBES is not set
 CONFIG_MODULES=y
 CONFIG_MODULE_UNLOAD=y
 # CONFIG_BLK_DEV_BSG is not set
@@ -26,6 +25,7 @@ CONFIG_CPU_FREQ=y
 CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND=y
 CONFIG_CPU_FREQ_GOV_USERSPACE=y
 CONFIG_CPU_FREQ_AT32AP=y
+CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS=y
 CONFIG_NET=y
 CONFIG_PACKET=y
 CONFIG_UNIX=y
@@ -35,6 +35,7 @@ CONFIG_INET=y
 CONFIG_IP_PNP=y
 CONFIG_IP_PNP_DHCP=y
 CONFIG_NET_IPIP=m
+CONFIG_NET_IPGRE_DEMUX=m
 CONFIG_NET_IPGRE=m
 CONFIG_INET_AH=m
 CONFIG_INET_ESP=m
@@ -58,16 +59,14 @@ CONFIG_MTD_BLOCK=y
 CONFIG_MTD_CFI=y
 CONFIG_MTD_CFI_AMDSTD=y
 CONFIG_MTD_PHYSMAP=y
-CONFIG_MTD_DATAFLASH=m
-CONFIG_MTD_M25P80=m
 CONFIG_MTD_UBI=y
 CONFIG_BLK_DEV_LOOP=m
 CONFIG_BLK_DEV_NBD=m
 CONFIG_BLK_DEV_RAM=m
+CONFIG_MISC_DEVICES=y
 CONFIG_ATMEL_PWM=m
 CONFIG_ATMEL_TCLIB=y
 CONFIG_ATMEL_SSC=m
-CONFIG_EEPROM_AT24=m
 # CONFIG_SCSI_PROC_FS is not set
 CONFIG_BLK_DEV_SD=m
 CONFIG_BLK_DEV_SR=m
@@ -120,7 +119,6 @@ CONFIG_SND_MIXER_OSS=m
 CONFIG_SND_PCM_OSS=m
 # CONFIG_SND_SUPPORT_OLD_API is not set
 # CONFIG_SND_VERBOSE_PROCFS is not set
-# CONFIG_SND_DRIVERS is not set
 CONFIG_SND_AT73C213=m
 # CONFIG_HID_SUPPORT is not set
 CONFIG_USB_GADGET=y
@@ -131,16 +129,15 @@ CONFIG_USB_FILE_STORAGE=m
 CONFIG_USB_G_SERIAL=m
 CONFIG_USB_CDC_COMPOSITE=m
 CONFIG_MMC=y
+CONFIG_MMC_TEST=m
 CONFIG_MMC_ATMELMCI=y
-CONFIG_MMC_SPI=m
 CONFIG_NEW_LEDS=y
-CONFIG_LEDS_CLASS=m
+CONFIG_LEDS_CLASS=y
 CONFIG_LEDS_ATMEL_PWM=m
 CONFIG_LEDS_GPIO=m
 CONFIG_LEDS_TRIGGERS=y
 CONFIG_LEDS_TRIGGER_TIMER=m
 CONFIG_LEDS_TRIGGER_HEARTBEAT=m
-CONFIG_LEDS_TRIGGER_DEFAULT_ON=m
 CONFIG_RTC_CLASS=y
 CONFIG_RTC_DRV_AT32AP700X=y
 CONFIG_DMADEVICES=y
@@ -149,20 +146,23 @@ CONFIG_EXT3_FS=y
 # CONFIG_EXT3_DEFAULTS_TO_ORDERED is not set
 # CONFIG_EXT3_FS_XATTR is not set
 CONFIG_EXT4_FS=y
+# CONFIG_EXT4_FS_XATTR is not set
 # CONFIG_DNOTIFY is not set
 CONFIG_FUSE_FS=m
 CONFIG_MSDOS_FS=m
 CONFIG_VFAT_FS=m
+CONFIG_FAT_DEFAULT_CODEPAGE=850
 CONFIG_PROC_KCORE=y
 CONFIG_TMPFS=y
+CONFIG_CONFIGFS_FS=y
 CONFIG_JFFS2_FS=y
-# CONFIG_JFFS2_FS_WRITEBUFFER is not set
 CONFIG_UBIFS_FS=y
-CONFIG_MINIX_FS=m
 CONFIG_NFS_FS=y
 CONFIG_NFS_V3=y
 CONFIG_ROOT_NFS=y
+CONFIG_CIFS=m
 CONFIG_NLS_CODEPAGE_437=m
+CONFIG_NLS_CODEPAGE_850=m
 CONFIG_NLS_ISO8859_1=m
 CONFIG_NLS_UTF8=m
 CONFIG_MAGIC_SYSRQ=y
@@ -170,6 +170,3 @@ CONFIG_DEBUG_FS=y
 CONFIG_DEBUG_KERNEL=y
 CONFIG_DETECT_HUNG_TASK=y
 CONFIG_FRAME_POINTER=y
-# CONFIG_RCU_CPU_STALL_DETECTOR is not set
-# CONFIG_CRYPTO_HW is not set
-CONFIG_CRC_T10DIF=m
diff --git a/arch/avr32/configs/atstk1003_defconfig b/arch/avr32/configs/atstk1003_defconfig
index 50ba3db682ca..9faaf9b900f2 100644
--- a/arch/avr32/configs/atstk1003_defconfig
+++ b/arch/avr32/configs/atstk1003_defconfig
@@ -2,22 +2,15 @@ CONFIG_EXPERIMENTAL=y
 # CONFIG_LOCALVERSION_AUTO is not set
 CONFIG_SYSVIPC=y
 CONFIG_POSIX_MQUEUE=y
-CONFIG_BSD_PROCESS_ACCT=y
-CONFIG_BSD_PROCESS_ACCT_V3=y
-CONFIG_TASKSTATS=y
-CONFIG_TASK_DELAY_ACCT=y
-CONFIG_AUDIT=y
 CONFIG_LOG_BUF_SHIFT=14
-CONFIG_SYSFS_DEPRECATED_V2=y
 CONFIG_RELAY=y
 CONFIG_BLK_DEV_INITRD=y
 # CONFIG_SYSCTL_SYSCALL is not set
 # CONFIG_BASE_FULL is not set
-# CONFIG_SLUB_DEBUG is not set
 # CONFIG_COMPAT_BRK is not set
 CONFIG_PROFILING=y
 CONFIG_OPROFILE=m
-CONFIG_KPROBES=y
+# CONFIG_KPROBES is not set
 CONFIG_MODULES=y
 CONFIG_MODULE_UNLOAD=y
 # CONFIG_BLK_DEV_BSG is not set
@@ -33,6 +26,7 @@ CONFIG_CPU_FREQ=y
 CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND=y
 CONFIG_CPU_FREQ_GOV_USERSPACE=y
 CONFIG_CPU_FREQ_AT32AP=y
+CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS=y
 CONFIG_NET=y
 CONFIG_PACKET=y
 CONFIG_UNIX=y
@@ -54,18 +48,18 @@ CONFIG_MTD_BLOCK=y
 CONFIG_MTD_CFI=y
 CONFIG_MTD_CFI_AMDSTD=y
 CONFIG_MTD_PHYSMAP=y
-CONFIG_MTD_DATAFLASH=m
-CONFIG_MTD_M25P80=m
+CONFIG_MTD_UBI=y
 CONFIG_BLK_DEV_LOOP=m
 CONFIG_BLK_DEV_NBD=m
 CONFIG_BLK_DEV_RAM=m
+CONFIG_MISC_DEVICES=y
 CONFIG_ATMEL_PWM=m
 CONFIG_ATMEL_TCLIB=y
 CONFIG_ATMEL_SSC=m
-CONFIG_EEPROM_AT24=m
 # CONFIG_SCSI_PROC_FS is not set
 CONFIG_BLK_DEV_SD=m
 CONFIG_BLK_DEV_SR=m
+# CONFIG_SCSI_LOWLEVEL is not set
 CONFIG_ATA=m
 # CONFIG_SATA_PMP is not set
 CONFIG_PATA_AT32=m
@@ -77,6 +71,7 @@ CONFIG_PPP_ASYNC=m
 CONFIG_PPP_DEFLATE=m
 CONFIG_PPP_BSDCOMP=m
 CONFIG_INPUT=m
+CONFIG_INPUT_EVDEV=m
 # CONFIG_KEYBOARD_ATKBD is not set
 CONFIG_KEYBOARD_GPIO=m
 # CONFIG_MOUSE_PS2 is not set
@@ -106,7 +101,6 @@ CONFIG_SND_PCM_OSS=m
 CONFIG_SND_AT73C213=m
 # CONFIG_HID_SUPPORT is not set
 CONFIG_USB_GADGET=y
-CONFIG_USB_GADGET_DEBUG_FS=y
 CONFIG_USB_ZERO=m
 CONFIG_USB_ETH=m
 CONFIG_USB_GADGETFS=m
@@ -116,36 +110,39 @@ CONFIG_USB_CDC_COMPOSITE=m
 CONFIG_MMC=y
 CONFIG_MMC_TEST=m
 CONFIG_MMC_ATMELMCI=y
-CONFIG_MMC_SPI=m
 CONFIG_NEW_LEDS=y
 CONFIG_LEDS_CLASS=y
 CONFIG_LEDS_ATMEL_PWM=m
-CONFIG_LEDS_GPIO=y
+CONFIG_LEDS_GPIO=m
 CONFIG_LEDS_TRIGGERS=y
-CONFIG_LEDS_TRIGGER_TIMER=y
-CONFIG_LEDS_TRIGGER_HEARTBEAT=y
-CONFIG_LEDS_TRIGGER_DEFAULT_ON=y
+CONFIG_LEDS_TRIGGER_TIMER=m
+CONFIG_LEDS_TRIGGER_HEARTBEAT=m
 CONFIG_RTC_CLASS=y
 CONFIG_RTC_DRV_AT32AP700X=y
 CONFIG_DMADEVICES=y
-CONFIG_DW_DMAC=y
-CONFIG_EXT2_FS=m
-CONFIG_EXT3_FS=m
+CONFIG_EXT2_FS=y
+CONFIG_EXT3_FS=y
+# CONFIG_EXT3_DEFAULTS_TO_ORDERED is not set
 # CONFIG_EXT3_FS_XATTR is not set
+CONFIG_EXT4_FS=y
+# CONFIG_EXT4_FS_XATTR is not set
 # CONFIG_DNOTIFY is not set
 CONFIG_FUSE_FS=m
 CONFIG_MSDOS_FS=m
 CONFIG_VFAT_FS=m
+CONFIG_FAT_DEFAULT_CODEPAGE=850
 CONFIG_PROC_KCORE=y
 CONFIG_TMPFS=y
-CONFIG_CONFIGFS_FS=m
+CONFIG_CONFIGFS_FS=y
 CONFIG_JFFS2_FS=y
+CONFIG_UBIFS_FS=y
 # CONFIG_NETWORK_FILESYSTEMS is not set
 CONFIG_NLS_CODEPAGE_437=m
+CONFIG_NLS_CODEPAGE_850=m
 CONFIG_NLS_ISO8859_1=m
 CONFIG_NLS_UTF8=m
 CONFIG_MAGIC_SYSRQ=y
 CONFIG_DEBUG_FS=y
 CONFIG_DEBUG_KERNEL=y
+CONFIG_DETECT_HUNG_TASK=y
 CONFIG_FRAME_POINTER=y
-CONFIG_CRC_T10DIF=m
diff --git a/arch/avr32/configs/atstk1004_defconfig b/arch/avr32/configs/atstk1004_defconfig
index 329e10ba3b54..3d2a5d85f970 100644
--- a/arch/avr32/configs/atstk1004_defconfig
+++ b/arch/avr32/configs/atstk1004_defconfig
@@ -1,19 +1,32 @@
 CONFIG_EXPERIMENTAL=y
 # CONFIG_LOCALVERSION_AUTO is not set
+CONFIG_SYSVIPC=y
+CONFIG_POSIX_MQUEUE=y
 CONFIG_LOG_BUF_SHIFT=14
-CONFIG_SYSFS_DEPRECATED_V2=y
+CONFIG_RELAY=y
+CONFIG_BLK_DEV_INITRD=y
 # CONFIG_SYSCTL_SYSCALL is not set
 # CONFIG_BASE_FULL is not set
-# CONFIG_FUTEX is not set
-# CONFIG_EPOLL is not set
-# CONFIG_SIGNALFD is not set
-# CONFIG_TIMERFD is not set
-# CONFIG_EVENTFD is not set
 # CONFIG_COMPAT_BRK is not set
-CONFIG_SLOB=y
-# CONFIG_BLOCK is not set
+CONFIG_PROFILING=y
+CONFIG_OPROFILE=m
+# CONFIG_KPROBES is not set
+CONFIG_MODULES=y
+CONFIG_MODULE_UNLOAD=y
+# CONFIG_BLK_DEV_BSG is not set
+# CONFIG_IOSCHED_DEADLINE is not set
+CONFIG_NO_HZ=y
+CONFIG_HIGH_RES_TIMERS=y
 CONFIG_BOARD_ATSTK1004=y
 # CONFIG_OWNERSHIP_TRACE is not set
+CONFIG_NMI_DEBUGGING=y
+CONFIG_PM=y
+CONFIG_CPU_FREQ=y
+# CONFIG_CPU_FREQ_STAT is not set
+CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND=y
+CONFIG_CPU_FREQ_GOV_USERSPACE=y
+CONFIG_CPU_FREQ_AT32AP=y
+CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS=y
 CONFIG_NET=y
 CONFIG_PACKET=y
 CONFIG_UNIX=y
@@ -31,40 +44,104 @@ CONFIG_MTD=y
 CONFIG_MTD_PARTITIONS=y
 CONFIG_MTD_CMDLINE_PARTS=y
 CONFIG_MTD_CHAR=y
+CONFIG_MTD_BLOCK=y
 CONFIG_MTD_CFI=y
 CONFIG_MTD_CFI_AMDSTD=y
 CONFIG_MTD_PHYSMAP=y
-# CONFIG_MISC_DEVICES is not set
-# CONFIG_INPUT is not set
+CONFIG_MTD_UBI=y
+CONFIG_BLK_DEV_LOOP=m
+CONFIG_BLK_DEV_NBD=m
+CONFIG_BLK_DEV_RAM=m
+CONFIG_MISC_DEVICES=y
+CONFIG_ATMEL_PWM=m
+CONFIG_ATMEL_TCLIB=y
+CONFIG_ATMEL_SSC=m
+# CONFIG_SCSI_PROC_FS is not set
+CONFIG_BLK_DEV_SD=m
+CONFIG_BLK_DEV_SR=m
+# CONFIG_SCSI_LOWLEVEL is not set
+CONFIG_ATA=m
+# CONFIG_SATA_PMP is not set
+CONFIG_PATA_AT32=m
+CONFIG_NETDEVICES=y
+# CONFIG_NETDEV_1000 is not set
+# CONFIG_NETDEV_10000 is not set
+CONFIG_PPP=m
+CONFIG_PPP_ASYNC=m
+CONFIG_PPP_DEFLATE=m
+CONFIG_PPP_BSDCOMP=m
+CONFIG_INPUT=m
+CONFIG_INPUT_EVDEV=m
+# CONFIG_KEYBOARD_ATKBD is not set
+CONFIG_KEYBOARD_GPIO=m
+# CONFIG_MOUSE_PS2 is not set
+CONFIG_MOUSE_GPIO=m
 # CONFIG_SERIO is not set
 # CONFIG_VT is not set
 # CONFIG_DEVKMEM is not set
 CONFIG_SERIAL_ATMEL=y
 CONFIG_SERIAL_ATMEL_CONSOLE=y
-# CONFIG_SERIAL_ATMEL_PDC is not set
 # CONFIG_LEGACY_PTYS is not set
 # CONFIG_HW_RANDOM is not set
+CONFIG_I2C=m
+CONFIG_I2C_CHARDEV=m
+CONFIG_I2C_GPIO=m
 CONFIG_SPI=y
 CONFIG_SPI_ATMEL=y
+CONFIG_SPI_SPIDEV=m
+CONFIG_GPIO_SYSFS=y
 # CONFIG_HWMON is not set
 CONFIG_WATCHDOG=y
 CONFIG_AT32AP700X_WDT=y
 CONFIG_FB=y
 CONFIG_FB_ATMEL=y
 CONFIG_BACKLIGHT_LCD_SUPPORT=y
+CONFIG_LCD_CLASS_DEVICE=y
 CONFIG_LCD_LTV350QV=y
 # CONFIG_BACKLIGHT_CLASS_DEVICE is not set
 CONFIG_USB_GADGET=y
-CONFIG_USB_ETH=y
-# CONFIG_USB_ETH_RNDIS is not set
+CONFIG_USB_ZERO=m
+CONFIG_USB_ETH=m
+CONFIG_USB_GADGETFS=m
+CONFIG_USB_FILE_STORAGE=m
+CONFIG_USB_G_SERIAL=m
+CONFIG_USB_CDC_COMPOSITE=m
+CONFIG_MMC=y
+CONFIG_MMC_TEST=m
+CONFIG_MMC_ATMELMCI=y
+CONFIG_NEW_LEDS=y
+CONFIG_LEDS_CLASS=y
+CONFIG_LEDS_ATMEL_PWM=m
+CONFIG_LEDS_GPIO=m
+CONFIG_LEDS_TRIGGERS=y
+CONFIG_LEDS_TRIGGER_TIMER=m
+CONFIG_LEDS_TRIGGER_HEARTBEAT=m
 CONFIG_RTC_CLASS=y
-# CONFIG_RTC_INTF_PROC is not set
 CONFIG_RTC_DRV_AT32AP700X=y
+CONFIG_DMADEVICES=y
+CONFIG_EXT2_FS=y
+CONFIG_EXT3_FS=y
+# CONFIG_EXT3_DEFAULTS_TO_ORDERED is not set
+# CONFIG_EXT3_FS_XATTR is not set
+CONFIG_EXT4_FS=y
+# CONFIG_EXT4_FS_XATTR is not set
 # CONFIG_DNOTIFY is not set
+CONFIG_FUSE_FS=m
+CONFIG_MSDOS_FS=m
+CONFIG_VFAT_FS=m
+CONFIG_FAT_DEFAULT_CODEPAGE=850
 CONFIG_PROC_KCORE=y
-# CONFIG_PROC_PAGE_MONITOR is not set
 CONFIG_TMPFS=y
+CONFIG_CONFIGFS_FS=y
 CONFIG_JFFS2_FS=y
-# CONFIG_JFFS2_FS_WRITEBUFFER is not set
+CONFIG_UBIFS_FS=y
 # CONFIG_NETWORK_FILESYSTEMS is not set
+CONFIG_NLS_CODEPAGE_437=m
+CONFIG_NLS_CODEPAGE_850=m
+CONFIG_NLS_ISO8859_1=m
+CONFIG_NLS_UTF8=m
 CONFIG_MAGIC_SYSRQ=y
+CONFIG_DEBUG_FS=y
+CONFIG_DEBUG_KERNEL=y
+CONFIG_DETECT_HUNG_TASK=y
+CONFIG_FRAME_POINTER=y
diff --git a/arch/avr32/configs/atstk1006_defconfig b/arch/avr32/configs/atstk1006_defconfig
index dbcc1b51e506..1ed8f22d4fe2 100644
--- a/arch/avr32/configs/atstk1006_defconfig
+++ b/arch/avr32/configs/atstk1006_defconfig
@@ -3,7 +3,6 @@ CONFIG_EXPERIMENTAL=y
 CONFIG_SYSVIPC=y
 CONFIG_POSIX_MQUEUE=y
 CONFIG_LOG_BUF_SHIFT=14
-CONFIG_SYSFS_DEPRECATED_V2=y
 CONFIG_RELAY=y
 CONFIG_BLK_DEV_INITRD=y
 # CONFIG_SYSCTL_SYSCALL is not set
@@ -11,7 +10,7 @@ CONFIG_BLK_DEV_INITRD=y
 # CONFIG_COMPAT_BRK is not set
 CONFIG_PROFILING=y
 CONFIG_OPROFILE=m
-CONFIG_KPROBES=y
+# CONFIG_KPROBES is not set
 CONFIG_MODULES=y
 CONFIG_MODULE_UNLOAD=y
 # CONFIG_BLK_DEV_BSG is not set
@@ -37,6 +36,7 @@ CONFIG_INET=y
 CONFIG_IP_PNP=y
 CONFIG_IP_PNP_DHCP=y
 CONFIG_NET_IPIP=m
+CONFIG_NET_IPGRE_DEMUX=m
 CONFIG_NET_IPGRE=m
 CONFIG_INET_AH=m
 CONFIG_INET_ESP=m
@@ -60,15 +60,13 @@ CONFIG_MTD_BLOCK=y
 CONFIG_MTD_CFI=y
 CONFIG_MTD_CFI_AMDSTD=y
 CONFIG_MTD_PHYSMAP=y
-CONFIG_MTD_DATAFLASH=m
-CONFIG_MTD_DATAFLASH_OTP=y
-CONFIG_MTD_M25P80=m
 CONFIG_MTD_NAND=y
 CONFIG_MTD_NAND_ATMEL=y
 CONFIG_MTD_UBI=y
 CONFIG_BLK_DEV_LOOP=m
 CONFIG_BLK_DEV_NBD=m
 CONFIG_BLK_DEV_RAM=m
+CONFIG_MISC_DEVICES=y
 CONFIG_ATMEL_PWM=m
 CONFIG_ATMEL_TCLIB=y
 CONFIG_ATMEL_SSC=m
@@ -132,17 +130,17 @@ CONFIG_USB_ETH=m
 CONFIG_USB_GADGETFS=m
 CONFIG_USB_FILE_STORAGE=m
 CONFIG_USB_G_SERIAL=m
+CONFIG_USB_CDC_COMPOSITE=m
 CONFIG_MMC=y
+CONFIG_MMC_TEST=m
 CONFIG_MMC_ATMELMCI=y
-CONFIG_MMC_SPI=m
 CONFIG_NEW_LEDS=y
-CONFIG_LEDS_CLASS=m
+CONFIG_LEDS_CLASS=y
 CONFIG_LEDS_ATMEL_PWM=m
 CONFIG_LEDS_GPIO=m
 CONFIG_LEDS_TRIGGERS=y
 CONFIG_LEDS_TRIGGER_TIMER=m
 CONFIG_LEDS_TRIGGER_HEARTBEAT=m
-CONFIG_LEDS_TRIGGER_DEFAULT_ON=m
 CONFIG_RTC_CLASS=y
 CONFIG_RTC_DRV_AT32AP700X=y
 CONFIG_DMADEVICES=y
@@ -156,15 +154,18 @@ CONFIG_EXT4_FS=y
 CONFIG_FUSE_FS=m
 CONFIG_MSDOS_FS=m
 CONFIG_VFAT_FS=m
+CONFIG_FAT_DEFAULT_CODEPAGE=850
 CONFIG_PROC_KCORE=y
 CONFIG_TMPFS=y
+CONFIG_CONFIGFS_FS=y
 CONFIG_JFFS2_FS=y
 CONFIG_UBIFS_FS=y
-CONFIG_MINIX_FS=m
 CONFIG_NFS_FS=y
 CONFIG_NFS_V3=y
 CONFIG_ROOT_NFS=y
+CONFIG_CIFS=m
 CONFIG_NLS_CODEPAGE_437=m
+CONFIG_NLS_CODEPAGE_850=m
 CONFIG_NLS_ISO8859_1=m
 CONFIG_NLS_UTF8=m
 CONFIG_MAGIC_SYSRQ=y
@@ -172,7 +173,3 @@ CONFIG_DEBUG_FS=y
 CONFIG_DEBUG_KERNEL=y
 CONFIG_DETECT_HUNG_TASK=y
 CONFIG_FRAME_POINTER=y
-# CONFIG_RCU_CPU_STALL_DETECTOR is not set
-CONFIG_CRYPTO_FIPS=y
-# CONFIG_CRYPTO_HW is not set
-CONFIG_CRC_T10DIF=m
diff --git a/arch/avr32/configs/favr-32_defconfig b/arch/avr32/configs/favr-32_defconfig
index 0c813b661a0a..aeadc955db32 100644
--- a/arch/avr32/configs/favr-32_defconfig
+++ b/arch/avr32/configs/favr-32_defconfig
@@ -11,7 +11,7 @@ CONFIG_BLK_DEV_INITRD=y
 # CONFIG_COMPAT_BRK is not set
 CONFIG_PROFILING=y
 CONFIG_OPROFILE=m
-CONFIG_KPROBES=y
+# CONFIG_KPROBES is not set
 CONFIG_MODULES=y
 CONFIG_MODULE_UNLOAD=y
 # CONFIG_BLK_DEV_BSG is not set
diff --git a/arch/avr32/configs/hammerhead_defconfig b/arch/avr32/configs/hammerhead_defconfig
index dcc01f0eb294..1692beeb7ed3 100644
--- a/arch/avr32/configs/hammerhead_defconfig
+++ b/arch/avr32/configs/hammerhead_defconfig
@@ -12,7 +12,7 @@ CONFIG_BLK_DEV_INITRD=y
 # CONFIG_COMPAT_BRK is not set
 CONFIG_PROFILING=y
 CONFIG_OPROFILE=m
-CONFIG_KPROBES=y
+# CONFIG_KPROBES is not set
 CONFIG_MODULES=y
 CONFIG_MODULE_UNLOAD=y
 CONFIG_MODULE_FORCE_UNLOAD=y
diff --git a/arch/avr32/include/asm/syscalls.h b/arch/avr32/include/asm/syscalls.h
index ab608b70b24d..244f2acab546 100644
--- a/arch/avr32/include/asm/syscalls.h
+++ b/arch/avr32/include/asm/syscalls.h
@@ -15,20 +15,6 @@
 #include <linux/types.h>
 #include <linux/signal.h>
 
-/* kernel/process.c */
-asmlinkage int sys_fork(struct pt_regs *);
-asmlinkage int sys_clone(unsigned long, unsigned long,
-                         unsigned long, unsigned long,
-                         struct pt_regs *);
-asmlinkage int sys_vfork(struct pt_regs *);
-asmlinkage int sys_execve(const char __user *, char __user *__user *,
-                          char __user *__user *, struct pt_regs *);
-
-/* kernel/signal.c */
-asmlinkage int sys_sigaltstack(const stack_t __user *, stack_t __user *,
-                               struct pt_regs *);
-asmlinkage int sys_rt_sigreturn(struct pt_regs *);
-
 /* mm/cache.c */
 asmlinkage int sys_cacheflush(int, void __user *, size_t);
 
diff --git a/arch/avr32/kernel/process.c b/arch/avr32/kernel/process.c
index 9c46aaad11ce..ef5a2a08fcca 100644
--- a/arch/avr32/kernel/process.c
+++ b/arch/avr32/kernel/process.c
@@ -367,14 +367,13 @@ asmlinkage int sys_fork(struct pt_regs *regs)
 }
 
 asmlinkage int sys_clone(unsigned long clone_flags, unsigned long newsp,
-                         unsigned long parent_tidptr,
-                         unsigned long child_tidptr, struct pt_regs *regs)
+                void __user *parent_tidptr, void __user *child_tidptr,
+                struct pt_regs *regs)
 {
         if (!newsp)
                 newsp = regs->sp;
-        return do_fork(clone_flags, newsp, regs, 0,
-                       (int __user *)parent_tidptr,
-                       (int __user *)child_tidptr);
+        return do_fork(clone_flags, newsp, regs, 0, parent_tidptr,
+                        child_tidptr);
 }
 
 asmlinkage int sys_vfork(struct pt_regs *regs)
diff --git a/arch/avr32/kernel/time.c b/arch/avr32/kernel/time.c
index 668ed2817e51..05ad29112ff4 100644
--- a/arch/avr32/kernel/time.c
+++ b/arch/avr32/kernel/time.c
@@ -35,7 +35,6 @@ static struct clocksource counter = {
         .rating         = 50,
         .read           = read_cycle_count,
         .mask           = CLOCKSOURCE_MASK(32),
-        .shift          = 16,
         .flags          = CLOCK_SOURCE_IS_CONTINUOUS,
 };
 
@@ -123,9 +122,7 @@ void __init time_init(void)
 
         /* figure rate for counter */
         counter_hz = clk_get_rate(boot_cpu_data.clk);
-        counter.mult = clocksource_hz2mult(counter_hz, counter.shift);
-
-        ret = clocksource_register(&counter);
+        ret = clocksource_register_hz(&counter, counter_hz);
         if (ret)
                 pr_debug("timer: could not register clocksource: %d\n", ret);
 
diff --git a/arch/avr32/mach-at32ap/pm.c b/arch/avr32/mach-at32ap/pm.c
index f021edfeaab0..32d680eb6f48 100644
--- a/arch/avr32/mach-at32ap/pm.c
+++ b/arch/avr32/mach-at32ap/pm.c
@@ -176,7 +176,7 @@ out:
         return 0;
 }
 
-static struct platform_suspend_ops avr32_pm_ops = {
+static const struct platform_suspend_ops avr32_pm_ops = {
         .valid  = avr32_pm_valid_state,
         .enter  = avr32_pm_enter,
 };
diff --git a/arch/blackfin/mach-bf537/include/mach/defBF534.h b/arch/blackfin/mach-bf537/include/mach/defBF534.h
index 32529a03b266..725bb35f3aaa 100644
--- a/arch/blackfin/mach-bf537/include/mach/defBF534.h
+++ b/arch/blackfin/mach-bf537/include/mach/defBF534.h
@@ -1418,7 +1418,7 @@
 #define SADD_LEN        0x0002  /* Slave Address Length                                                 */
 #define STDVAL          0x0004  /* Slave Transmit Data Valid                                    */
 #define NAK                     0x0008  /* NAK/ACK* Generated At Conclusion Of Transfer */
-#define GEN                     0x0010  /* General Call Adrress Matching Enabled                */
+#define GEN                     0x0010  /* General Call Address Matching Enabled                */
 
 /* TWI_SLAVE_STAT Masks                                                                                                                 */
 #define SDIR            0x0001  /* Slave Transfer Direction (Transmit/Receive*) */
diff --git a/arch/blackfin/mach-common/pm.c b/arch/blackfin/mach-common/pm.c
index 42fa87e8375c..3c648a077e75 100644
--- a/arch/blackfin/mach-common/pm.c
+++ b/arch/blackfin/mach-common/pm.c
@@ -223,7 +223,7 @@ static int bfin_pm_enter(suspend_state_t state)
         return 0;
 }
 
-struct platform_suspend_ops bfin_pm_ops = {
+static const struct platform_suspend_ops bfin_pm_ops = {
         .enter = bfin_pm_enter,
         .valid  = bfin_pm_valid,
 };
diff --git a/arch/cris/arch-v32/lib/nand_init.S b/arch/cris/arch-v32/lib/nand_init.S
index e705f5cce969..d671fed451c9 100644
--- a/arch/cris/arch-v32/lib/nand_init.S
+++ b/arch/cris/arch-v32/lib/nand_init.S
@@ -139,7 +139,7 @@ copy_nand_to_ram:
         lsrq    8, $r4
         move.b  $r4, [$r1]      ; Row address
         lsrq    8, $r4
-        move.b  $r4, [$r1]      ; Row adddress
+        move.b  $r4, [$r1]      ; Row address
         moveq   20, $r4
 2:      bne     2b
         subq    1, $r4
diff --git a/arch/cris/include/asm/etraxgpio.h b/arch/cris/include/asm/etraxgpio.h
index d474818a537e..461c089db765 100644
--- a/arch/cris/include/asm/etraxgpio.h
+++ b/arch/cris/include/asm/etraxgpio.h
@@ -1,5 +1,5 @@
 /*
- * The following devices are accessable using this driver using
+ * The following devices are accessible using this driver using
  * GPIO_MAJOR (120) and a couple of minor numbers.
  *
  * For ETRAX 100LX (CONFIG_ETRAX_ARCH_V10):
diff --git a/arch/cris/kernel/time.c b/arch/cris/kernel/time.c
index b5096430ce1c..4e73092e85c0 100644
--- a/arch/cris/kernel/time.c
+++ b/arch/cris/kernel/time.c
@@ -86,7 +86,7 @@ int set_rtc_mmss(unsigned long nowtime)
                 CMOS_WRITE(real_seconds,RTC_SECONDS);
                 CMOS_WRITE(real_minutes,RTC_MINUTES);
         } else {
-                printk(KERN_WARNING
+                printk_once(KERN_NOTICE
                        "set_rtc_mmss: can't update from %d to %d\n",
                        cmos_minutes, real_minutes);
                 retval = -1;
diff --git a/arch/h8300/Kconfig.debug b/arch/h8300/Kconfig.debug
index ee671c3f2c74..e8d1b236ad8c 100644
--- a/arch/h8300/Kconfig.debug
+++ b/arch/h8300/Kconfig.debug
@@ -48,7 +48,7 @@ config DEFAULT_CMDLINE
           builtin kernel commandline enabled.
 
 config KERNEL_COMMAND
-        string "Buildin commmand string"
+        string "Buildin command string"
         depends on DEFAULT_CMDLINE
         help
           builtin kernel commandline strings.
diff --git a/arch/ia64/include/asm/io.h b/arch/ia64/include/asm/io.h
index cc8335eb3110..e5a6c3530c6c 100644
--- a/arch/ia64/include/asm/io.h
+++ b/arch/ia64/include/asm/io.h
@@ -426,6 +426,11 @@ extern void __iomem * ioremap_nocache (unsigned long offset, unsigned long size)
 extern void iounmap (volatile void __iomem *addr);
 extern void __iomem * early_ioremap (unsigned long phys_addr, unsigned long size);
 extern void early_iounmap (volatile void __iomem *addr, unsigned long size);
+static inline void __iomem * ioremap_cache (unsigned long phys_addr, unsigned long size)
+{
+        return ioremap(phys_addr, size);
+}
+
 
 /*
  * String version of IO memory access ops:
diff --git a/arch/ia64/include/asm/kvm_host.h b/arch/ia64/include/asm/kvm_host.h
index 2f229e5de498..2689ee54a1c9 100644
--- a/arch/ia64/include/asm/kvm_host.h
+++ b/arch/ia64/include/asm/kvm_host.h
@@ -590,6 +590,10 @@ int kvm_emulate_halt(struct kvm_vcpu *vcpu);
 int kvm_pal_emul(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run);
 void kvm_sal_emul(struct kvm_vcpu *vcpu);
 
+#define __KVM_HAVE_ARCH_VM_ALLOC 1
+struct kvm *kvm_arch_alloc_vm(void);
+void kvm_arch_free_vm(struct kvm *kvm);
+
 #endif /* __ASSEMBLY__*/
 
 #endif
diff --git a/arch/ia64/include/asm/processor.h b/arch/ia64/include/asm/processor.h
index 348e44d08ce3..03afe7970748 100644
--- a/arch/ia64/include/asm/processor.h
+++ b/arch/ia64/include/asm/processor.h
@@ -717,8 +717,9 @@ prefetchw (const void *x)
 #define spin_lock_prefetch(x)   prefetchw(x)
 
 extern unsigned long boot_option_idle_override;
-extern unsigned long idle_halt;
-extern unsigned long idle_nomwait;
+
+enum idle_boot_override {IDLE_NO_OVERRIDE=0, IDLE_HALT, IDLE_FORCE_MWAIT,
+                         IDLE_NOMWAIT, IDLE_POLL};
 
 #endif /* !__ASSEMBLY__ */
 
diff --git a/arch/ia64/kernel/acpi.c b/arch/ia64/kernel/acpi.c
index 7b897b7b0ae6..90ebceb899a0 100644
--- a/arch/ia64/kernel/acpi.c
+++ b/arch/ia64/kernel/acpi.c
@@ -479,7 +479,7 @@ acpi_numa_processor_affinity_init(struct acpi_srat_cpu_affinity *pa)
 
         if (srat_num_cpus >= ARRAY_SIZE(node_cpuid)) {
                 printk_once(KERN_WARNING
-                            "node_cpuid[%d] is too small, may not be able to use all cpus\n",
+                            "node_cpuid[%ld] is too small, may not be able to use all cpus\n",
                             ARRAY_SIZE(node_cpuid));
                 return;
         }
diff --git a/arch/ia64/kernel/perfmon.c b/arch/ia64/kernel/perfmon.c
index d92d5b5161fc..89accc626b86 100644
--- a/arch/ia64/kernel/perfmon.c
+++ b/arch/ia64/kernel/perfmon.c
@@ -617,11 +617,14 @@ pfm_get_unmapped_area(struct file *file, unsigned long addr, unsigned long len,
         return get_unmapped_area(file, addr, len, pgoff, flags);
 }
 
+/* forward declaration */
+static const struct dentry_operations pfmfs_dentry_operations;
 
 static struct dentry *
 pfmfs_mount(struct file_system_type *fs_type, int flags, const char *dev_name, void *data)
 {
-        return mount_pseudo(fs_type, "pfm:", NULL, PFMFS_MAGIC);
+        return mount_pseudo(fs_type, "pfm:", NULL, &pfmfs_dentry_operations,
+                        PFMFS_MAGIC);
 }
 
 static struct file_system_type pfm_fs_type = {
@@ -2232,7 +2235,6 @@ pfm_alloc_file(pfm_context_t *ctx)
         }
         path.mnt = mntget(pfmfs_mnt);
 
-        d_set_d_op(path.dentry, &pfmfs_dentry_operations);
         d_add(path.dentry, inode);
 
         file = alloc_file(&path, FMODE_READ, &pfm_file_ops);
diff --git a/arch/ia64/kernel/process.c b/arch/ia64/kernel/process.c
index 16f1c7b04c69..6d33c5cc94f0 100644
--- a/arch/ia64/kernel/process.c
+++ b/arch/ia64/kernel/process.c
@@ -53,12 +53,8 @@
 
 void (*ia64_mark_idle)(int);
 
-unsigned long boot_option_idle_override = 0;
+unsigned long boot_option_idle_override = IDLE_NO_OVERRIDE;
 EXPORT_SYMBOL(boot_option_idle_override);
-unsigned long idle_halt;
-EXPORT_SYMBOL(idle_halt);
-unsigned long idle_nomwait;
-EXPORT_SYMBOL(idle_nomwait);
 void (*pm_idle) (void);
 EXPORT_SYMBOL(pm_idle);
 void (*pm_power_off) (void);
diff --git a/arch/ia64/kvm/kvm-ia64.c b/arch/ia64/kvm/kvm-ia64.c
index f56a6316e134..70d224d4264c 100644
--- a/arch/ia64/kvm/kvm-ia64.c
+++ b/arch/ia64/kvm/kvm-ia64.c
@@ -749,7 +749,7 @@ out:
         return r;
 }
 
-static struct kvm *kvm_alloc_kvm(void)
+struct kvm *kvm_arch_alloc_vm(void)
 {
 
         struct kvm *kvm;
@@ -760,7 +760,7 @@ static struct kvm *kvm_alloc_kvm(void)
         vm_base = __get_free_pages(GFP_KERNEL, get_order(KVM_VM_DATA_SIZE));
 
         if (!vm_base)
-                return ERR_PTR(-ENOMEM);
+                return NULL;
 
         memset((void *)vm_base, 0, KVM_VM_DATA_SIZE);
         kvm = (struct kvm *)(vm_base +
@@ -806,10 +806,12 @@ static void kvm_build_io_pmt(struct kvm *kvm)
 #define GUEST_PHYSICAL_RR4      0x2739
 #define VMM_INIT_RR             0x1660
 
-static void kvm_init_vm(struct kvm *kvm)
+int kvm_arch_init_vm(struct kvm *kvm)
 {
         BUG_ON(!kvm);
 
+        kvm->arch.is_sn2 = ia64_platform_is("sn2");
+
         kvm->arch.metaphysical_rr0 = GUEST_PHYSICAL_RR0;
         kvm->arch.metaphysical_rr4 = GUEST_PHYSICAL_RR4;
         kvm->arch.vmm_init_rr = VMM_INIT_RR;
@@ -823,21 +825,8 @@ static void kvm_init_vm(struct kvm *kvm)
 
         /* Reserve bit 0 of irq_sources_bitmap for userspace irq source */
         set_bit(KVM_USERSPACE_IRQ_SOURCE_ID, &kvm->arch.irq_sources_bitmap);
-}
-
-struct  kvm *kvm_arch_create_vm(void)
-{
-        struct kvm *kvm = kvm_alloc_kvm();
-
-        if (IS_ERR(kvm))
-                return ERR_PTR(-ENOMEM);
-
-        kvm->arch.is_sn2 = ia64_platform_is("sn2");
-
-        kvm_init_vm(kvm);
-
-        return kvm;
 
+        return 0;
 }
 
 static int kvm_vm_ioctl_get_irqchip(struct kvm *kvm,
@@ -962,7 +951,9 @@ long kvm_arch_vm_ioctl(struct file *filp,
                         goto out;
                 r = kvm_setup_default_irq_routing(kvm);
                 if (r) {
+                        mutex_lock(&kvm->slots_lock);
                         kvm_ioapic_destroy(kvm);
+                        mutex_unlock(&kvm->slots_lock);
                         goto out;
                 }
                 break;
@@ -1357,7 +1348,7 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
         return -EINVAL;
 }
 
-static void free_kvm(struct kvm *kvm)
+void kvm_arch_free_vm(struct kvm *kvm)
 {
         unsigned long vm_base = kvm->arch.vm_base;
 
@@ -1399,9 +1390,6 @@ void kvm_arch_destroy_vm(struct kvm *kvm)
 #endif
         kfree(kvm->arch.vioapic);
         kvm_release_vm_pages(kvm);
-        kvm_free_physmem(kvm);
-        cleanup_srcu_struct(&kvm->srcu);
-        free_kvm(kvm);
 }
 
 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
diff --git a/arch/ia64/kvm/mmio.c b/arch/ia64/kvm/mmio.c
index fb8f9f59a1ed..f1e17d3d6cd9 100644
--- a/arch/ia64/kvm/mmio.c
+++ b/arch/ia64/kvm/mmio.c
@@ -130,7 +130,7 @@ static void mmio_access(struct kvm_vcpu *vcpu, u64 src_pa, u64 *dest,
 
         local_irq_save(psr);
 
-        /*Intercept the acces for PIB range*/
+        /*Intercept the access for PIB range*/
         if (iot == GPFN_PIB) {
                 if (!dir)
                         lsapic_write(vcpu, src_pa, s, *dest);
diff --git a/arch/ia64/mm/hugetlbpage.c b/arch/ia64/mm/hugetlbpage.c
index 1841ee7e65f9..5ca674b74737 100644
--- a/arch/ia64/mm/hugetlbpage.c
+++ b/arch/ia64/mm/hugetlbpage.c
@@ -38,7 +38,7 @@ huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz)
         if (pud) {
                 pmd = pmd_alloc(mm, pud, taddr);
                 if (pmd)
-                        pte = pte_alloc_map(mm, pmd, taddr);
+                        pte = pte_alloc_map(mm, NULL, pmd, taddr);
         }
         return pte;
 }
diff --git a/arch/m68k/ifpsp060/src/fpsp.S b/arch/m68k/ifpsp060/src/fpsp.S
index 73613b5f1ee5..26e85e2b7a5e 100644
--- a/arch/m68k/ifpsp060/src/fpsp.S
+++ b/arch/m68k/ifpsp060/src/fpsp.S
@@ -3881,7 +3881,7 @@ _fpsp_fline:
 # FP Unimplemented Instruction stack frame and jump to that entry
 # point.
 #
-# but, if the FPU is disabled, then we need to jump to the FPU diabled
+# but, if the FPU is disabled, then we need to jump to the FPU disabled
 # entry point.
         movc            %pcr,%d0
         btst            &0x1,%d0
diff --git a/arch/m68k/include/asm/m548xgpt.h b/arch/m68k/include/asm/m548xgpt.h
index c8ef158a1c4e..33b2eef90f0a 100644
--- a/arch/m68k/include/asm/m548xgpt.h
+++ b/arch/m68k/include/asm/m548xgpt.h
@@ -59,11 +59,13 @@
 #define MCF_GPT_GMS_GPIO_INPUT     (0x00000000)
 #define MCF_GPT_GMS_GPIO_OUTLO     (0x00000020)
 #define MCF_GPT_GMS_GPIO_OUTHI     (0x00000030)
+#define MCF_GPT_GMS_GPIO_MASK      (0x00000030)
 #define MCF_GPT_GMS_TMS_DISABLE    (0x00000000)
 #define MCF_GPT_GMS_TMS_INCAPT     (0x00000001)
 #define MCF_GPT_GMS_TMS_OUTCAPT    (0x00000002)
 #define MCF_GPT_GMS_TMS_PWM        (0x00000003)
 #define MCF_GPT_GMS_TMS_GPIO       (0x00000004)
+#define MCF_GPT_GMS_TMS_MASK       (0x00000007)
 
 /* Bit definitions and macros for MCF_GPT_GCIR */
 #define MCF_GPT_GCIR_CNT(x)        (((x)&0x0000FFFF)<<0)
diff --git a/arch/m68k/mac/psc.c b/arch/m68k/mac/psc.c
index ba6ccab64018..a4c3eb60706e 100644
--- a/arch/m68k/mac/psc.c
+++ b/arch/m68k/mac/psc.c
@@ -88,7 +88,7 @@ void __init psc_init(void)
 
         /*
          * The PSC is always at the same spot, but using psc
-         * keeps things consisant with the psc_xxxx functions.
+         * keeps things consistent with the psc_xxxx functions.
          */
 
         psc = (void *) PSC_BASE;
diff --git a/arch/mips/alchemy/common/power.c b/arch/mips/alchemy/common/power.c
index e5916a516e58..647e518c90bc 100644
--- a/arch/mips/alchemy/common/power.c
+++ b/arch/mips/alchemy/common/power.c
@@ -130,7 +130,7 @@ static void restore_core_regs(void)
         au_writel(sleep_usb[1], USBD_ENABLE);
         au_sync();
 #else
-        /* enable accces to OTG memory */
+        /* enable access to OTG memory */
         au_writel(au_readl(USB_MSR_BASE + 4) | (1 << 6), USB_MSR_BASE + 4);
         au_sync();
 
diff --git a/arch/mips/alchemy/devboards/pm.c b/arch/mips/alchemy/devboards/pm.c
index 4bbd3133e451..acaf91b5e461 100644
--- a/arch/mips/alchemy/devboards/pm.c
+++ b/arch/mips/alchemy/devboards/pm.c
@@ -110,7 +110,7 @@ static void db1x_pm_end(void)
 
 }
 
-static struct platform_suspend_ops db1x_pm_ops = {
+static const struct platform_suspend_ops db1x_pm_ops = {
         .valid          = suspend_valid_only_mem,
         .begin          = db1x_pm_begin,
         .enter          = db1x_pm_enter,
diff --git a/arch/mips/dec/time.c b/arch/mips/dec/time.c
index 02f505f23c32..ea57f39e6736 100644
--- a/arch/mips/dec/time.c
+++ b/arch/mips/dec/time.c
@@ -104,7 +104,7 @@ int rtc_mips_set_mmss(unsigned long nowtime)
                 CMOS_WRITE(real_seconds, RTC_SECONDS);
                 CMOS_WRITE(real_minutes, RTC_MINUTES);
         } else {
-                printk(KERN_WARNING
+                printk_once(KERN_NOTICE
                        "set_rtc_mmss: can't update from %d to %d\n",
                        cmos_minutes, real_minutes);
                 retval = -1;
diff --git a/arch/mips/include/asm/mach-powertv/ioremap.h b/arch/mips/include/asm/mach-powertv/ioremap.h
index 076f2eeaa575..c86ef094ec37 100644
--- a/arch/mips/include/asm/mach-powertv/ioremap.h
+++ b/arch/mips/include/asm/mach-powertv/ioremap.h
@@ -88,7 +88,7 @@ static inline dma_addr_t _dma_to_phys_offset_raw(dma_addr_t dma)
 }
 
 /* These are not portable and should not be used in drivers. Drivers should
- * be using ioremap() and friends to map physical addreses to virtual
+ * be using ioremap() and friends to map physical addresses to virtual
  * addresses and dma_map*() and friends to map virtual addresses into DMA
  * addresses and back.
  */
diff --git a/arch/mips/include/asm/mc146818-time.h b/arch/mips/include/asm/mc146818-time.h
index 199b45733a95..4a08dbe37db8 100644
--- a/arch/mips/include/asm/mc146818-time.h
+++ b/arch/mips/include/asm/mc146818-time.h
@@ -66,7 +66,7 @@ static inline int mc146818_set_rtc_mmss(unsigned long nowtime)
                 CMOS_WRITE(real_seconds, RTC_SECONDS);
                 CMOS_WRITE(real_minutes, RTC_MINUTES);
         } else {
-                printk(KERN_WARNING
+                printk_once(KERN_NOTICE
                        "set_rtc_mmss: can't update from %d to %d\n",
                        cmos_minutes, real_minutes);
                 retval = -1;
diff --git a/arch/mips/include/asm/mman.h b/arch/mips/include/asm/mman.h
index c892bfb3e2c1..785b4ea4ec3f 100644
--- a/arch/mips/include/asm/mman.h
+++ b/arch/mips/include/asm/mman.h
@@ -77,6 +77,9 @@
 #define MADV_UNMERGEABLE 13             /* KSM may not merge identical pages */
 #define MADV_HWPOISON    100            /* poison a page for testing */
 
+#define MADV_HUGEPAGE   14              /* Worth backing with hugepages */
+#define MADV_NOHUGEPAGE 15              /* Not worth backing with hugepages */
+
 /* compatibility flags */
 #define MAP_FILE        0
 
diff --git a/arch/mips/jz4740/board-qi_lb60.c b/arch/mips/jz4740/board-qi_lb60.c
index 5c0a3575877c..2c0e107966ad 100644
--- a/arch/mips/jz4740/board-qi_lb60.c
+++ b/arch/mips/jz4740/board-qi_lb60.c
@@ -65,7 +65,7 @@ static struct nand_ecclayout qi_lb60_ecclayout_1gb = {
 
 /* Early prototypes of the QI LB60 had only 1GB of NAND.
  * In order to support these devices aswell the partition and ecc layout is
- * initalized depending on the NAND size */
+ * initialized depending on the NAND size */
 static struct mtd_partition qi_lb60_partitions_1gb[] = {
         {
                 .name = "NAND BOOT partition",
@@ -464,7 +464,7 @@ static int __init qi_lb60_board_setup(void)
         board_gpio_setup();
 
         if (qi_lb60_init_platform_devices())
-                panic("Failed to initalize platform devices\n");
+                panic("Failed to initialize platform devices\n");
 
         return 0;
 }
diff --git a/arch/mips/jz4740/gpio.c b/arch/mips/jz4740/gpio.c
index 38f60f35156c..88e6aeda5bf1 100644
--- a/arch/mips/jz4740/gpio.c
+++ b/arch/mips/jz4740/gpio.c
@@ -546,7 +546,7 @@ static int __init jz4740_gpio_init(void)
         for (i = 0; i < ARRAY_SIZE(jz4740_gpio_chips); ++i)
                 jz4740_gpio_chip_init(&jz4740_gpio_chips[i], i);
 
-        printk(KERN_INFO "JZ4740 GPIO initalized\n");
+        printk(KERN_INFO "JZ4740 GPIO initialized\n");
 
         return 0;
 }
diff --git a/arch/mips/jz4740/pm.c b/arch/mips/jz4740/pm.c
index a9994585424d..902d5b50124c 100644
--- a/arch/mips/jz4740/pm.c
+++ b/arch/mips/jz4740/pm.c
@@ -42,7 +42,7 @@ static int jz4740_pm_enter(suspend_state_t state)
         return 0;
 }
 
-static struct platform_suspend_ops jz4740_pm_ops = {
+static const struct platform_suspend_ops jz4740_pm_ops = {
         .valid          = suspend_valid_only_mem,
         .enter          = jz4740_pm_enter,
 };
diff --git a/arch/mips/kernel/module.c b/arch/mips/kernel/module.c
index 6f51dda87fce..d87a72e9fac7 100644
--- a/arch/mips/kernel/module.c
+++ b/arch/mips/kernel/module.c
@@ -46,17 +46,9 @@ static DEFINE_SPINLOCK(dbe_lock);
 void *module_alloc(unsigned long size)
 {
 #ifdef MODULE_START
-        struct vm_struct *area;
-
-        size = PAGE_ALIGN(size);
-        if (!size)
-                return NULL;
-
-        area = __get_vm_area(size, VM_ALLOC, MODULE_START, MODULE_END);
-        if (!area)
-                return NULL;
-
-        return __vmalloc_area(area, GFP_KERNEL, PAGE_KERNEL);
+        return __vmalloc_node_range(size, 1, MODULE_START, MODULE_END,
+                                GFP_KERNEL, PAGE_KERNEL, -1,
+                                __builtin_return_address(0));
 #else
         if (size == 0)
                 return NULL;
diff --git a/arch/mips/loongson/common/pm.c b/arch/mips/loongson/common/pm.c
index 6c1fd9001712..f55e07aee071 100644
--- a/arch/mips/loongson/common/pm.c
+++ b/arch/mips/loongson/common/pm.c
@@ -147,7 +147,7 @@ static int loongson_pm_valid_state(suspend_state_t state)
         }
 }
 
-static struct platform_suspend_ops loongson_pm_ops = {
+static const struct platform_suspend_ops loongson_pm_ops = {
         .valid  = loongson_pm_valid_state,
         .enter  = loongson_pm_enter,
 };
diff --git a/arch/mips/mti-malta/malta-memory.c b/arch/mips/mti-malta/malta-memory.c
index b27419c84919..a96d281f9221 100644
--- a/arch/mips/mti-malta/malta-memory.c
+++ b/arch/mips/mti-malta/malta-memory.c
@@ -43,7 +43,7 @@ static struct prom_pmemblock mdesc[PROM_MAX_PMEMBLOCKS];
 static char *mtypes[3] = {
         "Dont use memory",
         "YAMON PROM memory",
-        "Free memmory",
+        "Free memory",
 };
 #endif
 
diff --git a/arch/mips/pci/pcie-octeon.c b/arch/mips/pci/pcie-octeon.c
index 385f035b24e4..0583c463e5f1 100644
--- a/arch/mips/pci/pcie-octeon.c
+++ b/arch/mips/pci/pcie-octeon.c
@@ -900,7 +900,7 @@ static int cvmx_pcie_rc_initialize(int pcie_port)
         mem_access_subid.s.ror = 0;
         /* Disable Relaxed Ordering for Writes. */
         mem_access_subid.s.row = 0;
-        /* PCIe Adddress Bits <63:34>. */
+        /* PCIe Address Bits <63:34>. */
         mem_access_subid.s.ba = 0;
 
         /*
diff --git a/arch/mips/powertv/memory.c b/arch/mips/powertv/memory.c
index 73880ad29bc2..fb3d29660c42 100644
--- a/arch/mips/powertv/memory.c
+++ b/arch/mips/powertv/memory.c
@@ -57,7 +57,7 @@
 unsigned long ptv_memsize;
 
 /*
- * struct low_mem_reserved - Items in low memmory that are reserved
+ * struct low_mem_reserved - Items in low memory that are reserved
  * @start:      Physical address of item
  * @size:       Size, in bytes, of this item
  * @is_aliased: True if this is RAM aliased from another location. If false,
diff --git a/arch/mips/txx9/generic/pci.c b/arch/mips/txx9/generic/pci.c
index 9a0be810cafa..96e69a00ffc8 100644
--- a/arch/mips/txx9/generic/pci.c
+++ b/arch/mips/txx9/generic/pci.c
@@ -107,7 +107,7 @@ int txx9_pci_mem_high __initdata;
 
 /*
  * allocate pci_controller and resources.
- * mem_base, io_base: physical addresss.  0 for auto assignment.
+ * mem_base, io_base: physical address.  0 for auto assignment.
  * mem_size and io_size means max size on auto assignment.
  * pcic must be &txx9_primary_pcic or NULL.
  */
diff --git a/arch/mn10300/Kconfig b/arch/mn10300/Kconfig
index 41ba38513c89..8ed41cf2b08d 100644
--- a/arch/mn10300/Kconfig
+++ b/arch/mn10300/Kconfig
@@ -203,6 +203,7 @@ endmenu
 config SMP
         bool "Symmetric multi-processing support"
         default y
+        select USE_GENERIC_SMP_HELPERS
         depends on MN10300_PROC_MN2WS0038 || MN10300_PROC_MN2WS0050
         ---help---
           This enables support for systems with more than one CPU. If you have
@@ -226,11 +227,6 @@ config NR_CPUS
         depends on SMP
         default "2"
 
-config USE_GENERIC_SMP_HELPERS
-        bool
-        depends on SMP
-        default y
-
 source "kernel/Kconfig.preempt"
 
 config MN10300_CURRENT_IN_E2
diff --git a/arch/mn10300/kernel/rtc.c b/arch/mn10300/kernel/rtc.c
index e9e20f9a4dd3..48d7058b3295 100644
--- a/arch/mn10300/kernel/rtc.c
+++ b/arch/mn10300/kernel/rtc.c
@@ -89,7 +89,7 @@ static int set_rtc_mmss(unsigned long nowtime)
                 CMOS_WRITE(real_seconds, RTC_SECONDS);
                 CMOS_WRITE(real_minutes, RTC_MINUTES);
         } else {
-                printk(KERN_WARNING
+                printk_once(KERN_NOTICE
                        "set_rtc_mmss: can't update from %d to %d\n",
                        cmos_minutes, real_minutes);
                 retval = -1;
diff --git a/arch/parisc/include/asm/mman.h b/arch/parisc/include/asm/mman.h
index 9749c8afe83a..f5b7bf5fba68 100644
--- a/arch/parisc/include/asm/mman.h
+++ b/arch/parisc/include/asm/mman.h
@@ -59,6 +59,9 @@
 #define MADV_MERGEABLE   65             /* KSM may merge identical pages */
 #define MADV_UNMERGEABLE 66             /* KSM may not merge identical pages */
 
+#define MADV_HUGEPAGE   67              /* Worth backing with hugepages */
+#define MADV_NOHUGEPAGE 68              /* Not worth backing with hugepages */
+
 /* compatibility flags */
 #define MAP_FILE        0
 #define MAP_VARIABLE    0
diff --git a/arch/powerpc/include/asm/8xx_immap.h b/arch/powerpc/include/asm/8xx_immap.h
index 4b0e15206006..6b6dc20b0beb 100644
--- a/arch/powerpc/include/asm/8xx_immap.h
+++ b/arch/powerpc/include/asm/8xx_immap.h
@@ -93,7 +93,7 @@ typedef struct	mem_ctlr {
 } memctl8xx_t;
 
 /*-----------------------------------------------------------------------
- * BR - Memory Controler: Base Register                                 16-9
+ * BR - Memory Controller: Base Register                                        16-9
  */
 #define BR_BA_MSK       0xffff8000      /* Base Address Mask                    */
 #define BR_AT_MSK       0x00007000      /* Address Type Mask                    */
@@ -110,7 +110,7 @@ typedef struct	mem_ctlr {
 #define BR_V            0x00000001      /* Bank Valid                           */
 
 /*-----------------------------------------------------------------------
- * OR - Memory Controler: Option Register                               16-11
+ * OR - Memory Controller: Option Register                              16-11
  */
 #define OR_AM_MSK       0xffff8000      /* Address Mask Mask                    */
 #define OR_ATM_MSK      0x00007000      /* Address Type Mask Mask               */
diff --git a/arch/powerpc/kvm/book3s.c b/arch/powerpc/kvm/book3s.c
index e316847c08c0..badc983031b3 100644
--- a/arch/powerpc/kvm/book3s.c
+++ b/arch/powerpc/kvm/book3s.c
@@ -1307,12 +1307,10 @@ struct kvm_vcpu *kvmppc_core_vcpu_create(struct kvm *kvm, unsigned int id)
         int err = -ENOMEM;
         unsigned long p;
 
-        vcpu_book3s = vmalloc(sizeof(struct kvmppc_vcpu_book3s));
+        vcpu_book3s = vzalloc(sizeof(struct kvmppc_vcpu_book3s));
         if (!vcpu_book3s)
                 goto out;
 
-        memset(vcpu_book3s, 0, sizeof(struct kvmppc_vcpu_book3s));
-
         vcpu_book3s->shadow_vcpu = (struct kvmppc_book3s_shadow_vcpu *)
                 kzalloc(sizeof(*vcpu_book3s->shadow_vcpu), GFP_KERNEL);
         if (!vcpu_book3s->shadow_vcpu)
diff --git a/arch/powerpc/kvm/powerpc.c b/arch/powerpc/kvm/powerpc.c
index 38f756f25053..99758460efde 100644
--- a/arch/powerpc/kvm/powerpc.c
+++ b/arch/powerpc/kvm/powerpc.c
@@ -145,18 +145,12 @@ void kvm_arch_check_processor_compat(void *rtn)
         *(int *)rtn = kvmppc_core_check_processor_compat();
 }
 
-struct kvm *kvm_arch_create_vm(void)
+int kvm_arch_init_vm(struct kvm *kvm)
 {
-        struct kvm *kvm;
-
-        kvm = kzalloc(sizeof(struct kvm), GFP_KERNEL);
-        if (!kvm)
-                return ERR_PTR(-ENOMEM);
-
-        return kvm;
+        return 0;
 }
 
-static void kvmppc_free_vcpus(struct kvm *kvm)
+void kvm_arch_destroy_vm(struct kvm *kvm)
 {
         unsigned int i;
         struct kvm_vcpu *vcpu;
@@ -176,14 +170,6 @@ void kvm_arch_sync_events(struct kvm *kvm)
 {
 }
 
-void kvm_arch_destroy_vm(struct kvm *kvm)
-{
-        kvmppc_free_vcpus(kvm);
-        kvm_free_physmem(kvm);
-        cleanup_srcu_struct(&kvm->srcu);
-        kfree(kvm);
-}
-
 int kvm_dev_ioctl_check_extension(long ext)
 {
         int r;
diff --git a/arch/powerpc/mm/gup.c b/arch/powerpc/mm/gup.c
index d7efdbf640c7..fec13200868f 100644
--- a/arch/powerpc/mm/gup.c
+++ b/arch/powerpc/mm/gup.c
@@ -16,6 +16,16 @@
 
 #ifdef __HAVE_ARCH_PTE_SPECIAL
 
+static inline void get_huge_page_tail(struct page *page)
+{
+        /*
+         * __split_huge_page_refcount() cannot run
+         * from under us.
+         */
+        VM_BUG_ON(atomic_read(&page->_count) < 0);
+        atomic_inc(&page->_count);
+}
+
 /*
  * The performance critical leaf functions are made noinline otherwise gcc
  * inlines everything into a single function which results in too much
@@ -47,6 +57,8 @@ static noinline int gup_pte_range(pmd_t pmd, unsigned long addr,
                         put_page(page);
                         return 0;
                 }
+                if (PageTail(page))
+                        get_huge_page_tail(page);
                 pages[*nr] = page;
                 (*nr)++;
 
diff --git a/arch/powerpc/oprofile/op_model_cell.c b/arch/powerpc/oprofile/op_model_cell.c
index 7fd90d02d8c6..c4d2b7167568 100644
--- a/arch/powerpc/oprofile/op_model_cell.c
+++ b/arch/powerpc/oprofile/op_model_cell.c
@@ -1469,7 +1469,7 @@ static int cell_global_start(struct op_counter_config *ctr)
  * The pm_interval register is setup to write the SPU PC value into the
  * trace buffer at the maximum rate possible.  The trace buffer is configured
  * to store the PCs, wrapping when it is full.  The performance counter is
- * intialized to the max hardware count minus the number of events, N, between
+ * initialized to the max hardware count minus the number of events, N, between
  * samples.  Once the N events have occured, a HW counter overflow occurs
  * causing the generation of a HW counter interrupt which also stops the
  * writing of the SPU PC values to the trace buffer.  Hence the last PC
diff --git a/arch/powerpc/platforms/52xx/lite5200_pm.c b/arch/powerpc/platforms/52xx/lite5200_pm.c
index 80234e5921f5..eda0fc2a3914 100644
--- a/arch/powerpc/platforms/52xx/lite5200_pm.c
+++ b/arch/powerpc/platforms/52xx/lite5200_pm.c
@@ -232,7 +232,7 @@ static void lite5200_pm_end(void)
         lite5200_pm_target_state = PM_SUSPEND_ON;
 }
 
-static struct platform_suspend_ops lite5200_pm_ops = {
+static const struct platform_suspend_ops lite5200_pm_ops = {
         .valid          = lite5200_pm_valid,
         .begin          = lite5200_pm_begin,
         .prepare        = lite5200_pm_prepare,
diff --git a/arch/powerpc/platforms/52xx/mpc52xx_pm.c b/arch/powerpc/platforms/52xx/mpc52xx_pm.c
index 568cef636275..8310e8b5b57f 100644
--- a/arch/powerpc/platforms/52xx/mpc52xx_pm.c
+++ b/arch/powerpc/platforms/52xx/mpc52xx_pm.c
@@ -186,7 +186,7 @@ void mpc52xx_pm_finish(void)
         iounmap(mbar);
 }
 
-static struct platform_suspend_ops mpc52xx_pm_ops = {
+static const struct platform_suspend_ops mpc52xx_pm_ops = {
         .valid          = mpc52xx_pm_valid,
         .prepare        = mpc52xx_pm_prepare,
         .enter          = mpc52xx_pm_enter,
diff --git a/arch/powerpc/platforms/83xx/suspend-asm.S b/arch/powerpc/platforms/83xx/suspend-asm.S
index 1930543c98d3..3d1ecd211776 100644
--- a/arch/powerpc/platforms/83xx/suspend-asm.S
+++ b/arch/powerpc/platforms/83xx/suspend-asm.S
@@ -231,7 +231,7 @@ _GLOBAL(mpc83xx_enter_deep_sleep)
         ori     r4, r4, 0x002a
         mtspr   SPRN_DBAT0L, r4
         lis     r8, TMP_VIRT_IMMR@h
-        ori     r4, r8, 0x001e  /* 1 MByte accessable from Kernel Space only */
+        ori     r4, r8, 0x001e  /* 1 MByte accessible from Kernel Space only */
         mtspr   SPRN_DBAT0U, r4
         isync
 
@@ -241,7 +241,7 @@ _GLOBAL(mpc83xx_enter_deep_sleep)
         ori     r4, r4, 0x002a
         mtspr   SPRN_DBAT1L, r4
         lis     r9, (TMP_VIRT_IMMR + 0x01000000)@h
-        ori     r4, r9, 0x001e  /* 1 MByte accessable from Kernel Space only */
+        ori     r4, r9, 0x001e  /* 1 MByte accessible from Kernel Space only */
         mtspr   SPRN_DBAT1U, r4
         isync
 
@@ -253,7 +253,7 @@ _GLOBAL(mpc83xx_enter_deep_sleep)
         li      r4, 0x0002
         mtspr   SPRN_DBAT2L, r4
         lis     r4, KERNELBASE@h
-        ori     r4, r4, 0x001e  /* 1 MByte accessable from Kernel Space only */
+        ori     r4, r4, 0x001e  /* 1 MByte accessible from Kernel Space only */
         mtspr   SPRN_DBAT2U, r4
         isync
 
diff --git a/arch/powerpc/platforms/83xx/suspend.c b/arch/powerpc/platforms/83xx/suspend.c
index 75ae77f1af6a..fd4f2f2f19e6 100644
--- a/arch/powerpc/platforms/83xx/suspend.c
+++ b/arch/powerpc/platforms/83xx/suspend.c
@@ -311,7 +311,7 @@ static int mpc83xx_is_pci_agent(void)
         return ret;
 }
 
-static struct platform_suspend_ops mpc83xx_suspend_ops = {
+static const struct platform_suspend_ops mpc83xx_suspend_ops = {
         .valid = mpc83xx_suspend_valid,
         .begin = mpc83xx_suspend_begin,
         .enter = mpc83xx_suspend_enter,
diff --git a/arch/powerpc/platforms/85xx/mpc85xx_mds.c b/arch/powerpc/platforms/85xx/mpc85xx_mds.c
index aa34cac4eb5c..747d1ee661fd 100644
--- a/arch/powerpc/platforms/85xx/mpc85xx_mds.c
+++ b/arch/powerpc/platforms/85xx/mpc85xx_mds.c
@@ -309,7 +309,7 @@ static void __init mpc85xx_mds_qe_init(void)
                         /* P1021 has pins muxed for QE and other functions. To
                          * enable QE UEC mode, we need to set bit QE0 for UCC1
                          * in Eth mode, QE0 and QE3 for UCC5 in Eth mode, QE9
-                         * and QE12 for QE MII management singals in PMUXCR
+                         * and QE12 for QE MII management signals in PMUXCR
                          * register.
                          */
                                 setbits32(pmuxcr, MPC85xx_PMUXCR_QE0 |
diff --git a/arch/powerpc/platforms/ps3/device-init.c b/arch/powerpc/platforms/ps3/device-init.c
index b341018326df..6c4b5837fc8a 100644
--- a/arch/powerpc/platforms/ps3/device-init.c
+++ b/arch/powerpc/platforms/ps3/device-init.c
@@ -566,10 +566,10 @@ static int ps3_setup_dynamic_device(const struct ps3_repository_device *repo)
         case PS3_DEV_TYPE_STOR_DISK:
                 result = ps3_setup_storage_dev(repo, PS3_MATCH_ID_STOR_DISK);
 
-                /* Some devices are not accessable from the Other OS lpar. */
+                /* Some devices are not accessible from the Other OS lpar. */
                 if (result == -ENODEV) {
                         result = 0;
-                        pr_debug("%s:%u: not accessable\n", __func__,
+                        pr_debug("%s:%u: not accessible\n", __func__,
                                  __LINE__);
                 }
 
diff --git a/arch/powerpc/platforms/ps3/interrupt.c b/arch/powerpc/platforms/ps3/interrupt.c
index 59d9712d7364..92290ff4761a 100644
--- a/arch/powerpc/platforms/ps3/interrupt.c
+++ b/arch/powerpc/platforms/ps3/interrupt.c
@@ -44,7 +44,7 @@
  * @lock:
  * @ipi_debug_brk_mask:
  *
- * The HV mantains per SMT thread mappings of HV outlet to HV plug on
+ * The HV maintains per SMT thread mappings of HV outlet to HV plug on
  * behalf of the guest.  These mappings are implemented as 256 bit guest
  * supplied bitmaps indexed by plug number.  The addresses of the bitmaps
  * are registered with the HV through lv1_configure_irq_state_bitmap().
diff --git a/arch/powerpc/platforms/pseries/hvCall_inst.c b/arch/powerpc/platforms/pseries/hvCall_inst.c
index e19ff021e711..f106662f4381 100644
--- a/arch/powerpc/platforms/pseries/hvCall_inst.c
+++ b/arch/powerpc/platforms/pseries/hvCall_inst.c
@@ -55,7 +55,7 @@ static void hc_stop(struct seq_file *m, void *p)
 static int hc_show(struct seq_file *m, void *p)
 {
         unsigned long h_num = (unsigned long)p;
-        struct hcall_stats *hs = (struct hcall_stats *)m->private;
+        struct hcall_stats *hs = m->private;
 
         if (hs[h_num].num_calls) {
                 if (cpu_has_feature(CPU_FTR_PURR))
diff --git a/arch/powerpc/platforms/pseries/suspend.c b/arch/powerpc/platforms/pseries/suspend.c
index ed72098bb4e3..a8ca289ff267 100644
--- a/arch/powerpc/platforms/pseries/suspend.c
+++ b/arch/powerpc/platforms/pseries/suspend.c
@@ -153,7 +153,7 @@ static struct sysdev_class suspend_sysdev_class = {
         .name = "power",
 };
 
-static struct platform_suspend_ops pseries_suspend_ops = {
+static const struct platform_suspend_ops pseries_suspend_ops = {
         .valid          = suspend_valid_only_mem,
         .begin          = pseries_suspend_begin,
         .prepare_late   = pseries_prepare_late,
diff --git a/arch/powerpc/sysdev/fsl_pmc.c b/arch/powerpc/sysdev/fsl_pmc.c
index 44de8559c975..e9381bfefb21 100644
--- a/arch/powerpc/sysdev/fsl_pmc.c
+++ b/arch/powerpc/sysdev/fsl_pmc.c
@@ -53,7 +53,7 @@ static int pmc_suspend_valid(suspend_state_t state)
         return 1;
 }
 
-static struct platform_suspend_ops pmc_suspend_ops = {
+static const struct platform_suspend_ops pmc_suspend_ops = {
         .valid = pmc_suspend_valid,
         .enter = pmc_suspend_enter,
 };
diff --git a/arch/powerpc/sysdev/fsl_rio.c b/arch/powerpc/sysdev/fsl_rio.c
index 9725369d432a..9f99bef2adec 100644
--- a/arch/powerpc/sysdev/fsl_rio.c
+++ b/arch/powerpc/sysdev/fsl_rio.c
@@ -973,7 +973,6 @@ fsl_rio_dbell_handler(int irq, void *dev_instance)
         if (dsr & DOORBELL_DSR_QFI) {
                 pr_info("RIO: doorbell queue full\n");
                 out_be32(&priv->msg_regs->dsr, DOORBELL_DSR_QFI);
-                goto out;
         }
 
         /* XXX Need to check/dispatch until queue empty */
diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
index 985d825494f1..bade533ba288 100644
--- a/arch/s390/kvm/kvm-s390.c
+++ b/arch/s390/kvm/kvm-s390.c
@@ -164,24 +164,18 @@ long kvm_arch_vm_ioctl(struct file *filp,
         return r;
 }
 
-struct kvm *kvm_arch_create_vm(void)
+int kvm_arch_init_vm(struct kvm *kvm)
 {
-        struct kvm *kvm;
         int rc;
         char debug_name[16];
 
         rc = s390_enable_sie();
         if (rc)
-                goto out_nokvm;
-
-        rc = -ENOMEM;
-        kvm = kzalloc(sizeof(struct kvm), GFP_KERNEL);
-        if (!kvm)
-                goto out_nokvm;
+                goto out_err;
 
         kvm->arch.sca = (struct sca_block *) get_zeroed_page(GFP_KERNEL);
         if (!kvm->arch.sca)
-                goto out_nosca;
+                goto out_err;
 
         sprintf(debug_name, "kvm-%u", current->pid);
 
@@ -195,13 +189,11 @@ struct kvm *kvm_arch_create_vm(void)
         debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
         VM_EVENT(kvm, 3, "%s", "vm created");
 
-        return kvm;
+        return 0;
 out_nodbf:
         free_page((unsigned long)(kvm->arch.sca));
-out_nosca:
-        kfree(kvm);
-out_nokvm:
-        return ERR_PTR(rc);
+out_err:
+        return rc;
 }
 
 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
@@ -240,11 +232,8 @@ void kvm_arch_sync_events(struct kvm *kvm)
 void kvm_arch_destroy_vm(struct kvm *kvm)
 {
         kvm_free_vcpus(kvm);
-        kvm_free_physmem(kvm);
         free_page((unsigned long)(kvm->arch.sca));
         debug_unregister(kvm->arch.dbf);
-        cleanup_srcu_struct(&kvm->srcu);
-        kfree(kvm);
 }
 
 /* Section: vcpu related */
diff --git a/arch/sh/boards/Kconfig b/arch/sh/boards/Kconfig
index 2018c7ea4c93..d893411022d5 100644
--- a/arch/sh/boards/Kconfig
+++ b/arch/sh/boards/Kconfig
@@ -3,6 +3,9 @@ menu "Board support"
 config SOLUTION_ENGINE
         bool
 
+config SH_ALPHA_BOARD
+        bool
+
 config SH_SOLUTION_ENGINE
         bool "SolutionEngine"
         select SOLUTION_ENGINE
@@ -320,6 +323,21 @@ config SH_SH2007
           Compact Flash socket, two serial ports and PC-104 bus.
           More information at <http://sh2000.sh-linux.org>.
 
+config SH_APSH4A3A
+        bool "AP-SH4A-3A"
+        select SH_ALPHA_BOARD
+        depends on CPU_SUBTYPE_SH7785
+        help
+          Select AP-SH4A-3A if configuring for an ALPHAPROJECT AP-SH4A-3A.
+
+config SH_APSH4AD0A
+        bool "AP-SH4AD-0A"
+        select SH_ALPHA_BOARD
+        select SYS_SUPPORTS_PCI
+        depends on CPU_SUBTYPE_SH7786
+        help
+          Select AP-SH4AD-0A if configuring for an ALPHAPROJECT AP-SH4AD-0A.
+
 endmenu
 
 source "arch/sh/boards/mach-r2d/Kconfig"
diff --git a/arch/sh/boards/Makefile b/arch/sh/boards/Makefile
index be7d11d04b26..975a0f64ff20 100644
--- a/arch/sh/boards/Makefile
+++ b/arch/sh/boards/Makefile
@@ -13,3 +13,5 @@ obj-$(CONFIG_SH_ESPT)		+= board-espt.o
 obj-$(CONFIG_SH_POLARIS)        += board-polaris.o
 obj-$(CONFIG_SH_TITAN)          += board-titan.o
 obj-$(CONFIG_SH_SH7757LCR)      += board-sh7757lcr.o
+obj-$(CONFIG_SH_APSH4A3A)       += board-apsh4a3a.o
+obj-$(CONFIG_SH_APSH4AD0A)      += board-apsh4ad0a.o
diff --git a/arch/sh/boards/board-apsh4a3a.c b/arch/sh/boards/board-apsh4a3a.c
new file mode 100644
index 000000000000..8e2a27057bc9
--- /dev/null
+++ b/arch/sh/boards/board-apsh4a3a.c
@@ -0,0 +1,175 @@
+/*
+ * ALPHAPROJECT AP-SH4A-3A Support.
+ *
+ * Copyright (C) 2010 ALPHAPROJECT Co.,Ltd.
+ * Copyright (C) 2008  Yoshihiro Shimoda
+ * Copyright (C) 2009  Paul Mundt
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License.  See the file "COPYING" in the main directory of this archive
+ * for more details.
+ */
+#include <linux/init.h>
+#include <linux/platform_device.h>
+#include <linux/io.h>
+#include <linux/mtd/physmap.h>
+#include <linux/smsc911x.h>
+#include <linux/irq.h>
+#include <linux/clk.h>
+#include <asm/machvec.h>
+#include <asm/sizes.h>
+#include <asm/clock.h>
+
+static struct mtd_partition nor_flash_partitions[] = {
+        {
+                .name           = "loader",
+                .offset         = 0x00000000,
+                .size           = 512 * 1024,
+        },
+        {
+                .name           = "bootenv",
+                .offset         = MTDPART_OFS_APPEND,
+                .size           = 512 * 1024,
+        },
+        {
+                .name           = "kernel",
+                .offset         = MTDPART_OFS_APPEND,
+                .size           = 4 * 1024 * 1024,
+        },
+        {
+                .name           = "data",
+                .offset         = MTDPART_OFS_APPEND,
+                .size           = MTDPART_SIZ_FULL,
+        },
+};
+
+static struct physmap_flash_data nor_flash_data = {
+        .width          = 4,
+        .parts          = nor_flash_partitions,
+        .nr_parts       = ARRAY_SIZE(nor_flash_partitions),
+};
+
+static struct resource nor_flash_resources[] = {
+        [0]     = {
+                .start  = 0x00000000,
+                .end    = 0x01000000 - 1,
+                .flags  = IORESOURCE_MEM,
+        }
+};
+
+static struct platform_device nor_flash_device = {
+        .name           = "physmap-flash",
+        .dev            = {
+                .platform_data  = &nor_flash_data,
+        },
+        .num_resources  = ARRAY_SIZE(nor_flash_resources),
+        .resource       = nor_flash_resources,
+};
+
+static struct resource smsc911x_resources[] = {
+        [0] = {
+                .name           = "smsc911x-memory",
+                .start          = 0xA4000000,
+                .end            = 0xA4000000 + SZ_256 - 1,
+                .flags          = IORESOURCE_MEM,
+        },
+        [1] = {
+                .name           = "smsc911x-irq",
+                .start          = evt2irq(0x200),
+                .end            = evt2irq(0x200),
+                .flags          = IORESOURCE_IRQ,
+        },
+};
+
+static struct smsc911x_platform_config smsc911x_config = {
+        .irq_polarity   = SMSC911X_IRQ_POLARITY_ACTIVE_LOW,
+        .irq_type       = SMSC911X_IRQ_TYPE_OPEN_DRAIN,
+        .flags          = SMSC911X_USE_16BIT,
+        .phy_interface  = PHY_INTERFACE_MODE_MII,
+};
+
+static struct platform_device smsc911x_device = {
+        .name           = "smsc911x",
+        .id             = -1,
+        .num_resources  = ARRAY_SIZE(smsc911x_resources),
+        .resource       = smsc911x_resources,
+        .dev = {
+                .platform_data = &smsc911x_config,
+        },
+};
+
+static struct platform_device *apsh4a3a_devices[] __initdata = {
+        &nor_flash_device,
+        &smsc911x_device,
+};
+
+static int __init apsh4a3a_devices_setup(void)
+{
+        return platform_add_devices(apsh4a3a_devices,
+                                    ARRAY_SIZE(apsh4a3a_devices));
+}
+device_initcall(apsh4a3a_devices_setup);
+
+static int apsh4a3a_clk_init(void)
+{
+        struct clk *clk;
+        int ret;
+
+        clk = clk_get(NULL, "extal");
+        if (!clk || IS_ERR(clk))
+                return PTR_ERR(clk);
+        ret = clk_set_rate(clk, 33333000);
+        clk_put(clk);
+
+        return ret;
+}
+
+/* Initialize the board */
+static void __init apsh4a3a_setup(char **cmdline_p)
+{
+        printk(KERN_INFO "Alpha Project AP-SH4A-3A support:\n");
+}
+
+static void __init apsh4a3a_init_irq(void)
+{
+        plat_irq_setup_pins(IRQ_MODE_IRQ7654);
+}
+
+/* Return the board specific boot mode pin configuration */
+static int apsh4a3a_mode_pins(void)
+{
+        int value = 0;
+
+        /* These are the factory default settings of SW1 and SW2.
+         * If you change these dip switches then you will need to
+         * adjust the values below as well.
+         */
+        value &= ~MODE_PIN0;  /* Clock Mode 16 */
+        value &= ~MODE_PIN1;
+        value &= ~MODE_PIN2;
+        value &= ~MODE_PIN3;
+        value |=  MODE_PIN4;
+        value &= ~MODE_PIN5;  /* 16-bit Area0 bus width */
+        value |=  MODE_PIN6;  /* Area 0 SRAM interface */
+        value |=  MODE_PIN7;
+        value |=  MODE_PIN8;  /* Little Endian */
+        value |=  MODE_PIN9;  /* Master Mode */
+        value |=  MODE_PIN10; /* Crystal resonator */
+        value |=  MODE_PIN11; /* Display Unit */
+        value |=  MODE_PIN12;
+        value &= ~MODE_PIN13; /* 29-bit address mode */
+        value |=  MODE_PIN14; /* No PLL step-up */
+
+        return value;
+}
+
+/*
+ * The Machine Vector
+ */
+static struct sh_machine_vector mv_apsh4a3a __initmv = {
+        .mv_name                = "AP-SH4A-3A",
+        .mv_setup               = apsh4a3a_setup,
+        .mv_clk_init            = apsh4a3a_clk_init,
+        .mv_init_irq            = apsh4a3a_init_irq,
+        .mv_mode_pins           = apsh4a3a_mode_pins,
+};
diff --git a/arch/sh/boards/board-apsh4ad0a.c b/arch/sh/boards/board-apsh4ad0a.c
new file mode 100644
index 000000000000..e2bd218a054e
--- /dev/null
+++ b/arch/sh/boards/board-apsh4ad0a.c
@@ -0,0 +1,125 @@
+/*
+ * ALPHAPROJECT AP-SH4AD-0A Support.
+ *
+ * Copyright (C) 2010 ALPHAPROJECT Co.,Ltd.
+ * Copyright (C) 2010  Matt Fleming
+ * Copyright (C) 2010  Paul Mundt
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License.  See the file "COPYING" in the main directory of this archive
+ * for more details.
+ */
+#include <linux/init.h>
+#include <linux/platform_device.h>
+#include <linux/io.h>
+#include <linux/smsc911x.h>
+#include <linux/irq.h>
+#include <linux/clk.h>
+#include <asm/machvec.h>
+#include <asm/sizes.h>
+
+static struct resource smsc911x_resources[] = {
+        [0] = {
+                .name           = "smsc911x-memory",
+                .start          = 0xA4000000,
+                .end            = 0xA4000000 + SZ_256 - 1,
+                .flags          = IORESOURCE_MEM,
+        },
+        [1] = {
+                .name           = "smsc911x-irq",
+                .start          = evt2irq(0x200),
+                .end            = evt2irq(0x200),
+                .flags          = IORESOURCE_IRQ,
+        },
+};
+
+static struct smsc911x_platform_config smsc911x_config = {
+        .irq_polarity   = SMSC911X_IRQ_POLARITY_ACTIVE_LOW,
+        .irq_type       = SMSC911X_IRQ_TYPE_OPEN_DRAIN,
+        .flags          = SMSC911X_USE_16BIT,
+        .phy_interface  = PHY_INTERFACE_MODE_MII,
+};
+
+static struct platform_device smsc911x_device = {
+        .name           = "smsc911x",
+        .id             = -1,
+        .num_resources  = ARRAY_SIZE(smsc911x_resources),
+        .resource       = smsc911x_resources,
+        .dev = {
+                .platform_data = &smsc911x_config,
+        },
+};
+
+static struct platform_device *apsh4ad0a_devices[] __initdata = {
+        &smsc911x_device,
+};
+
+static int __init apsh4ad0a_devices_setup(void)
+{
+        return platform_add_devices(apsh4ad0a_devices,
+                                    ARRAY_SIZE(apsh4ad0a_devices));
+}
+device_initcall(apsh4ad0a_devices_setup);
+
+static int apsh4ad0a_mode_pins(void)
+{
+        int value = 0;
+
+        /* These are the factory default settings of SW1 and SW2.
+         * If you change these dip switches then you will need to
+         * adjust the values below as well.
+         */
+        value |=  MODE_PIN0;  /* Clock Mode 3 */
+        value |=  MODE_PIN1;
+        value &= ~MODE_PIN2;
+        value &= ~MODE_PIN3;
+        value &= ~MODE_PIN4;  /* 16-bit Area0 bus width  */
+        value |=  MODE_PIN5;
+        value |=  MODE_PIN6;
+        value |=  MODE_PIN7;  /* Normal mode */
+        value |=  MODE_PIN8;  /* Little Endian */
+        value |=  MODE_PIN9;  /* Crystal resonator */
+        value &= ~MODE_PIN10; /* 29-bit address mode */
+        value &= ~MODE_PIN11; /* PCI-E Root port */
+        value &= ~MODE_PIN12; /* 4 lane + 1 lane */
+        value |=  MODE_PIN13; /* AUD Enable */
+        value &= ~MODE_PIN14; /* Normal Operation */
+
+        return value;
+}
+
+static int apsh4ad0a_clk_init(void)
+{
+        struct clk *clk;
+        int ret;
+
+        clk = clk_get(NULL, "extal");
+        if (!clk || IS_ERR(clk))
+                return PTR_ERR(clk);
+        ret = clk_set_rate(clk, 33333000);
+        clk_put(clk);
+
+        return ret;
+}
+
+/* Initialize the board */
+static void __init apsh4ad0a_setup(char **cmdline_p)
+{
+        pr_info("Alpha Project AP-SH4AD-0A support:\n");
+}
+
+static void __init apsh4ad0a_init_irq(void)
+{
+        plat_irq_setup_pins(IRQ_MODE_IRQ3210);
+}
+
+/*
+ * The Machine Vector
+ */
+static struct sh_machine_vector mv_apsh4ad0a __initmv = {
+        .mv_name                = "AP-SH4AD-0A",
+        .mv_setup               = apsh4ad0a_setup,
+        .mv_mode_pins           = apsh4ad0a_mode_pins,
+        .mv_clk_init            = apsh4ad0a_clk_init,
+        .mv_init_irq            = apsh4ad0a_init_irq,
+};
diff --git a/arch/sh/boards/board-edosk7705.c b/arch/sh/boards/board-edosk7705.c
index 4cb3bb74c36f..541d8a281035 100644
--- a/arch/sh/boards/board-edosk7705.c
+++ b/arch/sh/boards/board-edosk7705.c
@@ -66,7 +66,7 @@ static int __init init_edosk7705_devices(void)
         return platform_add_devices(edosk7705_devices,
                                     ARRAY_SIZE(edosk7705_devices));
 }
-__initcall(init_edosk7705_devices);
+device_initcall(init_edosk7705_devices);
 
 /*
  * The Machine Vector
diff --git a/arch/sh/boards/board-edosk7760.c b/arch/sh/boards/board-edosk7760.c
index 35dc0994875d..f47ac82da876 100644
--- a/arch/sh/boards/board-edosk7760.c
+++ b/arch/sh/boards/board-edosk7760.c
@@ -182,7 +182,7 @@ static int __init init_edosk7760_devices(void)
         return platform_add_devices(edosk7760_devices,
                                     ARRAY_SIZE(edosk7760_devices));
 }
-__initcall(init_edosk7760_devices);
+device_initcall(init_edosk7760_devices);
 
 /*
  * The Machine Vector
diff --git a/arch/sh/boards/board-sh7785lcr.c b/arch/sh/boards/board-sh7785lcr.c
index fe7e686c94ac..ee65ff05c558 100644
--- a/arch/sh/boards/board-sh7785lcr.c
+++ b/arch/sh/boards/board-sh7785lcr.c
@@ -284,7 +284,7 @@ static int __init sh7785lcr_devices_setup(void)
         return platform_add_devices(sh7785lcr_devices,
                                     ARRAY_SIZE(sh7785lcr_devices));
 }
-__initcall(sh7785lcr_devices_setup);
+device_initcall(sh7785lcr_devices_setup);
 
 /* Initialize IRQ setting */
 void __init init_sh7785lcr_IRQ(void)
diff --git a/arch/sh/boards/mach-ap325rxa/setup.c b/arch/sh/boards/mach-ap325rxa/setup.c
index 07ea908c510d..3e5fc3bbf3ed 100644
--- a/arch/sh/boards/mach-ap325rxa/setup.c
+++ b/arch/sh/boards/mach-ap325rxa/setup.c
@@ -14,6 +14,8 @@
 #include <linux/device.h>
 #include <linux/interrupt.h>
 #include <linux/platform_device.h>
+#include <linux/mfd/sh_mobile_sdhi.h>
+#include <linux/mmc/host.h>
 #include <linux/mtd/physmap.h>
 #include <linux/mtd/sh_flctl.h>
 #include <linux/delay.h>
@@ -430,11 +432,18 @@ static struct resource sdhi0_cn3_resources[] = {
         },
 };
 
+static struct sh_mobile_sdhi_info sdhi0_cn3_data = {
+        .tmio_caps      = MMC_CAP_SDIO_IRQ,
+};
+
 static struct platform_device sdhi0_cn3_device = {
         .name           = "sh_mobile_sdhi",
         .id             = 0, /* "sdhi0" clock */
         .num_resources  = ARRAY_SIZE(sdhi0_cn3_resources),
         .resource       = sdhi0_cn3_resources,
+        .dev = {
+                .platform_data = &sdhi0_cn3_data,
+        },
         .archdata = {
                 .hwblk_id = HWBLK_SDHI0,
         },
@@ -453,11 +462,18 @@ static struct resource sdhi1_cn7_resources[] = {
         },
 };
 
+static struct sh_mobile_sdhi_info sdhi1_cn7_data = {
+        .tmio_caps      = MMC_CAP_SDIO_IRQ,
+};
+
 static struct platform_device sdhi1_cn7_device = {
         .name           = "sh_mobile_sdhi",
         .id             = 1, /* "sdhi1" clock */
         .num_resources  = ARRAY_SIZE(sdhi1_cn7_resources),
         .resource       = sdhi1_cn7_resources,
+        .dev = {
+                .platform_data = &sdhi1_cn7_data,
+        },
         .archdata = {
                 .hwblk_id = HWBLK_SDHI1,
         },
diff --git a/arch/sh/boards/mach-cayman/setup.c b/arch/sh/boards/mach-cayman/setup.c
index 7e8216ac31bd..e89e8e122a26 100644
--- a/arch/sh/boards/mach-cayman/setup.c
+++ b/arch/sh/boards/mach-cayman/setup.c
@@ -165,7 +165,7 @@ static int __init smsc_superio_setup(void)
 
         return 0;
 }
-__initcall(smsc_superio_setup);
+device_initcall(smsc_superio_setup);
 
 static void __iomem *cayman_ioport_map(unsigned long port, unsigned int len)
 {
diff --git a/arch/sh/boards/mach-ecovec24/setup.c b/arch/sh/boards/mach-ecovec24/setup.c
index f48c492a68d3..33b662999fc6 100644
--- a/arch/sh/boards/mach-ecovec24/setup.c
+++ b/arch/sh/boards/mach-ecovec24/setup.c
@@ -473,6 +473,7 @@ static struct sh_mobile_sdhi_info sdhi0_info = {
         .dma_slave_tx   = SHDMA_SLAVE_SDHI0_TX,
         .dma_slave_rx   = SHDMA_SLAVE_SDHI0_RX,
         .set_pwr        = sdhi0_set_pwr,
+        .tmio_caps      = MMC_CAP_SDIO_IRQ | MMC_CAP_POWER_OFF_CARD,
 };
 
 static struct resource sdhi0_resources[] = {
@@ -511,6 +512,7 @@ static void sdhi1_set_pwr(struct platform_device *pdev, int state)
 static struct sh_mobile_sdhi_info sdhi1_info = {
         .dma_slave_tx   = SHDMA_SLAVE_SDHI1_TX,
         .dma_slave_rx   = SHDMA_SLAVE_SDHI1_RX,
+        .tmio_caps      = MMC_CAP_SDIO_IRQ | MMC_CAP_POWER_OFF_CARD,
         .set_pwr        = sdhi1_set_pwr,
 };
 
diff --git a/arch/sh/boards/mach-hp6xx/pm.c b/arch/sh/boards/mach-hp6xx/pm.c
index 4499a3749d40..adc9b4bba828 100644
--- a/arch/sh/boards/mach-hp6xx/pm.c
+++ b/arch/sh/boards/mach-hp6xx/pm.c
@@ -143,7 +143,7 @@ static int hp6x0_pm_enter(suspend_state_t state)
         return 0;
 }
 
-static struct platform_suspend_ops hp6x0_pm_ops = {
+static const struct platform_suspend_ops hp6x0_pm_ops = {
         .enter          = hp6x0_pm_enter,
         .valid          = suspend_valid_only_mem,
 };
diff --git a/arch/sh/boards/mach-kfr2r09/setup.c b/arch/sh/boards/mach-kfr2r09/setup.c
index 9b60eaabf8f3..7504daaa85da 100644
--- a/arch/sh/boards/mach-kfr2r09/setup.c
+++ b/arch/sh/boards/mach-kfr2r09/setup.c
@@ -11,6 +11,7 @@
 #include <linux/platform_device.h>
 #include <linux/interrupt.h>
 #include <linux/mfd/sh_mobile_sdhi.h>
+#include <linux/mmc/host.h>
 #include <linux/mfd/tmio.h>
 #include <linux/mtd/physmap.h>
 #include <linux/mtd/onenand.h>
@@ -366,6 +367,7 @@ static struct sh_mobile_sdhi_info sh7724_sdhi0_data = {
         .dma_slave_tx   = SHDMA_SLAVE_SDHI0_TX,
         .dma_slave_rx   = SHDMA_SLAVE_SDHI0_RX,
         .tmio_flags     = TMIO_MMC_WRPROTECT_DISABLE,
+        .tmio_caps      = MMC_CAP_SDIO_IRQ,
 };
 
 static struct platform_device kfr2r09_sh_sdhi0_device = {
diff --git a/arch/sh/boards/mach-landisk/irq.c b/arch/sh/boards/mach-landisk/irq.c
index e79412a40490..c00ace38db3f 100644
--- a/arch/sh/boards/mach-landisk/irq.c
+++ b/arch/sh/boards/mach-landisk/irq.c
@@ -1,9 +1,10 @@
 /*
- * arch/sh/boards/landisk/irq.c
+ * arch/sh/boards/mach-landisk/irq.c
  *
  * I-O DATA Device, Inc. LANDISK Support
  *
  * Copyright (C) 2005-2007 kogiidena
+ * Copyright (C) 2011 Nobuhiro Iwamatsu
  *
  * Copyright (C) 2001  Ian da Silva, Jeremy Siegel
  * Based largely on io_se.c.
@@ -12,44 +13,54 @@
  * License.  See the file "COPYING" in the main directory of this archive
  * for more details.
  */
+
 #include <linux/init.h>
 #include <linux/irq.h>
 #include <linux/interrupt.h>
 #include <linux/io.h>
 #include <mach-landisk/mach/iodata_landisk.h>
 
-static void disable_landisk_irq(struct irq_data *data)
-{
-        unsigned char mask = 0xff ^ (0x01 << (data->irq - 5));
+enum {
+        UNUSED = 0,
 
-        __raw_writeb(__raw_readb(PA_IMASK) & mask, PA_IMASK);
-}
-
-static void enable_landisk_irq(struct irq_data *data)
-{
-        unsigned char value = (0x01 << (data->irq - 5));
+        PCI_INTA, /* PCI int A */
+        PCI_INTB, /* PCI int B */
+        PCI_INTC, /* PCI int C */
+        PCI_INTD, /* PCI int D */
+        ATA,      /* ATA */
+        FATA,     /* CF */
+        POWER,    /* Power swtich */
+        BUTTON,   /* Button swtich */
+};
 
-        __raw_writeb(__raw_readb(PA_IMASK) | value, PA_IMASK);
-}
+/* Vectors for LANDISK */
+static struct intc_vect vectors_landisk[] __initdata = {
+        INTC_IRQ(PCI_INTA, IRQ_PCIINTA),
+        INTC_IRQ(PCI_INTB, IRQ_PCIINTB),
+        INTC_IRQ(PCI_INTC, IRQ_PCIINTC),
+        INTC_IRQ(PCI_INTD, IRQ_PCIINTD),
+        INTC_IRQ(ATA, IRQ_ATA),
+        INTC_IRQ(FATA, IRQ_FATA),
+        INTC_IRQ(POWER, IRQ_POWER),
+        INTC_IRQ(BUTTON, IRQ_BUTTON),
+};
 
-static struct irq_chip landisk_irq_chip __read_mostly = {
-        .name           = "LANDISK",
-        .irq_mask       = disable_landisk_irq,
-        .irq_unmask     = enable_landisk_irq,
+/* IRLMSK mask register layout for LANDISK */
+static struct intc_mask_reg mask_registers_landisk[] __initdata = {
+        { PA_IMASK, 0, 8, /* IRLMSK */
+          {  BUTTON, POWER, FATA, ATA,
+             PCI_INTD, PCI_INTC, PCI_INTB, PCI_INTA,
+          }
+        },
 };
 
+static DECLARE_INTC_DESC(intc_desc_landisk, "landisk", vectors_landisk, NULL,
+                        mask_registers_landisk, NULL, NULL);
 /*
  * Initialize IRQ setting
  */
 void __init init_landisk_IRQ(void)
 {
-        int i;
-
-        for (i = 5; i < 14; i++) {
-                disable_irq_nosync(i);
-                set_irq_chip_and_handler_name(i, &landisk_irq_chip,
-                                              handle_level_irq, "level");
-                enable_landisk_irq(irq_get_irq_data(i));
-        }
+        register_intc_controller(&intc_desc_landisk);
         __raw_writeb(0x00, PA_PWRINT_CLR);
 }
diff --git a/arch/sh/boards/mach-landisk/setup.c b/arch/sh/boards/mach-landisk/setup.c
index 50337acc18c5..94186cf079b6 100644
--- a/arch/sh/boards/mach-landisk/setup.c
+++ b/arch/sh/boards/mach-landisk/setup.c
@@ -21,8 +21,6 @@
 #include <mach-landisk/mach/iodata_landisk.h>
 #include <asm/io.h>
 
-void init_landisk_IRQ(void);
-
 static void landisk_power_off(void)
 {
         __raw_writeb(0x01, PA_SHUTDOWN);
@@ -83,7 +81,7 @@ static int __init landisk_devices_setup(void)
                                     ARRAY_SIZE(landisk_devices));
 }
 
-__initcall(landisk_devices_setup);
+device_initcall(landisk_devices_setup);
 
 static void __init landisk_setup(char **cmdline_p)
 {
diff --git a/arch/sh/boards/mach-migor/setup.c b/arch/sh/boards/mach-migor/setup.c
index c8acfec98695..03a7ffe729d5 100644
--- a/arch/sh/boards/mach-migor/setup.c
+++ b/arch/sh/boards/mach-migor/setup.c
@@ -13,6 +13,7 @@
 #include <linux/input.h>
 #include <linux/input/sh_keysc.h>
 #include <linux/mfd/sh_mobile_sdhi.h>
+#include <linux/mmc/host.h>
 #include <linux/mtd/physmap.h>
 #include <linux/mtd/nand.h>
 #include <linux/i2c.h>
@@ -410,6 +411,7 @@ static struct resource sdhi_cn9_resources[] = {
 static struct sh_mobile_sdhi_info sh7724_sdhi_data = {
         .dma_slave_tx   = SHDMA_SLAVE_SDHI0_TX,
         .dma_slave_rx   = SHDMA_SLAVE_SDHI0_RX,
+        .tmio_caps      = MMC_CAP_SDIO_IRQ,
 };
 
 static struct platform_device sdhi_cn9_device = {
diff --git a/arch/sh/boards/mach-r2d/setup.c b/arch/sh/boards/mach-r2d/setup.c
index b84df6a3a93c..4b98a5251f83 100644
--- a/arch/sh/boards/mach-r2d/setup.c
+++ b/arch/sh/boards/mach-r2d/setup.c
@@ -258,7 +258,7 @@ static int __init rts7751r2d_devices_setup(void)
         return platform_add_devices(rts7751r2d_devices,
                                     ARRAY_SIZE(rts7751r2d_devices));
 }
-__initcall(rts7751r2d_devices_setup);
+device_initcall(rts7751r2d_devices_setup);
 
 static void rts7751r2d_power_off(void)
 {
diff --git a/arch/sh/boards/mach-sdk7786/setup.c b/arch/sh/boards/mach-sdk7786/setup.c
index 75e4ddbbec3e..1521aa75ee3a 100644
--- a/arch/sh/boards/mach-sdk7786/setup.c
+++ b/arch/sh/boards/mach-sdk7786/setup.c
@@ -15,13 +15,13 @@
 #include <linux/i2c.h>
 #include <linux/irq.h>
 #include <linux/clk.h>
+#include <linux/clkdev.h>
 #include <mach/fpga.h>
 #include <mach/irq.h>
 #include <asm/machvec.h>
 #include <asm/heartbeat.h>
 #include <asm/sizes.h>
 #include <asm/clock.h>
-#include <asm/clkdev.h>
 #include <asm/reboot.h>
 #include <asm/smp-ops.h>
 
@@ -135,7 +135,7 @@ static int __init sdk7786_devices_setup(void)
 
         return sdk7786_i2c_setup();
 }
-__initcall(sdk7786_devices_setup);
+device_initcall(sdk7786_devices_setup);
 
 static int sdk7786_mode_pins(void)
 {
diff --git a/arch/sh/boards/mach-se/7206/setup.c b/arch/sh/boards/mach-se/7206/setup.c
index 33039e0dc568..8ab8330e3fd1 100644
--- a/arch/sh/boards/mach-se/7206/setup.c
+++ b/arch/sh/boards/mach-se/7206/setup.c
@@ -77,7 +77,7 @@ static int __init se7206_devices_setup(void)
 {
         return platform_add_devices(se7206_devices, ARRAY_SIZE(se7206_devices));
 }
-__initcall(se7206_devices_setup);
+device_initcall(se7206_devices_setup);
 
 static int se7206_mode_pins(void)
 {
diff --git a/arch/sh/boards/mach-se/7724/setup.c b/arch/sh/boards/mach-se/7724/setup.c
index 527a0cd956b5..527679394a25 100644
--- a/arch/sh/boards/mach-se/7724/setup.c
+++ b/arch/sh/boards/mach-se/7724/setup.c
@@ -15,6 +15,7 @@
 #include <linux/interrupt.h>
 #include <linux/platform_device.h>
 #include <linux/mfd/sh_mobile_sdhi.h>
+#include <linux/mmc/host.h>
 #include <linux/mtd/physmap.h>
 #include <linux/delay.h>
 #include <linux/smc91x.h>
@@ -318,6 +319,10 @@ static struct platform_device fsi_device = {
         },
 };
 
+static struct platform_device fsi_ak4642_device = {
+        .name           = "sh_fsi_a_ak4642",
+};
+
 /* KEYSC in SoC (Needs SW33-2 set to ON) */
 static struct sh_keysc_info keysc_info = {
         .mode = SH_KEYSC_MODE_1,
@@ -467,6 +472,7 @@ static struct resource sdhi0_cn7_resources[] = {
 static struct sh_mobile_sdhi_info sh7724_sdhi0_data = {
         .dma_slave_tx   = SHDMA_SLAVE_SDHI0_TX,
         .dma_slave_rx   = SHDMA_SLAVE_SDHI0_RX,
+        .tmio_caps      = MMC_CAP_SDIO_IRQ,
 };
 
 static struct platform_device sdhi0_cn7_device = {
@@ -498,6 +504,7 @@ static struct resource sdhi1_cn8_resources[] = {
 static struct sh_mobile_sdhi_info sh7724_sdhi1_data = {
         .dma_slave_tx   = SHDMA_SLAVE_SDHI1_TX,
         .dma_slave_rx   = SHDMA_SLAVE_SDHI1_RX,
+        .tmio_caps      = MMC_CAP_SDIO_IRQ,
 };
 
 static struct platform_device sdhi1_cn8_device = {
@@ -590,6 +597,7 @@ static struct platform_device *ms7724se_devices[] __initdata = {
         &sh7724_usb0_host_device,
         &sh7724_usb1_gadget_device,
         &fsi_device,
+        &fsi_ak4642_device,
         &sdhi0_cn7_device,
         &sdhi1_cn8_device,
         &irda_device,
diff --git a/arch/sh/boards/mach-se/7751/setup.c b/arch/sh/boards/mach-se/7751/setup.c
index 9fbc51beb181..4ed60c5e221f 100644
--- a/arch/sh/boards/mach-se/7751/setup.c
+++ b/arch/sh/boards/mach-se/7751/setup.c
@@ -48,7 +48,7 @@ static int __init se7751_devices_setup(void)
 {
         return platform_add_devices(se7751_devices, ARRAY_SIZE(se7751_devices));
 }
-__initcall(se7751_devices_setup);
+device_initcall(se7751_devices_setup);
 
 /*
  * The Machine Vector
diff --git a/arch/sh/boards/mach-sh03/rtc.c b/arch/sh/boards/mach-sh03/rtc.c
index 1b200990500c..f83ac7995d0f 100644
--- a/arch/sh/boards/mach-sh03/rtc.c
+++ b/arch/sh/boards/mach-sh03/rtc.c
@@ -108,7 +108,7 @@ static int set_rtc_mmss(unsigned long nowtime)
                 __raw_writeb(real_minutes % 10, RTC_MIN1);
                 __raw_writeb(real_minutes / 10, RTC_MIN10);
         } else {
-                printk(KERN_WARNING
+                printk_once(KERN_NOTICE
                        "set_rtc_mmss: can't update from %d to %d\n",
                        cmos_minutes, real_minutes);
                 retval = -1;
diff --git a/arch/sh/boards/mach-sh03/setup.c b/arch/sh/boards/mach-sh03/setup.c
index af4a0c012a96..d4f79b2a6514 100644
--- a/arch/sh/boards/mach-sh03/setup.c
+++ b/arch/sh/boards/mach-sh03/setup.c
@@ -96,7 +96,7 @@ static int __init sh03_devices_setup(void)
 
         return platform_add_devices(sh03_devices, ARRAY_SIZE(sh03_devices));
 }
-__initcall(sh03_devices_setup);
+device_initcall(sh03_devices_setup);
 
 static struct sh_machine_vector mv_sh03 __initmv = {
         .mv_name                = "Interface (CTP/PCI-SH03)",
diff --git a/arch/sh/boot/romimage/mmcif-sh7724.c b/arch/sh/boot/romimage/mmcif-sh7724.c
index 14863d7292cb..c84e7831018d 100644
--- a/arch/sh/boot/romimage/mmcif-sh7724.c
+++ b/arch/sh/boot/romimage/mmcif-sh7724.c
@@ -21,9 +21,6 @@
 #define HIZCRC          0xa405015c
 #define DRVCRA          0xa405018a
 
-enum { MMCIF_PROGRESS_ENTER, MMCIF_PROGRESS_INIT,
-       MMCIF_PROGRESS_LOAD, MMCIF_PROGRESS_DONE };
-
 /* SH7724 specific MMCIF loader
  *
  * loads the romImage from an MMC card starting from block 512
@@ -63,7 +60,9 @@ asmlinkage void mmcif_loader(unsigned char *buf, unsigned long no_bytes)
         mmcif_update_progress(MMCIF_PROGRESS_LOAD);
 
         /* load kernel via MMCIF interface */
-        sh_mmcif_boot_slurp(MMCIF_BASE, buf, no_bytes);
+        sh_mmcif_boot_do_read(MMCIF_BASE, 512,
+                              (no_bytes + SH_MMCIF_BBS - 1) / SH_MMCIF_BBS,
+                              buf);
 
         /* disable clock to the MMCIF hardware block */
         __raw_writel(__raw_readl(MSTPCR2) | 0x20000000, MSTPCR2);
diff --git a/arch/sh/configs/apsh4a3a_defconfig b/arch/sh/configs/apsh4a3a_defconfig
new file mode 100644
index 000000000000..6cb327977d13
--- /dev/null
+++ b/arch/sh/configs/apsh4a3a_defconfig
@@ -0,0 +1,102 @@
+CONFIG_EXPERIMENTAL=y
+CONFIG_SYSVIPC=y
+CONFIG_BSD_PROCESS_ACCT=y
+CONFIG_IKCONFIG=y
+CONFIG_IKCONFIG_PROC=y
+CONFIG_LOG_BUF_SHIFT=14
+CONFIG_SYSFS_DEPRECATED=y
+CONFIG_SYSFS_DEPRECATED_V2=y
+CONFIG_BLK_DEV_INITRD=y
+CONFIG_SLAB=y
+CONFIG_PROFILING=y
+CONFIG_MODULES=y
+CONFIG_MODULE_UNLOAD=y
+# CONFIG_BLK_DEV_BSG is not set
+CONFIG_CPU_SUBTYPE_SH7785=y
+CONFIG_MEMORY_START=0x0C000000
+CONFIG_FLATMEM_MANUAL=y
+CONFIG_SH_STORE_QUEUES=y
+CONFIG_SH_APSH4A3A=y
+CONFIG_HIGH_RES_TIMERS=y
+CONFIG_KEXEC=y
+CONFIG_PREEMPT=y
+# CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set
+CONFIG_NET=y
+CONFIG_PACKET=y
+CONFIG_UNIX=y
+CONFIG_INET=y
+CONFIG_IP_ADVANCED_ROUTER=y
+CONFIG_IP_PNP=y
+CONFIG_IP_PNP_DHCP=y
+# CONFIG_INET_LRO is not set
+# CONFIG_IPV6 is not set
+# CONFIG_WIRELESS is not set
+CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug"
+# CONFIG_FW_LOADER is not set
+CONFIG_MTD=y
+CONFIG_MTD_CONCAT=y
+CONFIG_MTD_PARTITIONS=y
+CONFIG_MTD_CHAR=y
+CONFIG_MTD_BLOCK=y
+CONFIG_MTD_CFI=y
+CONFIG_MTD_CFI_AMDSTD=y
+CONFIG_MTD_PHYSMAP=y
+CONFIG_BLK_DEV_RAM=y
+CONFIG_BLK_DEV_RAM_SIZE=16384
+CONFIG_NETDEVICES=y
+CONFIG_NET_ETHERNET=y
+CONFIG_SMSC911X=y
+# CONFIG_NETDEV_1000 is not set
+# CONFIG_NETDEV_10000 is not set
+# CONFIG_WLAN is not set
+# CONFIG_INPUT_MOUSEDEV is not set
+# CONFIG_INPUT_KEYBOARD is not set
+# CONFIG_INPUT_MOUSE is not set
+# CONFIG_SERIO is not set
+CONFIG_VT_HW_CONSOLE_BINDING=y
+CONFIG_SERIAL_SH_SCI=y
+CONFIG_SERIAL_SH_SCI_NR_UARTS=6
+CONFIG_SERIAL_SH_SCI_CONSOLE=y
+CONFIG_HW_RANDOM=y
+# CONFIG_HWMON is not set
+CONFIG_FB=y
+CONFIG_FB_SH7785FB=y
+CONFIG_FRAMEBUFFER_CONSOLE=y
+CONFIG_FONTS=y
+CONFIG_FONT_8x8=y
+CONFIG_FONT_8x16=y
+CONFIG_LOGO=y
+# CONFIG_HID_SUPPORT is not set
+# CONFIG_USB_SUPPORT is not set
+CONFIG_EXT2_FS=y
+CONFIG_EXT3_FS=y
+# CONFIG_EXT3_DEFAULTS_TO_ORDERED is not set
+CONFIG_MSDOS_FS=y
+CONFIG_VFAT_FS=y
+CONFIG_NTFS_FS=y
+CONFIG_NTFS_RW=y
+CONFIG_PROC_KCORE=y
+CONFIG_TMPFS=y
+CONFIG_JFFS2_FS=y
+CONFIG_CRAMFS=y
+CONFIG_NFS_FS=y
+CONFIG_NFS_V3=y
+CONFIG_NFS_V4=y
+CONFIG_CIFS=y
+CONFIG_NLS_DEFAULT="utf8"
+CONFIG_NLS_CODEPAGE_437=y
+CONFIG_NLS_CODEPAGE_932=y
+CONFIG_NLS_ASCII=y
+CONFIG_NLS_ISO8859_1=y
+CONFIG_NLS_UTF8=y
+# CONFIG_ENABLE_WARN_DEPRECATED is not set
+# CONFIG_ENABLE_MUST_CHECK is not set
+CONFIG_DEBUG_FS=y
+CONFIG_DEBUG_KERNEL=y
+# CONFIG_DEBUG_PREEMPT is not set
+# CONFIG_DEBUG_BUGVERBOSE is not set
+CONFIG_DEBUG_INFO=y
+# CONFIG_RCU_CPU_STALL_DETECTOR is not set
+# CONFIG_FTRACE is not set
+# CONFIG_CRYPTO_ANSI_CPRNG is not set
+# CONFIG_CRYPTO_HW is not set
diff --git a/arch/sh/configs/apsh4ad0a_defconfig b/arch/sh/configs/apsh4ad0a_defconfig
new file mode 100644
index 000000000000..e71a531f1e31
--- /dev/null
+++ b/arch/sh/configs/apsh4ad0a_defconfig
@@ -0,0 +1,133 @@
+CONFIG_EXPERIMENTAL=y
+CONFIG_SYSVIPC=y
+CONFIG_POSIX_MQUEUE=y
+CONFIG_BSD_PROCESS_ACCT=y
+CONFIG_RCU_TRACE=y
+CONFIG_IKCONFIG=y
+CONFIG_IKCONFIG_PROC=y
+CONFIG_LOG_BUF_SHIFT=14
+CONFIG_CGROUPS=y
+CONFIG_CGROUP_NS=y
+CONFIG_CGROUP_FREEZER=y
+CONFIG_CGROUP_DEVICE=y
+CONFIG_CGROUP_CPUACCT=y
+CONFIG_RESOURCE_COUNTERS=y
+CONFIG_CGROUP_MEM_RES_CTLR=y
+CONFIG_BLK_CGROUP=y
+CONFIG_NAMESPACES=y
+CONFIG_BLK_DEV_INITRD=y
+CONFIG_KALLSYMS_ALL=y
+# CONFIG_COMPAT_BRK is not set
+CONFIG_SLAB=y
+CONFIG_PROFILING=y
+CONFIG_MODULES=y
+CONFIG_MODULE_UNLOAD=y
+# CONFIG_LBDAF is not set
+# CONFIG_BLK_DEV_BSG is not set
+CONFIG_CFQ_GROUP_IOSCHED=y
+CONFIG_CPU_SUBTYPE_SH7786=y
+CONFIG_MEMORY_SIZE=0x10000000
+CONFIG_HUGETLB_PAGE_SIZE_1MB=y
+CONFIG_MEMORY_HOTPLUG=y
+CONFIG_MEMORY_HOTREMOVE=y
+CONFIG_KSM=y
+CONFIG_SH_STORE_QUEUES=y
+CONFIG_SH_APSH4AD0A=y
+CONFIG_NO_HZ=y
+CONFIG_HIGH_RES_TIMERS=y
+CONFIG_CPU_FREQ=y
+CONFIG_CPU_FREQ_GOV_POWERSAVE=m
+CONFIG_CPU_FREQ_GOV_USERSPACE=m
+CONFIG_CPU_FREQ_GOV_ONDEMAND=m
+CONFIG_CPU_FREQ_GOV_CONSERVATIVE=m
+CONFIG_SH_CPU_FREQ=y
+CONFIG_KEXEC=y
+CONFIG_SECCOMP=y
+CONFIG_PREEMPT=y
+# CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set
+CONFIG_BINFMT_MISC=y
+CONFIG_PM=y
+CONFIG_PM_DEBUG=y
+CONFIG_PM_VERBOSE=y
+CONFIG_PM_RUNTIME=y
+CONFIG_CPU_IDLE=y
+CONFIG_NET=y
+CONFIG_PACKET=y
+CONFIG_UNIX=y
+CONFIG_NET_KEY=y
+CONFIG_INET=y
+# CONFIG_INET_LRO is not set
+# CONFIG_IPV6 is not set
+# CONFIG_WIRELESS is not set
+CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug"
+# CONFIG_FW_LOADER is not set
+CONFIG_MTD=y
+CONFIG_MTD_CFI=y
+CONFIG_BLK_DEV_RAM=y
+CONFIG_BLK_DEV_RAM_SIZE=16384
+CONFIG_SCSI=y
+CONFIG_BLK_DEV_SD=y
+CONFIG_SCSI_MULTI_LUN=y
+# CONFIG_SCSI_LOWLEVEL is not set
+CONFIG_NETDEVICES=y
+CONFIG_MDIO_BITBANG=y
+CONFIG_NET_ETHERNET=y
+CONFIG_SMSC911X=y
+# CONFIG_NETDEV_1000 is not set
+# CONFIG_NETDEV_10000 is not set
+# CONFIG_WLAN is not set
+CONFIG_INPUT_EVDEV=y
+# CONFIG_INPUT_KEYBOARD is not set
+# CONFIG_INPUT_MOUSE is not set
+# CONFIG_SERIO is not set
+CONFIG_SERIAL_SH_SCI=y
+CONFIG_SERIAL_SH_SCI_NR_UARTS=6
+CONFIG_SERIAL_SH_SCI_CONSOLE=y
+# CONFIG_LEGACY_PTYS is not set
+# CONFIG_HW_RANDOM is not set
+# CONFIG_HWMON is not set
+CONFIG_VIDEO_OUTPUT_CONTROL=y
+CONFIG_FB=y
+CONFIG_FB_SH7785FB=y
+CONFIG_FRAMEBUFFER_CONSOLE=y
+CONFIG_FONTS=y
+CONFIG_FONT_8x8=y
+CONFIG_FONT_8x16=y
+CONFIG_LOGO=y
+CONFIG_USB=y
+CONFIG_USB_DEBUG=y
+CONFIG_USB_MON=y
+CONFIG_USB_OHCI_HCD=y
+CONFIG_USB_STORAGE=y
+CONFIG_EXT2_FS=y
+CONFIG_EXT3_FS=y
+# CONFIG_EXT3_DEFAULTS_TO_ORDERED is not set
+CONFIG_MSDOS_FS=y
+CONFIG_VFAT_FS=y
+CONFIG_NTFS_FS=y
+CONFIG_NTFS_RW=y
+CONFIG_PROC_KCORE=y
+CONFIG_TMPFS=y
+CONFIG_HUGETLBFS=y
+CONFIG_JFFS2_FS=y
+CONFIG_CRAMFS=y
+CONFIG_NFS_FS=y
+CONFIG_NFS_V3=y
+CONFIG_NFS_V4=y
+CONFIG_CIFS=y
+CONFIG_NLS_DEFAULT="utf8"
+CONFIG_NLS_CODEPAGE_437=y
+CONFIG_NLS_CODEPAGE_932=y
+CONFIG_NLS_ASCII=y
+CONFIG_NLS_ISO8859_1=y
+CONFIG_NLS_UTF8=y
+# CONFIG_ENABLE_MUST_CHECK is not set
+CONFIG_MAGIC_SYSRQ=y
+CONFIG_DEBUG_KERNEL=y
+CONFIG_DEBUG_SHIRQ=y
+CONFIG_DETECT_HUNG_TASK=y
+CONFIG_DEBUG_INFO=y
+CONFIG_DEBUG_VM=y
+# CONFIG_RCU_CPU_STALL_DETECTOR is not set
+CONFIG_DWARF_UNWINDER=y
+# CONFIG_CRYPTO_ANSI_CPRNG is not set
diff --git a/arch/sh/configs/sh7757lcr_defconfig b/arch/sh/configs/sh7757lcr_defconfig
index 273f3fa198f7..5f7f667b9f3b 100644
--- a/arch/sh/configs/sh7757lcr_defconfig
+++ b/arch/sh/configs/sh7757lcr_defconfig
@@ -39,21 +39,15 @@ CONFIG_IPV6=y
 CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug"
 # CONFIG_FW_LOADER is not set
 CONFIG_BLK_DEV_RAM=y
-# CONFIG_MISC_DEVICES is not set
 CONFIG_NETDEVICES=y
-CONFIG_PHYLIB=y
 CONFIG_VITESSE_PHY=y
-CONFIG_MDIO_BITBANG=y
 CONFIG_NET_ETHERNET=y
-CONFIG_MII=y
+CONFIG_SH_ETH=y
 # CONFIG_NETDEV_10000 is not set
 # CONFIG_WLAN is not set
 # CONFIG_KEYBOARD_ATKBD is not set
 # CONFIG_MOUSE_PS2 is not set
 # CONFIG_SERIO is not set
-CONFIG_SERIAL_8250=y
-CONFIG_SERIAL_8250_CONSOLE=y
-CONFIG_SERIAL_8250_NR_UARTS=2
 CONFIG_SERIAL_SH_SCI=y
 CONFIG_SERIAL_SH_SCI_NR_UARTS=3
 CONFIG_SERIAL_SH_SCI_CONSOLE=y
@@ -63,7 +57,6 @@ CONFIG_SERIAL_SH_SCI_CONSOLE=y
 # CONFIG_USB_SUPPORT is not set
 CONFIG_EXT2_FS=y
 CONFIG_EXT3_FS=y
-CONFIG_INOTIFY=y
 CONFIG_ISO9660_FS=y
 CONFIG_VFAT_FS=y
 CONFIG_PROC_KCORE=y
@@ -76,10 +69,8 @@ CONFIG_NLS_CODEPAGE_437=y
 CONFIG_NLS_CODEPAGE_932=y
 CONFIG_NLS_ISO8859_1=y
 CONFIG_DEBUG_KERNEL=y
-# CONFIG_DETECT_SOFTLOCKUP is not set
 # CONFIG_SCHED_DEBUG is not set
 # CONFIG_DEBUG_BUGVERBOSE is not set
 CONFIG_DEBUG_INFO=y
-# CONFIG_RCU_CPU_STALL_DETECTOR is not set
 # CONFIG_FTRACE is not set
 # CONFIG_CRYPTO_ANSI_CPRNG is not set
diff --git a/arch/sh/drivers/pci/fixups-landisk.c b/arch/sh/drivers/pci/fixups-landisk.c
index bb1a6bb5149e..95c6e2d94a0a 100644
--- a/arch/sh/drivers/pci/fixups-landisk.c
+++ b/arch/sh/drivers/pci/fixups-landisk.c
@@ -1,9 +1,10 @@
 /*
- * arch/sh/drivers/pci/ops-landisk.c
+ * arch/sh/drivers/pci/fixups-landisk.c
  *
  * PCI initialization for the I-O DATA Device, Inc. LANDISK board
  *
  * Copyright (C) 2006 kogiidena
+ * Copyright (C) 2010 Nobuhiro Iwamatsu
  *
  * May be copied or modified under the terms of the GNU General Public
  * License.  See linux/COPYING for more information.
@@ -15,6 +16,9 @@
 #include <linux/pci.h>
 #include "pci-sh4.h"
 
+#define PCIMCR_MRSET_OFF        0xBFFFFFFF
+#define PCIMCR_RFSH_OFF         0xFFFFFFFB
+
 int pcibios_map_platform_irq(struct pci_dev *pdev, u8 slot, u8 pin)
 {
         /*
@@ -26,9 +30,29 @@ int pcibios_map_platform_irq(struct pci_dev *pdev, u8 slot, u8 pin)
         int irq = ((slot + pin - 1) & 0x3) + 5;
 
         if ((slot | (pin - 1)) > 0x3) {
-                printk("PCI: Bad IRQ mapping request for slot %d pin %c\n",
+                printk(KERN_WARNING "PCI: Bad IRQ mapping request for slot %d pin %c\n",
                        slot, pin - 1 + 'A');
                 return -1;
         }
         return irq;
 }
+
+int pci_fixup_pcic(struct pci_channel *chan)
+{
+        unsigned long bcr1, mcr;
+
+        bcr1 = __raw_readl(SH7751_BCR1);
+        bcr1 |= 0x40080000;     /* Enable Bit 19 BREQEN, set PCIC to slave */
+        pci_write_reg(chan, bcr1, SH4_PCIBCR1);
+
+        mcr = __raw_readl(SH7751_MCR);
+        mcr = (mcr & PCIMCR_MRSET_OFF) & PCIMCR_RFSH_OFF;
+        pci_write_reg(chan, mcr, SH4_PCIMCR);
+
+        pci_write_reg(chan, 0x0c000000, SH7751_PCICONF5);
+        pci_write_reg(chan, 0xd0000000, SH7751_PCICONF6);
+        pci_write_reg(chan, 0x0c000000, SH4_PCILAR0);
+        pci_write_reg(chan, 0x00000000, SH4_PCILAR1);
+
+        return 0;
+}
diff --git a/arch/sh/include/asm/io.h b/arch/sh/include/asm/io.h
index 89ab2c57a4c2..28c5aa58bb45 100644
--- a/arch/sh/include/asm/io.h
+++ b/arch/sh/include/asm/io.h
@@ -11,11 +11,6 @@
  *
  * While read{b,w,l,q} and write{b,w,l,q} contain memory barriers
  * automatically, there are also __raw versions, which do not.
- *
- * Historically, we have also had ctrl_in{b,w,l,q}/ctrl_out{b,w,l,q} for
- * SuperH specific I/O (raw I/O to on-chip CPU peripherals). In practice
- * these have the same semantics as the __raw variants, and as such, all
- * new code should be using the __raw versions.
  */
 #include <linux/errno.h>
 #include <asm/cache.h>
@@ -231,52 +226,6 @@ __BUILD_IOPORT_STRING(q, u64)
 
 #endif
 
-/*
- * Legacy SuperH on-chip I/O functions
- *
- * These are all deprecated, all new (and especially cross-platform) code
- * should be using the __raw_xxx() routines directly.
- */
-static inline u8 __deprecated ctrl_inb(unsigned long addr)
-{
-        return __raw_readb(addr);
-}
-
-static inline u16 __deprecated ctrl_inw(unsigned long addr)
-{
-        return __raw_readw(addr);
-}
-
-static inline u32 __deprecated ctrl_inl(unsigned long addr)
-{
-        return __raw_readl(addr);
-}
-
-static inline u64 __deprecated ctrl_inq(unsigned long addr)
-{
-        return __raw_readq(addr);
-}
-
-static inline void __deprecated ctrl_outb(u8 v, unsigned long addr)
-{
-        __raw_writeb(v, addr);
-}
-
-static inline void __deprecated ctrl_outw(u16 v, unsigned long addr)
-{
-        __raw_writew(v, addr);
-}
-
-static inline void __deprecated ctrl_outl(u32 v, unsigned long addr)
-{
-        __raw_writel(v, addr);
-}
-
-static inline void __deprecated ctrl_outq(u64 v, unsigned long addr)
-{
-        __raw_writeq(v, addr);
-}
-
 #define IO_SPACE_LIMIT 0xffffffff
 
 /* synco on SH-4A, otherwise a nop */
@@ -341,7 +290,15 @@ __ioremap_29bit(phys_addr_t offset, unsigned long size, pgprot_t prot)
          * mapping must be done by the PMB or by using page tables.
          */
         if (likely(PXSEG(offset) < P3SEG && PXSEG(last_addr) < P3SEG)) {
-                if (unlikely(pgprot_val(prot) & _PAGE_CACHABLE))
+                u64 flags = pgprot_val(prot);
+
+                /*
+                 * Anything using the legacy PTEA space attributes needs
+                 * to be kicked down to page table mappings.
+                 */
+                if (unlikely(flags & _PAGE_PCC_MASK))
+                        return NULL;
+                if (unlikely(flags & _PAGE_CACHABLE))
                         return (void __iomem *)P1SEGADDR(offset);
 
                 return (void __iomem *)P2SEGADDR(offset);
diff --git a/arch/sh/include/asm/machvec.h b/arch/sh/include/asm/machvec.h
index dd5d6e5bf204..57c5c3d0f39f 100644
--- a/arch/sh/include/asm/machvec.h
+++ b/arch/sh/include/asm/machvec.h
@@ -31,6 +31,7 @@ struct sh_machine_vector {
         int (*mv_mode_pins)(void);
 
         void (*mv_mem_init)(void);
+        void (*mv_mem_reserve)(void);
 };
 
 extern struct sh_machine_vector sh_mv;
diff --git a/arch/sh/include/asm/pgtable_32.h b/arch/sh/include/asm/pgtable_32.h
index 43528ec656ba..b799fe71114c 100644
--- a/arch/sh/include/asm/pgtable_32.h
+++ b/arch/sh/include/asm/pgtable_32.h
@@ -76,6 +76,10 @@
 /* Wrapper for extended mode pgprot twiddling */
 #define _PAGE_EXT(x)            ((unsigned long long)(x) << 32)
 
+#ifdef CONFIG_X2TLB
+#define _PAGE_PCC_MASK  0x00000000      /* No legacy PTEA support */
+#else
+
 /* software: moves to PTEA.TC (Timing Control) */
 #define _PAGE_PCC_AREA5 0x00000000      /* use BSC registers for area5 */
 #define _PAGE_PCC_AREA6 0x80000000      /* use BSC registers for area6 */
@@ -89,7 +93,8 @@
 #define _PAGE_PCC_ATR8  0x60000000      /* Attribute Memory space, 8 bit bus */
 #define _PAGE_PCC_ATR16 0x60000001      /* Attribute Memory space, 6 bit bus */
 
-#ifndef CONFIG_X2TLB
+#define _PAGE_PCC_MASK  0xe0000001
+
 /* copy the ptea attributes */
 static inline unsigned long copy_ptea_attributes(unsigned long x)
 {
@@ -231,13 +236,7 @@ static inline unsigned long copy_ptea_attributes(unsigned long x)
                                            _PAGE_EXT_KERN_EXEC))
 
 #define PAGE_KERNEL_PCC(slot, type) \
-                        __pgprot(_PAGE_PRESENT | _PAGE_DIRTY | \
-                                 _PAGE_ACCESSED | _PAGE_FLAGS_HARD | \
-                                 _PAGE_EXT(_PAGE_EXT_KERN_READ | \
-                                           _PAGE_EXT_KERN_WRITE | \
-                                           _PAGE_EXT_KERN_EXEC) \
-                                 (slot ? _PAGE_PCC_AREA5 : _PAGE_PCC_AREA6) | \
-                                 (type))
+                        __pgprot(0)
 
 #elif defined(CONFIG_MMU) /* SH-X TLB */
 #define PAGE_NONE       __pgprot(_PAGE_PROTNONE | _PAGE_CACHABLE | \
diff --git a/arch/sh/include/asm/processor.h b/arch/sh/include/asm/processor.h
index c9e7cbc4768a..9c7bdfcaebbd 100644
--- a/arch/sh/include/asm/processor.h
+++ b/arch/sh/include/asm/processor.h
@@ -35,7 +35,7 @@ enum cpu_type {
         CPU_SH7723, CPU_SH7724, CPU_SH7757, CPU_SHX3,
 
         /* SH4AL-DSP types */
-        CPU_SH7343, CPU_SH7722, CPU_SH7366,
+        CPU_SH7343, CPU_SH7722, CPU_SH7366, CPU_SH7372,
 
         /* SH-5 types */
         CPU_SH5_101, CPU_SH5_103,
diff --git a/arch/sh/include/asm/processor_32.h b/arch/sh/include/asm/processor_32.h
index e3c73cdd8c90..900f8d72ffe2 100644
--- a/arch/sh/include/asm/processor_32.h
+++ b/arch/sh/include/asm/processor_32.h
@@ -194,15 +194,17 @@ extern unsigned long get_wchan(struct task_struct *p);
 #define KSTK_ESP(tsk)  (task_pt_regs(tsk)->regs[15])
 
 #if defined(CONFIG_CPU_SH2A) || defined(CONFIG_CPU_SH4)
+
 #define PREFETCH_STRIDE         L1_CACHE_BYTES
 #define ARCH_HAS_PREFETCH
 #define ARCH_HAS_PREFETCHW
-static inline void prefetch(void *x)
+
+static inline void prefetch(const void *x)
 {
         __builtin_prefetch(x, 0, 3);
 }
 
-static inline void prefetchw(void *x)
+static inline void prefetchw(const void *x)
 {
         __builtin_prefetch(x, 1, 3);
 }
diff --git a/arch/sh/include/mach-landisk/mach/iodata_landisk.h b/arch/sh/include/mach-landisk/mach/iodata_landisk.h
index 6fb04ab38b9f..f432773a9571 100644
--- a/arch/sh/include/mach-landisk/mach/iodata_landisk.h
+++ b/arch/sh/include/mach-landisk/mach/iodata_landisk.h
@@ -2,7 +2,7 @@
 #define __ASM_SH_IODATA_LANDISK_H
 
 /*
- * linux/include/asm-sh/landisk/iodata_landisk.h
+ * arch/sh/include/mach-landisk/mach/iodata_landisk.h
  *
  * Copyright (C) 2000  Atom Create Engineering Co., Ltd.
  *
@@ -27,7 +27,7 @@
 
 #define IRQ_PCIINTA     5               /* PCI INTA IRQ */
 #define IRQ_PCIINTB     6               /* PCI INTB IRQ */
-#define IRQ_PCIINDC     7               /* PCI INTC IRQ */
+#define IRQ_PCIINTC     7               /* PCI INTC IRQ */
 #define IRQ_PCIINTD     8               /* PCI INTD IRQ */
 #define IRQ_ATA         9               /* ATA IRQ */
 #define IRQ_FATA        10              /* FATA IRQ */
@@ -35,6 +35,8 @@
 #define IRQ_BUTTON      12              /* USL-5P Button IRQ */
 #define IRQ_FAULT       13              /* USL-5P Fault  IRQ */
 
+void init_landisk_IRQ(void);
+
 #define __IO_PREFIX landisk
 #include <asm/io_generic.h>
 
diff --git a/arch/sh/kernel/cpu/proc.c b/arch/sh/kernel/cpu/proc.c
index e80a936f409a..f47be8727b3b 100644
--- a/arch/sh/kernel/cpu/proc.c
+++ b/arch/sh/kernel/cpu/proc.c
@@ -25,7 +25,7 @@ static const char *cpu_name[] = {
         [CPU_SH5_101]   = "SH5-101",    [CPU_SH5_103]   = "SH5-103",
         [CPU_MXG]       = "MX-G",       [CPU_SH7723]    = "SH7723",
         [CPU_SH7366]    = "SH7366",     [CPU_SH7724]    = "SH7724",
-        [CPU_SH_NONE]   = "Unknown"
+        [CPU_SH7372]    = "SH7372",     [CPU_SH_NONE]   = "Unknown"
 };
 
 const char *get_cpu_subtype(struct sh_cpuinfo *c)
diff --git a/arch/sh/kernel/cpu/sh2/setup-sh7619.c b/arch/sh/kernel/cpu/sh2/setup-sh7619.c
index c3638516bffc..0f8befccf9fa 100644
--- a/arch/sh/kernel/cpu/sh2/setup-sh7619.c
+++ b/arch/sh/kernel/cpu/sh2/setup-sh7619.c
@@ -62,6 +62,8 @@ static DECLARE_INTC_DESC(intc_desc, "sh7619", vectors, NULL,
 static struct plat_sci_port scif0_platform_data = {
         .mapbase        = 0xf8400000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 88, 88, 88, 88 },
 };
@@ -77,6 +79,8 @@ static struct platform_device scif0_device = {
 static struct plat_sci_port scif1_platform_data = {
         .mapbase        = 0xf8410000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 92, 92, 92, 92 },
 };
@@ -92,6 +96,8 @@ static struct platform_device scif1_device = {
 static struct plat_sci_port scif2_platform_data = {
         .mapbase        = 0xf8420000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 96, 96, 96, 96 },
 };
diff --git a/arch/sh/kernel/cpu/sh2a/setup-mxg.c b/arch/sh/kernel/cpu/sh2a/setup-mxg.c
index 6c96ea02bf8d..949bf2bac28c 100644
--- a/arch/sh/kernel/cpu/sh2a/setup-mxg.c
+++ b/arch/sh/kernel/cpu/sh2a/setup-mxg.c
@@ -201,6 +201,8 @@ static struct platform_device mtu2_2_device = {
 static struct plat_sci_port scif0_platform_data = {
         .mapbase        = 0xff804000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 220, 220, 220, 220 },
 };
diff --git a/arch/sh/kernel/cpu/sh2a/setup-sh7201.c b/arch/sh/kernel/cpu/sh2a/setup-sh7201.c
index d08bf4c07d60..9df558dcdb86 100644
--- a/arch/sh/kernel/cpu/sh2a/setup-sh7201.c
+++ b/arch/sh/kernel/cpu/sh2a/setup-sh7201.c
@@ -180,6 +180,8 @@ static DECLARE_INTC_DESC(intc_desc, "sh7201", vectors, groups,
 static struct plat_sci_port scif0_platform_data = {
         .mapbase        = 0xfffe8000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 180, 180, 180, 180 }
 };
@@ -195,6 +197,8 @@ static struct platform_device scif0_device = {
 static struct plat_sci_port scif1_platform_data = {
         .mapbase        = 0xfffe8800,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 184, 184, 184, 184 }
 };
@@ -210,6 +214,8 @@ static struct platform_device scif1_device = {
 static struct plat_sci_port scif2_platform_data = {
         .mapbase        = 0xfffe9000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 188, 188, 188, 188 }
 };
@@ -225,6 +231,8 @@ static struct platform_device scif2_device = {
 static struct plat_sci_port scif3_platform_data = {
         .mapbase        = 0xfffe9800,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 192, 192, 192, 192 }
 };
@@ -240,6 +248,8 @@ static struct platform_device scif3_device = {
 static struct plat_sci_port scif4_platform_data = {
         .mapbase        = 0xfffea000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 196, 196, 196, 196 }
 };
@@ -255,6 +265,8 @@ static struct platform_device scif4_device = {
 static struct plat_sci_port scif5_platform_data = {
         .mapbase        = 0xfffea800,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 200, 200, 200, 200 }
 };
@@ -270,6 +282,8 @@ static struct platform_device scif5_device = {
 static struct plat_sci_port scif6_platform_data = {
         .mapbase        = 0xfffeb000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 204, 204, 204, 204 }
 };
@@ -285,6 +299,8 @@ static struct platform_device scif6_device = {
 static struct plat_sci_port scif7_platform_data = {
         .mapbase        = 0xfffeb800,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 208, 208, 208, 208 }
 };
diff --git a/arch/sh/kernel/cpu/sh2a/setup-sh7203.c b/arch/sh/kernel/cpu/sh2a/setup-sh7203.c
index 832f401b5860..a43124e608c3 100644
--- a/arch/sh/kernel/cpu/sh2a/setup-sh7203.c
+++ b/arch/sh/kernel/cpu/sh2a/setup-sh7203.c
@@ -176,6 +176,8 @@ static DECLARE_INTC_DESC(intc_desc, "sh7203", vectors, groups,
 static struct plat_sci_port scif0_platform_data = {
         .mapbase        = 0xfffe8000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           =  { 192, 192, 192, 192 },
 };
@@ -191,6 +193,8 @@ static struct platform_device scif0_device = {
 static struct plat_sci_port scif1_platform_data = {
         .mapbase        = 0xfffe8800,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           =  { 196, 196, 196, 196 },
 };
@@ -206,6 +210,8 @@ static struct platform_device scif1_device = {
 static struct plat_sci_port scif2_platform_data = {
         .mapbase        = 0xfffe9000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           =  { 200, 200, 200, 200 },
 };
@@ -221,6 +227,8 @@ static struct platform_device scif2_device = {
 static struct plat_sci_port scif3_platform_data = {
         .mapbase        = 0xfffe9800,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           =  { 204, 204, 204, 204 },
 };
diff --git a/arch/sh/kernel/cpu/sh2a/setup-sh7206.c b/arch/sh/kernel/cpu/sh2a/setup-sh7206.c
index dc47b04e1049..5d14f849aea3 100644
--- a/arch/sh/kernel/cpu/sh2a/setup-sh7206.c
+++ b/arch/sh/kernel/cpu/sh2a/setup-sh7206.c
@@ -136,6 +136,8 @@ static DECLARE_INTC_DESC(intc_desc, "sh7206", vectors, groups,
 static struct plat_sci_port scif0_platform_data = {
         .mapbase        = 0xfffe8000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 240, 240, 240, 240 },
 };
@@ -151,6 +153,8 @@ static struct platform_device scif0_device = {
 static struct plat_sci_port scif1_platform_data = {
         .mapbase        = 0xfffe8800,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 244, 244, 244, 244 },
 };
@@ -166,6 +170,8 @@ static struct platform_device scif1_device = {
 static struct plat_sci_port scif2_platform_data = {
         .mapbase        = 0xfffe9000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 248, 248, 248, 248 },
 };
@@ -181,6 +187,8 @@ static struct platform_device scif2_device = {
 static struct plat_sci_port scif3_platform_data = {
         .mapbase        = 0xfffe9800,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 252, 252, 252, 252 },
 };
diff --git a/arch/sh/kernel/cpu/sh3/setup-sh7705.c b/arch/sh/kernel/cpu/sh3/setup-sh7705.c
index baadd7f54d94..cd2e702feb7e 100644
--- a/arch/sh/kernel/cpu/sh3/setup-sh7705.c
+++ b/arch/sh/kernel/cpu/sh3/setup-sh7705.c
@@ -70,6 +70,9 @@ static DECLARE_INTC_DESC(intc_desc, "sh7705", vectors, NULL,
 static struct plat_sci_port scif0_platform_data = {
         .mapbase        = 0xa4410000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_TIE | SCSCR_RIE  | SCSCR_TE |
+                          SCSCR_RE  | SCSCR_CKE1 | SCSCR_CKE0,
+        .scbrr_algo_id  = SCBRR_ALGO_4,
         .type           = PORT_SCIF,
         .irqs           = { 56, 56, 56 },
 };
@@ -85,6 +88,8 @@ static struct platform_device scif0_device = {
 static struct plat_sci_port scif1_platform_data = {
         .mapbase        = 0xa4400000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_TIE | SCSCR_RIE | SCSCR_TE | SCSCR_RE,
+        .scbrr_algo_id  = SCBRR_ALGO_4,
         .type           = PORT_SCIF,
         .irqs           = { 52, 52, 52 },
 };
diff --git a/arch/sh/kernel/cpu/sh3/setup-sh770x.c b/arch/sh/kernel/cpu/sh3/setup-sh770x.c
index 3cf8c8ef7b32..4551ad647c2c 100644
--- a/arch/sh/kernel/cpu/sh3/setup-sh770x.c
+++ b/arch/sh/kernel/cpu/sh3/setup-sh770x.c
@@ -109,6 +109,8 @@ static struct platform_device rtc_device = {
 static struct plat_sci_port scif0_platform_data = {
         .mapbase        = 0xfffffe80,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_TE | SCSCR_RE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCI,
         .irqs           = { 23, 23, 23, 0 },
 };
@@ -126,6 +128,8 @@ static struct platform_device scif0_device = {
 static struct plat_sci_port scif1_platform_data = {
         .mapbase        = 0xa4000150,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_TE | SCSCR_RE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 56, 56, 56, 56 },
 };
@@ -143,6 +147,8 @@ static struct platform_device scif1_device = {
 static struct plat_sci_port scif2_platform_data = {
         .mapbase        = 0xa4000140,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_TE | SCSCR_RE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_IRDA,
         .irqs           = { 52, 52, 52, 52 },
 };
diff --git a/arch/sh/kernel/cpu/sh3/setup-sh7710.c b/arch/sh/kernel/cpu/sh3/setup-sh7710.c
index b0c2fb4ab479..78f6b01d42c3 100644
--- a/arch/sh/kernel/cpu/sh3/setup-sh7710.c
+++ b/arch/sh/kernel/cpu/sh3/setup-sh7710.c
@@ -99,6 +99,9 @@ static struct platform_device rtc_device = {
 static struct plat_sci_port scif0_platform_data = {
         .mapbase        = 0xa4400000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_TE | SCSCR_RE | SCSCR_REIE |
+                          SCSCR_CKE1 | SCSCR_CKE0,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 52, 52, 52, 52 },
 };
@@ -114,6 +117,9 @@ static struct platform_device scif0_device = {
 static struct plat_sci_port scif1_platform_data = {
         .mapbase        = 0xa4410000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_TE | SCSCR_RE | SCSCR_REIE |
+                          SCSCR_CKE1 | SCSCR_CKE0,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 56, 56, 56, 56 },
 };
diff --git a/arch/sh/kernel/cpu/sh3/setup-sh7720.c b/arch/sh/kernel/cpu/sh3/setup-sh7720.c
index 24b17135d5d2..365b94a6fcb7 100644
--- a/arch/sh/kernel/cpu/sh3/setup-sh7720.c
+++ b/arch/sh/kernel/cpu/sh3/setup-sh7720.c
@@ -1,5 +1,5 @@
 /*
- * SH7720 Setup
+ * Setup code for SH7720, SH7721.
  *
  *  Copyright (C) 2007  Markus Brunner, Mark Jonas
  *  Copyright (C) 2009  Paul Mundt
@@ -51,6 +51,8 @@ static struct platform_device rtc_device = {
 static struct plat_sci_port scif0_platform_data = {
         .mapbase        = 0xa4430000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE,
+        .scbrr_algo_id  = SCBRR_ALGO_4,
         .type           = PORT_SCIF,
         .irqs           = { 80, 80, 80, 80 },
 };
@@ -66,6 +68,8 @@ static struct platform_device scif0_device = {
 static struct plat_sci_port scif1_platform_data = {
         .mapbase        = 0xa4438000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE,
+        .scbrr_algo_id  = SCBRR_ALGO_4,
         .type           = PORT_SCIF,
         .irqs           = { 81, 81, 81, 81 },
 };
diff --git a/arch/sh/kernel/cpu/sh4/probe.c b/arch/sh/kernel/cpu/sh4/probe.c
index b93458f33b74..971cf0fce4f5 100644
--- a/arch/sh/kernel/cpu/sh4/probe.c
+++ b/arch/sh/kernel/cpu/sh4/probe.c
@@ -151,8 +151,14 @@ void __cpuinit cpu_probe(void)
                         boot_cpu_data.flags |= CPU_HAS_L2_CACHE;
                         break;
                 case 0x10:
+                case 0x11:
                         boot_cpu_data.type = CPU_SH7757;
                         break;
+                case 0xd0:
+                case 0x40: /* yon-ten-go */
+                        boot_cpu_data.type = CPU_SH7372;
+                        break;
+
                 }
                 break;
         case 0x4000:    /* 1st cut */
diff --git a/arch/sh/kernel/cpu/sh4/setup-sh4-202.c b/arch/sh/kernel/cpu/sh4/setup-sh4-202.c
index e916b18e1f7c..5b2833159b7d 100644
--- a/arch/sh/kernel/cpu/sh4/setup-sh4-202.c
+++ b/arch/sh/kernel/cpu/sh4/setup-sh4-202.c
@@ -18,6 +18,8 @@
 static struct plat_sci_port scif0_platform_data = {
         .mapbase        = 0xffe80000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 40, 41, 43, 42 },
 };
diff --git a/arch/sh/kernel/cpu/sh4/setup-sh7750.c b/arch/sh/kernel/cpu/sh4/setup-sh7750.c
index 911d196e86b5..c2b0aaaedcae 100644
--- a/arch/sh/kernel/cpu/sh4/setup-sh7750.c
+++ b/arch/sh/kernel/cpu/sh4/setup-sh7750.c
@@ -14,6 +14,7 @@
 #include <linux/io.h>
 #include <linux/sh_timer.h>
 #include <linux/serial_sci.h>
+#include <asm/machtypes.h>
 
 static struct resource rtc_resources[] = {
         [0] = {
@@ -35,33 +36,37 @@ static struct platform_device rtc_device = {
         .resource       = rtc_resources,
 };
 
-static struct plat_sci_port scif0_platform_data = {
+static struct plat_sci_port sci_platform_data = {
         .mapbase        = 0xffe00000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_TE | SCSCR_RE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCI,
         .irqs           = { 23, 23, 23, 0 },
 };
 
-static struct platform_device scif0_device = {
+static struct platform_device sci_device = {
         .name           = "sh-sci",
         .id             = 0,
         .dev            = {
-                .platform_data  = &scif0_platform_data,
+                .platform_data  = &sci_platform_data,
         },
 };
 
-static struct plat_sci_port scif1_platform_data = {
+static struct plat_sci_port scif_platform_data = {
         .mapbase        = 0xffe80000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_TE | SCSCR_RE | SCSCR_REIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 40, 40, 40, 40 },
 };
 
-static struct platform_device scif1_device = {
+static struct platform_device scif_device = {
         .name           = "sh-sci",
         .id             = 1,
         .dev            = {
-                .platform_data  = &scif1_platform_data,
+                .platform_data  = &scif_platform_data,
         },
 };
 
@@ -210,8 +215,6 @@ static struct platform_device tmu4_device = {
 #endif
 
 static struct platform_device *sh7750_devices[] __initdata = {
-        &scif0_device,
-        &scif1_device,
         &rtc_device,
         &tmu0_device,
         &tmu1_device,
@@ -226,14 +229,19 @@ static struct platform_device *sh7750_devices[] __initdata = {
 
 static int __init sh7750_devices_setup(void)
 {
+        if (mach_is_rts7751r2d()) {
+                platform_register_device(&scif_device);
+        } else {
+                platform_register_device(&sci_device);
+                platform_register_device(&scif_device);
+        }
+
         return platform_add_devices(sh7750_devices,
                                     ARRAY_SIZE(sh7750_devices));
 }
 arch_initcall(sh7750_devices_setup);
 
 static struct platform_device *sh7750_early_devices[] __initdata = {
-        &scif0_device,
-        &scif1_device,
         &tmu0_device,
         &tmu1_device,
         &tmu2_device,
@@ -247,6 +255,14 @@ static struct platform_device *sh7750_early_devices[] __initdata = {
 
 void __init plat_early_device_setup(void)
 {
+        if (mach_is_rts7751r2d()) {
+                scif_platform_data.scscr |= SCSCR_CKE1;
+                early_platform_add_devices(&scif_device, 1);
+        } else {
+                early_platform_add_devices(&sci_device, 1);
+                early_platform_add_devices(&scif_device, 1);
+        }
+
         early_platform_add_devices(sh7750_early_devices,
                                    ARRAY_SIZE(sh7750_early_devices));
 }
diff --git a/arch/sh/kernel/cpu/sh4/setup-sh7760.c b/arch/sh/kernel/cpu/sh4/setup-sh7760.c
index 48ea8fe85dc5..78bbf232e391 100644
--- a/arch/sh/kernel/cpu/sh4/setup-sh7760.c
+++ b/arch/sh/kernel/cpu/sh4/setup-sh7760.c
@@ -129,6 +129,8 @@ static DECLARE_INTC_DESC(intc_desc_irq, "sh7760-irq", vectors_irq, groups,
 static struct plat_sci_port scif0_platform_data = {
         .mapbase        = 0xfe600000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 52, 53, 55, 54 },
 };
@@ -145,6 +147,8 @@ static struct plat_sci_port scif1_platform_data = {
         .mapbase        = 0xfe610000,
         .flags          = UPF_BOOT_AUTOCONF,
         .type           = PORT_SCIF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .irqs           = { 72, 73, 75, 74 },
 };
 
@@ -159,6 +163,8 @@ static struct platform_device scif1_device = {
 static struct plat_sci_port scif2_platform_data = {
         .mapbase        = 0xfe620000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 76, 77, 79, 78 },
 };
@@ -174,6 +180,8 @@ static struct platform_device scif2_device = {
 static struct plat_sci_port scif3_platform_data = {
         .mapbase        = 0xfe480000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCI,
         .irqs           = { 80, 81, 82, 0 },
 };
diff --git a/arch/sh/kernel/cpu/sh4a/setup-sh7343.c b/arch/sh/kernel/cpu/sh4a/setup-sh7343.c
index 3681cafdb4af..1b8848317e9c 100644
--- a/arch/sh/kernel/cpu/sh4a/setup-sh7343.c
+++ b/arch/sh/kernel/cpu/sh4a/setup-sh7343.c
@@ -19,6 +19,8 @@
 static struct plat_sci_port scif0_platform_data = {
         .mapbase        = 0xffe00000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_CKE1,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 80, 80, 80, 80 },
 };
@@ -34,6 +36,8 @@ static struct platform_device scif0_device = {
 static struct plat_sci_port scif1_platform_data = {
         .mapbase        = 0xffe10000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_CKE1,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 81, 81, 81, 81 },
 };
@@ -49,6 +53,8 @@ static struct platform_device scif1_device = {
 static struct plat_sci_port scif2_platform_data = {
         .mapbase        = 0xffe20000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_CKE1,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 82, 82, 82, 82 },
 };
@@ -64,6 +70,8 @@ static struct platform_device scif2_device = {
 static struct plat_sci_port scif3_platform_data = {
         .mapbase        = 0xffe30000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_CKE1,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 83, 83, 83, 83 },
 };
@@ -360,6 +368,8 @@ void __init plat_early_device_setup(void)
 
 enum {
         UNUSED = 0,
+        ENABLED,
+        DISABLED,
 
         /* interrupt sources */
         IRQ0, IRQ1, IRQ2, IRQ3, IRQ4, IRQ5, IRQ6, IRQ7,
@@ -375,15 +385,13 @@ enum {
         I2C0_ALI, I2C0_TACKI, I2C0_WAITI, I2C0_DTEI,
         I2C1_ALI, I2C1_TACKI, I2C1_WAITI, I2C1_DTEI,
         SIM_TEI, SIM_TXI, SIM_RXI, SIM_ERI,
-        IRDA,
-        SDHI0, SDHI1, SDHI2, SDHI3,
-        CMT, TSIF, SIU,
+        IRDA, SDHI, CMT, TSIF, SIU,
         TMU0, TMU1, TMU2,
         JPU, LCDC,
 
         /* interrupt groups */
 
-        DMAC0123, VIOVOU, MMC, DMAC45, FLCTL, I2C0, I2C1, SIM, SDHI, USB,
+        DMAC0123, VIOVOU, MMC, DMAC45, FLCTL, I2C0, I2C1, SIM, USB,
 };
 
 static struct intc_vect vectors[] __initdata = {
@@ -412,8 +420,8 @@ static struct intc_vect vectors[] __initdata = {
         INTC_VECT(FLCTL_FLTREQ0I, 0xdc0), INTC_VECT(FLCTL_FLTREQ1I, 0xde0),
         INTC_VECT(I2C0_ALI, 0xe00), INTC_VECT(I2C0_TACKI, 0xe20),
         INTC_VECT(I2C0_WAITI, 0xe40), INTC_VECT(I2C0_DTEI, 0xe60),
-        INTC_VECT(SDHI0, 0xe80), INTC_VECT(SDHI1, 0xea0),
-        INTC_VECT(SDHI2, 0xec0), INTC_VECT(SDHI3, 0xee0),
+        INTC_VECT(SDHI, 0xe80), INTC_VECT(SDHI, 0xea0),
+        INTC_VECT(SDHI, 0xec0), INTC_VECT(SDHI, 0xee0),
         INTC_VECT(CMT, 0xf00), INTC_VECT(TSIF, 0xf20),
         INTC_VECT(SIU, 0xf80),
         INTC_VECT(TMU0, 0x400), INTC_VECT(TMU1, 0x420),
@@ -431,7 +439,6 @@ static struct intc_group groups[] __initdata = {
         INTC_GROUP(I2C0, I2C0_ALI, I2C0_TACKI, I2C0_WAITI, I2C0_DTEI),
         INTC_GROUP(I2C1, I2C1_ALI, I2C1_TACKI, I2C1_WAITI, I2C1_DTEI),
         INTC_GROUP(SIM, SIM_TEI, SIM_TXI, SIM_RXI, SIM_ERI),
-        INTC_GROUP(SDHI, SDHI0, SDHI1, SDHI2, SDHI3),
         INTC_GROUP(USB, USBI0, USBI1),
 };
 
@@ -452,7 +459,7 @@ static struct intc_mask_reg mask_registers[] __initdata = {
           { I2C0_DTEI, I2C0_WAITI, I2C0_TACKI, I2C0_ALI,
             FLCTL_FLTREQ1I, FLCTL_FLTREQ0I, FLCTL_FLENDI, FLCTL_FLSTEI } },
         { 0xa40800a0, 0xa40800e0, 8, /* IMR8 / IMCR8 */
-          { SDHI3, SDHI2, SDHI1, SDHI0, 0, 0, 0, SIU } },
+          { DISABLED, ENABLED, ENABLED, ENABLED, 0, 0, 0, SIU } },
         { 0xa40800a4, 0xa40800e4, 8, /* IMR9 / IMCR9 */
           { 0, 0, 0, CMT, 0, USBI1, USBI0 } },
         { 0xa40800a8, 0xa40800e8, 8, /* IMR10 / IMCR10 */
@@ -488,9 +495,13 @@ static struct intc_mask_reg ack_registers[] __initdata = {
           { IRQ0, IRQ1, IRQ2, IRQ3, IRQ4, IRQ5, IRQ6, IRQ7 } },
 };
 
-static DECLARE_INTC_DESC_ACK(intc_desc, "sh7343", vectors, groups,
-                             mask_registers, prio_registers, sense_registers,
-                             ack_registers);
+static struct intc_desc intc_desc __initdata = {
+        .name = "sh7343",
+        .force_enable = ENABLED,
+        .force_disable = DISABLED,
+        .hw = INTC_HW_DESC(vectors, groups, mask_registers,
+                           prio_registers, sense_registers, ack_registers),
+};
 
 void __init plat_irq_setup(void)
 {
diff --git a/arch/sh/kernel/cpu/sh4a/setup-sh7366.c b/arch/sh/kernel/cpu/sh4a/setup-sh7366.c
index 8dab9e1bbd89..82616af64d62 100644
--- a/arch/sh/kernel/cpu/sh4a/setup-sh7366.c
+++ b/arch/sh/kernel/cpu/sh4a/setup-sh7366.c
@@ -21,6 +21,8 @@
 static struct plat_sci_port scif0_platform_data = {
         .mapbase        = 0xffe00000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 80, 80, 80, 80 },
 };
@@ -319,6 +321,8 @@ void __init plat_early_device_setup(void)
 
 enum {
         UNUSED=0,
+        ENABLED,
+        DISABLED,
 
         /* interrupt sources */
         IRQ0, IRQ1, IRQ2, IRQ3, IRQ4, IRQ5, IRQ6, IRQ7,
@@ -332,14 +336,13 @@ enum {
         DENC, MSIOF,
         FLCTL_FLSTEI, FLCTL_FLENDI, FLCTL_FLTREQ0I, FLCTL_FLTREQ1I,
         I2C_ALI, I2C_TACKI, I2C_WAITI, I2C_DTEI,
-        SDHI0, SDHI1, SDHI2, SDHI3,
-        CMT, TSIF, SIU,
+        SDHI, CMT, TSIF, SIU,
         TMU0, TMU1, TMU2,
         VEU2, LCDC,
 
         /* interrupt groups */
 
-        DMAC0123, VIOVOU, MMC, DMAC45, FLCTL, I2C, SDHI,
+        DMAC0123, VIOVOU, MMC, DMAC45, FLCTL, I2C,
 };
 
 static struct intc_vect vectors[] __initdata = {
@@ -364,8 +367,8 @@ static struct intc_vect vectors[] __initdata = {
         INTC_VECT(FLCTL_FLTREQ0I, 0xdc0), INTC_VECT(FLCTL_FLTREQ1I, 0xde0),
         INTC_VECT(I2C_ALI, 0xe00), INTC_VECT(I2C_TACKI, 0xe20),
         INTC_VECT(I2C_WAITI, 0xe40), INTC_VECT(I2C_DTEI, 0xe60),
-        INTC_VECT(SDHI0, 0xe80), INTC_VECT(SDHI1, 0xea0),
-        INTC_VECT(SDHI2, 0xec0), INTC_VECT(SDHI3, 0xee0),
+        INTC_VECT(SDHI, 0xe80), INTC_VECT(SDHI, 0xea0),
+        INTC_VECT(SDHI, 0xec0), INTC_VECT(SDHI, 0xee0),
         INTC_VECT(CMT, 0xf00), INTC_VECT(TSIF, 0xf20),
         INTC_VECT(SIU, 0xf80),
         INTC_VECT(TMU0, 0x400), INTC_VECT(TMU1, 0x420),
@@ -381,7 +384,6 @@ static struct intc_group groups[] __initdata = {
         INTC_GROUP(FLCTL, FLCTL_FLSTEI, FLCTL_FLENDI,
                    FLCTL_FLTREQ0I, FLCTL_FLTREQ1I),
         INTC_GROUP(I2C, I2C_ALI, I2C_TACKI, I2C_WAITI, I2C_DTEI),
-        INTC_GROUP(SDHI, SDHI0, SDHI1, SDHI2, SDHI3),
 };
 
 static struct intc_mask_reg mask_registers[] __initdata = {
@@ -403,7 +405,7 @@ static struct intc_mask_reg mask_registers[] __initdata = {
           { I2C_DTEI, I2C_WAITI, I2C_TACKI, I2C_ALI,
             FLCTL_FLTREQ1I, FLCTL_FLTREQ0I, FLCTL_FLENDI, FLCTL_FLSTEI } },
         { 0xa40800a0, 0xa40800e0, 8, /* IMR8 / IMCR8 */
-          { SDHI3, SDHI2, SDHI1, SDHI0, 0, 0, 0, SIU } },
+          { DISABLED, ENABLED, ENABLED, ENABLED, 0, 0, 0, SIU } },
         { 0xa40800a4, 0xa40800e4, 8, /* IMR9 / IMCR9 */
           { 0, 0, 0, CMT, 0, USB, } },
         { 0xa40800a8, 0xa40800e8, 8, /* IMR10 / IMCR10 */
@@ -441,9 +443,13 @@ static struct intc_mask_reg ack_registers[] __initdata = {
           { IRQ0, IRQ1, IRQ2, IRQ3, IRQ4, IRQ5, IRQ6, IRQ7 } },
 };
 
-static DECLARE_INTC_DESC_ACK(intc_desc, "sh7366", vectors, groups,
-                             mask_registers, prio_registers, sense_registers,
-                             ack_registers);
+static struct intc_desc intc_desc __initdata = {
+        .name = "sh7366",
+        .force_enable = ENABLED,
+        .force_disable = DISABLED,
+        .hw = INTC_HW_DESC(vectors, groups, mask_registers,
+                           prio_registers, sense_registers, ack_registers),
+};
 
 void __init plat_irq_setup(void)
 {
diff --git a/arch/sh/kernel/cpu/sh4a/setup-sh7722.c b/arch/sh/kernel/cpu/sh4a/setup-sh7722.c
index d551ed8dea95..5813d8023619 100644
--- a/arch/sh/kernel/cpu/sh4a/setup-sh7722.c
+++ b/arch/sh/kernel/cpu/sh4a/setup-sh7722.c
@@ -181,6 +181,8 @@ struct platform_device dma_device = {
 static struct plat_sci_port scif0_platform_data = {
         .mapbase        = 0xffe00000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 80, 80, 80, 80 },
 };
@@ -196,6 +198,8 @@ static struct platform_device scif0_device = {
 static struct plat_sci_port scif1_platform_data = {
         .mapbase        = 0xffe10000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 81, 81, 81, 81 },
 };
@@ -211,6 +215,8 @@ static struct platform_device scif1_device = {
 static struct plat_sci_port scif2_platform_data = {
         .mapbase        = 0xffe20000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 82, 82, 82, 82 },
 };
@@ -699,7 +705,7 @@ static struct intc_mask_reg mask_registers[] __initdata = {
           { I2C_DTEI, I2C_WAITI, I2C_TACKI, I2C_ALI,
             FLCTL_FLTREQ1I, FLCTL_FLTREQ0I, FLCTL_FLENDI, FLCTL_FLSTEI } },
         { 0xa40800a0, 0xa40800e0, 8, /* IMR8 / IMCR8 */
-          { DISABLED, DISABLED, ENABLED, ENABLED, 0, 0, TWODG, SIU } },
+          { DISABLED, ENABLED, ENABLED, ENABLED, 0, 0, TWODG, SIU } },
         { 0xa40800a4, 0xa40800e4, 8, /* IMR9 / IMCR9 */
           { 0, 0, 0, CMT, 0, USB_USBI1, USB_USBI0, } },
         { 0xa40800a8, 0xa40800e8, 8, /* IMR10 / IMCR10 */
diff --git a/arch/sh/kernel/cpu/sh4a/setup-sh7723.c b/arch/sh/kernel/cpu/sh4a/setup-sh7723.c
index 0eadefdbbba1..072382280f96 100644
--- a/arch/sh/kernel/cpu/sh4a/setup-sh7723.c
+++ b/arch/sh/kernel/cpu/sh4a/setup-sh7723.c
@@ -24,6 +24,8 @@
 static struct plat_sci_port scif0_platform_data = {
         .mapbase        = 0xffe00000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 80, 80, 80, 80 },
 };
@@ -39,6 +41,8 @@ static struct platform_device scif0_device = {
 static struct plat_sci_port scif1_platform_data = {
         .mapbase        = 0xffe10000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 81, 81, 81, 81 },
 };
@@ -54,6 +58,8 @@ static struct platform_device scif1_device = {
 static struct plat_sci_port scif2_platform_data = {
         .mapbase        = 0xffe20000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 82, 82, 82, 82 },
 };
@@ -69,6 +75,8 @@ static struct platform_device scif2_device = {
 static struct plat_sci_port scif3_platform_data = {
         .mapbase        = 0xa4e30000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE,
+        .scbrr_algo_id  = SCBRR_ALGO_3,
         .type           = PORT_SCIFA,
         .irqs           = { 56, 56, 56, 56 },
 };
@@ -84,6 +92,8 @@ static struct platform_device scif3_device = {
 static struct plat_sci_port scif4_platform_data = {
         .mapbase        = 0xa4e40000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE,
+        .scbrr_algo_id  = SCBRR_ALGO_3,
         .type           = PORT_SCIFA,
         .irqs           = { 88, 88, 88, 88 },
 };
@@ -99,6 +109,8 @@ static struct platform_device scif4_device = {
 static struct plat_sci_port scif5_platform_data = {
         .mapbase        = 0xa4e50000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE,
+        .scbrr_algo_id  = SCBRR_ALGO_3,
         .type           = PORT_SCIFA,
         .irqs           = { 109, 109, 109, 109 },
 };
@@ -719,7 +731,7 @@ static struct intc_group groups[] __initdata = {
 static struct intc_mask_reg mask_registers[] __initdata = {
         { 0xa4080080, 0xa40800c0, 8, /* IMR0 / IMCR0 */
           { 0, TMU1_TUNI2, TMU1_TUNI1, TMU1_TUNI0,
-            0, DISABLED, ENABLED, ENABLED } },
+            0, ENABLED, ENABLED, ENABLED } },
         { 0xa4080084, 0xa40800c4, 8, /* IMR1 / IMCR1 */
           { VIO_VOUI, VIO_VEU2HI,VIO_BEUI,VIO_CEUI,DMAC0A_DEI3,DMAC0A_DEI2,DMAC0A_DEI1,DMAC0A_DEI0 } },
         { 0xa4080088, 0xa40800c8, 8, /* IMR2 / IMCR2 */
@@ -736,7 +748,7 @@ static struct intc_mask_reg mask_registers[] __initdata = {
           { I2C_DTEI, I2C_WAITI, I2C_TACKI, I2C_ALI,
             FLCTL_FLTREQ1I, FLCTL_FLTREQ0I, FLCTL_FLTENDI, FLCTL_FLSTEI } },
         { 0xa40800a0, 0xa40800e0, 8, /* IMR8 / IMCR8 */
-          { 0, DISABLED, ENABLED, ENABLED,
+          { 0, ENABLED, ENABLED, ENABLED,
             0, 0, SCIFA_SCIFA2, SIU_SIUI } },
         { 0xa40800a4, 0xa40800e4, 8, /* IMR9 / IMCR9 */
           { 0, 0, 0, CMT_CMTI, 0, 0, USB_USI0,0 } },
diff --git a/arch/sh/kernel/cpu/sh4a/setup-sh7724.c b/arch/sh/kernel/cpu/sh4a/setup-sh7724.c
index 828c9657eb52..0333fe9e3881 100644
--- a/arch/sh/kernel/cpu/sh4a/setup-sh7724.c
+++ b/arch/sh/kernel/cpu/sh4a/setup-sh7724.c
@@ -257,6 +257,8 @@ static struct platform_device dma1_device = {
 static struct plat_sci_port scif0_platform_data = {
         .mapbase        = 0xffe00000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 80, 80, 80, 80 },
 };
@@ -272,6 +274,8 @@ static struct platform_device scif0_device = {
 static struct plat_sci_port scif1_platform_data = {
         .mapbase        = 0xffe10000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 81, 81, 81, 81 },
 };
@@ -287,6 +291,8 @@ static struct platform_device scif1_device = {
 static struct plat_sci_port scif2_platform_data = {
         .mapbase        = 0xffe20000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 82, 82, 82, 82 },
 };
@@ -302,6 +308,8 @@ static struct platform_device scif2_device = {
 static struct plat_sci_port scif3_platform_data = {
         .mapbase        = 0xa4e30000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE,
+        .scbrr_algo_id  = SCBRR_ALGO_3,
         .type           = PORT_SCIFA,
         .irqs           = { 56, 56, 56, 56 },
 };
@@ -317,6 +325,8 @@ static struct platform_device scif3_device = {
 static struct plat_sci_port scif4_platform_data = {
         .mapbase        = 0xa4e40000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE,
+        .scbrr_algo_id  = SCBRR_ALGO_3,
         .type           = PORT_SCIFA,
         .irqs           = { 88, 88, 88, 88 },
 };
@@ -332,6 +342,8 @@ static struct platform_device scif4_device = {
 static struct plat_sci_port scif5_platform_data = {
         .mapbase        = 0xa4e50000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE,
+        .scbrr_algo_id  = SCBRR_ALGO_3,
         .type           = PORT_SCIFA,
         .irqs           = { 109, 109, 109, 109 },
 };
@@ -1144,7 +1156,7 @@ static struct intc_group groups[] __initdata = {
 static struct intc_mask_reg mask_registers[] __initdata = {
         { 0xa4080080, 0xa40800c0, 8, /* IMR0 / IMCR0 */
           { 0, TMU1_TUNI2, TMU1_TUNI1, TMU1_TUNI0,
-            0, DISABLED, ENABLED, ENABLED } },
+            0, ENABLED, ENABLED, ENABLED } },
         { 0xa4080084, 0xa40800c4, 8, /* IMR1 / IMCR1 */
           { VIO_VOU, VIO_VEU1, VIO_BEU0, VIO_CEU0,
             DMAC0A_DEI3, DMAC0A_DEI2, DMAC0A_DEI1, DMAC0A_DEI0 } },
@@ -1166,7 +1178,7 @@ static struct intc_mask_reg mask_registers[] __initdata = {
           { I2C0_DTEI, I2C0_WAITI, I2C0_TACKI, I2C0_ALI,
             I2C1_DTEI, I2C1_WAITI, I2C1_TACKI, I2C1_ALI } },
         { 0xa40800a0, 0xa40800e0, 8, /* IMR8 / IMCR8 */
-          { DISABLED, DISABLED, ENABLED, ENABLED,
+          { DISABLED, ENABLED, ENABLED, ENABLED,
             0, 0, SCIFA5, FSI } },
         { 0xa40800a4, 0xa40800e4, 8, /* IMR9 / IMCR9 */
           { 0, 0, 0, CMT, 0, USB1, USB0, 0 } },
diff --git a/arch/sh/kernel/cpu/sh4a/setup-sh7757.c b/arch/sh/kernel/cpu/sh4a/setup-sh7757.c
index 749c6388d5a5..9c1de2633ac3 100644
--- a/arch/sh/kernel/cpu/sh4a/setup-sh7757.c
+++ b/arch/sh/kernel/cpu/sh4a/setup-sh7757.c
@@ -20,6 +20,8 @@
 static struct plat_sci_port scif2_platform_data = {
         .mapbase        = 0xfe4b0000,           /* SCIF2 */
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 40, 40, 40, 40 },
 };
@@ -35,6 +37,8 @@ static struct platform_device scif2_device = {
 static struct plat_sci_port scif3_platform_data = {
         .mapbase        = 0xfe4c0000,           /* SCIF3 */
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 76, 76, 76, 76 },
 };
@@ -50,6 +54,8 @@ static struct platform_device scif3_device = {
 static struct plat_sci_port scif4_platform_data = {
         .mapbase        = 0xfe4d0000,           /* SCIF4 */
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 104, 104, 104, 104 },
 };
diff --git a/arch/sh/kernel/cpu/sh4a/setup-sh7763.c b/arch/sh/kernel/cpu/sh4a/setup-sh7763.c
index 5b5f6b005fc5..593eca6509b5 100644
--- a/arch/sh/kernel/cpu/sh4a/setup-sh7763.c
+++ b/arch/sh/kernel/cpu/sh4a/setup-sh7763.c
@@ -19,6 +19,8 @@
 static struct plat_sci_port scif0_platform_data = {
         .mapbase        = 0xffe00000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 40, 40, 40, 40 },
 };
@@ -34,6 +36,8 @@ static struct platform_device scif0_device = {
 static struct plat_sci_port scif1_platform_data = {
         .mapbase        = 0xffe08000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 76, 76, 76, 76 },
 };
@@ -49,6 +53,8 @@ static struct platform_device scif1_device = {
 static struct plat_sci_port scif2_platform_data = {
         .mapbase        = 0xffe10000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 104, 104, 104, 104 },
 };
diff --git a/arch/sh/kernel/cpu/sh4a/setup-sh7770.c b/arch/sh/kernel/cpu/sh4a/setup-sh7770.c
index 7270d7fd6761..2c6aa22cf5f6 100644
--- a/arch/sh/kernel/cpu/sh4a/setup-sh7770.c
+++ b/arch/sh/kernel/cpu/sh4a/setup-sh7770.c
@@ -17,6 +17,8 @@
 static struct plat_sci_port scif0_platform_data = {
         .mapbase        = 0xff923000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_TOIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 61, 61, 61, 61 },
 };
@@ -32,6 +34,8 @@ static struct platform_device scif0_device = {
 static struct plat_sci_port scif1_platform_data = {
         .mapbase        = 0xff924000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_TOIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 62, 62, 62, 62 },
 };
@@ -47,6 +51,8 @@ static struct platform_device scif1_device = {
 static struct plat_sci_port scif2_platform_data = {
         .mapbase        = 0xff925000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_TOIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 63, 63, 63, 63 },
 };
@@ -62,6 +68,8 @@ static struct platform_device scif2_device = {
 static struct plat_sci_port scif3_platform_data = {
         .mapbase        = 0xff926000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_TOIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 64, 64, 64, 64 },
 };
@@ -77,6 +85,8 @@ static struct platform_device scif3_device = {
 static struct plat_sci_port scif4_platform_data = {
         .mapbase        = 0xff927000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_TOIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 65, 65, 65, 65 },
 };
@@ -92,6 +102,8 @@ static struct platform_device scif4_device = {
 static struct plat_sci_port scif5_platform_data = {
         .mapbase        = 0xff928000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_TOIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 66, 66, 66, 66 },
 };
@@ -107,6 +119,8 @@ static struct platform_device scif5_device = {
 static struct plat_sci_port scif6_platform_data = {
         .mapbase        = 0xff929000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_TOIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 67, 67, 67, 67 },
 };
@@ -122,6 +136,8 @@ static struct platform_device scif6_device = {
 static struct plat_sci_port scif7_platform_data = {
         .mapbase        = 0xff92a000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_TOIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 68, 68, 68, 68 },
 };
@@ -137,6 +153,8 @@ static struct platform_device scif7_device = {
 static struct plat_sci_port scif8_platform_data = {
         .mapbase        = 0xff92b000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_TOIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 69, 69, 69, 69 },
 };
@@ -152,6 +170,8 @@ static struct platform_device scif8_device = {
 static struct plat_sci_port scif9_platform_data = {
         .mapbase        = 0xff92c000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_TOIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 70, 70, 70, 70 },
 };
diff --git a/arch/sh/kernel/cpu/sh4a/setup-sh7780.c b/arch/sh/kernel/cpu/sh4a/setup-sh7780.c
index 0f414864f76b..08add7fa6849 100644
--- a/arch/sh/kernel/cpu/sh4a/setup-sh7780.c
+++ b/arch/sh/kernel/cpu/sh4a/setup-sh7780.c
@@ -20,6 +20,8 @@
 static struct plat_sci_port scif0_platform_data = {
         .mapbase        = 0xffe00000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_CKE1,
+        .scbrr_algo_id  = SCBRR_ALGO_1,
         .type           = PORT_SCIF,
         .irqs           = { 40, 40, 40, 40 },
 };
@@ -35,6 +37,8 @@ static struct platform_device scif0_device = {
 static struct plat_sci_port scif1_platform_data = {
         .mapbase        = 0xffe10000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_CKE1,
+        .scbrr_algo_id  = SCBRR_ALGO_1,
         .type           = PORT_SCIF,
         .irqs           = { 76, 76, 76, 76 },
 };
@@ -379,6 +383,7 @@ static int __init sh7780_devices_setup(void)
                                     ARRAY_SIZE(sh7780_devices));
 }
 arch_initcall(sh7780_devices_setup);
+
 static struct platform_device *sh7780_early_devices[] __initdata = {
         &scif0_device,
         &scif1_device,
@@ -392,6 +397,13 @@ static struct platform_device *sh7780_early_devices[] __initdata = {
 
 void __init plat_early_device_setup(void)
 {
+        if (mach_is_sh2007()) {
+                scif0_platform_data.scscr &= ~SCSCR_CKE1;
+                scif0_platform_data.scbrr_algo_id = SCBRR_ALGO_2;
+                scif1_platform_data.scscr &= ~SCSCR_CKE1;
+                scif1_platform_data.scbrr_algo_id = SCBRR_ALGO_2;
+        }
+
         early_platform_add_devices(sh7780_early_devices,
                                    ARRAY_SIZE(sh7780_early_devices));
 }
diff --git a/arch/sh/kernel/cpu/sh4a/setup-sh7785.c b/arch/sh/kernel/cpu/sh4a/setup-sh7785.c
index c9a572bc6dc8..18d8fc136fb2 100644
--- a/arch/sh/kernel/cpu/sh4a/setup-sh7785.c
+++ b/arch/sh/kernel/cpu/sh4a/setup-sh7785.c
@@ -23,6 +23,8 @@
 static struct plat_sci_port scif0_platform_data = {
         .mapbase        = 0xffea0000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_CKE1,
+        .scbrr_algo_id  = SCBRR_ALGO_1,
         .type           = PORT_SCIF,
         .irqs           = { 40, 40, 40, 40 },
 };
@@ -38,6 +40,8 @@ static struct platform_device scif0_device = {
 static struct plat_sci_port scif1_platform_data = {
         .mapbase        = 0xffeb0000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_CKE1,
+        .scbrr_algo_id  = SCBRR_ALGO_1,
         .type           = PORT_SCIF,
         .irqs           = { 44, 44, 44, 44 },
 };
@@ -53,6 +57,8 @@ static struct platform_device scif1_device = {
 static struct plat_sci_port scif2_platform_data = {
         .mapbase        = 0xffec0000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_CKE1,
+        .scbrr_algo_id  = SCBRR_ALGO_1,
         .type           = PORT_SCIF,
         .irqs           = { 60, 60, 60, 60 },
 };
@@ -68,6 +74,8 @@ static struct platform_device scif2_device = {
 static struct plat_sci_port scif3_platform_data = {
         .mapbase        = 0xffed0000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_CKE1,
+        .scbrr_algo_id  = SCBRR_ALGO_1,
         .type           = PORT_SCIF,
         .irqs           = { 61, 61, 61, 61 },
 };
@@ -83,6 +91,8 @@ static struct platform_device scif3_device = {
 static struct plat_sci_port scif4_platform_data = {
         .mapbase        = 0xffee0000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_CKE1,
+        .scbrr_algo_id  = SCBRR_ALGO_1,
         .type           = PORT_SCIF,
         .irqs           = { 62, 62, 62, 62 },
 };
@@ -98,6 +108,8 @@ static struct platform_device scif4_device = {
 static struct plat_sci_port scif5_platform_data = {
         .mapbase        = 0xffef0000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_CKE1,
+        .scbrr_algo_id  = SCBRR_ALGO_1,
         .type           = PORT_SCIF,
         .irqs           = { 63, 63, 63, 63 },
 };
diff --git a/arch/sh/kernel/cpu/sh4a/setup-sh7786.c b/arch/sh/kernel/cpu/sh4a/setup-sh7786.c
index 0170dbda1d00..1656b8c91faf 100644
--- a/arch/sh/kernel/cpu/sh4a/setup-sh7786.c
+++ b/arch/sh/kernel/cpu/sh4a/setup-sh7786.c
@@ -29,6 +29,8 @@
 static struct plat_sci_port scif0_platform_data = {
         .mapbase        = 0xffea0000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_CKE1,
+        .scbrr_algo_id  = SCBRR_ALGO_1,
         .type           = PORT_SCIF,
         .irqs           = { 40, 41, 43, 42 },
 };
@@ -47,6 +49,8 @@ static struct platform_device scif0_device = {
 static struct plat_sci_port scif1_platform_data = {
         .mapbase        = 0xffeb0000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_CKE1,
+        .scbrr_algo_id  = SCBRR_ALGO_1,
         .type           = PORT_SCIF,
         .irqs           = { 44, 44, 44, 44 },
 };
@@ -62,6 +66,8 @@ static struct platform_device scif1_device = {
 static struct plat_sci_port scif2_platform_data = {
         .mapbase        = 0xffec0000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_CKE1,
+        .scbrr_algo_id  = SCBRR_ALGO_1,
         .type           = PORT_SCIF,
         .irqs           = { 50, 50, 50, 50 },
 };
@@ -77,6 +83,8 @@ static struct platform_device scif2_device = {
 static struct plat_sci_port scif3_platform_data = {
         .mapbase        = 0xffed0000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_CKE1,
+        .scbrr_algo_id  = SCBRR_ALGO_1,
         .type           = PORT_SCIF,
         .irqs           = { 51, 51, 51, 51 },
 };
@@ -92,6 +100,8 @@ static struct platform_device scif3_device = {
 static struct plat_sci_port scif4_platform_data = {
         .mapbase        = 0xffee0000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_CKE1,
+        .scbrr_algo_id  = SCBRR_ALGO_1,
         .type           = PORT_SCIF,
         .irqs           = { 52, 52, 52, 52 },
 };
@@ -107,6 +117,8 @@ static struct platform_device scif4_device = {
 static struct plat_sci_port scif5_platform_data = {
         .mapbase        = 0xffef0000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_CKE1,
+        .scbrr_algo_id  = SCBRR_ALGO_1,
         .type           = PORT_SCIF,
         .irqs           = { 53, 53, 53, 53 },
 };
diff --git a/arch/sh/kernel/cpu/sh4a/setup-shx3.c b/arch/sh/kernel/cpu/sh4a/setup-shx3.c
index 013f0b144489..bb208806dc1a 100644
--- a/arch/sh/kernel/cpu/sh4a/setup-shx3.c
+++ b/arch/sh/kernel/cpu/sh4a/setup-shx3.c
@@ -29,6 +29,8 @@
 static struct plat_sci_port scif0_platform_data = {
         .mapbase        = 0xffc30000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 40, 41, 43, 42 },
 };
@@ -44,6 +46,8 @@ static struct platform_device scif0_device = {
 static struct plat_sci_port scif1_platform_data = {
         .mapbase        = 0xffc40000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 44, 45, 47, 46 },
 };
@@ -59,6 +63,8 @@ static struct platform_device scif1_device = {
 static struct plat_sci_port scif2_platform_data = {
         .mapbase        = 0xffc60000,
         .flags          = UPF_BOOT_AUTOCONF,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 52, 53, 55, 54 },
 };
diff --git a/arch/sh/kernel/cpu/sh5/setup-sh5.c b/arch/sh/kernel/cpu/sh5/setup-sh5.c
index d910666142b1..18419f1de963 100644
--- a/arch/sh/kernel/cpu/sh5/setup-sh5.c
+++ b/arch/sh/kernel/cpu/sh5/setup-sh5.c
@@ -19,6 +19,8 @@
 static struct plat_sci_port scif0_platform_data = {
         .mapbase        = PHYS_PERIPHERAL_BLOCK + 0x01030000,
         .flags          = UPF_BOOT_AUTOCONF | UPF_IOREMAP,
+        .scscr          = SCSCR_RE | SCSCR_TE | SCSCR_REIE,
+        .scbrr_algo_id  = SCBRR_ALGO_2,
         .type           = PORT_SCIF,
         .irqs           = { 39, 40, 42, 0 },
 };
diff --git a/arch/sh/kernel/cpu/shmobile/cpuidle.c b/arch/sh/kernel/cpu/shmobile/cpuidle.c
index 83972aa319c2..c19e2a940e3f 100644
--- a/arch/sh/kernel/cpu/shmobile/cpuidle.c
+++ b/arch/sh/kernel/cpu/shmobile/cpuidle.c
@@ -81,7 +81,6 @@ void sh_mobile_setup_cpuidle(void)
         state->target_residency = 1 * 2;
         state->power_usage = 3;
         state->flags = 0;
-        state->flags |= CPUIDLE_FLAG_SHALLOW;
         state->flags |= CPUIDLE_FLAG_TIME_VALID;
         state->enter = cpuidle_sleep_enter;
 
diff --git a/arch/sh/kernel/cpu/shmobile/pm.c b/arch/sh/kernel/cpu/shmobile/pm.c
index e55968712706..a6f95ae4aae7 100644
--- a/arch/sh/kernel/cpu/shmobile/pm.c
+++ b/arch/sh/kernel/cpu/shmobile/pm.c
@@ -141,7 +141,7 @@ static int sh_pm_enter(suspend_state_t state)
         return 0;
 }
 
-static struct platform_suspend_ops sh_pm_ops = {
+static const struct platform_suspend_ops sh_pm_ops = {
         .enter          = sh_pm_enter,
         .valid          = suspend_valid_only_mem,
 };
diff --git a/arch/sh/mm/cache-sh5.c b/arch/sh/mm/cache-sh5.c
index eb4cc4ec7952..d1bffbcd9d52 100644
--- a/arch/sh/mm/cache-sh5.c
+++ b/arch/sh/mm/cache-sh5.c
@@ -568,7 +568,7 @@ static void sh5_flush_dcache_page(void *page)
 }
 
 /*
- * Flush the range [start,end] of kernel virtual adddress space from
+ * Flush the range [start,end] of kernel virtual address space from
  * the I-cache.  The corresponding range must be purged from the
  * D-cache also because the SH-5 doesn't have cache snooping between
  * the caches.  The addresses will be visible through the superpage
diff --git a/arch/sh/mm/hugetlbpage.c b/arch/sh/mm/hugetlbpage.c
index 9163db3e8d15..d7762349ea48 100644
--- a/arch/sh/mm/hugetlbpage.c
+++ b/arch/sh/mm/hugetlbpage.c
@@ -35,7 +35,7 @@ pte_t *huge_pte_alloc(struct mm_struct *mm,
                 if (pud) {
                         pmd = pmd_alloc(mm, pud, addr);
                         if (pmd)
-                                pte = pte_alloc_map(mm, pmd, addr);
+                                pte = pte_alloc_map(mm, NULL, pmd, addr);
                 }
         }
 
diff --git a/arch/sh/mm/init.c b/arch/sh/mm/init.c
index 3385b28acaac..0d3f912e3334 100644
--- a/arch/sh/mm/init.c
+++ b/arch/sh/mm/init.c
@@ -2,7 +2,7 @@
  * linux/arch/sh/mm/init.c
  *
  *  Copyright (C) 1999  Niibe Yutaka
- *  Copyright (C) 2002 - 2010  Paul Mundt
+ *  Copyright (C) 2002 - 2011  Paul Mundt
  *
  *  Based on linux/arch/i386/mm/init.c:
  *   Copyright (C) 1995  Linus Torvalds
@@ -325,11 +325,17 @@ void __init paging_init(void)
         int nid;
 
         memblock_init();
-
         sh_mv.mv_mem_init();
 
         early_reserve_mem();
 
+        /*
+         * Once the early reservations are out of the way, give the
+         * platforms a chance to kick out some memory.
+         */
+        if (sh_mv.mv_mem_reserve)
+                sh_mv.mv_mem_reserve();
+
         memblock_enforce_memory_limit(memory_limit);
         memblock_analyze();
 
diff --git a/arch/sh/tools/mach-types b/arch/sh/tools/mach-types
index 0e68465e7b50..6dd56c4d0054 100644
--- a/arch/sh/tools/mach-types
+++ b/arch/sh/tools/mach-types
@@ -9,6 +9,7 @@ SE			SH_SOLUTION_ENGINE
 HIGHLANDER              SH_HIGHLANDER
 RTS7751R2D              SH_RTS7751R2D
 RSK                     SH_RSK
+ALPHA_BOARD             SH_ALPHA_BOARD
 
 #
 # List of companion chips / MFDs.
@@ -61,3 +62,5 @@ ESPT			SH_ESPT
 POLARIS                 SH_POLARIS
 KFR2R09                 SH_KFR2R09
 ECOVEC                  SH_ECOVEC
+APSH4A3A                SH_APSH4A3A
+APSH4AD0A               SH_APSH4AD0A
diff --git a/arch/sparc/kernel/module.c b/arch/sparc/kernel/module.c
index ee3c7dde8d9f..8d348c474a2f 100644
--- a/arch/sparc/kernel/module.c
+++ b/arch/sparc/kernel/module.c
@@ -23,17 +23,11 @@
 
 static void *module_map(unsigned long size)
 {
-        struct vm_struct *area;
-
-        size = PAGE_ALIGN(size);
-        if (!size || size > MODULES_LEN)
-                return NULL;
-
-        area = __get_vm_area(size, VM_ALLOC, MODULES_VADDR, MODULES_END);
-        if (!area)
+        if (PAGE_ALIGN(size) > MODULES_LEN)
                 return NULL;
-
-        return __vmalloc_area(area, GFP_KERNEL, PAGE_KERNEL);
+        return __vmalloc_node_range(size, 1, MODULES_VADDR, MODULES_END,
+                                GFP_KERNEL, PAGE_KERNEL, -1,
+                                __builtin_return_address(0));
 }
 
 static char *dot2underscore(char *name)
diff --git a/arch/sparc/kernel/traps_64.c b/arch/sparc/kernel/traps_64.c
index 42ad2ba85010..1e9770936c3b 100644
--- a/arch/sparc/kernel/traps_64.c
+++ b/arch/sparc/kernel/traps_64.c
@@ -622,7 +622,7 @@ static const char CHAFSR_PERR_msg[] =
 static const char CHAFSR_IERR_msg[] =
         "Internal processor error";
 static const char CHAFSR_ISAP_msg[] =
-        "System request parity error on incoming addresss";
+        "System request parity error on incoming address";
 static const char CHAFSR_UCU_msg[] =
         "Uncorrectable E-cache ECC error for ifetch/data";
 static const char CHAFSR_UCC_msg[] =
diff --git a/arch/sparc/mm/generic_32.c b/arch/sparc/mm/generic_32.c
index 5edcac184eaf..e6067b75f11c 100644
--- a/arch/sparc/mm/generic_32.c
+++ b/arch/sparc/mm/generic_32.c
@@ -50,7 +50,7 @@ static inline int io_remap_pmd_range(struct mm_struct *mm, pmd_t * pmd, unsigned
                 end = PGDIR_SIZE;
         offset -= address;
         do {
-                pte_t * pte = pte_alloc_map(mm, pmd, address);
+                pte_t *pte = pte_alloc_map(mm, NULL, pmd, address);
                 if (!pte)
                         return -ENOMEM;
                 io_remap_pte_range(mm, pte, address, end - address, address + offset, prot, space);
diff --git a/arch/sparc/mm/generic_64.c b/arch/sparc/mm/generic_64.c
index 04f2bf4cd571..3cb00dfd4bd6 100644
--- a/arch/sparc/mm/generic_64.c
+++ b/arch/sparc/mm/generic_64.c
@@ -92,7 +92,7 @@ static inline int io_remap_pmd_range(struct mm_struct *mm, pmd_t * pmd, unsigned
                 end = PGDIR_SIZE;
         offset -= address;
         do {
-                pte_t * pte = pte_alloc_map(mm, pmd, address);
+                pte_t *pte = pte_alloc_map(mm, NULL, pmd, address);
                 if (!pte)
                         return -ENOMEM;
                 io_remap_pte_range(mm, pte, address, end - address, address + offset, prot, space);
diff --git a/arch/sparc/mm/hugetlbpage.c b/arch/sparc/mm/hugetlbpage.c
index 5fdddf134caa..f4e97646ce23 100644
--- a/arch/sparc/mm/hugetlbpage.c
+++ b/arch/sparc/mm/hugetlbpage.c
@@ -214,7 +214,7 @@ pte_t *huge_pte_alloc(struct mm_struct *mm,
         if (pud) {
                 pmd = pmd_alloc(mm, pud, addr);
                 if (pmd)
-                        pte = pte_alloc_map(mm, pmd, addr);
+                        pte = pte_alloc_map(mm, NULL, pmd, addr);
         }
         return pte;
 }
diff --git a/arch/um/Kconfig.um b/arch/um/Kconfig.um
index 50d6aa20c353..f8d1d0d47fe6 100644
--- a/arch/um/Kconfig.um
+++ b/arch/um/Kconfig.um
@@ -131,7 +131,7 @@ config NR_CPUS
 
 config HIGHMEM
         bool "Highmem support (EXPERIMENTAL)"
-        depends on !64BIT && EXPERIMENTAL
+        depends on !64BIT && BROKEN
         default n
         help
           This was used to allow UML to run with big amounts of memory.
diff --git a/arch/um/drivers/line.c b/arch/um/drivers/line.c
index 1664cce7b0ac..050e4ddbbb65 100644
--- a/arch/um/drivers/line.c
+++ b/arch/um/drivers/line.c
@@ -821,12 +821,12 @@ void register_winch_irq(int fd, int tty_fd, int pid, struct tty_struct *tty,
 
 static void unregister_winch(struct tty_struct *tty)
 {
-        struct list_head *ele;
+        struct list_head *ele, *next;
         struct winch *winch;
 
         spin_lock(&winch_handler_lock);
 
-        list_for_each(ele, &winch_handlers) {
+        list_for_each_safe(ele, next, &winch_handlers) {
                 winch = list_entry(ele, struct winch, list);
                 if (winch->tty == tty) {
                         free_winch(winch, 1);
diff --git a/arch/um/drivers/mmapper_kern.c b/arch/um/drivers/mmapper_kern.c
index 8501e7d0015c..7e0619c2c2c6 100644
--- a/arch/um/drivers/mmapper_kern.c
+++ b/arch/um/drivers/mmapper_kern.c
@@ -37,13 +37,7 @@ static ssize_t mmapper_write(struct file *file, const char __user *buf,
         if (*ppos > mmapper_size)
                 return -EINVAL;
 
-        if (count > mmapper_size - *ppos)
-                count = mmapper_size - *ppos;
-
-        if (copy_from_user(&v_buf[*ppos], buf, count))
-                return -EFAULT;
-
-        return count;
+        return simple_write_to_buffer(v_buf, mmapper_size, ppos, buf, count);
 }
 
 static long mmapper_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
@@ -137,3 +131,4 @@ module_exit(mmapper_exit);
 
 MODULE_AUTHOR("Greg Lonnon <glonnon@ridgerun.com>");
 MODULE_DESCRIPTION("DSPLinux simulator mmapper driver");
+MODULE_LICENSE("GPL");
diff --git a/arch/um/kernel/skas/mmu.c b/arch/um/kernel/skas/mmu.c
index 3d099f974785..1aee587e9c5d 100644
--- a/arch/um/kernel/skas/mmu.c
+++ b/arch/um/kernel/skas/mmu.c
@@ -31,7 +31,7 @@ static int init_stub_pte(struct mm_struct *mm, unsigned long proc,
         if (!pmd)
                 goto out_pmd;
 
-        pte = pte_alloc_map(mm, pmd, proc);
+        pte = pte_alloc_map(mm, NULL, pmd, proc);
         if (!pte)
                 goto out_pte;
 
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index b6fccb07123e..36ed2e2c896b 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -51,6 +51,7 @@ config X86
         select HAVE_KERNEL_GZIP
         select HAVE_KERNEL_BZIP2
         select HAVE_KERNEL_LZMA
+        select HAVE_KERNEL_XZ
         select HAVE_KERNEL_LZO
         select HAVE_HW_BREAKPOINT
         select HAVE_MIXED_BREAKPOINTS_REGS
@@ -65,6 +66,7 @@ config X86
         select HAVE_SPARSE_IRQ
         select GENERIC_IRQ_PROBE
         select GENERIC_PENDING_IRQ if SMP
+        select USE_GENERIC_SMP_HELPERS if SMP
 
 config INSTRUCTION_DECODER
         def_bool (KPROBES || PERF_EVENTS)
@@ -203,10 +205,6 @@ config HAVE_INTEL_TXT
         def_bool y
         depends on EXPERIMENTAL && DMAR && ACPI
 
-config USE_GENERIC_SMP_HELPERS
-        def_bool y
-        depends on SMP
-
 config X86_32_SMP
         def_bool y
         depends on X86_32 && SMP
@@ -2078,11 +2076,17 @@ config OLPC_OPENFIRMWARE
         bool "Support for OLPC's Open Firmware"
         depends on !X86_64 && !X86_PAE
         default n
+        select OF
         help
           This option adds support for the implementation of Open Firmware
           that is used on the OLPC XO-1 Children's Machine.
           If unsure, say N here.
 
+config OLPC_OPENFIRMWARE_DT
+        bool
+        default y if OLPC_OPENFIRMWARE && PROC_DEVICETREE
+        select OF_PROMTREE
+
 endif # X86_32
 
 config AMD_NB
diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile
index 0c229551eead..09664efb9cee 100644
--- a/arch/x86/boot/compressed/Makefile
+++ b/arch/x86/boot/compressed/Makefile
@@ -4,7 +4,7 @@
 # create a compressed vmlinux image from the original vmlinux
 #
 
-targets := vmlinux.lds vmlinux vmlinux.bin vmlinux.bin.gz vmlinux.bin.bz2 vmlinux.bin.lzma vmlinux.bin.lzo head_$(BITS).o misc.o string.o cmdline.o early_serial_console.o piggy.o
+targets := vmlinux.lds vmlinux vmlinux.bin vmlinux.bin.gz vmlinux.bin.bz2 vmlinux.bin.lzma vmlinux.bin.xz vmlinux.bin.lzo head_$(BITS).o misc.o string.o cmdline.o early_serial_console.o piggy.o
 
 KBUILD_CFLAGS := -m$(BITS) -D__KERNEL__ $(LINUX_INCLUDE) -O2
 KBUILD_CFLAGS += -fno-strict-aliasing -fPIC
@@ -49,12 +49,15 @@ $(obj)/vmlinux.bin.bz2: $(vmlinux.bin.all-y) FORCE
         $(call if_changed,bzip2)
 $(obj)/vmlinux.bin.lzma: $(vmlinux.bin.all-y) FORCE
         $(call if_changed,lzma)
+$(obj)/vmlinux.bin.xz: $(vmlinux.bin.all-y) FORCE
+        $(call if_changed,xzkern)
 $(obj)/vmlinux.bin.lzo: $(vmlinux.bin.all-y) FORCE
         $(call if_changed,lzo)
 
 suffix-$(CONFIG_KERNEL_GZIP)    := gz
 suffix-$(CONFIG_KERNEL_BZIP2)   := bz2
 suffix-$(CONFIG_KERNEL_LZMA)    := lzma
+suffix-$(CONFIG_KERNEL_XZ)      := xz
 suffix-$(CONFIG_KERNEL_LZO)     := lzo
 
 quiet_cmd_mkpiggy = MKPIGGY $@
diff --git a/arch/x86/boot/compressed/misc.c b/arch/x86/boot/compressed/misc.c
index 325c05294fc4..3a19d04cebeb 100644
--- a/arch/x86/boot/compressed/misc.c
+++ b/arch/x86/boot/compressed/misc.c
@@ -139,6 +139,10 @@ static int lines, cols;
 #include "../../../../lib/decompress_unlzma.c"
 #endif
 
+#ifdef CONFIG_KERNEL_XZ
+#include "../../../../lib/decompress_unxz.c"
+#endif
+
 #ifdef CONFIG_KERNEL_LZO
 #include "../../../../lib/decompress_unlzo.c"
 #endif
diff --git a/arch/x86/boot/compressed/mkpiggy.c b/arch/x86/boot/compressed/mkpiggy.c
index 5c228129d175..646aa78ba5fd 100644
--- a/arch/x86/boot/compressed/mkpiggy.c
+++ b/arch/x86/boot/compressed/mkpiggy.c
@@ -74,7 +74,7 @@ int main(int argc, char *argv[])
 
         offs = (olen > ilen) ? olen - ilen : 0;
         offs += olen >> 12;     /* Add 8 bytes for each 32K block */
-        offs += 32*1024 + 18;   /* Add 32K + 18 bytes slack */
+        offs += 64*1024 + 128;  /* Add 64K + 128 bytes slack */
         offs = (offs+4095) & ~4095; /* Round to a 4K boundary */
 
         printf(".section \".rodata..compressed\",\"a\",@progbits\n");
diff --git a/arch/x86/crypto/aesni-intel_asm.S b/arch/x86/crypto/aesni-intel_asm.S
index ff16756a51c1..8fe2a4966b7a 100644
--- a/arch/x86/crypto/aesni-intel_asm.S
+++ b/arch/x86/crypto/aesni-intel_asm.S
@@ -9,6 +9,20 @@
  *            Vinodh Gopal <vinodh.gopal@intel.com>
  *            Kahraman Akdemir
  *
+ * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
+ * interface for 64-bit kernels.
+ *    Authors: Erdinc Ozturk (erdinc.ozturk@intel.com)
+ *             Aidan O'Mahony (aidan.o.mahony@intel.com)
+ *             Adrian Hoban <adrian.hoban@intel.com>
+ *             James Guilford (james.guilford@intel.com)
+ *             Gabriele Paoloni <gabriele.paoloni@intel.com>
+ *             Tadeusz Struk (tadeusz.struk@intel.com)
+ *             Wajdi Feghali (wajdi.k.feghali@intel.com)
+ *    Copyright (c) 2010, Intel Corporation.
+ *
+ * Ported x86_64 version to x86:
+ *    Author: Mathias Krause <minipli@googlemail.com>
+ *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
  * the Free Software Foundation; either version 2 of the License, or
@@ -18,8 +32,62 @@
 #include <linux/linkage.h>
 #include <asm/inst.h>
 
+#ifdef __x86_64__
+.data
+POLY:   .octa 0xC2000000000000000000000000000001
+TWOONE: .octa 0x00000001000000000000000000000001
+
+# order of these constants should not change.
+# more specifically, ALL_F should follow SHIFT_MASK,
+# and ZERO should follow ALL_F
+
+SHUF_MASK:  .octa 0x000102030405060708090A0B0C0D0E0F
+MASK1:      .octa 0x0000000000000000ffffffffffffffff
+MASK2:      .octa 0xffffffffffffffff0000000000000000
+SHIFT_MASK: .octa 0x0f0e0d0c0b0a09080706050403020100
+ALL_F:      .octa 0xffffffffffffffffffffffffffffffff
+ZERO:       .octa 0x00000000000000000000000000000000
+ONE:        .octa 0x00000000000000000000000000000001
+F_MIN_MASK: .octa 0xf1f2f3f4f5f6f7f8f9fafbfcfdfeff0
+dec:        .octa 0x1
+enc:        .octa 0x2
+
+
 .text
 
+
+#define STACK_OFFSET    8*3
+#define HashKey         16*0    // store HashKey <<1 mod poly here
+#define HashKey_2       16*1    // store HashKey^2 <<1 mod poly here
+#define HashKey_3       16*2    // store HashKey^3 <<1 mod poly here
+#define HashKey_4       16*3    // store HashKey^4 <<1 mod poly here
+#define HashKey_k       16*4    // store XOR of High 64 bits and Low 64
+                                // bits of  HashKey <<1 mod poly here
+                                //(for Karatsuba purposes)
+#define HashKey_2_k     16*5    // store XOR of High 64 bits and Low 64
+                                // bits of  HashKey^2 <<1 mod poly here
+                                // (for Karatsuba purposes)
+#define HashKey_3_k     16*6    // store XOR of High 64 bits and Low 64
+                                // bits of  HashKey^3 <<1 mod poly here
+                                // (for Karatsuba purposes)
+#define HashKey_4_k     16*7    // store XOR of High 64 bits and Low 64
+                                // bits of  HashKey^4 <<1 mod poly here
+                                // (for Karatsuba purposes)
+#define VARIABLE_OFFSET 16*8
+
+#define arg1 rdi
+#define arg2 rsi
+#define arg3 rdx
+#define arg4 rcx
+#define arg5 r8
+#define arg6 r9
+#define arg7 STACK_OFFSET+8(%r14)
+#define arg8 STACK_OFFSET+16(%r14)
+#define arg9 STACK_OFFSET+24(%r14)
+#define arg10 STACK_OFFSET+32(%r14)
+#endif
+
+
 #define STATE1  %xmm0
 #define STATE2  %xmm4
 #define STATE3  %xmm5
@@ -32,12 +100,16 @@
 #define IN      IN1
 #define KEY     %xmm2
 #define IV      %xmm3
+
 #define BSWAP_MASK %xmm10
 #define CTR     %xmm11
 #define INC     %xmm12
 
+#ifdef __x86_64__
+#define AREG    %rax
 #define KEYP    %rdi
 #define OUTP    %rsi
+#define UKEYP   OUTP
 #define INP     %rdx
 #define LEN     %rcx
 #define IVP     %r8
@@ -46,6 +118,1588 @@
 #define TKEYP   T1
 #define T2      %r11
 #define TCTR_LOW T2
+#else
+#define AREG    %eax
+#define KEYP    %edi
+#define OUTP    AREG
+#define UKEYP   OUTP
+#define INP     %edx
+#define LEN     %esi
+#define IVP     %ebp
+#define KLEN    %ebx
+#define T1      %ecx
+#define TKEYP   T1
+#endif
+
+
+#ifdef __x86_64__
+/* GHASH_MUL MACRO to implement: Data*HashKey mod (128,127,126,121,0)
+*
+*
+* Input: A and B (128-bits each, bit-reflected)
+* Output: C = A*B*x mod poly, (i.e. >>1 )
+* To compute GH = GH*HashKey mod poly, give HK = HashKey<<1 mod poly as input
+* GH = GH * HK * x mod poly which is equivalent to GH*HashKey mod poly.
+*
+*/
+.macro GHASH_MUL GH HK TMP1 TMP2 TMP3 TMP4 TMP5
+        movdqa    \GH, \TMP1
+        pshufd    $78, \GH, \TMP2
+        pshufd    $78, \HK, \TMP3
+        pxor      \GH, \TMP2            # TMP2 = a1+a0
+        pxor      \HK, \TMP3            # TMP3 = b1+b0
+        PCLMULQDQ 0x11, \HK, \TMP1     # TMP1 = a1*b1
+        PCLMULQDQ 0x00, \HK, \GH       # GH = a0*b0
+        PCLMULQDQ 0x00, \TMP3, \TMP2   # TMP2 = (a0+a1)*(b1+b0)
+        pxor      \GH, \TMP2
+        pxor      \TMP1, \TMP2          # TMP2 = (a0*b0)+(a1*b0)
+        movdqa    \TMP2, \TMP3
+        pslldq    $8, \TMP3             # left shift TMP3 2 DWs
+        psrldq    $8, \TMP2             # right shift TMP2 2 DWs
+        pxor      \TMP3, \GH
+        pxor      \TMP2, \TMP1          # TMP2:GH holds the result of GH*HK
+
+        # first phase of the reduction
+
+        movdqa    \GH, \TMP2
+        movdqa    \GH, \TMP3
+        movdqa    \GH, \TMP4            # copy GH into TMP2,TMP3 and TMP4
+                                        # in in order to perform
+                                        # independent shifts
+        pslld     $31, \TMP2            # packed right shift <<31
+        pslld     $30, \TMP3            # packed right shift <<30
+        pslld     $25, \TMP4            # packed right shift <<25
+        pxor      \TMP3, \TMP2          # xor the shifted versions
+        pxor      \TMP4, \TMP2
+        movdqa    \TMP2, \TMP5
+        psrldq    $4, \TMP5             # right shift TMP5 1 DW
+        pslldq    $12, \TMP2            # left shift TMP2 3 DWs
+        pxor      \TMP2, \GH
+
+        # second phase of the reduction
+
+        movdqa    \GH,\TMP2             # copy GH into TMP2,TMP3 and TMP4
+                                        # in in order to perform
+                                        # independent shifts
+        movdqa    \GH,\TMP3
+        movdqa    \GH,\TMP4
+        psrld     $1,\TMP2              # packed left shift >>1
+        psrld     $2,\TMP3              # packed left shift >>2
+        psrld     $7,\TMP4              # packed left shift >>7
+        pxor      \TMP3,\TMP2           # xor the shifted versions
+        pxor      \TMP4,\TMP2
+        pxor      \TMP5, \TMP2
+        pxor      \TMP2, \GH
+        pxor      \TMP1, \GH            # result is in TMP1
+.endm
+
+/*
+* if a = number of total plaintext bytes
+* b = floor(a/16)
+* num_initial_blocks = b mod 4
+* encrypt the initial num_initial_blocks blocks and apply ghash on
+* the ciphertext
+* %r10, %r11, %r12, %rax, %xmm5, %xmm6, %xmm7, %xmm8, %xmm9 registers
+* are clobbered
+* arg1, %arg2, %arg3, %r14 are used as a pointer only, not modified
+*/
+
+
+.macro INITIAL_BLOCKS_DEC num_initial_blocks TMP1 TMP2 TMP3 TMP4 TMP5 XMM0 XMM1 \
+XMM2 XMM3 XMM4 XMMDst TMP6 TMP7 i i_seq operation
+        mov        arg7, %r10           # %r10 = AAD
+        mov        arg8, %r12           # %r12 = aadLen
+        mov        %r12, %r11
+        pxor       %xmm\i, %xmm\i
+_get_AAD_loop\num_initial_blocks\operation:
+        movd       (%r10), \TMP1
+        pslldq     $12, \TMP1
+        psrldq     $4, %xmm\i
+        pxor       \TMP1, %xmm\i
+        add        $4, %r10
+        sub        $4, %r12
+        jne        _get_AAD_loop\num_initial_blocks\operation
+        cmp        $16, %r11
+        je         _get_AAD_loop2_done\num_initial_blocks\operation
+        mov        $16, %r12
+_get_AAD_loop2\num_initial_blocks\operation:
+        psrldq     $4, %xmm\i
+        sub        $4, %r12
+        cmp        %r11, %r12
+        jne        _get_AAD_loop2\num_initial_blocks\operation
+_get_AAD_loop2_done\num_initial_blocks\operation:
+        movdqa     SHUF_MASK(%rip), %xmm14
+        PSHUFB_XMM   %xmm14, %xmm\i # byte-reflect the AAD data
+
+        xor        %r11, %r11 # initialise the data pointer offset as zero
+
+        # start AES for num_initial_blocks blocks
+
+        mov        %arg5, %rax                      # %rax = *Y0
+        movdqu     (%rax), \XMM0                    # XMM0 = Y0
+        movdqa     SHUF_MASK(%rip), %xmm14
+        PSHUFB_XMM   %xmm14, \XMM0
+
+.if (\i == 5) || (\i == 6) || (\i == 7)
+.irpc index, \i_seq
+        paddd      ONE(%rip), \XMM0                 # INCR Y0
+        movdqa     \XMM0, %xmm\index
+        movdqa     SHUF_MASK(%rip), %xmm14
+        PSHUFB_XMM   %xmm14, %xmm\index      # perform a 16 byte swap
+
+.endr
+.irpc index, \i_seq
+        pxor       16*0(%arg1), %xmm\index
+.endr
+.irpc index, \i_seq
+        movaps 0x10(%rdi), \TMP1
+        AESENC     \TMP1, %xmm\index          # Round 1
+.endr
+.irpc index, \i_seq
+        movaps 0x20(%arg1), \TMP1
+        AESENC     \TMP1, %xmm\index          # Round 2
+.endr
+.irpc index, \i_seq
+        movaps 0x30(%arg1), \TMP1
+        AESENC     \TMP1, %xmm\index          # Round 2
+.endr
+.irpc index, \i_seq
+        movaps 0x40(%arg1), \TMP1
+        AESENC     \TMP1, %xmm\index          # Round 2
+.endr
+.irpc index, \i_seq
+        movaps 0x50(%arg1), \TMP1
+        AESENC     \TMP1, %xmm\index          # Round 2
+.endr
+.irpc index, \i_seq
+        movaps 0x60(%arg1), \TMP1
+        AESENC     \TMP1, %xmm\index          # Round 2
+.endr
+.irpc index, \i_seq
+        movaps 0x70(%arg1), \TMP1
+        AESENC     \TMP1, %xmm\index          # Round 2
+.endr
+.irpc index, \i_seq
+        movaps 0x80(%arg1), \TMP1
+        AESENC     \TMP1, %xmm\index          # Round 2
+.endr
+.irpc index, \i_seq
+        movaps 0x90(%arg1), \TMP1
+        AESENC     \TMP1, %xmm\index          # Round 2
+.endr
+.irpc index, \i_seq
+        movaps 0xa0(%arg1), \TMP1
+        AESENCLAST \TMP1, %xmm\index         # Round 10
+.endr
+.irpc index, \i_seq
+        movdqu     (%arg3 , %r11, 1), \TMP1
+        pxor       \TMP1, %xmm\index
+        movdqu     %xmm\index, (%arg2 , %r11, 1)
+        # write back plaintext/ciphertext for num_initial_blocks
+        add        $16, %r11
+
+        movdqa     \TMP1, %xmm\index
+        movdqa     SHUF_MASK(%rip), %xmm14
+        PSHUFB_XMM         %xmm14, %xmm\index
+
+                # prepare plaintext/ciphertext for GHASH computation
+.endr
+.endif
+        GHASH_MUL  %xmm\i, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
+        # apply GHASH on num_initial_blocks blocks
+
+.if \i == 5
+        pxor       %xmm5, %xmm6
+        GHASH_MUL  %xmm6, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
+        pxor       %xmm6, %xmm7
+        GHASH_MUL  %xmm7, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
+        pxor       %xmm7, %xmm8
+        GHASH_MUL  %xmm8, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
+.elseif \i == 6
+        pxor       %xmm6, %xmm7
+        GHASH_MUL  %xmm7, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
+        pxor       %xmm7, %xmm8
+        GHASH_MUL  %xmm8, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
+.elseif \i == 7
+        pxor       %xmm7, %xmm8
+        GHASH_MUL  %xmm8, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
+.endif
+        cmp        $64, %r13
+        jl      _initial_blocks_done\num_initial_blocks\operation
+        # no need for precomputed values
+/*
+*
+* Precomputations for HashKey parallel with encryption of first 4 blocks.
+* Haskey_i_k holds XORed values of the low and high parts of the Haskey_i
+*/
+        paddd      ONE(%rip), \XMM0              # INCR Y0
+        movdqa     \XMM0, \XMM1
+        movdqa     SHUF_MASK(%rip), %xmm14
+        PSHUFB_XMM  %xmm14, \XMM1        # perform a 16 byte swap
+
+        paddd      ONE(%rip), \XMM0              # INCR Y0
+        movdqa     \XMM0, \XMM2
+        movdqa     SHUF_MASK(%rip), %xmm14
+        PSHUFB_XMM  %xmm14, \XMM2        # perform a 16 byte swap
+
+        paddd      ONE(%rip), \XMM0              # INCR Y0
+        movdqa     \XMM0, \XMM3
+        movdqa     SHUF_MASK(%rip), %xmm14
+        PSHUFB_XMM %xmm14, \XMM3        # perform a 16 byte swap
+
+        paddd      ONE(%rip), \XMM0              # INCR Y0
+        movdqa     \XMM0, \XMM4
+        movdqa     SHUF_MASK(%rip), %xmm14
+        PSHUFB_XMM %xmm14, \XMM4        # perform a 16 byte swap
+
+        pxor       16*0(%arg1), \XMM1
+        pxor       16*0(%arg1), \XMM2
+        pxor       16*0(%arg1), \XMM3
+        pxor       16*0(%arg1), \XMM4
+        movdqa     \TMP3, \TMP5
+        pshufd     $78, \TMP3, \TMP1
+        pxor       \TMP3, \TMP1
+        movdqa     \TMP1, HashKey_k(%rsp)
+        GHASH_MUL  \TMP5, \TMP3, \TMP1, \TMP2, \TMP4, \TMP6, \TMP7
+# TMP5 = HashKey^2<<1 (mod poly)
+        movdqa     \TMP5, HashKey_2(%rsp)
+# HashKey_2 = HashKey^2<<1 (mod poly)
+        pshufd     $78, \TMP5, \TMP1
+        pxor       \TMP5, \TMP1
+        movdqa     \TMP1, HashKey_2_k(%rsp)
+.irpc index, 1234 # do 4 rounds
+        movaps 0x10*\index(%arg1), \TMP1
+        AESENC     \TMP1, \XMM1
+        AESENC     \TMP1, \XMM2
+        AESENC     \TMP1, \XMM3
+        AESENC     \TMP1, \XMM4
+.endr
+        GHASH_MUL  \TMP5, \TMP3, \TMP1, \TMP2, \TMP4, \TMP6, \TMP7
+# TMP5 = HashKey^3<<1 (mod poly)
+        movdqa     \TMP5, HashKey_3(%rsp)
+        pshufd     $78, \TMP5, \TMP1
+        pxor       \TMP5, \TMP1
+        movdqa     \TMP1, HashKey_3_k(%rsp)
+.irpc index, 56789 # do next 5 rounds
+        movaps 0x10*\index(%arg1), \TMP1
+        AESENC     \TMP1, \XMM1
+        AESENC     \TMP1, \XMM2
+        AESENC     \TMP1, \XMM3
+        AESENC     \TMP1, \XMM4
+.endr
+        GHASH_MUL  \TMP5, \TMP3, \TMP1, \TMP2, \TMP4, \TMP6, \TMP7
+# TMP5 = HashKey^3<<1 (mod poly)
+        movdqa     \TMP5, HashKey_4(%rsp)
+        pshufd     $78, \TMP5, \TMP1
+        pxor       \TMP5, \TMP1
+        movdqa     \TMP1, HashKey_4_k(%rsp)
+        movaps 0xa0(%arg1), \TMP2
+        AESENCLAST \TMP2, \XMM1
+        AESENCLAST \TMP2, \XMM2
+        AESENCLAST \TMP2, \XMM3
+        AESENCLAST \TMP2, \XMM4
+        movdqu     16*0(%arg3 , %r11 , 1), \TMP1
+        pxor       \TMP1, \XMM1
+        movdqu     \XMM1, 16*0(%arg2 , %r11 , 1)
+        movdqa     \TMP1, \XMM1
+        movdqu     16*1(%arg3 , %r11 , 1), \TMP1
+        pxor       \TMP1, \XMM2
+        movdqu     \XMM2, 16*1(%arg2 , %r11 , 1)
+        movdqa     \TMP1, \XMM2
+        movdqu     16*2(%arg3 , %r11 , 1), \TMP1
+        pxor       \TMP1, \XMM3
+        movdqu     \XMM3, 16*2(%arg2 , %r11 , 1)
+        movdqa     \TMP1, \XMM3
+        movdqu     16*3(%arg3 , %r11 , 1), \TMP1
+        pxor       \TMP1, \XMM4
+        movdqu     \XMM4, 16*3(%arg2 , %r11 , 1)
+        movdqa     \TMP1, \XMM4
+        add        $64, %r11
+        movdqa     SHUF_MASK(%rip), %xmm14
+        PSHUFB_XMM %xmm14, \XMM1 # perform a 16 byte swap
+        pxor       \XMMDst, \XMM1
+# combine GHASHed value with the corresponding ciphertext
+        movdqa     SHUF_MASK(%rip), %xmm14
+        PSHUFB_XMM %xmm14, \XMM2 # perform a 16 byte swap
+        movdqa     SHUF_MASK(%rip), %xmm14
+        PSHUFB_XMM %xmm14, \XMM3 # perform a 16 byte swap
+        movdqa     SHUF_MASK(%rip), %xmm14
+        PSHUFB_XMM %xmm14, \XMM4 # perform a 16 byte swap
+
+_initial_blocks_done\num_initial_blocks\operation:
+
+.endm
+
+
+/*
+* if a = number of total plaintext bytes
+* b = floor(a/16)
+* num_initial_blocks = b mod 4
+* encrypt the initial num_initial_blocks blocks and apply ghash on
+* the ciphertext
+* %r10, %r11, %r12, %rax, %xmm5, %xmm6, %xmm7, %xmm8, %xmm9 registers
+* are clobbered
+* arg1, %arg2, %arg3, %r14 are used as a pointer only, not modified
+*/
+
+
+.macro INITIAL_BLOCKS_ENC num_initial_blocks TMP1 TMP2 TMP3 TMP4 TMP5 XMM0 XMM1 \
+XMM2 XMM3 XMM4 XMMDst TMP6 TMP7 i i_seq operation
+        mov        arg7, %r10           # %r10 = AAD
+        mov        arg8, %r12           # %r12 = aadLen
+        mov        %r12, %r11
+        pxor       %xmm\i, %xmm\i
+_get_AAD_loop\num_initial_blocks\operation:
+        movd       (%r10), \TMP1
+        pslldq     $12, \TMP1
+        psrldq     $4, %xmm\i
+        pxor       \TMP1, %xmm\i
+        add        $4, %r10
+        sub        $4, %r12
+        jne        _get_AAD_loop\num_initial_blocks\operation
+        cmp        $16, %r11
+        je         _get_AAD_loop2_done\num_initial_blocks\operation
+        mov        $16, %r12
+_get_AAD_loop2\num_initial_blocks\operation:
+        psrldq     $4, %xmm\i
+        sub        $4, %r12
+        cmp        %r11, %r12
+        jne        _get_AAD_loop2\num_initial_blocks\operation
+_get_AAD_loop2_done\num_initial_blocks\operation:
+        movdqa     SHUF_MASK(%rip), %xmm14
+        PSHUFB_XMM   %xmm14, %xmm\i # byte-reflect the AAD data
+
+        xor        %r11, %r11 # initialise the data pointer offset as zero
+
+        # start AES for num_initial_blocks blocks
+
+        mov        %arg5, %rax                      # %rax = *Y0
+        movdqu     (%rax), \XMM0                    # XMM0 = Y0
+        movdqa     SHUF_MASK(%rip), %xmm14
+        PSHUFB_XMM   %xmm14, \XMM0
+
+.if (\i == 5) || (\i == 6) || (\i == 7)
+.irpc index, \i_seq
+        paddd      ONE(%rip), \XMM0                 # INCR Y0
+        movdqa     \XMM0, %xmm\index
+        movdqa     SHUF_MASK(%rip), %xmm14
+        PSHUFB_XMM   %xmm14, %xmm\index      # perform a 16 byte swap
+
+.endr
+.irpc index, \i_seq
+        pxor       16*0(%arg1), %xmm\index
+.endr
+.irpc index, \i_seq
+        movaps 0x10(%rdi), \TMP1
+        AESENC     \TMP1, %xmm\index          # Round 1
+.endr
+.irpc index, \i_seq
+        movaps 0x20(%arg1), \TMP1
+        AESENC     \TMP1, %xmm\index          # Round 2
+.endr
+.irpc index, \i_seq
+        movaps 0x30(%arg1), \TMP1
+        AESENC     \TMP1, %xmm\index          # Round 2
+.endr
+.irpc index, \i_seq
+        movaps 0x40(%arg1), \TMP1
+        AESENC     \TMP1, %xmm\index          # Round 2
+.endr
+.irpc index, \i_seq
+        movaps 0x50(%arg1), \TMP1
+        AESENC     \TMP1, %xmm\index          # Round 2
+.endr
+.irpc index, \i_seq
+        movaps 0x60(%arg1), \TMP1
+        AESENC     \TMP1, %xmm\index          # Round 2
+.endr
+.irpc index, \i_seq
+        movaps 0x70(%arg1), \TMP1
+        AESENC     \TMP1, %xmm\index          # Round 2
+.endr
+.irpc index, \i_seq
+        movaps 0x80(%arg1), \TMP1
+        AESENC     \TMP1, %xmm\index          # Round 2
+.endr
+.irpc index, \i_seq
+        movaps 0x90(%arg1), \TMP1
+        AESENC     \TMP1, %xmm\index          # Round 2
+.endr
+.irpc index, \i_seq
+        movaps 0xa0(%arg1), \TMP1
+        AESENCLAST \TMP1, %xmm\index         # Round 10
+.endr
+.irpc index, \i_seq
+        movdqu     (%arg3 , %r11, 1), \TMP1
+        pxor       \TMP1, %xmm\index
+        movdqu     %xmm\index, (%arg2 , %r11, 1)
+        # write back plaintext/ciphertext for num_initial_blocks
+        add        $16, %r11
+
+        movdqa     SHUF_MASK(%rip), %xmm14
+        PSHUFB_XMM         %xmm14, %xmm\index
+
+                # prepare plaintext/ciphertext for GHASH computation
+.endr
+.endif
+        GHASH_MUL  %xmm\i, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
+        # apply GHASH on num_initial_blocks blocks
+
+.if \i == 5
+        pxor       %xmm5, %xmm6
+        GHASH_MUL  %xmm6, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
+        pxor       %xmm6, %xmm7
+        GHASH_MUL  %xmm7, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
+        pxor       %xmm7, %xmm8
+        GHASH_MUL  %xmm8, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
+.elseif \i == 6
+        pxor       %xmm6, %xmm7
+        GHASH_MUL  %xmm7, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
+        pxor       %xmm7, %xmm8
+        GHASH_MUL  %xmm8, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
+.elseif \i == 7
+        pxor       %xmm7, %xmm8
+        GHASH_MUL  %xmm8, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
+.endif
+        cmp        $64, %r13
+        jl      _initial_blocks_done\num_initial_blocks\operation
+        # no need for precomputed values
+/*
+*
+* Precomputations for HashKey parallel with encryption of first 4 blocks.
+* Haskey_i_k holds XORed values of the low and high parts of the Haskey_i
+*/
+        paddd      ONE(%rip), \XMM0              # INCR Y0
+        movdqa     \XMM0, \XMM1
+        movdqa     SHUF_MASK(%rip), %xmm14
+        PSHUFB_XMM  %xmm14, \XMM1        # perform a 16 byte swap
+
+        paddd      ONE(%rip), \XMM0              # INCR Y0
+        movdqa     \XMM0, \XMM2
+        movdqa     SHUF_MASK(%rip), %xmm14
+        PSHUFB_XMM  %xmm14, \XMM2        # perform a 16 byte swap
+
+        paddd      ONE(%rip), \XMM0              # INCR Y0
+        movdqa     \XMM0, \XMM3
+        movdqa     SHUF_MASK(%rip), %xmm14
+        PSHUFB_XMM %xmm14, \XMM3        # perform a 16 byte swap
+
+        paddd      ONE(%rip), \XMM0              # INCR Y0
+        movdqa     \XMM0, \XMM4
+        movdqa     SHUF_MASK(%rip), %xmm14
+        PSHUFB_XMM %xmm14, \XMM4        # perform a 16 byte swap
+
+        pxor       16*0(%arg1), \XMM1
+        pxor       16*0(%arg1), \XMM2
+        pxor       16*0(%arg1), \XMM3
+        pxor       16*0(%arg1), \XMM4
+        movdqa     \TMP3, \TMP5
+        pshufd     $78, \TMP3, \TMP1
+        pxor       \TMP3, \TMP1
+        movdqa     \TMP1, HashKey_k(%rsp)
+        GHASH_MUL  \TMP5, \TMP3, \TMP1, \TMP2, \TMP4, \TMP6, \TMP7
+# TMP5 = HashKey^2<<1 (mod poly)
+        movdqa     \TMP5, HashKey_2(%rsp)
+# HashKey_2 = HashKey^2<<1 (mod poly)
+        pshufd     $78, \TMP5, \TMP1
+        pxor       \TMP5, \TMP1
+        movdqa     \TMP1, HashKey_2_k(%rsp)
+.irpc index, 1234 # do 4 rounds
+        movaps 0x10*\index(%arg1), \TMP1
+        AESENC     \TMP1, \XMM1
+        AESENC     \TMP1, \XMM2
+        AESENC     \TMP1, \XMM3
+        AESENC     \TMP1, \XMM4
+.endr
+        GHASH_MUL  \TMP5, \TMP3, \TMP1, \TMP2, \TMP4, \TMP6, \TMP7
+# TMP5 = HashKey^3<<1 (mod poly)
+        movdqa     \TMP5, HashKey_3(%rsp)
+        pshufd     $78, \TMP5, \TMP1
+        pxor       \TMP5, \TMP1
+        movdqa     \TMP1, HashKey_3_k(%rsp)
+.irpc index, 56789 # do next 5 rounds
+        movaps 0x10*\index(%arg1), \TMP1
+        AESENC     \TMP1, \XMM1
+        AESENC     \TMP1, \XMM2
+        AESENC     \TMP1, \XMM3
+        AESENC     \TMP1, \XMM4
+.endr
+        GHASH_MUL  \TMP5, \TMP3, \TMP1, \TMP2, \TMP4, \TMP6, \TMP7
+# TMP5 = HashKey^3<<1 (mod poly)
+        movdqa     \TMP5, HashKey_4(%rsp)
+        pshufd     $78, \TMP5, \TMP1
+        pxor       \TMP5, \TMP1
+        movdqa     \TMP1, HashKey_4_k(%rsp)
+        movaps 0xa0(%arg1), \TMP2
+        AESENCLAST \TMP2, \XMM1
+        AESENCLAST \TMP2, \XMM2
+        AESENCLAST \TMP2, \XMM3
+        AESENCLAST \TMP2, \XMM4
+        movdqu     16*0(%arg3 , %r11 , 1), \TMP1
+        pxor       \TMP1, \XMM1
+        movdqu     16*1(%arg3 , %r11 , 1), \TMP1
+        pxor       \TMP1, \XMM2
+        movdqu     16*2(%arg3 , %r11 , 1), \TMP1
+        pxor       \TMP1, \XMM3
+        movdqu     16*3(%arg3 , %r11 , 1), \TMP1
+        pxor       \TMP1, \XMM4
+        movdqu     \XMM1, 16*0(%arg2 , %r11 , 1)
+        movdqu     \XMM2, 16*1(%arg2 , %r11 , 1)
+        movdqu     \XMM3, 16*2(%arg2 , %r11 , 1)
+        movdqu     \XMM4, 16*3(%arg2 , %r11 , 1)
+
+        add        $64, %r11
+        movdqa     SHUF_MASK(%rip), %xmm14
+        PSHUFB_XMM %xmm14, \XMM1 # perform a 16 byte swap
+        pxor       \XMMDst, \XMM1
+# combine GHASHed value with the corresponding ciphertext
+        movdqa     SHUF_MASK(%rip), %xmm14
+        PSHUFB_XMM %xmm14, \XMM2 # perform a 16 byte swap
+        movdqa     SHUF_MASK(%rip), %xmm14
+        PSHUFB_XMM %xmm14, \XMM3 # perform a 16 byte swap
+        movdqa     SHUF_MASK(%rip), %xmm14
+        PSHUFB_XMM %xmm14, \XMM4 # perform a 16 byte swap
+
+_initial_blocks_done\num_initial_blocks\operation:
+
+.endm
+
+/*
+* encrypt 4 blocks at a time
+* ghash the 4 previously encrypted ciphertext blocks
+* arg1, %arg2, %arg3 are used as pointers only, not modified
+* %r11 is the data offset value
+*/
+.macro GHASH_4_ENCRYPT_4_PARALLEL_ENC TMP1 TMP2 TMP3 TMP4 TMP5 \
+TMP6 XMM0 XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 operation
+
+        movdqa    \XMM1, \XMM5
+        movdqa    \XMM2, \XMM6
+        movdqa    \XMM3, \XMM7
+        movdqa    \XMM4, \XMM8
+
+        movdqa    SHUF_MASK(%rip), %xmm15
+        # multiply TMP5 * HashKey using karatsuba
+
+        movdqa    \XMM5, \TMP4
+        pshufd    $78, \XMM5, \TMP6
+        pxor      \XMM5, \TMP6
+        paddd     ONE(%rip), \XMM0              # INCR CNT
+        movdqa    HashKey_4(%rsp), \TMP5
+        PCLMULQDQ 0x11, \TMP5, \TMP4           # TMP4 = a1*b1
+        movdqa    \XMM0, \XMM1
+        paddd     ONE(%rip), \XMM0              # INCR CNT
+        movdqa    \XMM0, \XMM2
+        paddd     ONE(%rip), \XMM0              # INCR CNT
+        movdqa    \XMM0, \XMM3
+        paddd     ONE(%rip), \XMM0              # INCR CNT
+        movdqa    \XMM0, \XMM4
+        PSHUFB_XMM %xmm15, \XMM1        # perform a 16 byte swap
+        PCLMULQDQ 0x00, \TMP5, \XMM5           # XMM5 = a0*b0
+        PSHUFB_XMM %xmm15, \XMM2        # perform a 16 byte swap
+        PSHUFB_XMM %xmm15, \XMM3        # perform a 16 byte swap
+        PSHUFB_XMM %xmm15, \XMM4        # perform a 16 byte swap
+
+        pxor      (%arg1), \XMM1
+        pxor      (%arg1), \XMM2
+        pxor      (%arg1), \XMM3
+        pxor      (%arg1), \XMM4
+        movdqa    HashKey_4_k(%rsp), \TMP5
+        PCLMULQDQ 0x00, \TMP5, \TMP6           # TMP6 = (a1+a0)*(b1+b0)
+        movaps 0x10(%arg1), \TMP1
+        AESENC    \TMP1, \XMM1              # Round 1
+        AESENC    \TMP1, \XMM2
+        AESENC    \TMP1, \XMM3
+        AESENC    \TMP1, \XMM4
+        movaps 0x20(%arg1), \TMP1
+        AESENC    \TMP1, \XMM1              # Round 2
+        AESENC    \TMP1, \XMM2
+        AESENC    \TMP1, \XMM3
+        AESENC    \TMP1, \XMM4
+        movdqa    \XMM6, \TMP1
+        pshufd    $78, \XMM6, \TMP2
+        pxor      \XMM6, \TMP2
+        movdqa    HashKey_3(%rsp), \TMP5
+        PCLMULQDQ 0x11, \TMP5, \TMP1           # TMP1 = a1 * b1
+        movaps 0x30(%arg1), \TMP3
+        AESENC    \TMP3, \XMM1              # Round 3
+        AESENC    \TMP3, \XMM2
+        AESENC    \TMP3, \XMM3
+        AESENC    \TMP3, \XMM4
+        PCLMULQDQ 0x00, \TMP5, \XMM6           # XMM6 = a0*b0
+        movaps 0x40(%arg1), \TMP3
+        AESENC    \TMP3, \XMM1              # Round 4
+        AESENC    \TMP3, \XMM2
+        AESENC    \TMP3, \XMM3
+        AESENC    \TMP3, \XMM4
+        movdqa    HashKey_3_k(%rsp), \TMP5
+        PCLMULQDQ 0x00, \TMP5, \TMP2           # TMP2 = (a1+a0)*(b1+b0)
+        movaps 0x50(%arg1), \TMP3
+        AESENC    \TMP3, \XMM1              # Round 5
+        AESENC    \TMP3, \XMM2
+        AESENC    \TMP3, \XMM3
+        AESENC    \TMP3, \XMM4
+        pxor      \TMP1, \TMP4
+# accumulate the results in TMP4:XMM5, TMP6 holds the middle part
+        pxor      \XMM6, \XMM5
+        pxor      \TMP2, \TMP6
+        movdqa    \XMM7, \TMP1
+        pshufd    $78, \XMM7, \TMP2
+        pxor      \XMM7, \TMP2
+        movdqa    HashKey_2(%rsp ), \TMP5
+
+        # Multiply TMP5 * HashKey using karatsuba
+
+        PCLMULQDQ 0x11, \TMP5, \TMP1           # TMP1 = a1*b1
+        movaps 0x60(%arg1), \TMP3
+        AESENC    \TMP3, \XMM1              # Round 6
+        AESENC    \TMP3, \XMM2
+        AESENC    \TMP3, \XMM3
+        AESENC    \TMP3, \XMM4
+        PCLMULQDQ 0x00, \TMP5, \XMM7           # XMM7 = a0*b0
+        movaps 0x70(%arg1), \TMP3
+        AESENC    \TMP3, \XMM1             # Round 7
+        AESENC    \TMP3, \XMM2
+        AESENC    \TMP3, \XMM3
+        AESENC    \TMP3, \XMM4
+        movdqa    HashKey_2_k(%rsp), \TMP5
+        PCLMULQDQ 0x00, \TMP5, \TMP2           # TMP2 = (a1+a0)*(b1+b0)
+        movaps 0x80(%arg1), \TMP3
+        AESENC    \TMP3, \XMM1             # Round 8
+        AESENC    \TMP3, \XMM2
+        AESENC    \TMP3, \XMM3
+        AESENC    \TMP3, \XMM4
+        pxor      \TMP1, \TMP4
+# accumulate the results in TMP4:XMM5, TMP6 holds the middle part
+        pxor      \XMM7, \XMM5
+        pxor      \TMP2, \TMP6
+
+        # Multiply XMM8 * HashKey
+        # XMM8 and TMP5 hold the values for the two operands
+
+        movdqa    \XMM8, \TMP1
+        pshufd    $78, \XMM8, \TMP2
+        pxor      \XMM8, \TMP2
+        movdqa    HashKey(%rsp), \TMP5
+        PCLMULQDQ 0x11, \TMP5, \TMP1          # TMP1 = a1*b1
+        movaps 0x90(%arg1), \TMP3
+        AESENC    \TMP3, \XMM1            # Round 9
+        AESENC    \TMP3, \XMM2
+        AESENC    \TMP3, \XMM3
+        AESENC    \TMP3, \XMM4
+        PCLMULQDQ 0x00, \TMP5, \XMM8          # XMM8 = a0*b0
+        movaps 0xa0(%arg1), \TMP3
+        AESENCLAST \TMP3, \XMM1           # Round 10
+        AESENCLAST \TMP3, \XMM2
+        AESENCLAST \TMP3, \XMM3
+        AESENCLAST \TMP3, \XMM4
+        movdqa    HashKey_k(%rsp), \TMP5
+        PCLMULQDQ 0x00, \TMP5, \TMP2          # TMP2 = (a1+a0)*(b1+b0)
+        movdqu    (%arg3,%r11,1), \TMP3
+        pxor      \TMP3, \XMM1                 # Ciphertext/Plaintext XOR EK
+        movdqu    16(%arg3,%r11,1), \TMP3
+        pxor      \TMP3, \XMM2                 # Ciphertext/Plaintext XOR EK
+        movdqu    32(%arg3,%r11,1), \TMP3
+        pxor      \TMP3, \XMM3                 # Ciphertext/Plaintext XOR EK
+        movdqu    48(%arg3,%r11,1), \TMP3
+        pxor      \TMP3, \XMM4                 # Ciphertext/Plaintext XOR EK
+        movdqu    \XMM1, (%arg2,%r11,1)        # Write to the ciphertext buffer
+        movdqu    \XMM2, 16(%arg2,%r11,1)      # Write to the ciphertext buffer
+        movdqu    \XMM3, 32(%arg2,%r11,1)      # Write to the ciphertext buffer
+        movdqu    \XMM4, 48(%arg2,%r11,1)      # Write to the ciphertext buffer
+        PSHUFB_XMM %xmm15, \XMM1        # perform a 16 byte swap
+        PSHUFB_XMM %xmm15, \XMM2        # perform a 16 byte swap
+        PSHUFB_XMM %xmm15, \XMM3        # perform a 16 byte swap
+        PSHUFB_XMM %xmm15, \XMM4        # perform a 16 byte swap
+
+        pxor      \TMP4, \TMP1
+        pxor      \XMM8, \XMM5
+        pxor      \TMP6, \TMP2
+        pxor      \TMP1, \TMP2
+        pxor      \XMM5, \TMP2
+        movdqa    \TMP2, \TMP3
+        pslldq    $8, \TMP3                    # left shift TMP3 2 DWs
+        psrldq    $8, \TMP2                    # right shift TMP2 2 DWs
+        pxor      \TMP3, \XMM5
+        pxor      \TMP2, \TMP1    # accumulate the results in TMP1:XMM5
+
+        # first phase of reduction
+
+        movdqa    \XMM5, \TMP2
+        movdqa    \XMM5, \TMP3
+        movdqa    \XMM5, \TMP4
+# move XMM5 into TMP2, TMP3, TMP4 in order to perform shifts independently
+        pslld     $31, \TMP2                   # packed right shift << 31
+        pslld     $30, \TMP3                   # packed right shift << 30
+        pslld     $25, \TMP4                   # packed right shift << 25
+        pxor      \TMP3, \TMP2                 # xor the shifted versions
+        pxor      \TMP4, \TMP2
+        movdqa    \TMP2, \TMP5
+        psrldq    $4, \TMP5                    # right shift T5 1 DW
+        pslldq    $12, \TMP2                   # left shift T2 3 DWs
+        pxor      \TMP2, \XMM5
+
+        # second phase of reduction
+
+        movdqa    \XMM5,\TMP2 # make 3 copies of XMM5 into TMP2, TMP3, TMP4
+        movdqa    \XMM5,\TMP3
+        movdqa    \XMM5,\TMP4
+        psrld     $1, \TMP2                    # packed left shift >>1
+        psrld     $2, \TMP3                    # packed left shift >>2
+        psrld     $7, \TMP4                    # packed left shift >>7
+        pxor      \TMP3,\TMP2                  # xor the shifted versions
+        pxor      \TMP4,\TMP2
+        pxor      \TMP5, \TMP2
+        pxor      \TMP2, \XMM5
+        pxor      \TMP1, \XMM5                 # result is in TMP1
+
+        pxor      \XMM5, \XMM1
+.endm
+
+/*
+* decrypt 4 blocks at a time
+* ghash the 4 previously decrypted ciphertext blocks
+* arg1, %arg2, %arg3 are used as pointers only, not modified
+* %r11 is the data offset value
+*/
+.macro GHASH_4_ENCRYPT_4_PARALLEL_DEC TMP1 TMP2 TMP3 TMP4 TMP5 \
+TMP6 XMM0 XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 operation
+
+        movdqa    \XMM1, \XMM5
+        movdqa    \XMM2, \XMM6
+        movdqa    \XMM3, \XMM7
+        movdqa    \XMM4, \XMM8
+
+        movdqa    SHUF_MASK(%rip), %xmm15
+        # multiply TMP5 * HashKey using karatsuba
+
+        movdqa    \XMM5, \TMP4
+        pshufd    $78, \XMM5, \TMP6
+        pxor      \XMM5, \TMP6
+        paddd     ONE(%rip), \XMM0              # INCR CNT
+        movdqa    HashKey_4(%rsp), \TMP5
+        PCLMULQDQ 0x11, \TMP5, \TMP4           # TMP4 = a1*b1
+        movdqa    \XMM0, \XMM1
+        paddd     ONE(%rip), \XMM0              # INCR CNT
+        movdqa    \XMM0, \XMM2
+        paddd     ONE(%rip), \XMM0              # INCR CNT
+        movdqa    \XMM0, \XMM3
+        paddd     ONE(%rip), \XMM0              # INCR CNT
+        movdqa    \XMM0, \XMM4
+        PSHUFB_XMM %xmm15, \XMM1        # perform a 16 byte swap
+        PCLMULQDQ 0x00, \TMP5, \XMM5           # XMM5 = a0*b0
+        PSHUFB_XMM %xmm15, \XMM2        # perform a 16 byte swap
+        PSHUFB_XMM %xmm15, \XMM3        # perform a 16 byte swap
+        PSHUFB_XMM %xmm15, \XMM4        # perform a 16 byte swap
+
+        pxor      (%arg1), \XMM1
+        pxor      (%arg1), \XMM2
+        pxor      (%arg1), \XMM3
+        pxor      (%arg1), \XMM4
+        movdqa    HashKey_4_k(%rsp), \TMP5
+        PCLMULQDQ 0x00, \TMP5, \TMP6           # TMP6 = (a1+a0)*(b1+b0)
+        movaps 0x10(%arg1), \TMP1
+        AESENC    \TMP1, \XMM1              # Round 1
+        AESENC    \TMP1, \XMM2
+        AESENC    \TMP1, \XMM3
+        AESENC    \TMP1, \XMM4
+        movaps 0x20(%arg1), \TMP1
+        AESENC    \TMP1, \XMM1              # Round 2
+        AESENC    \TMP1, \XMM2
+        AESENC    \TMP1, \XMM3
+        AESENC    \TMP1, \XMM4
+        movdqa    \XMM6, \TMP1
+        pshufd    $78, \XMM6, \TMP2
+        pxor      \XMM6, \TMP2
+        movdqa    HashKey_3(%rsp), \TMP5
+        PCLMULQDQ 0x11, \TMP5, \TMP1           # TMP1 = a1 * b1
+        movaps 0x30(%arg1), \TMP3
+        AESENC    \TMP3, \XMM1              # Round 3
+        AESENC    \TMP3, \XMM2
+        AESENC    \TMP3, \XMM3
+        AESENC    \TMP3, \XMM4
+        PCLMULQDQ 0x00, \TMP5, \XMM6           # XMM6 = a0*b0
+        movaps 0x40(%arg1), \TMP3
+        AESENC    \TMP3, \XMM1              # Round 4
+        AESENC    \TMP3, \XMM2
+        AESENC    \TMP3, \XMM3
+        AESENC    \TMP3, \XMM4
+        movdqa    HashKey_3_k(%rsp), \TMP5
+        PCLMULQDQ 0x00, \TMP5, \TMP2           # TMP2 = (a1+a0)*(b1+b0)
+        movaps 0x50(%arg1), \TMP3
+        AESENC    \TMP3, \XMM1              # Round 5
+        AESENC    \TMP3, \XMM2
+        AESENC    \TMP3, \XMM3
+        AESENC    \TMP3, \XMM4
+        pxor      \TMP1, \TMP4
+# accumulate the results in TMP4:XMM5, TMP6 holds the middle part
+        pxor      \XMM6, \XMM5
+        pxor      \TMP2, \TMP6
+        movdqa    \XMM7, \TMP1
+        pshufd    $78, \XMM7, \TMP2
+        pxor      \XMM7, \TMP2
+        movdqa    HashKey_2(%rsp ), \TMP5
+
+        # Multiply TMP5 * HashKey using karatsuba
+
+        PCLMULQDQ 0x11, \TMP5, \TMP1           # TMP1 = a1*b1
+        movaps 0x60(%arg1), \TMP3
+        AESENC    \TMP3, \XMM1              # Round 6
+        AESENC    \TMP3, \XMM2
+        AESENC    \TMP3, \XMM3
+        AESENC    \TMP3, \XMM4
+        PCLMULQDQ 0x00, \TMP5, \XMM7           # XMM7 = a0*b0
+        movaps 0x70(%arg1), \TMP3
+        AESENC    \TMP3, \XMM1             # Round 7
+        AESENC    \TMP3, \XMM2
+        AESENC    \TMP3, \XMM3
+        AESENC    \TMP3, \XMM4
+        movdqa    HashKey_2_k(%rsp), \TMP5
+        PCLMULQDQ 0x00, \TMP5, \TMP2           # TMP2 = (a1+a0)*(b1+b0)
+        movaps 0x80(%arg1), \TMP3
+        AESENC    \TMP3, \XMM1             # Round 8
+        AESENC    \TMP3, \XMM2
+        AESENC    \TMP3, \XMM3
+        AESENC    \TMP3, \XMM4
+        pxor      \TMP1, \TMP4
+# accumulate the results in TMP4:XMM5, TMP6 holds the middle part
+        pxor      \XMM7, \XMM5
+        pxor      \TMP2, \TMP6
+
+        # Multiply XMM8 * HashKey
+        # XMM8 and TMP5 hold the values for the two operands
+
+        movdqa    \XMM8, \TMP1
+        pshufd    $78, \XMM8, \TMP2
+        pxor      \XMM8, \TMP2
+        movdqa    HashKey(%rsp), \TMP5
+        PCLMULQDQ 0x11, \TMP5, \TMP1          # TMP1 = a1*b1
+        movaps 0x90(%arg1), \TMP3
+        AESENC    \TMP3, \XMM1            # Round 9
+        AESENC    \TMP3, \XMM2
+        AESENC    \TMP3, \XMM3
+        AESENC    \TMP3, \XMM4
+        PCLMULQDQ 0x00, \TMP5, \XMM8          # XMM8 = a0*b0
+        movaps 0xa0(%arg1), \TMP3
+        AESENCLAST \TMP3, \XMM1           # Round 10
+        AESENCLAST \TMP3, \XMM2
+        AESENCLAST \TMP3, \XMM3
+        AESENCLAST \TMP3, \XMM4
+        movdqa    HashKey_k(%rsp), \TMP5
+        PCLMULQDQ 0x00, \TMP5, \TMP2          # TMP2 = (a1+a0)*(b1+b0)
+        movdqu    (%arg3,%r11,1), \TMP3
+        pxor      \TMP3, \XMM1                 # Ciphertext/Plaintext XOR EK
+        movdqu    \XMM1, (%arg2,%r11,1)        # Write to plaintext buffer
+        movdqa    \TMP3, \XMM1
+        movdqu    16(%arg3,%r11,1), \TMP3
+        pxor      \TMP3, \XMM2                 # Ciphertext/Plaintext XOR EK
+        movdqu    \XMM2, 16(%arg2,%r11,1)      # Write to plaintext buffer
+        movdqa    \TMP3, \XMM2
+        movdqu    32(%arg3,%r11,1), \TMP3
+        pxor      \TMP3, \XMM3                 # Ciphertext/Plaintext XOR EK
+        movdqu    \XMM3, 32(%arg2,%r11,1)      # Write to plaintext buffer
+        movdqa    \TMP3, \XMM3
+        movdqu    48(%arg3,%r11,1), \TMP3
+        pxor      \TMP3, \XMM4                 # Ciphertext/Plaintext XOR EK
+        movdqu    \XMM4, 48(%arg2,%r11,1)      # Write to plaintext buffer
+        movdqa    \TMP3, \XMM4
+        PSHUFB_XMM %xmm15, \XMM1        # perform a 16 byte swap
+        PSHUFB_XMM %xmm15, \XMM2        # perform a 16 byte swap
+        PSHUFB_XMM %xmm15, \XMM3        # perform a 16 byte swap
+        PSHUFB_XMM %xmm15, \XMM4        # perform a 16 byte swap
+
+        pxor      \TMP4, \TMP1
+        pxor      \XMM8, \XMM5
+        pxor      \TMP6, \TMP2
+        pxor      \TMP1, \TMP2
+        pxor      \XMM5, \TMP2
+        movdqa    \TMP2, \TMP3
+        pslldq    $8, \TMP3                    # left shift TMP3 2 DWs
+        psrldq    $8, \TMP2                    # right shift TMP2 2 DWs
+        pxor      \TMP3, \XMM5
+        pxor      \TMP2, \TMP1    # accumulate the results in TMP1:XMM5
+
+        # first phase of reduction
+
+        movdqa    \XMM5, \TMP2
+        movdqa    \XMM5, \TMP3
+        movdqa    \XMM5, \TMP4
+# move XMM5 into TMP2, TMP3, TMP4 in order to perform shifts independently
+        pslld     $31, \TMP2                   # packed right shift << 31
+        pslld     $30, \TMP3                   # packed right shift << 30
+        pslld     $25, \TMP4                   # packed right shift << 25
+        pxor      \TMP3, \TMP2                 # xor the shifted versions
+        pxor      \TMP4, \TMP2
+        movdqa    \TMP2, \TMP5
+        psrldq    $4, \TMP5                    # right shift T5 1 DW
+        pslldq    $12, \TMP2                   # left shift T2 3 DWs
+        pxor      \TMP2, \XMM5
+
+        # second phase of reduction
+
+        movdqa    \XMM5,\TMP2 # make 3 copies of XMM5 into TMP2, TMP3, TMP4
+        movdqa    \XMM5,\TMP3
+        movdqa    \XMM5,\TMP4
+        psrld     $1, \TMP2                    # packed left shift >>1
+        psrld     $2, \TMP3                    # packed left shift >>2
+        psrld     $7, \TMP4                    # packed left shift >>7
+        pxor      \TMP3,\TMP2                  # xor the shifted versions
+        pxor      \TMP4,\TMP2
+        pxor      \TMP5, \TMP2
+        pxor      \TMP2, \XMM5
+        pxor      \TMP1, \XMM5                 # result is in TMP1
+
+        pxor      \XMM5, \XMM1
+.endm
+
+/* GHASH the last 4 ciphertext blocks. */
+.macro  GHASH_LAST_4 TMP1 TMP2 TMP3 TMP4 TMP5 TMP6 \
+TMP7 XMM1 XMM2 XMM3 XMM4 XMMDst
+
+        # Multiply TMP6 * HashKey (using Karatsuba)
+
+        movdqa    \XMM1, \TMP6
+        pshufd    $78, \XMM1, \TMP2
+        pxor      \XMM1, \TMP2
+        movdqa    HashKey_4(%rsp), \TMP5
+        PCLMULQDQ 0x11, \TMP5, \TMP6       # TMP6 = a1*b1
+        PCLMULQDQ 0x00, \TMP5, \XMM1       # XMM1 = a0*b0
+        movdqa    HashKey_4_k(%rsp), \TMP4
+        PCLMULQDQ 0x00, \TMP4, \TMP2       # TMP2 = (a1+a0)*(b1+b0)
+        movdqa    \XMM1, \XMMDst
+        movdqa    \TMP2, \XMM1              # result in TMP6, XMMDst, XMM1
+
+        # Multiply TMP1 * HashKey (using Karatsuba)
+
+        movdqa    \XMM2, \TMP1
+        pshufd    $78, \XMM2, \TMP2
+        pxor      \XMM2, \TMP2
+        movdqa    HashKey_3(%rsp), \TMP5
+        PCLMULQDQ 0x11, \TMP5, \TMP1       # TMP1 = a1*b1
+        PCLMULQDQ 0x00, \TMP5, \XMM2       # XMM2 = a0*b0
+        movdqa    HashKey_3_k(%rsp), \TMP4
+        PCLMULQDQ 0x00, \TMP4, \TMP2       # TMP2 = (a1+a0)*(b1+b0)
+        pxor      \TMP1, \TMP6
+        pxor      \XMM2, \XMMDst
+        pxor      \TMP2, \XMM1
+# results accumulated in TMP6, XMMDst, XMM1
+
+        # Multiply TMP1 * HashKey (using Karatsuba)
+
+        movdqa    \XMM3, \TMP1
+        pshufd    $78, \XMM3, \TMP2
+        pxor      \XMM3, \TMP2
+        movdqa    HashKey_2(%rsp), \TMP5
+        PCLMULQDQ 0x11, \TMP5, \TMP1       # TMP1 = a1*b1
+        PCLMULQDQ 0x00, \TMP5, \XMM3       # XMM3 = a0*b0
+        movdqa    HashKey_2_k(%rsp), \TMP4
+        PCLMULQDQ 0x00, \TMP4, \TMP2       # TMP2 = (a1+a0)*(b1+b0)
+        pxor      \TMP1, \TMP6
+        pxor      \XMM3, \XMMDst
+        pxor      \TMP2, \XMM1   # results accumulated in TMP6, XMMDst, XMM1
+
+        # Multiply TMP1 * HashKey (using Karatsuba)
+        movdqa    \XMM4, \TMP1
+        pshufd    $78, \XMM4, \TMP2
+        pxor      \XMM4, \TMP2
+        movdqa    HashKey(%rsp), \TMP5
+        PCLMULQDQ 0x11, \TMP5, \TMP1        # TMP1 = a1*b1
+        PCLMULQDQ 0x00, \TMP5, \XMM4       # XMM4 = a0*b0
+        movdqa    HashKey_k(%rsp), \TMP4
+        PCLMULQDQ 0x00, \TMP4, \TMP2       # TMP2 = (a1+a0)*(b1+b0)
+        pxor      \TMP1, \TMP6
+        pxor      \XMM4, \XMMDst
+        pxor      \XMM1, \TMP2
+        pxor      \TMP6, \TMP2
+        pxor      \XMMDst, \TMP2
+        # middle section of the temp results combined as in karatsuba algorithm
+        movdqa    \TMP2, \TMP4
+        pslldq    $8, \TMP4                 # left shift TMP4 2 DWs
+        psrldq    $8, \TMP2                 # right shift TMP2 2 DWs
+        pxor      \TMP4, \XMMDst
+        pxor      \TMP2, \TMP6
+# TMP6:XMMDst holds the result of the accumulated carry-less multiplications
+        # first phase of the reduction
+        movdqa    \XMMDst, \TMP2
+        movdqa    \XMMDst, \TMP3
+        movdqa    \XMMDst, \TMP4
+# move XMMDst into TMP2, TMP3, TMP4 in order to perform 3 shifts independently
+        pslld     $31, \TMP2                # packed right shifting << 31
+        pslld     $30, \TMP3                # packed right shifting << 30
+        pslld     $25, \TMP4                # packed right shifting << 25
+        pxor      \TMP3, \TMP2              # xor the shifted versions
+        pxor      \TMP4, \TMP2
+        movdqa    \TMP2, \TMP7
+        psrldq    $4, \TMP7                 # right shift TMP7 1 DW
+        pslldq    $12, \TMP2                # left shift TMP2 3 DWs
+        pxor      \TMP2, \XMMDst
+
+        # second phase of the reduction
+        movdqa    \XMMDst, \TMP2
+        # make 3 copies of XMMDst for doing 3 shift operations
+        movdqa    \XMMDst, \TMP3
+        movdqa    \XMMDst, \TMP4
+        psrld     $1, \TMP2                 # packed left shift >> 1
+        psrld     $2, \TMP3                 # packed left shift >> 2
+        psrld     $7, \TMP4                 # packed left shift >> 7
+        pxor      \TMP3, \TMP2              # xor the shifted versions
+        pxor      \TMP4, \TMP2
+        pxor      \TMP7, \TMP2
+        pxor      \TMP2, \XMMDst
+        pxor      \TMP6, \XMMDst            # reduced result is in XMMDst
+.endm
+
+/* Encryption of a single block done*/
+.macro ENCRYPT_SINGLE_BLOCK XMM0 TMP1
+
+        pxor    (%arg1), \XMM0
+        movaps 16(%arg1), \TMP1
+        AESENC  \TMP1, \XMM0
+        movaps 32(%arg1), \TMP1
+        AESENC  \TMP1, \XMM0
+        movaps 48(%arg1), \TMP1
+        AESENC  \TMP1, \XMM0
+        movaps 64(%arg1), \TMP1
+        AESENC  \TMP1, \XMM0
+        movaps 80(%arg1), \TMP1
+        AESENC  \TMP1, \XMM0
+        movaps 96(%arg1), \TMP1
+        AESENC  \TMP1, \XMM0
+        movaps 112(%arg1), \TMP1
+        AESENC  \TMP1, \XMM0
+        movaps 128(%arg1), \TMP1
+        AESENC  \TMP1, \XMM0
+        movaps 144(%arg1), \TMP1
+        AESENC  \TMP1, \XMM0
+        movaps 160(%arg1), \TMP1
+        AESENCLAST      \TMP1, \XMM0
+.endm
+
+
+/*****************************************************************************
+* void aesni_gcm_dec(void *aes_ctx,    // AES Key schedule. Starts on a 16 byte boundary.
+*                   u8 *out,           // Plaintext output. Encrypt in-place is allowed.
+*                   const u8 *in,      // Ciphertext input
+*                   u64 plaintext_len, // Length of data in bytes for decryption.
+*                   u8 *iv,            // Pre-counter block j0: 4 byte salt (from Security Association)
+*                                      // concatenated with 8 byte Initialisation Vector (from IPSec ESP Payload)
+*                                      // concatenated with 0x00000001. 16-byte aligned pointer.
+*                   u8 *hash_subkey,   // H, the Hash sub key input. Data starts on a 16-byte boundary.
+*                   const u8 *aad,     // Additional Authentication Data (AAD)
+*                   u64 aad_len,       // Length of AAD in bytes. With RFC4106 this is going to be 8 or 12 bytes
+*                   u8  *auth_tag,     // Authenticated Tag output. The driver will compare this to the
+*                                      // given authentication tag and only return the plaintext if they match.
+*                   u64 auth_tag_len); // Authenticated Tag Length in bytes. Valid values are 16
+*                                      // (most likely), 12 or 8.
+*
+* Assumptions:
+*
+* keys:
+*       keys are pre-expanded and aligned to 16 bytes. we are using the first
+*       set of 11 keys in the data structure void *aes_ctx
+*
+* iv:
+*       0                   1                   2                   3
+*       0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+*       |                             Salt  (From the SA)               |
+*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+*       |                     Initialization Vector                     |
+*       |         (This is the sequence number from IPSec header)       |
+*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+*       |                              0x1                              |
+*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+*
+*
+*
+* AAD:
+*       AAD padded to 128 bits with 0
+*       for example, assume AAD is a u32 vector
+*
+*       if AAD is 8 bytes:
+*       AAD[3] = {A0, A1};
+*       padded AAD in xmm register = {A1 A0 0 0}
+*
+*       0                   1                   2                   3
+*       0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+*       |                               SPI (A1)                        |
+*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+*       |                     32-bit Sequence Number (A0)               |
+*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+*       |                              0x0                              |
+*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+*
+*                                       AAD Format with 32-bit Sequence Number
+*
+*       if AAD is 12 bytes:
+*       AAD[3] = {A0, A1, A2};
+*       padded AAD in xmm register = {A2 A1 A0 0}
+*
+*       0                   1                   2                   3
+*       0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+*       0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+*       |                               SPI (A2)                        |
+*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+*       |                 64-bit Extended Sequence Number {A1,A0}       |
+*       |                                                               |
+*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+*       |                              0x0                              |
+*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+*
+*                        AAD Format with 64-bit Extended Sequence Number
+*
+* aadLen:
+*       from the definition of the spec, aadLen can only be 8 or 12 bytes.
+*       The code supports 16 too but for other sizes, the code will fail.
+*
+* TLen:
+*       from the definition of the spec, TLen can only be 8, 12 or 16 bytes.
+*       For other sizes, the code will fail.
+*
+* poly = x^128 + x^127 + x^126 + x^121 + 1
+*
+*****************************************************************************/
+
+ENTRY(aesni_gcm_dec)
+        push    %r12
+        push    %r13
+        push    %r14
+        mov     %rsp, %r14
+/*
+* states of %xmm registers %xmm6:%xmm15 not saved
+* all %xmm registers are clobbered
+*/
+        sub     $VARIABLE_OFFSET, %rsp
+        and     $~63, %rsp                        # align rsp to 64 bytes
+        mov     %arg6, %r12
+        movdqu  (%r12), %xmm13                    # %xmm13 = HashKey
+        movdqa  SHUF_MASK(%rip), %xmm2
+        PSHUFB_XMM %xmm2, %xmm13
+
+
+# Precompute HashKey<<1 (mod poly) from the hash key (required for GHASH)
+
+        movdqa  %xmm13, %xmm2
+        psllq   $1, %xmm13
+        psrlq   $63, %xmm2
+        movdqa  %xmm2, %xmm1
+        pslldq  $8, %xmm2
+        psrldq  $8, %xmm1
+        por     %xmm2, %xmm13
+
+        # Reduction
+
+        pshufd  $0x24, %xmm1, %xmm2
+        pcmpeqd TWOONE(%rip), %xmm2
+        pand    POLY(%rip), %xmm2
+        pxor    %xmm2, %xmm13     # %xmm13 holds the HashKey<<1 (mod poly)
+
+
+        # Decrypt first few blocks
+
+        movdqa %xmm13, HashKey(%rsp)           # store HashKey<<1 (mod poly)
+        mov %arg4, %r13    # save the number of bytes of plaintext/ciphertext
+        and $-16, %r13                      # %r13 = %r13 - (%r13 mod 16)
+        mov %r13, %r12
+        and $(3<<4), %r12
+        jz _initial_num_blocks_is_0_decrypt
+        cmp $(2<<4), %r12
+        jb _initial_num_blocks_is_1_decrypt
+        je _initial_num_blocks_is_2_decrypt
+_initial_num_blocks_is_3_decrypt:
+        INITIAL_BLOCKS_DEC 3, %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \
+%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 5, 678, dec
+        sub     $48, %r13
+        jmp     _initial_blocks_decrypted
+_initial_num_blocks_is_2_decrypt:
+        INITIAL_BLOCKS_DEC      2, %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \
+%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 6, 78, dec
+        sub     $32, %r13
+        jmp     _initial_blocks_decrypted
+_initial_num_blocks_is_1_decrypt:
+        INITIAL_BLOCKS_DEC      1, %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \
+%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 7, 8, dec
+        sub     $16, %r13
+        jmp     _initial_blocks_decrypted
+_initial_num_blocks_is_0_decrypt:
+        INITIAL_BLOCKS_DEC      0, %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \
+%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 8, 0, dec
+_initial_blocks_decrypted:
+        cmp     $0, %r13
+        je      _zero_cipher_left_decrypt
+        sub     $64, %r13
+        je      _four_cipher_left_decrypt
+_decrypt_by_4:
+        GHASH_4_ENCRYPT_4_PARALLEL_DEC  %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, \
+%xmm14, %xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, dec
+        add     $64, %r11
+        sub     $64, %r13
+        jne     _decrypt_by_4
+_four_cipher_left_decrypt:
+        GHASH_LAST_4    %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, \
+%xmm15, %xmm1, %xmm2, %xmm3, %xmm4, %xmm8
+_zero_cipher_left_decrypt:
+        mov     %arg4, %r13
+        and     $15, %r13                               # %r13 = arg4 (mod 16)
+        je      _multiple_of_16_bytes_decrypt
+
+        # Handle the last <16 byte block seperately
+
+        paddd ONE(%rip), %xmm0         # increment CNT to get Yn
+        movdqa SHUF_MASK(%rip), %xmm10
+        PSHUFB_XMM %xmm10, %xmm0
+
+        ENCRYPT_SINGLE_BLOCK  %xmm0, %xmm1    # E(K, Yn)
+        sub $16, %r11
+        add %r13, %r11
+        movdqu (%arg3,%r11,1), %xmm1   # recieve the last <16 byte block
+        lea SHIFT_MASK+16(%rip), %r12
+        sub %r13, %r12
+# adjust the shuffle mask pointer to be able to shift 16-%r13 bytes
+# (%r13 is the number of bytes in plaintext mod 16)
+        movdqu (%r12), %xmm2           # get the appropriate shuffle mask
+        PSHUFB_XMM %xmm2, %xmm1            # right shift 16-%r13 butes
+
+        movdqa  %xmm1, %xmm2
+        pxor %xmm1, %xmm0            # Ciphertext XOR E(K, Yn)
+        movdqu ALL_F-SHIFT_MASK(%r12), %xmm1
+        # get the appropriate mask to mask out top 16-%r13 bytes of %xmm0
+        pand %xmm1, %xmm0            # mask out top 16-%r13 bytes of %xmm0
+        pand    %xmm1, %xmm2
+        movdqa SHUF_MASK(%rip), %xmm10
+        PSHUFB_XMM %xmm10 ,%xmm2
+
+        pxor %xmm2, %xmm8
+        GHASH_MUL %xmm8, %xmm13, %xmm9, %xmm10, %xmm11, %xmm5, %xmm6
+                  # GHASH computation for the last <16 byte block
+        sub %r13, %r11
+        add $16, %r11
+
+        # output %r13 bytes
+        MOVQ_R64_XMM    %xmm0, %rax
+        cmp     $8, %r13
+        jle     _less_than_8_bytes_left_decrypt
+        mov     %rax, (%arg2 , %r11, 1)
+        add     $8, %r11
+        psrldq  $8, %xmm0
+        MOVQ_R64_XMM    %xmm0, %rax
+        sub     $8, %r13
+_less_than_8_bytes_left_decrypt:
+        mov     %al,  (%arg2, %r11, 1)
+        add     $1, %r11
+        shr     $8, %rax
+        sub     $1, %r13
+        jne     _less_than_8_bytes_left_decrypt
+_multiple_of_16_bytes_decrypt:
+        mov     arg8, %r12                # %r13 = aadLen (number of bytes)
+        shl     $3, %r12                  # convert into number of bits
+        movd    %r12d, %xmm15             # len(A) in %xmm15
+        shl     $3, %arg4                 # len(C) in bits (*128)
+        MOVQ_R64_XMM    %arg4, %xmm1
+        pslldq  $8, %xmm15                # %xmm15 = len(A)||0x0000000000000000
+        pxor    %xmm1, %xmm15             # %xmm15 = len(A)||len(C)
+        pxor    %xmm15, %xmm8
+        GHASH_MUL       %xmm8, %xmm13, %xmm9, %xmm10, %xmm11, %xmm5, %xmm6
+                 # final GHASH computation
+        movdqa SHUF_MASK(%rip), %xmm10
+        PSHUFB_XMM %xmm10, %xmm8
+
+        mov     %arg5, %rax               # %rax = *Y0
+        movdqu  (%rax), %xmm0             # %xmm0 = Y0
+        ENCRYPT_SINGLE_BLOCK    %xmm0,  %xmm1     # E(K, Y0)
+        pxor    %xmm8, %xmm0
+_return_T_decrypt:
+        mov     arg9, %r10                # %r10 = authTag
+        mov     arg10, %r11               # %r11 = auth_tag_len
+        cmp     $16, %r11
+        je      _T_16_decrypt
+        cmp     $12, %r11
+        je      _T_12_decrypt
+_T_8_decrypt:
+        MOVQ_R64_XMM    %xmm0, %rax
+        mov     %rax, (%r10)
+        jmp     _return_T_done_decrypt
+_T_12_decrypt:
+        MOVQ_R64_XMM    %xmm0, %rax
+        mov     %rax, (%r10)
+        psrldq  $8, %xmm0
+        movd    %xmm0, %eax
+        mov     %eax, 8(%r10)
+        jmp     _return_T_done_decrypt
+_T_16_decrypt:
+        movdqu  %xmm0, (%r10)
+_return_T_done_decrypt:
+        mov     %r14, %rsp
+        pop     %r14
+        pop     %r13
+        pop     %r12
+        ret
+
+
+/*****************************************************************************
+* void aesni_gcm_enc(void *aes_ctx,      // AES Key schedule. Starts on a 16 byte boundary.
+*                    u8 *out,            // Ciphertext output. Encrypt in-place is allowed.
+*                    const u8 *in,       // Plaintext input
+*                    u64 plaintext_len,  // Length of data in bytes for encryption.
+*                    u8 *iv,             // Pre-counter block j0: 4 byte salt (from Security Association)
+*                                        // concatenated with 8 byte Initialisation Vector (from IPSec ESP Payload)
+*                                        // concatenated with 0x00000001. 16-byte aligned pointer.
+*                    u8 *hash_subkey,    // H, the Hash sub key input. Data starts on a 16-byte boundary.
+*                    const u8 *aad,      // Additional Authentication Data (AAD)
+*                    u64 aad_len,        // Length of AAD in bytes. With RFC4106 this is going to be 8 or 12 bytes
+*                    u8 *auth_tag,       // Authenticated Tag output.
+*                    u64 auth_tag_len);  // Authenticated Tag Length in bytes. Valid values are 16 (most likely),
+*                                        // 12 or 8.
+*
+* Assumptions:
+*
+* keys:
+*       keys are pre-expanded and aligned to 16 bytes. we are using the
+*       first set of 11 keys in the data structure void *aes_ctx
+*
+*
+* iv:
+*       0                   1                   2                   3
+*       0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+*       |                             Salt  (From the SA)               |
+*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+*       |                     Initialization Vector                     |
+*       |         (This is the sequence number from IPSec header)       |
+*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+*       |                              0x1                              |
+*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+*
+*
+*
+* AAD:
+*       AAD padded to 128 bits with 0
+*       for example, assume AAD is a u32 vector
+*
+*       if AAD is 8 bytes:
+*       AAD[3] = {A0, A1};
+*       padded AAD in xmm register = {A1 A0 0 0}
+*
+*       0                   1                   2                   3
+*       0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+*       |                               SPI (A1)                        |
+*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+*       |                     32-bit Sequence Number (A0)               |
+*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+*       |                              0x0                              |
+*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+*
+*                                 AAD Format with 32-bit Sequence Number
+*
+*       if AAD is 12 bytes:
+*       AAD[3] = {A0, A1, A2};
+*       padded AAD in xmm register = {A2 A1 A0 0}
+*
+*       0                   1                   2                   3
+*       0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+*       |                               SPI (A2)                        |
+*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+*       |                 64-bit Extended Sequence Number {A1,A0}       |
+*       |                                                               |
+*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+*       |                              0x0                              |
+*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+*
+*                         AAD Format with 64-bit Extended Sequence Number
+*
+* aadLen:
+*       from the definition of the spec, aadLen can only be 8 or 12 bytes.
+*       The code supports 16 too but for other sizes, the code will fail.
+*
+* TLen:
+*       from the definition of the spec, TLen can only be 8, 12 or 16 bytes.
+*       For other sizes, the code will fail.
+*
+* poly = x^128 + x^127 + x^126 + x^121 + 1
+***************************************************************************/
+ENTRY(aesni_gcm_enc)
+        push    %r12
+        push    %r13
+        push    %r14
+        mov     %rsp, %r14
+#
+# states of %xmm registers %xmm6:%xmm15 not saved
+# all %xmm registers are clobbered
+#
+        sub     $VARIABLE_OFFSET, %rsp
+        and     $~63, %rsp
+        mov     %arg6, %r12
+        movdqu  (%r12), %xmm13
+        movdqa  SHUF_MASK(%rip), %xmm2
+        PSHUFB_XMM %xmm2, %xmm13
+
+
+# precompute HashKey<<1 mod poly from the HashKey (required for GHASH)
+
+        movdqa  %xmm13, %xmm2
+        psllq   $1, %xmm13
+        psrlq   $63, %xmm2
+        movdqa  %xmm2, %xmm1
+        pslldq  $8, %xmm2
+        psrldq  $8, %xmm1
+        por     %xmm2, %xmm13
+
+        # reduce HashKey<<1
+
+        pshufd  $0x24, %xmm1, %xmm2
+        pcmpeqd TWOONE(%rip), %xmm2
+        pand    POLY(%rip), %xmm2
+        pxor    %xmm2, %xmm13
+        movdqa  %xmm13, HashKey(%rsp)
+        mov     %arg4, %r13            # %xmm13 holds HashKey<<1 (mod poly)
+        and     $-16, %r13
+        mov     %r13, %r12
+
+        # Encrypt first few blocks
+
+        and     $(3<<4), %r12
+        jz      _initial_num_blocks_is_0_encrypt
+        cmp     $(2<<4), %r12
+        jb      _initial_num_blocks_is_1_encrypt
+        je      _initial_num_blocks_is_2_encrypt
+_initial_num_blocks_is_3_encrypt:
+        INITIAL_BLOCKS_ENC      3, %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \
+%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 5, 678, enc
+        sub     $48, %r13
+        jmp     _initial_blocks_encrypted
+_initial_num_blocks_is_2_encrypt:
+        INITIAL_BLOCKS_ENC      2, %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \
+%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 6, 78, enc
+        sub     $32, %r13
+        jmp     _initial_blocks_encrypted
+_initial_num_blocks_is_1_encrypt:
+        INITIAL_BLOCKS_ENC      1, %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \
+%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 7, 8, enc
+        sub     $16, %r13
+        jmp     _initial_blocks_encrypted
+_initial_num_blocks_is_0_encrypt:
+        INITIAL_BLOCKS_ENC      0, %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \
+%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 8, 0, enc
+_initial_blocks_encrypted:
+
+        # Main loop - Encrypt remaining blocks
+
+        cmp     $0, %r13
+        je      _zero_cipher_left_encrypt
+        sub     $64, %r13
+        je      _four_cipher_left_encrypt
+_encrypt_by_4_encrypt:
+        GHASH_4_ENCRYPT_4_PARALLEL_ENC  %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, \
+%xmm14, %xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, enc
+        add     $64, %r11
+        sub     $64, %r13
+        jne     _encrypt_by_4_encrypt
+_four_cipher_left_encrypt:
+        GHASH_LAST_4    %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, \
+%xmm15, %xmm1, %xmm2, %xmm3, %xmm4, %xmm8
+_zero_cipher_left_encrypt:
+        mov     %arg4, %r13
+        and     $15, %r13                       # %r13 = arg4 (mod 16)
+        je      _multiple_of_16_bytes_encrypt
+
+         # Handle the last <16 Byte block seperately
+        paddd ONE(%rip), %xmm0                # INCR CNT to get Yn
+        movdqa SHUF_MASK(%rip), %xmm10
+        PSHUFB_XMM %xmm10, %xmm0
+
+        ENCRYPT_SINGLE_BLOCK    %xmm0, %xmm1        # Encrypt(K, Yn)
+        sub $16, %r11
+        add %r13, %r11
+        movdqu (%arg3,%r11,1), %xmm1     # receive the last <16 byte blocks
+        lea SHIFT_MASK+16(%rip), %r12
+        sub %r13, %r12
+        # adjust the shuffle mask pointer to be able to shift 16-r13 bytes
+        # (%r13 is the number of bytes in plaintext mod 16)
+        movdqu  (%r12), %xmm2           # get the appropriate shuffle mask
+        PSHUFB_XMM      %xmm2, %xmm1            # shift right 16-r13 byte
+        pxor    %xmm1, %xmm0            # Plaintext XOR Encrypt(K, Yn)
+        movdqu  ALL_F-SHIFT_MASK(%r12), %xmm1
+        # get the appropriate mask to mask out top 16-r13 bytes of xmm0
+        pand    %xmm1, %xmm0            # mask out top 16-r13 bytes of xmm0
+        movdqa SHUF_MASK(%rip), %xmm10
+        PSHUFB_XMM %xmm10,%xmm0
+
+        pxor    %xmm0, %xmm8
+        GHASH_MUL %xmm8, %xmm13, %xmm9, %xmm10, %xmm11, %xmm5, %xmm6
+        # GHASH computation for the last <16 byte block
+        sub     %r13, %r11
+        add     $16, %r11
+        PSHUFB_XMM %xmm10, %xmm1
+
+        # shuffle xmm0 back to output as ciphertext
+
+        # Output %r13 bytes
+        MOVQ_R64_XMM %xmm0, %rax
+        cmp $8, %r13
+        jle _less_than_8_bytes_left_encrypt
+        mov %rax, (%arg2 , %r11, 1)
+        add $8, %r11
+        psrldq $8, %xmm0
+        MOVQ_R64_XMM %xmm0, %rax
+        sub $8, %r13
+_less_than_8_bytes_left_encrypt:
+        mov %al,  (%arg2, %r11, 1)
+        add $1, %r11
+        shr $8, %rax
+        sub $1, %r13
+        jne _less_than_8_bytes_left_encrypt
+_multiple_of_16_bytes_encrypt:
+        mov     arg8, %r12    # %r12 = addLen (number of bytes)
+        shl     $3, %r12
+        movd    %r12d, %xmm15       # len(A) in %xmm15
+        shl     $3, %arg4               # len(C) in bits (*128)
+        MOVQ_R64_XMM    %arg4, %xmm1
+        pslldq  $8, %xmm15          # %xmm15 = len(A)||0x0000000000000000
+        pxor    %xmm1, %xmm15       # %xmm15 = len(A)||len(C)
+        pxor    %xmm15, %xmm8
+        GHASH_MUL       %xmm8, %xmm13, %xmm9, %xmm10, %xmm11, %xmm5, %xmm6
+        # final GHASH computation
+        movdqa SHUF_MASK(%rip), %xmm10
+        PSHUFB_XMM %xmm10, %xmm8         # perform a 16 byte swap
+
+        mov     %arg5, %rax                    # %rax  = *Y0
+        movdqu  (%rax), %xmm0                  # %xmm0 = Y0
+        ENCRYPT_SINGLE_BLOCK    %xmm0, %xmm15         # Encrypt(K, Y0)
+        pxor    %xmm8, %xmm0
+_return_T_encrypt:
+        mov     arg9, %r10                     # %r10 = authTag
+        mov     arg10, %r11                    # %r11 = auth_tag_len
+        cmp     $16, %r11
+        je      _T_16_encrypt
+        cmp     $12, %r11
+        je      _T_12_encrypt
+_T_8_encrypt:
+        MOVQ_R64_XMM    %xmm0, %rax
+        mov     %rax, (%r10)
+        jmp     _return_T_done_encrypt
+_T_12_encrypt:
+        MOVQ_R64_XMM    %xmm0, %rax
+        mov     %rax, (%r10)
+        psrldq  $8, %xmm0
+        movd    %xmm0, %eax
+        mov     %eax, 8(%r10)
+        jmp     _return_T_done_encrypt
+_T_16_encrypt:
+        movdqu  %xmm0, (%r10)
+_return_T_done_encrypt:
+        mov     %r14, %rsp
+        pop     %r14
+        pop     %r13
+        pop     %r12
+        ret
+
+#endif
+
 
 _key_expansion_128:
 _key_expansion_256a:
@@ -55,10 +1709,11 @@ _key_expansion_256a:
         shufps $0b10001100, %xmm0, %xmm4
         pxor %xmm4, %xmm0
         pxor %xmm1, %xmm0
-        movaps %xmm0, (%rcx)
-        add $0x10, %rcx
+        movaps %xmm0, (TKEYP)
+        add $0x10, TKEYP
         ret
 
+.align 4
 _key_expansion_192a:
         pshufd $0b01010101, %xmm1, %xmm1
         shufps $0b00010000, %xmm0, %xmm4
@@ -76,12 +1731,13 @@ _key_expansion_192a:
 
         movaps %xmm0, %xmm1
         shufps $0b01000100, %xmm0, %xmm6
-        movaps %xmm6, (%rcx)
+        movaps %xmm6, (TKEYP)
         shufps $0b01001110, %xmm2, %xmm1
-        movaps %xmm1, 16(%rcx)
-        add $0x20, %rcx
+        movaps %xmm1, 0x10(TKEYP)
+        add $0x20, TKEYP
         ret
 
+.align 4
 _key_expansion_192b:
         pshufd $0b01010101, %xmm1, %xmm1
         shufps $0b00010000, %xmm0, %xmm4
@@ -96,10 +1752,11 @@ _key_expansion_192b:
         pxor %xmm3, %xmm2
         pxor %xmm5, %xmm2
 
-        movaps %xmm0, (%rcx)
-        add $0x10, %rcx
+        movaps %xmm0, (TKEYP)
+        add $0x10, TKEYP
         ret
 
+.align 4
 _key_expansion_256b:
         pshufd $0b10101010, %xmm1, %xmm1
         shufps $0b00010000, %xmm2, %xmm4
@@ -107,8 +1764,8 @@ _key_expansion_256b:
         shufps $0b10001100, %xmm2, %xmm4
         pxor %xmm4, %xmm2
         pxor %xmm1, %xmm2
-        movaps %xmm2, (%rcx)
-        add $0x10, %rcx
+        movaps %xmm2, (TKEYP)
+        add $0x10, TKEYP
         ret
 
 /*
@@ -116,17 +1773,23 @@ _key_expansion_256b:
  *                   unsigned int key_len)
  */
 ENTRY(aesni_set_key)
-        movups (%rsi), %xmm0            # user key (first 16 bytes)
-        movaps %xmm0, (%rdi)
-        lea 0x10(%rdi), %rcx            # key addr
-        movl %edx, 480(%rdi)
+#ifndef __x86_64__
+        pushl KEYP
+        movl 8(%esp), KEYP              # ctx
+        movl 12(%esp), UKEYP            # in_key
+        movl 16(%esp), %edx             # key_len
+#endif
+        movups (UKEYP), %xmm0           # user key (first 16 bytes)
+        movaps %xmm0, (KEYP)
+        lea 0x10(KEYP), TKEYP           # key addr
+        movl %edx, 480(KEYP)
         pxor %xmm4, %xmm4               # xmm4 is assumed 0 in _key_expansion_x
         cmp $24, %dl
         jb .Lenc_key128
         je .Lenc_key192
-        movups 0x10(%rsi), %xmm2        # other user key
-        movaps %xmm2, (%rcx)
-        add $0x10, %rcx
+        movups 0x10(UKEYP), %xmm2       # other user key
+        movaps %xmm2, (TKEYP)
+        add $0x10, TKEYP
         AESKEYGENASSIST 0x1 %xmm2 %xmm1         # round 1
         call _key_expansion_256a
         AESKEYGENASSIST 0x1 %xmm0 %xmm1
@@ -155,7 +1818,7 @@ ENTRY(aesni_set_key)
         call _key_expansion_256a
         jmp .Ldec_key
 .Lenc_key192:
-        movq 0x10(%rsi), %xmm2          # other user key
+        movq 0x10(UKEYP), %xmm2         # other user key
         AESKEYGENASSIST 0x1 %xmm2 %xmm1         # round 1
         call _key_expansion_192a
         AESKEYGENASSIST 0x2 %xmm2 %xmm1         # round 2
@@ -195,33 +1858,47 @@ ENTRY(aesni_set_key)
         AESKEYGENASSIST 0x36 %xmm0 %xmm1        # round 10
         call _key_expansion_128
 .Ldec_key:
-        sub $0x10, %rcx
-        movaps (%rdi), %xmm0
-        movaps (%rcx), %xmm1
-        movaps %xmm0, 240(%rcx)
-        movaps %xmm1, 240(%rdi)
-        add $0x10, %rdi
-        lea 240-16(%rcx), %rsi
+        sub $0x10, TKEYP
+        movaps (KEYP), %xmm0
+        movaps (TKEYP), %xmm1
+        movaps %xmm0, 240(TKEYP)
+        movaps %xmm1, 240(KEYP)
+        add $0x10, KEYP
+        lea 240-16(TKEYP), UKEYP
 .align 4
 .Ldec_key_loop:
-        movaps (%rdi), %xmm0
+        movaps (KEYP), %xmm0
         AESIMC %xmm0 %xmm1
-        movaps %xmm1, (%rsi)
-        add $0x10, %rdi
-        sub $0x10, %rsi
-        cmp %rcx, %rdi
+        movaps %xmm1, (UKEYP)
+        add $0x10, KEYP
+        sub $0x10, UKEYP
+        cmp TKEYP, KEYP
         jb .Ldec_key_loop
-        xor %rax, %rax
+        xor AREG, AREG
+#ifndef __x86_64__
+        popl KEYP
+#endif
         ret
 
 /*
  * void aesni_enc(struct crypto_aes_ctx *ctx, u8 *dst, const u8 *src)
  */
 ENTRY(aesni_enc)
+#ifndef __x86_64__
+        pushl KEYP
+        pushl KLEN
+        movl 12(%esp), KEYP
+        movl 16(%esp), OUTP
+        movl 20(%esp), INP
+#endif
         movl 480(KEYP), KLEN            # key length
         movups (INP), STATE             # input
         call _aesni_enc1
         movups STATE, (OUTP)            # output
+#ifndef __x86_64__
+        popl KLEN
+        popl KEYP
+#endif
         ret
 
 /*
@@ -236,6 +1913,7 @@ ENTRY(aesni_enc)
  *      KEY
  *      TKEYP (T1)
  */
+.align 4
 _aesni_enc1:
         movaps (KEYP), KEY              # key
         mov KEYP, TKEYP
@@ -298,6 +1976,7 @@ _aesni_enc1:
  *      KEY
  *      TKEYP (T1)
  */
+.align 4
 _aesni_enc4:
         movaps (KEYP), KEY              # key
         mov KEYP, TKEYP
@@ -391,11 +2070,22 @@ _aesni_enc4:
  * void aesni_dec (struct crypto_aes_ctx *ctx, u8 *dst, const u8 *src)
  */
 ENTRY(aesni_dec)
+#ifndef __x86_64__
+        pushl KEYP
+        pushl KLEN
+        movl 12(%esp), KEYP
+        movl 16(%esp), OUTP
+        movl 20(%esp), INP
+#endif
         mov 480(KEYP), KLEN             # key length
         add $240, KEYP
         movups (INP), STATE             # input
         call _aesni_dec1
         movups STATE, (OUTP)            #output
+#ifndef __x86_64__
+        popl KLEN
+        popl KEYP
+#endif
         ret
 
 /*
@@ -410,6 +2100,7 @@ ENTRY(aesni_dec)
  *      KEY
  *      TKEYP (T1)
  */
+.align 4
 _aesni_dec1:
         movaps (KEYP), KEY              # key
         mov KEYP, TKEYP
@@ -472,6 +2163,7 @@ _aesni_dec1:
  *      KEY
  *      TKEYP (T1)
  */
+.align 4
 _aesni_dec4:
         movaps (KEYP), KEY              # key
         mov KEYP, TKEYP
@@ -566,6 +2258,15 @@ _aesni_dec4:
  *                    size_t len)
  */
 ENTRY(aesni_ecb_enc)
+#ifndef __x86_64__
+        pushl LEN
+        pushl KEYP
+        pushl KLEN
+        movl 16(%esp), KEYP
+        movl 20(%esp), OUTP
+        movl 24(%esp), INP
+        movl 28(%esp), LEN
+#endif
         test LEN, LEN           # check length
         jz .Lecb_enc_ret
         mov 480(KEYP), KLEN
@@ -602,6 +2303,11 @@ ENTRY(aesni_ecb_enc)
         cmp $16, LEN
         jge .Lecb_enc_loop1
 .Lecb_enc_ret:
+#ifndef __x86_64__
+        popl KLEN
+        popl KEYP
+        popl LEN
+#endif
         ret
 
 /*
@@ -609,6 +2315,15 @@ ENTRY(aesni_ecb_enc)
  *                    size_t len);
  */
 ENTRY(aesni_ecb_dec)
+#ifndef __x86_64__
+        pushl LEN
+        pushl KEYP
+        pushl KLEN
+        movl 16(%esp), KEYP
+        movl 20(%esp), OUTP
+        movl 24(%esp), INP
+        movl 28(%esp), LEN
+#endif
         test LEN, LEN
         jz .Lecb_dec_ret
         mov 480(KEYP), KLEN
@@ -646,6 +2361,11 @@ ENTRY(aesni_ecb_dec)
         cmp $16, LEN
         jge .Lecb_dec_loop1
 .Lecb_dec_ret:
+#ifndef __x86_64__
+        popl KLEN
+        popl KEYP
+        popl LEN
+#endif
         ret
 
 /*
@@ -653,6 +2373,17 @@ ENTRY(aesni_ecb_dec)
  *                    size_t len, u8 *iv)
  */
 ENTRY(aesni_cbc_enc)
+#ifndef __x86_64__
+        pushl IVP
+        pushl LEN
+        pushl KEYP
+        pushl KLEN
+        movl 20(%esp), KEYP
+        movl 24(%esp), OUTP
+        movl 28(%esp), INP
+        movl 32(%esp), LEN
+        movl 36(%esp), IVP
+#endif
         cmp $16, LEN
         jb .Lcbc_enc_ret
         mov 480(KEYP), KLEN
@@ -670,6 +2401,12 @@ ENTRY(aesni_cbc_enc)
         jge .Lcbc_enc_loop
         movups STATE, (IVP)
 .Lcbc_enc_ret:
+#ifndef __x86_64__
+        popl KLEN
+        popl KEYP
+        popl LEN
+        popl IVP
+#endif
         ret
 
 /*
@@ -677,6 +2414,17 @@ ENTRY(aesni_cbc_enc)
  *                    size_t len, u8 *iv)
  */
 ENTRY(aesni_cbc_dec)
+#ifndef __x86_64__
+        pushl IVP
+        pushl LEN
+        pushl KEYP
+        pushl KLEN
+        movl 20(%esp), KEYP
+        movl 24(%esp), OUTP
+        movl 28(%esp), INP
+        movl 32(%esp), LEN
+        movl 36(%esp), IVP
+#endif
         cmp $16, LEN
         jb .Lcbc_dec_just_ret
         mov 480(KEYP), KLEN
@@ -690,16 +2438,30 @@ ENTRY(aesni_cbc_dec)
         movaps IN1, STATE1
         movups 0x10(INP), IN2
         movaps IN2, STATE2
+#ifdef __x86_64__
         movups 0x20(INP), IN3
         movaps IN3, STATE3
         movups 0x30(INP), IN4
         movaps IN4, STATE4
+#else
+        movups 0x20(INP), IN1
+        movaps IN1, STATE3
+        movups 0x30(INP), IN2
+        movaps IN2, STATE4
+#endif
         call _aesni_dec4
         pxor IV, STATE1
+#ifdef __x86_64__
         pxor IN1, STATE2
         pxor IN2, STATE3
         pxor IN3, STATE4
         movaps IN4, IV
+#else
+        pxor (INP), STATE2
+        pxor 0x10(INP), STATE3
+        pxor IN1, STATE4
+        movaps IN2, IV
+#endif
         movups STATE1, (OUTP)
         movups STATE2, 0x10(OUTP)
         movups STATE3, 0x20(OUTP)
@@ -727,8 +2489,15 @@ ENTRY(aesni_cbc_dec)
 .Lcbc_dec_ret:
         movups IV, (IVP)
 .Lcbc_dec_just_ret:
+#ifndef __x86_64__
+        popl KLEN
+        popl KEYP
+        popl LEN
+        popl IVP
+#endif
         ret
 
+#ifdef __x86_64__
 .align 16
 .Lbswap_mask:
         .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
@@ -744,6 +2513,7 @@ ENTRY(aesni_cbc_dec)
  *      INC:    == 1, in little endian
  *      BSWAP_MASK == endian swapping mask
  */
+.align 4
 _aesni_inc_init:
         movaps .Lbswap_mask, BSWAP_MASK
         movaps IV, CTR
@@ -768,6 +2538,7 @@ _aesni_inc_init:
  *      CTR:    == output IV, in little endian
  *      TCTR_LOW: == lower qword of CTR
  */
+.align 4
 _aesni_inc:
         paddq INC, CTR
         add $1, TCTR_LOW
@@ -839,3 +2610,4 @@ ENTRY(aesni_ctr_enc)
         movups IV, (IVP)
 .Lctr_enc_just_ret:
         ret
+#endif
diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c
index 2cb3dcc4490a..e1e60c7d5813 100644
--- a/arch/x86/crypto/aesni-intel_glue.c
+++ b/arch/x86/crypto/aesni-intel_glue.c
@@ -5,6 +5,14 @@
  * Copyright (C) 2008, Intel Corp.
  *    Author: Huang Ying <ying.huang@intel.com>
  *
+ * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
+ * interface for 64-bit kernels.
+ *    Authors: Adrian Hoban <adrian.hoban@intel.com>
+ *             Gabriele Paoloni <gabriele.paoloni@intel.com>
+ *             Tadeusz Struk (tadeusz.struk@intel.com)
+ *             Aidan O'Mahony (aidan.o.mahony@intel.com)
+ *    Copyright (c) 2010, Intel Corporation.
+ *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
  * the Free Software Foundation; either version 2 of the License, or
@@ -21,6 +29,10 @@
 #include <crypto/ctr.h>
 #include <asm/i387.h>
 #include <asm/aes.h>
+#include <crypto/scatterwalk.h>
+#include <crypto/internal/aead.h>
+#include <linux/workqueue.h>
+#include <linux/spinlock.h>
 
 #if defined(CONFIG_CRYPTO_CTR) || defined(CONFIG_CRYPTO_CTR_MODULE)
 #define HAS_CTR
@@ -42,8 +54,31 @@ struct async_aes_ctx {
         struct cryptd_ablkcipher *cryptd_tfm;
 };
 
-#define AESNI_ALIGN     16
+/* This data is stored at the end of the crypto_tfm struct.
+ * It's a type of per "session" data storage location.
+ * This needs to be 16 byte aligned.
+ */
+struct aesni_rfc4106_gcm_ctx {
+        u8 hash_subkey[16];
+        struct crypto_aes_ctx aes_key_expanded;
+        u8 nonce[4];
+        struct cryptd_aead *cryptd_tfm;
+};
+
+struct aesni_gcm_set_hash_subkey_result {
+        int err;
+        struct completion completion;
+};
+
+struct aesni_hash_subkey_req_data {
+        u8 iv[16];
+        struct aesni_gcm_set_hash_subkey_result result;
+        struct scatterlist sg;
+};
+
+#define AESNI_ALIGN     (16)
 #define AES_BLOCK_MASK  (~(AES_BLOCK_SIZE-1))
+#define RFC4106_HASH_SUBKEY_SIZE 16
 
 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
                              unsigned int key_len);
@@ -59,9 +94,62 @@ asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
                               const u8 *in, unsigned int len, u8 *iv);
 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
                               const u8 *in, unsigned int len, u8 *iv);
+#ifdef CONFIG_X86_64
 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out,
                               const u8 *in, unsigned int len, u8 *iv);
 
+/* asmlinkage void aesni_gcm_enc()
+ * void *ctx,  AES Key schedule. Starts on a 16 byte boundary.
+ * u8 *out, Ciphertext output. Encrypt in-place is allowed.
+ * const u8 *in, Plaintext input
+ * unsigned long plaintext_len, Length of data in bytes for encryption.
+ * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association)
+ *         concatenated with 8 byte Initialisation Vector (from IPSec ESP
+ *         Payload) concatenated with 0x00000001. 16-byte aligned pointer.
+ * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
+ * const u8 *aad, Additional Authentication Data (AAD)
+ * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this
+ *          is going to be 8 or 12 bytes
+ * u8 *auth_tag, Authenticated Tag output.
+ * unsigned long auth_tag_len), Authenticated Tag Length in bytes.
+ *          Valid values are 16 (most likely), 12 or 8.
+ */
+asmlinkage void aesni_gcm_enc(void *ctx, u8 *out,
+                        const u8 *in, unsigned long plaintext_len, u8 *iv,
+                        u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
+                        u8 *auth_tag, unsigned long auth_tag_len);
+
+/* asmlinkage void aesni_gcm_dec()
+ * void *ctx, AES Key schedule. Starts on a 16 byte boundary.
+ * u8 *out, Plaintext output. Decrypt in-place is allowed.
+ * const u8 *in, Ciphertext input
+ * unsigned long ciphertext_len, Length of data in bytes for decryption.
+ * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association)
+ *         concatenated with 8 byte Initialisation Vector (from IPSec ESP
+ *         Payload) concatenated with 0x00000001. 16-byte aligned pointer.
+ * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
+ * const u8 *aad, Additional Authentication Data (AAD)
+ * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this is going
+ * to be 8 or 12 bytes
+ * u8 *auth_tag, Authenticated Tag output.
+ * unsigned long auth_tag_len) Authenticated Tag Length in bytes.
+ * Valid values are 16 (most likely), 12 or 8.
+ */
+asmlinkage void aesni_gcm_dec(void *ctx, u8 *out,
+                        const u8 *in, unsigned long ciphertext_len, u8 *iv,
+                        u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
+                        u8 *auth_tag, unsigned long auth_tag_len);
+
+static inline struct
+aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm)
+{
+        return
+                (struct aesni_rfc4106_gcm_ctx *)
+                PTR_ALIGN((u8 *)
+                crypto_tfm_ctx(crypto_aead_tfm(tfm)), AESNI_ALIGN);
+}
+#endif
+
 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx)
 {
         unsigned long addr = (unsigned long)raw_ctx;
@@ -324,6 +412,7 @@ static struct crypto_alg blk_cbc_alg = {
         },
 };
 
+#ifdef CONFIG_X86_64
 static void ctr_crypt_final(struct crypto_aes_ctx *ctx,
                             struct blkcipher_walk *walk)
 {
@@ -389,6 +478,7 @@ static struct crypto_alg blk_ctr_alg = {
                 },
         },
 };
+#endif
 
 static int ablk_set_key(struct crypto_ablkcipher *tfm, const u8 *key,
                         unsigned int key_len)
@@ -536,6 +626,7 @@ static struct crypto_alg ablk_cbc_alg = {
         },
 };
 
+#ifdef CONFIG_X86_64
 static int ablk_ctr_init(struct crypto_tfm *tfm)
 {
         struct cryptd_ablkcipher *cryptd_tfm;
@@ -612,6 +703,7 @@ static struct crypto_alg ablk_rfc3686_ctr_alg = {
         },
 };
 #endif
+#endif
 
 #ifdef HAS_LRW
 static int ablk_lrw_init(struct crypto_tfm *tfm)
@@ -730,6 +822,424 @@ static struct crypto_alg ablk_xts_alg = {
 };
 #endif
 
+#ifdef CONFIG_X86_64
+static int rfc4106_init(struct crypto_tfm *tfm)
+{
+        struct cryptd_aead *cryptd_tfm;
+        struct aesni_rfc4106_gcm_ctx *ctx = (struct aesni_rfc4106_gcm_ctx *)
+                PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN);
+        cryptd_tfm = cryptd_alloc_aead("__driver-gcm-aes-aesni", 0, 0);
+        if (IS_ERR(cryptd_tfm))
+                return PTR_ERR(cryptd_tfm);
+        ctx->cryptd_tfm = cryptd_tfm;
+        tfm->crt_aead.reqsize = sizeof(struct aead_request)
+                + crypto_aead_reqsize(&cryptd_tfm->base);
+        return 0;
+}
+
+static void rfc4106_exit(struct crypto_tfm *tfm)
+{
+        struct aesni_rfc4106_gcm_ctx *ctx =
+                (struct aesni_rfc4106_gcm_ctx *)
+                PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN);
+        if (!IS_ERR(ctx->cryptd_tfm))
+                cryptd_free_aead(ctx->cryptd_tfm);
+        return;
+}
+
+static void
+rfc4106_set_hash_subkey_done(struct crypto_async_request *req, int err)
+{
+        struct aesni_gcm_set_hash_subkey_result *result = req->data;
+
+        if (err == -EINPROGRESS)
+                return;
+        result->err = err;
+        complete(&result->completion);
+}
+
+static int
+rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len)
+{
+        struct crypto_ablkcipher *ctr_tfm;
+        struct ablkcipher_request *req;
+        int ret = -EINVAL;
+        struct aesni_hash_subkey_req_data *req_data;
+
+        ctr_tfm = crypto_alloc_ablkcipher("ctr(aes)", 0, 0);
+        if (IS_ERR(ctr_tfm))
+                return PTR_ERR(ctr_tfm);
+
+        crypto_ablkcipher_clear_flags(ctr_tfm, ~0);
+
+        ret = crypto_ablkcipher_setkey(ctr_tfm, key, key_len);
+        if (ret) {
+                crypto_free_ablkcipher(ctr_tfm);
+                return ret;
+        }
+
+        req = ablkcipher_request_alloc(ctr_tfm, GFP_KERNEL);
+        if (!req) {
+                crypto_free_ablkcipher(ctr_tfm);
+                return -EINVAL;
+        }
+
+        req_data = kmalloc(sizeof(*req_data), GFP_KERNEL);
+        if (!req_data) {
+                crypto_free_ablkcipher(ctr_tfm);
+                return -ENOMEM;
+        }
+        memset(req_data->iv, 0, sizeof(req_data->iv));
+
+        /* Clear the data in the hash sub key container to zero.*/
+        /* We want to cipher all zeros to create the hash sub key. */
+        memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE);
+
+        init_completion(&req_data->result.completion);
+        sg_init_one(&req_data->sg, hash_subkey, RFC4106_HASH_SUBKEY_SIZE);
+        ablkcipher_request_set_tfm(req, ctr_tfm);
+        ablkcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP |
+                                        CRYPTO_TFM_REQ_MAY_BACKLOG,
+                                        rfc4106_set_hash_subkey_done,
+                                        &req_data->result);
+
+        ablkcipher_request_set_crypt(req, &req_data->sg,
+                &req_data->sg, RFC4106_HASH_SUBKEY_SIZE, req_data->iv);
+
+        ret = crypto_ablkcipher_encrypt(req);
+        if (ret == -EINPROGRESS || ret == -EBUSY) {
+                ret = wait_for_completion_interruptible
+                        (&req_data->result.completion);
+                if (!ret)
+                        ret = req_data->result.err;
+        }
+        ablkcipher_request_free(req);
+        kfree(req_data);
+        crypto_free_ablkcipher(ctr_tfm);
+        return ret;
+}
+
+static int rfc4106_set_key(struct crypto_aead *parent, const u8 *key,
+                                                   unsigned int key_len)
+{
+        int ret = 0;
+        struct crypto_tfm *tfm = crypto_aead_tfm(parent);
+        struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent);
+        u8 *new_key_mem = NULL;
+
+        if (key_len < 4) {
+                crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
+                return -EINVAL;
+        }
+        /*Account for 4 byte nonce at the end.*/
+        key_len -= 4;
+        if (key_len != AES_KEYSIZE_128) {
+                crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
+                return -EINVAL;
+        }
+
+        memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce));
+        /*This must be on a 16 byte boundary!*/
+        if ((unsigned long)(&(ctx->aes_key_expanded.key_enc[0])) % AESNI_ALIGN)
+                return -EINVAL;
+
+        if ((unsigned long)key % AESNI_ALIGN) {
+                /*key is not aligned: use an auxuliar aligned pointer*/
+                new_key_mem = kmalloc(key_len+AESNI_ALIGN, GFP_KERNEL);
+                if (!new_key_mem)
+                        return -ENOMEM;
+
+                new_key_mem = PTR_ALIGN(new_key_mem, AESNI_ALIGN);
+                memcpy(new_key_mem, key, key_len);
+                key = new_key_mem;
+        }
+
+        if (!irq_fpu_usable())
+                ret = crypto_aes_expand_key(&(ctx->aes_key_expanded),
+                key, key_len);
+        else {
+                kernel_fpu_begin();
+                ret = aesni_set_key(&(ctx->aes_key_expanded), key, key_len);
+                kernel_fpu_end();
+        }
+        /*This must be on a 16 byte boundary!*/
+        if ((unsigned long)(&(ctx->hash_subkey[0])) % AESNI_ALIGN) {
+                ret = -EINVAL;
+                goto exit;
+        }
+        ret = rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
+exit:
+        kfree(new_key_mem);
+        return ret;
+}
+
+/* This is the Integrity Check Value (aka the authentication tag length and can
+ * be 8, 12 or 16 bytes long. */
+static int rfc4106_set_authsize(struct crypto_aead *parent,
+                                unsigned int authsize)
+{
+        struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent);
+        struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
+
+        switch (authsize) {
+        case 8:
+        case 12:
+        case 16:
+                break;
+        default:
+                return -EINVAL;
+        }
+        crypto_aead_crt(parent)->authsize = authsize;
+        crypto_aead_crt(cryptd_child)->authsize = authsize;
+        return 0;
+}
+
+static int rfc4106_encrypt(struct aead_request *req)
+{
+        int ret;
+        struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+        struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
+        struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
+
+        if (!irq_fpu_usable()) {
+                struct aead_request *cryptd_req =
+                        (struct aead_request *) aead_request_ctx(req);
+                memcpy(cryptd_req, req, sizeof(*req));
+                aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
+                return crypto_aead_encrypt(cryptd_req);
+        } else {
+                kernel_fpu_begin();
+                ret = cryptd_child->base.crt_aead.encrypt(req);
+                kernel_fpu_end();
+                return ret;
+        }
+}
+
+static int rfc4106_decrypt(struct aead_request *req)
+{
+        int ret;
+        struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+        struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
+        struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
+
+        if (!irq_fpu_usable()) {
+                struct aead_request *cryptd_req =
+                        (struct aead_request *) aead_request_ctx(req);
+                memcpy(cryptd_req, req, sizeof(*req));
+                aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
+                return crypto_aead_decrypt(cryptd_req);
+        } else {
+                kernel_fpu_begin();
+                ret = cryptd_child->base.crt_aead.decrypt(req);
+                kernel_fpu_end();
+                return ret;
+        }
+}
+
+static struct crypto_alg rfc4106_alg = {
+        .cra_name = "rfc4106(gcm(aes))",
+        .cra_driver_name = "rfc4106-gcm-aesni",
+        .cra_priority = 400,
+        .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
+        .cra_blocksize = 1,
+        .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx) + AESNI_ALIGN,
+        .cra_alignmask = 0,
+        .cra_type = &crypto_nivaead_type,
+        .cra_module = THIS_MODULE,
+        .cra_list = LIST_HEAD_INIT(rfc4106_alg.cra_list),
+        .cra_init = rfc4106_init,
+        .cra_exit = rfc4106_exit,
+        .cra_u = {
+                .aead = {
+                        .setkey = rfc4106_set_key,
+                        .setauthsize = rfc4106_set_authsize,
+                        .encrypt = rfc4106_encrypt,
+                        .decrypt = rfc4106_decrypt,
+                        .geniv = "seqiv",
+                        .ivsize = 8,
+                        .maxauthsize = 16,
+                },
+        },
+};
+
+static int __driver_rfc4106_encrypt(struct aead_request *req)
+{
+        u8 one_entry_in_sg = 0;
+        u8 *src, *dst, *assoc;
+        __be32 counter = cpu_to_be32(1);
+        struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+        struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
+        void *aes_ctx = &(ctx->aes_key_expanded);
+        unsigned long auth_tag_len = crypto_aead_authsize(tfm);
+        u8 iv_tab[16+AESNI_ALIGN];
+        u8* iv = (u8 *) PTR_ALIGN((u8 *)iv_tab, AESNI_ALIGN);
+        struct scatter_walk src_sg_walk;
+        struct scatter_walk assoc_sg_walk;
+        struct scatter_walk dst_sg_walk;
+        unsigned int i;
+
+        /* Assuming we are supporting rfc4106 64-bit extended */
+        /* sequence numbers We need to have the AAD length equal */
+        /* to 8 or 12 bytes */
+        if (unlikely(req->assoclen != 8 && req->assoclen != 12))
+                return -EINVAL;
+        /* IV below built */
+        for (i = 0; i < 4; i++)
+                *(iv+i) = ctx->nonce[i];
+        for (i = 0; i < 8; i++)
+                *(iv+4+i) = req->iv[i];
+        *((__be32 *)(iv+12)) = counter;
+
+        if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) {
+                one_entry_in_sg = 1;
+                scatterwalk_start(&src_sg_walk, req->src);
+                scatterwalk_start(&assoc_sg_walk, req->assoc);
+                src = scatterwalk_map(&src_sg_walk, 0);
+                assoc = scatterwalk_map(&assoc_sg_walk, 0);
+                dst = src;
+                if (unlikely(req->src != req->dst)) {
+                        scatterwalk_start(&dst_sg_walk, req->dst);
+                        dst = scatterwalk_map(&dst_sg_walk, 0);
+                }
+
+        } else {
+                /* Allocate memory for src, dst, assoc */
+                src = kmalloc(req->cryptlen + auth_tag_len + req->assoclen,
+                        GFP_ATOMIC);
+                if (unlikely(!src))
+                        return -ENOMEM;
+                assoc = (src + req->cryptlen + auth_tag_len);
+                scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0);
+                scatterwalk_map_and_copy(assoc, req->assoc, 0,
+                                        req->assoclen, 0);
+                dst = src;
+        }
+
+        aesni_gcm_enc(aes_ctx, dst, src, (unsigned long)req->cryptlen, iv,
+                ctx->hash_subkey, assoc, (unsigned long)req->assoclen, dst
+                + ((unsigned long)req->cryptlen), auth_tag_len);
+
+        /* The authTag (aka the Integrity Check Value) needs to be written
+         * back to the packet. */
+        if (one_entry_in_sg) {
+                if (unlikely(req->src != req->dst)) {
+                        scatterwalk_unmap(dst, 0);
+                        scatterwalk_done(&dst_sg_walk, 0, 0);
+                }
+                scatterwalk_unmap(src, 0);
+                scatterwalk_unmap(assoc, 0);
+                scatterwalk_done(&src_sg_walk, 0, 0);
+                scatterwalk_done(&assoc_sg_walk, 0, 0);
+        } else {
+                scatterwalk_map_and_copy(dst, req->dst, 0,
+                        req->cryptlen + auth_tag_len, 1);
+                kfree(src);
+        }
+        return 0;
+}
+
+static int __driver_rfc4106_decrypt(struct aead_request *req)
+{
+        u8 one_entry_in_sg = 0;
+        u8 *src, *dst, *assoc;
+        unsigned long tempCipherLen = 0;
+        __be32 counter = cpu_to_be32(1);
+        int retval = 0;
+        struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+        struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
+        void *aes_ctx = &(ctx->aes_key_expanded);
+        unsigned long auth_tag_len = crypto_aead_authsize(tfm);
+        u8 iv_and_authTag[32+AESNI_ALIGN];
+        u8 *iv = (u8 *) PTR_ALIGN((u8 *)iv_and_authTag, AESNI_ALIGN);
+        u8 *authTag = iv + 16;
+        struct scatter_walk src_sg_walk;
+        struct scatter_walk assoc_sg_walk;
+        struct scatter_walk dst_sg_walk;
+        unsigned int i;
+
+        if (unlikely((req->cryptlen < auth_tag_len) ||
+                (req->assoclen != 8 && req->assoclen != 12)))
+                return -EINVAL;
+        /* Assuming we are supporting rfc4106 64-bit extended */
+        /* sequence numbers We need to have the AAD length */
+        /* equal to 8 or 12 bytes */
+
+        tempCipherLen = (unsigned long)(req->cryptlen - auth_tag_len);
+        /* IV below built */
+        for (i = 0; i < 4; i++)
+                *(iv+i) = ctx->nonce[i];
+        for (i = 0; i < 8; i++)
+                *(iv+4+i) = req->iv[i];
+        *((__be32 *)(iv+12)) = counter;
+
+        if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) {
+                one_entry_in_sg = 1;
+                scatterwalk_start(&src_sg_walk, req->src);
+                scatterwalk_start(&assoc_sg_walk, req->assoc);
+                src = scatterwalk_map(&src_sg_walk, 0);
+                assoc = scatterwalk_map(&assoc_sg_walk, 0);
+                dst = src;
+                if (unlikely(req->src != req->dst)) {
+                        scatterwalk_start(&dst_sg_walk, req->dst);
+                        dst = scatterwalk_map(&dst_sg_walk, 0);
+                }
+
+        } else {
+                /* Allocate memory for src, dst, assoc */
+                src = kmalloc(req->cryptlen + req->assoclen, GFP_ATOMIC);
+                if (!src)
+                        return -ENOMEM;
+                assoc = (src + req->cryptlen + auth_tag_len);
+                scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0);
+                scatterwalk_map_and_copy(assoc, req->assoc, 0,
+                        req->assoclen, 0);
+                dst = src;
+        }
+
+        aesni_gcm_dec(aes_ctx, dst, src, tempCipherLen, iv,
+                ctx->hash_subkey, assoc, (unsigned long)req->assoclen,
+                authTag, auth_tag_len);
+
+        /* Compare generated tag with passed in tag. */
+        retval = memcmp(src + tempCipherLen, authTag, auth_tag_len) ?
+                -EBADMSG : 0;
+
+        if (one_entry_in_sg) {
+                if (unlikely(req->src != req->dst)) {
+                        scatterwalk_unmap(dst, 0);
+                        scatterwalk_done(&dst_sg_walk, 0, 0);
+                }
+                scatterwalk_unmap(src, 0);
+                scatterwalk_unmap(assoc, 0);
+                scatterwalk_done(&src_sg_walk, 0, 0);
+                scatterwalk_done(&assoc_sg_walk, 0, 0);
+        } else {
+                scatterwalk_map_and_copy(dst, req->dst, 0, req->cryptlen, 1);
+                kfree(src);
+        }
+        return retval;
+}
+
+static struct crypto_alg __rfc4106_alg = {
+        .cra_name               = "__gcm-aes-aesni",
+        .cra_driver_name        = "__driver-gcm-aes-aesni",
+        .cra_priority           = 0,
+        .cra_flags              = CRYPTO_ALG_TYPE_AEAD,
+        .cra_blocksize          = 1,
+        .cra_ctxsize    = sizeof(struct aesni_rfc4106_gcm_ctx) + AESNI_ALIGN,
+        .cra_alignmask          = 0,
+        .cra_type               = &crypto_aead_type,
+        .cra_module             = THIS_MODULE,
+        .cra_list               = LIST_HEAD_INIT(__rfc4106_alg.cra_list),
+        .cra_u = {
+                .aead = {
+                        .encrypt        = __driver_rfc4106_encrypt,
+                        .decrypt        = __driver_rfc4106_decrypt,
+                },
+        },
+};
+#endif
+
 static int __init aesni_init(void)
 {
         int err;
@@ -738,6 +1248,7 @@ static int __init aesni_init(void)
                 printk(KERN_INFO "Intel AES-NI instructions are not detected.\n");
                 return -ENODEV;
         }
+
         if ((err = crypto_register_alg(&aesni_alg)))
                 goto aes_err;
         if ((err = crypto_register_alg(&__aesni_alg)))
@@ -746,18 +1257,24 @@ static int __init aesni_init(void)
                 goto blk_ecb_err;
         if ((err = crypto_register_alg(&blk_cbc_alg)))
                 goto blk_cbc_err;
-        if ((err = crypto_register_alg(&blk_ctr_alg)))
-                goto blk_ctr_err;
         if ((err = crypto_register_alg(&ablk_ecb_alg)))
                 goto ablk_ecb_err;
         if ((err = crypto_register_alg(&ablk_cbc_alg)))
                 goto ablk_cbc_err;
+#ifdef CONFIG_X86_64
+        if ((err = crypto_register_alg(&blk_ctr_alg)))
+                goto blk_ctr_err;
         if ((err = crypto_register_alg(&ablk_ctr_alg)))
                 goto ablk_ctr_err;
+        if ((err = crypto_register_alg(&__rfc4106_alg)))
+                goto __aead_gcm_err;
+        if ((err = crypto_register_alg(&rfc4106_alg)))
+                goto aead_gcm_err;
 #ifdef HAS_CTR
         if ((err = crypto_register_alg(&ablk_rfc3686_ctr_alg)))
                 goto ablk_rfc3686_ctr_err;
 #endif
+#endif
 #ifdef HAS_LRW
         if ((err = crypto_register_alg(&ablk_lrw_alg)))
                 goto ablk_lrw_err;
@@ -770,7 +1287,6 @@ static int __init aesni_init(void)
         if ((err = crypto_register_alg(&ablk_xts_alg)))
                 goto ablk_xts_err;
 #endif
-
         return err;
 
 #ifdef HAS_XTS
@@ -784,18 +1300,24 @@ ablk_pcbc_err:
         crypto_unregister_alg(&ablk_lrw_alg);
 ablk_lrw_err:
 #endif
+#ifdef CONFIG_X86_64
 #ifdef HAS_CTR
         crypto_unregister_alg(&ablk_rfc3686_ctr_alg);
 ablk_rfc3686_ctr_err:
 #endif
+        crypto_unregister_alg(&rfc4106_alg);
+aead_gcm_err:
+        crypto_unregister_alg(&__rfc4106_alg);
+__aead_gcm_err:
         crypto_unregister_alg(&ablk_ctr_alg);
 ablk_ctr_err:
+        crypto_unregister_alg(&blk_ctr_alg);
+blk_ctr_err:
+#endif
         crypto_unregister_alg(&ablk_cbc_alg);
 ablk_cbc_err:
         crypto_unregister_alg(&ablk_ecb_alg);
 ablk_ecb_err:
-        crypto_unregister_alg(&blk_ctr_alg);
-blk_ctr_err:
         crypto_unregister_alg(&blk_cbc_alg);
 blk_cbc_err:
         crypto_unregister_alg(&blk_ecb_alg);
@@ -818,13 +1340,17 @@ static void __exit aesni_exit(void)
 #ifdef HAS_LRW
         crypto_unregister_alg(&ablk_lrw_alg);
 #endif
+#ifdef CONFIG_X86_64
 #ifdef HAS_CTR
         crypto_unregister_alg(&ablk_rfc3686_ctr_alg);
 #endif
+        crypto_unregister_alg(&rfc4106_alg);
+        crypto_unregister_alg(&__rfc4106_alg);
         crypto_unregister_alg(&ablk_ctr_alg);
+        crypto_unregister_alg(&blk_ctr_alg);
+#endif
         crypto_unregister_alg(&ablk_cbc_alg);
         crypto_unregister_alg(&ablk_ecb_alg);
-        crypto_unregister_alg(&blk_ctr_alg);
         crypto_unregister_alg(&blk_cbc_alg);
         crypto_unregister_alg(&blk_ecb_alg);
         crypto_unregister_alg(&__aesni_alg);
diff --git a/arch/x86/include/asm/boot.h b/arch/x86/include/asm/boot.h
index 3b62ab56c7a0..5e1a2eef3e7c 100644
--- a/arch/x86/include/asm/boot.h
+++ b/arch/x86/include/asm/boot.h
@@ -32,11 +32,7 @@
 #define BOOT_HEAP_SIZE             0x400000
 #else /* !CONFIG_KERNEL_BZIP2 */
 
-#ifdef CONFIG_X86_64
-#define BOOT_HEAP_SIZE  0x7000
-#else
-#define BOOT_HEAP_SIZE  0x4000
-#endif
+#define BOOT_HEAP_SIZE  0x8000
 
 #endif /* !CONFIG_KERNEL_BZIP2 */
 
diff --git a/arch/x86/include/asm/irq.h b/arch/x86/include/asm/irq.h
index ba870bb6dd8e..c704b38c57a2 100644
--- a/arch/x86/include/asm/irq.h
+++ b/arch/x86/include/asm/irq.h
@@ -10,6 +10,9 @@
 #include <asm/apicdef.h>
 #include <asm/irq_vectors.h>
 
+/* Even though we don't support this, supply it to appease OF */
+static inline void irq_dispose_mapping(unsigned int virq) { }
+
 static inline int irq_canonicalize(int irq)
 {
         return ((irq == 2) ? 9 : irq);
diff --git a/arch/x86/include/asm/kvm_emulate.h b/arch/x86/include/asm/kvm_emulate.h
index b36c6b3fe144..8e37deb1eb38 100644
--- a/arch/x86/include/asm/kvm_emulate.h
+++ b/arch/x86/include/asm/kvm_emulate.h
@@ -15,6 +15,14 @@
 
 struct x86_emulate_ctxt;
 
+struct x86_exception {
+        u8 vector;
+        bool error_code_valid;
+        u16 error_code;
+        bool nested_page_fault;
+        u64 address; /* cr2 or nested page fault gpa */
+};
+
 /*
  * x86_emulate_ops:
  *
@@ -64,7 +72,8 @@ struct x86_emulate_ops {
          *  @bytes: [IN ] Number of bytes to read from memory.
          */
         int (*read_std)(unsigned long addr, void *val,
-                        unsigned int bytes, struct kvm_vcpu *vcpu, u32 *error);
+                        unsigned int bytes, struct kvm_vcpu *vcpu,
+                        struct x86_exception *fault);
 
         /*
          * write_std: Write bytes of standard (non-emulated/special) memory.
@@ -74,7 +83,8 @@ struct x86_emulate_ops {
          *  @bytes: [IN ] Number of bytes to write to memory.
          */
         int (*write_std)(unsigned long addr, void *val,
-                         unsigned int bytes, struct kvm_vcpu *vcpu, u32 *error);
+                         unsigned int bytes, struct kvm_vcpu *vcpu,
+                         struct x86_exception *fault);
         /*
          * fetch: Read bytes of standard (non-emulated/special) memory.
          *        Used for instruction fetch.
@@ -83,7 +93,8 @@ struct x86_emulate_ops {
          *  @bytes: [IN ] Number of bytes to read from memory.
          */
         int (*fetch)(unsigned long addr, void *val,
-                        unsigned int bytes, struct kvm_vcpu *vcpu, u32 *error);
+                     unsigned int bytes, struct kvm_vcpu *vcpu,
+                     struct x86_exception *fault);
 
         /*
          * read_emulated: Read bytes from emulated/special memory area.
@@ -94,7 +105,7 @@ struct x86_emulate_ops {
         int (*read_emulated)(unsigned long addr,
                              void *val,
                              unsigned int bytes,
-                             unsigned int *error,
+                             struct x86_exception *fault,
                              struct kvm_vcpu *vcpu);
 
         /*
@@ -107,7 +118,7 @@ struct x86_emulate_ops {
         int (*write_emulated)(unsigned long addr,
                               const void *val,
                               unsigned int bytes,
-                              unsigned int *error,
+                              struct x86_exception *fault,
                               struct kvm_vcpu *vcpu);
 
         /*
@@ -122,7 +133,7 @@ struct x86_emulate_ops {
                                 const void *old,
                                 const void *new,
                                 unsigned int bytes,
-                                unsigned int *error,
+                                struct x86_exception *fault,
                                 struct kvm_vcpu *vcpu);
 
         int (*pio_in_emulated)(int size, unsigned short port, void *val,
@@ -159,7 +170,10 @@ struct operand {
         };
         union {
                 unsigned long *reg;
-                unsigned long mem;
+                struct segmented_address {
+                        ulong ea;
+                        unsigned seg;
+                } mem;
         } addr;
         union {
                 unsigned long val;
@@ -226,9 +240,8 @@ struct x86_emulate_ctxt {
 
         bool perm_ok; /* do not check permissions if true */
 
-        int exception; /* exception that happens during emulation or -1 */
-        u32 error_code; /* error code for exception */
-        bool error_code_valid;
+        bool have_exception;
+        struct x86_exception exception;
 
         /* decode cache */
         struct decode_cache decode;
@@ -252,7 +265,7 @@ struct x86_emulate_ctxt {
 #define X86EMUL_MODE_HOST X86EMUL_MODE_PROT64
 #endif
 
-int x86_decode_insn(struct x86_emulate_ctxt *ctxt);
+int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len);
 #define EMULATION_FAILED -1
 #define EMULATION_OK 0
 #define EMULATION_RESTART 1
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index f702f82aa1eb..ffd7f8d29187 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -83,11 +83,14 @@
 #define KVM_NR_FIXED_MTRR_REGION 88
 #define KVM_NR_VAR_MTRR 8
 
+#define ASYNC_PF_PER_VCPU 64
+
 extern spinlock_t kvm_lock;
 extern struct list_head vm_list;
 
 struct kvm_vcpu;
 struct kvm;
+struct kvm_async_pf;
 
 enum kvm_reg {
         VCPU_REGS_RAX = 0,
@@ -114,6 +117,7 @@ enum kvm_reg {
 
 enum kvm_reg_ex {
         VCPU_EXREG_PDPTR = NR_VCPU_REGS,
+        VCPU_EXREG_CR3,
 };
 
 enum {
@@ -238,16 +242,18 @@ struct kvm_mmu {
         void (*new_cr3)(struct kvm_vcpu *vcpu);
         void (*set_cr3)(struct kvm_vcpu *vcpu, unsigned long root);
         unsigned long (*get_cr3)(struct kvm_vcpu *vcpu);
-        int (*page_fault)(struct kvm_vcpu *vcpu, gva_t gva, u32 err);
-        void (*inject_page_fault)(struct kvm_vcpu *vcpu);
+        int (*page_fault)(struct kvm_vcpu *vcpu, gva_t gva, u32 err,
+                          bool prefault);
+        void (*inject_page_fault)(struct kvm_vcpu *vcpu,
+                                  struct x86_exception *fault);
         void (*free)(struct kvm_vcpu *vcpu);
         gpa_t (*gva_to_gpa)(struct kvm_vcpu *vcpu, gva_t gva, u32 access,
-                            u32 *error);
+                            struct x86_exception *exception);
         gpa_t (*translate_gpa)(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access);
         void (*prefetch_page)(struct kvm_vcpu *vcpu,
                               struct kvm_mmu_page *page);
         int (*sync_page)(struct kvm_vcpu *vcpu,
-                         struct kvm_mmu_page *sp, bool clear_unsync);
+                         struct kvm_mmu_page *sp);
         void (*invlpg)(struct kvm_vcpu *vcpu, gva_t gva);
         hpa_t root_hpa;
         int root_level;
@@ -315,16 +321,6 @@ struct kvm_vcpu_arch {
          */
         struct kvm_mmu *walk_mmu;
 
-        /*
-         * This struct is filled with the necessary information to propagate a
-         * page fault into the guest
-         */
-        struct {
-                u64      address;
-                unsigned error_code;
-                bool     nested;
-        } fault;
-
         /* only needed in kvm_pv_mmu_op() path, but it's hot so
          * put it here to avoid allocation */
         struct kvm_pv_mmu_op_buffer mmu_op_buffer;
@@ -412,6 +408,15 @@ struct kvm_vcpu_arch {
         u64 hv_vapic;
 
         cpumask_var_t wbinvd_dirty_mask;
+
+        struct {
+                bool halted;
+                gfn_t gfns[roundup_pow_of_two(ASYNC_PF_PER_VCPU)];
+                struct gfn_to_hva_cache data;
+                u64 msr_val;
+                u32 id;
+                bool send_user_only;
+        } apf;
 };
 
 struct kvm_arch {
@@ -456,6 +461,10 @@ struct kvm_arch {
         /* fields used by HYPER-V emulation */
         u64 hv_guest_os_id;
         u64 hv_hypercall;
+
+        #ifdef CONFIG_KVM_MMU_AUDIT
+        int audit_point;
+        #endif
 };
 
 struct kvm_vm_stat {
@@ -529,6 +538,7 @@ struct kvm_x86_ops {
                             struct kvm_segment *var, int seg);
         void (*get_cs_db_l_bits)(struct kvm_vcpu *vcpu, int *db, int *l);
         void (*decache_cr0_guest_bits)(struct kvm_vcpu *vcpu);
+        void (*decache_cr3)(struct kvm_vcpu *vcpu);
         void (*decache_cr4_guest_bits)(struct kvm_vcpu *vcpu);
         void (*set_cr0)(struct kvm_vcpu *vcpu, unsigned long cr0);
         void (*set_cr3)(struct kvm_vcpu *vcpu, unsigned long cr3);
@@ -582,9 +592,17 @@ struct kvm_x86_ops {
 
         void (*write_tsc_offset)(struct kvm_vcpu *vcpu, u64 offset);
 
+        void (*get_exit_info)(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2);
         const struct trace_print_flags *exit_reasons_str;
 };
 
+struct kvm_arch_async_pf {
+        u32 token;
+        gfn_t gfn;
+        unsigned long cr3;
+        bool direct_map;
+};
+
 extern struct kvm_x86_ops *kvm_x86_ops;
 
 int kvm_mmu_module_init(void);
@@ -594,7 +612,6 @@ void kvm_mmu_destroy(struct kvm_vcpu *vcpu);
 int kvm_mmu_create(struct kvm_vcpu *vcpu);
 int kvm_mmu_setup(struct kvm_vcpu *vcpu);
 void kvm_mmu_set_nonpresent_ptes(u64 trap_pte, u64 notrap_pte);
-void kvm_mmu_set_base_ptes(u64 base_pte);
 void kvm_mmu_set_mask_ptes(u64 user_mask, u64 accessed_mask,
                 u64 dirty_mask, u64 nx_mask, u64 x_mask);
 
@@ -623,8 +640,15 @@ enum emulation_result {
 #define EMULTYPE_NO_DECODE          (1 << 0)
 #define EMULTYPE_TRAP_UD            (1 << 1)
 #define EMULTYPE_SKIP               (1 << 2)
-int emulate_instruction(struct kvm_vcpu *vcpu,
-                        unsigned long cr2, u16 error_code, int emulation_type);
+int x86_emulate_instruction(struct kvm_vcpu *vcpu, unsigned long cr2,
+                            int emulation_type, void *insn, int insn_len);
+
+static inline int emulate_instruction(struct kvm_vcpu *vcpu,
+                        int emulation_type)
+{
+        return x86_emulate_instruction(vcpu, 0, emulation_type, NULL, 0);
+}
+
 void realmode_lgdt(struct kvm_vcpu *vcpu, u16 size, unsigned long address);
 void realmode_lidt(struct kvm_vcpu *vcpu, u16 size, unsigned long address);
 
@@ -650,7 +674,7 @@ int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int reason,
 int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0);
 int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3);
 int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4);
-void kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8);
+int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8);
 int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val);
 int kvm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *val);
 unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu);
@@ -668,11 +692,11 @@ void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr);
 void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code);
 void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr);
 void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code);
-void kvm_inject_page_fault(struct kvm_vcpu *vcpu);
+void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault);
 int kvm_read_guest_page_mmu(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
                             gfn_t gfn, void *data, int offset, int len,
                             u32 access);
-void kvm_propagate_fault(struct kvm_vcpu *vcpu);
+void kvm_propagate_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault);
 bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl);
 
 int kvm_pic_set_irq(void *opaque, int irq, int level);
@@ -690,16 +714,21 @@ void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu);
 int kvm_mmu_load(struct kvm_vcpu *vcpu);
 void kvm_mmu_unload(struct kvm_vcpu *vcpu);
 void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu);
-gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva, u32 *error);
-gpa_t kvm_mmu_gva_to_gpa_fetch(struct kvm_vcpu *vcpu, gva_t gva, u32 *error);
-gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva, u32 *error);
-gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva, u32 *error);
+gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva,
+                              struct x86_exception *exception);
+gpa_t kvm_mmu_gva_to_gpa_fetch(struct kvm_vcpu *vcpu, gva_t gva,
+                               struct x86_exception *exception);
+gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva,
+                               struct x86_exception *exception);
+gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva,
+                                struct x86_exception *exception);
 
 int kvm_emulate_hypercall(struct kvm_vcpu *vcpu);
 
 int kvm_fix_hypercall(struct kvm_vcpu *vcpu);
 
-int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t gva, u32 error_code);
+int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t gva, u32 error_code,
+                       void *insn, int insn_len);
 void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva);
 
 void kvm_enable_tdp(void);
@@ -766,20 +795,25 @@ enum {
 #define HF_VINTR_MASK           (1 << 2)
 #define HF_NMI_MASK             (1 << 3)
 #define HF_IRET_MASK            (1 << 4)
+#define HF_GUEST_MASK           (1 << 5) /* VCPU is in guest-mode */
 
 /*
  * Hardware virtualization extension instructions may fault if a
  * reboot turns off virtualization while processes are running.
  * Trap the fault and ignore the instruction if that happens.
  */
-asmlinkage void kvm_handle_fault_on_reboot(void);
+asmlinkage void kvm_spurious_fault(void);
+extern bool kvm_rebooting;
 
 #define __kvm_handle_fault_on_reboot(insn) \
         "666: " insn "\n\t" \
+        "668: \n\t"                           \
         ".pushsection .fixup, \"ax\" \n" \
         "667: \n\t" \
+        "cmpb $0, kvm_rebooting \n\t"         \
+        "jne 668b \n\t"                       \
         __ASM_SIZE(push) " $666b \n\t"        \
-        "jmp kvm_handle_fault_on_reboot \n\t" \
+        "call kvm_spurious_fault \n\t"        \
         ".popsection \n\t" \
         ".pushsection __ex_table, \"a\" \n\t" \
         _ASM_PTR " 666b, 667b \n\t" \
@@ -788,6 +822,7 @@ asmlinkage void kvm_handle_fault_on_reboot(void);
 #define KVM_ARCH_WANT_MMU_NOTIFIER
 int kvm_unmap_hva(struct kvm *kvm, unsigned long hva);
 int kvm_age_hva(struct kvm *kvm, unsigned long hva);
+int kvm_test_age_hva(struct kvm *kvm, unsigned long hva);
 void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte);
 int cpuid_maxphyaddr(struct kvm_vcpu *vcpu);
 int kvm_cpu_has_interrupt(struct kvm_vcpu *vcpu);
@@ -799,4 +834,15 @@ void kvm_set_shared_msr(unsigned index, u64 val, u64 mask);
 
 bool kvm_is_linear_rip(struct kvm_vcpu *vcpu, unsigned long linear_rip);
 
+void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
+                                     struct kvm_async_pf *work);
+void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
+                                 struct kvm_async_pf *work);
+void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
+                               struct kvm_async_pf *work);
+bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu);
+extern bool kvm_find_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn);
+
+void kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err);
+
 #endif /* _ASM_X86_KVM_HOST_H */
diff --git a/arch/x86/include/asm/kvm_para.h b/arch/x86/include/asm/kvm_para.h
index 7b562b6184bc..a427bf77a93d 100644
--- a/arch/x86/include/asm/kvm_para.h
+++ b/arch/x86/include/asm/kvm_para.h
@@ -20,6 +20,7 @@
  * are available. The use of 0x11 and 0x12 is deprecated
  */
 #define KVM_FEATURE_CLOCKSOURCE2        3
+#define KVM_FEATURE_ASYNC_PF            4
 
 /* The last 8 bits are used to indicate how to interpret the flags field
  * in pvclock structure. If no bits are set, all flags are ignored.
@@ -32,9 +33,13 @@
 /* Custom MSRs falls in the range 0x4b564d00-0x4b564dff */
 #define MSR_KVM_WALL_CLOCK_NEW  0x4b564d00
 #define MSR_KVM_SYSTEM_TIME_NEW 0x4b564d01
+#define MSR_KVM_ASYNC_PF_EN 0x4b564d02
 
 #define KVM_MAX_MMU_OP_BATCH           32
 
+#define KVM_ASYNC_PF_ENABLED                    (1 << 0)
+#define KVM_ASYNC_PF_SEND_ALWAYS                (1 << 1)
+
 /* Operations for KVM_HC_MMU_OP */
 #define KVM_MMU_OP_WRITE_PTE            1
 #define KVM_MMU_OP_FLUSH_TLB            2
@@ -61,10 +66,20 @@ struct kvm_mmu_op_release_pt {
         __u64 pt_phys;
 };
 
+#define KVM_PV_REASON_PAGE_NOT_PRESENT 1
+#define KVM_PV_REASON_PAGE_READY 2
+
+struct kvm_vcpu_pv_apf_data {
+        __u32 reason;
+        __u8 pad[60];
+        __u32 enabled;
+};
+
 #ifdef __KERNEL__
 #include <asm/processor.h>
 
 extern void kvmclock_init(void);
+extern int kvm_register_clock(char *txt);
 
 
 /* This instruction is vmcall.  On non-VT architectures, it will generate a
@@ -160,8 +175,17 @@ static inline unsigned int kvm_arch_para_features(void)
 
 #ifdef CONFIG_KVM_GUEST
 void __init kvm_guest_init(void);
+void kvm_async_pf_task_wait(u32 token);
+void kvm_async_pf_task_wake(u32 token);
+u32 kvm_read_and_reset_pf_reason(void);
 #else
 #define kvm_guest_init() do { } while (0)
+#define kvm_async_pf_task_wait(T) do {} while(0)
+#define kvm_async_pf_task_wake(T) do {} while(0)
+static inline u32 kvm_read_and_reset_pf_reason(void)
+{
+        return 0;
+}
 #endif
 
 #endif /* __KERNEL__ */
diff --git a/arch/x86/include/asm/olpc.h b/arch/x86/include/asm/olpc.h
index 42a978c0c1b3..f482010350fb 100644
--- a/arch/x86/include/asm/olpc.h
+++ b/arch/x86/include/asm/olpc.h
@@ -107,10 +107,14 @@ extern int olpc_ec_mask_unset(uint8_t bits);
 /* GPIO assignments */
 
 #define OLPC_GPIO_MIC_AC        1
-#define OLPC_GPIO_DCON_IRQ      geode_gpio(7)
+#define OLPC_GPIO_DCON_STAT0    5
+#define OLPC_GPIO_DCON_STAT1    6
+#define OLPC_GPIO_DCON_IRQ      7
 #define OLPC_GPIO_THRM_ALRM     geode_gpio(10)
-#define OLPC_GPIO_SMB_CLK       geode_gpio(14)
-#define OLPC_GPIO_SMB_DATA      geode_gpio(15)
+#define OLPC_GPIO_DCON_LOAD    11
+#define OLPC_GPIO_DCON_BLANK   12
+#define OLPC_GPIO_SMB_CLK      14
+#define OLPC_GPIO_SMB_DATA     15
 #define OLPC_GPIO_WORKAUX       geode_gpio(24)
 #define OLPC_GPIO_LID           geode_gpio(26)
 #define OLPC_GPIO_ECSCI         geode_gpio(27)
diff --git a/arch/x86/include/asm/olpc_ofw.h b/arch/x86/include/asm/olpc_ofw.h
index 2a8478140bb3..641988efe063 100644
--- a/arch/x86/include/asm/olpc_ofw.h
+++ b/arch/x86/include/asm/olpc_ofw.h
@@ -8,6 +8,8 @@
 
 #ifdef CONFIG_OLPC_OPENFIRMWARE
 
+extern bool olpc_ofw_is_installed(void);
+
 /* run an OFW command by calling into the firmware */
 #define olpc_ofw(name, args, res) \
         __olpc_ofw((name), ARRAY_SIZE(args), args, ARRAY_SIZE(res), res)
@@ -26,10 +28,17 @@ extern bool olpc_ofw_present(void);
 
 #else /* !CONFIG_OLPC_OPENFIRMWARE */
 
+static inline bool olpc_ofw_is_installed(void) { return false; }
 static inline void olpc_ofw_detect(void) { }
 static inline void setup_olpc_ofw_pgd(void) { }
 static inline bool olpc_ofw_present(void) { return false; }
 
 #endif /* !CONFIG_OLPC_OPENFIRMWARE */
 
+#ifdef CONFIG_OLPC_OPENFIRMWARE_DT
+extern void olpc_dt_build_devicetree(void);
+#else
+static inline void olpc_dt_build_devicetree(void) { }
+#endif /* CONFIG_OLPC_OPENFIRMWARE_DT */
+
 #endif /* _ASM_X86_OLPC_OFW_H */
diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h
index 7709c12431b8..2071a8b2b32f 100644
--- a/arch/x86/include/asm/paravirt.h
+++ b/arch/x86/include/asm/paravirt.h
@@ -435,6 +435,11 @@ static inline void pte_update(struct mm_struct *mm, unsigned long addr,
 {
         PVOP_VCALL3(pv_mmu_ops.pte_update, mm, addr, ptep);
 }
+static inline void pmd_update(struct mm_struct *mm, unsigned long addr,
+                              pmd_t *pmdp)
+{
+        PVOP_VCALL3(pv_mmu_ops.pmd_update, mm, addr, pmdp);
+}
 
 static inline void pte_update_defer(struct mm_struct *mm, unsigned long addr,
                                     pte_t *ptep)
@@ -442,6 +447,12 @@ static inline void pte_update_defer(struct mm_struct *mm, unsigned long addr,
         PVOP_VCALL3(pv_mmu_ops.pte_update_defer, mm, addr, ptep);
 }
 
+static inline void pmd_update_defer(struct mm_struct *mm, unsigned long addr,
+                                    pmd_t *pmdp)
+{
+        PVOP_VCALL3(pv_mmu_ops.pmd_update_defer, mm, addr, pmdp);
+}
+
 static inline pte_t __pte(pteval_t val)
 {
         pteval_t ret;
@@ -543,6 +554,20 @@ static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
                 PVOP_VCALL4(pv_mmu_ops.set_pte_at, mm, addr, ptep, pte.pte);
 }
 
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,
+                              pmd_t *pmdp, pmd_t pmd)
+{
+#if PAGETABLE_LEVELS >= 3
+        if (sizeof(pmdval_t) > sizeof(long))
+                /* 5 arg words */
+                pv_mmu_ops.set_pmd_at(mm, addr, pmdp, pmd);
+        else
+                PVOP_VCALL4(pv_mmu_ops.set_pmd_at, mm, addr, pmdp, pmd.pmd);
+#endif
+}
+#endif
+
 static inline void set_pmd(pmd_t *pmdp, pmd_t pmd)
 {
         pmdval_t val = native_pmd_val(pmd);
diff --git a/arch/x86/include/asm/paravirt_types.h b/arch/x86/include/asm/paravirt_types.h
index b82bac975250..82885099c869 100644
--- a/arch/x86/include/asm/paravirt_types.h
+++ b/arch/x86/include/asm/paravirt_types.h
@@ -265,10 +265,16 @@ struct pv_mmu_ops {
         void (*set_pte_at)(struct mm_struct *mm, unsigned long addr,
                            pte_t *ptep, pte_t pteval);
         void (*set_pmd)(pmd_t *pmdp, pmd_t pmdval);
+        void (*set_pmd_at)(struct mm_struct *mm, unsigned long addr,
+                           pmd_t *pmdp, pmd_t pmdval);
         void (*pte_update)(struct mm_struct *mm, unsigned long addr,
                            pte_t *ptep);
         void (*pte_update_defer)(struct mm_struct *mm,
                                  unsigned long addr, pte_t *ptep);
+        void (*pmd_update)(struct mm_struct *mm, unsigned long addr,
+                           pmd_t *pmdp);
+        void (*pmd_update_defer)(struct mm_struct *mm,
+                                 unsigned long addr, pmd_t *pmdp);
 
         pte_t (*ptep_modify_prot_start)(struct mm_struct *mm, unsigned long addr,
                                         pte_t *ptep);
diff --git a/arch/x86/include/asm/pgalloc.h b/arch/x86/include/asm/pgalloc.h
index 271de94c3810..b4389a468fb6 100644
--- a/arch/x86/include/asm/pgalloc.h
+++ b/arch/x86/include/asm/pgalloc.h
@@ -92,7 +92,7 @@ static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
 extern void ___pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd);
 
 static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd,
-                                  unsigned long adddress)
+                                  unsigned long address)
 {
         ___pmd_free_tlb(tlb, pmd);
 }
diff --git a/arch/x86/include/asm/pgtable-2level.h b/arch/x86/include/asm/pgtable-2level.h
index 2334982b339e..98391db840c6 100644
--- a/arch/x86/include/asm/pgtable-2level.h
+++ b/arch/x86/include/asm/pgtable-2level.h
@@ -46,6 +46,15 @@ static inline pte_t native_ptep_get_and_clear(pte_t *xp)
 #define native_ptep_get_and_clear(xp) native_local_ptep_get_and_clear(xp)
 #endif
 
+#ifdef CONFIG_SMP
+static inline pmd_t native_pmdp_get_and_clear(pmd_t *xp)
+{
+        return __pmd(xchg((pmdval_t *)xp, 0));
+}
+#else
+#define native_pmdp_get_and_clear(xp) native_local_pmdp_get_and_clear(xp)
+#endif
+
 /*
  * Bits _PAGE_BIT_PRESENT, _PAGE_BIT_FILE and _PAGE_BIT_PROTNONE are taken,
  * split up the 29 bits of offset into this range:
diff --git a/arch/x86/include/asm/pgtable-3level.h b/arch/x86/include/asm/pgtable-3level.h
index 177b0165ea01..94b979d1b58d 100644
--- a/arch/x86/include/asm/pgtable-3level.h
+++ b/arch/x86/include/asm/pgtable-3level.h
@@ -104,6 +104,29 @@ static inline pte_t native_ptep_get_and_clear(pte_t *ptep)
 #define native_ptep_get_and_clear(xp) native_local_ptep_get_and_clear(xp)
 #endif
 
+#ifdef CONFIG_SMP
+union split_pmd {
+        struct {
+                u32 pmd_low;
+                u32 pmd_high;
+        };
+        pmd_t pmd;
+};
+static inline pmd_t native_pmdp_get_and_clear(pmd_t *pmdp)
+{
+        union split_pmd res, *orig = (union split_pmd *)pmdp;
+
+        /* xchg acts as a barrier before setting of the high bits */
+        res.pmd_low = xchg(&orig->pmd_low, 0);
+        res.pmd_high = orig->pmd_high;
+        orig->pmd_high = 0;
+
+        return res.pmd;
+}
+#else
+#define native_pmdp_get_and_clear(xp) native_local_pmdp_get_and_clear(xp)
+#endif
+
 /*
  * Bits 0, 6 and 7 are taken in the low part of the pte,
  * put the 32 bits of offset into the high part.
diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h
index ada823a13c7c..18601c86fab1 100644
--- a/arch/x86/include/asm/pgtable.h
+++ b/arch/x86/include/asm/pgtable.h
@@ -35,6 +35,7 @@ extern struct mm_struct *pgd_page_get_mm(struct page *page);
 #else  /* !CONFIG_PARAVIRT */
 #define set_pte(ptep, pte)              native_set_pte(ptep, pte)
 #define set_pte_at(mm, addr, ptep, pte) native_set_pte_at(mm, addr, ptep, pte)
+#define set_pmd_at(mm, addr, pmdp, pmd) native_set_pmd_at(mm, addr, pmdp, pmd)
 
 #define set_pte_atomic(ptep, pte)                                       \
         native_set_pte_atomic(ptep, pte)
@@ -59,6 +60,8 @@ extern struct mm_struct *pgd_page_get_mm(struct page *page);
 
 #define pte_update(mm, addr, ptep)              do { } while (0)
 #define pte_update_defer(mm, addr, ptep)        do { } while (0)
+#define pmd_update(mm, addr, ptep)              do { } while (0)
+#define pmd_update_defer(mm, addr, ptep)        do { } while (0)
 
 #define pgd_val(x)      native_pgd_val(x)
 #define __pgd(x)        native_make_pgd(x)
@@ -94,6 +97,11 @@ static inline int pte_young(pte_t pte)
         return pte_flags(pte) & _PAGE_ACCESSED;
 }
 
+static inline int pmd_young(pmd_t pmd)
+{
+        return pmd_flags(pmd) & _PAGE_ACCESSED;
+}
+
 static inline int pte_write(pte_t pte)
 {
         return pte_flags(pte) & _PAGE_RW;
@@ -142,6 +150,23 @@ static inline int pmd_large(pmd_t pte)
                 (_PAGE_PSE | _PAGE_PRESENT);
 }
 
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+static inline int pmd_trans_splitting(pmd_t pmd)
+{
+        return pmd_val(pmd) & _PAGE_SPLITTING;
+}
+
+static inline int pmd_trans_huge(pmd_t pmd)
+{
+        return pmd_val(pmd) & _PAGE_PSE;
+}
+
+static inline int has_transparent_hugepage(void)
+{
+        return cpu_has_pse;
+}
+#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
+
 static inline pte_t pte_set_flags(pte_t pte, pteval_t set)
 {
         pteval_t v = native_pte_val(pte);
@@ -216,6 +241,55 @@ static inline pte_t pte_mkspecial(pte_t pte)
         return pte_set_flags(pte, _PAGE_SPECIAL);
 }
 
+static inline pmd_t pmd_set_flags(pmd_t pmd, pmdval_t set)
+{
+        pmdval_t v = native_pmd_val(pmd);
+
+        return __pmd(v | set);
+}
+
+static inline pmd_t pmd_clear_flags(pmd_t pmd, pmdval_t clear)
+{
+        pmdval_t v = native_pmd_val(pmd);
+
+        return __pmd(v & ~clear);
+}
+
+static inline pmd_t pmd_mkold(pmd_t pmd)
+{
+        return pmd_clear_flags(pmd, _PAGE_ACCESSED);
+}
+
+static inline pmd_t pmd_wrprotect(pmd_t pmd)
+{
+        return pmd_clear_flags(pmd, _PAGE_RW);
+}
+
+static inline pmd_t pmd_mkdirty(pmd_t pmd)
+{
+        return pmd_set_flags(pmd, _PAGE_DIRTY);
+}
+
+static inline pmd_t pmd_mkhuge(pmd_t pmd)
+{
+        return pmd_set_flags(pmd, _PAGE_PSE);
+}
+
+static inline pmd_t pmd_mkyoung(pmd_t pmd)
+{
+        return pmd_set_flags(pmd, _PAGE_ACCESSED);
+}
+
+static inline pmd_t pmd_mkwrite(pmd_t pmd)
+{
+        return pmd_set_flags(pmd, _PAGE_RW);
+}
+
+static inline pmd_t pmd_mknotpresent(pmd_t pmd)
+{
+        return pmd_clear_flags(pmd, _PAGE_PRESENT);
+}
+
 /*
  * Mask out unsupported bits in a present pgprot.  Non-present pgprots
  * can use those bits for other purposes, so leave them be.
@@ -256,6 +330,16 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
         return __pte(val);
 }
 
+static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
+{
+        pmdval_t val = pmd_val(pmd);
+
+        val &= _HPAGE_CHG_MASK;
+        val |= massage_pgprot(newprot) & ~_HPAGE_CHG_MASK;
+
+        return __pmd(val);
+}
+
 /* mprotect needs to preserve PAT bits when updating vm_page_prot */
 #define pgprot_modify pgprot_modify
 static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot)
@@ -350,7 +434,7 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd)
  * Currently stuck as a macro due to indirect forward reference to
  * linux/mmzone.h's __section_mem_map_addr() definition:
  */
-#define pmd_page(pmd)   pfn_to_page(pmd_val(pmd) >> PAGE_SHIFT)
+#define pmd_page(pmd)   pfn_to_page((pmd_val(pmd) & PTE_PFN_MASK) >> PAGE_SHIFT)
 
 /*
  * the pmd page can be thought of an array like this: pmd_t[PTRS_PER_PMD]
@@ -524,12 +608,26 @@ static inline pte_t native_local_ptep_get_and_clear(pte_t *ptep)
         return res;
 }
 
+static inline pmd_t native_local_pmdp_get_and_clear(pmd_t *pmdp)
+{
+        pmd_t res = *pmdp;
+
+        native_pmd_clear(pmdp);
+        return res;
+}
+
 static inline void native_set_pte_at(struct mm_struct *mm, unsigned long addr,
                                      pte_t *ptep , pte_t pte)
 {
         native_set_pte(ptep, pte);
 }
 
+static inline void native_set_pmd_at(struct mm_struct *mm, unsigned long addr,
+                                     pmd_t *pmdp , pmd_t pmd)
+{
+        native_set_pmd(pmdp, pmd);
+}
+
 #ifndef CONFIG_PARAVIRT
 /*
  * Rules for using pte_update - it must be called after any PTE update which
@@ -607,6 +705,49 @@ static inline void ptep_set_wrprotect(struct mm_struct *mm,
 
 #define flush_tlb_fix_spurious_fault(vma, address)
 
+#define mk_pmd(page, pgprot)   pfn_pmd(page_to_pfn(page), (pgprot))
+
+#define  __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
+extern int pmdp_set_access_flags(struct vm_area_struct *vma,
+                                 unsigned long address, pmd_t *pmdp,
+                                 pmd_t entry, int dirty);
+
+#define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG
+extern int pmdp_test_and_clear_young(struct vm_area_struct *vma,
+                                     unsigned long addr, pmd_t *pmdp);
+
+#define __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH
+extern int pmdp_clear_flush_young(struct vm_area_struct *vma,
+                                  unsigned long address, pmd_t *pmdp);
+
+
+#define __HAVE_ARCH_PMDP_SPLITTING_FLUSH
+extern void pmdp_splitting_flush(struct vm_area_struct *vma,
+                                 unsigned long addr, pmd_t *pmdp);
+
+#define __HAVE_ARCH_PMD_WRITE
+static inline int pmd_write(pmd_t pmd)
+{
+        return pmd_flags(pmd) & _PAGE_RW;
+}
+
+#define __HAVE_ARCH_PMDP_GET_AND_CLEAR
+static inline pmd_t pmdp_get_and_clear(struct mm_struct *mm, unsigned long addr,
+                                       pmd_t *pmdp)
+{
+        pmd_t pmd = native_pmdp_get_and_clear(pmdp);
+        pmd_update(mm, addr, pmdp);
+        return pmd;
+}
+
+#define __HAVE_ARCH_PMDP_SET_WRPROTECT
+static inline void pmdp_set_wrprotect(struct mm_struct *mm,
+                                      unsigned long addr, pmd_t *pmdp)
+{
+        clear_bit(_PAGE_BIT_RW, (unsigned long *)pmdp);
+        pmd_update(mm, addr, pmdp);
+}
+
 /*
  * clone_pgd_range(pgd_t *dst, pgd_t *src, int count);
  *
diff --git a/arch/x86/include/asm/pgtable_64.h b/arch/x86/include/asm/pgtable_64.h
index f86da20347f2..975f709e09ae 100644
--- a/arch/x86/include/asm/pgtable_64.h
+++ b/arch/x86/include/asm/pgtable_64.h
@@ -59,6 +59,16 @@ static inline void native_set_pte_atomic(pte_t *ptep, pte_t pte)
         native_set_pte(ptep, pte);
 }
 
+static inline void native_set_pmd(pmd_t *pmdp, pmd_t pmd)
+{
+        *pmdp = pmd;
+}
+
+static inline void native_pmd_clear(pmd_t *pmd)
+{
+        native_set_pmd(pmd, native_make_pmd(0));
+}
+
 static inline pte_t native_ptep_get_and_clear(pte_t *xp)
 {
 #ifdef CONFIG_SMP
@@ -72,14 +82,17 @@ static inline pte_t native_ptep_get_and_clear(pte_t *xp)
 #endif
 }
 
-static inline void native_set_pmd(pmd_t *pmdp, pmd_t pmd)
+static inline pmd_t native_pmdp_get_and_clear(pmd_t *xp)
 {
-        *pmdp = pmd;
-}
-
-static inline void native_pmd_clear(pmd_t *pmd)
-{
-        native_set_pmd(pmd, native_make_pmd(0));
+#ifdef CONFIG_SMP
+        return native_make_pmd(xchg(&xp->pmd, 0));
+#else
+        /* native_local_pmdp_get_and_clear,
+           but duplicated because of cyclic dependency */
+        pmd_t ret = *xp;
+        native_pmd_clear(xp);
+        return ret;
+#endif
 }
 
 static inline void native_set_pud(pud_t *pudp, pud_t pud)
@@ -168,6 +181,7 @@ extern void cleanup_highmap(void);
 #define kc_offset_to_vaddr(o) ((o) | ~__VIRTUAL_MASK)
 
 #define __HAVE_ARCH_PTE_SAME
+
 #endif /* !__ASSEMBLY__ */
 
 #endif /* _ASM_X86_PGTABLE_64_H */
diff --git a/arch/x86/include/asm/pgtable_types.h b/arch/x86/include/asm/pgtable_types.h
index d1f4a760be23..7db7723d1f32 100644
--- a/arch/x86/include/asm/pgtable_types.h
+++ b/arch/x86/include/asm/pgtable_types.h
@@ -22,6 +22,7 @@
 #define _PAGE_BIT_PAT_LARGE     12      /* On 2MB or 1GB pages */
 #define _PAGE_BIT_SPECIAL       _PAGE_BIT_UNUSED1
 #define _PAGE_BIT_CPA_TEST      _PAGE_BIT_UNUSED1
+#define _PAGE_BIT_SPLITTING     _PAGE_BIT_UNUSED1 /* only valid on a PSE pmd */
 #define _PAGE_BIT_NX           63       /* No execute: only valid after cpuid check */
 
 /* If _PAGE_BIT_PRESENT is clear, we use these: */
@@ -45,6 +46,7 @@
 #define _PAGE_PAT_LARGE (_AT(pteval_t, 1) << _PAGE_BIT_PAT_LARGE)
 #define _PAGE_SPECIAL   (_AT(pteval_t, 1) << _PAGE_BIT_SPECIAL)
 #define _PAGE_CPA_TEST  (_AT(pteval_t, 1) << _PAGE_BIT_CPA_TEST)
+#define _PAGE_SPLITTING (_AT(pteval_t, 1) << _PAGE_BIT_SPLITTING)
 #define __HAVE_ARCH_PTE_SPECIAL
 
 #ifdef CONFIG_KMEMCHECK
@@ -70,6 +72,7 @@
 /* Set of bits not changed in pte_modify */
 #define _PAGE_CHG_MASK  (PTE_PFN_MASK | _PAGE_PCD | _PAGE_PWT |         \
                          _PAGE_SPECIAL | _PAGE_ACCESSED | _PAGE_DIRTY)
+#define _HPAGE_CHG_MASK (_PAGE_CHG_MASK | _PAGE_PSE)
 
 #define _PAGE_CACHE_MASK        (_PAGE_PCD | _PAGE_PWT)
 #define _PAGE_CACHE_WB          (0)
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index c6efecf85a6a..45636cefa186 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -761,10 +761,11 @@ extern void select_idle_routine(const struct cpuinfo_x86 *c);
 extern void init_c1e_mask(void);
 
 extern unsigned long            boot_option_idle_override;
-extern unsigned long            idle_halt;
-extern unsigned long            idle_nomwait;
 extern bool                     c1e_detected;
 
+enum idle_boot_override {IDLE_NO_OVERRIDE=0, IDLE_HALT, IDLE_NOMWAIT,
+                         IDLE_POLL, IDLE_FORCE_MWAIT};
+
 extern void enable_sep_cpu(void);
 extern int sysenter_setup(void);
 
@@ -901,7 +902,7 @@ extern unsigned long thread_saved_pc(struct task_struct *tsk);
 /*
  * The below -8 is to reserve 8 bytes on top of the ring0 stack.
  * This is necessary to guarantee that the entire "struct pt_regs"
- * is accessable even if the CPU haven't stored the SS/ESP registers
+ * is accessible even if the CPU haven't stored the SS/ESP registers
  * on the stack (interrupt gate does not save these registers
  * when switching to the same priv ring).
  * Therefore beware: accessing the ss/esp fields of the
diff --git a/arch/x86/include/asm/prom.h b/arch/x86/include/asm/prom.h
new file mode 100644
index 000000000000..b4ec95f07518
--- /dev/null
+++ b/arch/x86/include/asm/prom.h
@@ -0,0 +1 @@
+/* dummy prom.h; here to make linux/of.h's #includes happy */
diff --git a/arch/x86/include/asm/svm.h b/arch/x86/include/asm/svm.h
index 0e831059ac5a..f2b83bc7d784 100644
--- a/arch/x86/include/asm/svm.h
+++ b/arch/x86/include/asm/svm.h
@@ -47,14 +47,13 @@ enum {
         INTERCEPT_MONITOR,
         INTERCEPT_MWAIT,
         INTERCEPT_MWAIT_COND,
+        INTERCEPT_XSETBV,
 };
 
 
 struct __attribute__ ((__packed__)) vmcb_control_area {
-        u16 intercept_cr_read;
-        u16 intercept_cr_write;
-        u16 intercept_dr_read;
-        u16 intercept_dr_write;
+        u32 intercept_cr;
+        u32 intercept_dr;
         u32 intercept_exceptions;
         u64 intercept;
         u8 reserved_1[42];
@@ -81,14 +80,19 @@ struct __attribute__ ((__packed__)) vmcb_control_area {
         u32 event_inj_err;
         u64 nested_cr3;
         u64 lbr_ctl;
-        u64 reserved_5;
+        u32 clean;
+        u32 reserved_5;
         u64 next_rip;
-        u8 reserved_6[816];
+        u8 insn_len;
+        u8 insn_bytes[15];
+        u8 reserved_6[800];
 };
 
 
 #define TLB_CONTROL_DO_NOTHING 0
 #define TLB_CONTROL_FLUSH_ALL_ASID 1
+#define TLB_CONTROL_FLUSH_ASID 3
+#define TLB_CONTROL_FLUSH_ASID_LOCAL 7
 
 #define V_TPR_MASK 0x0f
 
@@ -204,19 +208,31 @@ struct __attribute__ ((__packed__)) vmcb {
 #define SVM_SELECTOR_READ_MASK SVM_SELECTOR_WRITE_MASK
 #define SVM_SELECTOR_CODE_MASK (1 << 3)
 
-#define INTERCEPT_CR0_MASK 1
-#define INTERCEPT_CR3_MASK (1 << 3)
-#define INTERCEPT_CR4_MASK (1 << 4)
-#define INTERCEPT_CR8_MASK (1 << 8)
-
-#define INTERCEPT_DR0_MASK 1
-#define INTERCEPT_DR1_MASK (1 << 1)
-#define INTERCEPT_DR2_MASK (1 << 2)
-#define INTERCEPT_DR3_MASK (1 << 3)
-#define INTERCEPT_DR4_MASK (1 << 4)
-#define INTERCEPT_DR5_MASK (1 << 5)
-#define INTERCEPT_DR6_MASK (1 << 6)
-#define INTERCEPT_DR7_MASK (1 << 7)
+#define INTERCEPT_CR0_READ      0
+#define INTERCEPT_CR3_READ      3
+#define INTERCEPT_CR4_READ      4
+#define INTERCEPT_CR8_READ      8
+#define INTERCEPT_CR0_WRITE     (16 + 0)
+#define INTERCEPT_CR3_WRITE     (16 + 3)
+#define INTERCEPT_CR4_WRITE     (16 + 4)
+#define INTERCEPT_CR8_WRITE     (16 + 8)
+
+#define INTERCEPT_DR0_READ      0
+#define INTERCEPT_DR1_READ      1
+#define INTERCEPT_DR2_READ      2
+#define INTERCEPT_DR3_READ      3
+#define INTERCEPT_DR4_READ      4
+#define INTERCEPT_DR5_READ      5
+#define INTERCEPT_DR6_READ      6
+#define INTERCEPT_DR7_READ      7
+#define INTERCEPT_DR0_WRITE     (16 + 0)
+#define INTERCEPT_DR1_WRITE     (16 + 1)
+#define INTERCEPT_DR2_WRITE     (16 + 2)
+#define INTERCEPT_DR3_WRITE     (16 + 3)
+#define INTERCEPT_DR4_WRITE     (16 + 4)
+#define INTERCEPT_DR5_WRITE     (16 + 5)
+#define INTERCEPT_DR6_WRITE     (16 + 6)
+#define INTERCEPT_DR7_WRITE     (16 + 7)
 
 #define SVM_EVTINJ_VEC_MASK 0xff
 
@@ -246,6 +262,8 @@ struct __attribute__ ((__packed__)) vmcb {
 #define SVM_EXITINFOSHIFT_TS_REASON_JMP 38
 #define SVM_EXITINFOSHIFT_TS_HAS_ERROR_CODE 44
 
+#define SVM_EXITINFO_REG_MASK 0x0F
+
 #define SVM_EXIT_READ_CR0       0x000
 #define SVM_EXIT_READ_CR3       0x003
 #define SVM_EXIT_READ_CR4       0x004
@@ -316,6 +334,7 @@ struct __attribute__ ((__packed__)) vmcb {
 #define SVM_EXIT_MONITOR        0x08a
 #define SVM_EXIT_MWAIT          0x08b
 #define SVM_EXIT_MWAIT_COND     0x08c
+#define SVM_EXIT_XSETBV         0x08d
 #define SVM_EXIT_NPF            0x400
 
 #define SVM_EXIT_ERR            -1
diff --git a/arch/x86/include/asm/traps.h b/arch/x86/include/asm/traps.h
index f66cda56781d..0310da67307f 100644
--- a/arch/x86/include/asm/traps.h
+++ b/arch/x86/include/asm/traps.h
@@ -30,6 +30,7 @@ asmlinkage void segment_not_present(void);
 asmlinkage void stack_segment(void);
 asmlinkage void general_protection(void);
 asmlinkage void page_fault(void);
+asmlinkage void async_page_fault(void);
 asmlinkage void spurious_interrupt_bug(void);
 asmlinkage void coprocessor_error(void);
 asmlinkage void alignment_check(void);
diff --git a/arch/x86/include/asm/vmx.h b/arch/x86/include/asm/vmx.h
index 9f0cbd987d50..84471b810460 100644
--- a/arch/x86/include/asm/vmx.h
+++ b/arch/x86/include/asm/vmx.h
@@ -66,15 +66,23 @@
 #define PIN_BASED_NMI_EXITING                   0x00000008
 #define PIN_BASED_VIRTUAL_NMIS                  0x00000020
 
+#define VM_EXIT_SAVE_DEBUG_CONTROLS             0x00000002
 #define VM_EXIT_HOST_ADDR_SPACE_SIZE            0x00000200
+#define VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL      0x00001000
 #define VM_EXIT_ACK_INTR_ON_EXIT                0x00008000
 #define VM_EXIT_SAVE_IA32_PAT                   0x00040000
 #define VM_EXIT_LOAD_IA32_PAT                   0x00080000
+#define VM_EXIT_SAVE_IA32_EFER                  0x00100000
+#define VM_EXIT_LOAD_IA32_EFER                  0x00200000
+#define VM_EXIT_SAVE_VMX_PREEMPTION_TIMER       0x00400000
 
+#define VM_ENTRY_LOAD_DEBUG_CONTROLS            0x00000002
 #define VM_ENTRY_IA32E_MODE                     0x00000200
 #define VM_ENTRY_SMM                            0x00000400
 #define VM_ENTRY_DEACT_DUAL_MONITOR             0x00000800
+#define VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL     0x00002000
 #define VM_ENTRY_LOAD_IA32_PAT                  0x00004000
+#define VM_ENTRY_LOAD_IA32_EFER                 0x00008000
 
 /* VMCS Encodings */
 enum vmcs_field {
@@ -239,6 +247,7 @@ enum vmcs_field {
 #define EXIT_REASON_TASK_SWITCH         9
 #define EXIT_REASON_CPUID               10
 #define EXIT_REASON_HLT                 12
+#define EXIT_REASON_INVD                13
 #define EXIT_REASON_INVLPG              14
 #define EXIT_REASON_RDPMC               15
 #define EXIT_REASON_RDTSC               16
@@ -296,6 +305,12 @@ enum vmcs_field {
 #define GUEST_INTR_STATE_SMI            0x00000004
 #define GUEST_INTR_STATE_NMI            0x00000008
 
+/* GUEST_ACTIVITY_STATE flags */
+#define GUEST_ACTIVITY_ACTIVE           0
+#define GUEST_ACTIVITY_HLT              1
+#define GUEST_ACTIVITY_SHUTDOWN         2
+#define GUEST_ACTIVITY_WAIT_SIPI        3
+
 /*
  * Exit Qualifications for MOV for Control Register Access
  */
diff --git a/arch/x86/include/asm/xen/page.h b/arch/x86/include/asm/xen/page.h
index 8760cc60a21c..f25bdf238a33 100644
--- a/arch/x86/include/asm/xen/page.h
+++ b/arch/x86/include/asm/xen/page.h
@@ -42,6 +42,11 @@ extern unsigned int   machine_to_phys_order;
 extern unsigned long get_phys_to_machine(unsigned long pfn);
 extern bool set_phys_to_machine(unsigned long pfn, unsigned long mfn);
 
+extern int m2p_add_override(unsigned long mfn, struct page *page);
+extern int m2p_remove_override(struct page *page);
+extern struct page *m2p_find_override(unsigned long mfn);
+extern unsigned long m2p_find_override_pfn(unsigned long mfn, unsigned long pfn);
+
 static inline unsigned long pfn_to_mfn(unsigned long pfn)
 {
         unsigned long mfn;
@@ -72,9 +77,6 @@ static inline unsigned long mfn_to_pfn(unsigned long mfn)
         if (xen_feature(XENFEAT_auto_translated_physmap))
                 return mfn;
 
-        if (unlikely((mfn >> machine_to_phys_order) != 0))
-                return ~0;
-
         pfn = 0;
         /*
          * The array access can fail (e.g., device space beyond end of RAM).
@@ -83,6 +85,14 @@ static inline unsigned long mfn_to_pfn(unsigned long mfn)
          */
         __get_user(pfn, &machine_to_phys_mapping[mfn]);
 
+        /*
+         * If this appears to be a foreign mfn (because the pfn
+         * doesn't map back to the mfn), then check the local override
+         * table to see if there's a better pfn to use.
+         */
+        if (get_phys_to_machine(pfn) != mfn)
+                pfn = m2p_find_override_pfn(mfn, pfn);
+
         return pfn;
 }
 
diff --git a/arch/x86/kernel/acpi/boot.c b/arch/x86/kernel/acpi/boot.c
index ec881c6bfee0..b3a71137983a 100644
--- a/arch/x86/kernel/acpi/boot.c
+++ b/arch/x86/kernel/acpi/boot.c
@@ -509,6 +509,7 @@ int acpi_gsi_to_irq(u32 gsi, unsigned int *irq)
 
         return 0;
 }
+EXPORT_SYMBOL_GPL(acpi_gsi_to_irq);
 
 int acpi_isa_irq_to_gsi(unsigned isa_irq, u32 *gsi)
 {
diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c
index d2fdb0826df2..57ca77787220 100644
--- a/arch/x86/kernel/amd_iommu.c
+++ b/arch/x86/kernel/amd_iommu.c
@@ -1086,7 +1086,7 @@ static int alloc_new_range(struct dma_ops_domain *dma_dom,
 
         dma_dom->aperture_size += APERTURE_RANGE_SIZE;
 
-        /* Intialize the exclusion range if necessary */
+        /* Initialize the exclusion range if necessary */
         for_each_iommu(iommu) {
                 if (iommu->exclusion_start &&
                     iommu->exclusion_start >= dma_dom->aperture[index]->offset
@@ -1353,7 +1353,7 @@ static void dma_ops_domain_free(struct dma_ops_domain *dom)
 
 /*
  * Allocates a new protection domain usable for the dma_ops functions.
- * It also intializes the page table and the address allocator data
+ * It also initializes the page table and the address allocator data
  * structures required for the dma_ops interface
  */
 static struct dma_ops_domain *dma_ops_domain_alloc(void)
diff --git a/arch/x86/kernel/dumpstack.c b/arch/x86/kernel/dumpstack.c
index d6fb146c0d8b..df20723a6a1b 100644
--- a/arch/x86/kernel/dumpstack.c
+++ b/arch/x86/kernel/dumpstack.c
@@ -234,6 +234,7 @@ unsigned __kprobes long oops_begin(void)
         bust_spinlocks(1);
         return flags;
 }
+EXPORT_SYMBOL_GPL(oops_begin);
 
 void __kprobes oops_end(unsigned long flags, struct pt_regs *regs, int signr)
 {
diff --git a/arch/x86/kernel/e820.c b/arch/x86/kernel/e820.c
index 0c2b7ef7a34d..294f26da0c0c 100644
--- a/arch/x86/kernel/e820.c
+++ b/arch/x86/kernel/e820.c
@@ -14,6 +14,7 @@
 #include <linux/bootmem.h>
 #include <linux/pfn.h>
 #include <linux/suspend.h>
+#include <linux/acpi.h>
 #include <linux/firmware-map.h>
 #include <linux/memblock.h>
 
diff --git a/arch/x86/kernel/entry_32.S b/arch/x86/kernel/entry_32.S
index 591e60104278..c8b4efad7ebb 100644
--- a/arch/x86/kernel/entry_32.S
+++ b/arch/x86/kernel/entry_32.S
@@ -1406,6 +1406,16 @@ ENTRY(general_protection)
         CFI_ENDPROC
 END(general_protection)
 
+#ifdef CONFIG_KVM_GUEST
+ENTRY(async_page_fault)
+        RING0_EC_FRAME
+        pushl $do_async_page_fault
+        CFI_ADJUST_CFA_OFFSET 4
+        jmp error_code
+        CFI_ENDPROC
+END(apf_page_fault)
+#endif
+
 /*
  * End of kprobes section
  */
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index d3b895f375d3..aed1ffbeb0c9 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -1329,6 +1329,9 @@ errorentry xen_stack_segment do_stack_segment
 #endif
 errorentry general_protection do_general_protection
 errorentry page_fault do_page_fault
+#ifdef CONFIG_KVM_GUEST
+errorentry async_page_fault do_async_page_fault
+#endif
 #ifdef CONFIG_X86_MCE
 paranoidzeroentry machine_check *machine_check_vector(%rip)
 #endif
diff --git a/arch/x86/kernel/head_32.S b/arch/x86/kernel/head_32.S
index 9f54b209c378..fc293dc8dc35 100644
--- a/arch/x86/kernel/head_32.S
+++ b/arch/x86/kernel/head_32.S
@@ -126,7 +126,7 @@ ENTRY(startup_32)
         movsl
         movl pa(boot_params) + NEW_CL_POINTER,%esi
         andl %esi,%esi
-        jz 1f                   # No comand line
+        jz 1f                   # No command line
         movl $pa(boot_command_line),%edi
         movl $(COMMAND_LINE_SIZE/4),%ecx
         rep
diff --git a/arch/x86/kernel/i387.c b/arch/x86/kernel/i387.c
index 58bb239a2fd7..e60c38cc0eed 100644
--- a/arch/x86/kernel/i387.c
+++ b/arch/x86/kernel/i387.c
@@ -169,6 +169,7 @@ int init_fpu(struct task_struct *tsk)
         set_stopped_child_used_math(tsk);
         return 0;
 }
+EXPORT_SYMBOL_GPL(init_fpu);
 
 /*
  * The xstateregs_active() routine is the same as the fpregs_active() routine,
diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c
index 3a43caa3beb7..52945da52a94 100644
--- a/arch/x86/kernel/irq.c
+++ b/arch/x86/kernel/irq.c
@@ -4,6 +4,7 @@
 #include <linux/cpu.h>
 #include <linux/interrupt.h>
 #include <linux/kernel_stat.h>
+#include <linux/of.h>
 #include <linux/seq_file.h>
 #include <linux/smp.h>
 #include <linux/ftrace.h>
@@ -275,6 +276,15 @@ void smp_x86_platform_ipi(struct pt_regs *regs)
 
 EXPORT_SYMBOL_GPL(vector_used_by_percpu_irq);
 
+#ifdef CONFIG_OF
+unsigned int irq_create_of_mapping(struct device_node *controller,
+                const u32 *intspec, unsigned int intsize)
+{
+        return intspec[0];
+}
+EXPORT_SYMBOL_GPL(irq_create_of_mapping);
+#endif
+
 #ifdef CONFIG_HOTPLUG_CPU
 /* A cpu has been removed from cpu_online_mask.  Reset irq affinities. */
 void fixup_irqs(void)
diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c
index 63b0ec8d3d4a..8dc44662394b 100644
--- a/arch/x86/kernel/kvm.c
+++ b/arch/x86/kernel/kvm.c
@@ -27,16 +27,37 @@
 #include <linux/mm.h>
 #include <linux/highmem.h>
 #include <linux/hardirq.h>
+#include <linux/notifier.h>
+#include <linux/reboot.h>
+#include <linux/hash.h>
+#include <linux/sched.h>
+#include <linux/slab.h>
+#include <linux/kprobes.h>
 #include <asm/timer.h>
+#include <asm/cpu.h>
+#include <asm/traps.h>
+#include <asm/desc.h>
+#include <asm/tlbflush.h>
 
 #define MMU_QUEUE_SIZE 1024
 
+static int kvmapf = 1;
+
+static int parse_no_kvmapf(char *arg)
+{
+        kvmapf = 0;
+        return 0;
+}
+
+early_param("no-kvmapf", parse_no_kvmapf);
+
 struct kvm_para_state {
         u8 mmu_queue[MMU_QUEUE_SIZE];
         int mmu_queue_len;
 };
 
 static DEFINE_PER_CPU(struct kvm_para_state, para_state);
+static DEFINE_PER_CPU(struct kvm_vcpu_pv_apf_data, apf_reason) __aligned(64);
 
 static struct kvm_para_state *kvm_para_state(void)
 {
@@ -50,6 +71,195 @@ static void kvm_io_delay(void)
 {
 }
 
+#define KVM_TASK_SLEEP_HASHBITS 8
+#define KVM_TASK_SLEEP_HASHSIZE (1<<KVM_TASK_SLEEP_HASHBITS)
+
+struct kvm_task_sleep_node {
+        struct hlist_node link;
+        wait_queue_head_t wq;
+        u32 token;
+        int cpu;
+        bool halted;
+        struct mm_struct *mm;
+};
+
+static struct kvm_task_sleep_head {
+        spinlock_t lock;
+        struct hlist_head list;
+} async_pf_sleepers[KVM_TASK_SLEEP_HASHSIZE];
+
+static struct kvm_task_sleep_node *_find_apf_task(struct kvm_task_sleep_head *b,
+                                                  u32 token)
+{
+        struct hlist_node *p;
+
+        hlist_for_each(p, &b->list) {
+                struct kvm_task_sleep_node *n =
+                        hlist_entry(p, typeof(*n), link);
+                if (n->token == token)
+                        return n;
+        }
+
+        return NULL;
+}
+
+void kvm_async_pf_task_wait(u32 token)
+{
+        u32 key = hash_32(token, KVM_TASK_SLEEP_HASHBITS);
+        struct kvm_task_sleep_head *b = &async_pf_sleepers[key];
+        struct kvm_task_sleep_node n, *e;
+        DEFINE_WAIT(wait);
+        int cpu, idle;
+
+        cpu = get_cpu();
+        idle = idle_cpu(cpu);
+        put_cpu();
+
+        spin_lock(&b->lock);
+        e = _find_apf_task(b, token);
+        if (e) {
+                /* dummy entry exist -> wake up was delivered ahead of PF */
+                hlist_del(&e->link);
+                kfree(e);
+                spin_unlock(&b->lock);
+                return;
+        }
+
+        n.token = token;
+        n.cpu = smp_processor_id();
+        n.mm = current->active_mm;
+        n.halted = idle || preempt_count() > 1;
+        atomic_inc(&n.mm->mm_count);
+        init_waitqueue_head(&n.wq);
+        hlist_add_head(&n.link, &b->list);
+        spin_unlock(&b->lock);
+
+        for (;;) {
+                if (!n.halted)
+                        prepare_to_wait(&n.wq, &wait, TASK_UNINTERRUPTIBLE);
+                if (hlist_unhashed(&n.link))
+                        break;
+
+                if (!n.halted) {
+                        local_irq_enable();
+                        schedule();
+                        local_irq_disable();
+                } else {
+                        /*
+                         * We cannot reschedule. So halt.
+                         */
+                        native_safe_halt();
+                        local_irq_disable();
+                }
+        }
+        if (!n.halted)
+                finish_wait(&n.wq, &wait);
+
+        return;
+}
+EXPORT_SYMBOL_GPL(kvm_async_pf_task_wait);
+
+static void apf_task_wake_one(struct kvm_task_sleep_node *n)
+{
+        hlist_del_init(&n->link);
+        if (!n->mm)
+                return;
+        mmdrop(n->mm);
+        if (n->halted)
+                smp_send_reschedule(n->cpu);
+        else if (waitqueue_active(&n->wq))
+                wake_up(&n->wq);
+}
+
+static void apf_task_wake_all(void)
+{
+        int i;
+
+        for (i = 0; i < KVM_TASK_SLEEP_HASHSIZE; i++) {
+                struct hlist_node *p, *next;
+                struct kvm_task_sleep_head *b = &async_pf_sleepers[i];
+                spin_lock(&b->lock);
+                hlist_for_each_safe(p, next, &b->list) {
+                        struct kvm_task_sleep_node *n =
+                                hlist_entry(p, typeof(*n), link);
+                        if (n->cpu == smp_processor_id())
+                                apf_task_wake_one(n);
+                }
+                spin_unlock(&b->lock);
+        }
+}
+
+void kvm_async_pf_task_wake(u32 token)
+{
+        u32 key = hash_32(token, KVM_TASK_SLEEP_HASHBITS);
+        struct kvm_task_sleep_head *b = &async_pf_sleepers[key];
+        struct kvm_task_sleep_node *n;
+
+        if (token == ~0) {
+                apf_task_wake_all();
+                return;
+        }
+
+again:
+        spin_lock(&b->lock);
+        n = _find_apf_task(b, token);
+        if (!n) {
+                /*
+                 * async PF was not yet handled.
+                 * Add dummy entry for the token.
+                 */
+                n = kmalloc(sizeof(*n), GFP_ATOMIC);
+                if (!n) {
+                        /*
+                         * Allocation failed! Busy wait while other cpu
+                         * handles async PF.
+                         */
+                        spin_unlock(&b->lock);
+                        cpu_relax();
+                        goto again;
+                }
+                n->token = token;
+                n->cpu = smp_processor_id();
+                n->mm = NULL;
+                init_waitqueue_head(&n->wq);
+                hlist_add_head(&n->link, &b->list);
+        } else
+                apf_task_wake_one(n);
+        spin_unlock(&b->lock);
+        return;
+}
+EXPORT_SYMBOL_GPL(kvm_async_pf_task_wake);
+
+u32 kvm_read_and_reset_pf_reason(void)
+{
+        u32 reason = 0;
+
+        if (__get_cpu_var(apf_reason).enabled) {
+                reason = __get_cpu_var(apf_reason).reason;
+                __get_cpu_var(apf_reason).reason = 0;
+        }
+
+        return reason;
+}
+EXPORT_SYMBOL_GPL(kvm_read_and_reset_pf_reason);
+
+dotraplinkage void __kprobes
+do_async_page_fault(struct pt_regs *regs, unsigned long error_code)
+{
+        switch (kvm_read_and_reset_pf_reason()) {
+        default:
+                do_page_fault(regs, error_code);
+                break;
+        case KVM_PV_REASON_PAGE_NOT_PRESENT:
+                /* page is swapped out by the host. */
+                kvm_async_pf_task_wait((u32)read_cr2());
+                break;
+        case KVM_PV_REASON_PAGE_READY:
+                kvm_async_pf_task_wake((u32)read_cr2());
+                break;
+        }
+}
+
 static void kvm_mmu_op(void *buffer, unsigned len)
 {
         int r;
@@ -231,10 +441,117 @@ static void __init paravirt_ops_setup(void)
 #endif
 }
 
+void __cpuinit kvm_guest_cpu_init(void)
+{
+        if (!kvm_para_available())
+                return;
+
+        if (kvm_para_has_feature(KVM_FEATURE_ASYNC_PF) && kvmapf) {
+                u64 pa = __pa(&__get_cpu_var(apf_reason));
+
+#ifdef CONFIG_PREEMPT
+                pa |= KVM_ASYNC_PF_SEND_ALWAYS;
+#endif
+                wrmsrl(MSR_KVM_ASYNC_PF_EN, pa | KVM_ASYNC_PF_ENABLED);
+                __get_cpu_var(apf_reason).enabled = 1;
+                printk(KERN_INFO"KVM setup async PF for cpu %d\n",
+                       smp_processor_id());
+        }
+}
+
+static void kvm_pv_disable_apf(void *unused)
+{
+        if (!__get_cpu_var(apf_reason).enabled)
+                return;
+
+        wrmsrl(MSR_KVM_ASYNC_PF_EN, 0);
+        __get_cpu_var(apf_reason).enabled = 0;
+
+        printk(KERN_INFO"Unregister pv shared memory for cpu %d\n",
+               smp_processor_id());
+}
+
+static int kvm_pv_reboot_notify(struct notifier_block *nb,
+                                unsigned long code, void *unused)
+{
+        if (code == SYS_RESTART)
+                on_each_cpu(kvm_pv_disable_apf, NULL, 1);
+        return NOTIFY_DONE;
+}
+
+static struct notifier_block kvm_pv_reboot_nb = {
+        .notifier_call = kvm_pv_reboot_notify,
+};
+
+#ifdef CONFIG_SMP
+static void __init kvm_smp_prepare_boot_cpu(void)
+{
+#ifdef CONFIG_KVM_CLOCK
+        WARN_ON(kvm_register_clock("primary cpu clock"));
+#endif
+        kvm_guest_cpu_init();
+        native_smp_prepare_boot_cpu();
+}
+
+static void kvm_guest_cpu_online(void *dummy)
+{
+        kvm_guest_cpu_init();
+}
+
+static void kvm_guest_cpu_offline(void *dummy)
+{
+        kvm_pv_disable_apf(NULL);
+        apf_task_wake_all();
+}
+
+static int __cpuinit kvm_cpu_notify(struct notifier_block *self,
+                                    unsigned long action, void *hcpu)
+{
+        int cpu = (unsigned long)hcpu;
+        switch (action) {
+        case CPU_ONLINE:
+        case CPU_DOWN_FAILED:
+        case CPU_ONLINE_FROZEN:
+                smp_call_function_single(cpu, kvm_guest_cpu_online, NULL, 0);
+                break;
+        case CPU_DOWN_PREPARE:
+        case CPU_DOWN_PREPARE_FROZEN:
+                smp_call_function_single(cpu, kvm_guest_cpu_offline, NULL, 1);
+                break;
+        default:
+                break;
+        }
+        return NOTIFY_OK;
+}
+
+static struct notifier_block __cpuinitdata kvm_cpu_notifier = {
+        .notifier_call  = kvm_cpu_notify,
+};
+#endif
+
+static void __init kvm_apf_trap_init(void)
+{
+        set_intr_gate(14, &async_page_fault);
+}
+
 void __init kvm_guest_init(void)
 {
+        int i;
+
         if (!kvm_para_available())
                 return;
 
         paravirt_ops_setup();
+        register_reboot_notifier(&kvm_pv_reboot_nb);
+        for (i = 0; i < KVM_TASK_SLEEP_HASHSIZE; i++)
+                spin_lock_init(&async_pf_sleepers[i].lock);
+        if (kvm_para_has_feature(KVM_FEATURE_ASYNC_PF))
+                x86_init.irqs.trap_init = kvm_apf_trap_init;
+
+#ifdef CONFIG_SMP
+        smp_ops.smp_prepare_boot_cpu = kvm_smp_prepare_boot_cpu;
+        register_cpu_notifier(&kvm_cpu_notifier);
+#else
+        kvm_guest_cpu_init();
+#endif
 }
diff --git a/arch/x86/kernel/kvmclock.c b/arch/x86/kernel/kvmclock.c
index ca43ce31a19c..f98d3eafe07a 100644
--- a/arch/x86/kernel/kvmclock.c
+++ b/arch/x86/kernel/kvmclock.c
@@ -125,7 +125,7 @@ static struct clocksource kvm_clock = {
         .flags = CLOCK_SOURCE_IS_CONTINUOUS,
 };
 
-static int kvm_register_clock(char *txt)
+int kvm_register_clock(char *txt)
 {
         int cpu = smp_processor_id();
         int low, high, ret;
@@ -152,14 +152,6 @@ static void __cpuinit kvm_setup_secondary_clock(void)
 }
 #endif
 
-#ifdef CONFIG_SMP
-static void __init kvm_smp_prepare_boot_cpu(void)
-{
-        WARN_ON(kvm_register_clock("primary cpu clock"));
-        native_smp_prepare_boot_cpu();
-}
-#endif
-
 /*
  * After the clock is registered, the host will keep writing to the
  * registered memory location. If the guest happens to shutdown, this memory
@@ -206,9 +198,6 @@ void __init kvmclock_init(void)
         x86_cpuinit.setup_percpu_clockev =
                 kvm_setup_secondary_clock;
 #endif
-#ifdef CONFIG_SMP
-        smp_ops.smp_prepare_boot_cpu = kvm_smp_prepare_boot_cpu;
-#endif
         machine_ops.shutdown  = kvm_shutdown;
 #ifdef CONFIG_KEXEC
         machine_ops.crash_shutdown  = kvm_crash_shutdown;
diff --git a/arch/x86/kernel/module.c b/arch/x86/kernel/module.c
index 8f2956091735..ab23f1ad4bf1 100644
--- a/arch/x86/kernel/module.c
+++ b/arch/x86/kernel/module.c
@@ -37,20 +37,11 @@
 
 void *module_alloc(unsigned long size)
 {
-        struct vm_struct *area;
-
-        if (!size)
-                return NULL;
-        size = PAGE_ALIGN(size);
-        if (size > MODULES_LEN)
+        if (PAGE_ALIGN(size) > MODULES_LEN)
                 return NULL;
-
-        area = __get_vm_area(size, VM_ALLOC, MODULES_VADDR, MODULES_END);
-        if (!area)
-                return NULL;
-
-        return __vmalloc_area(area, GFP_KERNEL | __GFP_HIGHMEM,
-                                        PAGE_KERNEL_EXEC);
+        return __vmalloc_node_range(size, 1, MODULES_VADDR, MODULES_END,
+                                GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL_EXEC,
+                                -1, __builtin_return_address(0));
 }
 
 /* Free memory returned from module_alloc */
diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c
index c5b250011fd4..869e1aeeb71b 100644
--- a/arch/x86/kernel/paravirt.c
+++ b/arch/x86/kernel/paravirt.c
@@ -421,8 +421,11 @@ struct pv_mmu_ops pv_mmu_ops = {
         .set_pte = native_set_pte,
         .set_pte_at = native_set_pte_at,
         .set_pmd = native_set_pmd,
+        .set_pmd_at = native_set_pmd_at,
         .pte_update = paravirt_nop,
         .pte_update_defer = paravirt_nop,
+        .pmd_update = paravirt_nop,
+        .pmd_update_defer = paravirt_nop,
 
         .ptep_modify_prot_start = __ptep_modify_prot_start,
         .ptep_modify_prot_commit = __ptep_modify_prot_commit,
diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c
index 09c08a1c706f..d8286ed54ffa 100644
--- a/arch/x86/kernel/process.c
+++ b/arch/x86/kernel/process.c
@@ -22,11 +22,6 @@
 #include <asm/i387.h>
 #include <asm/debugreg.h>
 
-unsigned long idle_halt;
-EXPORT_SYMBOL(idle_halt);
-unsigned long idle_nomwait;
-EXPORT_SYMBOL(idle_nomwait);
-
 struct kmem_cache *task_xstate_cachep;
 EXPORT_SYMBOL_GPL(task_xstate_cachep);
 
@@ -327,7 +322,7 @@ long sys_execve(const char __user *name,
 /*
  * Idle related variables and functions
  */
-unsigned long boot_option_idle_override = 0;
+unsigned long boot_option_idle_override = IDLE_NO_OVERRIDE;
 EXPORT_SYMBOL(boot_option_idle_override);
 
 /*
@@ -386,6 +381,8 @@ void default_idle(void)
                 else
                         local_irq_enable();
                 current_thread_info()->status |= TS_POLLING;
+                trace_power_end(smp_processor_id());
+                trace_cpu_idle(PWR_EVENT_EXIT, smp_processor_id());
         } else {
                 local_irq_enable();
                 /* loop is done by the caller */
@@ -443,8 +440,6 @@ EXPORT_SYMBOL_GPL(cpu_idle_wait);
  */
 void mwait_idle_with_hints(unsigned long ax, unsigned long cx)
 {
-        trace_power_start(POWER_CSTATE, (ax>>4)+1, smp_processor_id());
-        trace_cpu_idle((ax>>4)+1, smp_processor_id());
         if (!need_resched()) {
                 if (cpu_has(__this_cpu_ptr(&cpu_info), X86_FEATURE_CLFLUSH_MONITOR))
                         clflush((void *)&current_thread_info()->flags);
@@ -471,6 +466,8 @@ static void mwait_idle(void)
                         __sti_mwait(0, 0);
                 else
                         local_irq_enable();
+                trace_power_end(smp_processor_id());
+                trace_cpu_idle(PWR_EVENT_EXIT, smp_processor_id());
         } else
                 local_irq_enable();
 }
@@ -503,7 +500,6 @@ static void poll_idle(void)
  *
  * idle=mwait overrides this decision and forces the usage of mwait.
  */
-static int __cpuinitdata force_mwait;
 
 #define MWAIT_INFO                      0x05
 #define MWAIT_ECX_EXTENDED_INFO         0x01
@@ -513,7 +509,7 @@ static int __cpuinit mwait_usable(const struct cpuinfo_x86 *c)
 {
         u32 eax, ebx, ecx, edx;
 
-        if (force_mwait)
+        if (boot_option_idle_override == IDLE_FORCE_MWAIT)
                 return 1;
 
         if (c->cpuid_level < MWAIT_INFO)
@@ -633,9 +629,10 @@ static int __init idle_setup(char *str)
         if (!strcmp(str, "poll")) {
                 printk("using polling idle threads.\n");
                 pm_idle = poll_idle;
-        } else if (!strcmp(str, "mwait"))
-                force_mwait = 1;
-        else if (!strcmp(str, "halt")) {
+                boot_option_idle_override = IDLE_POLL;
+        } else if (!strcmp(str, "mwait")) {
+                boot_option_idle_override = IDLE_FORCE_MWAIT;
+        } else if (!strcmp(str, "halt")) {
                 /*
                  * When the boot option of idle=halt is added, halt is
                  * forced to be used for CPU idle. In such case CPU C2/C3
@@ -644,8 +641,7 @@ static int __init idle_setup(char *str)
                  * the boot_option_idle_override.
                  */
                 pm_idle = default_idle;
-                idle_halt = 1;
-                return 0;
+                boot_option_idle_override = IDLE_HALT;
         } else if (!strcmp(str, "nomwait")) {
                 /*
                  * If the boot option of "idle=nomwait" is added,
@@ -653,12 +649,10 @@ static int __init idle_setup(char *str)
                  * states. In such case it won't touch the variable
                  * of boot_option_idle_override.
                  */
-                idle_nomwait = 1;
-                return 0;
+                boot_option_idle_override = IDLE_NOMWAIT;
         } else
                 return -1;
 
-        boot_option_idle_override = 1;
         return 0;
 }
 early_param("idle", idle_setup);
diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c
index 4b9befa0e347..8d128783af47 100644
--- a/arch/x86/kernel/process_32.c
+++ b/arch/x86/kernel/process_32.c
@@ -57,8 +57,6 @@
 #include <asm/syscalls.h>
 #include <asm/debugreg.h>
 
-#include <trace/events/power.h>
-
 asmlinkage void ret_from_fork(void) __asm__("ret_from_fork");
 
 /*
@@ -113,8 +111,6 @@ void cpu_idle(void)
                         stop_critical_timings();
                         pm_idle();
                         start_critical_timings();
-                        trace_power_end(smp_processor_id());
-                        trace_cpu_idle(PWR_EVENT_EXIT, smp_processor_id());
                 }
                 tick_nohz_restart_sched_tick();
                 preempt_enable_no_resched();
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index 4c818a738396..bd387e8f73b4 100644
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -51,8 +51,6 @@
 #include <asm/syscalls.h>
 #include <asm/debugreg.h>
 
-#include <trace/events/power.h>
-
 asmlinkage extern void ret_from_fork(void);
 
 DEFINE_PER_CPU(unsigned long, old_rsp);
@@ -141,10 +139,6 @@ void cpu_idle(void)
                         pm_idle();
                         start_critical_timings();
 
-                        trace_power_end(smp_processor_id());
-                        trace_cpu_idle(PWR_EVENT_EXIT,
-                                       smp_processor_id());
-
                         /* In many cases the interrupt that ended idle
                            has already called exit_idle. But some idle
                            loops can be woken up without interrupt. */
diff --git a/arch/x86/kernel/rtc.c b/arch/x86/kernel/rtc.c
index 1cfbbfc3ae26..6f39cab052d5 100644
--- a/arch/x86/kernel/rtc.c
+++ b/arch/x86/kernel/rtc.c
@@ -76,7 +76,7 @@ int mach_set_rtc_mmss(unsigned long nowtime)
                 CMOS_WRITE(real_seconds, RTC_SECONDS);
                 CMOS_WRITE(real_minutes, RTC_MINUTES);
         } else {
-                printk(KERN_WARNING
+                printk_once(KERN_NOTICE
                        "set_rtc_mmss: can't update from %d to %d\n",
                        cmos_minutes, real_minutes);
                 retval = -1;
diff --git a/arch/x86/kernel/tboot.c b/arch/x86/kernel/tboot.c
index c2f1b26141e2..998e972f3b1a 100644
--- a/arch/x86/kernel/tboot.c
+++ b/arch/x86/kernel/tboot.c
@@ -133,7 +133,7 @@ static int map_tboot_page(unsigned long vaddr, unsigned long pfn,
         pmd = pmd_alloc(&tboot_mm, pud, vaddr);
         if (!pmd)
                 return -1;
-        pte = pte_alloc_map(&tboot_mm, pmd, vaddr);
+        pte = pte_alloc_map(&tboot_mm, NULL, pmd, vaddr);
         if (!pte)
                 return -1;
         set_pte_at(&tboot_mm, vaddr, pte, pfn_pte(pfn, prot));
diff --git a/arch/x86/kernel/vm86_32.c b/arch/x86/kernel/vm86_32.c
index 61fb98519622..863f8753ab0a 100644
--- a/arch/x86/kernel/vm86_32.c
+++ b/arch/x86/kernel/vm86_32.c
@@ -179,6 +179,7 @@ static void mark_screen_rdonly(struct mm_struct *mm)
         if (pud_none_or_clear_bad(pud))
                 goto out;
         pmd = pmd_offset(pud, 0xA0000);
+        split_huge_page_pmd(mm, pmd);
         if (pmd_none_or_clear_bad(pmd))
                 goto out;
         pte = pte_offset_map_lock(mm, pmd, 0xA0000, &ptl);
diff --git a/arch/x86/kvm/Kconfig b/arch/x86/kvm/Kconfig
index ddc131ff438f..50f63648ce1b 100644
--- a/arch/x86/kvm/Kconfig
+++ b/arch/x86/kvm/Kconfig
@@ -28,6 +28,7 @@ config KVM
         select HAVE_KVM_IRQCHIP
         select HAVE_KVM_EVENTFD
         select KVM_APIC_ARCHITECTURE
+        select KVM_ASYNC_PF
         select USER_RETURN_NOTIFIER
         select KVM_MMIO
         ---help---
diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile
index 31a7035c4bd9..f15501f431c8 100644
--- a/arch/x86/kvm/Makefile
+++ b/arch/x86/kvm/Makefile
@@ -1,5 +1,5 @@
 
-EXTRA_CFLAGS += -Ivirt/kvm -Iarch/x86/kvm
+ccflags-y += -Ivirt/kvm -Iarch/x86/kvm
 
 CFLAGS_x86.o := -I.
 CFLAGS_svm.o := -I.
@@ -9,6 +9,7 @@ kvm-y			+= $(addprefix ../../../virt/kvm/, kvm_main.o ioapic.o \
                                 coalesced_mmio.o irq_comm.o eventfd.o \
                                 assigned-dev.o)
 kvm-$(CONFIG_IOMMU_API) += $(addprefix ../../../virt/kvm/, iommu.o)
+kvm-$(CONFIG_KVM_ASYNC_PF)      += $(addprefix ../../../virt/kvm/, async_pf.o)
 
 kvm-y                   += x86.o mmu.o emulate.o i8259.o irq.o lapic.o \
                            i8254.o timer.o
diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c
index 38b6e8dafaff..caf966781d25 100644
--- a/arch/x86/kvm/emulate.c
+++ b/arch/x86/kvm/emulate.c
@@ -20,16 +20,8 @@
  * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
  */
 
-#ifndef __KERNEL__
-#include <stdio.h>
-#include <stdint.h>
-#include <public/xen.h>
-#define DPRINTF(_f, _a ...) printf(_f , ## _a)
-#else
 #include <linux/kvm_host.h>
 #include "kvm_cache_regs.h"
-#define DPRINTF(x...) do {} while (0)
-#endif
 #include <linux/module.h>
 #include <asm/kvm_emulate.h>
 
@@ -418,9 +410,9 @@ address_mask(struct decode_cache *c, unsigned long reg)
 }
 
 static inline unsigned long
-register_address(struct decode_cache *c, unsigned long base, unsigned long reg)
+register_address(struct decode_cache *c, unsigned long reg)
 {
-        return base + address_mask(c, reg);
+        return address_mask(c, reg);
 }
 
 static inline void
@@ -452,60 +444,55 @@ static unsigned long seg_base(struct x86_emulate_ctxt *ctxt,
         return ops->get_cached_segment_base(seg, ctxt->vcpu);
 }
 
-static unsigned long seg_override_base(struct x86_emulate_ctxt *ctxt,
-                                       struct x86_emulate_ops *ops,
-                                       struct decode_cache *c)
+static unsigned seg_override(struct x86_emulate_ctxt *ctxt,
+                             struct x86_emulate_ops *ops,
+                             struct decode_cache *c)
 {
         if (!c->has_seg_override)
                 return 0;
 
-        return seg_base(ctxt, ops, c->seg_override);
+        return c->seg_override;
 }
 
-static unsigned long es_base(struct x86_emulate_ctxt *ctxt,
-                             struct x86_emulate_ops *ops)
+static ulong linear(struct x86_emulate_ctxt *ctxt,
+                    struct segmented_address addr)
 {
-        return seg_base(ctxt, ops, VCPU_SREG_ES);
-}
-
-static unsigned long ss_base(struct x86_emulate_ctxt *ctxt,
-                             struct x86_emulate_ops *ops)
-{
-        return seg_base(ctxt, ops, VCPU_SREG_SS);
-}
+        struct decode_cache *c = &ctxt->decode;
+        ulong la;
 
-static void emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
-                                      u32 error, bool valid)
-{
-        ctxt->exception = vec;
-        ctxt->error_code = error;
-        ctxt->error_code_valid = valid;
+        la = seg_base(ctxt, ctxt->ops, addr.seg) + addr.ea;
+        if (c->ad_bytes != 8)
+                la &= (u32)-1;
+        return la;
 }
 
-static void emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
+static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
+                             u32 error, bool valid)
 {
-        emulate_exception(ctxt, GP_VECTOR, err, true);
+        ctxt->exception.vector = vec;
+        ctxt->exception.error_code = error;
+        ctxt->exception.error_code_valid = valid;
+        return X86EMUL_PROPAGATE_FAULT;
 }
 
-static void emulate_pf(struct x86_emulate_ctxt *ctxt)
+static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
 {
-        emulate_exception(ctxt, PF_VECTOR, 0, true);
+        return emulate_exception(ctxt, GP_VECTOR, err, true);
 }
 
-static void emulate_ud(struct x86_emulate_ctxt *ctxt)
+static int emulate_ud(struct x86_emulate_ctxt *ctxt)
 {
-        emulate_exception(ctxt, UD_VECTOR, 0, false);
+        return emulate_exception(ctxt, UD_VECTOR, 0, false);
 }
 
-static void emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
+static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
 {
-        emulate_exception(ctxt, TS_VECTOR, err, true);
+        return emulate_exception(ctxt, TS_VECTOR, err, true);
 }
 
 static int emulate_de(struct x86_emulate_ctxt *ctxt)
 {
-        emulate_exception(ctxt, DE_VECTOR, 0, false);
-        return X86EMUL_PROPAGATE_FAULT;
+        return emulate_exception(ctxt, DE_VECTOR, 0, false);
 }
 
 static int do_fetch_insn_byte(struct x86_emulate_ctxt *ctxt,
@@ -520,7 +507,7 @@ static int do_fetch_insn_byte(struct x86_emulate_ctxt *ctxt,
                 cur_size = fc->end - fc->start;
                 size = min(15UL - cur_size, PAGE_SIZE - offset_in_page(eip));
                 rc = ops->fetch(ctxt->cs_base + eip, fc->data + cur_size,
-                                size, ctxt->vcpu, NULL);
+                                size, ctxt->vcpu, &ctxt->exception);
                 if (rc != X86EMUL_CONTINUE)
                         return rc;
                 fc->end += size;
@@ -564,7 +551,7 @@ static void *decode_register(u8 modrm_reg, unsigned long *regs,
 
 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
                            struct x86_emulate_ops *ops,
-                           ulong addr,
+                           struct segmented_address addr,
                            u16 *size, unsigned long *address, int op_bytes)
 {
         int rc;
@@ -572,10 +559,13 @@ static int read_descriptor(struct x86_emulate_ctxt *ctxt,
         if (op_bytes == 2)
                 op_bytes = 3;
         *address = 0;
-        rc = ops->read_std(addr, (unsigned long *)size, 2, ctxt->vcpu, NULL);
+        rc = ops->read_std(linear(ctxt, addr), (unsigned long *)size, 2,
+                           ctxt->vcpu, &ctxt->exception);
         if (rc != X86EMUL_CONTINUE)
                 return rc;
-        rc = ops->read_std(addr + 2, address, op_bytes, ctxt->vcpu, NULL);
+        addr.ea += 2;
+        rc = ops->read_std(linear(ctxt, addr), address, op_bytes,
+                           ctxt->vcpu, &ctxt->exception);
         return rc;
 }
 
@@ -768,7 +758,7 @@ static int decode_modrm(struct x86_emulate_ctxt *ctxt,
                         break;
                 }
         }
-        op->addr.mem = modrm_ea;
+        op->addr.mem.ea = modrm_ea;
 done:
         return rc;
 }
@@ -783,13 +773,13 @@ static int decode_abs(struct x86_emulate_ctxt *ctxt,
         op->type = OP_MEM;
         switch (c->ad_bytes) {
         case 2:
-                op->addr.mem = insn_fetch(u16, 2, c->eip);
+                op->addr.mem.ea = insn_fetch(u16, 2, c->eip);
                 break;
         case 4:
-                op->addr.mem = insn_fetch(u32, 4, c->eip);
+                op->addr.mem.ea = insn_fetch(u32, 4, c->eip);
                 break;
         case 8:
-                op->addr.mem = insn_fetch(u64, 8, c->eip);
+                op->addr.mem.ea = insn_fetch(u64, 8, c->eip);
                 break;
         }
 done:
@@ -808,7 +798,7 @@ static void fetch_bit_operand(struct decode_cache *c)
                 else if (c->src.bytes == 4)
                         sv = (s32)c->src.val & (s32)mask;
 
-                c->dst.addr.mem += (sv >> 3);
+                c->dst.addr.mem.ea += (sv >> 3);
         }
 
         /* only subword offset */
@@ -821,7 +811,6 @@ static int read_emulated(struct x86_emulate_ctxt *ctxt,
 {
         int rc;
         struct read_cache *mc = &ctxt->decode.mem_read;
-        u32 err;
 
         while (size) {
                 int n = min(size, 8u);
@@ -829,10 +818,8 @@ static int read_emulated(struct x86_emulate_ctxt *ctxt,
                 if (mc->pos < mc->end)
                         goto read_cached;
 
-                rc = ops->read_emulated(addr, mc->data + mc->end, n, &err,
-                                        ctxt->vcpu);
-                if (rc == X86EMUL_PROPAGATE_FAULT)
-                        emulate_pf(ctxt);
+                rc = ops->read_emulated(addr, mc->data + mc->end, n,
+                                        &ctxt->exception, ctxt->vcpu);
                 if (rc != X86EMUL_CONTINUE)
                         return rc;
                 mc->end += n;
@@ -907,19 +894,15 @@ static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
         struct desc_ptr dt;
         u16 index = selector >> 3;
         int ret;
-        u32 err;
         ulong addr;
 
         get_descriptor_table_ptr(ctxt, ops, selector, &dt);
 
-        if (dt.size < index * 8 + 7) {
-                emulate_gp(ctxt, selector & 0xfffc);
-                return X86EMUL_PROPAGATE_FAULT;
-        }
+        if (dt.size < index * 8 + 7)
+                return emulate_gp(ctxt, selector & 0xfffc);
         addr = dt.address + index * 8;
-        ret = ops->read_std(addr, desc, sizeof *desc, ctxt->vcpu,  &err);
-        if (ret == X86EMUL_PROPAGATE_FAULT)
-                emulate_pf(ctxt);
+        ret = ops->read_std(addr, desc, sizeof *desc, ctxt->vcpu,
+                            &ctxt->exception);
 
        return ret;
 }
@@ -931,21 +914,17 @@ static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
 {
         struct desc_ptr dt;
         u16 index = selector >> 3;
-        u32 err;
         ulong addr;
         int ret;
 
         get_descriptor_table_ptr(ctxt, ops, selector, &dt);
 
-        if (dt.size < index * 8 + 7) {
-                emulate_gp(ctxt, selector & 0xfffc);
-                return X86EMUL_PROPAGATE_FAULT;
-        }
+        if (dt.size < index * 8 + 7)
+                return emulate_gp(ctxt, selector & 0xfffc);
 
         addr = dt.address + index * 8;
-        ret = ops->write_std(addr, desc, sizeof *desc, ctxt->vcpu, &err);
-        if (ret == X86EMUL_PROPAGATE_FAULT)
-                emulate_pf(ctxt);
+        ret = ops->write_std(addr, desc, sizeof *desc, ctxt->vcpu,
+                             &ctxt->exception);
 
         return ret;
 }
@@ -1092,7 +1071,6 @@ static inline int writeback(struct x86_emulate_ctxt *ctxt,
 {
         int rc;
         struct decode_cache *c = &ctxt->decode;
-        u32 err;
 
         switch (c->dst.type) {
         case OP_REG:
@@ -1101,21 +1079,19 @@ static inline int writeback(struct x86_emulate_ctxt *ctxt,
         case OP_MEM:
                 if (c->lock_prefix)
                         rc = ops->cmpxchg_emulated(
-                                        c->dst.addr.mem,
+                                        linear(ctxt, c->dst.addr.mem),
                                         &c->dst.orig_val,
                                         &c->dst.val,
                                         c->dst.bytes,
-                                        &err,
+                                        &ctxt->exception,
                                         ctxt->vcpu);
                 else
                         rc = ops->write_emulated(
-                                        c->dst.addr.mem,
+                                        linear(ctxt, c->dst.addr.mem),
                                         &c->dst.val,
                                         c->dst.bytes,
-                                        &err,
+                                        &ctxt->exception,
                                         ctxt->vcpu);
-                if (rc == X86EMUL_PROPAGATE_FAULT)
-                        emulate_pf(ctxt);
                 if (rc != X86EMUL_CONTINUE)
                         return rc;
                 break;
@@ -1137,8 +1113,8 @@ static inline void emulate_push(struct x86_emulate_ctxt *ctxt,
         c->dst.bytes = c->op_bytes;
         c->dst.val = c->src.val;
         register_address_increment(c, &c->regs[VCPU_REGS_RSP], -c->op_bytes);
-        c->dst.addr.mem = register_address(c, ss_base(ctxt, ops),
-                                           c->regs[VCPU_REGS_RSP]);
+        c->dst.addr.mem.ea = register_address(c, c->regs[VCPU_REGS_RSP]);
+        c->dst.addr.mem.seg = VCPU_SREG_SS;
 }
 
 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
@@ -1147,10 +1123,11 @@ static int emulate_pop(struct x86_emulate_ctxt *ctxt,
 {
         struct decode_cache *c = &ctxt->decode;
         int rc;
+        struct segmented_address addr;
 
-        rc = read_emulated(ctxt, ops, register_address(c, ss_base(ctxt, ops),
-                                                       c->regs[VCPU_REGS_RSP]),
-                           dest, len);
+        addr.ea = register_address(c, c->regs[VCPU_REGS_RSP]);
+        addr.seg = VCPU_SREG_SS;
+        rc = read_emulated(ctxt, ops, linear(ctxt, addr), dest, len);
         if (rc != X86EMUL_CONTINUE)
                 return rc;
 
@@ -1184,10 +1161,8 @@ static int emulate_popf(struct x86_emulate_ctxt *ctxt,
                         change_mask |= EFLG_IF;
                 break;
         case X86EMUL_MODE_VM86:
-                if (iopl < 3) {
-                        emulate_gp(ctxt, 0);
-                        return X86EMUL_PROPAGATE_FAULT;
-                }
+                if (iopl < 3)
+                        return emulate_gp(ctxt, 0);
                 change_mask |= EFLG_IF;
                 break;
         default: /* real mode */
@@ -1198,9 +1173,6 @@ static int emulate_popf(struct x86_emulate_ctxt *ctxt,
         *(unsigned long *)dest =
                 (ctxt->eflags & ~change_mask) | (val & change_mask);
 
-        if (rc == X86EMUL_PROPAGATE_FAULT)
-                emulate_pf(ctxt);
-
         return rc;
 }
 
@@ -1287,7 +1259,6 @@ int emulate_int_real(struct x86_emulate_ctxt *ctxt,
         gva_t cs_addr;
         gva_t eip_addr;
         u16 cs, eip;
-        u32 err;
 
         /* TODO: Add limit checks */
         c->src.val = ctxt->eflags;
@@ -1317,11 +1288,11 @@ int emulate_int_real(struct x86_emulate_ctxt *ctxt,
         eip_addr = dt.address + (irq << 2);
         cs_addr = dt.address + (irq << 2) + 2;
 
-        rc = ops->read_std(cs_addr, &cs, 2, ctxt->vcpu, &err);
+        rc = ops->read_std(cs_addr, &cs, 2, ctxt->vcpu, &ctxt->exception);
         if (rc != X86EMUL_CONTINUE)
                 return rc;
 
-        rc = ops->read_std(eip_addr, &eip, 2, ctxt->vcpu, &err);
+        rc = ops->read_std(eip_addr, &eip, 2, ctxt->vcpu, &ctxt->exception);
         if (rc != X86EMUL_CONTINUE)
                 return rc;
 
@@ -1370,10 +1341,8 @@ static int emulate_iret_real(struct x86_emulate_ctxt *ctxt,
         if (rc != X86EMUL_CONTINUE)
                 return rc;
 
-        if (temp_eip & ~0xffff) {
-                emulate_gp(ctxt, 0);
-                return X86EMUL_PROPAGATE_FAULT;
-        }
+        if (temp_eip & ~0xffff)
+                return emulate_gp(ctxt, 0);
 
         rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
 
@@ -1624,10 +1593,8 @@ emulate_syscall(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
 
         /* syscall is not available in real mode */
         if (ctxt->mode == X86EMUL_MODE_REAL ||
-            ctxt->mode == X86EMUL_MODE_VM86) {
-                emulate_ud(ctxt);
-                return X86EMUL_PROPAGATE_FAULT;
-        }
+            ctxt->mode == X86EMUL_MODE_VM86)
+                return emulate_ud(ctxt);
 
         setup_syscalls_segments(ctxt, ops, &cs, &ss);
 
@@ -1678,34 +1645,26 @@ emulate_sysenter(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
         u16 cs_sel, ss_sel;
 
         /* inject #GP if in real mode */
-        if (ctxt->mode == X86EMUL_MODE_REAL) {
-                emulate_gp(ctxt, 0);
-                return X86EMUL_PROPAGATE_FAULT;
-        }
+        if (ctxt->mode == X86EMUL_MODE_REAL)
+                return emulate_gp(ctxt, 0);
 
         /* XXX sysenter/sysexit have not been tested in 64bit mode.
         * Therefore, we inject an #UD.
         */
-        if (ctxt->mode == X86EMUL_MODE_PROT64) {
-                emulate_ud(ctxt);
-                return X86EMUL_PROPAGATE_FAULT;
-        }
+        if (ctxt->mode == X86EMUL_MODE_PROT64)
+                return emulate_ud(ctxt);
 
         setup_syscalls_segments(ctxt, ops, &cs, &ss);
 
         ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
         switch (ctxt->mode) {
         case X86EMUL_MODE_PROT32:
-                if ((msr_data & 0xfffc) == 0x0) {
-                        emulate_gp(ctxt, 0);
-                        return X86EMUL_PROPAGATE_FAULT;
-                }
+                if ((msr_data & 0xfffc) == 0x0)
+                        return emulate_gp(ctxt, 0);
                 break;
         case X86EMUL_MODE_PROT64:
-                if (msr_data == 0x0) {
-                        emulate_gp(ctxt, 0);
-                        return X86EMUL_PROPAGATE_FAULT;
-                }
+                if (msr_data == 0x0)
+                        return emulate_gp(ctxt, 0);
                 break;
         }
 
@@ -1745,10 +1704,8 @@ emulate_sysexit(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
 
         /* inject #GP if in real mode or Virtual 8086 mode */
         if (ctxt->mode == X86EMUL_MODE_REAL ||
-            ctxt->mode == X86EMUL_MODE_VM86) {
-                emulate_gp(ctxt, 0);
-                return X86EMUL_PROPAGATE_FAULT;
-        }
+            ctxt->mode == X86EMUL_MODE_VM86)
+                return emulate_gp(ctxt, 0);
 
         setup_syscalls_segments(ctxt, ops, &cs, &ss);
 
@@ -1763,18 +1720,14 @@ emulate_sysexit(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
         switch (usermode) {
         case X86EMUL_MODE_PROT32:
                 cs_sel = (u16)(msr_data + 16);
-                if ((msr_data & 0xfffc) == 0x0) {
-                        emulate_gp(ctxt, 0);
-                        return X86EMUL_PROPAGATE_FAULT;
-                }
+                if ((msr_data & 0xfffc) == 0x0)
+                        return emulate_gp(ctxt, 0);
                 ss_sel = (u16)(msr_data + 24);
                 break;
         case X86EMUL_MODE_PROT64:
                 cs_sel = (u16)(msr_data + 32);
-                if (msr_data == 0x0) {
-                        emulate_gp(ctxt, 0);
-                        return X86EMUL_PROPAGATE_FAULT;
-                }
+                if (msr_data == 0x0)
+                        return emulate_gp(ctxt, 0);
                 ss_sel = cs_sel + 8;
                 cs.d = 0;
                 cs.l = 1;
@@ -1934,33 +1887,27 @@ static int task_switch_16(struct x86_emulate_ctxt *ctxt,
 {
         struct tss_segment_16 tss_seg;
         int ret;
-        u32 err, new_tss_base = get_desc_base(new_desc);
+        u32 new_tss_base = get_desc_base(new_desc);
 
         ret = ops->read_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
-                            &err);
-        if (ret == X86EMUL_PROPAGATE_FAULT) {
+                            &ctxt->exception);
+        if (ret != X86EMUL_CONTINUE)
                 /* FIXME: need to provide precise fault address */
-                emulate_pf(ctxt);
                 return ret;
-        }
 
         save_state_to_tss16(ctxt, ops, &tss_seg);
 
         ret = ops->write_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
-                             &err);
-        if (ret == X86EMUL_PROPAGATE_FAULT) {
+                             &ctxt->exception);
+        if (ret != X86EMUL_CONTINUE)
                 /* FIXME: need to provide precise fault address */
-                emulate_pf(ctxt);
                 return ret;
-        }
 
         ret = ops->read_std(new_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
-                            &err);
-        if (ret == X86EMUL_PROPAGATE_FAULT) {
+                            &ctxt->exception);
+        if (ret != X86EMUL_CONTINUE)
                 /* FIXME: need to provide precise fault address */
-                emulate_pf(ctxt);
                 return ret;
-        }
 
         if (old_tss_sel != 0xffff) {
                 tss_seg.prev_task_link = old_tss_sel;
@@ -1968,12 +1915,10 @@ static int task_switch_16(struct x86_emulate_ctxt *ctxt,
                 ret = ops->write_std(new_tss_base,
                                      &tss_seg.prev_task_link,
                                      sizeof tss_seg.prev_task_link,
-                                     ctxt->vcpu, &err);
-                if (ret == X86EMUL_PROPAGATE_FAULT) {
+                                     ctxt->vcpu, &ctxt->exception);
+                if (ret != X86EMUL_CONTINUE)
                         /* FIXME: need to provide precise fault address */
-                        emulate_pf(ctxt);
                         return ret;
-                }
         }
 
         return load_state_from_tss16(ctxt, ops, &tss_seg);
@@ -2013,10 +1958,8 @@ static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
         struct decode_cache *c = &ctxt->decode;
         int ret;
 
-        if (ops->set_cr(3, tss->cr3, ctxt->vcpu)) {
-                emulate_gp(ctxt, 0);
-                return X86EMUL_PROPAGATE_FAULT;
-        }
+        if (ops->set_cr(3, tss->cr3, ctxt->vcpu))
+                return emulate_gp(ctxt, 0);
         c->eip = tss->eip;
         ctxt->eflags = tss->eflags | 2;
         c->regs[VCPU_REGS_RAX] = tss->eax;
@@ -2076,33 +2019,27 @@ static int task_switch_32(struct x86_emulate_ctxt *ctxt,
 {
         struct tss_segment_32 tss_seg;
         int ret;
-        u32 err, new_tss_base = get_desc_base(new_desc);
+        u32 new_tss_base = get_desc_base(new_desc);
 
         ret = ops->read_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
-                            &err);
-        if (ret == X86EMUL_PROPAGATE_FAULT) {
+                            &ctxt->exception);
+        if (ret != X86EMUL_CONTINUE)
                 /* FIXME: need to provide precise fault address */
-                emulate_pf(ctxt);
                 return ret;
-        }
 
         save_state_to_tss32(ctxt, ops, &tss_seg);
 
         ret = ops->write_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
-                             &err);
-        if (ret == X86EMUL_PROPAGATE_FAULT) {
+                             &ctxt->exception);
+        if (ret != X86EMUL_CONTINUE)
                 /* FIXME: need to provide precise fault address */
-                emulate_pf(ctxt);
                 return ret;
-        }
 
         ret = ops->read_std(new_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
-                            &err);
-        if (ret == X86EMUL_PROPAGATE_FAULT) {
+                            &ctxt->exception);
+        if (ret != X86EMUL_CONTINUE)
                 /* FIXME: need to provide precise fault address */
-                emulate_pf(ctxt);
                 return ret;
-        }
 
         if (old_tss_sel != 0xffff) {
                 tss_seg.prev_task_link = old_tss_sel;
@@ -2110,12 +2047,10 @@ static int task_switch_32(struct x86_emulate_ctxt *ctxt,
                 ret = ops->write_std(new_tss_base,
                                      &tss_seg.prev_task_link,
                                      sizeof tss_seg.prev_task_link,
-                                     ctxt->vcpu, &err);
-                if (ret == X86EMUL_PROPAGATE_FAULT) {
+                                     ctxt->vcpu, &ctxt->exception);
+                if (ret != X86EMUL_CONTINUE)
                         /* FIXME: need to provide precise fault address */
-                        emulate_pf(ctxt);
                         return ret;
-                }
         }
 
         return load_state_from_tss32(ctxt, ops, &tss_seg);
@@ -2146,10 +2081,8 @@ static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
 
         if (reason != TASK_SWITCH_IRET) {
                 if ((tss_selector & 3) > next_tss_desc.dpl ||
-                    ops->cpl(ctxt->vcpu) > next_tss_desc.dpl) {
-                        emulate_gp(ctxt, 0);
-                        return X86EMUL_PROPAGATE_FAULT;
-                }
+                    ops->cpl(ctxt->vcpu) > next_tss_desc.dpl)
+                        return emulate_gp(ctxt, 0);
         }
 
         desc_limit = desc_limit_scaled(&next_tss_desc);
@@ -2231,14 +2164,15 @@ int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
         return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0;
 }
 
-static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned long base,
+static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned seg,
                             int reg, struct operand *op)
 {
         struct decode_cache *c = &ctxt->decode;
         int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;
 
         register_address_increment(c, &c->regs[reg], df * op->bytes);
-        op->addr.mem = register_address(c,  base, c->regs[reg]);
+        op->addr.mem.ea = register_address(c, c->regs[reg]);
+        op->addr.mem.seg = seg;
 }
 
 static int em_push(struct x86_emulate_ctxt *ctxt)
@@ -2369,10 +2303,8 @@ static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
         struct decode_cache *c = &ctxt->decode;
         u64 tsc = 0;
 
-        if (cpl > 0 && (ctxt->ops->get_cr(4, ctxt->vcpu) & X86_CR4_TSD)) {
-                emulate_gp(ctxt, 0);
-                return X86EMUL_PROPAGATE_FAULT;
-        }
+        if (cpl > 0 && (ctxt->ops->get_cr(4, ctxt->vcpu) & X86_CR4_TSD))
+                return emulate_gp(ctxt, 0);
         ctxt->ops->get_msr(ctxt->vcpu, MSR_IA32_TSC, &tsc);
         c->regs[VCPU_REGS_RAX] = (u32)tsc;
         c->regs[VCPU_REGS_RDX] = tsc >> 32;
@@ -2647,7 +2579,7 @@ static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
 
         op->type = OP_IMM;
         op->bytes = size;
-        op->addr.mem = c->eip;
+        op->addr.mem.ea = c->eip;
         /* NB. Immediates are sign-extended as necessary. */
         switch (op->bytes) {
         case 1:
@@ -2678,7 +2610,7 @@ done:
 }
 
 int
-x86_decode_insn(struct x86_emulate_ctxt *ctxt)
+x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
 {
         struct x86_emulate_ops *ops = ctxt->ops;
         struct decode_cache *c = &ctxt->decode;
@@ -2689,7 +2621,10 @@ x86_decode_insn(struct x86_emulate_ctxt *ctxt)
         struct operand memop = { .type = OP_NONE };
 
         c->eip = ctxt->eip;
-        c->fetch.start = c->fetch.end = c->eip;
+        c->fetch.start = c->eip;
+        c->fetch.end = c->fetch.start + insn_len;
+        if (insn_len > 0)
+                memcpy(c->fetch.data, insn, insn_len);
         ctxt->cs_base = seg_base(ctxt, ops, VCPU_SREG_CS);
 
         switch (mode) {
@@ -2803,10 +2738,8 @@ done_prefixes:
         c->execute = opcode.u.execute;
 
         /* Unrecognised? */
-        if (c->d == 0 || (c->d & Undefined)) {
-                DPRINTF("Cannot emulate %02x\n", c->b);
+        if (c->d == 0 || (c->d & Undefined))
                 return -1;
-        }
 
         if (mode == X86EMUL_MODE_PROT64 && (c->d & Stack))
                 c->op_bytes = 8;
@@ -2831,14 +2764,13 @@ done_prefixes:
         if (!c->has_seg_override)
                 set_seg_override(c, VCPU_SREG_DS);
 
-        if (memop.type == OP_MEM && !(!c->twobyte && c->b == 0x8d))
-                memop.addr.mem += seg_override_base(ctxt, ops, c);
+        memop.addr.mem.seg = seg_override(ctxt, ops, c);
 
         if (memop.type == OP_MEM && c->ad_bytes != 8)
-                memop.addr.mem = (u32)memop.addr.mem;
+                memop.addr.mem.ea = (u32)memop.addr.mem.ea;
 
         if (memop.type == OP_MEM && c->rip_relative)
-                memop.addr.mem += c->eip;
+                memop.addr.mem.ea += c->eip;
 
         /*
          * Decode and fetch the source operand: register, memory
@@ -2890,14 +2822,14 @@ done_prefixes:
         case SrcSI:
                 c->src.type = OP_MEM;
                 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
-                c->src.addr.mem =
-                        register_address(c,  seg_override_base(ctxt, ops, c),
-                                         c->regs[VCPU_REGS_RSI]);
+                c->src.addr.mem.ea =
+                        register_address(c, c->regs[VCPU_REGS_RSI]);
+                c->src.addr.mem.seg = seg_override(ctxt, ops, c),
                 c->src.val = 0;
                 break;
         case SrcImmFAddr:
                 c->src.type = OP_IMM;
-                c->src.addr.mem = c->eip;
+                c->src.addr.mem.ea = c->eip;
                 c->src.bytes = c->op_bytes + 2;
                 insn_fetch_arr(c->src.valptr, c->src.bytes, c->eip);
                 break;
@@ -2944,7 +2876,7 @@ done_prefixes:
                 break;
         case DstImmUByte:
                 c->dst.type = OP_IMM;
-                c->dst.addr.mem = c->eip;
+                c->dst.addr.mem.ea = c->eip;
                 c->dst.bytes = 1;
                 c->dst.val = insn_fetch(u8, 1, c->eip);
                 break;
@@ -2969,9 +2901,9 @@ done_prefixes:
         case DstDI:
                 c->dst.type = OP_MEM;
                 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
-                c->dst.addr.mem =
-                        register_address(c, es_base(ctxt, ops),
-                                         c->regs[VCPU_REGS_RDI]);
+                c->dst.addr.mem.ea =
+                        register_address(c, c->regs[VCPU_REGS_RDI]);
+                c->dst.addr.mem.seg = VCPU_SREG_ES;
                 c->dst.val = 0;
                 break;
         case ImplicitOps:
@@ -3020,24 +2952,24 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
         ctxt->decode.mem_read.pos = 0;
 
         if (ctxt->mode == X86EMUL_MODE_PROT64 && (c->d & No64)) {
-                emulate_ud(ctxt);
+                rc = emulate_ud(ctxt);
                 goto done;
         }
 
         /* LOCK prefix is allowed only with some instructions */
         if (c->lock_prefix && (!(c->d & Lock) || c->dst.type != OP_MEM)) {
-                emulate_ud(ctxt);
+                rc = emulate_ud(ctxt);
                 goto done;
         }
 
         if ((c->d & SrcMask) == SrcMemFAddr && c->src.type != OP_MEM) {
-                emulate_ud(ctxt);
+                rc = emulate_ud(ctxt);
                 goto done;
         }
 
         /* Privileged instruction can be executed only in CPL=0 */
         if ((c->d & Priv) && ops->cpl(ctxt->vcpu)) {
-                emulate_gp(ctxt, 0);
+                rc = emulate_gp(ctxt, 0);
                 goto done;
         }
 
@@ -3050,7 +2982,7 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
         }
 
         if ((c->src.type == OP_MEM) && !(c->d & NoAccess)) {
-                rc = read_emulated(ctxt, ops, c->src.addr.mem,
+                rc = read_emulated(ctxt, ops, linear(ctxt, c->src.addr.mem),
                                         c->src.valptr, c->src.bytes);
                 if (rc != X86EMUL_CONTINUE)
                         goto done;
@@ -3058,7 +2990,7 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
         }
 
         if (c->src2.type == OP_MEM) {
-                rc = read_emulated(ctxt, ops, c->src2.addr.mem,
+                rc = read_emulated(ctxt, ops, linear(ctxt, c->src2.addr.mem),
                                         &c->src2.val, c->src2.bytes);
                 if (rc != X86EMUL_CONTINUE)
                         goto done;
@@ -3070,7 +3002,7 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
 
         if ((c->dst.type == OP_MEM) && !(c->d & Mov)) {
                 /* optimisation - avoid slow emulated read if Mov */
-                rc = read_emulated(ctxt, ops, c->dst.addr.mem,
+                rc = read_emulated(ctxt, ops, linear(ctxt, c->dst.addr.mem),
                                    &c->dst.val, c->dst.bytes);
                 if (rc != X86EMUL_CONTINUE)
                         goto done;
@@ -3215,13 +3147,13 @@ special_insn:
                 break;
         case 0x8c:  /* mov r/m, sreg */
                 if (c->modrm_reg > VCPU_SREG_GS) {
-                        emulate_ud(ctxt);
+                        rc = emulate_ud(ctxt);
                         goto done;
                 }
                 c->dst.val = ops->get_segment_selector(c->modrm_reg, ctxt->vcpu);
                 break;
         case 0x8d: /* lea r16/r32, m */
-                c->dst.val = c->src.addr.mem;
+                c->dst.val = c->src.addr.mem.ea;
                 break;
         case 0x8e: { /* mov seg, r/m16 */
                 uint16_t sel;
@@ -3230,7 +3162,7 @@ special_insn:
 
                 if (c->modrm_reg == VCPU_SREG_CS ||
                     c->modrm_reg > VCPU_SREG_GS) {
-                        emulate_ud(ctxt);
+                        rc = emulate_ud(ctxt);
                         goto done;
                 }
 
@@ -3268,7 +3200,6 @@ special_insn:
                 break;
         case 0xa6 ... 0xa7:     /* cmps */
                 c->dst.type = OP_NONE; /* Disable writeback. */
-                DPRINTF("cmps: mem1=0x%p mem2=0x%p\n", c->src.addr.mem, c->dst.addr.mem);
                 goto cmp;
         case 0xa8 ... 0xa9:     /* test ax, imm */
                 goto test;
@@ -3363,7 +3294,7 @@ special_insn:
         do_io_in:
                 c->dst.bytes = min(c->dst.bytes, 4u);
                 if (!emulator_io_permited(ctxt, ops, c->src.val, c->dst.bytes)) {
-                        emulate_gp(ctxt, 0);
+                        rc = emulate_gp(ctxt, 0);
                         goto done;
                 }
                 if (!pio_in_emulated(ctxt, ops, c->dst.bytes, c->src.val,
@@ -3377,7 +3308,7 @@ special_insn:
                 c->src.bytes = min(c->src.bytes, 4u);
                 if (!emulator_io_permited(ctxt, ops, c->dst.val,
                                           c->src.bytes)) {
-                        emulate_gp(ctxt, 0);
+                        rc = emulate_gp(ctxt, 0);
                         goto done;
                 }
                 ops->pio_out_emulated(c->src.bytes, c->dst.val,
@@ -3402,14 +3333,14 @@ special_insn:
                 break;
         case 0xfa: /* cli */
                 if (emulator_bad_iopl(ctxt, ops)) {
-                        emulate_gp(ctxt, 0);
+                        rc = emulate_gp(ctxt, 0);
                         goto done;
                 } else
                         ctxt->eflags &= ~X86_EFLAGS_IF;
                 break;
         case 0xfb: /* sti */
                 if (emulator_bad_iopl(ctxt, ops)) {
-                        emulate_gp(ctxt, 0);
+                        rc = emulate_gp(ctxt, 0);
                         goto done;
                 } else {
                         ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
@@ -3449,11 +3380,11 @@ writeback:
         c->dst.type = saved_dst_type;
 
         if ((c->d & SrcMask) == SrcSI)
-                string_addr_inc(ctxt, seg_override_base(ctxt, ops, c),
+                string_addr_inc(ctxt, seg_override(ctxt, ops, c),
                                 VCPU_REGS_RSI, &c->src);
 
         if ((c->d & DstMask) == DstDI)
-                string_addr_inc(ctxt, es_base(ctxt, ops), VCPU_REGS_RDI,
+                string_addr_inc(ctxt, VCPU_SREG_ES, VCPU_REGS_RDI,
                                 &c->dst);
 
         if (c->rep_prefix && (c->d & String)) {
@@ -3482,6 +3413,8 @@ writeback:
         ctxt->eip = c->eip;
 
 done:
+        if (rc == X86EMUL_PROPAGATE_FAULT)
+                ctxt->have_exception = true;
         return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
 
 twobyte_insn:
@@ -3544,9 +3477,11 @@ twobyte_insn:
                         break;
                 case 5: /* not defined */
                         emulate_ud(ctxt);
+                        rc = X86EMUL_PROPAGATE_FAULT;
                         goto done;
                 case 7: /* invlpg*/
-                        emulate_invlpg(ctxt->vcpu, c->src.addr.mem);
+                        emulate_invlpg(ctxt->vcpu,
+                                       linear(ctxt, c->src.addr.mem));
                         /* Disable writeback. */
                         c->dst.type = OP_NONE;
                         break;
@@ -3573,6 +3508,7 @@ twobyte_insn:
                 case 5 ... 7:
                 case 9 ... 15:
                         emulate_ud(ctxt);
+                        rc = X86EMUL_PROPAGATE_FAULT;
                         goto done;
                 }
                 c->dst.val = ops->get_cr(c->modrm_reg, ctxt->vcpu);
@@ -3581,6 +3517,7 @@ twobyte_insn:
                 if ((ops->get_cr(4, ctxt->vcpu) & X86_CR4_DE) &&
                     (c->modrm_reg == 4 || c->modrm_reg == 5)) {
                         emulate_ud(ctxt);
+                        rc = X86EMUL_PROPAGATE_FAULT;
                         goto done;
                 }
                 ops->get_dr(c->modrm_reg, &c->dst.val, ctxt->vcpu);
@@ -3588,6 +3525,7 @@ twobyte_insn:
         case 0x22: /* mov reg, cr */
                 if (ops->set_cr(c->modrm_reg, c->src.val, ctxt->vcpu)) {
                         emulate_gp(ctxt, 0);
+                        rc = X86EMUL_PROPAGATE_FAULT;
                         goto done;
                 }
                 c->dst.type = OP_NONE;
@@ -3596,6 +3534,7 @@ twobyte_insn:
                 if ((ops->get_cr(4, ctxt->vcpu) & X86_CR4_DE) &&
                     (c->modrm_reg == 4 || c->modrm_reg == 5)) {
                         emulate_ud(ctxt);
+                        rc = X86EMUL_PROPAGATE_FAULT;
                         goto done;
                 }
 
@@ -3604,6 +3543,7 @@ twobyte_insn:
                                  ~0ULL : ~0U), ctxt->vcpu) < 0) {
                         /* #UD condition is already handled by the code above */
                         emulate_gp(ctxt, 0);
+                        rc = X86EMUL_PROPAGATE_FAULT;
                         goto done;
                 }
 
@@ -3615,6 +3555,7 @@ twobyte_insn:
                         | ((u64)c->regs[VCPU_REGS_RDX] << 32);
                 if (ops->set_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], msr_data)) {
                         emulate_gp(ctxt, 0);
+                        rc = X86EMUL_PROPAGATE_FAULT;
                         goto done;
                 }
                 rc = X86EMUL_CONTINUE;
@@ -3623,6 +3564,7 @@ twobyte_insn:
                 /* rdmsr */
                 if (ops->get_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], &msr_data)) {
                         emulate_gp(ctxt, 0);
+                        rc = X86EMUL_PROPAGATE_FAULT;
                         goto done;
                 } else {
                         c->regs[VCPU_REGS_RAX] = (u32)msr_data;
@@ -3785,6 +3727,5 @@ twobyte_insn:
         goto writeback;
 
 cannot_emulate:
-        DPRINTF("Cannot emulate %02x\n", c->b);
         return -1;
 }
diff --git a/arch/x86/kvm/kvm_cache_regs.h b/arch/x86/kvm/kvm_cache_regs.h
index 975bb45329a1..3377d53fcd36 100644
--- a/arch/x86/kvm/kvm_cache_regs.h
+++ b/arch/x86/kvm/kvm_cache_regs.h
@@ -73,6 +73,13 @@ static inline ulong kvm_read_cr4_bits(struct kvm_vcpu *vcpu, ulong mask)
         return vcpu->arch.cr4 & mask;
 }
 
+static inline ulong kvm_read_cr3(struct kvm_vcpu *vcpu)
+{
+        if (!test_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail))
+                kvm_x86_ops->decache_cr3(vcpu);
+        return vcpu->arch.cr3;
+}
+
 static inline ulong kvm_read_cr4(struct kvm_vcpu *vcpu)
 {
         return kvm_read_cr4_bits(vcpu, ~0UL);
@@ -84,4 +91,19 @@ static inline u64 kvm_read_edx_eax(struct kvm_vcpu *vcpu)
                 | ((u64)(kvm_register_read(vcpu, VCPU_REGS_RDX) & -1u) << 32);
 }
 
+static inline void enter_guest_mode(struct kvm_vcpu *vcpu)
+{
+        vcpu->arch.hflags |= HF_GUEST_MASK;
+}
+
+static inline void leave_guest_mode(struct kvm_vcpu *vcpu)
+{
+        vcpu->arch.hflags &= ~HF_GUEST_MASK;
+}
+
+static inline bool is_guest_mode(struct kvm_vcpu *vcpu)
+{
+        return vcpu->arch.hflags & HF_GUEST_MASK;
+}
+
 #endif
diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c
index 413f8973a855..93cf9d0d3653 100644
--- a/arch/x86/kvm/lapic.c
+++ b/arch/x86/kvm/lapic.c
@@ -277,7 +277,8 @@ static void apic_update_ppr(struct kvm_lapic *apic)
 
         if (old_ppr != ppr) {
                 apic_set_reg(apic, APIC_PROCPRI, ppr);
-                kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
+                if (ppr < old_ppr)
+                        kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
         }
 }
 
diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c
index fbb04aee8301..f02b8edc3d44 100644
--- a/arch/x86/kvm/mmu.c
+++ b/arch/x86/kvm/mmu.c
@@ -18,9 +18,11 @@
  *
  */
 
+#include "irq.h"
 #include "mmu.h"
 #include "x86.h"
 #include "kvm_cache_regs.h"
+#include "x86.h"
 
 #include <linux/kvm_host.h>
 #include <linux/types.h>
@@ -194,7 +196,6 @@ static struct percpu_counter kvm_total_used_mmu_pages;
 
 static u64 __read_mostly shadow_trap_nonpresent_pte;
 static u64 __read_mostly shadow_notrap_nonpresent_pte;
-static u64 __read_mostly shadow_base_present_pte;
 static u64 __read_mostly shadow_nx_mask;
 static u64 __read_mostly shadow_x_mask; /* mutual exclusive with nx_mask */
 static u64 __read_mostly shadow_user_mask;
@@ -213,12 +214,6 @@ void kvm_mmu_set_nonpresent_ptes(u64 trap_pte, u64 notrap_pte)
 }
 EXPORT_SYMBOL_GPL(kvm_mmu_set_nonpresent_ptes);
 
-void kvm_mmu_set_base_ptes(u64 base_pte)
-{
-        shadow_base_present_pte = base_pte;
-}
-EXPORT_SYMBOL_GPL(kvm_mmu_set_base_ptes);
-
 void kvm_mmu_set_mask_ptes(u64 user_mask, u64 accessed_mask,
                 u64 dirty_mask, u64 nx_mask, u64 x_mask)
 {
@@ -482,46 +477,46 @@ static void kvm_mmu_page_set_gfn(struct kvm_mmu_page *sp, int index, gfn_t gfn)
 }
 
 /*
- * Return the pointer to the largepage write count for a given
- * gfn, handling slots that are not large page aligned.
+ * Return the pointer to the large page information for a given gfn,
+ * handling slots that are not large page aligned.
  */
-static int *slot_largepage_idx(gfn_t gfn,
-                               struct kvm_memory_slot *slot,
-                               int level)
+static struct kvm_lpage_info *lpage_info_slot(gfn_t gfn,
+                                              struct kvm_memory_slot *slot,
+                                              int level)
 {
         unsigned long idx;
 
         idx = (gfn >> KVM_HPAGE_GFN_SHIFT(level)) -
               (slot->base_gfn >> KVM_HPAGE_GFN_SHIFT(level));
-        return &slot->lpage_info[level - 2][idx].write_count;
+        return &slot->lpage_info[level - 2][idx];
 }
 
 static void account_shadowed(struct kvm *kvm, gfn_t gfn)
 {
         struct kvm_memory_slot *slot;
-        int *write_count;
+        struct kvm_lpage_info *linfo;
         int i;
 
         slot = gfn_to_memslot(kvm, gfn);
         for (i = PT_DIRECTORY_LEVEL;
              i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
-                write_count   = slot_largepage_idx(gfn, slot, i);
-                *write_count += 1;
+                linfo = lpage_info_slot(gfn, slot, i);
+                linfo->write_count += 1;
         }
 }
 
 static void unaccount_shadowed(struct kvm *kvm, gfn_t gfn)
 {
         struct kvm_memory_slot *slot;
-        int *write_count;
+        struct kvm_lpage_info *linfo;
         int i;
 
         slot = gfn_to_memslot(kvm, gfn);
         for (i = PT_DIRECTORY_LEVEL;
              i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
-                write_count   = slot_largepage_idx(gfn, slot, i);
-                *write_count -= 1;
-                WARN_ON(*write_count < 0);
+                linfo = lpage_info_slot(gfn, slot, i);
+                linfo->write_count -= 1;
+                WARN_ON(linfo->write_count < 0);
         }
 }
 
@@ -530,12 +525,12 @@ static int has_wrprotected_page(struct kvm *kvm,
                                 int level)
 {
         struct kvm_memory_slot *slot;
-        int *largepage_idx;
+        struct kvm_lpage_info *linfo;
 
         slot = gfn_to_memslot(kvm, gfn);
         if (slot) {
-                largepage_idx = slot_largepage_idx(gfn, slot, level);
-                return *largepage_idx;
+                linfo = lpage_info_slot(gfn, slot, level);
+                return linfo->write_count;
         }
 
         return 1;
@@ -559,14 +554,18 @@ static int host_mapping_level(struct kvm *kvm, gfn_t gfn)
         return ret;
 }
 
-static int mapping_level(struct kvm_vcpu *vcpu, gfn_t large_gfn)
+static bool mapping_level_dirty_bitmap(struct kvm_vcpu *vcpu, gfn_t large_gfn)
 {
         struct kvm_memory_slot *slot;
-        int host_level, level, max_level;
-
         slot = gfn_to_memslot(vcpu->kvm, large_gfn);
         if (slot && slot->dirty_bitmap)
-                return PT_PAGE_TABLE_LEVEL;
+                return true;
+        return false;
+}
+
+static int mapping_level(struct kvm_vcpu *vcpu, gfn_t large_gfn)
+{
+        int host_level, level, max_level;
 
         host_level = host_mapping_level(vcpu->kvm, large_gfn);
 
@@ -590,16 +589,15 @@ static int mapping_level(struct kvm_vcpu *vcpu, gfn_t large_gfn)
 static unsigned long *gfn_to_rmap(struct kvm *kvm, gfn_t gfn, int level)
 {
         struct kvm_memory_slot *slot;
-        unsigned long idx;
+        struct kvm_lpage_info *linfo;
 
         slot = gfn_to_memslot(kvm, gfn);
         if (likely(level == PT_PAGE_TABLE_LEVEL))
                 return &slot->rmap[gfn - slot->base_gfn];
 
-        idx = (gfn >> KVM_HPAGE_GFN_SHIFT(level)) -
-                (slot->base_gfn >> KVM_HPAGE_GFN_SHIFT(level));
+        linfo = lpage_info_slot(gfn, slot, level);
 
-        return &slot->lpage_info[level - 2][idx].rmap_pde;
+        return &linfo->rmap_pde;
 }
 
 /*
@@ -887,19 +885,16 @@ static int kvm_handle_hva(struct kvm *kvm, unsigned long hva,
                 end = start + (memslot->npages << PAGE_SHIFT);
                 if (hva >= start && hva < end) {
                         gfn_t gfn_offset = (hva - start) >> PAGE_SHIFT;
+                        gfn_t gfn = memslot->base_gfn + gfn_offset;
 
                         ret = handler(kvm, &memslot->rmap[gfn_offset], data);
 
                         for (j = 0; j < KVM_NR_PAGE_SIZES - 1; ++j) {
-                                unsigned long idx;
-                                int sh;
-
-                                sh = KVM_HPAGE_GFN_SHIFT(PT_DIRECTORY_LEVEL+j);
-                                idx = ((memslot->base_gfn+gfn_offset) >> sh) -
-                                        (memslot->base_gfn >> sh);
-                                ret |= handler(kvm,
-                                        &memslot->lpage_info[j][idx].rmap_pde,
-                                        data);
+                                struct kvm_lpage_info *linfo;
+
+                                linfo = lpage_info_slot(gfn, memslot,
+                                                        PT_DIRECTORY_LEVEL + j);
+                                ret |= handler(kvm, &linfo->rmap_pde, data);
                         }
                         trace_kvm_age_page(hva, memslot, ret);
                         retval |= ret;
@@ -950,6 +945,35 @@ static int kvm_age_rmapp(struct kvm *kvm, unsigned long *rmapp,
         return young;
 }
 
+static int kvm_test_age_rmapp(struct kvm *kvm, unsigned long *rmapp,
+                              unsigned long data)
+{
+        u64 *spte;
+        int young = 0;
+
+        /*
+         * If there's no access bit in the secondary pte set by the
+         * hardware it's up to gup-fast/gup to set the access bit in
+         * the primary pte or in the page structure.
+         */
+        if (!shadow_accessed_mask)
+                goto out;
+
+        spte = rmap_next(kvm, rmapp, NULL);
+        while (spte) {
+                u64 _spte = *spte;
+                BUG_ON(!(_spte & PT_PRESENT_MASK));
+                young = _spte & PT_ACCESSED_MASK;
+                if (young) {
+                        young = 1;
+                        break;
+                }
+                spte = rmap_next(kvm, rmapp, spte);
+        }
+out:
+        return young;
+}
+
 #define RMAP_RECYCLE_THRESHOLD 1000
 
 static void rmap_recycle(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn)
@@ -970,6 +994,11 @@ int kvm_age_hva(struct kvm *kvm, unsigned long hva)
         return kvm_handle_hva(kvm, hva, 0, kvm_age_rmapp);
 }
 
+int kvm_test_age_hva(struct kvm *kvm, unsigned long hva)
+{
+        return kvm_handle_hva(kvm, hva, 0, kvm_test_age_rmapp);
+}
+
 #ifdef MMU_DEBUG
 static int is_empty_shadow_page(u64 *spt)
 {
@@ -1161,7 +1190,7 @@ static void nonpaging_prefetch_page(struct kvm_vcpu *vcpu,
 }
 
 static int nonpaging_sync_page(struct kvm_vcpu *vcpu,
-                               struct kvm_mmu_page *sp, bool clear_unsync)
+                               struct kvm_mmu_page *sp)
 {
         return 1;
 }
@@ -1291,7 +1320,7 @@ static int __kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
         if (clear_unsync)
                 kvm_unlink_unsync_page(vcpu->kvm, sp);
 
-        if (vcpu->arch.mmu.sync_page(vcpu, sp, clear_unsync)) {
+        if (vcpu->arch.mmu.sync_page(vcpu, sp)) {
                 kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list);
                 return 1;
         }
@@ -1332,12 +1361,12 @@ static void kvm_sync_pages(struct kvm_vcpu *vcpu,  gfn_t gfn)
                         continue;
 
                 WARN_ON(s->role.level != PT_PAGE_TABLE_LEVEL);
+                kvm_unlink_unsync_page(vcpu->kvm, s);
                 if ((s->role.cr4_pae != !!is_pae(vcpu)) ||
-                        (vcpu->arch.mmu.sync_page(vcpu, s, true))) {
+                        (vcpu->arch.mmu.sync_page(vcpu, s))) {
                         kvm_mmu_prepare_zap_page(vcpu->kvm, s, &invalid_list);
                         continue;
                 }
-                kvm_unlink_unsync_page(vcpu->kvm, s);
                 flush = true;
         }
 
@@ -1963,9 +1992,9 @@ static int set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
                     unsigned pte_access, int user_fault,
                     int write_fault, int dirty, int level,
                     gfn_t gfn, pfn_t pfn, bool speculative,
-                    bool can_unsync, bool reset_host_protection)
+                    bool can_unsync, bool host_writable)
 {
-        u64 spte;
+        u64 spte, entry = *sptep;
         int ret = 0;
 
         /*
@@ -1973,7 +2002,7 @@ static int set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
          * whether the guest actually used the pte (in order to detect
          * demand paging).
          */
-        spte = shadow_base_present_pte;
+        spte = PT_PRESENT_MASK;
         if (!speculative)
                 spte |= shadow_accessed_mask;
         if (!dirty)
@@ -1990,8 +2019,10 @@ static int set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
                 spte |= kvm_x86_ops->get_mt_mask(vcpu, gfn,
                         kvm_is_mmio_pfn(pfn));
 
-        if (reset_host_protection)
+        if (host_writable)
                 spte |= SPTE_HOST_WRITEABLE;
+        else
+                pte_access &= ~ACC_WRITE_MASK;
 
         spte |= (u64)pfn << PAGE_SHIFT;
 
@@ -2036,6 +2067,14 @@ static int set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
 
 set_pte:
         update_spte(sptep, spte);
+        /*
+         * If we overwrite a writable spte with a read-only one we
+         * should flush remote TLBs. Otherwise rmap_write_protect
+         * will find a read-only spte, even though the writable spte
+         * might be cached on a CPU's TLB.
+         */
+        if (is_writable_pte(entry) && !is_writable_pte(*sptep))
+                kvm_flush_remote_tlbs(vcpu->kvm);
 done:
         return ret;
 }
@@ -2045,7 +2084,7 @@ static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
                          int user_fault, int write_fault, int dirty,
                          int *ptwrite, int level, gfn_t gfn,
                          pfn_t pfn, bool speculative,
-                         bool reset_host_protection)
+                         bool host_writable)
 {
         int was_rmapped = 0;
         int rmap_count;
@@ -2080,7 +2119,7 @@ static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
 
         if (set_spte(vcpu, sptep, pte_access, user_fault, write_fault,
                       dirty, level, gfn, pfn, speculative, true,
-                      reset_host_protection)) {
+                      host_writable)) {
                 if (write_fault)
                         *ptwrite = 1;
                 kvm_mmu_flush_tlb(vcpu);
@@ -2211,7 +2250,8 @@ static void direct_pte_prefetch(struct kvm_vcpu *vcpu, u64 *sptep)
 }
 
 static int __direct_map(struct kvm_vcpu *vcpu, gpa_t v, int write,
-                        int level, gfn_t gfn, pfn_t pfn)
+                        int map_writable, int level, gfn_t gfn, pfn_t pfn,
+                        bool prefault)
 {
         struct kvm_shadow_walk_iterator iterator;
         struct kvm_mmu_page *sp;
@@ -2220,9 +2260,11 @@ static int __direct_map(struct kvm_vcpu *vcpu, gpa_t v, int write,
 
         for_each_shadow_entry(vcpu, (u64)gfn << PAGE_SHIFT, iterator) {
                 if (iterator.level == level) {
-                        mmu_set_spte(vcpu, iterator.sptep, ACC_ALL, ACC_ALL,
+                        unsigned pte_access = ACC_ALL;
+
+                        mmu_set_spte(vcpu, iterator.sptep, ACC_ALL, pte_access,
                                      0, write, 1, &pt_write,
-                                     level, gfn, pfn, false, true);
+                                     level, gfn, pfn, prefault, map_writable);
                         direct_pte_prefetch(vcpu, iterator.sptep);
                         ++vcpu->stat.pf_fixed;
                         break;
@@ -2277,27 +2319,81 @@ static int kvm_handle_bad_page(struct kvm *kvm, gfn_t gfn, pfn_t pfn)
         return 1;
 }
 
-static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, int write, gfn_t gfn)
+static void transparent_hugepage_adjust(struct kvm_vcpu *vcpu,
+                                        gfn_t *gfnp, pfn_t *pfnp, int *levelp)
+{
+        pfn_t pfn = *pfnp;
+        gfn_t gfn = *gfnp;
+        int level = *levelp;
+
+        /*
+         * Check if it's a transparent hugepage. If this would be an
+         * hugetlbfs page, level wouldn't be set to
+         * PT_PAGE_TABLE_LEVEL and there would be no adjustment done
+         * here.
+         */
+        if (!is_error_pfn(pfn) && !kvm_is_mmio_pfn(pfn) &&
+            level == PT_PAGE_TABLE_LEVEL &&
+            PageTransCompound(pfn_to_page(pfn)) &&
+            !has_wrprotected_page(vcpu->kvm, gfn, PT_DIRECTORY_LEVEL)) {
+                unsigned long mask;
+                /*
+                 * mmu_notifier_retry was successful and we hold the
+                 * mmu_lock here, so the pmd can't become splitting
+                 * from under us, and in turn
+                 * __split_huge_page_refcount() can't run from under
+                 * us and we can safely transfer the refcount from
+                 * PG_tail to PG_head as we switch the pfn to tail to
+                 * head.
+                 */
+                *levelp = level = PT_DIRECTORY_LEVEL;
+                mask = KVM_PAGES_PER_HPAGE(level) - 1;
+                VM_BUG_ON((gfn & mask) != (pfn & mask));
+                if (pfn & mask) {
+                        gfn &= ~mask;
+                        *gfnp = gfn;
+                        kvm_release_pfn_clean(pfn);
+                        pfn &= ~mask;
+                        if (!get_page_unless_zero(pfn_to_page(pfn)))
+                                BUG();
+                        *pfnp = pfn;
+                }
+        }
+}
+
+static bool try_async_pf(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn,
+                         gva_t gva, pfn_t *pfn, bool write, bool *writable);
+
+static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, int write, gfn_t gfn,
+                         bool prefault)
 {
         int r;
         int level;
+        int force_pt_level;
         pfn_t pfn;
         unsigned long mmu_seq;
+        bool map_writable;
 
-        level = mapping_level(vcpu, gfn);
-
-        /*
-         * This path builds a PAE pagetable - so we can map 2mb pages at
-         * maximum. Therefore check if the level is larger than that.
-         */
-        if (level > PT_DIRECTORY_LEVEL)
-                level = PT_DIRECTORY_LEVEL;
+        force_pt_level = mapping_level_dirty_bitmap(vcpu, gfn);
+        if (likely(!force_pt_level)) {
+                level = mapping_level(vcpu, gfn);
+                /*
+                 * This path builds a PAE pagetable - so we can map
+                 * 2mb pages at maximum. Therefore check if the level
+                 * is larger than that.
+                 */
+                if (level > PT_DIRECTORY_LEVEL)
+                        level = PT_DIRECTORY_LEVEL;
 
-        gfn &= ~(KVM_PAGES_PER_HPAGE(level) - 1);
+                gfn &= ~(KVM_PAGES_PER_HPAGE(level) - 1);
+        } else
+                level = PT_PAGE_TABLE_LEVEL;
 
         mmu_seq = vcpu->kvm->mmu_notifier_seq;
         smp_rmb();
-        pfn = gfn_to_pfn(vcpu->kvm, gfn);
+
+        if (try_async_pf(vcpu, prefault, gfn, v, &pfn, write, &map_writable))
+                return 0;
 
         /* mmio */
         if (is_error_pfn(pfn))
@@ -2307,7 +2403,10 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, int write, gfn_t gfn)
         if (mmu_notifier_retry(vcpu, mmu_seq))
                 goto out_unlock;
         kvm_mmu_free_some_pages(vcpu);
-        r = __direct_map(vcpu, v, write, level, gfn, pfn);
+        if (likely(!force_pt_level))
+                transparent_hugepage_adjust(vcpu, &gfn, &pfn, &level);
+        r = __direct_map(vcpu, v, write, map_writable, level, gfn, pfn,
+                         prefault);
         spin_unlock(&vcpu->kvm->mmu_lock);
 
 
@@ -2530,6 +2629,7 @@ static void mmu_sync_roots(struct kvm_vcpu *vcpu)
                 hpa_t root = vcpu->arch.mmu.root_hpa;
                 sp = page_header(root);
                 mmu_sync_children(vcpu, sp);
+                trace_kvm_mmu_audit(vcpu, AUDIT_POST_SYNC);
                 return;
         }
         for (i = 0; i < 4; ++i) {
@@ -2552,23 +2652,24 @@ void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu)
 }
 
 static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr,
-                                  u32 access, u32 *error)
+                                  u32 access, struct x86_exception *exception)
 {
-        if (error)
-                *error = 0;
+        if (exception)
+                exception->error_code = 0;
         return vaddr;
 }
 
 static gpa_t nonpaging_gva_to_gpa_nested(struct kvm_vcpu *vcpu, gva_t vaddr,
-                                         u32 access, u32 *error)
+                                         u32 access,
+                                         struct x86_exception *exception)
 {
-        if (error)
-                *error = 0;
+        if (exception)
+                exception->error_code = 0;
         return vcpu->arch.nested_mmu.translate_gpa(vcpu, vaddr, access);
 }
 
 static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva,
-                                u32 error_code)
+                                u32 error_code, bool prefault)
 {
         gfn_t gfn;
         int r;
@@ -2584,17 +2685,68 @@ static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva,
         gfn = gva >> PAGE_SHIFT;
 
         return nonpaging_map(vcpu, gva & PAGE_MASK,
-                             error_code & PFERR_WRITE_MASK, gfn);
+                             error_code & PFERR_WRITE_MASK, gfn, prefault);
+}
+
+static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu, gva_t gva, gfn_t gfn)
+{
+        struct kvm_arch_async_pf arch;
+
+        arch.token = (vcpu->arch.apf.id++ << 12) | vcpu->vcpu_id;
+        arch.gfn = gfn;
+        arch.direct_map = vcpu->arch.mmu.direct_map;
+        arch.cr3 = vcpu->arch.mmu.get_cr3(vcpu);
+
+        return kvm_setup_async_pf(vcpu, gva, gfn, &arch);
+}
+
+static bool can_do_async_pf(struct kvm_vcpu *vcpu)
+{
+        if (unlikely(!irqchip_in_kernel(vcpu->kvm) ||
+                     kvm_event_needs_reinjection(vcpu)))
+                return false;
+
+        return kvm_x86_ops->interrupt_allowed(vcpu);
 }
 
-static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa,
-                                u32 error_code)
+static bool try_async_pf(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn,
+                         gva_t gva, pfn_t *pfn, bool write, bool *writable)
+{
+        bool async;
+
+        *pfn = gfn_to_pfn_async(vcpu->kvm, gfn, &async, write, writable);
+
+        if (!async)
+                return false; /* *pfn has correct page already */
+
+        put_page(pfn_to_page(*pfn));
+
+        if (!prefault && can_do_async_pf(vcpu)) {
+                trace_kvm_try_async_get_page(gva, gfn);
+                if (kvm_find_async_pf_gfn(vcpu, gfn)) {
+                        trace_kvm_async_pf_doublefault(gva, gfn);
+                        kvm_make_request(KVM_REQ_APF_HALT, vcpu);
+                        return true;
+                } else if (kvm_arch_setup_async_pf(vcpu, gva, gfn))
+                        return true;
+        }
+
+        *pfn = gfn_to_pfn_prot(vcpu->kvm, gfn, write, writable);
+
+        return false;
+}
+
+static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa, u32 error_code,
+                          bool prefault)
 {
         pfn_t pfn;
         int r;
         int level;
+        int force_pt_level;
         gfn_t gfn = gpa >> PAGE_SHIFT;
         unsigned long mmu_seq;
+        int write = error_code & PFERR_WRITE_MASK;
+        bool map_writable;
 
         ASSERT(vcpu);
         ASSERT(VALID_PAGE(vcpu->arch.mmu.root_hpa));
@@ -2603,21 +2755,30 @@ static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa,
         if (r)
                 return r;
 
-        level = mapping_level(vcpu, gfn);
-
-        gfn &= ~(KVM_PAGES_PER_HPAGE(level) - 1);
+        force_pt_level = mapping_level_dirty_bitmap(vcpu, gfn);
+        if (likely(!force_pt_level)) {
+                level = mapping_level(vcpu, gfn);
+                gfn &= ~(KVM_PAGES_PER_HPAGE(level) - 1);
+        } else
+                level = PT_PAGE_TABLE_LEVEL;
 
         mmu_seq = vcpu->kvm->mmu_notifier_seq;
         smp_rmb();
-        pfn = gfn_to_pfn(vcpu->kvm, gfn);
+
+        if (try_async_pf(vcpu, prefault, gfn, gpa, &pfn, write, &map_writable))
+                return 0;
+
+        /* mmio */
         if (is_error_pfn(pfn))
                 return kvm_handle_bad_page(vcpu->kvm, gfn, pfn);
         spin_lock(&vcpu->kvm->mmu_lock);
         if (mmu_notifier_retry(vcpu, mmu_seq))
                 goto out_unlock;
         kvm_mmu_free_some_pages(vcpu);
-        r = __direct_map(vcpu, gpa, error_code & PFERR_WRITE_MASK,
-                         level, gfn, pfn);
+        if (likely(!force_pt_level))
+                transparent_hugepage_adjust(vcpu, &gfn, &pfn, &level);
+        r = __direct_map(vcpu, gpa, write, map_writable,
+                         level, gfn, pfn, prefault);
         spin_unlock(&vcpu->kvm->mmu_lock);
 
         return r;
@@ -2659,18 +2820,19 @@ void kvm_mmu_flush_tlb(struct kvm_vcpu *vcpu)
 
 static void paging_new_cr3(struct kvm_vcpu *vcpu)
 {
-        pgprintk("%s: cr3 %lx\n", __func__, vcpu->arch.cr3);
+        pgprintk("%s: cr3 %lx\n", __func__, kvm_read_cr3(vcpu));
         mmu_free_roots(vcpu);
 }
 
 static unsigned long get_cr3(struct kvm_vcpu *vcpu)
 {
-        return vcpu->arch.cr3;
+        return kvm_read_cr3(vcpu);
 }
 
-static void inject_page_fault(struct kvm_vcpu *vcpu)
+static void inject_page_fault(struct kvm_vcpu *vcpu,
+                              struct x86_exception *fault)
 {
-        vcpu->arch.mmu.inject_page_fault(vcpu);
+        vcpu->arch.mmu.inject_page_fault(vcpu, fault);
 }
 
 static void paging_free(struct kvm_vcpu *vcpu)
@@ -2816,6 +2978,7 @@ static int init_kvm_tdp_mmu(struct kvm_vcpu *vcpu)
 {
         struct kvm_mmu *context = vcpu->arch.walk_mmu;
 
+        context->base_role.word = 0;
         context->new_cr3 = nonpaging_new_cr3;
         context->page_fault = tdp_page_fault;
         context->free = nonpaging_free;
@@ -3008,9 +3171,6 @@ static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu,
                 return;
         }
 
-        if (is_rsvd_bits_set(&vcpu->arch.mmu, *(u64 *)new, PT_PAGE_TABLE_LEVEL))
-                return;
-
         ++vcpu->kvm->stat.mmu_pte_updated;
         if (!sp->role.cr4_pae)
                 paging32_update_pte(vcpu, sp, spte, new);
@@ -3264,12 +3424,13 @@ void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu)
         }
 }
 
-int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code)
+int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code,
+                       void *insn, int insn_len)
 {
         int r;
         enum emulation_result er;
 
-        r = vcpu->arch.mmu.page_fault(vcpu, cr2, error_code);
+        r = vcpu->arch.mmu.page_fault(vcpu, cr2, error_code, false);
         if (r < 0)
                 goto out;
 
@@ -3282,7 +3443,7 @@ int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code)
         if (r)
                 goto out;
 
-        er = emulate_instruction(vcpu, cr2, error_code, 0);
+        er = x86_emulate_instruction(vcpu, cr2, 0, insn, insn_len);
 
         switch (er) {
         case EMULATE_DONE:
@@ -3377,11 +3538,14 @@ void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot)
                 if (!test_bit(slot, sp->slot_bitmap))
                         continue;
 
+                if (sp->role.level != PT_PAGE_TABLE_LEVEL)
+                        continue;
+
                 pt = sp->spt;
                 for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
                         /* avoid RMW */
                         if (is_writable_pte(pt[i]))
-                                pt[i] &= ~PT_WRITABLE_MASK;
+                                update_spte(&pt[i], pt[i] & ~PT_WRITABLE_MASK);
         }
         kvm_flush_remote_tlbs(kvm);
 }
@@ -3463,13 +3627,6 @@ static void mmu_destroy_caches(void)
                 kmem_cache_destroy(mmu_page_header_cache);
 }
 
-void kvm_mmu_module_exit(void)
-{
-        mmu_destroy_caches();
-        percpu_counter_destroy(&kvm_total_used_mmu_pages);
-        unregister_shrinker(&mmu_shrinker);
-}
-
 int kvm_mmu_module_init(void)
 {
         pte_chain_cache = kmem_cache_create("kvm_pte_chain",
@@ -3566,7 +3723,7 @@ static int kvm_pv_mmu_write(struct kvm_vcpu *vcpu,
 
 static int kvm_pv_mmu_flush_tlb(struct kvm_vcpu *vcpu)
 {
-        (void)kvm_set_cr3(vcpu, vcpu->arch.cr3);
+        (void)kvm_set_cr3(vcpu, kvm_read_cr3(vcpu));
         return 1;
 }
 
@@ -3662,12 +3819,6 @@ int kvm_mmu_get_spte_hierarchy(struct kvm_vcpu *vcpu, u64 addr, u64 sptes[4])
 }
 EXPORT_SYMBOL_GPL(kvm_mmu_get_spte_hierarchy);
 
-#ifdef CONFIG_KVM_MMU_AUDIT
-#include "mmu_audit.c"
-#else
-static void mmu_audit_disable(void) { }
-#endif
-
 void kvm_mmu_destroy(struct kvm_vcpu *vcpu)
 {
         ASSERT(vcpu);
@@ -3675,5 +3826,18 @@ void kvm_mmu_destroy(struct kvm_vcpu *vcpu)
         destroy_kvm_mmu(vcpu);
         free_mmu_pages(vcpu);
         mmu_free_memory_caches(vcpu);
+}
+
+#ifdef CONFIG_KVM_MMU_AUDIT
+#include "mmu_audit.c"
+#else
+static void mmu_audit_disable(void) { }
+#endif
+
+void kvm_mmu_module_exit(void)
+{
+        mmu_destroy_caches();
+        percpu_counter_destroy(&kvm_total_used_mmu_pages);
+        unregister_shrinker(&mmu_shrinker);
         mmu_audit_disable();
 }
diff --git a/arch/x86/kvm/mmu_audit.c b/arch/x86/kvm/mmu_audit.c
index ba2bcdde6221..5f6223b8bcf7 100644
--- a/arch/x86/kvm/mmu_audit.c
+++ b/arch/x86/kvm/mmu_audit.c
@@ -19,11 +19,9 @@
 
 #include <linux/ratelimit.h>
 
-static int audit_point;
-
-#define audit_printk(fmt, args...)              \
+#define audit_printk(kvm, fmt, args...)         \
         printk(KERN_ERR "audit: (%s) error: "   \
-                fmt, audit_point_name[audit_point], ##args)
+                fmt, audit_point_name[kvm->arch.audit_point], ##args)
 
 typedef void (*inspect_spte_fn) (struct kvm_vcpu *vcpu, u64 *sptep, int level);
 
@@ -97,18 +95,21 @@ static void audit_mappings(struct kvm_vcpu *vcpu, u64 *sptep, int level)
 
         if (sp->unsync) {
                 if (level != PT_PAGE_TABLE_LEVEL) {
-                        audit_printk("unsync sp: %p level = %d\n", sp, level);
+                        audit_printk(vcpu->kvm, "unsync sp: %p "
+                                     "level = %d\n", sp, level);
                         return;
                 }
 
                 if (*sptep == shadow_notrap_nonpresent_pte) {
-                        audit_printk("notrap spte in unsync sp: %p\n", sp);
+                        audit_printk(vcpu->kvm, "notrap spte in unsync "
+                                     "sp: %p\n", sp);
                         return;
                 }
         }
 
         if (sp->role.direct && *sptep == shadow_notrap_nonpresent_pte) {
-                audit_printk("notrap spte in direct sp: %p\n", sp);
+                audit_printk(vcpu->kvm, "notrap spte in direct sp: %p\n",
+                             sp);
                 return;
         }
 
@@ -125,8 +126,9 @@ static void audit_mappings(struct kvm_vcpu *vcpu, u64 *sptep, int level)
 
         hpa =  pfn << PAGE_SHIFT;
         if ((*sptep & PT64_BASE_ADDR_MASK) != hpa)
-                audit_printk("levels %d pfn %llx hpa %llx ent %llxn",
-                                   vcpu->arch.mmu.root_level, pfn, hpa, *sptep);
+                audit_printk(vcpu->kvm, "levels %d pfn %llx hpa %llx "
+                             "ent %llxn", vcpu->arch.mmu.root_level, pfn,
+                             hpa, *sptep);
 }
 
 static void inspect_spte_has_rmap(struct kvm *kvm, u64 *sptep)
@@ -142,8 +144,8 @@ static void inspect_spte_has_rmap(struct kvm *kvm, u64 *sptep)
         if (!gfn_to_memslot(kvm, gfn)) {
                 if (!printk_ratelimit())
                         return;
-                audit_printk("no memslot for gfn %llx\n", gfn);
-                audit_printk("index %ld of sp (gfn=%llx)\n",
+                audit_printk(kvm, "no memslot for gfn %llx\n", gfn);
+                audit_printk(kvm, "index %ld of sp (gfn=%llx)\n",
                        (long int)(sptep - rev_sp->spt), rev_sp->gfn);
                 dump_stack();
                 return;
@@ -153,7 +155,8 @@ static void inspect_spte_has_rmap(struct kvm *kvm, u64 *sptep)
         if (!*rmapp) {
                 if (!printk_ratelimit())
                         return;
-                audit_printk("no rmap for writable spte %llx\n", *sptep);
+                audit_printk(kvm, "no rmap for writable spte %llx\n",
+                             *sptep);
                 dump_stack();
         }
 }
@@ -168,8 +171,9 @@ static void audit_spte_after_sync(struct kvm_vcpu *vcpu, u64 *sptep, int level)
 {
         struct kvm_mmu_page *sp = page_header(__pa(sptep));
 
-        if (audit_point == AUDIT_POST_SYNC && sp->unsync)
-                audit_printk("meet unsync sp(%p) after sync root.\n", sp);
+        if (vcpu->kvm->arch.audit_point == AUDIT_POST_SYNC && sp->unsync)
+                audit_printk(vcpu->kvm, "meet unsync sp(%p) after sync "
+                             "root.\n", sp);
 }
 
 static void check_mappings_rmap(struct kvm *kvm, struct kvm_mmu_page *sp)
@@ -202,8 +206,9 @@ static void audit_write_protection(struct kvm *kvm, struct kvm_mmu_page *sp)
         spte = rmap_next(kvm, rmapp, NULL);
         while (spte) {
                 if (is_writable_pte(*spte))
-                        audit_printk("shadow page has writable mappings: gfn "
-                                     "%llx role %x\n", sp->gfn, sp->role.word);
+                        audit_printk(kvm, "shadow page has writable "
+                                     "mappings: gfn %llx role %x\n",
+                                     sp->gfn, sp->role.word);
                 spte = rmap_next(kvm, rmapp, spte);
         }
 }
@@ -238,7 +243,7 @@ static void kvm_mmu_audit(void *ignore, struct kvm_vcpu *vcpu, int point)
         if (!__ratelimit(&ratelimit_state))
                 return;
 
-        audit_point = point;
+        vcpu->kvm->arch.audit_point = point;
         audit_all_active_sps(vcpu->kvm);
         audit_vcpu_spte(vcpu);
 }
diff --git a/arch/x86/kvm/paging_tmpl.h b/arch/x86/kvm/paging_tmpl.h
index cd7a833a3b52..6bccc24c4181 100644
--- a/arch/x86/kvm/paging_tmpl.h
+++ b/arch/x86/kvm/paging_tmpl.h
@@ -72,7 +72,7 @@ struct guest_walker {
         unsigned pt_access;
         unsigned pte_access;
         gfn_t gfn;
-        u32 error_code;
+        struct x86_exception fault;
 };
 
 static gfn_t gpte_to_gfn_lvl(pt_element_t gpte, int lvl)
@@ -266,21 +266,23 @@ walk:
         return 1;
 
 error:
-        walker->error_code = 0;
+        walker->fault.vector = PF_VECTOR;
+        walker->fault.error_code_valid = true;
+        walker->fault.error_code = 0;
         if (present)
-                walker->error_code |= PFERR_PRESENT_MASK;
+                walker->fault.error_code |= PFERR_PRESENT_MASK;
 
-        walker->error_code |= write_fault | user_fault;
+        walker->fault.error_code |= write_fault | user_fault;
 
         if (fetch_fault && mmu->nx)
-                walker->error_code |= PFERR_FETCH_MASK;
+                walker->fault.error_code |= PFERR_FETCH_MASK;
         if (rsvd_fault)
-                walker->error_code |= PFERR_RSVD_MASK;
+                walker->fault.error_code |= PFERR_RSVD_MASK;
 
-        vcpu->arch.fault.address    = addr;
-        vcpu->arch.fault.error_code = walker->error_code;
+        walker->fault.address = addr;
+        walker->fault.nested_page_fault = mmu != vcpu->arch.walk_mmu;
 
-        trace_kvm_mmu_walker_error(walker->error_code);
+        trace_kvm_mmu_walker_error(walker->fault.error_code);
         return 0;
 }
 
@@ -299,25 +301,42 @@ static int FNAME(walk_addr_nested)(struct guest_walker *walker,
                                         addr, access);
 }
 
+static bool FNAME(prefetch_invalid_gpte)(struct kvm_vcpu *vcpu,
+                                    struct kvm_mmu_page *sp, u64 *spte,
+                                    pt_element_t gpte)
+{
+        u64 nonpresent = shadow_trap_nonpresent_pte;
+
+        if (is_rsvd_bits_set(&vcpu->arch.mmu, gpte, PT_PAGE_TABLE_LEVEL))
+                goto no_present;
+
+        if (!is_present_gpte(gpte)) {
+                if (!sp->unsync)
+                        nonpresent = shadow_notrap_nonpresent_pte;
+                goto no_present;
+        }
+
+        if (!(gpte & PT_ACCESSED_MASK))
+                goto no_present;
+
+        return false;
+
+no_present:
+        drop_spte(vcpu->kvm, spte, nonpresent);
+        return true;
+}
+
 static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
                               u64 *spte, const void *pte)
 {
         pt_element_t gpte;
         unsigned pte_access;
         pfn_t pfn;
-        u64 new_spte;
 
         gpte = *(const pt_element_t *)pte;
-        if (~gpte & (PT_PRESENT_MASK | PT_ACCESSED_MASK)) {
-                if (!is_present_gpte(gpte)) {
-                        if (sp->unsync)
-                                new_spte = shadow_trap_nonpresent_pte;
-                        else
-                                new_spte = shadow_notrap_nonpresent_pte;
-                        __set_spte(spte, new_spte);
-                }
+        if (FNAME(prefetch_invalid_gpte)(vcpu, sp, spte, gpte))
                 return;
-        }
+
         pgprintk("%s: gpte %llx spte %p\n", __func__, (u64)gpte, spte);
         pte_access = sp->role.access & FNAME(gpte_access)(vcpu, gpte);
         if (gpte_to_gfn(gpte) != vcpu->arch.update_pte.gfn)
@@ -329,7 +348,7 @@ static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
                 return;
         kvm_get_pfn(pfn);
         /*
-         * we call mmu_set_spte() with reset_host_protection = true beacuse that
+         * we call mmu_set_spte() with host_writable = true beacuse that
          * vcpu->arch.update_pte.pfn was fetched from get_user_pages(write = 1).
          */
         mmu_set_spte(vcpu, spte, sp->role.access, pte_access, 0, 0,
@@ -364,7 +383,6 @@ static void FNAME(pte_prefetch)(struct kvm_vcpu *vcpu, struct guest_walker *gw,
                                 u64 *sptep)
 {
         struct kvm_mmu_page *sp;
-        struct kvm_mmu *mmu = &vcpu->arch.mmu;
         pt_element_t *gptep = gw->prefetch_ptes;
         u64 *spte;
         int i;
@@ -395,14 +413,7 @@ static void FNAME(pte_prefetch)(struct kvm_vcpu *vcpu, struct guest_walker *gw,
 
                 gpte = gptep[i];
 
-                if (!is_present_gpte(gpte) ||
-                      is_rsvd_bits_set(mmu, gpte, PT_PAGE_TABLE_LEVEL)) {
-                        if (!sp->unsync)
-                                __set_spte(spte, shadow_notrap_nonpresent_pte);
-                        continue;
-                }
-
-                if (!(gpte & PT_ACCESSED_MASK))
+                if (FNAME(prefetch_invalid_gpte)(vcpu, sp, spte, gpte))
                         continue;
 
                 pte_access = sp->role.access & FNAME(gpte_access)(vcpu, gpte);
@@ -427,7 +438,8 @@ static void FNAME(pte_prefetch)(struct kvm_vcpu *vcpu, struct guest_walker *gw,
 static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr,
                          struct guest_walker *gw,
                          int user_fault, int write_fault, int hlevel,
-                         int *ptwrite, pfn_t pfn)
+                         int *ptwrite, pfn_t pfn, bool map_writable,
+                         bool prefault)
 {
         unsigned access = gw->pt_access;
         struct kvm_mmu_page *sp = NULL;
@@ -501,7 +513,7 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr,
 
         mmu_set_spte(vcpu, it.sptep, access, gw->pte_access & access,
                      user_fault, write_fault, dirty, ptwrite, it.level,
-                     gw->gfn, pfn, false, true);
+                     gw->gfn, pfn, prefault, map_writable);
         FNAME(pte_prefetch)(vcpu, gw, it.sptep);
 
         return it.sptep;
@@ -527,8 +539,8 @@ out_gpte_changed:
  *  Returns: 1 if we need to emulate the instruction, 0 otherwise, or
  *           a negative value on error.
  */
-static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr,
-                               u32 error_code)
+static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr, u32 error_code,
+                             bool prefault)
 {
         int write_fault = error_code & PFERR_WRITE_MASK;
         int user_fault = error_code & PFERR_USER_MASK;
@@ -538,7 +550,9 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr,
         int r;
         pfn_t pfn;
         int level = PT_PAGE_TABLE_LEVEL;
+        int force_pt_level;
         unsigned long mmu_seq;
+        bool map_writable;
 
         pgprintk("%s: addr %lx err %x\n", __func__, addr, error_code);
 
@@ -556,19 +570,29 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr,
          */
         if (!r) {
                 pgprintk("%s: guest page fault\n", __func__);
-                inject_page_fault(vcpu);
-                vcpu->arch.last_pt_write_count = 0; /* reset fork detector */
+                if (!prefault) {
+                        inject_page_fault(vcpu, &walker.fault);
+                        /* reset fork detector */
+                        vcpu->arch.last_pt_write_count = 0;
+                }
                 return 0;
         }
 
-        if (walker.level >= PT_DIRECTORY_LEVEL) {
+        if (walker.level >= PT_DIRECTORY_LEVEL)
+                force_pt_level = mapping_level_dirty_bitmap(vcpu, walker.gfn);
+        else
+                force_pt_level = 1;
+        if (!force_pt_level) {
                 level = min(walker.level, mapping_level(vcpu, walker.gfn));
                 walker.gfn = walker.gfn & ~(KVM_PAGES_PER_HPAGE(level) - 1);
         }
 
         mmu_seq = vcpu->kvm->mmu_notifier_seq;
         smp_rmb();
-        pfn = gfn_to_pfn(vcpu->kvm, walker.gfn);
+
+        if (try_async_pf(vcpu, prefault, walker.gfn, addr, &pfn, write_fault,
+                         &map_writable))
+                return 0;
 
         /* mmio */
         if (is_error_pfn(pfn))
@@ -580,8 +604,10 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr,
 
         trace_kvm_mmu_audit(vcpu, AUDIT_PRE_PAGE_FAULT);
         kvm_mmu_free_some_pages(vcpu);
+        if (!force_pt_level)
+                transparent_hugepage_adjust(vcpu, &walker.gfn, &pfn, &level);
         sptep = FNAME(fetch)(vcpu, addr, &walker, user_fault, write_fault,
-                             level, &write_pt, pfn);
+                             level, &write_pt, pfn, map_writable, prefault);
         (void)sptep;
         pgprintk("%s: shadow pte %p %llx ptwrite %d\n", __func__,
                  sptep, *sptep, write_pt);
@@ -661,7 +687,7 @@ static void FNAME(invlpg)(struct kvm_vcpu *vcpu, gva_t gva)
 }
 
 static gpa_t FNAME(gva_to_gpa)(struct kvm_vcpu *vcpu, gva_t vaddr, u32 access,
-                               u32 *error)
+                               struct x86_exception *exception)
 {
         struct guest_walker walker;
         gpa_t gpa = UNMAPPED_GVA;
@@ -672,14 +698,15 @@ static gpa_t FNAME(gva_to_gpa)(struct kvm_vcpu *vcpu, gva_t vaddr, u32 access,
         if (r) {
                 gpa = gfn_to_gpa(walker.gfn);
                 gpa |= vaddr & ~PAGE_MASK;
-        } else if (error)
-                *error = walker.error_code;
+        } else if (exception)
+                *exception = walker.fault;
 
         return gpa;
 }
 
 static gpa_t FNAME(gva_to_gpa_nested)(struct kvm_vcpu *vcpu, gva_t vaddr,
-                                      u32 access, u32 *error)
+                                      u32 access,
+                                      struct x86_exception *exception)
 {
         struct guest_walker walker;
         gpa_t gpa = UNMAPPED_GVA;
@@ -690,8 +717,8 @@ static gpa_t FNAME(gva_to_gpa_nested)(struct kvm_vcpu *vcpu, gva_t vaddr,
         if (r) {
                 gpa = gfn_to_gpa(walker.gfn);
                 gpa |= vaddr & ~PAGE_MASK;
-        } else if (error)
-                *error = walker.error_code;
+        } else if (exception)
+                *exception = walker.fault;
 
         return gpa;
 }
@@ -730,12 +757,19 @@ static void FNAME(prefetch_page)(struct kvm_vcpu *vcpu,
  * Using the cached information from sp->gfns is safe because:
  * - The spte has a reference to the struct page, so the pfn for a given gfn
  *   can't change unless all sptes pointing to it are nuked first.
+ *
+ * Note:
+ *   We should flush all tlbs if spte is dropped even though guest is
+ *   responsible for it. Since if we don't, kvm_mmu_notifier_invalidate_page
+ *   and kvm_mmu_notifier_invalidate_range_start detect the mapping page isn't
+ *   used by guest then tlbs are not flushed, so guest is allowed to access the
+ *   freed pages.
+ *   And we increase kvm->tlbs_dirty to delay tlbs flush in this case.
  */
-static int FNAME(sync_page)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
-                            bool clear_unsync)
+static int FNAME(sync_page)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
 {
         int i, offset, nr_present;
-        bool reset_host_protection;
+        bool host_writable;
         gpa_t first_pte_gpa;
 
         offset = nr_present = 0;
@@ -764,31 +798,27 @@ static int FNAME(sync_page)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
                         return -EINVAL;
 
                 gfn = gpte_to_gfn(gpte);
-                if (is_rsvd_bits_set(&vcpu->arch.mmu, gpte, PT_PAGE_TABLE_LEVEL)
-                      || gfn != sp->gfns[i] || !is_present_gpte(gpte)
-                      || !(gpte & PT_ACCESSED_MASK)) {
-                        u64 nonpresent;
 
-                        if (is_present_gpte(gpte) || !clear_unsync)
-                                nonpresent = shadow_trap_nonpresent_pte;
-                        else
-                                nonpresent = shadow_notrap_nonpresent_pte;
-                        drop_spte(vcpu->kvm, &sp->spt[i], nonpresent);
+                if (FNAME(prefetch_invalid_gpte)(vcpu, sp, &sp->spt[i], gpte)) {
+                        vcpu->kvm->tlbs_dirty++;
+                        continue;
+                }
+
+                if (gfn != sp->gfns[i]) {
+                        drop_spte(vcpu->kvm, &sp->spt[i],
+                                      shadow_trap_nonpresent_pte);
+                        vcpu->kvm->tlbs_dirty++;
                         continue;
                 }
 
                 nr_present++;
                 pte_access = sp->role.access & FNAME(gpte_access)(vcpu, gpte);
-                if (!(sp->spt[i] & SPTE_HOST_WRITEABLE)) {
-                        pte_access &= ~ACC_WRITE_MASK;
-                        reset_host_protection = 0;
-                } else {
-                        reset_host_protection = 1;
-                }
+                host_writable = sp->spt[i] & SPTE_HOST_WRITEABLE;
+
                 set_spte(vcpu, &sp->spt[i], pte_access, 0, 0,
                          is_dirty_gpte(gpte), PT_PAGE_TABLE_LEVEL, gfn,
                          spte_to_pfn(sp->spt[i]), true, false,
-                         reset_host_protection);
+                         host_writable);
         }
 
         return !nr_present;
diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c
index b81a9b7c2ca4..25bd1bc5aad2 100644
--- a/arch/x86/kvm/svm.c
+++ b/arch/x86/kvm/svm.c
@@ -31,6 +31,7 @@
 
 #include <asm/tlbflush.h>
 #include <asm/desc.h>
+#include <asm/kvm_para.h>
 
 #include <asm/virtext.h>
 #include "trace.h"
@@ -50,6 +51,10 @@ MODULE_LICENSE("GPL");
 #define SVM_FEATURE_LBRV           (1 <<  1)
 #define SVM_FEATURE_SVML           (1 <<  2)
 #define SVM_FEATURE_NRIP           (1 <<  3)
+#define SVM_FEATURE_TSC_RATE       (1 <<  4)
+#define SVM_FEATURE_VMCB_CLEAN     (1 <<  5)
+#define SVM_FEATURE_FLUSH_ASID     (1 <<  6)
+#define SVM_FEATURE_DECODE_ASSIST  (1 <<  7)
 #define SVM_FEATURE_PAUSE_FILTER   (1 << 10)
 
 #define NESTED_EXIT_HOST        0       /* Exit handled on host level */
@@ -97,10 +102,8 @@ struct nested_state {
         unsigned long vmexit_rax;
 
         /* cache for intercepts of the guest */
-        u16 intercept_cr_read;
-        u16 intercept_cr_write;
-        u16 intercept_dr_read;
-        u16 intercept_dr_write;
+        u32 intercept_cr;
+        u32 intercept_dr;
         u32 intercept_exceptions;
         u64 intercept;
 
@@ -123,7 +126,12 @@ struct vcpu_svm {
         u64 next_rip;
 
         u64 host_user_msrs[NR_HOST_SAVE_USER_MSRS];
-        u64 host_gs_base;
+        struct {
+                u16 fs;
+                u16 gs;
+                u16 ldt;
+                u64 gs_base;
+        } host;
 
         u32 *msrpm;
 
@@ -133,6 +141,7 @@ struct vcpu_svm {
 
         unsigned int3_injected;
         unsigned long int3_rip;
+        u32 apf_reason;
 };
 
 #define MSR_INVALID                     0xffffffffU
@@ -180,14 +189,151 @@ static int nested_svm_vmexit(struct vcpu_svm *svm);
 static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr,
                                       bool has_error_code, u32 error_code);
 
+enum {
+        VMCB_INTERCEPTS, /* Intercept vectors, TSC offset,
+                            pause filter count */
+        VMCB_PERM_MAP,   /* IOPM Base and MSRPM Base */
+        VMCB_ASID,       /* ASID */
+        VMCB_INTR,       /* int_ctl, int_vector */
+        VMCB_NPT,        /* npt_en, nCR3, gPAT */
+        VMCB_CR,         /* CR0, CR3, CR4, EFER */
+        VMCB_DR,         /* DR6, DR7 */
+        VMCB_DT,         /* GDT, IDT */
+        VMCB_SEG,        /* CS, DS, SS, ES, CPL */
+        VMCB_CR2,        /* CR2 only */
+        VMCB_LBR,        /* DBGCTL, BR_FROM, BR_TO, LAST_EX_FROM, LAST_EX_TO */
+        VMCB_DIRTY_MAX,
+};
+
+/* TPR and CR2 are always written before VMRUN */
+#define VMCB_ALWAYS_DIRTY_MASK  ((1U << VMCB_INTR) | (1U << VMCB_CR2))
+
+static inline void mark_all_dirty(struct vmcb *vmcb)
+{
+        vmcb->control.clean = 0;
+}
+
+static inline void mark_all_clean(struct vmcb *vmcb)
+{
+        vmcb->control.clean = ((1 << VMCB_DIRTY_MAX) - 1)
+                               & ~VMCB_ALWAYS_DIRTY_MASK;
+}
+
+static inline void mark_dirty(struct vmcb *vmcb, int bit)
+{
+        vmcb->control.clean &= ~(1 << bit);
+}
+
 static inline struct vcpu_svm *to_svm(struct kvm_vcpu *vcpu)
 {
         return container_of(vcpu, struct vcpu_svm, vcpu);
 }
 
-static inline bool is_nested(struct vcpu_svm *svm)
+static void recalc_intercepts(struct vcpu_svm *svm)
+{
+        struct vmcb_control_area *c, *h;
+        struct nested_state *g;
+
+        mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
+
+        if (!is_guest_mode(&svm->vcpu))
+                return;
+
+        c = &svm->vmcb->control;
+        h = &svm->nested.hsave->control;
+        g = &svm->nested;
+
+        c->intercept_cr = h->intercept_cr | g->intercept_cr;
+        c->intercept_dr = h->intercept_dr | g->intercept_dr;
+        c->intercept_exceptions = h->intercept_exceptions | g->intercept_exceptions;
+        c->intercept = h->intercept | g->intercept;
+}
+
+static inline struct vmcb *get_host_vmcb(struct vcpu_svm *svm)
+{
+        if (is_guest_mode(&svm->vcpu))
+                return svm->nested.hsave;
+        else
+                return svm->vmcb;
+}
+
+static inline void set_cr_intercept(struct vcpu_svm *svm, int bit)
+{
+        struct vmcb *vmcb = get_host_vmcb(svm);
+
+        vmcb->control.intercept_cr |= (1U << bit);
+
+        recalc_intercepts(svm);
+}
+
+static inline void clr_cr_intercept(struct vcpu_svm *svm, int bit)
+{
+        struct vmcb *vmcb = get_host_vmcb(svm);
+
+        vmcb->control.intercept_cr &= ~(1U << bit);
+
+        recalc_intercepts(svm);
+}
+
+static inline bool is_cr_intercept(struct vcpu_svm *svm, int bit)
+{
+        struct vmcb *vmcb = get_host_vmcb(svm);
+
+        return vmcb->control.intercept_cr & (1U << bit);
+}
+
+static inline void set_dr_intercept(struct vcpu_svm *svm, int bit)
+{
+        struct vmcb *vmcb = get_host_vmcb(svm);
+
+        vmcb->control.intercept_dr |= (1U << bit);
+
+        recalc_intercepts(svm);
+}
+
+static inline void clr_dr_intercept(struct vcpu_svm *svm, int bit)
+{
+        struct vmcb *vmcb = get_host_vmcb(svm);
+
+        vmcb->control.intercept_dr &= ~(1U << bit);
+
+        recalc_intercepts(svm);
+}
+
+static inline void set_exception_intercept(struct vcpu_svm *svm, int bit)
+{
+        struct vmcb *vmcb = get_host_vmcb(svm);
+
+        vmcb->control.intercept_exceptions |= (1U << bit);
+
+        recalc_intercepts(svm);
+}
+
+static inline void clr_exception_intercept(struct vcpu_svm *svm, int bit)
 {
-        return svm->nested.vmcb;
+        struct vmcb *vmcb = get_host_vmcb(svm);
+
+        vmcb->control.intercept_exceptions &= ~(1U << bit);
+
+        recalc_intercepts(svm);
+}
+
+static inline void set_intercept(struct vcpu_svm *svm, int bit)
+{
+        struct vmcb *vmcb = get_host_vmcb(svm);
+
+        vmcb->control.intercept |= (1ULL << bit);
+
+        recalc_intercepts(svm);
+}
+
+static inline void clr_intercept(struct vcpu_svm *svm, int bit)
+{
+        struct vmcb *vmcb = get_host_vmcb(svm);
+
+        vmcb->control.intercept &= ~(1ULL << bit);
+
+        recalc_intercepts(svm);
 }
 
 static inline void enable_gif(struct vcpu_svm *svm)
@@ -264,11 +410,6 @@ static u32 svm_msrpm_offset(u32 msr)
 
 #define MAX_INST_SIZE 15
 
-static inline u32 svm_has(u32 feat)
-{
-        return svm_features & feat;
-}
-
 static inline void clgi(void)
 {
         asm volatile (__ex(SVM_CLGI));
@@ -284,16 +425,6 @@ static inline void invlpga(unsigned long addr, u32 asid)
         asm volatile (__ex(SVM_INVLPGA) : : "a"(addr), "c"(asid));
 }
 
-static inline void force_new_asid(struct kvm_vcpu *vcpu)
-{
-        to_svm(vcpu)->asid_generation--;
-}
-
-static inline void flush_guest_tlb(struct kvm_vcpu *vcpu)
-{
-        force_new_asid(vcpu);
-}
-
 static int get_npt_level(void)
 {
 #ifdef CONFIG_X86_64
@@ -310,6 +441,7 @@ static void svm_set_efer(struct kvm_vcpu *vcpu, u64 efer)
                 efer &= ~EFER_LME;
 
         to_svm(vcpu)->vmcb->save.efer = efer | EFER_SVME;
+        mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR);
 }
 
 static int is_external_interrupt(u32 info)
@@ -347,7 +479,7 @@ static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
                 svm->next_rip = svm->vmcb->control.next_rip;
 
         if (!svm->next_rip) {
-                if (emulate_instruction(vcpu, 0, 0, EMULTYPE_SKIP) !=
+                if (emulate_instruction(vcpu, EMULTYPE_SKIP) !=
                                 EMULATE_DONE)
                         printk(KERN_DEBUG "%s: NOP\n", __func__);
                 return;
@@ -374,7 +506,7 @@ static void svm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr,
             nested_svm_check_exception(svm, nr, has_error_code, error_code))
                 return;
 
-        if (nr == BP_VECTOR && !svm_has(SVM_FEATURE_NRIP)) {
+        if (nr == BP_VECTOR && !static_cpu_has(X86_FEATURE_NRIPS)) {
                 unsigned long rip, old_rip = kvm_rip_read(&svm->vcpu);
 
                 /*
@@ -670,7 +802,7 @@ static __init int svm_hardware_setup(void)
 
         svm_features = cpuid_edx(SVM_CPUID_FUNC);
 
-        if (!svm_has(SVM_FEATURE_NPT))
+        if (!boot_cpu_has(X86_FEATURE_NPT))
                 npt_enabled = false;
 
         if (npt_enabled && !npt) {
@@ -725,13 +857,15 @@ static void svm_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)
         struct vcpu_svm *svm = to_svm(vcpu);
         u64 g_tsc_offset = 0;
 
-        if (is_nested(svm)) {
+        if (is_guest_mode(vcpu)) {
                 g_tsc_offset = svm->vmcb->control.tsc_offset -
                                svm->nested.hsave->control.tsc_offset;
                 svm->nested.hsave->control.tsc_offset = offset;
         }
 
         svm->vmcb->control.tsc_offset = offset + g_tsc_offset;
+
+        mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
 }
 
 static void svm_adjust_tsc_offset(struct kvm_vcpu *vcpu, s64 adjustment)
@@ -739,8 +873,9 @@ static void svm_adjust_tsc_offset(struct kvm_vcpu *vcpu, s64 adjustment)
         struct vcpu_svm *svm = to_svm(vcpu);
 
         svm->vmcb->control.tsc_offset += adjustment;
-        if (is_nested(svm))
+        if (is_guest_mode(vcpu))
                 svm->nested.hsave->control.tsc_offset += adjustment;
+        mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
 }
 
 static void init_vmcb(struct vcpu_svm *svm)
@@ -749,62 +884,62 @@ static void init_vmcb(struct vcpu_svm *svm)
         struct vmcb_save_area *save = &svm->vmcb->save;
 
         svm->vcpu.fpu_active = 1;
+        svm->vcpu.arch.hflags = 0;
 
-        control->intercept_cr_read =    INTERCEPT_CR0_MASK |
-                                        INTERCEPT_CR3_MASK |
-                                        INTERCEPT_CR4_MASK;
-
-        control->intercept_cr_write =   INTERCEPT_CR0_MASK |
-                                        INTERCEPT_CR3_MASK |
-                                        INTERCEPT_CR4_MASK |
-                                        INTERCEPT_CR8_MASK;
-
-        control->intercept_dr_read =    INTERCEPT_DR0_MASK |
-                                        INTERCEPT_DR1_MASK |
-                                        INTERCEPT_DR2_MASK |
-                                        INTERCEPT_DR3_MASK |
-                                        INTERCEPT_DR4_MASK |
-                                        INTERCEPT_DR5_MASK |
-                                        INTERCEPT_DR6_MASK |
-                                        INTERCEPT_DR7_MASK;
-
-        control->intercept_dr_write =   INTERCEPT_DR0_MASK |
-                                        INTERCEPT_DR1_MASK |
-                                        INTERCEPT_DR2_MASK |
-                                        INTERCEPT_DR3_MASK |
-                                        INTERCEPT_DR4_MASK |
-                                        INTERCEPT_DR5_MASK |
-                                        INTERCEPT_DR6_MASK |
-                                        INTERCEPT_DR7_MASK;
-
-        control->intercept_exceptions = (1 << PF_VECTOR) |
-                                        (1 << UD_VECTOR) |
-                                        (1 << MC_VECTOR);
-
-
-        control->intercept =    (1ULL << INTERCEPT_INTR) |
-                                (1ULL << INTERCEPT_NMI) |
-                                (1ULL << INTERCEPT_SMI) |
-                                (1ULL << INTERCEPT_SELECTIVE_CR0) |
-                                (1ULL << INTERCEPT_CPUID) |
-                                (1ULL << INTERCEPT_INVD) |
-                                (1ULL << INTERCEPT_HLT) |
-                                (1ULL << INTERCEPT_INVLPG) |
-                                (1ULL << INTERCEPT_INVLPGA) |
-                                (1ULL << INTERCEPT_IOIO_PROT) |
-                                (1ULL << INTERCEPT_MSR_PROT) |
-                                (1ULL << INTERCEPT_TASK_SWITCH) |
-                                (1ULL << INTERCEPT_SHUTDOWN) |
-                                (1ULL << INTERCEPT_VMRUN) |
-                                (1ULL << INTERCEPT_VMMCALL) |
-                                (1ULL << INTERCEPT_VMLOAD) |
-                                (1ULL << INTERCEPT_VMSAVE) |
-                                (1ULL << INTERCEPT_STGI) |
-                                (1ULL << INTERCEPT_CLGI) |
-                                (1ULL << INTERCEPT_SKINIT) |
-                                (1ULL << INTERCEPT_WBINVD) |
-                                (1ULL << INTERCEPT_MONITOR) |
-                                (1ULL << INTERCEPT_MWAIT);
+        set_cr_intercept(svm, INTERCEPT_CR0_READ);
+        set_cr_intercept(svm, INTERCEPT_CR3_READ);
+        set_cr_intercept(svm, INTERCEPT_CR4_READ);
+        set_cr_intercept(svm, INTERCEPT_CR0_WRITE);
+        set_cr_intercept(svm, INTERCEPT_CR3_WRITE);
+        set_cr_intercept(svm, INTERCEPT_CR4_WRITE);
+        set_cr_intercept(svm, INTERCEPT_CR8_WRITE);
+
+        set_dr_intercept(svm, INTERCEPT_DR0_READ);
+        set_dr_intercept(svm, INTERCEPT_DR1_READ);
+        set_dr_intercept(svm, INTERCEPT_DR2_READ);
+        set_dr_intercept(svm, INTERCEPT_DR3_READ);
+        set_dr_intercept(svm, INTERCEPT_DR4_READ);
+        set_dr_intercept(svm, INTERCEPT_DR5_READ);
+        set_dr_intercept(svm, INTERCEPT_DR6_READ);
+        set_dr_intercept(svm, INTERCEPT_DR7_READ);
+
+        set_dr_intercept(svm, INTERCEPT_DR0_WRITE);
+        set_dr_intercept(svm, INTERCEPT_DR1_WRITE);
+        set_dr_intercept(svm, INTERCEPT_DR2_WRITE);
+        set_dr_intercept(svm, INTERCEPT_DR3_WRITE);
+        set_dr_intercept(svm, INTERCEPT_DR4_WRITE);
+        set_dr_intercept(svm, INTERCEPT_DR5_WRITE);
+        set_dr_intercept(svm, INTERCEPT_DR6_WRITE);
+        set_dr_intercept(svm, INTERCEPT_DR7_WRITE);
+
+        set_exception_intercept(svm, PF_VECTOR);
+        set_exception_intercept(svm, UD_VECTOR);
+        set_exception_intercept(svm, MC_VECTOR);
+
+        set_intercept(svm, INTERCEPT_INTR);
+        set_intercept(svm, INTERCEPT_NMI);
+        set_intercept(svm, INTERCEPT_SMI);
+        set_intercept(svm, INTERCEPT_SELECTIVE_CR0);
+        set_intercept(svm, INTERCEPT_CPUID);
+        set_intercept(svm, INTERCEPT_INVD);
+        set_intercept(svm, INTERCEPT_HLT);
+        set_intercept(svm, INTERCEPT_INVLPG);
+        set_intercept(svm, INTERCEPT_INVLPGA);
+        set_intercept(svm, INTERCEPT_IOIO_PROT);
+        set_intercept(svm, INTERCEPT_MSR_PROT);
+        set_intercept(svm, INTERCEPT_TASK_SWITCH);
+        set_intercept(svm, INTERCEPT_SHUTDOWN);
+        set_intercept(svm, INTERCEPT_VMRUN);
+        set_intercept(svm, INTERCEPT_VMMCALL);
+        set_intercept(svm, INTERCEPT_VMLOAD);
+        set_intercept(svm, INTERCEPT_VMSAVE);
+        set_intercept(svm, INTERCEPT_STGI);
+        set_intercept(svm, INTERCEPT_CLGI);
+        set_intercept(svm, INTERCEPT_SKINIT);
+        set_intercept(svm, INTERCEPT_WBINVD);
+        set_intercept(svm, INTERCEPT_MONITOR);
+        set_intercept(svm, INTERCEPT_MWAIT);
+        set_intercept(svm, INTERCEPT_XSETBV);
 
         control->iopm_base_pa = iopm_base;
         control->msrpm_base_pa = __pa(svm->msrpm);
@@ -855,25 +990,27 @@ static void init_vmcb(struct vcpu_svm *svm)
         if (npt_enabled) {
                 /* Setup VMCB for Nested Paging */
                 control->nested_ctl = 1;
-                control->intercept &= ~((1ULL << INTERCEPT_TASK_SWITCH) |
-                                        (1ULL << INTERCEPT_INVLPG));
-                control->intercept_exceptions &= ~(1 << PF_VECTOR);
-                control->intercept_cr_read &= ~INTERCEPT_CR3_MASK;
-                control->intercept_cr_write &= ~INTERCEPT_CR3_MASK;
+                clr_intercept(svm, INTERCEPT_TASK_SWITCH);
+                clr_intercept(svm, INTERCEPT_INVLPG);
+                clr_exception_intercept(svm, PF_VECTOR);
+                clr_cr_intercept(svm, INTERCEPT_CR3_READ);
+                clr_cr_intercept(svm, INTERCEPT_CR3_WRITE);
                 save->g_pat = 0x0007040600070406ULL;
                 save->cr3 = 0;
                 save->cr4 = 0;
         }
-        force_new_asid(&svm->vcpu);
+        svm->asid_generation = 0;
 
         svm->nested.vmcb = 0;
         svm->vcpu.arch.hflags = 0;
 
-        if (svm_has(SVM_FEATURE_PAUSE_FILTER)) {
+        if (boot_cpu_has(X86_FEATURE_PAUSEFILTER)) {
                 control->pause_filter_count = 3000;
-                control->intercept |= (1ULL << INTERCEPT_PAUSE);
+                set_intercept(svm, INTERCEPT_PAUSE);
         }
 
+        mark_all_dirty(svm->vmcb);
+
         enable_gif(svm);
 }
 
@@ -990,8 +1127,16 @@ static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 
         if (unlikely(cpu != vcpu->cpu)) {
                 svm->asid_generation = 0;
+                mark_all_dirty(svm->vmcb);
         }
 
+#ifdef CONFIG_X86_64
+        rdmsrl(MSR_GS_BASE, to_svm(vcpu)->host.gs_base);
+#endif
+        savesegment(fs, svm->host.fs);
+        savesegment(gs, svm->host.gs);
+        svm->host.ldt = kvm_read_ldt();
+
         for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
                 rdmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
 }
@@ -1002,6 +1147,14 @@ static void svm_vcpu_put(struct kvm_vcpu *vcpu)
         int i;
 
         ++vcpu->stat.host_state_reload;
+        kvm_load_ldt(svm->host.ldt);
+#ifdef CONFIG_X86_64
+        loadsegment(fs, svm->host.fs);
+        load_gs_index(svm->host.gs);
+        wrmsrl(MSR_KERNEL_GS_BASE, current->thread.gs);
+#else
+        loadsegment(gs, svm->host.gs);
+#endif
         for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
                 wrmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
 }
@@ -1021,7 +1174,7 @@ static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
         switch (reg) {
         case VCPU_EXREG_PDPTR:
                 BUG_ON(!npt_enabled);
-                load_pdptrs(vcpu, vcpu->arch.walk_mmu, vcpu->arch.cr3);
+                load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu));
                 break;
         default:
                 BUG();
@@ -1030,12 +1183,12 @@ static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
 
 static void svm_set_vintr(struct vcpu_svm *svm)
 {
-        svm->vmcb->control.intercept |= 1ULL << INTERCEPT_VINTR;
+        set_intercept(svm, INTERCEPT_VINTR);
 }
 
 static void svm_clear_vintr(struct vcpu_svm *svm)
 {
-        svm->vmcb->control.intercept &= ~(1ULL << INTERCEPT_VINTR);
+        clr_intercept(svm, INTERCEPT_VINTR);
 }
 
 static struct vmcb_seg *svm_seg(struct kvm_vcpu *vcpu, int seg)
@@ -1150,6 +1303,7 @@ static void svm_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
 
         svm->vmcb->save.idtr.limit = dt->size;
         svm->vmcb->save.idtr.base = dt->address ;
+        mark_dirty(svm->vmcb, VMCB_DT);
 }
 
 static void svm_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
@@ -1166,19 +1320,23 @@ static void svm_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
 
         svm->vmcb->save.gdtr.limit = dt->size;
         svm->vmcb->save.gdtr.base = dt->address ;
+        mark_dirty(svm->vmcb, VMCB_DT);
 }
 
 static void svm_decache_cr0_guest_bits(struct kvm_vcpu *vcpu)
 {
 }
 
+static void svm_decache_cr3(struct kvm_vcpu *vcpu)
+{
+}
+
 static void svm_decache_cr4_guest_bits(struct kvm_vcpu *vcpu)
 {
 }
 
 static void update_cr0_intercept(struct vcpu_svm *svm)
 {
-        struct vmcb *vmcb = svm->vmcb;
         ulong gcr0 = svm->vcpu.arch.cr0;
         u64 *hcr0 = &svm->vmcb->save.cr0;
 
@@ -1188,27 +1346,14 @@ static void update_cr0_intercept(struct vcpu_svm *svm)
                 *hcr0 = (*hcr0 & ~SVM_CR0_SELECTIVE_MASK)
                         | (gcr0 & SVM_CR0_SELECTIVE_MASK);
 
+        mark_dirty(svm->vmcb, VMCB_CR);
 
         if (gcr0 == *hcr0 && svm->vcpu.fpu_active) {
-                vmcb->control.intercept_cr_read &= ~INTERCEPT_CR0_MASK;
-                vmcb->control.intercept_cr_write &= ~INTERCEPT_CR0_MASK;
-                if (is_nested(svm)) {
-                        struct vmcb *hsave = svm->nested.hsave;
-
-                        hsave->control.intercept_cr_read  &= ~INTERCEPT_CR0_MASK;
-                        hsave->control.intercept_cr_write &= ~INTERCEPT_CR0_MASK;
-                        vmcb->control.intercept_cr_read  |= svm->nested.intercept_cr_read;
-                        vmcb->control.intercept_cr_write |= svm->nested.intercept_cr_write;
-                }
+                clr_cr_intercept(svm, INTERCEPT_CR0_READ);
+                clr_cr_intercept(svm, INTERCEPT_CR0_WRITE);
         } else {
-                svm->vmcb->control.intercept_cr_read |= INTERCEPT_CR0_MASK;
-                svm->vmcb->control.intercept_cr_write |= INTERCEPT_CR0_MASK;
-                if (is_nested(svm)) {
-                        struct vmcb *hsave = svm->nested.hsave;
-
-                        hsave->control.intercept_cr_read |= INTERCEPT_CR0_MASK;
-                        hsave->control.intercept_cr_write |= INTERCEPT_CR0_MASK;
-                }
+                set_cr_intercept(svm, INTERCEPT_CR0_READ);
+                set_cr_intercept(svm, INTERCEPT_CR0_WRITE);
         }
 }
 
@@ -1216,7 +1361,7 @@ static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
 {
         struct vcpu_svm *svm = to_svm(vcpu);
 
-        if (is_nested(svm)) {
+        if (is_guest_mode(vcpu)) {
                 /*
                  * We are here because we run in nested mode, the host kvm
                  * intercepts cr0 writes but the l1 hypervisor does not.
@@ -1268,6 +1413,7 @@ static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
          */
         cr0 &= ~(X86_CR0_CD | X86_CR0_NW);
         svm->vmcb->save.cr0 = cr0;
+        mark_dirty(svm->vmcb, VMCB_CR);
         update_cr0_intercept(svm);
 }
 
@@ -1277,13 +1423,14 @@ static void svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
         unsigned long old_cr4 = to_svm(vcpu)->vmcb->save.cr4;
 
         if (npt_enabled && ((old_cr4 ^ cr4) & X86_CR4_PGE))
-                force_new_asid(vcpu);
+                svm_flush_tlb(vcpu);
 
         vcpu->arch.cr4 = cr4;
         if (!npt_enabled)
                 cr4 |= X86_CR4_PAE;
         cr4 |= host_cr4_mce;
         to_svm(vcpu)->vmcb->save.cr4 = cr4;
+        mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR);
 }
 
 static void svm_set_segment(struct kvm_vcpu *vcpu,
@@ -1312,26 +1459,25 @@ static void svm_set_segment(struct kvm_vcpu *vcpu,
                         = (svm->vmcb->save.cs.attrib
                            >> SVM_SELECTOR_DPL_SHIFT) & 3;
 
+        mark_dirty(svm->vmcb, VMCB_SEG);
 }
 
 static void update_db_intercept(struct kvm_vcpu *vcpu)
 {
         struct vcpu_svm *svm = to_svm(vcpu);
 
-        svm->vmcb->control.intercept_exceptions &=
-                ~((1 << DB_VECTOR) | (1 << BP_VECTOR));
+        clr_exception_intercept(svm, DB_VECTOR);
+        clr_exception_intercept(svm, BP_VECTOR);
 
         if (svm->nmi_singlestep)
-                svm->vmcb->control.intercept_exceptions |= (1 << DB_VECTOR);
+                set_exception_intercept(svm, DB_VECTOR);
 
         if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) {
                 if (vcpu->guest_debug &
                     (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))
-                        svm->vmcb->control.intercept_exceptions |=
-                                1 << DB_VECTOR;
+                        set_exception_intercept(svm, DB_VECTOR);
                 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
-                        svm->vmcb->control.intercept_exceptions |=
-                                1 << BP_VECTOR;
+                        set_exception_intercept(svm, BP_VECTOR);
         } else
                 vcpu->guest_debug = 0;
 }
@@ -1345,21 +1491,9 @@ static void svm_guest_debug(struct kvm_vcpu *vcpu, struct kvm_guest_debug *dbg)
         else
                 svm->vmcb->save.dr7 = vcpu->arch.dr7;
 
-        update_db_intercept(vcpu);
-}
-
-static void load_host_msrs(struct kvm_vcpu *vcpu)
-{
-#ifdef CONFIG_X86_64
-        wrmsrl(MSR_GS_BASE, to_svm(vcpu)->host_gs_base);
-#endif
-}
+        mark_dirty(svm->vmcb, VMCB_DR);
 
-static void save_host_msrs(struct kvm_vcpu *vcpu)
-{
-#ifdef CONFIG_X86_64
-        rdmsrl(MSR_GS_BASE, to_svm(vcpu)->host_gs_base);
-#endif
+        update_db_intercept(vcpu);
 }
 
 static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd)
@@ -1372,6 +1506,8 @@ static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd)
 
         svm->asid_generation = sd->asid_generation;
         svm->vmcb->control.asid = sd->next_asid++;
+
+        mark_dirty(svm->vmcb, VMCB_ASID);
 }
 
 static void svm_set_dr7(struct kvm_vcpu *vcpu, unsigned long value)
@@ -1379,20 +1515,40 @@ static void svm_set_dr7(struct kvm_vcpu *vcpu, unsigned long value)
         struct vcpu_svm *svm = to_svm(vcpu);
 
         svm->vmcb->save.dr7 = value;
+        mark_dirty(svm->vmcb, VMCB_DR);
 }
 
 static int pf_interception(struct vcpu_svm *svm)
 {
-        u64 fault_address;
+        u64 fault_address = svm->vmcb->control.exit_info_2;
         u32 error_code;
+        int r = 1;
 
-        fault_address  = svm->vmcb->control.exit_info_2;
-        error_code = svm->vmcb->control.exit_info_1;
-
-        trace_kvm_page_fault(fault_address, error_code);
-        if (!npt_enabled && kvm_event_needs_reinjection(&svm->vcpu))
-                kvm_mmu_unprotect_page_virt(&svm->vcpu, fault_address);
-        return kvm_mmu_page_fault(&svm->vcpu, fault_address, error_code);
+        switch (svm->apf_reason) {
+        default:
+                error_code = svm->vmcb->control.exit_info_1;
+
+                trace_kvm_page_fault(fault_address, error_code);
+                if (!npt_enabled && kvm_event_needs_reinjection(&svm->vcpu))
+                        kvm_mmu_unprotect_page_virt(&svm->vcpu, fault_address);
+                r = kvm_mmu_page_fault(&svm->vcpu, fault_address, error_code,
+                        svm->vmcb->control.insn_bytes,
+                        svm->vmcb->control.insn_len);
+                break;
+        case KVM_PV_REASON_PAGE_NOT_PRESENT:
+                svm->apf_reason = 0;
+                local_irq_disable();
+                kvm_async_pf_task_wait(fault_address);
+                local_irq_enable();
+                break;
+        case KVM_PV_REASON_PAGE_READY:
+                svm->apf_reason = 0;
+                local_irq_disable();
+                kvm_async_pf_task_wake(fault_address);
+                local_irq_enable();
+                break;
+        }
+        return r;
 }
 
 static int db_interception(struct vcpu_svm *svm)
@@ -1440,7 +1596,7 @@ static int ud_interception(struct vcpu_svm *svm)
 {
         int er;
 
-        er = emulate_instruction(&svm->vcpu, 0, 0, EMULTYPE_TRAP_UD);
+        er = emulate_instruction(&svm->vcpu, EMULTYPE_TRAP_UD);
         if (er != EMULATE_DONE)
                 kvm_queue_exception(&svm->vcpu, UD_VECTOR);
         return 1;
@@ -1449,21 +1605,8 @@ static int ud_interception(struct vcpu_svm *svm)
 static void svm_fpu_activate(struct kvm_vcpu *vcpu)
 {
         struct vcpu_svm *svm = to_svm(vcpu);
-        u32 excp;
-
-        if (is_nested(svm)) {
-                u32 h_excp, n_excp;
-
-                h_excp  = svm->nested.hsave->control.intercept_exceptions;
-                n_excp  = svm->nested.intercept_exceptions;
-                h_excp &= ~(1 << NM_VECTOR);
-                excp    = h_excp | n_excp;
-        } else {
-                excp  = svm->vmcb->control.intercept_exceptions;
-                excp &= ~(1 << NM_VECTOR);
-        }
 
-        svm->vmcb->control.intercept_exceptions = excp;
+        clr_exception_intercept(svm, NM_VECTOR);
 
         svm->vcpu.fpu_active = 1;
         update_cr0_intercept(svm);
@@ -1570,7 +1713,7 @@ static int io_interception(struct vcpu_svm *svm)
         string = (io_info & SVM_IOIO_STR_MASK) != 0;
         in = (io_info & SVM_IOIO_TYPE_MASK) != 0;
         if (string || in)
-                return emulate_instruction(vcpu, 0, 0, 0) == EMULATE_DONE;
+                return emulate_instruction(vcpu, 0) == EMULATE_DONE;
 
         port = io_info >> 16;
         size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT;
@@ -1624,17 +1767,19 @@ static void nested_svm_set_tdp_cr3(struct kvm_vcpu *vcpu,
         struct vcpu_svm *svm = to_svm(vcpu);
 
         svm->vmcb->control.nested_cr3 = root;
-        force_new_asid(vcpu);
+        mark_dirty(svm->vmcb, VMCB_NPT);
+        svm_flush_tlb(vcpu);
 }
 
-static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu)
+static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu,
+                                       struct x86_exception *fault)
 {
         struct vcpu_svm *svm = to_svm(vcpu);
 
         svm->vmcb->control.exit_code = SVM_EXIT_NPF;
         svm->vmcb->control.exit_code_hi = 0;
-        svm->vmcb->control.exit_info_1 = vcpu->arch.fault.error_code;
-        svm->vmcb->control.exit_info_2 = vcpu->arch.fault.address;
+        svm->vmcb->control.exit_info_1 = fault->error_code;
+        svm->vmcb->control.exit_info_2 = fault->address;
 
         nested_svm_vmexit(svm);
 }
@@ -1680,7 +1825,7 @@ static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr,
 {
         int vmexit;
 
-        if (!is_nested(svm))
+        if (!is_guest_mode(&svm->vcpu))
                 return 0;
 
         svm->vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + nr;
@@ -1698,7 +1843,7 @@ static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr,
 /* This function returns true if it is save to enable the irq window */
 static inline bool nested_svm_intr(struct vcpu_svm *svm)
 {
-        if (!is_nested(svm))
+        if (!is_guest_mode(&svm->vcpu))
                 return true;
 
         if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK))
@@ -1737,7 +1882,7 @@ static inline bool nested_svm_intr(struct vcpu_svm *svm)
 /* This function returns true if it is save to enable the nmi window */
 static inline bool nested_svm_nmi(struct vcpu_svm *svm)
 {
-        if (!is_nested(svm))
+        if (!is_guest_mode(&svm->vcpu))
                 return true;
 
         if (!(svm->nested.intercept & (1ULL << INTERCEPT_NMI)))
@@ -1836,8 +1981,8 @@ static int nested_svm_exit_special(struct vcpu_svm *svm)
                         return NESTED_EXIT_HOST;
                 break;
         case SVM_EXIT_EXCP_BASE + PF_VECTOR:
-                /* When we're shadowing, trap PFs */
-                if (!npt_enabled)
+                /* When we're shadowing, trap PFs, but not async PF */
+                if (!npt_enabled && svm->apf_reason == 0)
                         return NESTED_EXIT_HOST;
                 break;
         case SVM_EXIT_EXCP_BASE + NM_VECTOR:
@@ -1865,27 +2010,15 @@ static int nested_svm_intercept(struct vcpu_svm *svm)
         case SVM_EXIT_IOIO:
                 vmexit = nested_svm_intercept_ioio(svm);
                 break;
-        case SVM_EXIT_READ_CR0 ... SVM_EXIT_READ_CR8: {
-                u32 cr_bits = 1 << (exit_code - SVM_EXIT_READ_CR0);
-                if (svm->nested.intercept_cr_read & cr_bits)
+        case SVM_EXIT_READ_CR0 ... SVM_EXIT_WRITE_CR8: {
+                u32 bit = 1U << (exit_code - SVM_EXIT_READ_CR0);
+                if (svm->nested.intercept_cr & bit)
                         vmexit = NESTED_EXIT_DONE;
                 break;
         }
-        case SVM_EXIT_WRITE_CR0 ... SVM_EXIT_WRITE_CR8: {
-                u32 cr_bits = 1 << (exit_code - SVM_EXIT_WRITE_CR0);
-                if (svm->nested.intercept_cr_write & cr_bits)
-                        vmexit = NESTED_EXIT_DONE;
-                break;
-        }
-        case SVM_EXIT_READ_DR0 ... SVM_EXIT_READ_DR7: {
-                u32 dr_bits = 1 << (exit_code - SVM_EXIT_READ_DR0);
-                if (svm->nested.intercept_dr_read & dr_bits)
-                        vmexit = NESTED_EXIT_DONE;
-                break;
-        }
-        case SVM_EXIT_WRITE_DR0 ... SVM_EXIT_WRITE_DR7: {
-                u32 dr_bits = 1 << (exit_code - SVM_EXIT_WRITE_DR0);
-                if (svm->nested.intercept_dr_write & dr_bits)
+        case SVM_EXIT_READ_DR0 ... SVM_EXIT_WRITE_DR7: {
+                u32 bit = 1U << (exit_code - SVM_EXIT_READ_DR0);
+                if (svm->nested.intercept_dr & bit)
                         vmexit = NESTED_EXIT_DONE;
                 break;
         }
@@ -1893,6 +2026,10 @@ static int nested_svm_intercept(struct vcpu_svm *svm)
                 u32 excp_bits = 1 << (exit_code - SVM_EXIT_EXCP_BASE);
                 if (svm->nested.intercept_exceptions & excp_bits)
                         vmexit = NESTED_EXIT_DONE;
+                /* async page fault always cause vmexit */
+                else if ((exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR) &&
+                         svm->apf_reason != 0)
+                        vmexit = NESTED_EXIT_DONE;
                 break;
         }
         case SVM_EXIT_ERR: {
@@ -1926,10 +2063,8 @@ static inline void copy_vmcb_control_area(struct vmcb *dst_vmcb, struct vmcb *fr
         struct vmcb_control_area *dst  = &dst_vmcb->control;
         struct vmcb_control_area *from = &from_vmcb->control;
 
-        dst->intercept_cr_read    = from->intercept_cr_read;
-        dst->intercept_cr_write   = from->intercept_cr_write;
-        dst->intercept_dr_read    = from->intercept_dr_read;
-        dst->intercept_dr_write   = from->intercept_dr_write;
+        dst->intercept_cr         = from->intercept_cr;
+        dst->intercept_dr         = from->intercept_dr;
         dst->intercept_exceptions = from->intercept_exceptions;
         dst->intercept            = from->intercept;
         dst->iopm_base_pa         = from->iopm_base_pa;
@@ -1970,7 +2105,8 @@ static int nested_svm_vmexit(struct vcpu_svm *svm)
         if (!nested_vmcb)
                 return 1;
 
-        /* Exit nested SVM mode */
+        /* Exit Guest-Mode */
+        leave_guest_mode(&svm->vcpu);
         svm->nested.vmcb = 0;
 
         /* Give the current vmcb to the guest */
@@ -1984,7 +2120,7 @@ static int nested_svm_vmexit(struct vcpu_svm *svm)
         nested_vmcb->save.idtr   = vmcb->save.idtr;
         nested_vmcb->save.efer   = svm->vcpu.arch.efer;
         nested_vmcb->save.cr0    = kvm_read_cr0(&svm->vcpu);
-        nested_vmcb->save.cr3    = svm->vcpu.arch.cr3;
+        nested_vmcb->save.cr3    = kvm_read_cr3(&svm->vcpu);
         nested_vmcb->save.cr2    = vmcb->save.cr2;
         nested_vmcb->save.cr4    = svm->vcpu.arch.cr4;
         nested_vmcb->save.rflags = vmcb->save.rflags;
@@ -2061,6 +2197,8 @@ static int nested_svm_vmexit(struct vcpu_svm *svm)
         svm->vmcb->save.cpl = 0;
         svm->vmcb->control.exit_int_info = 0;
 
+        mark_all_dirty(svm->vmcb);
+
         nested_svm_unmap(page);
 
         nested_svm_uninit_mmu_context(&svm->vcpu);
@@ -2148,8 +2286,8 @@ static bool nested_svm_vmrun(struct vcpu_svm *svm)
                                nested_vmcb->control.event_inj,
                                nested_vmcb->control.nested_ctl);
 
-        trace_kvm_nested_intercepts(nested_vmcb->control.intercept_cr_read,
-                                    nested_vmcb->control.intercept_cr_write,
+        trace_kvm_nested_intercepts(nested_vmcb->control.intercept_cr & 0xffff,
+                                    nested_vmcb->control.intercept_cr >> 16,
                                     nested_vmcb->control.intercept_exceptions,
                                     nested_vmcb->control.intercept);
 
@@ -2177,7 +2315,7 @@ static bool nested_svm_vmrun(struct vcpu_svm *svm)
         if (npt_enabled)
                 hsave->save.cr3    = vmcb->save.cr3;
         else
-                hsave->save.cr3    = svm->vcpu.arch.cr3;
+                hsave->save.cr3    = kvm_read_cr3(&svm->vcpu);
 
         copy_vmcb_control_area(hsave, vmcb);
 
@@ -2229,14 +2367,12 @@ static bool nested_svm_vmrun(struct vcpu_svm *svm)
         svm->nested.vmcb_iopm  = nested_vmcb->control.iopm_base_pa  & ~0x0fffULL;
 
         /* cache intercepts */
-        svm->nested.intercept_cr_read    = nested_vmcb->control.intercept_cr_read;
-        svm->nested.intercept_cr_write   = nested_vmcb->control.intercept_cr_write;
-        svm->nested.intercept_dr_read    = nested_vmcb->control.intercept_dr_read;
-        svm->nested.intercept_dr_write   = nested_vmcb->control.intercept_dr_write;
+        svm->nested.intercept_cr         = nested_vmcb->control.intercept_cr;
+        svm->nested.intercept_dr         = nested_vmcb->control.intercept_dr;
         svm->nested.intercept_exceptions = nested_vmcb->control.intercept_exceptions;
         svm->nested.intercept            = nested_vmcb->control.intercept;
 
-        force_new_asid(&svm->vcpu);
+        svm_flush_tlb(&svm->vcpu);
         svm->vmcb->control.int_ctl = nested_vmcb->control.int_ctl | V_INTR_MASKING_MASK;
         if (nested_vmcb->control.int_ctl & V_INTR_MASKING_MASK)
                 svm->vcpu.arch.hflags |= HF_VINTR_MASK;
@@ -2245,29 +2381,12 @@ static bool nested_svm_vmrun(struct vcpu_svm *svm)
 
         if (svm->vcpu.arch.hflags & HF_VINTR_MASK) {
                 /* We only want the cr8 intercept bits of the guest */
-                svm->vmcb->control.intercept_cr_read &= ~INTERCEPT_CR8_MASK;
-                svm->vmcb->control.intercept_cr_write &= ~INTERCEPT_CR8_MASK;
+                clr_cr_intercept(svm, INTERCEPT_CR8_READ);
+                clr_cr_intercept(svm, INTERCEPT_CR8_WRITE);
         }
 
         /* We don't want to see VMMCALLs from a nested guest */
-        svm->vmcb->control.intercept &= ~(1ULL << INTERCEPT_VMMCALL);
-
-        /*
-         * We don't want a nested guest to be more powerful than the guest, so
-         * all intercepts are ORed
-         */
-        svm->vmcb->control.intercept_cr_read |=
-                nested_vmcb->control.intercept_cr_read;
-        svm->vmcb->control.intercept_cr_write |=
-                nested_vmcb->control.intercept_cr_write;
-        svm->vmcb->control.intercept_dr_read |=
-                nested_vmcb->control.intercept_dr_read;
-        svm->vmcb->control.intercept_dr_write |=
-                nested_vmcb->control.intercept_dr_write;
-        svm->vmcb->control.intercept_exceptions |=
-                nested_vmcb->control.intercept_exceptions;
-
-        svm->vmcb->control.intercept |= nested_vmcb->control.intercept;
+        clr_intercept(svm, INTERCEPT_VMMCALL);
 
         svm->vmcb->control.lbr_ctl = nested_vmcb->control.lbr_ctl;
         svm->vmcb->control.int_vector = nested_vmcb->control.int_vector;
@@ -2278,11 +2397,21 @@ static bool nested_svm_vmrun(struct vcpu_svm *svm)
 
         nested_svm_unmap(page);
 
-        /* nested_vmcb is our indicator if nested SVM is activated */
+        /* Enter Guest-Mode */
+        enter_guest_mode(&svm->vcpu);
+
+        /*
+         * Merge guest and host intercepts - must be called  with vcpu in
+         * guest-mode to take affect here
+         */
+        recalc_intercepts(svm);
+
         svm->nested.vmcb = vmcb_gpa;
 
         enable_gif(svm);
 
+        mark_all_dirty(svm->vmcb);
+
         return true;
 }
 
@@ -2400,6 +2529,8 @@ static int clgi_interception(struct vcpu_svm *svm)
         svm_clear_vintr(svm);
         svm->vmcb->control.int_ctl &= ~V_IRQ_MASK;
 
+        mark_dirty(svm->vmcb, VMCB_INTR);
+
         return 1;
 }
 
@@ -2426,6 +2557,19 @@ static int skinit_interception(struct vcpu_svm *svm)
         return 1;
 }
 
+static int xsetbv_interception(struct vcpu_svm *svm)
+{
+        u64 new_bv = kvm_read_edx_eax(&svm->vcpu);
+        u32 index = kvm_register_read(&svm->vcpu, VCPU_REGS_RCX);
+
+        if (kvm_set_xcr(&svm->vcpu, index, new_bv) == 0) {
+                svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
+                skip_emulated_instruction(&svm->vcpu);
+        }
+
+        return 1;
+}
+
 static int invalid_op_interception(struct vcpu_svm *svm)
 {
         kvm_queue_exception(&svm->vcpu, UD_VECTOR);
@@ -2507,19 +2651,92 @@ static int cpuid_interception(struct vcpu_svm *svm)
 static int iret_interception(struct vcpu_svm *svm)
 {
         ++svm->vcpu.stat.nmi_window_exits;
-        svm->vmcb->control.intercept &= ~(1ULL << INTERCEPT_IRET);
+        clr_intercept(svm, INTERCEPT_IRET);
         svm->vcpu.arch.hflags |= HF_IRET_MASK;
         return 1;
 }
 
 static int invlpg_interception(struct vcpu_svm *svm)
 {
-        return emulate_instruction(&svm->vcpu, 0, 0, 0) == EMULATE_DONE;
+        if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
+                return emulate_instruction(&svm->vcpu, 0) == EMULATE_DONE;
+
+        kvm_mmu_invlpg(&svm->vcpu, svm->vmcb->control.exit_info_1);
+        skip_emulated_instruction(&svm->vcpu);
+        return 1;
 }
 
 static int emulate_on_interception(struct vcpu_svm *svm)
 {
-        return emulate_instruction(&svm->vcpu, 0, 0, 0) == EMULATE_DONE;
+        return emulate_instruction(&svm->vcpu, 0) == EMULATE_DONE;
+}
+
+#define CR_VALID (1ULL << 63)
+
+static int cr_interception(struct vcpu_svm *svm)
+{
+        int reg, cr;
+        unsigned long val;
+        int err;
+
+        if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
+                return emulate_on_interception(svm);
+
+        if (unlikely((svm->vmcb->control.exit_info_1 & CR_VALID) == 0))
+                return emulate_on_interception(svm);
+
+        reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
+        cr = svm->vmcb->control.exit_code - SVM_EXIT_READ_CR0;
+
+        err = 0;
+        if (cr >= 16) { /* mov to cr */
+                cr -= 16;
+                val = kvm_register_read(&svm->vcpu, reg);
+                switch (cr) {
+                case 0:
+                        err = kvm_set_cr0(&svm->vcpu, val);
+                        break;
+                case 3:
+                        err = kvm_set_cr3(&svm->vcpu, val);
+                        break;
+                case 4:
+                        err = kvm_set_cr4(&svm->vcpu, val);
+                        break;
+                case 8:
+                        err = kvm_set_cr8(&svm->vcpu, val);
+                        break;
+                default:
+                        WARN(1, "unhandled write to CR%d", cr);
+                        kvm_queue_exception(&svm->vcpu, UD_VECTOR);
+                        return 1;
+                }
+        } else { /* mov from cr */
+                switch (cr) {
+                case 0:
+                        val = kvm_read_cr0(&svm->vcpu);
+                        break;
+                case 2:
+                        val = svm->vcpu.arch.cr2;
+                        break;
+                case 3:
+                        val = kvm_read_cr3(&svm->vcpu);
+                        break;
+                case 4:
+                        val = kvm_read_cr4(&svm->vcpu);
+                        break;
+                case 8:
+                        val = kvm_get_cr8(&svm->vcpu);
+                        break;
+                default:
+                        WARN(1, "unhandled read from CR%d", cr);
+                        kvm_queue_exception(&svm->vcpu, UD_VECTOR);
+                        return 1;
+                }
+                kvm_register_write(&svm->vcpu, reg, val);
+        }
+        kvm_complete_insn_gp(&svm->vcpu, err);
+
+        return 1;
 }
 
 static int cr0_write_interception(struct vcpu_svm *svm)
@@ -2527,7 +2744,7 @@ static int cr0_write_interception(struct vcpu_svm *svm)
         struct kvm_vcpu *vcpu = &svm->vcpu;
         int r;
 
-        r = emulate_instruction(&svm->vcpu, 0, 0, 0);
+        r = cr_interception(svm);
 
         if (svm->nested.vmexit_rip) {
                 kvm_register_write(vcpu, VCPU_REGS_RIP, svm->nested.vmexit_rip);
@@ -2536,22 +2753,47 @@ static int cr0_write_interception(struct vcpu_svm *svm)
                 svm->nested.vmexit_rip = 0;
         }
 
-        return r == EMULATE_DONE;
+        return r;
+}
+
+static int dr_interception(struct vcpu_svm *svm)
+{
+        int reg, dr;
+        unsigned long val;
+        int err;
+
+        if (!boot_cpu_has(X86_FEATURE_DECODEASSISTS))
+                return emulate_on_interception(svm);
+
+        reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
+        dr = svm->vmcb->control.exit_code - SVM_EXIT_READ_DR0;
+
+        if (dr >= 16) { /* mov to DRn */
+                val = kvm_register_read(&svm->vcpu, reg);
+                kvm_set_dr(&svm->vcpu, dr - 16, val);
+        } else {
+                err = kvm_get_dr(&svm->vcpu, dr, &val);
+                if (!err)
+                        kvm_register_write(&svm->vcpu, reg, val);
+        }
+
+        return 1;
 }
 
 static int cr8_write_interception(struct vcpu_svm *svm)
 {
         struct kvm_run *kvm_run = svm->vcpu.run;
+        int r;
 
         u8 cr8_prev = kvm_get_cr8(&svm->vcpu);
         /* instruction emulation calls kvm_set_cr8() */
-        emulate_instruction(&svm->vcpu, 0, 0, 0);
+        r = cr_interception(svm);
         if (irqchip_in_kernel(svm->vcpu.kvm)) {
-                svm->vmcb->control.intercept_cr_write &= ~INTERCEPT_CR8_MASK;
-                return 1;
+                clr_cr_intercept(svm, INTERCEPT_CR8_WRITE);
+                return r;
         }
         if (cr8_prev <= kvm_get_cr8(&svm->vcpu))
-                return 1;
+                return r;
         kvm_run->exit_reason = KVM_EXIT_SET_TPR;
         return 0;
 }
@@ -2562,14 +2804,9 @@ static int svm_get_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 *data)
 
         switch (ecx) {
         case MSR_IA32_TSC: {
-                u64 tsc_offset;
+                struct vmcb *vmcb = get_host_vmcb(svm);
 
-                if (is_nested(svm))
-                        tsc_offset = svm->nested.hsave->control.tsc_offset;
-                else
-                        tsc_offset = svm->vmcb->control.tsc_offset;
-
-                *data = tsc_offset + native_read_tsc();
+                *data = vmcb->control.tsc_offset + native_read_tsc();
                 break;
         }
         case MSR_STAR:
@@ -2714,7 +2951,7 @@ static int svm_set_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 data)
                 svm->vmcb->save.sysenter_esp = data;
                 break;
         case MSR_IA32_DEBUGCTLMSR:
-                if (!svm_has(SVM_FEATURE_LBRV)) {
+                if (!boot_cpu_has(X86_FEATURE_LBRV)) {
                         pr_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTL 0x%llx, nop\n",
                                         __func__, data);
                         break;
@@ -2723,6 +2960,7 @@ static int svm_set_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 data)
                         return 1;
 
                 svm->vmcb->save.dbgctl = data;
+                mark_dirty(svm->vmcb, VMCB_LBR);
                 if (data & (1ULL<<0))
                         svm_enable_lbrv(svm);
                 else
@@ -2775,6 +3013,7 @@ static int interrupt_window_interception(struct vcpu_svm *svm)
         kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
         svm_clear_vintr(svm);
         svm->vmcb->control.int_ctl &= ~V_IRQ_MASK;
+        mark_dirty(svm->vmcb, VMCB_INTR);
         /*
          * If the user space waits to inject interrupts, exit as soon as
          * possible
@@ -2797,31 +3036,31 @@ static int pause_interception(struct vcpu_svm *svm)
 }
 
 static int (*svm_exit_handlers[])(struct vcpu_svm *svm) = {
-        [SVM_EXIT_READ_CR0]                     = emulate_on_interception,
-        [SVM_EXIT_READ_CR3]                     = emulate_on_interception,
-        [SVM_EXIT_READ_CR4]                     = emulate_on_interception,
-        [SVM_EXIT_READ_CR8]                     = emulate_on_interception,
+        [SVM_EXIT_READ_CR0]                     = cr_interception,
+        [SVM_EXIT_READ_CR3]                     = cr_interception,
+        [SVM_EXIT_READ_CR4]                     = cr_interception,
+        [SVM_EXIT_READ_CR8]                     = cr_interception,
         [SVM_EXIT_CR0_SEL_WRITE]                = emulate_on_interception,
         [SVM_EXIT_WRITE_CR0]                    = cr0_write_interception,
-        [SVM_EXIT_WRITE_CR3]                    = emulate_on_interception,
-        [SVM_EXIT_WRITE_CR4]                    = emulate_on_interception,
+        [SVM_EXIT_WRITE_CR3]                    = cr_interception,
+        [SVM_EXIT_WRITE_CR4]                    = cr_interception,
         [SVM_EXIT_WRITE_CR8]                    = cr8_write_interception,
-        [SVM_EXIT_READ_DR0]                     = emulate_on_interception,
-        [SVM_EXIT_READ_DR1]                     = emulate_on_interception,
-        [SVM_EXIT_READ_DR2]                     = emulate_on_interception,
-        [SVM_EXIT_READ_DR3]                     = emulate_on_interception,
-        [SVM_EXIT_READ_DR4]                     = emulate_on_interception,
-        [SVM_EXIT_READ_DR5]                     = emulate_on_interception,
-        [SVM_EXIT_READ_DR6]                     = emulate_on_interception,
-        [SVM_EXIT_READ_DR7]                     = emulate_on_interception,
-        [SVM_EXIT_WRITE_DR0]                    = emulate_on_interception,
-        [SVM_EXIT_WRITE_DR1]                    = emulate_on_interception,
-        [SVM_EXIT_WRITE_DR2]                    = emulate_on_interception,
-        [SVM_EXIT_WRITE_DR3]                    = emulate_on_interception,
-        [SVM_EXIT_WRITE_DR4]                    = emulate_on_interception,
-        [SVM_EXIT_WRITE_DR5]                    = emulate_on_interception,
-        [SVM_EXIT_WRITE_DR6]                    = emulate_on_interception,
-        [SVM_EXIT_WRITE_DR7]                    = emulate_on_interception,
+        [SVM_EXIT_READ_DR0]                     = dr_interception,
+        [SVM_EXIT_READ_DR1]                     = dr_interception,
+        [SVM_EXIT_READ_DR2]                     = dr_interception,
+        [SVM_EXIT_READ_DR3]                     = dr_interception,
+        [SVM_EXIT_READ_DR4]                     = dr_interception,
+        [SVM_EXIT_READ_DR5]                     = dr_interception,
+        [SVM_EXIT_READ_DR6]                     = dr_interception,
+        [SVM_EXIT_READ_DR7]                     = dr_interception,
+        [SVM_EXIT_WRITE_DR0]                    = dr_interception,
+        [SVM_EXIT_WRITE_DR1]                    = dr_interception,
+        [SVM_EXIT_WRITE_DR2]                    = dr_interception,
+        [SVM_EXIT_WRITE_DR3]                    = dr_interception,
+        [SVM_EXIT_WRITE_DR4]                    = dr_interception,
+        [SVM_EXIT_WRITE_DR5]                    = dr_interception,
+        [SVM_EXIT_WRITE_DR6]                    = dr_interception,
+        [SVM_EXIT_WRITE_DR7]                    = dr_interception,
         [SVM_EXIT_EXCP_BASE + DB_VECTOR]        = db_interception,
         [SVM_EXIT_EXCP_BASE + BP_VECTOR]        = bp_interception,
         [SVM_EXIT_EXCP_BASE + UD_VECTOR]        = ud_interception,
@@ -2854,6 +3093,7 @@ static int (*svm_exit_handlers[])(struct vcpu_svm *svm) = {
         [SVM_EXIT_WBINVD]                       = emulate_on_interception,
         [SVM_EXIT_MONITOR]                      = invalid_op_interception,
         [SVM_EXIT_MWAIT]                        = invalid_op_interception,
+        [SVM_EXIT_XSETBV]                       = xsetbv_interception,
         [SVM_EXIT_NPF]                          = pf_interception,
 };
 
@@ -2864,10 +3104,10 @@ void dump_vmcb(struct kvm_vcpu *vcpu)
         struct vmcb_save_area *save = &svm->vmcb->save;
 
         pr_err("VMCB Control Area:\n");
-        pr_err("cr_read:            %04x\n", control->intercept_cr_read);
-        pr_err("cr_write:           %04x\n", control->intercept_cr_write);
-        pr_err("dr_read:            %04x\n", control->intercept_dr_read);
-        pr_err("dr_write:           %04x\n", control->intercept_dr_write);
+        pr_err("cr_read:            %04x\n", control->intercept_cr & 0xffff);
+        pr_err("cr_write:           %04x\n", control->intercept_cr >> 16);
+        pr_err("dr_read:            %04x\n", control->intercept_dr & 0xffff);
+        pr_err("dr_write:           %04x\n", control->intercept_dr >> 16);
         pr_err("exceptions:         %08x\n", control->intercept_exceptions);
         pr_err("intercepts:         %016llx\n", control->intercept);
         pr_err("pause filter count: %d\n", control->pause_filter_count);
@@ -2950,15 +3190,23 @@ void dump_vmcb(struct kvm_vcpu *vcpu)
 
 }
 
+static void svm_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2)
+{
+        struct vmcb_control_area *control = &to_svm(vcpu)->vmcb->control;
+
+        *info1 = control->exit_info_1;
+        *info2 = control->exit_info_2;
+}
+
 static int handle_exit(struct kvm_vcpu *vcpu)
 {
         struct vcpu_svm *svm = to_svm(vcpu);
         struct kvm_run *kvm_run = vcpu->run;
         u32 exit_code = svm->vmcb->control.exit_code;
 
-        trace_kvm_exit(exit_code, vcpu);
+        trace_kvm_exit(exit_code, vcpu, KVM_ISA_SVM);
 
-        if (!(svm->vmcb->control.intercept_cr_write & INTERCEPT_CR0_MASK))
+        if (!is_cr_intercept(svm, INTERCEPT_CR0_WRITE))
                 vcpu->arch.cr0 = svm->vmcb->save.cr0;
         if (npt_enabled)
                 vcpu->arch.cr3 = svm->vmcb->save.cr3;
@@ -2970,7 +3218,7 @@ static int handle_exit(struct kvm_vcpu *vcpu)
                 return 1;
         }
 
-        if (is_nested(svm)) {
+        if (is_guest_mode(vcpu)) {
                 int vmexit;
 
                 trace_kvm_nested_vmexit(svm->vmcb->save.rip, exit_code,
@@ -3033,7 +3281,6 @@ static void pre_svm_run(struct vcpu_svm *svm)
 
         struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
 
-        svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING;
         /* FIXME: handle wraparound of asid_generation */
         if (svm->asid_generation != sd->asid_generation)
                 new_asid(svm, sd);
@@ -3045,7 +3292,7 @@ static void svm_inject_nmi(struct kvm_vcpu *vcpu)
 
         svm->vmcb->control.event_inj = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI;
         vcpu->arch.hflags |= HF_NMI_MASK;
-        svm->vmcb->control.intercept |= (1ULL << INTERCEPT_IRET);
+        set_intercept(svm, INTERCEPT_IRET);
         ++vcpu->stat.nmi_injections;
 }
 
@@ -3058,6 +3305,7 @@ static inline void svm_inject_irq(struct vcpu_svm *svm, int irq)
         control->int_ctl &= ~V_INTR_PRIO_MASK;
         control->int_ctl |= V_IRQ_MASK |
                 ((/*control->int_vector >> 4*/ 0xf) << V_INTR_PRIO_SHIFT);
+        mark_dirty(svm->vmcb, VMCB_INTR);
 }
 
 static void svm_set_irq(struct kvm_vcpu *vcpu)
@@ -3077,14 +3325,14 @@ static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
 {
         struct vcpu_svm *svm = to_svm(vcpu);
 
-        if (is_nested(svm) && (vcpu->arch.hflags & HF_VINTR_MASK))
+        if (is_guest_mode(vcpu) && (vcpu->arch.hflags & HF_VINTR_MASK))
                 return;
 
         if (irr == -1)
                 return;
 
         if (tpr >= irr)
-                svm->vmcb->control.intercept_cr_write |= INTERCEPT_CR8_MASK;
+                set_cr_intercept(svm, INTERCEPT_CR8_WRITE);
 }
 
 static int svm_nmi_allowed(struct kvm_vcpu *vcpu)
@@ -3112,10 +3360,10 @@ static void svm_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked)
 
         if (masked) {
                 svm->vcpu.arch.hflags |= HF_NMI_MASK;
-                svm->vmcb->control.intercept |= (1ULL << INTERCEPT_IRET);
+                set_intercept(svm, INTERCEPT_IRET);
         } else {
                 svm->vcpu.arch.hflags &= ~HF_NMI_MASK;
-                svm->vmcb->control.intercept &= ~(1ULL << INTERCEPT_IRET);
+                clr_intercept(svm, INTERCEPT_IRET);
         }
 }
 
@@ -3131,7 +3379,7 @@ static int svm_interrupt_allowed(struct kvm_vcpu *vcpu)
 
         ret = !!(vmcb->save.rflags & X86_EFLAGS_IF);
 
-        if (is_nested(svm))
+        if (is_guest_mode(vcpu))
                 return ret && !(svm->vcpu.arch.hflags & HF_VINTR_MASK);
 
         return ret;
@@ -3177,7 +3425,12 @@ static int svm_set_tss_addr(struct kvm *kvm, unsigned int addr)
 
 static void svm_flush_tlb(struct kvm_vcpu *vcpu)
 {
-        force_new_asid(vcpu);
+        struct vcpu_svm *svm = to_svm(vcpu);
+
+        if (static_cpu_has(X86_FEATURE_FLUSHBYASID))
+                svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID;
+        else
+                svm->asid_generation--;
 }
 
 static void svm_prepare_guest_switch(struct kvm_vcpu *vcpu)
@@ -3188,10 +3441,10 @@ static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu)
 {
         struct vcpu_svm *svm = to_svm(vcpu);
 
-        if (is_nested(svm) && (vcpu->arch.hflags & HF_VINTR_MASK))
+        if (is_guest_mode(vcpu) && (vcpu->arch.hflags & HF_VINTR_MASK))
                 return;
 
-        if (!(svm->vmcb->control.intercept_cr_write & INTERCEPT_CR8_MASK)) {
+        if (!is_cr_intercept(svm, INTERCEPT_CR8_WRITE)) {
                 int cr8 = svm->vmcb->control.int_ctl & V_TPR_MASK;
                 kvm_set_cr8(vcpu, cr8);
         }
@@ -3202,7 +3455,7 @@ static inline void sync_lapic_to_cr8(struct kvm_vcpu *vcpu)
         struct vcpu_svm *svm = to_svm(vcpu);
         u64 cr8;
 
-        if (is_nested(svm) && (vcpu->arch.hflags & HF_VINTR_MASK))
+        if (is_guest_mode(vcpu) && (vcpu->arch.hflags & HF_VINTR_MASK))
                 return;
 
         cr8 = kvm_get_cr8(vcpu);
@@ -3289,9 +3542,6 @@ static void svm_cancel_injection(struct kvm_vcpu *vcpu)
 static void svm_vcpu_run(struct kvm_vcpu *vcpu)
 {
         struct vcpu_svm *svm = to_svm(vcpu);
-        u16 fs_selector;
-        u16 gs_selector;
-        u16 ldt_selector;
 
         svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
         svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
@@ -3308,10 +3558,6 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu)
 
         sync_lapic_to_cr8(vcpu);
 
-        save_host_msrs(vcpu);
-        savesegment(fs, fs_selector);
-        savesegment(gs, gs_selector);
-        ldt_selector = kvm_read_ldt();
         svm->vmcb->save.cr2 = vcpu->arch.cr2;
 
         clgi();
@@ -3389,19 +3635,10 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu)
 #endif
                 );
 
-        vcpu->arch.cr2 = svm->vmcb->save.cr2;
-        vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax;
-        vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp;
-        vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip;
-
-        load_host_msrs(vcpu);
-        kvm_load_ldt(ldt_selector);
-        loadsegment(fs, fs_selector);
 #ifdef CONFIG_X86_64
-        load_gs_index(gs_selector);
-        wrmsrl(MSR_KERNEL_GS_BASE, current->thread.gs);
+        wrmsrl(MSR_GS_BASE, svm->host.gs_base);
 #else
-        loadsegment(gs, gs_selector);
+        loadsegment(fs, svm->host.fs);
 #endif
 
         reload_tss(vcpu);
@@ -3410,10 +3647,21 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu)
 
         stgi();
 
+        vcpu->arch.cr2 = svm->vmcb->save.cr2;
+        vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax;
+        vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp;
+        vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip;
+
         sync_cr8_to_lapic(vcpu);
 
         svm->next_rip = 0;
 
+        svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING;
+
+        /* if exit due to PF check for async PF */
+        if (svm->vmcb->control.exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR)
+                svm->apf_reason = kvm_read_and_reset_pf_reason();
+
         if (npt_enabled) {
                 vcpu->arch.regs_avail &= ~(1 << VCPU_EXREG_PDPTR);
                 vcpu->arch.regs_dirty &= ~(1 << VCPU_EXREG_PDPTR);
@@ -3426,6 +3674,8 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu)
         if (unlikely(svm->vmcb->control.exit_code ==
                      SVM_EXIT_EXCP_BASE + MC_VECTOR))
                 svm_handle_mce(svm);
+
+        mark_all_clean(svm->vmcb);
 }
 
 #undef R
@@ -3435,7 +3685,8 @@ static void svm_set_cr3(struct kvm_vcpu *vcpu, unsigned long root)
         struct vcpu_svm *svm = to_svm(vcpu);
 
         svm->vmcb->save.cr3 = root;
-        force_new_asid(vcpu);
+        mark_dirty(svm->vmcb, VMCB_CR);
+        svm_flush_tlb(vcpu);
 }
 
 static void set_tdp_cr3(struct kvm_vcpu *vcpu, unsigned long root)
@@ -3443,11 +3694,13 @@ static void set_tdp_cr3(struct kvm_vcpu *vcpu, unsigned long root)
         struct vcpu_svm *svm = to_svm(vcpu);
 
         svm->vmcb->control.nested_cr3 = root;
+        mark_dirty(svm->vmcb, VMCB_NPT);
 
         /* Also sync guest cr3 here in case we live migrate */
-        svm->vmcb->save.cr3 = vcpu->arch.cr3;
+        svm->vmcb->save.cr3 = kvm_read_cr3(vcpu);
+        mark_dirty(svm->vmcb, VMCB_CR);
 
-        force_new_asid(vcpu);
+        svm_flush_tlb(vcpu);
 }
 
 static int is_disabled(void)
@@ -3494,10 +3747,6 @@ static void svm_cpuid_update(struct kvm_vcpu *vcpu)
 static void svm_set_supported_cpuid(u32 func, struct kvm_cpuid_entry2 *entry)
 {
         switch (func) {
-        case 0x00000001:
-                /* Mask out xsave bit as long as it is not supported by SVM */
-                entry->ecx &= ~(bit(X86_FEATURE_XSAVE));
-                break;
         case 0x80000001:
                 if (nested)
                         entry->ecx |= (1 << 2); /* Set SVM bit */
@@ -3511,7 +3760,7 @@ static void svm_set_supported_cpuid(u32 func, struct kvm_cpuid_entry2 *entry)
                                    additional features */
 
                 /* Support next_rip if host supports it */
-                if (svm_has(SVM_FEATURE_NRIP))
+                if (boot_cpu_has(X86_FEATURE_NRIPS))
                         entry->edx |= SVM_FEATURE_NRIP;
 
                 /* Support NPT for the guest if enabled */
@@ -3571,6 +3820,7 @@ static const struct trace_print_flags svm_exit_reasons_str[] = {
         { SVM_EXIT_WBINVD,                      "wbinvd" },
         { SVM_EXIT_MONITOR,                     "monitor" },
         { SVM_EXIT_MWAIT,                       "mwait" },
+        { SVM_EXIT_XSETBV,                      "xsetbv" },
         { SVM_EXIT_NPF,                         "npf" },
         { -1, NULL }
 };
@@ -3594,9 +3844,7 @@ static void svm_fpu_deactivate(struct kvm_vcpu *vcpu)
 {
         struct vcpu_svm *svm = to_svm(vcpu);
 
-        svm->vmcb->control.intercept_exceptions |= 1 << NM_VECTOR;
-        if (is_nested(svm))
-                svm->nested.hsave->control.intercept_exceptions |= 1 << NM_VECTOR;
+        set_exception_intercept(svm, NM_VECTOR);
         update_cr0_intercept(svm);
 }
 
@@ -3627,6 +3875,7 @@ static struct kvm_x86_ops svm_x86_ops = {
         .get_cpl = svm_get_cpl,
         .get_cs_db_l_bits = kvm_get_cs_db_l_bits,
         .decache_cr0_guest_bits = svm_decache_cr0_guest_bits,
+        .decache_cr3 = svm_decache_cr3,
         .decache_cr4_guest_bits = svm_decache_cr4_guest_bits,
         .set_cr0 = svm_set_cr0,
         .set_cr3 = svm_set_cr3,
@@ -3667,7 +3916,9 @@ static struct kvm_x86_ops svm_x86_ops = {
         .get_tdp_level = get_npt_level,
         .get_mt_mask = svm_get_mt_mask,
 
+        .get_exit_info = svm_get_exit_info,
         .exit_reasons_str = svm_exit_reasons_str,
+
         .get_lpage_level = svm_get_lpage_level,
 
         .cpuid_update = svm_cpuid_update,
diff --git a/arch/x86/kvm/trace.h b/arch/x86/kvm/trace.h
index a6544b8e7c0f..1357d7cf4ec8 100644
--- a/arch/x86/kvm/trace.h
+++ b/arch/x86/kvm/trace.h
@@ -178,27 +178,36 @@ TRACE_EVENT(kvm_apic,
 #define trace_kvm_apic_read(reg, val)           trace_kvm_apic(0, reg, val)
 #define trace_kvm_apic_write(reg, val)          trace_kvm_apic(1, reg, val)
 
+#define KVM_ISA_VMX   1
+#define KVM_ISA_SVM   2
+
 /*
  * Tracepoint for kvm guest exit:
  */
 TRACE_EVENT(kvm_exit,
-        TP_PROTO(unsigned int exit_reason, struct kvm_vcpu *vcpu),
-        TP_ARGS(exit_reason, vcpu),
+        TP_PROTO(unsigned int exit_reason, struct kvm_vcpu *vcpu, u32 isa),
+        TP_ARGS(exit_reason, vcpu, isa),
 
         TP_STRUCT__entry(
                 __field(        unsigned int,   exit_reason     )
                 __field(        unsigned long,  guest_rip       )
+                __field(        u32,            isa             )
+                __field(        u64,            info1           )
+                __field(        u64,            info2           )
         ),
 
         TP_fast_assign(
                 __entry->exit_reason    = exit_reason;
                 __entry->guest_rip      = kvm_rip_read(vcpu);
+                __entry->isa            = isa;
+                kvm_x86_ops->get_exit_info(vcpu, &__entry->info1,
+                                           &__entry->info2);
         ),
 
-        TP_printk("reason %s rip 0x%lx",
+        TP_printk("reason %s rip 0x%lx info %llx %llx",
                  ftrace_print_symbols_seq(p, __entry->exit_reason,
                                           kvm_x86_ops->exit_reasons_str),
-                 __entry->guest_rip)
+                 __entry->guest_rip, __entry->info1, __entry->info2)
 );
 
 /*
diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index 81fcbe9515c5..bf89ec2cfb82 100644
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -69,6 +69,9 @@ module_param(emulate_invalid_guest_state, bool, S_IRUGO);
 static int __read_mostly vmm_exclusive = 1;
 module_param(vmm_exclusive, bool, S_IRUGO);
 
+static int __read_mostly yield_on_hlt = 1;
+module_param(yield_on_hlt, bool, S_IRUGO);
+
 #define KVM_GUEST_CR0_MASK_UNRESTRICTED_GUEST                           \
         (X86_CR0_WP | X86_CR0_NE | X86_CR0_NW | X86_CR0_CD)
 #define KVM_GUEST_CR0_MASK                                              \
@@ -177,6 +180,7 @@ static int init_rmode(struct kvm *kvm);
 static u64 construct_eptp(unsigned long root_hpa);
 static void kvm_cpu_vmxon(u64 addr);
 static void kvm_cpu_vmxoff(void);
+static void vmx_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3);
 
 static DEFINE_PER_CPU(struct vmcs *, vmxarea);
 static DEFINE_PER_CPU(struct vmcs *, current_vmcs);
@@ -188,6 +192,8 @@ static unsigned long *vmx_io_bitmap_b;
 static unsigned long *vmx_msr_bitmap_legacy;
 static unsigned long *vmx_msr_bitmap_longmode;
 
+static bool cpu_has_load_ia32_efer;
+
 static DECLARE_BITMAP(vmx_vpid_bitmap, VMX_NR_VPIDS);
 static DEFINE_SPINLOCK(vmx_vpid_lock);
 
@@ -472,7 +478,7 @@ static void vmcs_clear(struct vmcs *vmcs)
         u8 error;
 
         asm volatile (__ex(ASM_VMX_VMCLEAR_RAX) "; setna %0"
-                      : "=g"(error) : "a"(&phys_addr), "m"(phys_addr)
+                      : "=qm"(error) : "a"(&phys_addr), "m"(phys_addr)
                       : "cc", "memory");
         if (error)
                 printk(KERN_ERR "kvm: vmclear fail: %p/%llx\n",
@@ -485,7 +491,7 @@ static void vmcs_load(struct vmcs *vmcs)
         u8 error;
 
         asm volatile (__ex(ASM_VMX_VMPTRLD_RAX) "; setna %0"
-                        : "=g"(error) : "a"(&phys_addr), "m"(phys_addr)
+                        : "=qm"(error) : "a"(&phys_addr), "m"(phys_addr)
                         : "cc", "memory");
         if (error)
                 printk(KERN_ERR "kvm: vmptrld %p/%llx fail\n",
@@ -565,10 +571,10 @@ static inline void ept_sync_individual_addr(u64 eptp, gpa_t gpa)
 
 static unsigned long vmcs_readl(unsigned long field)
 {
-        unsigned long value;
+        unsigned long value = 0;
 
         asm volatile (__ex(ASM_VMX_VMREAD_RDX_RAX)
-                      : "=a"(value) : "d"(field) : "cc");
+                      : "+a"(value) : "d"(field) : "cc");
         return value;
 }
 
@@ -661,6 +667,12 @@ static void clear_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr)
         unsigned i;
         struct msr_autoload *m = &vmx->msr_autoload;
 
+        if (msr == MSR_EFER && cpu_has_load_ia32_efer) {
+                vmcs_clear_bits(VM_ENTRY_CONTROLS, VM_ENTRY_LOAD_IA32_EFER);
+                vmcs_clear_bits(VM_EXIT_CONTROLS, VM_EXIT_LOAD_IA32_EFER);
+                return;
+        }
+
         for (i = 0; i < m->nr; ++i)
                 if (m->guest[i].index == msr)
                         break;
@@ -680,6 +692,14 @@ static void add_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr,
         unsigned i;
         struct msr_autoload *m = &vmx->msr_autoload;
 
+        if (msr == MSR_EFER && cpu_has_load_ia32_efer) {
+                vmcs_write64(GUEST_IA32_EFER, guest_val);
+                vmcs_write64(HOST_IA32_EFER, host_val);
+                vmcs_set_bits(VM_ENTRY_CONTROLS, VM_ENTRY_LOAD_IA32_EFER);
+                vmcs_set_bits(VM_EXIT_CONTROLS, VM_EXIT_LOAD_IA32_EFER);
+                return;
+        }
+
         for (i = 0; i < m->nr; ++i)
                 if (m->guest[i].index == msr)
                         break;
@@ -1009,6 +1029,17 @@ static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
         vmx_set_interrupt_shadow(vcpu, 0);
 }
 
+static void vmx_clear_hlt(struct kvm_vcpu *vcpu)
+{
+        /* Ensure that we clear the HLT state in the VMCS.  We don't need to
+         * explicitly skip the instruction because if the HLT state is set, then
+         * the instruction is already executing and RIP has already been
+         * advanced. */
+        if (!yield_on_hlt &&
+            vmcs_read32(GUEST_ACTIVITY_STATE) == GUEST_ACTIVITY_HLT)
+                vmcs_write32(GUEST_ACTIVITY_STATE, GUEST_ACTIVITY_ACTIVE);
+}
+
 static void vmx_queue_exception(struct kvm_vcpu *vcpu, unsigned nr,
                                 bool has_error_code, u32 error_code,
                                 bool reinject)
@@ -1035,6 +1066,7 @@ static void vmx_queue_exception(struct kvm_vcpu *vcpu, unsigned nr,
                 intr_info |= INTR_TYPE_HARD_EXCEPTION;
 
         vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr_info);
+        vmx_clear_hlt(vcpu);
 }
 
 static bool vmx_rdtscp_supported(void)
@@ -1305,8 +1337,11 @@ static __init int vmx_disabled_by_bios(void)
                         && tboot_enabled())
                         return 1;
                 if (!(msr & FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX)
-                        && !tboot_enabled())
+                        && !tboot_enabled()) {
+                        printk(KERN_WARNING "kvm: disable TXT in the BIOS or "
+                                " activate TXT before enabling KVM\n");
                         return 1;
+                }
         }
 
         return 0;
@@ -1400,6 +1435,14 @@ static __init int adjust_vmx_controls(u32 ctl_min, u32 ctl_opt,
         return 0;
 }
 
+static __init bool allow_1_setting(u32 msr, u32 ctl)
+{
+        u32 vmx_msr_low, vmx_msr_high;
+
+        rdmsr(msr, vmx_msr_low, vmx_msr_high);
+        return vmx_msr_high & ctl;
+}
+
 static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf)
 {
         u32 vmx_msr_low, vmx_msr_high;
@@ -1416,7 +1459,7 @@ static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf)
                                 &_pin_based_exec_control) < 0)
                 return -EIO;
 
-        min = CPU_BASED_HLT_EXITING |
+        min =
 #ifdef CONFIG_X86_64
               CPU_BASED_CR8_LOAD_EXITING |
               CPU_BASED_CR8_STORE_EXITING |
@@ -1429,6 +1472,10 @@ static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf)
               CPU_BASED_MWAIT_EXITING |
               CPU_BASED_MONITOR_EXITING |
               CPU_BASED_INVLPG_EXITING;
+
+        if (yield_on_hlt)
+                min |= CPU_BASED_HLT_EXITING;
+
         opt = CPU_BASED_TPR_SHADOW |
               CPU_BASED_USE_MSR_BITMAPS |
               CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
@@ -1510,6 +1557,12 @@ static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf)
         vmcs_conf->vmexit_ctrl         = _vmexit_control;
         vmcs_conf->vmentry_ctrl        = _vmentry_control;
 
+        cpu_has_load_ia32_efer =
+                allow_1_setting(MSR_IA32_VMX_ENTRY_CTLS,
+                                VM_ENTRY_LOAD_IA32_EFER)
+                && allow_1_setting(MSR_IA32_VMX_EXIT_CTLS,
+                                   VM_EXIT_LOAD_IA32_EFER);
+
         return 0;
 }
 
@@ -1683,9 +1736,13 @@ static void fix_rmode_seg(int seg, struct kvm_save_segment *save)
         save->limit = vmcs_read32(sf->limit);
         save->ar = vmcs_read32(sf->ar_bytes);
         vmcs_write16(sf->selector, save->base >> 4);
-        vmcs_write32(sf->base, save->base & 0xfffff);
+        vmcs_write32(sf->base, save->base & 0xffff0);
         vmcs_write32(sf->limit, 0xffff);
         vmcs_write32(sf->ar_bytes, 0xf3);
+        if (save->base & 0xf)
+                printk_once(KERN_WARNING "kvm: segment base is not paragraph"
+                            " aligned when entering protected mode (seg=%d)",
+                            seg);
 }
 
 static void enter_rmode(struct kvm_vcpu *vcpu)
@@ -1814,6 +1871,13 @@ static void vmx_decache_cr0_guest_bits(struct kvm_vcpu *vcpu)
         vcpu->arch.cr0 |= vmcs_readl(GUEST_CR0) & cr0_guest_owned_bits;
 }
 
+static void vmx_decache_cr3(struct kvm_vcpu *vcpu)
+{
+        if (enable_ept && is_paging(vcpu))
+                vcpu->arch.cr3 = vmcs_readl(GUEST_CR3);
+        __set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail);
+}
+
 static void vmx_decache_cr4_guest_bits(struct kvm_vcpu *vcpu)
 {
         ulong cr4_guest_owned_bits = vcpu->arch.cr4_guest_owned_bits;
@@ -1857,6 +1921,7 @@ static void ept_update_paging_mode_cr0(unsigned long *hw_cr0,
                                         unsigned long cr0,
                                         struct kvm_vcpu *vcpu)
 {
+        vmx_decache_cr3(vcpu);
         if (!(cr0 & X86_CR0_PG)) {
                 /* From paging/starting to nonpaging */
                 vmcs_write32(CPU_BASED_VM_EXEC_CONTROL,
@@ -1937,7 +2002,7 @@ static void vmx_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
         if (enable_ept) {
                 eptp = construct_eptp(cr3);
                 vmcs_write64(EPT_POINTER, eptp);
-                guest_cr3 = is_paging(vcpu) ? vcpu->arch.cr3 :
+                guest_cr3 = is_paging(vcpu) ? kvm_read_cr3(vcpu) :
                         vcpu->kvm->arch.ept_identity_map_addr;
                 ept_load_pdptrs(vcpu);
         }
@@ -2725,7 +2790,7 @@ static int vmx_vcpu_reset(struct kvm_vcpu *vcpu)
         vmcs_writel(GUEST_IDTR_BASE, 0);
         vmcs_write32(GUEST_IDTR_LIMIT, 0xffff);
 
-        vmcs_write32(GUEST_ACTIVITY_STATE, 0);
+        vmcs_write32(GUEST_ACTIVITY_STATE, GUEST_ACTIVITY_ACTIVE);
         vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 0);
         vmcs_write32(GUEST_PENDING_DBG_EXCEPTIONS, 0);
 
@@ -2787,6 +2852,10 @@ static void enable_nmi_window(struct kvm_vcpu *vcpu)
                 return;
         }
 
+        if (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & GUEST_INTR_STATE_STI) {
+                enable_irq_window(vcpu);
+                return;
+        }
         cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
         cpu_based_vm_exec_control |= CPU_BASED_VIRTUAL_NMI_PENDING;
         vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
@@ -2814,6 +2883,7 @@ static void vmx_inject_irq(struct kvm_vcpu *vcpu)
         } else
                 intr |= INTR_TYPE_EXT_INTR;
         vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr);
+        vmx_clear_hlt(vcpu);
 }
 
 static void vmx_inject_nmi(struct kvm_vcpu *vcpu)
@@ -2841,6 +2911,7 @@ static void vmx_inject_nmi(struct kvm_vcpu *vcpu)
         }
         vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
                         INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK | NMI_VECTOR);
+        vmx_clear_hlt(vcpu);
 }
 
 static int vmx_nmi_allowed(struct kvm_vcpu *vcpu)
@@ -2849,7 +2920,8 @@ static int vmx_nmi_allowed(struct kvm_vcpu *vcpu)
                 return 0;
 
         return  !(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) &
-                        (GUEST_INTR_STATE_MOV_SS | GUEST_INTR_STATE_NMI));
+                  (GUEST_INTR_STATE_MOV_SS | GUEST_INTR_STATE_STI
+                   | GUEST_INTR_STATE_NMI));
 }
 
 static bool vmx_get_nmi_mask(struct kvm_vcpu *vcpu)
@@ -2910,7 +2982,7 @@ static int handle_rmode_exception(struct kvm_vcpu *vcpu,
          * Cause the #SS fault with 0 error code in VM86 mode.
          */
         if (((vec == GP_VECTOR) || (vec == SS_VECTOR)) && err_code == 0)
-                if (emulate_instruction(vcpu, 0, 0, 0) == EMULATE_DONE)
+                if (emulate_instruction(vcpu, 0) == EMULATE_DONE)
                         return 1;
         /*
          * Forward all other exceptions that are valid in real mode.
@@ -3007,7 +3079,7 @@ static int handle_exception(struct kvm_vcpu *vcpu)
         }
 
         if (is_invalid_opcode(intr_info)) {
-                er = emulate_instruction(vcpu, 0, 0, EMULTYPE_TRAP_UD);
+                er = emulate_instruction(vcpu, EMULTYPE_TRAP_UD);
                 if (er != EMULATE_DONE)
                         kvm_queue_exception(vcpu, UD_VECTOR);
                 return 1;
@@ -3026,7 +3098,7 @@ static int handle_exception(struct kvm_vcpu *vcpu)
 
                 if (kvm_event_needs_reinjection(vcpu))
                         kvm_mmu_unprotect_page_virt(vcpu, cr2);
-                return kvm_mmu_page_fault(vcpu, cr2, error_code);
+                return kvm_mmu_page_fault(vcpu, cr2, error_code, NULL, 0);
         }
 
         if (vmx->rmode.vm86_active &&
@@ -3098,7 +3170,7 @@ static int handle_io(struct kvm_vcpu *vcpu)
         ++vcpu->stat.io_exits;
 
         if (string || in)
-                return emulate_instruction(vcpu, 0, 0, 0) == EMULATE_DONE;
+                return emulate_instruction(vcpu, 0) == EMULATE_DONE;
 
         port = exit_qualification >> 16;
         size = (exit_qualification & 7) + 1;
@@ -3118,14 +3190,6 @@ vmx_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
         hypercall[2] = 0xc1;
 }
 
-static void complete_insn_gp(struct kvm_vcpu *vcpu, int err)
-{
-        if (err)
-                kvm_inject_gp(vcpu, 0);
-        else
-                skip_emulated_instruction(vcpu);
-}
-
 static int handle_cr(struct kvm_vcpu *vcpu)
 {
         unsigned long exit_qualification, val;
@@ -3143,21 +3207,21 @@ static int handle_cr(struct kvm_vcpu *vcpu)
                 switch (cr) {
                 case 0:
                         err = kvm_set_cr0(vcpu, val);
-                        complete_insn_gp(vcpu, err);
+                        kvm_complete_insn_gp(vcpu, err);
                         return 1;
                 case 3:
                         err = kvm_set_cr3(vcpu, val);
-                        complete_insn_gp(vcpu, err);
+                        kvm_complete_insn_gp(vcpu, err);
                         return 1;
                 case 4:
                         err = kvm_set_cr4(vcpu, val);
-                        complete_insn_gp(vcpu, err);
+                        kvm_complete_insn_gp(vcpu, err);
                         return 1;
                 case 8: {
                                 u8 cr8_prev = kvm_get_cr8(vcpu);
                                 u8 cr8 = kvm_register_read(vcpu, reg);
-                                kvm_set_cr8(vcpu, cr8);
-                                skip_emulated_instruction(vcpu);
+                                err = kvm_set_cr8(vcpu, cr8);
+                                kvm_complete_insn_gp(vcpu, err);
                                 if (irqchip_in_kernel(vcpu->kvm))
                                         return 1;
                                 if (cr8_prev <= cr8)
@@ -3176,8 +3240,9 @@ static int handle_cr(struct kvm_vcpu *vcpu)
         case 1: /*mov from cr*/
                 switch (cr) {
                 case 3:
-                        kvm_register_write(vcpu, reg, vcpu->arch.cr3);
-                        trace_kvm_cr_read(cr, vcpu->arch.cr3);
+                        val = kvm_read_cr3(vcpu);
+                        kvm_register_write(vcpu, reg, val);
+                        trace_kvm_cr_read(cr, val);
                         skip_emulated_instruction(vcpu);
                         return 1;
                 case 8:
@@ -3349,6 +3414,11 @@ static int handle_vmx_insn(struct kvm_vcpu *vcpu)
         return 1;
 }
 
+static int handle_invd(struct kvm_vcpu *vcpu)
+{
+        return emulate_instruction(vcpu, 0) == EMULATE_DONE;
+}
+
 static int handle_invlpg(struct kvm_vcpu *vcpu)
 {
         unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
@@ -3377,7 +3447,7 @@ static int handle_xsetbv(struct kvm_vcpu *vcpu)
 
 static int handle_apic_access(struct kvm_vcpu *vcpu)
 {
-        return emulate_instruction(vcpu, 0, 0, 0) == EMULATE_DONE;
+        return emulate_instruction(vcpu, 0) == EMULATE_DONE;
 }
 
 static int handle_task_switch(struct kvm_vcpu *vcpu)
@@ -3476,7 +3546,7 @@ static int handle_ept_violation(struct kvm_vcpu *vcpu)
 
         gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS);
         trace_kvm_page_fault(gpa, exit_qualification);
-        return kvm_mmu_page_fault(vcpu, gpa & PAGE_MASK, 0);
+        return kvm_mmu_page_fault(vcpu, gpa, exit_qualification & 0x3, NULL, 0);
 }
 
 static u64 ept_rsvd_mask(u64 spte, int level)
@@ -3592,7 +3662,7 @@ static int handle_invalid_guest_state(struct kvm_vcpu *vcpu)
                     && (kvm_get_rflags(&vmx->vcpu) & X86_EFLAGS_IF))
                         return handle_interrupt_window(&vmx->vcpu);
 
-                err = emulate_instruction(vcpu, 0, 0, 0);
+                err = emulate_instruction(vcpu, 0);
 
                 if (err == EMULATE_DO_MMIO) {
                         ret = 0;
@@ -3649,6 +3719,7 @@ static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = {
         [EXIT_REASON_MSR_WRITE]               = handle_wrmsr,
         [EXIT_REASON_PENDING_INTERRUPT]       = handle_interrupt_window,
         [EXIT_REASON_HLT]                     = handle_halt,
+        [EXIT_REASON_INVD]                    = handle_invd,
         [EXIT_REASON_INVLPG]                  = handle_invlpg,
         [EXIT_REASON_VMCALL]                  = handle_vmcall,
         [EXIT_REASON_VMCLEAR]                 = handle_vmx_insn,
@@ -3676,6 +3747,12 @@ static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = {
 static const int kvm_vmx_max_exit_handlers =
         ARRAY_SIZE(kvm_vmx_exit_handlers);
 
+static void vmx_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2)
+{
+        *info1 = vmcs_readl(EXIT_QUALIFICATION);
+        *info2 = vmcs_read32(VM_EXIT_INTR_INFO);
+}
+
 /*
  * The guest has exited.  See if we can fix it or if we need userspace
  * assistance.
@@ -3686,17 +3763,12 @@ static int vmx_handle_exit(struct kvm_vcpu *vcpu)
         u32 exit_reason = vmx->exit_reason;
         u32 vectoring_info = vmx->idt_vectoring_info;
 
-        trace_kvm_exit(exit_reason, vcpu);
+        trace_kvm_exit(exit_reason, vcpu, KVM_ISA_VMX);
 
         /* If guest state is invalid, start emulating */
         if (vmx->emulation_required && emulate_invalid_guest_state)
                 return handle_invalid_guest_state(vcpu);
 
-        /* Access CR3 don't cause VMExit in paging mode, so we need
-         * to sync with guest real CR3. */
-        if (enable_ept && is_paging(vcpu))
-                vcpu->arch.cr3 = vmcs_readl(GUEST_CR3);
-
         if (exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY) {
                 vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY;
                 vcpu->run->fail_entry.hardware_entry_failure_reason
@@ -4013,7 +4085,8 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu)
               );
 
         vcpu->arch.regs_avail = ~((1 << VCPU_REGS_RIP) | (1 << VCPU_REGS_RSP)
-                                  | (1 << VCPU_EXREG_PDPTR));
+                                  | (1 << VCPU_EXREG_PDPTR)
+                                  | (1 << VCPU_EXREG_CR3));
         vcpu->arch.regs_dirty = 0;
 
         vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD);
@@ -4280,6 +4353,7 @@ static struct kvm_x86_ops vmx_x86_ops = {
         .get_cpl = vmx_get_cpl,
         .get_cs_db_l_bits = vmx_get_cs_db_l_bits,
         .decache_cr0_guest_bits = vmx_decache_cr0_guest_bits,
+        .decache_cr3 = vmx_decache_cr3,
         .decache_cr4_guest_bits = vmx_decache_cr4_guest_bits,
         .set_cr0 = vmx_set_cr0,
         .set_cr3 = vmx_set_cr3,
@@ -4320,7 +4394,9 @@ static struct kvm_x86_ops vmx_x86_ops = {
         .get_tdp_level = get_ept_level,
         .get_mt_mask = vmx_get_mt_mask,
 
+        .get_exit_info = vmx_get_exit_info,
         .exit_reasons_str = vmx_exit_reasons_str,
+
         .get_lpage_level = vmx_get_lpage_level,
 
         .cpuid_update = vmx_cpuid_update,
@@ -4396,8 +4472,6 @@ static int __init vmx_init(void)
 
         if (enable_ept) {
                 bypass_guest_pf = 0;
-                kvm_mmu_set_base_ptes(VMX_EPT_READABLE_MASK |
-                        VMX_EPT_WRITABLE_MASK);
                 kvm_mmu_set_mask_ptes(0ull, 0ull, 0ull, 0ull,
                                 VMX_EPT_EXECUTABLE_MASK);
                 kvm_enable_tdp();
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 46a368cb651e..bcc0efce85bf 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -43,6 +43,7 @@
 #include <linux/slab.h>
 #include <linux/perf_event.h>
 #include <linux/uaccess.h>
+#include <linux/hash.h>
 #include <trace/events/kvm.h>
 
 #define CREATE_TRACE_POINTS
@@ -155,6 +156,13 @@ struct kvm_stats_debugfs_item debugfs_entries[] = {
 
 u64 __read_mostly host_xcr0;
 
+static inline void kvm_async_pf_hash_reset(struct kvm_vcpu *vcpu)
+{
+        int i;
+        for (i = 0; i < roundup_pow_of_two(ASYNC_PF_PER_VCPU); i++)
+                vcpu->arch.apf.gfns[i] = ~0;
+}
+
 static void kvm_on_user_return(struct user_return_notifier *urn)
 {
         unsigned slot;
@@ -326,23 +334,28 @@ void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr)
 }
 EXPORT_SYMBOL_GPL(kvm_requeue_exception);
 
-void kvm_inject_page_fault(struct kvm_vcpu *vcpu)
+void kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err)
 {
-        unsigned error_code = vcpu->arch.fault.error_code;
+        if (err)
+                kvm_inject_gp(vcpu, 0);
+        else
+                kvm_x86_ops->skip_emulated_instruction(vcpu);
+}
+EXPORT_SYMBOL_GPL(kvm_complete_insn_gp);
 
+void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault)
+{
         ++vcpu->stat.pf_guest;
-        vcpu->arch.cr2 = vcpu->arch.fault.address;
-        kvm_queue_exception_e(vcpu, PF_VECTOR, error_code);
+        vcpu->arch.cr2 = fault->address;
+        kvm_queue_exception_e(vcpu, PF_VECTOR, fault->error_code);
 }
 
-void kvm_propagate_fault(struct kvm_vcpu *vcpu)
+void kvm_propagate_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault)
 {
-        if (mmu_is_nested(vcpu) && !vcpu->arch.fault.nested)
-                vcpu->arch.nested_mmu.inject_page_fault(vcpu);
+        if (mmu_is_nested(vcpu) && !fault->nested_page_fault)
+                vcpu->arch.nested_mmu.inject_page_fault(vcpu, fault);
         else
-                vcpu->arch.mmu.inject_page_fault(vcpu);
-
-        vcpu->arch.fault.nested = false;
+                vcpu->arch.mmu.inject_page_fault(vcpu, fault);
 }
 
 void kvm_inject_nmi(struct kvm_vcpu *vcpu)
@@ -460,8 +473,8 @@ static bool pdptrs_changed(struct kvm_vcpu *vcpu)
                       (unsigned long *)&vcpu->arch.regs_avail))
                 return true;
 
-        gfn = (vcpu->arch.cr3 & ~31u) >> PAGE_SHIFT;
-        offset = (vcpu->arch.cr3 & ~31u) & (PAGE_SIZE - 1);
+        gfn = (kvm_read_cr3(vcpu) & ~31u) >> PAGE_SHIFT;
+        offset = (kvm_read_cr3(vcpu) & ~31u) & (PAGE_SIZE - 1);
         r = kvm_read_nested_guest_page(vcpu, gfn, pdpte, offset, sizeof(pdpte),
                                        PFERR_USER_MASK | PFERR_WRITE_MASK);
         if (r < 0)
@@ -506,12 +519,15 @@ int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
                 } else
 #endif
                 if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->arch.walk_mmu,
-                                                 vcpu->arch.cr3))
+                                                 kvm_read_cr3(vcpu)))
                         return 1;
         }
 
         kvm_x86_ops->set_cr0(vcpu, cr0);
 
+        if ((cr0 ^ old_cr0) & X86_CR0_PG)
+                kvm_clear_async_pf_completion_queue(vcpu);
+
         if ((cr0 ^ old_cr0) & update_bits)
                 kvm_mmu_reset_context(vcpu);
         return 0;
@@ -595,7 +611,8 @@ int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
                         return 1;
         } else if (is_paging(vcpu) && (cr4 & X86_CR4_PAE)
                    && ((cr4 ^ old_cr4) & pdptr_bits)
-                   && !load_pdptrs(vcpu, vcpu->arch.walk_mmu, vcpu->arch.cr3))
+                   && !load_pdptrs(vcpu, vcpu->arch.walk_mmu,
+                                   kvm_read_cr3(vcpu)))
                 return 1;
 
         if (cr4 & X86_CR4_VMXE)
@@ -615,7 +632,7 @@ EXPORT_SYMBOL_GPL(kvm_set_cr4);
 
 int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
 {
-        if (cr3 == vcpu->arch.cr3 && !pdptrs_changed(vcpu)) {
+        if (cr3 == kvm_read_cr3(vcpu) && !pdptrs_changed(vcpu)) {
                 kvm_mmu_sync_roots(vcpu);
                 kvm_mmu_flush_tlb(vcpu);
                 return 0;
@@ -650,12 +667,13 @@ int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
         if (unlikely(!gfn_to_memslot(vcpu->kvm, cr3 >> PAGE_SHIFT)))
                 return 1;
         vcpu->arch.cr3 = cr3;
+        __set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail);
         vcpu->arch.mmu.new_cr3(vcpu);
         return 0;
 }
 EXPORT_SYMBOL_GPL(kvm_set_cr3);
 
-int __kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
+int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
 {
         if (cr8 & CR8_RESERVED_BITS)
                 return 1;
@@ -665,12 +683,6 @@ int __kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
                 vcpu->arch.cr8 = cr8;
         return 0;
 }
-
-void kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
-{
-        if (__kvm_set_cr8(vcpu, cr8))
-                kvm_inject_gp(vcpu, 0);
-}
 EXPORT_SYMBOL_GPL(kvm_set_cr8);
 
 unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu)
@@ -775,12 +787,12 @@ EXPORT_SYMBOL_GPL(kvm_get_dr);
  * kvm-specific. Those are put in the beginning of the list.
  */
 
-#define KVM_SAVE_MSRS_BEGIN     7
+#define KVM_SAVE_MSRS_BEGIN     8
 static u32 msrs_to_save[] = {
         MSR_KVM_SYSTEM_TIME, MSR_KVM_WALL_CLOCK,
         MSR_KVM_SYSTEM_TIME_NEW, MSR_KVM_WALL_CLOCK_NEW,
         HV_X64_MSR_GUEST_OS_ID, HV_X64_MSR_HYPERCALL,
-        HV_X64_MSR_APIC_ASSIST_PAGE,
+        HV_X64_MSR_APIC_ASSIST_PAGE, MSR_KVM_ASYNC_PF_EN,
         MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
         MSR_STAR,
 #ifdef CONFIG_X86_64
@@ -830,7 +842,6 @@ static int set_efer(struct kvm_vcpu *vcpu, u64 efer)
         kvm_x86_ops->set_efer(vcpu, efer);
 
         vcpu->arch.mmu.base_role.nxe = (efer & EFER_NX) && !tdp_enabled;
-        kvm_mmu_reset_context(vcpu);
 
         /* Update reserved bits */
         if ((efer ^ old_efer) & EFER_NX)
@@ -1418,6 +1429,30 @@ static int set_msr_hyperv(struct kvm_vcpu *vcpu, u32 msr, u64 data)
         return 0;
 }
 
+static int kvm_pv_enable_async_pf(struct kvm_vcpu *vcpu, u64 data)
+{
+        gpa_t gpa = data & ~0x3f;
+
+        /* Bits 2:5 are resrved, Should be zero */
+        if (data & 0x3c)
+                return 1;
+
+        vcpu->arch.apf.msr_val = data;
+
+        if (!(data & KVM_ASYNC_PF_ENABLED)) {
+                kvm_clear_async_pf_completion_queue(vcpu);
+                kvm_async_pf_hash_reset(vcpu);
+                return 0;
+        }
+
+        if (kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.apf.data, gpa))
+                return 1;
+
+        vcpu->arch.apf.send_user_only = !(data & KVM_ASYNC_PF_SEND_ALWAYS);
+        kvm_async_pf_wakeup_all(vcpu);
+        return 0;
+}
+
 int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data)
 {
         switch (msr) {
@@ -1499,6 +1534,10 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data)
                 }
                 break;
         }
+        case MSR_KVM_ASYNC_PF_EN:
+                if (kvm_pv_enable_async_pf(vcpu, data))
+                        return 1;
+                break;
         case MSR_IA32_MCG_CTL:
         case MSR_IA32_MCG_STATUS:
         case MSR_IA32_MC0_CTL ... MSR_IA32_MC0_CTL + 4 * KVM_MAX_MCE_BANKS - 1:
@@ -1775,6 +1814,9 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
         case MSR_KVM_SYSTEM_TIME_NEW:
                 data = vcpu->arch.time;
                 break;
+        case MSR_KVM_ASYNC_PF_EN:
+                data = vcpu->arch.apf.msr_val;
+                break;
         case MSR_IA32_P5_MC_ADDR:
         case MSR_IA32_P5_MC_TYPE:
         case MSR_IA32_MCG_CAP:
@@ -1904,6 +1946,7 @@ int kvm_dev_ioctl_check_extension(long ext)
         case KVM_CAP_NOP_IO_DELAY:
         case KVM_CAP_MP_STATE:
         case KVM_CAP_SYNC_MMU:
+        case KVM_CAP_USER_NMI:
         case KVM_CAP_REINJECT_CONTROL:
         case KVM_CAP_IRQ_INJECT_STATUS:
         case KVM_CAP_ASSIGN_DEV_IRQ:
@@ -1922,6 +1965,7 @@ int kvm_dev_ioctl_check_extension(long ext)
         case KVM_CAP_DEBUGREGS:
         case KVM_CAP_X86_ROBUST_SINGLESTEP:
         case KVM_CAP_XSAVE:
+        case KVM_CAP_ASYNC_PF:
                 r = 1;
                 break;
         case KVM_CAP_COALESCED_MMIO:
@@ -2185,6 +2229,11 @@ out:
         return r;
 }
 
+static void cpuid_mask(u32 *word, int wordnum)
+{
+        *word &= boot_cpu_data.x86_capability[wordnum];
+}
+
 static void do_cpuid_1_ent(struct kvm_cpuid_entry2 *entry, u32 function,
                            u32 index)
 {
@@ -2259,7 +2308,9 @@ static void do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function,
                 break;
         case 1:
                 entry->edx &= kvm_supported_word0_x86_features;
+                cpuid_mask(&entry->edx, 0);
                 entry->ecx &= kvm_supported_word4_x86_features;
+                cpuid_mask(&entry->ecx, 4);
                 /* we support x2apic emulation even if host does not support
                  * it since we emulate x2apic in software */
                 entry->ecx |= F(X2APIC);
@@ -2350,7 +2401,9 @@ static void do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function,
                 break;
         case 0x80000001:
                 entry->edx &= kvm_supported_word1_x86_features;
+                cpuid_mask(&entry->edx, 1);
                 entry->ecx &= kvm_supported_word6_x86_features;
+                cpuid_mask(&entry->ecx, 6);
                 break;
         }
 
@@ -3169,20 +3222,18 @@ int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
                 struct kvm_memslots *slots, *old_slots;
                 unsigned long *dirty_bitmap;
 
-                r = -ENOMEM;
-                dirty_bitmap = vmalloc(n);
-                if (!dirty_bitmap)
-                        goto out;
+                dirty_bitmap = memslot->dirty_bitmap_head;
+                if (memslot->dirty_bitmap == dirty_bitmap)
+                        dirty_bitmap += n / sizeof(long);
                 memset(dirty_bitmap, 0, n);
 
                 r = -ENOMEM;
                 slots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL);
-                if (!slots) {
-                        vfree(dirty_bitmap);
+                if (!slots)
                         goto out;
-                }
                 memcpy(slots, kvm->memslots, sizeof(struct kvm_memslots));
                 slots->memslots[log->slot].dirty_bitmap = dirty_bitmap;
+                slots->generation++;
 
                 old_slots = kvm->memslots;
                 rcu_assign_pointer(kvm->memslots, slots);
@@ -3195,11 +3246,8 @@ int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
                 spin_unlock(&kvm->mmu_lock);
 
                 r = -EFAULT;
-                if (copy_to_user(log->dirty_bitmap, dirty_bitmap, n)) {
-                        vfree(dirty_bitmap);
+                if (copy_to_user(log->dirty_bitmap, dirty_bitmap, n))
                         goto out;
-                }
-                vfree(dirty_bitmap);
         } else {
                 r = -EFAULT;
                 if (clear_user(log->dirty_bitmap, n))
@@ -3266,8 +3314,10 @@ long kvm_arch_vm_ioctl(struct file *filp,
                 if (vpic) {
                         r = kvm_ioapic_init(kvm);
                         if (r) {
+                                mutex_lock(&kvm->slots_lock);
                                 kvm_io_bus_unregister_dev(kvm, KVM_PIO_BUS,
                                                           &vpic->dev);
+                                mutex_unlock(&kvm->slots_lock);
                                 kfree(vpic);
                                 goto create_irqchip_unlock;
                         }
@@ -3278,10 +3328,12 @@ long kvm_arch_vm_ioctl(struct file *filp,
                 smp_wmb();
                 r = kvm_setup_default_irq_routing(kvm);
                 if (r) {
+                        mutex_lock(&kvm->slots_lock);
                         mutex_lock(&kvm->irq_lock);
                         kvm_ioapic_destroy(kvm);
                         kvm_destroy_pic(kvm);
                         mutex_unlock(&kvm->irq_lock);
+                        mutex_unlock(&kvm->slots_lock);
                 }
         create_irqchip_unlock:
                 mutex_unlock(&kvm->lock);
@@ -3557,63 +3609,63 @@ static gpa_t translate_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access)
 static gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access)
 {
         gpa_t t_gpa;
-        u32 error;
+        struct x86_exception exception;
 
         BUG_ON(!mmu_is_nested(vcpu));
 
         /* NPT walks are always user-walks */
         access |= PFERR_USER_MASK;
-        t_gpa  = vcpu->arch.mmu.gva_to_gpa(vcpu, gpa, access, &error);
-        if (t_gpa == UNMAPPED_GVA)
-                vcpu->arch.fault.nested = true;
+        t_gpa  = vcpu->arch.mmu.gva_to_gpa(vcpu, gpa, access, &exception);
 
         return t_gpa;
 }
 
-gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva, u32 *error)
+gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva,
+                              struct x86_exception *exception)
 {
         u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
-        return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, error);
+        return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
 }
 
- gpa_t kvm_mmu_gva_to_gpa_fetch(struct kvm_vcpu *vcpu, gva_t gva, u32 *error)
+ gpa_t kvm_mmu_gva_to_gpa_fetch(struct kvm_vcpu *vcpu, gva_t gva,
+                                struct x86_exception *exception)
 {
         u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
         access |= PFERR_FETCH_MASK;
-        return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, error);
+        return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
 }
 
-gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva, u32 *error)
+gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva,
+                               struct x86_exception *exception)
 {
         u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
         access |= PFERR_WRITE_MASK;
-        return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, error);
+        return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
 }
 
 /* uses this to access any guest's mapped memory without checking CPL */
-gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva, u32 *error)
+gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva,
+                                struct x86_exception *exception)
 {
-        return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, 0, error);
+        return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, 0, exception);
 }
 
 static int kvm_read_guest_virt_helper(gva_t addr, void *val, unsigned int bytes,
                                       struct kvm_vcpu *vcpu, u32 access,
-                                      u32 *error)
+                                      struct x86_exception *exception)
 {
         void *data = val;
         int r = X86EMUL_CONTINUE;
 
         while (bytes) {
                 gpa_t gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr, access,
-                                                            error);
+                                                            exception);
                 unsigned offset = addr & (PAGE_SIZE-1);
                 unsigned toread = min(bytes, (unsigned)PAGE_SIZE - offset);
                 int ret;
 
-                if (gpa == UNMAPPED_GVA) {
-                        r = X86EMUL_PROPAGATE_FAULT;
-                        goto out;
-                }
+                if (gpa == UNMAPPED_GVA)
+                        return X86EMUL_PROPAGATE_FAULT;
                 ret = kvm_read_guest(vcpu->kvm, gpa, data, toread);
                 if (ret < 0) {
                         r = X86EMUL_IO_NEEDED;
@@ -3630,31 +3682,35 @@ out:
 
 /* used for instruction fetching */
 static int kvm_fetch_guest_virt(gva_t addr, void *val, unsigned int bytes,
-                                struct kvm_vcpu *vcpu, u32 *error)
+                                struct kvm_vcpu *vcpu,
+                                struct x86_exception *exception)
 {
         u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
         return kvm_read_guest_virt_helper(addr, val, bytes, vcpu,
-                                          access | PFERR_FETCH_MASK, error);
+                                          access | PFERR_FETCH_MASK,
+                                          exception);
 }
 
 static int kvm_read_guest_virt(gva_t addr, void *val, unsigned int bytes,
-                               struct kvm_vcpu *vcpu, u32 *error)
+                               struct kvm_vcpu *vcpu,
+                               struct x86_exception *exception)
 {
         u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
         return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access,
-                                          error);
+                                          exception);
 }
 
 static int kvm_read_guest_virt_system(gva_t addr, void *val, unsigned int bytes,
-                               struct kvm_vcpu *vcpu, u32 *error)
+                                      struct kvm_vcpu *vcpu,
+                                      struct x86_exception *exception)
 {
-        return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, 0, error);
+        return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, 0, exception);
 }
 
 static int kvm_write_guest_virt_system(gva_t addr, void *val,
                                        unsigned int bytes,
                                        struct kvm_vcpu *vcpu,
-                                       u32 *error)
+                                       struct x86_exception *exception)
 {
         void *data = val;
         int r = X86EMUL_CONTINUE;
@@ -3662,15 +3718,13 @@ static int kvm_write_guest_virt_system(gva_t addr, void *val,
         while (bytes) {
                 gpa_t gpa =  vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr,
                                                              PFERR_WRITE_MASK,
-                                                             error);
+                                                             exception);
                 unsigned offset = addr & (PAGE_SIZE-1);
                 unsigned towrite = min(bytes, (unsigned)PAGE_SIZE - offset);
                 int ret;
 
-                if (gpa == UNMAPPED_GVA) {
-                        r = X86EMUL_PROPAGATE_FAULT;
-                        goto out;
-                }
+                if (gpa == UNMAPPED_GVA)
+                        return X86EMUL_PROPAGATE_FAULT;
                 ret = kvm_write_guest(vcpu->kvm, gpa, data, towrite);
                 if (ret < 0) {
                         r = X86EMUL_IO_NEEDED;
@@ -3688,7 +3742,7 @@ out:
 static int emulator_read_emulated(unsigned long addr,
                                   void *val,
                                   unsigned int bytes,
-                                  unsigned int *error_code,
+                                  struct x86_exception *exception,
                                   struct kvm_vcpu *vcpu)
 {
         gpa_t                 gpa;
@@ -3701,7 +3755,7 @@ static int emulator_read_emulated(unsigned long addr,
                 return X86EMUL_CONTINUE;
         }
 
-        gpa = kvm_mmu_gva_to_gpa_read(vcpu, addr, error_code);
+        gpa = kvm_mmu_gva_to_gpa_read(vcpu, addr, exception);
 
         if (gpa == UNMAPPED_GVA)
                 return X86EMUL_PROPAGATE_FAULT;
@@ -3710,8 +3764,8 @@ static int emulator_read_emulated(unsigned long addr,
         if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
                 goto mmio;
 
-        if (kvm_read_guest_virt(addr, val, bytes, vcpu, NULL)
-                                == X86EMUL_CONTINUE)
+        if (kvm_read_guest_virt(addr, val, bytes, vcpu, exception)
+            == X86EMUL_CONTINUE)
                 return X86EMUL_CONTINUE;
 
 mmio:
@@ -3735,7 +3789,7 @@ mmio:
 }
 
 int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
-                          const void *val, int bytes)
+                        const void *val, int bytes)
 {
         int ret;
 
@@ -3749,12 +3803,12 @@ int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
 static int emulator_write_emulated_onepage(unsigned long addr,
                                            const void *val,
                                            unsigned int bytes,
-                                           unsigned int *error_code,
+                                           struct x86_exception *exception,
                                            struct kvm_vcpu *vcpu)
 {
         gpa_t                 gpa;
 
-        gpa = kvm_mmu_gva_to_gpa_write(vcpu, addr, error_code);
+        gpa = kvm_mmu_gva_to_gpa_write(vcpu, addr, exception);
 
         if (gpa == UNMAPPED_GVA)
                 return X86EMUL_PROPAGATE_FAULT;
@@ -3787,7 +3841,7 @@ mmio:
 int emulator_write_emulated(unsigned long addr,
                             const void *val,
                             unsigned int bytes,
-                            unsigned int *error_code,
+                            struct x86_exception *exception,
                             struct kvm_vcpu *vcpu)
 {
         /* Crossing a page boundary? */
@@ -3795,7 +3849,7 @@ int emulator_write_emulated(unsigned long addr,
                 int rc, now;
 
                 now = -addr & ~PAGE_MASK;
-                rc = emulator_write_emulated_onepage(addr, val, now, error_code,
+                rc = emulator_write_emulated_onepage(addr, val, now, exception,
                                                      vcpu);
                 if (rc != X86EMUL_CONTINUE)
                         return rc;
@@ -3803,7 +3857,7 @@ int emulator_write_emulated(unsigned long addr,
                 val += now;
                 bytes -= now;
         }
-        return emulator_write_emulated_onepage(addr, val, bytes, error_code,
+        return emulator_write_emulated_onepage(addr, val, bytes, exception,
                                                vcpu);
 }
 
@@ -3821,7 +3875,7 @@ static int emulator_cmpxchg_emulated(unsigned long addr,
                                      const void *old,
                                      const void *new,
                                      unsigned int bytes,
-                                     unsigned int *error_code,
+                                     struct x86_exception *exception,
                                      struct kvm_vcpu *vcpu)
 {
         gpa_t gpa;
@@ -3879,7 +3933,7 @@ static int emulator_cmpxchg_emulated(unsigned long addr,
 emul_write:
         printk_once(KERN_WARNING "kvm: emulating exchange as write\n");
 
-        return emulator_write_emulated(addr, new, bytes, error_code, vcpu);
+        return emulator_write_emulated(addr, new, bytes, exception, vcpu);
 }
 
 static int kernel_pio(struct kvm_vcpu *vcpu, void *pd)
@@ -3904,7 +3958,7 @@ static int emulator_pio_in_emulated(int size, unsigned short port, void *val,
         if (vcpu->arch.pio.count)
                 goto data_avail;
 
-        trace_kvm_pio(0, port, size, 1);
+        trace_kvm_pio(0, port, size, count);
 
         vcpu->arch.pio.port = port;
         vcpu->arch.pio.in = 1;
@@ -3932,7 +3986,7 @@ static int emulator_pio_out_emulated(int size, unsigned short port,
                               const void *val, unsigned int count,
                               struct kvm_vcpu *vcpu)
 {
-        trace_kvm_pio(1, port, size, 1);
+        trace_kvm_pio(1, port, size, count);
 
         vcpu->arch.pio.port = port;
         vcpu->arch.pio.in = 0;
@@ -3973,13 +4027,15 @@ int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu)
                 return X86EMUL_CONTINUE;
 
         if (kvm_x86_ops->has_wbinvd_exit()) {
-                preempt_disable();
+                int cpu = get_cpu();
+
+                cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);
                 smp_call_function_many(vcpu->arch.wbinvd_dirty_mask,
                                 wbinvd_ipi, NULL, 1);
-                preempt_enable();
+                put_cpu();
                 cpumask_clear(vcpu->arch.wbinvd_dirty_mask);
-        }
-        wbinvd();
+        } else
+                wbinvd();
         return X86EMUL_CONTINUE;
 }
 EXPORT_SYMBOL_GPL(kvm_emulate_wbinvd);
@@ -4019,7 +4075,7 @@ static unsigned long emulator_get_cr(int cr, struct kvm_vcpu *vcpu)
                 value = vcpu->arch.cr2;
                 break;
         case 3:
-                value = vcpu->arch.cr3;
+                value = kvm_read_cr3(vcpu);
                 break;
         case 4:
                 value = kvm_read_cr4(vcpu);
@@ -4053,7 +4109,7 @@ static int emulator_set_cr(int cr, unsigned long val, struct kvm_vcpu *vcpu)
                 res = kvm_set_cr4(vcpu, mk_cr_64(kvm_read_cr4(vcpu), val));
                 break;
         case 8:
-                res = __kvm_set_cr8(vcpu, val & 0xfUL);
+                res = kvm_set_cr8(vcpu, val);
                 break;
         default:
                 vcpu_printf(vcpu, "%s: unexpected cr %u\n", __func__, cr);
@@ -4206,12 +4262,13 @@ static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask)
 static void inject_emulated_exception(struct kvm_vcpu *vcpu)
 {
         struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
-        if (ctxt->exception == PF_VECTOR)
-                kvm_propagate_fault(vcpu);
-        else if (ctxt->error_code_valid)
-                kvm_queue_exception_e(vcpu, ctxt->exception, ctxt->error_code);
+        if (ctxt->exception.vector == PF_VECTOR)
+                kvm_propagate_fault(vcpu, &ctxt->exception);
+        else if (ctxt->exception.error_code_valid)
+                kvm_queue_exception_e(vcpu, ctxt->exception.vector,
+                                      ctxt->exception.error_code);
         else
-                kvm_queue_exception(vcpu, ctxt->exception);
+                kvm_queue_exception(vcpu, ctxt->exception.vector);
 }
 
 static void init_emulate_ctxt(struct kvm_vcpu *vcpu)
@@ -4267,13 +4324,19 @@ EXPORT_SYMBOL_GPL(kvm_inject_realmode_interrupt);
 
 static int handle_emulation_failure(struct kvm_vcpu *vcpu)
 {
+        int r = EMULATE_DONE;
+
         ++vcpu->stat.insn_emulation_fail;
         trace_kvm_emulate_insn_failed(vcpu);
-        vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
-        vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
-        vcpu->run->internal.ndata = 0;
+        if (!is_guest_mode(vcpu)) {
+                vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
+                vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
+                vcpu->run->internal.ndata = 0;
+                r = EMULATE_FAIL;
+        }
         kvm_queue_exception(vcpu, UD_VECTOR);
-        return EMULATE_FAIL;
+
+        return r;
 }
 
 static bool reexecute_instruction(struct kvm_vcpu *vcpu, gva_t gva)
@@ -4302,10 +4365,11 @@ static bool reexecute_instruction(struct kvm_vcpu *vcpu, gva_t gva)
         return false;
 }
 
-int emulate_instruction(struct kvm_vcpu *vcpu,
-                        unsigned long cr2,
-                        u16 error_code,
-                        int emulation_type)
+int x86_emulate_instruction(struct kvm_vcpu *vcpu,
+                            unsigned long cr2,
+                            int emulation_type,
+                            void *insn,
+                            int insn_len)
 {
         int r;
         struct decode_cache *c = &vcpu->arch.emulate_ctxt.decode;
@@ -4323,10 +4387,10 @@ int emulate_instruction(struct kvm_vcpu *vcpu,
         if (!(emulation_type & EMULTYPE_NO_DECODE)) {
                 init_emulate_ctxt(vcpu);
                 vcpu->arch.emulate_ctxt.interruptibility = 0;
-                vcpu->arch.emulate_ctxt.exception = -1;
+                vcpu->arch.emulate_ctxt.have_exception = false;
                 vcpu->arch.emulate_ctxt.perm_ok = false;
 
-                r = x86_decode_insn(&vcpu->arch.emulate_ctxt);
+                r = x86_decode_insn(&vcpu->arch.emulate_ctxt, insn, insn_len);
                 if (r == X86EMUL_PROPAGATE_FAULT)
                         goto done;
 
@@ -4389,7 +4453,7 @@ restart:
         }
 
 done:
-        if (vcpu->arch.emulate_ctxt.exception >= 0) {
+        if (vcpu->arch.emulate_ctxt.have_exception) {
                 inject_emulated_exception(vcpu);
                 r = EMULATE_DONE;
         } else if (vcpu->arch.pio.count) {
@@ -4413,7 +4477,7 @@ done:
 
         return r;
 }
-EXPORT_SYMBOL_GPL(emulate_instruction);
+EXPORT_SYMBOL_GPL(x86_emulate_instruction);
 
 int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size, unsigned short port)
 {
@@ -4653,7 +4717,6 @@ int kvm_arch_init(void *opaque)
 
         kvm_x86_ops = ops;
         kvm_mmu_set_nonpresent_ptes(0ull, 0ull);
-        kvm_mmu_set_base_ptes(PT_PRESENT_MASK);
         kvm_mmu_set_mask_ptes(PT_USER_MASK, PT_ACCESSED_MASK,
                         PT_DIRTY_MASK, PT64_NX_MASK, 0);
 
@@ -5116,6 +5179,12 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
                         vcpu->fpu_active = 0;
                         kvm_x86_ops->fpu_deactivate(vcpu);
                 }
+                if (kvm_check_request(KVM_REQ_APF_HALT, vcpu)) {
+                        /* Page is swapped out. Do synthetic halt */
+                        vcpu->arch.apf.halted = true;
+                        r = 1;
+                        goto out;
+                }
         }
 
         r = kvm_mmu_reload(vcpu);
@@ -5244,7 +5313,8 @@ static int __vcpu_run(struct kvm_vcpu *vcpu)
 
         r = 1;
         while (r > 0) {
-                if (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE)
+                if (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE &&
+                    !vcpu->arch.apf.halted)
                         r = vcpu_enter_guest(vcpu);
                 else {
                         srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
@@ -5257,6 +5327,7 @@ static int __vcpu_run(struct kvm_vcpu *vcpu)
                                         vcpu->arch.mp_state =
                                                 KVM_MP_STATE_RUNNABLE;
                                 case KVM_MP_STATE_RUNNABLE:
+                                        vcpu->arch.apf.halted = false;
                                         break;
                                 case KVM_MP_STATE_SIPI_RECEIVED:
                                 default:
@@ -5278,6 +5349,9 @@ static int __vcpu_run(struct kvm_vcpu *vcpu)
                         vcpu->run->exit_reason = KVM_EXIT_INTR;
                         ++vcpu->stat.request_irq_exits;
                 }
+
+                kvm_check_async_pf_completion(vcpu);
+
                 if (signal_pending(current)) {
                         r = -EINTR;
                         vcpu->run->exit_reason = KVM_EXIT_INTR;
@@ -5302,6 +5376,9 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
         int r;
         sigset_t sigsaved;
 
+        if (!tsk_used_math(current) && init_fpu(current))
+                return -ENOMEM;
+
         if (vcpu->sigset_active)
                 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
 
@@ -5313,8 +5390,12 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
         }
 
         /* re-sync apic's tpr */
-        if (!irqchip_in_kernel(vcpu->kvm))
-                kvm_set_cr8(vcpu, kvm_run->cr8);
+        if (!irqchip_in_kernel(vcpu->kvm)) {
+                if (kvm_set_cr8(vcpu, kvm_run->cr8) != 0) {
+                        r = -EINVAL;
+                        goto out;
+                }
+        }
 
         if (vcpu->arch.pio.count || vcpu->mmio_needed) {
                 if (vcpu->mmio_needed) {
@@ -5323,7 +5404,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
                         vcpu->mmio_needed = 0;
                 }
                 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
-                r = emulate_instruction(vcpu, 0, 0, EMULTYPE_NO_DECODE);
+                r = emulate_instruction(vcpu, EMULTYPE_NO_DECODE);
                 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
                 if (r != EMULATE_DONE) {
                         r = 0;
@@ -5436,7 +5517,7 @@ int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
 
         sregs->cr0 = kvm_read_cr0(vcpu);
         sregs->cr2 = vcpu->arch.cr2;
-        sregs->cr3 = vcpu->arch.cr3;
+        sregs->cr3 = kvm_read_cr3(vcpu);
         sregs->cr4 = kvm_read_cr4(vcpu);
         sregs->cr8 = kvm_get_cr8(vcpu);
         sregs->efer = vcpu->arch.efer;
@@ -5504,8 +5585,9 @@ int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
         kvm_x86_ops->set_gdt(vcpu, &dt);
 
         vcpu->arch.cr2 = sregs->cr2;
-        mmu_reset_needed |= vcpu->arch.cr3 != sregs->cr3;
+        mmu_reset_needed |= kvm_read_cr3(vcpu) != sregs->cr3;
         vcpu->arch.cr3 = sregs->cr3;
+        __set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail);
 
         kvm_set_cr8(vcpu, sregs->cr8);
 
@@ -5522,7 +5604,7 @@ int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
         if (sregs->cr4 & X86_CR4_OSXSAVE)
                 update_cpuid(vcpu);
         if (!is_long_mode(vcpu) && is_pae(vcpu)) {
-                load_pdptrs(vcpu, vcpu->arch.walk_mmu, vcpu->arch.cr3);
+                load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu));
                 mmu_reset_needed = 1;
         }
 
@@ -5773,6 +5855,8 @@ free_vcpu:
 
 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
 {
+        vcpu->arch.apf.msr_val = 0;
+
         vcpu_load(vcpu);
         kvm_mmu_unload(vcpu);
         vcpu_put(vcpu);
@@ -5792,6 +5876,11 @@ int kvm_arch_vcpu_reset(struct kvm_vcpu *vcpu)
         vcpu->arch.dr7 = DR7_FIXED_1;
 
         kvm_make_request(KVM_REQ_EVENT, vcpu);
+        vcpu->arch.apf.msr_val = 0;
+
+        kvm_clear_async_pf_completion_queue(vcpu);
+        kvm_async_pf_hash_reset(vcpu);
+        vcpu->arch.apf.halted = false;
 
         return kvm_x86_ops->vcpu_reset(vcpu);
 }
@@ -5881,6 +5970,8 @@ int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
         if (!zalloc_cpumask_var(&vcpu->arch.wbinvd_dirty_mask, GFP_KERNEL))
                 goto fail_free_mce_banks;
 
+        kvm_async_pf_hash_reset(vcpu);
+
         return 0;
 fail_free_mce_banks:
         kfree(vcpu->arch.mce_banks);
@@ -5906,13 +5997,8 @@ void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu)
         free_page((unsigned long)vcpu->arch.pio_data);
 }
 
-struct  kvm *kvm_arch_create_vm(void)
+int kvm_arch_init_vm(struct kvm *kvm)
 {
-        struct kvm *kvm = kzalloc(sizeof(struct kvm), GFP_KERNEL);
-
-        if (!kvm)
-                return ERR_PTR(-ENOMEM);
-
         INIT_LIST_HEAD(&kvm->arch.active_mmu_pages);
         INIT_LIST_HEAD(&kvm->arch.assigned_dev_head);
 
@@ -5921,7 +6007,7 @@ struct  kvm *kvm_arch_create_vm(void)
 
         spin_lock_init(&kvm->arch.tsc_write_lock);
 
-        return kvm;
+        return 0;
 }
 
 static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu)
@@ -5939,8 +6025,10 @@ static void kvm_free_vcpus(struct kvm *kvm)
         /*
          * Unpin any mmu pages first.
          */
-        kvm_for_each_vcpu(i, vcpu, kvm)
+        kvm_for_each_vcpu(i, vcpu, kvm) {
+                kvm_clear_async_pf_completion_queue(vcpu);
                 kvm_unload_vcpu_mmu(vcpu);
+        }
         kvm_for_each_vcpu(i, vcpu, kvm)
                 kvm_arch_vcpu_free(vcpu);
 
@@ -5964,13 +6052,10 @@ void kvm_arch_destroy_vm(struct kvm *kvm)
         kfree(kvm->arch.vpic);
         kfree(kvm->arch.vioapic);
         kvm_free_vcpus(kvm);
-        kvm_free_physmem(kvm);
         if (kvm->arch.apic_access_page)
                 put_page(kvm->arch.apic_access_page);
         if (kvm->arch.ept_identity_pagetable)
                 put_page(kvm->arch.ept_identity_pagetable);
-        cleanup_srcu_struct(&kvm->srcu);
-        kfree(kvm);
 }
 
 int kvm_arch_prepare_memory_region(struct kvm *kvm,
@@ -6051,7 +6136,9 @@ void kvm_arch_flush_shadow(struct kvm *kvm)
 
 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
 {
-        return vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE
+        return (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE &&
+                !vcpu->arch.apf.halted)
+                || !list_empty_careful(&vcpu->async_pf.done)
                 || vcpu->arch.mp_state == KVM_MP_STATE_SIPI_RECEIVED
                 || vcpu->arch.nmi_pending ||
                 (kvm_arch_interrupt_allowed(vcpu) &&
@@ -6110,6 +6197,147 @@ void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
 }
 EXPORT_SYMBOL_GPL(kvm_set_rflags);
 
+void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, struct kvm_async_pf *work)
+{
+        int r;
+
+        if ((vcpu->arch.mmu.direct_map != work->arch.direct_map) ||
+              is_error_page(work->page))
+                return;
+
+        r = kvm_mmu_reload(vcpu);
+        if (unlikely(r))
+                return;
+
+        if (!vcpu->arch.mmu.direct_map &&
+              work->arch.cr3 != vcpu->arch.mmu.get_cr3(vcpu))
+                return;
+
+        vcpu->arch.mmu.page_fault(vcpu, work->gva, 0, true);
+}
+
+static inline u32 kvm_async_pf_hash_fn(gfn_t gfn)
+{
+        return hash_32(gfn & 0xffffffff, order_base_2(ASYNC_PF_PER_VCPU));
+}
+
+static inline u32 kvm_async_pf_next_probe(u32 key)
+{
+        return (key + 1) & (roundup_pow_of_two(ASYNC_PF_PER_VCPU) - 1);
+}
+
+static void kvm_add_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
+{
+        u32 key = kvm_async_pf_hash_fn(gfn);
+
+        while (vcpu->arch.apf.gfns[key] != ~0)
+                key = kvm_async_pf_next_probe(key);
+
+        vcpu->arch.apf.gfns[key] = gfn;
+}
+
+static u32 kvm_async_pf_gfn_slot(struct kvm_vcpu *vcpu, gfn_t gfn)
+{
+        int i;
+        u32 key = kvm_async_pf_hash_fn(gfn);
+
+        for (i = 0; i < roundup_pow_of_two(ASYNC_PF_PER_VCPU) &&
+                     (vcpu->arch.apf.gfns[key] != gfn &&
+                      vcpu->arch.apf.gfns[key] != ~0); i++)
+                key = kvm_async_pf_next_probe(key);
+
+        return key;
+}
+
+bool kvm_find_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
+{
+        return vcpu->arch.apf.gfns[kvm_async_pf_gfn_slot(vcpu, gfn)] == gfn;
+}
+
+static void kvm_del_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
+{
+        u32 i, j, k;
+
+        i = j = kvm_async_pf_gfn_slot(vcpu, gfn);
+        while (true) {
+                vcpu->arch.apf.gfns[i] = ~0;
+                do {
+                        j = kvm_async_pf_next_probe(j);
+                        if (vcpu->arch.apf.gfns[j] == ~0)
+                                return;
+                        k = kvm_async_pf_hash_fn(vcpu->arch.apf.gfns[j]);
+                        /*
+                         * k lies cyclically in ]i,j]
+                         * |    i.k.j |
+                         * |....j i.k.| or  |.k..j i...|
+                         */
+                } while ((i <= j) ? (i < k && k <= j) : (i < k || k <= j));
+                vcpu->arch.apf.gfns[i] = vcpu->arch.apf.gfns[j];
+                i = j;
+        }
+}
+
+static int apf_put_user(struct kvm_vcpu *vcpu, u32 val)
+{
+
+        return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apf.data, &val,
+                                      sizeof(val));
+}
+
+void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
+                                     struct kvm_async_pf *work)
+{
+        struct x86_exception fault;
+
+        trace_kvm_async_pf_not_present(work->arch.token, work->gva);
+        kvm_add_async_pf_gfn(vcpu, work->arch.gfn);
+
+        if (!(vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED) ||
+            (vcpu->arch.apf.send_user_only &&
+             kvm_x86_ops->get_cpl(vcpu) == 0))
+                kvm_make_request(KVM_REQ_APF_HALT, vcpu);
+        else if (!apf_put_user(vcpu, KVM_PV_REASON_PAGE_NOT_PRESENT)) {
+                fault.vector = PF_VECTOR;
+                fault.error_code_valid = true;
+                fault.error_code = 0;
+                fault.nested_page_fault = false;
+                fault.address = work->arch.token;
+                kvm_inject_page_fault(vcpu, &fault);
+        }
+}
+
+void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
+                                 struct kvm_async_pf *work)
+{
+        struct x86_exception fault;
+
+        trace_kvm_async_pf_ready(work->arch.token, work->gva);
+        if (is_error_page(work->page))
+                work->arch.token = ~0; /* broadcast wakeup */
+        else
+                kvm_del_async_pf_gfn(vcpu, work->arch.gfn);
+
+        if ((vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED) &&
+            !apf_put_user(vcpu, KVM_PV_REASON_PAGE_READY)) {
+                fault.vector = PF_VECTOR;
+                fault.error_code_valid = true;
+                fault.error_code = 0;
+                fault.nested_page_fault = false;
+                fault.address = work->arch.token;
+                kvm_inject_page_fault(vcpu, &fault);
+        }
+        vcpu->arch.apf.halted = false;
+}
+
+bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
+{
+        if (!(vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED))
+                return true;
+        else
+                return !kvm_event_needs_reinjection(vcpu) &&
+                        kvm_x86_ops->interrupt_allowed(vcpu);
+}
+
 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_exit);
 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_inj_virq);
 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_page_fault);
diff --git a/arch/x86/mm/gup.c b/arch/x86/mm/gup.c
index 738e6593799d..dbe34b931374 100644
--- a/arch/x86/mm/gup.c
+++ b/arch/x86/mm/gup.c
@@ -8,6 +8,7 @@
 #include <linux/mm.h>
 #include <linux/vmstat.h>
 #include <linux/highmem.h>
+#include <linux/swap.h>
 
 #include <asm/pgtable.h>
 
@@ -89,6 +90,7 @@ static noinline int gup_pte_range(pmd_t pmd, unsigned long addr,
                 VM_BUG_ON(!pfn_valid(pte_pfn(pte)));
                 page = pte_page(pte);
                 get_page(page);
+                SetPageReferenced(page);
                 pages[*nr] = page;
                 (*nr)++;
 
@@ -103,6 +105,17 @@ static inline void get_head_page_multiple(struct page *page, int nr)
         VM_BUG_ON(page != compound_head(page));
         VM_BUG_ON(page_count(page) == 0);
         atomic_add(nr, &page->_count);
+        SetPageReferenced(page);
+}
+
+static inline void get_huge_page_tail(struct page *page)
+{
+        /*
+         * __split_huge_page_refcount() cannot run
+         * from under us.
+         */
+        VM_BUG_ON(atomic_read(&page->_count) < 0);
+        atomic_inc(&page->_count);
 }
 
 static noinline int gup_huge_pmd(pmd_t pmd, unsigned long addr,
@@ -128,6 +141,8 @@ static noinline int gup_huge_pmd(pmd_t pmd, unsigned long addr,
         do {
                 VM_BUG_ON(compound_head(page) != head);
                 pages[*nr] = page;
+                if (PageTail(page))
+                        get_huge_page_tail(page);
                 (*nr)++;
                 page++;
                 refs++;
@@ -148,7 +163,18 @@ static int gup_pmd_range(pud_t pud, unsigned long addr, unsigned long end,
                 pmd_t pmd = *pmdp;
 
                 next = pmd_addr_end(addr, end);
-                if (pmd_none(pmd))
+                /*
+                 * The pmd_trans_splitting() check below explains why
+                 * pmdp_splitting_flush has to flush the tlb, to stop
+                 * this gup-fast code from running while we set the
+                 * splitting bit in the pmd. Returning zero will take
+                 * the slow path that will call wait_split_huge_page()
+                 * if the pmd is still in splitting state. gup-fast
+                 * can't because it has irq disabled and
+                 * wait_split_huge_page() would never return as the
+                 * tlb flush IPI wouldn't run.
+                 */
+                if (pmd_none(pmd) || pmd_trans_splitting(pmd))
                         return 0;
                 if (unlikely(pmd_large(pmd))) {
                         if (!gup_huge_pmd(pmd, addr, next, write, pages, nr))
diff --git a/arch/x86/mm/init_32.c b/arch/x86/mm/init_32.c
index f89b5bb4e93f..c821074b7f0b 100644
--- a/arch/x86/mm/init_32.c
+++ b/arch/x86/mm/init_32.c
@@ -45,6 +45,7 @@
 #include <asm/bugs.h>
 #include <asm/tlb.h>
 #include <asm/tlbflush.h>
+#include <asm/olpc_ofw.h>
 #include <asm/pgalloc.h>
 #include <asm/sections.h>
 #include <asm/paravirt.h>
@@ -715,6 +716,7 @@ void __init paging_init(void)
         /*
          * NOTE: at this point the bootmem allocator is fully available.
          */
+        olpc_dt_build_devicetree();
         sparse_init();
         zone_sizes_init();
 }
diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c
index 8be8c7d7bc89..500242d3c96d 100644
--- a/arch/x86/mm/pgtable.c
+++ b/arch/x86/mm/pgtable.c
@@ -320,6 +320,25 @@ int ptep_set_access_flags(struct vm_area_struct *vma,
         return changed;
 }
 
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+int pmdp_set_access_flags(struct vm_area_struct *vma,
+                          unsigned long address, pmd_t *pmdp,
+                          pmd_t entry, int dirty)
+{
+        int changed = !pmd_same(*pmdp, entry);
+
+        VM_BUG_ON(address & ~HPAGE_PMD_MASK);
+
+        if (changed && dirty) {
+                *pmdp = entry;
+                pmd_update_defer(vma->vm_mm, address, pmdp);
+                flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
+        }
+
+        return changed;
+}
+#endif
+
 int ptep_test_and_clear_young(struct vm_area_struct *vma,
                               unsigned long addr, pte_t *ptep)
 {
@@ -335,6 +354,23 @@ int ptep_test_and_clear_young(struct vm_area_struct *vma,
         return ret;
 }
 
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+int pmdp_test_and_clear_young(struct vm_area_struct *vma,
+                              unsigned long addr, pmd_t *pmdp)
+{
+        int ret = 0;
+
+        if (pmd_young(*pmdp))
+                ret = test_and_clear_bit(_PAGE_BIT_ACCESSED,
+                                         (unsigned long *)pmdp);
+
+        if (ret)
+                pmd_update(vma->vm_mm, addr, pmdp);
+
+        return ret;
+}
+#endif
+
 int ptep_clear_flush_young(struct vm_area_struct *vma,
                            unsigned long address, pte_t *ptep)
 {
@@ -347,6 +383,36 @@ int ptep_clear_flush_young(struct vm_area_struct *vma,
         return young;
 }
 
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+int pmdp_clear_flush_young(struct vm_area_struct *vma,
+                           unsigned long address, pmd_t *pmdp)
+{
+        int young;
+
+        VM_BUG_ON(address & ~HPAGE_PMD_MASK);
+
+        young = pmdp_test_and_clear_young(vma, address, pmdp);
+        if (young)
+                flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
+
+        return young;
+}
+
+void pmdp_splitting_flush(struct vm_area_struct *vma,
+                          unsigned long address, pmd_t *pmdp)
+{
+        int set;
+        VM_BUG_ON(address & ~HPAGE_PMD_MASK);
+        set = !test_and_set_bit(_PAGE_BIT_SPLITTING,
+                                (unsigned long *)pmdp);
+        if (set) {
+                pmd_update(vma->vm_mm, address, pmdp);
+                /* need tlb flush only to serialize against gup-fast */
+                flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
+        }
+}
+#endif
+
 /**
  * reserve_top_address - reserves a hole in the top of kernel address space
  * @reserve - size of hole to reserve
diff --git a/arch/x86/platform/mrst/early_printk_mrst.c b/arch/x86/platform/mrst/early_printk_mrst.c
index 65df603622b2..25bfdbb5b130 100644
--- a/arch/x86/platform/mrst/early_printk_mrst.c
+++ b/arch/x86/platform/mrst/early_printk_mrst.c
@@ -103,7 +103,7 @@ struct dw_spi_reg {
 static unsigned long mrst_spi_paddr = MRST_REGBASE_SPI0;
 
 static u32 *pclk_spi0;
-/* Always contains an accessable address, start with 0 */
+/* Always contains an accessible address, start with 0 */
 static struct dw_spi_reg *pspi;
 
 static struct kmsg_dumper dw_dumper;
diff --git a/arch/x86/platform/olpc/Makefile b/arch/x86/platform/olpc/Makefile
index c31b8fcb5a86..e797428b163b 100644
--- a/arch/x86/platform/olpc/Makefile
+++ b/arch/x86/platform/olpc/Makefile
@@ -1,3 +1,4 @@
 obj-$(CONFIG_OLPC)              += olpc.o
 obj-$(CONFIG_OLPC_XO1)          += olpc-xo1.o
 obj-$(CONFIG_OLPC_OPENFIRMWARE) += olpc_ofw.o
+obj-$(CONFIG_OLPC_OPENFIRMWARE_DT)      += olpc_dt.o
diff --git a/arch/x86/platform/olpc/olpc_dt.c b/arch/x86/platform/olpc/olpc_dt.c
new file mode 100644
index 000000000000..dab874647530
--- /dev/null
+++ b/arch/x86/platform/olpc/olpc_dt.c
@@ -0,0 +1,183 @@
+/*
+ * OLPC-specific OFW device tree support code.
+ *
+ * Paul Mackerras       August 1996.
+ * Copyright (C) 1996-2005 Paul Mackerras.
+ *
+ *  Adapted for 64bit PowerPC by Dave Engebretsen and Peter Bergner.
+ *    {engebret|bergner}@us.ibm.com
+ *
+ *  Adapted for sparc by David S. Miller davem@davemloft.net
+ *  Adapted for x86/OLPC by Andres Salomon <dilinger@queued.net>
+ *
+ *      This program is free software; you can redistribute it and/or
+ *      modify it under the terms of the GNU General Public License
+ *      as published by the Free Software Foundation; either version
+ *      2 of the License, or (at your option) any later version.
+ */
+
+#include <linux/kernel.h>
+#include <linux/bootmem.h>
+#include <linux/of.h>
+#include <linux/of_pdt.h>
+#include <asm/olpc_ofw.h>
+
+static phandle __init olpc_dt_getsibling(phandle node)
+{
+        const void *args[] = { (void *)node };
+        void *res[] = { &node };
+
+        if ((s32)node == -1)
+                return 0;
+
+        if (olpc_ofw("peer", args, res) || (s32)node == -1)
+                return 0;
+
+        return node;
+}
+
+static phandle __init olpc_dt_getchild(phandle node)
+{
+        const void *args[] = { (void *)node };
+        void *res[] = { &node };
+
+        if ((s32)node == -1)
+                return 0;
+
+        if (olpc_ofw("child", args, res) || (s32)node == -1) {
+                pr_err("PROM: %s: fetching child failed!\n", __func__);
+                return 0;
+        }
+
+        return node;
+}
+
+static int __init olpc_dt_getproplen(phandle node, const char *prop)
+{
+        const void *args[] = { (void *)node, prop };
+        int len;
+        void *res[] = { &len };
+
+        if ((s32)node == -1)
+                return -1;
+
+        if (olpc_ofw("getproplen", args, res)) {
+                pr_err("PROM: %s: getproplen failed!\n", __func__);
+                return -1;
+        }
+
+        return len;
+}
+
+static int __init olpc_dt_getproperty(phandle node, const char *prop,
+                char *buf, int bufsize)
+{
+        int plen;
+
+        plen = olpc_dt_getproplen(node, prop);
+        if (plen > bufsize || plen < 1) {
+                return -1;
+        } else {
+                const void *args[] = { (void *)node, prop, buf, (void *)plen };
+                void *res[] = { &plen };
+
+                if (olpc_ofw("getprop", args, res)) {
+                        pr_err("PROM: %s: getprop failed!\n", __func__);
+                        return -1;
+                }
+        }
+
+        return plen;
+}
+
+static int __init olpc_dt_nextprop(phandle node, char *prev, char *buf)
+{
+        const void *args[] = { (void *)node, prev, buf };
+        int success;
+        void *res[] = { &success };
+
+        buf[0] = '\0';
+
+        if ((s32)node == -1)
+                return -1;
+
+        if (olpc_ofw("nextprop", args, res) || success != 1)
+                return -1;
+
+        return 0;
+}
+
+static int __init olpc_dt_pkg2path(phandle node, char *buf,
+                const int buflen, int *len)
+{
+        const void *args[] = { (void *)node, buf, (void *)buflen };
+        void *res[] = { len };
+
+        if ((s32)node == -1)
+                return -1;
+
+        if (olpc_ofw("package-to-path", args, res) || *len < 1)
+                return -1;
+
+        return 0;
+}
+
+static unsigned int prom_early_allocated __initdata;
+
+void * __init prom_early_alloc(unsigned long size)
+{
+        static u8 *mem;
+        static size_t free_mem;
+        void *res;
+
+        if (free_mem < size) {
+                const size_t chunk_size = max(PAGE_SIZE, size);
+
+                /*
+                 * To mimimize the number of allocations, grab at least
+                 * PAGE_SIZE of memory (that's an arbitrary choice that's
+                 * fast enough on the platforms we care about while minimizing
+                 * wasted bootmem) and hand off chunks of it to callers.
+                 */
+                res = alloc_bootmem(chunk_size);
+                if (!res)
+                        return NULL;
+                prom_early_allocated += chunk_size;
+                memset(res, 0, chunk_size);
+                free_mem = chunk_size;
+                mem = res;
+        }
+
+        /* allocate from the local cache */
+        free_mem -= size;
+        res = mem;
+        mem += size;
+        return res;
+}
+
+static struct of_pdt_ops prom_olpc_ops __initdata = {
+        .nextprop = olpc_dt_nextprop,
+        .getproplen = olpc_dt_getproplen,
+        .getproperty = olpc_dt_getproperty,
+        .getchild = olpc_dt_getchild,
+        .getsibling = olpc_dt_getsibling,
+        .pkg2path = olpc_dt_pkg2path,
+};
+
+void __init olpc_dt_build_devicetree(void)
+{
+        phandle root;
+
+        if (!olpc_ofw_is_installed())
+                return;
+
+        root = olpc_dt_getsibling(0);
+        if (!root) {
+                pr_err("PROM: unable to get root node from OFW!\n");
+                return;
+        }
+        of_pdt_build_devicetree(root, &prom_olpc_ops);
+
+        pr_info("PROM DT: Built device tree with %u bytes of memory.\n",
+                        prom_early_allocated);
+}
diff --git a/arch/x86/platform/olpc/olpc_ofw.c b/arch/x86/platform/olpc/olpc_ofw.c
index 787320464379..e7604f62870d 100644
--- a/arch/x86/platform/olpc/olpc_ofw.c
+++ b/arch/x86/platform/olpc/olpc_ofw.c
@@ -110,3 +110,8 @@ void __init olpc_ofw_detect(void)
                         (unsigned long)olpc_ofw_cif, (-start) >> 20);
         reserve_top_address(-start);
 }
+
+bool __init olpc_ofw_is_installed(void)
+{
+        return olpc_ofw_cif != NULL;
+}
diff --git a/arch/x86/xen/Makefile b/arch/x86/xen/Makefile
index 779385158915..17c565de3d64 100644
--- a/arch/x86/xen/Makefile
+++ b/arch/x86/xen/Makefile
@@ -12,7 +12,8 @@ CFLAGS_mmu.o			:= $(nostackp)
 
 obj-y           := enlighten.o setup.o multicalls.o mmu.o irq.o \
                         time.o xen-asm.o xen-asm_$(BITS).o \
-                        grant-table.o suspend.o platform-pci-unplug.o
+                        grant-table.o suspend.o platform-pci-unplug.o \
+                        p2m.o
 
 obj-$(CONFIG_SMP)               += smp.o
 obj-$(CONFIG_PARAVIRT_SPINLOCKS)+= spinlock.o
diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
index 44924e551fde..7575e55cd52e 100644
--- a/arch/x86/xen/mmu.c
+++ b/arch/x86/xen/mmu.c
@@ -173,371 +173,6 @@ DEFINE_PER_CPU(unsigned long, xen_current_cr3);	 /* actual vcpu cr3 */
  */
 #define USER_LIMIT      ((STACK_TOP_MAX + PGDIR_SIZE - 1) & PGDIR_MASK)
 
-/*
- * Xen leaves the responsibility for maintaining p2m mappings to the
- * guests themselves, but it must also access and update the p2m array
- * during suspend/resume when all the pages are reallocated.
- *
- * The p2m table is logically a flat array, but we implement it as a
- * three-level tree to allow the address space to be sparse.
- *
- *                               Xen
- *                                |
- *     p2m_top              p2m_top_mfn
- *       /  \                   /   \
- * p2m_mid p2m_mid      p2m_mid_mfn p2m_mid_mfn
- *    / \      / \         /           /
- *  p2m p2m p2m p2m p2m p2m p2m ...
- *
- * The p2m_mid_mfn pages are mapped by p2m_top_mfn_p.
- *
- * The p2m_top and p2m_top_mfn levels are limited to 1 page, so the
- * maximum representable pseudo-physical address space is:
- *  P2M_TOP_PER_PAGE * P2M_MID_PER_PAGE * P2M_PER_PAGE pages
- *
- * P2M_PER_PAGE depends on the architecture, as a mfn is always
- * unsigned long (8 bytes on 64-bit, 4 bytes on 32), leading to
- * 512 and 1024 entries respectively. 
- */
-
-unsigned long xen_max_p2m_pfn __read_mostly;
-
-#define P2M_PER_PAGE            (PAGE_SIZE / sizeof(unsigned long))
-#define P2M_MID_PER_PAGE        (PAGE_SIZE / sizeof(unsigned long *))
-#define P2M_TOP_PER_PAGE        (PAGE_SIZE / sizeof(unsigned long **))
-
-#define MAX_P2M_PFN             (P2M_TOP_PER_PAGE * P2M_MID_PER_PAGE * P2M_PER_PAGE)
-
-/* Placeholders for holes in the address space */
-static RESERVE_BRK_ARRAY(unsigned long, p2m_missing, P2M_PER_PAGE);
-static RESERVE_BRK_ARRAY(unsigned long *, p2m_mid_missing, P2M_MID_PER_PAGE);
-static RESERVE_BRK_ARRAY(unsigned long, p2m_mid_missing_mfn, P2M_MID_PER_PAGE);
-
-static RESERVE_BRK_ARRAY(unsigned long **, p2m_top, P2M_TOP_PER_PAGE);
-static RESERVE_BRK_ARRAY(unsigned long, p2m_top_mfn, P2M_TOP_PER_PAGE);
-static RESERVE_BRK_ARRAY(unsigned long *, p2m_top_mfn_p, P2M_TOP_PER_PAGE);
-
-RESERVE_BRK(p2m_mid, PAGE_SIZE * (MAX_DOMAIN_PAGES / (P2M_PER_PAGE * P2M_MID_PER_PAGE)));
-RESERVE_BRK(p2m_mid_mfn, PAGE_SIZE * (MAX_DOMAIN_PAGES / (P2M_PER_PAGE * P2M_MID_PER_PAGE)));
-
-static inline unsigned p2m_top_index(unsigned long pfn)
-{
-        BUG_ON(pfn >= MAX_P2M_PFN);
-        return pfn / (P2M_MID_PER_PAGE * P2M_PER_PAGE);
-}
-
-static inline unsigned p2m_mid_index(unsigned long pfn)
-{
-        return (pfn / P2M_PER_PAGE) % P2M_MID_PER_PAGE;
-}
-
-static inline unsigned p2m_index(unsigned long pfn)
-{
-        return pfn % P2M_PER_PAGE;
-}
-
-static void p2m_top_init(unsigned long ***top)
-{
-        unsigned i;
-
-        for (i = 0; i < P2M_TOP_PER_PAGE; i++)
-                top[i] = p2m_mid_missing;
-}
-
-static void p2m_top_mfn_init(unsigned long *top)
-{
-        unsigned i;
-
-        for (i = 0; i < P2M_TOP_PER_PAGE; i++)
-                top[i] = virt_to_mfn(p2m_mid_missing_mfn);
-}
-
-static void p2m_top_mfn_p_init(unsigned long **top)
-{
-        unsigned i;
-
-        for (i = 0; i < P2M_TOP_PER_PAGE; i++)
-                top[i] = p2m_mid_missing_mfn;
-}
-
-static void p2m_mid_init(unsigned long **mid)
-{
-        unsigned i;
-
-        for (i = 0; i < P2M_MID_PER_PAGE; i++)
-                mid[i] = p2m_missing;
-}
-
-static void p2m_mid_mfn_init(unsigned long *mid)
-{
-        unsigned i;
-
-        for (i = 0; i < P2M_MID_PER_PAGE; i++)
-                mid[i] = virt_to_mfn(p2m_missing);
-}
-
-static void p2m_init(unsigned long *p2m)
-{
-        unsigned i;
-
-        for (i = 0; i < P2M_MID_PER_PAGE; i++)
-                p2m[i] = INVALID_P2M_ENTRY;
-}
-
-/*
- * Build the parallel p2m_top_mfn and p2m_mid_mfn structures
- *
- * This is called both at boot time, and after resuming from suspend:
- * - At boot time we're called very early, and must use extend_brk()
- *   to allocate memory.
- *
- * - After resume we're called from within stop_machine, but the mfn
- *   tree should alreay be completely allocated.
- */
-void xen_build_mfn_list_list(void)
-{
-        unsigned long pfn;
-
-        /* Pre-initialize p2m_top_mfn to be completely missing */
-        if (p2m_top_mfn == NULL) {
-                p2m_mid_missing_mfn = extend_brk(PAGE_SIZE, PAGE_SIZE);
-                p2m_mid_mfn_init(p2m_mid_missing_mfn);
-
-                p2m_top_mfn_p = extend_brk(PAGE_SIZE, PAGE_SIZE);
-                p2m_top_mfn_p_init(p2m_top_mfn_p);
-
-                p2m_top_mfn = extend_brk(PAGE_SIZE, PAGE_SIZE);
-                p2m_top_mfn_init(p2m_top_mfn);
-        } else {
-                /* Reinitialise, mfn's all change after migration */
-                p2m_mid_mfn_init(p2m_mid_missing_mfn);
-        }
-
-        for (pfn = 0; pfn < xen_max_p2m_pfn; pfn += P2M_PER_PAGE) {
-                unsigned topidx = p2m_top_index(pfn);
-                unsigned mididx = p2m_mid_index(pfn);
-                unsigned long **mid;
-                unsigned long *mid_mfn_p;
-
-                mid = p2m_top[topidx];
-                mid_mfn_p = p2m_top_mfn_p[topidx];
-
-                /* Don't bother allocating any mfn mid levels if
-                 * they're just missing, just update the stored mfn,
-                 * since all could have changed over a migrate.
-                 */
-                if (mid == p2m_mid_missing) {
-                        BUG_ON(mididx);
-                        BUG_ON(mid_mfn_p != p2m_mid_missing_mfn);
-                        p2m_top_mfn[topidx] = virt_to_mfn(p2m_mid_missing_mfn);
-                        pfn += (P2M_MID_PER_PAGE - 1) * P2M_PER_PAGE;
-                        continue;
-                }
-
-                if (mid_mfn_p == p2m_mid_missing_mfn) {
-                        /*
-                         * XXX boot-time only!  We should never find
-                         * missing parts of the mfn tree after
-                         * runtime.  extend_brk() will BUG if we call
-                         * it too late.
-                         */
-                        mid_mfn_p = extend_brk(PAGE_SIZE, PAGE_SIZE);
-                        p2m_mid_mfn_init(mid_mfn_p);
-
-                        p2m_top_mfn_p[topidx] = mid_mfn_p;
-                }
-
-                p2m_top_mfn[topidx] = virt_to_mfn(mid_mfn_p);
-                mid_mfn_p[mididx] = virt_to_mfn(mid[mididx]);
-        }
-}
-
-void xen_setup_mfn_list_list(void)
-{
-        BUG_ON(HYPERVISOR_shared_info == &xen_dummy_shared_info);
-
-        HYPERVISOR_shared_info->arch.pfn_to_mfn_frame_list_list =
-                virt_to_mfn(p2m_top_mfn);
-        HYPERVISOR_shared_info->arch.max_pfn = xen_max_p2m_pfn;
-}
-
-/* Set up p2m_top to point to the domain-builder provided p2m pages */
-void __init xen_build_dynamic_phys_to_machine(void)
-{
-        unsigned long *mfn_list = (unsigned long *)xen_start_info->mfn_list;
-        unsigned long max_pfn = min(MAX_DOMAIN_PAGES, xen_start_info->nr_pages);
-        unsigned long pfn;
-
-        xen_max_p2m_pfn = max_pfn;
-
-        p2m_missing = extend_brk(PAGE_SIZE, PAGE_SIZE);
-        p2m_init(p2m_missing);
-
-        p2m_mid_missing = extend_brk(PAGE_SIZE, PAGE_SIZE);
-        p2m_mid_init(p2m_mid_missing);
-
-        p2m_top = extend_brk(PAGE_SIZE, PAGE_SIZE);
-        p2m_top_init(p2m_top);
-
-        /*
-         * The domain builder gives us a pre-constructed p2m array in
-         * mfn_list for all the pages initially given to us, so we just
-         * need to graft that into our tree structure.
-         */
-        for (pfn = 0; pfn < max_pfn; pfn += P2M_PER_PAGE) {
-                unsigned topidx = p2m_top_index(pfn);
-                unsigned mididx = p2m_mid_index(pfn);
-
-                if (p2m_top[topidx] == p2m_mid_missing) {
-                        unsigned long **mid = extend_brk(PAGE_SIZE, PAGE_SIZE);
-                        p2m_mid_init(mid);
-
-                        p2m_top[topidx] = mid;
-                }
-
-                p2m_top[topidx][mididx] = &mfn_list[pfn];
-        }
-}
-
-unsigned long get_phys_to_machine(unsigned long pfn)
-{
-        unsigned topidx, mididx, idx;
-
-        if (unlikely(pfn >= MAX_P2M_PFN))
-                return INVALID_P2M_ENTRY;
-
-        topidx = p2m_top_index(pfn);
-        mididx = p2m_mid_index(pfn);
-        idx = p2m_index(pfn);
-
-        return p2m_top[topidx][mididx][idx];
-}
-EXPORT_SYMBOL_GPL(get_phys_to_machine);
-
-static void *alloc_p2m_page(void)
-{
-        return (void *)__get_free_page(GFP_KERNEL | __GFP_REPEAT);
-}
-
-static void free_p2m_page(void *p)
-{
-        free_page((unsigned long)p);
-}
-
-/* 
- * Fully allocate the p2m structure for a given pfn.  We need to check
- * that both the top and mid levels are allocated, and make sure the
- * parallel mfn tree is kept in sync.  We may race with other cpus, so
- * the new pages are installed with cmpxchg; if we lose the race then
- * simply free the page we allocated and use the one that's there.
- */
-static bool alloc_p2m(unsigned long pfn)
-{
-        unsigned topidx, mididx;
-        unsigned long ***top_p, **mid;
-        unsigned long *top_mfn_p, *mid_mfn;
-
-        topidx = p2m_top_index(pfn);
-        mididx = p2m_mid_index(pfn);
-
-        top_p = &p2m_top[topidx];
-        mid = *top_p;
-
-        if (mid == p2m_mid_missing) {
-                /* Mid level is missing, allocate a new one */
-                mid = alloc_p2m_page();
-                if (!mid)
-                        return false;
-
-                p2m_mid_init(mid);
-
-                if (cmpxchg(top_p, p2m_mid_missing, mid) != p2m_mid_missing)
-                        free_p2m_page(mid);
-        }
-
-        top_mfn_p = &p2m_top_mfn[topidx];
-        mid_mfn = p2m_top_mfn_p[topidx];
-
-        BUG_ON(virt_to_mfn(mid_mfn) != *top_mfn_p);
-
-        if (mid_mfn == p2m_mid_missing_mfn) {
-                /* Separately check the mid mfn level */
-                unsigned long missing_mfn;
-                unsigned long mid_mfn_mfn;
-
-                mid_mfn = alloc_p2m_page();
-                if (!mid_mfn)
-                        return false;
-
-                p2m_mid_mfn_init(mid_mfn);
-
-                missing_mfn = virt_to_mfn(p2m_mid_missing_mfn);
-                mid_mfn_mfn = virt_to_mfn(mid_mfn);
-                if (cmpxchg(top_mfn_p, missing_mfn, mid_mfn_mfn) != missing_mfn)
-                        free_p2m_page(mid_mfn);
-                else
-                        p2m_top_mfn_p[topidx] = mid_mfn;
-        }
-
-        if (p2m_top[topidx][mididx] == p2m_missing) {
-                /* p2m leaf page is missing */
-                unsigned long *p2m;
-
-                p2m = alloc_p2m_page();
-                if (!p2m)
-                        return false;
-
-                p2m_init(p2m);
-
-                if (cmpxchg(&mid[mididx], p2m_missing, p2m) != p2m_missing)
-                        free_p2m_page(p2m);
-                else
-                        mid_mfn[mididx] = virt_to_mfn(p2m);
-        }
-
-        return true;
-}
-
-/* Try to install p2m mapping; fail if intermediate bits missing */
-bool __set_phys_to_machine(unsigned long pfn, unsigned long mfn)
-{
-        unsigned topidx, mididx, idx;
-
-        if (unlikely(pfn >= MAX_P2M_PFN)) {
-                BUG_ON(mfn != INVALID_P2M_ENTRY);
-                return true;
-        }
-
-        topidx = p2m_top_index(pfn);
-        mididx = p2m_mid_index(pfn);
-        idx = p2m_index(pfn);
-
-        if (p2m_top[topidx][mididx] == p2m_missing)
-                return mfn == INVALID_P2M_ENTRY;
-
-        p2m_top[topidx][mididx][idx] = mfn;
-
-        return true;
-}
-
-bool set_phys_to_machine(unsigned long pfn, unsigned long mfn)
-{
-        if (unlikely(xen_feature(XENFEAT_auto_translated_physmap))) {
-                BUG_ON(pfn != mfn && mfn != INVALID_P2M_ENTRY);
-                return true;
-        }
-
-        if (unlikely(!__set_phys_to_machine(pfn, mfn)))  {
-                if (!alloc_p2m(pfn))
-                        return false;
-
-                if (!__set_phys_to_machine(pfn, mfn))
-                        return false;
-        }
-
-        return true;
-}
-
 unsigned long arbitrary_virt_to_mfn(void *vaddr)
 {
         xmaddr_t maddr = arbitrary_virt_to_machine(vaddr);
diff --git a/arch/x86/xen/p2m.c b/arch/x86/xen/p2m.c
new file mode 100644
index 000000000000..8f2251d2a3f8
--- /dev/null
+++ b/arch/x86/xen/p2m.c
@@ -0,0 +1,510 @@
+/*
+ * Xen leaves the responsibility for maintaining p2m mappings to the
+ * guests themselves, but it must also access and update the p2m array
+ * during suspend/resume when all the pages are reallocated.
+ *
+ * The p2m table is logically a flat array, but we implement it as a
+ * three-level tree to allow the address space to be sparse.
+ *
+ *                               Xen
+ *                                |
+ *     p2m_top              p2m_top_mfn
+ *       /  \                   /   \
+ * p2m_mid p2m_mid      p2m_mid_mfn p2m_mid_mfn
+ *    / \      / \         /           /
+ *  p2m p2m p2m p2m p2m p2m p2m ...
+ *
+ * The p2m_mid_mfn pages are mapped by p2m_top_mfn_p.
+ *
+ * The p2m_top and p2m_top_mfn levels are limited to 1 page, so the
+ * maximum representable pseudo-physical address space is:
+ *  P2M_TOP_PER_PAGE * P2M_MID_PER_PAGE * P2M_PER_PAGE pages
+ *
+ * P2M_PER_PAGE depends on the architecture, as a mfn is always
+ * unsigned long (8 bytes on 64-bit, 4 bytes on 32), leading to
+ * 512 and 1024 entries respectively. 
+ */
+
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/list.h>
+#include <linux/hash.h>
+#include <linux/sched.h>
+
+#include <asm/cache.h>
+#include <asm/setup.h>
+
+#include <asm/xen/page.h>
+#include <asm/xen/hypercall.h>
+#include <asm/xen/hypervisor.h>
+
+#include "xen-ops.h"
+
+static void __init m2p_override_init(void);
+
+unsigned long xen_max_p2m_pfn __read_mostly;
+
+#define P2M_PER_PAGE            (PAGE_SIZE / sizeof(unsigned long))
+#define P2M_MID_PER_PAGE        (PAGE_SIZE / sizeof(unsigned long *))
+#define P2M_TOP_PER_PAGE        (PAGE_SIZE / sizeof(unsigned long **))
+
+#define MAX_P2M_PFN             (P2M_TOP_PER_PAGE * P2M_MID_PER_PAGE * P2M_PER_PAGE)
+
+/* Placeholders for holes in the address space */
+static RESERVE_BRK_ARRAY(unsigned long, p2m_missing, P2M_PER_PAGE);
+static RESERVE_BRK_ARRAY(unsigned long *, p2m_mid_missing, P2M_MID_PER_PAGE);
+static RESERVE_BRK_ARRAY(unsigned long, p2m_mid_missing_mfn, P2M_MID_PER_PAGE);
+
+static RESERVE_BRK_ARRAY(unsigned long **, p2m_top, P2M_TOP_PER_PAGE);
+static RESERVE_BRK_ARRAY(unsigned long, p2m_top_mfn, P2M_TOP_PER_PAGE);
+static RESERVE_BRK_ARRAY(unsigned long *, p2m_top_mfn_p, P2M_TOP_PER_PAGE);
+
+RESERVE_BRK(p2m_mid, PAGE_SIZE * (MAX_DOMAIN_PAGES / (P2M_PER_PAGE * P2M_MID_PER_PAGE)));
+RESERVE_BRK(p2m_mid_mfn, PAGE_SIZE * (MAX_DOMAIN_PAGES / (P2M_PER_PAGE * P2M_MID_PER_PAGE)));
+
+static inline unsigned p2m_top_index(unsigned long pfn)
+{
+        BUG_ON(pfn >= MAX_P2M_PFN);
+        return pfn / (P2M_MID_PER_PAGE * P2M_PER_PAGE);
+}
+
+static inline unsigned p2m_mid_index(unsigned long pfn)
+{
+        return (pfn / P2M_PER_PAGE) % P2M_MID_PER_PAGE;
+}
+
+static inline unsigned p2m_index(unsigned long pfn)
+{
+        return pfn % P2M_PER_PAGE;
+}
+
+static void p2m_top_init(unsigned long ***top)
+{
+        unsigned i;
+
+        for (i = 0; i < P2M_TOP_PER_PAGE; i++)
+                top[i] = p2m_mid_missing;
+}
+
+static void p2m_top_mfn_init(unsigned long *top)
+{
+        unsigned i;
+
+        for (i = 0; i < P2M_TOP_PER_PAGE; i++)
+                top[i] = virt_to_mfn(p2m_mid_missing_mfn);
+}
+
+static void p2m_top_mfn_p_init(unsigned long **top)
+{
+        unsigned i;
+
+        for (i = 0; i < P2M_TOP_PER_PAGE; i++)
+                top[i] = p2m_mid_missing_mfn;
+}
+
+static void p2m_mid_init(unsigned long **mid)
+{
+        unsigned i;
+
+        for (i = 0; i < P2M_MID_PER_PAGE; i++)
+                mid[i] = p2m_missing;
+}
+
+static void p2m_mid_mfn_init(unsigned long *mid)
+{
+        unsigned i;
+
+        for (i = 0; i < P2M_MID_PER_PAGE; i++)
+                mid[i] = virt_to_mfn(p2m_missing);
+}
+
+static void p2m_init(unsigned long *p2m)
+{
+        unsigned i;
+
+        for (i = 0; i < P2M_MID_PER_PAGE; i++)
+                p2m[i] = INVALID_P2M_ENTRY;
+}
+
+/*
+ * Build the parallel p2m_top_mfn and p2m_mid_mfn structures
+ *
+ * This is called both at boot time, and after resuming from suspend:
+ * - At boot time we're called very early, and must use extend_brk()
+ *   to allocate memory.
+ *
+ * - After resume we're called from within stop_machine, but the mfn
+ *   tree should alreay be completely allocated.
+ */
+void xen_build_mfn_list_list(void)
+{
+        unsigned long pfn;
+
+        /* Pre-initialize p2m_top_mfn to be completely missing */
+        if (p2m_top_mfn == NULL) {
+                p2m_mid_missing_mfn = extend_brk(PAGE_SIZE, PAGE_SIZE);
+                p2m_mid_mfn_init(p2m_mid_missing_mfn);
+
+                p2m_top_mfn_p = extend_brk(PAGE_SIZE, PAGE_SIZE);
+                p2m_top_mfn_p_init(p2m_top_mfn_p);
+
+                p2m_top_mfn = extend_brk(PAGE_SIZE, PAGE_SIZE);
+                p2m_top_mfn_init(p2m_top_mfn);
+        } else {
+                /* Reinitialise, mfn's all change after migration */
+                p2m_mid_mfn_init(p2m_mid_missing_mfn);
+        }
+
+        for (pfn = 0; pfn < xen_max_p2m_pfn; pfn += P2M_PER_PAGE) {
+                unsigned topidx = p2m_top_index(pfn);
+                unsigned mididx = p2m_mid_index(pfn);
+                unsigned long **mid;
+                unsigned long *mid_mfn_p;
+
+                mid = p2m_top[topidx];
+                mid_mfn_p = p2m_top_mfn_p[topidx];
+
+                /* Don't bother allocating any mfn mid levels if
+                 * they're just missing, just update the stored mfn,
+                 * since all could have changed over a migrate.
+                 */
+                if (mid == p2m_mid_missing) {
+                        BUG_ON(mididx);
+                        BUG_ON(mid_mfn_p != p2m_mid_missing_mfn);
+                        p2m_top_mfn[topidx] = virt_to_mfn(p2m_mid_missing_mfn);
+                        pfn += (P2M_MID_PER_PAGE - 1) * P2M_PER_PAGE;
+                        continue;
+                }
+
+                if (mid_mfn_p == p2m_mid_missing_mfn) {
+                        /*
+                         * XXX boot-time only!  We should never find
+                         * missing parts of the mfn tree after
+                         * runtime.  extend_brk() will BUG if we call
+                         * it too late.
+                         */
+                        mid_mfn_p = extend_brk(PAGE_SIZE, PAGE_SIZE);
+                        p2m_mid_mfn_init(mid_mfn_p);
+
+                        p2m_top_mfn_p[topidx] = mid_mfn_p;
+                }
+
+                p2m_top_mfn[topidx] = virt_to_mfn(mid_mfn_p);
+                mid_mfn_p[mididx] = virt_to_mfn(mid[mididx]);
+        }
+}
+
+void xen_setup_mfn_list_list(void)
+{
+        BUG_ON(HYPERVISOR_shared_info == &xen_dummy_shared_info);
+
+        HYPERVISOR_shared_info->arch.pfn_to_mfn_frame_list_list =
+                virt_to_mfn(p2m_top_mfn);
+        HYPERVISOR_shared_info->arch.max_pfn = xen_max_p2m_pfn;
+}
+
+/* Set up p2m_top to point to the domain-builder provided p2m pages */
+void __init xen_build_dynamic_phys_to_machine(void)
+{
+        unsigned long *mfn_list = (unsigned long *)xen_start_info->mfn_list;
+        unsigned long max_pfn = min(MAX_DOMAIN_PAGES, xen_start_info->nr_pages);
+        unsigned long pfn;
+
+        xen_max_p2m_pfn = max_pfn;
+
+        p2m_missing = extend_brk(PAGE_SIZE, PAGE_SIZE);
+        p2m_init(p2m_missing);
+
+        p2m_mid_missing = extend_brk(PAGE_SIZE, PAGE_SIZE);
+        p2m_mid_init(p2m_mid_missing);
+
+        p2m_top = extend_brk(PAGE_SIZE, PAGE_SIZE);
+        p2m_top_init(p2m_top);
+
+        /*
+         * The domain builder gives us a pre-constructed p2m array in
+         * mfn_list for all the pages initially given to us, so we just
+         * need to graft that into our tree structure.
+         */
+        for (pfn = 0; pfn < max_pfn; pfn += P2M_PER_PAGE) {
+                unsigned topidx = p2m_top_index(pfn);
+                unsigned mididx = p2m_mid_index(pfn);
+
+                if (p2m_top[topidx] == p2m_mid_missing) {
+                        unsigned long **mid = extend_brk(PAGE_SIZE, PAGE_SIZE);
+                        p2m_mid_init(mid);
+
+                        p2m_top[topidx] = mid;
+                }
+
+                p2m_top[topidx][mididx] = &mfn_list[pfn];
+        }
+
+        m2p_override_init();
+}
+
+unsigned long get_phys_to_machine(unsigned long pfn)
+{
+        unsigned topidx, mididx, idx;
+
+        if (unlikely(pfn >= MAX_P2M_PFN))
+                return INVALID_P2M_ENTRY;
+
+        topidx = p2m_top_index(pfn);
+        mididx = p2m_mid_index(pfn);
+        idx = p2m_index(pfn);
+
+        return p2m_top[topidx][mididx][idx];
+}
+EXPORT_SYMBOL_GPL(get_phys_to_machine);
+
+static void *alloc_p2m_page(void)
+{
+        return (void *)__get_free_page(GFP_KERNEL | __GFP_REPEAT);
+}
+
+static void free_p2m_page(void *p)
+{
+        free_page((unsigned long)p);
+}
+
+/* 
+ * Fully allocate the p2m structure for a given pfn.  We need to check
+ * that both the top and mid levels are allocated, and make sure the
+ * parallel mfn tree is kept in sync.  We may race with other cpus, so
+ * the new pages are installed with cmpxchg; if we lose the race then
+ * simply free the page we allocated and use the one that's there.
+ */
+static bool alloc_p2m(unsigned long pfn)
+{
+        unsigned topidx, mididx;
+        unsigned long ***top_p, **mid;
+        unsigned long *top_mfn_p, *mid_mfn;
+
+        topidx = p2m_top_index(pfn);
+        mididx = p2m_mid_index(pfn);
+
+        top_p = &p2m_top[topidx];
+        mid = *top_p;
+
+        if (mid == p2m_mid_missing) {
+                /* Mid level is missing, allocate a new one */
+                mid = alloc_p2m_page();
+                if (!mid)
+                        return false;
+
+                p2m_mid_init(mid);
+
+                if (cmpxchg(top_p, p2m_mid_missing, mid) != p2m_mid_missing)
+                        free_p2m_page(mid);
+        }
+
+        top_mfn_p = &p2m_top_mfn[topidx];
+        mid_mfn = p2m_top_mfn_p[topidx];
+
+        BUG_ON(virt_to_mfn(mid_mfn) != *top_mfn_p);
+
+        if (mid_mfn == p2m_mid_missing_mfn) {
+                /* Separately check the mid mfn level */
+                unsigned long missing_mfn;
+                unsigned long mid_mfn_mfn;
+
+                mid_mfn = alloc_p2m_page();
+                if (!mid_mfn)
+                        return false;
+
+                p2m_mid_mfn_init(mid_mfn);
+
+                missing_mfn = virt_to_mfn(p2m_mid_missing_mfn);
+                mid_mfn_mfn = virt_to_mfn(mid_mfn);
+                if (cmpxchg(top_mfn_p, missing_mfn, mid_mfn_mfn) != missing_mfn)
+                        free_p2m_page(mid_mfn);
+                else
+                        p2m_top_mfn_p[topidx] = mid_mfn;
+        }
+
+        if (p2m_top[topidx][mididx] == p2m_missing) {
+                /* p2m leaf page is missing */
+                unsigned long *p2m;
+
+                p2m = alloc_p2m_page();
+                if (!p2m)
+                        return false;
+
+                p2m_init(p2m);
+
+                if (cmpxchg(&mid[mididx], p2m_missing, p2m) != p2m_missing)
+                        free_p2m_page(p2m);
+                else
+                        mid_mfn[mididx] = virt_to_mfn(p2m);
+        }
+
+        return true;
+}
+
+/* Try to install p2m mapping; fail if intermediate bits missing */
+bool __set_phys_to_machine(unsigned long pfn, unsigned long mfn)
+{
+        unsigned topidx, mididx, idx;
+
+        if (unlikely(pfn >= MAX_P2M_PFN)) {
+                BUG_ON(mfn != INVALID_P2M_ENTRY);
+                return true;
+        }
+
+        topidx = p2m_top_index(pfn);
+        mididx = p2m_mid_index(pfn);
+        idx = p2m_index(pfn);
+
+        if (p2m_top[topidx][mididx] == p2m_missing)
+                return mfn == INVALID_P2M_ENTRY;
+
+        p2m_top[topidx][mididx][idx] = mfn;
+
+        return true;
+}
+
+bool set_phys_to_machine(unsigned long pfn, unsigned long mfn)
+{
+        if (unlikely(xen_feature(XENFEAT_auto_translated_physmap))) {
+                BUG_ON(pfn != mfn && mfn != INVALID_P2M_ENTRY);
+                return true;
+        }
+
+        if (unlikely(!__set_phys_to_machine(pfn, mfn)))  {
+                if (!alloc_p2m(pfn))
+                        return false;
+
+                if (!__set_phys_to_machine(pfn, mfn))
+                        return false;
+        }
+
+        return true;
+}
+
+#define M2P_OVERRIDE_HASH_SHIFT 10
+#define M2P_OVERRIDE_HASH       (1 << M2P_OVERRIDE_HASH_SHIFT)
+
+static RESERVE_BRK_ARRAY(struct list_head, m2p_overrides, M2P_OVERRIDE_HASH);
+static DEFINE_SPINLOCK(m2p_override_lock);
+
+static void __init m2p_override_init(void)
+{
+        unsigned i;
+
+        m2p_overrides = extend_brk(sizeof(*m2p_overrides) * M2P_OVERRIDE_HASH,
+                                   sizeof(unsigned long));
+
+        for (i = 0; i < M2P_OVERRIDE_HASH; i++)
+                INIT_LIST_HEAD(&m2p_overrides[i]);
+}
+
+static unsigned long mfn_hash(unsigned long mfn)
+{
+        return hash_long(mfn, M2P_OVERRIDE_HASH_SHIFT);
+}
+
+/* Add an MFN override for a particular page */
+int m2p_add_override(unsigned long mfn, struct page *page)
+{
+        unsigned long flags;
+        unsigned long pfn;
+        unsigned long address;
+        unsigned level;
+        pte_t *ptep = NULL;
+
+        pfn = page_to_pfn(page);
+        if (!PageHighMem(page)) {
+                address = (unsigned long)__va(pfn << PAGE_SHIFT);
+                ptep = lookup_address(address, &level);
+
+                if (WARN(ptep == NULL || level != PG_LEVEL_4K,
+                                        "m2p_add_override: pfn %lx not mapped", pfn))
+                        return -EINVAL;
+        }
+
+        page->private = mfn;
+        page->index = pfn_to_mfn(pfn);
+
+        __set_phys_to_machine(pfn, FOREIGN_FRAME(mfn));
+        if (!PageHighMem(page))
+                /* Just zap old mapping for now */
+                pte_clear(&init_mm, address, ptep);
+
+        spin_lock_irqsave(&m2p_override_lock, flags);
+        list_add(&page->lru,  &m2p_overrides[mfn_hash(mfn)]);
+        spin_unlock_irqrestore(&m2p_override_lock, flags);
+
+        return 0;
+}
+
+int m2p_remove_override(struct page *page)
+{
+        unsigned long flags;
+        unsigned long mfn;
+        unsigned long pfn;
+        unsigned long address;
+        unsigned level;
+        pte_t *ptep = NULL;
+
+        pfn = page_to_pfn(page);
+        mfn = get_phys_to_machine(pfn);
+        if (mfn == INVALID_P2M_ENTRY || !(mfn & FOREIGN_FRAME_BIT))
+                return -EINVAL;
+
+        if (!PageHighMem(page)) {
+                address = (unsigned long)__va(pfn << PAGE_SHIFT);
+                ptep = lookup_address(address, &level);
+
+                if (WARN(ptep == NULL || level != PG_LEVEL_4K,
+                                        "m2p_remove_override: pfn %lx not mapped", pfn))
+                        return -EINVAL;
+        }
+
+        spin_lock_irqsave(&m2p_override_lock, flags);
+        list_del(&page->lru);
+        spin_unlock_irqrestore(&m2p_override_lock, flags);
+        __set_phys_to_machine(pfn, page->index);
+
+        if (!PageHighMem(page))
+                set_pte_at(&init_mm, address, ptep,
+                                pfn_pte(pfn, PAGE_KERNEL));
+                /* No tlb flush necessary because the caller already
+                 * left the pte unmapped. */
+
+        return 0;
+}
+
+struct page *m2p_find_override(unsigned long mfn)
+{
+        unsigned long flags;
+        struct list_head *bucket = &m2p_overrides[mfn_hash(mfn)];
+        struct page *p, *ret;
+
+        ret = NULL;
+
+        spin_lock_irqsave(&m2p_override_lock, flags);
+
+        list_for_each_entry(p, bucket, lru) {
+                if (p->private == mfn) {
+                        ret = p;
+                        break;
+                }
+        }
+
+        spin_unlock_irqrestore(&m2p_override_lock, flags);
+
+        return ret;
+}
+
+unsigned long m2p_find_override_pfn(unsigned long mfn, unsigned long pfn)
+{
+        struct page *p = m2p_find_override(mfn);
+        unsigned long ret = pfn;
+
+        if (p)
+                ret = page_to_pfn(p);
+
+        return ret;
+}
+EXPORT_SYMBOL_GPL(m2p_find_override_pfn);
diff --git a/arch/xtensa/include/asm/mman.h b/arch/xtensa/include/asm/mman.h
index fca4db425f6e..30789010733d 100644
--- a/arch/xtensa/include/asm/mman.h
+++ b/arch/xtensa/include/asm/mman.h
@@ -83,6 +83,9 @@
 #define MADV_MERGEABLE   12             /* KSM may merge identical pages */
 #define MADV_UNMERGEABLE 13             /* KSM may not merge identical pages */
 
+#define MADV_HUGEPAGE   14              /* Worth backing with hugepages */
+#define MADV_NOHUGEPAGE 15              /* Not worth backing with hugepages */
+
 /* compatibility flags */
 #define MAP_FILE        0