149 files changed, 4914 insertions, 2676 deletions
diff --git a/arch/arm/boot/dts/Makefile b/arch/arm/boot/dts/Makefile
index 0f441740c22a..ca6fb8e7f17d 100644
--- a/arch/arm/boot/dts/Makefile
+++ b/arch/arm/boot/dts/Makefile
@@ -131,8 +131,8 @@ dtb-$(CONFIG_ARCH_SPEAR3XX)+= spear300-evb.dtb \
        spear320-evb.dtb \
        spear320-hmi.dtb
 dtb-$(CONFIG_ARCH_SPEAR6XX)+= spear600-evb.dtb
-dtb-$(CONFIG_ARCH_SUNXI) += sun4i-cubieboard.dtb \
+dtb-$(CONFIG_ARCH_SUNXI) += sun4i-a10-cubieboard.dtb \
-        sun5i-olinuxino.dtb
+        sun5i-a13-olinuxino.dtb
 dtb-$(CONFIG_ARCH_TEGRA) += tegra20-harmony.dtb \
        tegra20-medcom-wide.dtb \
        tegra20-paz00.dtb \
diff --git a/arch/arm/boot/dts/sun4i-cubieboard.dts b/arch/arm/boot/dts/sun4i-cubieboard.dts
index f4ca126ad994..5cab82540437 100644
--- a/arch/arm/boot/dts/sun4i-cubieboard.dts
+++ b/arch/arm/boot/dts/sun4i-cubieboard.dts
@@ -11,11 +11,11 @@
 */
 /dts-v1/;
-/include/ "sun4i.dtsi"
+/include/ "sun4i-a10.dtsi"
 / {
        model = "Cubietech Cubieboard";
-        compatible = "cubietech,cubieboard", "allwinner,sun4i";
+        compatible = "cubietech,a10-cubieboard", "allwinner,sun4i-a10";
        aliases {
                serial0 = &uart0;
diff --git a/arch/arm/boot/dts/sun5i-olinuxino.dts b/arch/arm/boot/dts/sun5i-olinuxino.dts
index d6ff889a5d87..498a091a4ea2 100644
--- a/arch/arm/boot/dts/sun5i-olinuxino.dts
+++ b/arch/arm/boot/dts/sun5i-olinuxino.dts
@@ -12,11 +12,11 @@
 */
 /dts-v1/;
-/include/ "sun5i.dtsi"
+/include/ "sun5i-a13.dtsi"
 / {
        model = "Olimex A13-Olinuxino";
-        compatible = "olimex,a13-olinuxino", "allwinner,sun5i";
+        compatible = "olimex,a13-olinuxino", "allwinner,sun5i-a13";
        chosen {
                bootargs = "earlyprintk console=ttyS0,115200";
diff --git a/arch/arm/mach-davinci/board-da850-evm.c b/arch/arm/mach-davinci/board-da850-evm.c
index 7211772edd9d..0299915575a8 100644
--- a/arch/arm/mach-davinci/board-da850-evm.c
+++ b/arch/arm/mach-davinci/board-da850-evm.c
@@ -41,6 +41,7 @@
 #include <mach/cp_intc.h>
 #include <mach/da8xx.h>
 #include <mach/mux.h>
+#include <mach/sram.h>
 #include <asm/mach-types.h>
 #include <asm/mach/arch.h>
diff --git a/arch/arm/mach-davinci/da830.c b/arch/arm/mach-davinci/da830.c
index 510648e0394b..678a54a64dae 100644
--- a/arch/arm/mach-davinci/da830.c
+++ b/arch/arm/mach-davinci/da830.c
@@ -408,7 +408,7 @@ static struct clk_lookup da830_clks[] = {
        CLK(NULL,               "pwm2",         &pwm2_clk),
        CLK("eqep.0",           NULL,           &eqep0_clk),
        CLK("eqep.1",           NULL,           &eqep1_clk),
-        CLK("da8xx_lcdc.0",     NULL,           &lcdc_clk),
+        CLK("da8xx_lcdc.0",     "fck",          &lcdc_clk),
        CLK("davinci-mcasp.0",  NULL,           &mcasp0_clk),
        CLK("davinci-mcasp.1",  NULL,           &mcasp1_clk),
        CLK("davinci-mcasp.2",  NULL,           &mcasp2_clk),
diff --git a/arch/arm/mach-davinci/da850.c b/arch/arm/mach-davinci/da850.c
index 68c5fe01857c..6b9154e9f908 100644
--- a/arch/arm/mach-davinci/da850.c
+++ b/arch/arm/mach-davinci/da850.c
@@ -403,7 +403,7 @@ static struct clk_lookup da850_clks[] = {
        CLK(NULL,               "rmii",         &rmii_clk),
        CLK("davinci_emac.1",   NULL,           &emac_clk),
        CLK("davinci-mcasp.0",  NULL,           &mcasp_clk),
-        CLK("da8xx_lcdc.0",     NULL,           &lcdc_clk),
+        CLK("da8xx_lcdc.0",     "fck",          &lcdc_clk),
        CLK("davinci_mmc.0",    NULL,           &mmcsd0_clk),
        CLK("davinci_mmc.1",    NULL,           &mmcsd1_clk),
        CLK(NULL,               "aemif",        &aemif_clk),
diff --git a/arch/arm/mach-davinci/devices-da8xx.c b/arch/arm/mach-davinci/devices-da8xx.c
index 46c9a0c09ae5..fcdbe437409e 100644
--- a/arch/arm/mach-davinci/devices-da8xx.c
+++ b/arch/arm/mach-davinci/devices-da8xx.c
@@ -589,29 +589,9 @@ int __init da8xx_register_uio_pruss(void)
        return platform_device_register(&da8xx_uio_pruss_dev);
 }
-static const struct display_panel disp_panel = {
-        QVGA,
-        16,
-        16,
-        COLOR_ACTIVE,
-};
 static struct lcd_ctrl_config lcd_cfg = {
-        &disp_panel,
+        .panel_shade            = COLOR_ACTIVE,
-        .ac_bias                = 255,
-        .ac_bias_intrpt         = 0,
-        .dma_burst_sz           = 16,
        .bpp                    = 16,
-        .fdd                    = 255,
-        .tft_alt_mode           = 0,
-        .stn_565_mode           = 0,
-        .mono_8bit_mode         = 0,
-        .invert_line_clock      = 1,
-        .invert_frm_clock       = 1,
-        .sync_edge              = 0,
-        .sync_ctrl              = 1,
-        .raster_order           = 0,
-        .fifo_th                = 6,
 };
 struct da8xx_lcdc_platform_data sharp_lcd035q3dg01_pdata = {
diff --git a/arch/arm/mach-davinci/pm_domain.c b/arch/arm/mach-davinci/pm_domain.c
index 00946e23c1ee..c90250e3bef8 100644
--- a/arch/arm/mach-davinci/pm_domain.c
+++ b/arch/arm/mach-davinci/pm_domain.c
@@ -53,6 +53,7 @@ static struct dev_pm_domain davinci_pm_domain = {
 static struct pm_clk_notifier_block platform_bus_notifier = {
        .pm_domain = &davinci_pm_domain,
+        .con_ids = { "fck", NULL, },
 };
 static int __init davinci_pm_runtime_init(void)
diff --git a/arch/arm/mach-exynos/common.h b/arch/arm/mach-exynos/common.h
index dac146df79ac..04744f9c120f 100644
--- a/arch/arm/mach-exynos/common.h
+++ b/arch/arm/mach-exynos/common.h
@@ -25,7 +25,7 @@ void exynos_init_late(void);
 #ifdef CONFIG_PM_GENERIC_DOMAINS
 int exynos_pm_late_initcall(void);
 #else
-static int exynos_pm_late_initcall(void) { return 0; }
+static inline int exynos_pm_late_initcall(void) { return 0; }
 #endif
 #ifdef CONFIG_ARCH_EXYNOS4
diff --git a/arch/arm/mach-omap2/board-rx51-video.c b/arch/arm/mach-omap2/board-rx51-video.c
index c22e111bcd00..46f4fc982766 100644
--- a/arch/arm/mach-omap2/board-rx51-video.c
+++ b/arch/arm/mach-omap2/board-rx51-video.c
@@ -16,7 +16,6 @@
 #include <linux/mm.h>
 #include <asm/mach-types.h>
 #include <video/omapdss.h>
-#include <plat/vram.h>
 #include <linux/platform_data/spi-omap2-mcspi.h>
 #include "board-rx51.h"
@@ -87,17 +86,4 @@ static int __init rx51_video_init(void)
 }
 subsys_initcall(rx51_video_init);
-void __init rx51_video_mem_init(void)
-{
-        /*
-         * GFX 864x480x32bpp
-         * VID1/2 1280x720x32bpp double buffered
-         */
-        omap_vram_set_sdram_vram(PAGE_ALIGN(864 * 480 * 4) +
-                        2 * PAGE_ALIGN(1280 * 720 * 4 * 2), 0);
-}
-#else
-void __init rx51_video_mem_init(void) { }
 #endif /* defined(CONFIG_FB_OMAP2) || defined(CONFIG_FB_OMAP2_MODULE) */
diff --git a/arch/arm/mach-omap2/board-rx51.c b/arch/arm/mach-omap2/board-rx51.c
index f1d6efe079ca..d0374ea2dfb0 100644
--- a/arch/arm/mach-omap2/board-rx51.c
+++ b/arch/arm/mach-omap2/board-rx51.c
@@ -34,8 +34,6 @@
 #define RX51_GPIO_SLEEP_IND 162
-extern void rx51_video_mem_init(void);
 static struct gpio_led gpio_leds[] = {
        {
                .name   = "sleep_ind",
@@ -112,7 +110,6 @@ static void __init rx51_init(void)
 static void __init rx51_reserve(void)
 {
-        rx51_video_mem_init();
        omap_reserve();
 }
diff --git a/arch/arm/mach-omap2/display.c b/arch/arm/mach-omap2/display.c
index 38ba58c97628..cc75aaf6e764 100644
--- a/arch/arm/mach-omap2/display.c
+++ b/arch/arm/mach-omap2/display.c
@@ -102,17 +102,20 @@ static const struct omap_dss_hwmod_data omap4_dss_hwmod_data[] __initconst = {
        { "dss_hdmi", "omapdss_hdmi", -1 },
 };
-static void __init omap4_hdmi_mux_pads(enum omap_hdmi_flags flags)
+static void __init omap4_tpd12s015_mux_pads(void)
 {
-        u32 reg;
-        u16 control_i2c_1;
        omap_mux_init_signal("hdmi_cec",
                        OMAP_PIN_INPUT_PULLUP);
        omap_mux_init_signal("hdmi_ddc_scl",
                        OMAP_PIN_INPUT_PULLUP);
        omap_mux_init_signal("hdmi_ddc_sda",
                        OMAP_PIN_INPUT_PULLUP);
+}
+static void __init omap4_hdmi_mux_pads(enum omap_hdmi_flags flags)
+{
+        u32 reg;
+        u16 control_i2c_1;
        /*
         * CONTROL_I2C_1: HDMI_DDC_SDA_PULLUPRESX (bit 28) and
@@ -163,8 +166,10 @@ static int omap4_dsi_mux_pads(int dsi_id, unsigned lanes)
 int __init omap_hdmi_init(enum omap_hdmi_flags flags)
 {
-        if (cpu_is_omap44xx())
+        if (cpu_is_omap44xx()) {
                omap4_hdmi_mux_pads(flags);
+                omap4_tpd12s015_mux_pads();
+        }
        return 0;
 }
diff --git a/arch/arm/mach-shmobile/board-ag5evm.c b/arch/arm/mach-shmobile/board-ag5evm.c
index 25eb88a923e6..032d10817e79 100644
--- a/arch/arm/mach-shmobile/board-ag5evm.c
+++ b/arch/arm/mach-shmobile/board-ag5evm.c
@@ -213,95 +213,6 @@ static struct platform_device irda_device = {
        .num_resources  = ARRAY_SIZE(irda_resources),
 };
-static unsigned char lcd_backlight_seq[3][2] = {
-        { 0x04, 0x07 },
-        { 0x23, 0x80 },
-        { 0x03, 0x01 },
-};
-static void lcd_backlight_on(void)
-{
-        struct i2c_adapter *a;
-        struct i2c_msg msg;
-        int k;
-        a = i2c_get_adapter(1);
-        for (k = 0; a && k < 3; k++) {
-                msg.addr = 0x6d;
-                msg.buf = &lcd_backlight_seq[k][0];
-                msg.len = 2;
-                msg.flags = 0;
-                if (i2c_transfer(a, &msg, 1) != 1)
-                        break;
-        }
-}
-static void lcd_backlight_reset(void)
-{
-        gpio_set_value(GPIO_PORT235, 0);
-        mdelay(24);
-        gpio_set_value(GPIO_PORT235, 1);
-}
-/* LCDC0 */
-static const struct fb_videomode lcdc0_modes[] = {
-        {
-                .name           = "R63302(QHD)",
-                .xres           = 544,
-                .yres           = 961,
-                .left_margin    = 72,
-                .right_margin   = 600,
-                .hsync_len      = 16,
-                .upper_margin   = 8,
-                .lower_margin   = 8,
-                .vsync_len      = 2,
-                .sync           = FB_SYNC_VERT_HIGH_ACT | FB_SYNC_HOR_HIGH_ACT,
-        },
-};
-static struct sh_mobile_lcdc_info lcdc0_info = {
-        .clock_source = LCDC_CLK_PERIPHERAL,
-        .ch[0] = {
-                .chan = LCDC_CHAN_MAINLCD,
-                .interface_type = RGB24,
-                .clock_divider = 1,
-                .flags = LCDC_FLAGS_DWPOL,
-                .fourcc = V4L2_PIX_FMT_RGB565,
-                .lcd_modes = lcdc0_modes,
-                .num_modes = ARRAY_SIZE(lcdc0_modes),
-                .panel_cfg = {
-                        .width = 44,
-                        .height = 79,
-                        .display_on = lcd_backlight_on,
-                        .display_off = lcd_backlight_reset,
-                },
-        }
-};
-static struct resource lcdc0_resources[] = {
-        [0] = {
-                .name   = "LCDC0",
-                .start  = 0xfe940000, /* P4-only space */
-                .end    = 0xfe943fff,
-                .flags  = IORESOURCE_MEM,
-        },
-        [1] = {
-                .start  = intcs_evt2irq(0x580),
-                .flags  = IORESOURCE_IRQ,
-        },
-};
-static struct platform_device lcdc0_device = {
-        .name           = "sh_mobile_lcdc_fb",
-        .num_resources  = ARRAY_SIZE(lcdc0_resources),
-        .resource       = lcdc0_resources,
-        .id             = 0,
-        .dev    = {
-                .platform_data  = &lcdc0_info,
-                .coherent_dma_mask = ~0,
-        },
-};
 /* MIPI-DSI */
 static struct resource mipidsi0_resources[] = {
        [0] = {
@@ -358,7 +269,7 @@ sh_mipi_set_dot_clock_pck_err:
 static struct sh_mipi_dsi_info mipidsi0_info = {
        .data_format    = MIPI_RGB888,
-        .lcd_chan       = &lcdc0_info.ch[0],
+        .channel        = LCDC_CHAN_MAINLCD,
        .lane           = 2,
        .vsynw_offset   = 20,
        .clksrc         = 1,
@@ -378,6 +289,109 @@ static struct platform_device mipidsi0_device = {
        },
 };
+static unsigned char lcd_backlight_seq[3][2] = {
+        { 0x04, 0x07 },
+        { 0x23, 0x80 },
+        { 0x03, 0x01 },
+};
+static int lcd_backlight_set_brightness(int brightness)
+{
+        struct i2c_adapter *adap;
+        struct i2c_msg msg;
+        unsigned int i;
+        int ret;
+        if (brightness == 0) {
+                /* Reset the chip */
+                gpio_set_value(GPIO_PORT235, 0);
+                mdelay(24);
+                gpio_set_value(GPIO_PORT235, 1);
+                return 0;
+        }
+        adap = i2c_get_adapter(1);
+        if (adap == NULL)
+                return -ENODEV;
+        for (i = 0; i < ARRAY_SIZE(lcd_backlight_seq); i++) {
+                msg.addr = 0x6d;
+                msg.buf = &lcd_backlight_seq[i][0];
+                msg.len = 2;
+                msg.flags = 0;
+                ret = i2c_transfer(adap, &msg, 1);
+                if (ret < 0)
+                        break;
+        }
+        i2c_put_adapter(adap);
+        return ret < 0 ? ret : 0;
+}
+/* LCDC0 */
+static const struct fb_videomode lcdc0_modes[] = {
+        {
+                .name           = "R63302(QHD)",
+                .xres           = 544,
+                .yres           = 961,
+                .left_margin    = 72,
+                .right_margin   = 600,
+                .hsync_len      = 16,
+                .upper_margin   = 8,
+                .lower_margin   = 8,
+                .vsync_len      = 2,
+                .sync           = FB_SYNC_VERT_HIGH_ACT | FB_SYNC_HOR_HIGH_ACT,
+        },
+};
+static struct sh_mobile_lcdc_info lcdc0_info = {
+        .clock_source = LCDC_CLK_PERIPHERAL,
+        .ch[0] = {
+                .chan = LCDC_CHAN_MAINLCD,
+                .interface_type = RGB24,
+                .clock_divider = 1,
+                .flags = LCDC_FLAGS_DWPOL,
+                .fourcc = V4L2_PIX_FMT_RGB565,
+                .lcd_modes = lcdc0_modes,
+                .num_modes = ARRAY_SIZE(lcdc0_modes),
+                .panel_cfg = {
+                        .width = 44,
+                        .height = 79,
+                },
+                .bl_info = {
+                        .name = "sh_mobile_lcdc_bl",
+                        .max_brightness = 1,
+                        .set_brightness = lcd_backlight_set_brightness,
+                },
+                .tx_dev = &mipidsi0_device,
+        }
+};
+static struct resource lcdc0_resources[] = {
+        [0] = {
+                .name   = "LCDC0",
+                .start  = 0xfe940000, /* P4-only space */
+                .end    = 0xfe943fff,
+                .flags  = IORESOURCE_MEM,
+        },
+        [1] = {
+                .start  = intcs_evt2irq(0x580),
+                .flags  = IORESOURCE_IRQ,
+        },
+};
+static struct platform_device lcdc0_device = {
+        .name           = "sh_mobile_lcdc_fb",
+        .num_resources  = ARRAY_SIZE(lcdc0_resources),
+        .resource       = lcdc0_resources,
+        .id             = 0,
+        .dev    = {
+                .platform_data  = &lcdc0_info,
+                .coherent_dma_mask = ~0,
+        },
+};
 /* Fixed 2.8V regulators to be used by SDHI0 */
 static struct regulator_consumer_supply fixed2v8_power_consumers[] =
 {
@@ -531,8 +545,8 @@ static struct platform_device *ag5evm_devices[] __initdata = {
        &fsi_device,
        &mmc_device,
        &irda_device,
-        &lcdc0_device,
        &mipidsi0_device,
+        &lcdc0_device,
        &sdhi0_device,
        &sdhi1_device,
 };
@@ -621,7 +635,7 @@ static void __init ag5evm_init(void)
        /* LCD backlight controller */
        gpio_request(GPIO_PORT235, NULL); /* RESET */
        gpio_direction_output(GPIO_PORT235, 0);
-        lcd_backlight_reset();
+        lcd_backlight_set_brightness(0);
        /* enable SDHI0 on CN15 [SD I/F] */
        gpio_request(GPIO_FN_SDHIWP0, NULL);
diff --git a/arch/arm/mach-shmobile/board-ap4evb.c b/arch/arm/mach-shmobile/board-ap4evb.c
index 40657854e3ad..99ef190d0909 100644
--- a/arch/arm/mach-shmobile/board-ap4evb.c
+++ b/arch/arm/mach-shmobile/board-ap4evb.c
@@ -552,11 +552,9 @@ static struct resource mipidsi0_resources[] = {
        },
 };
-static struct sh_mobile_lcdc_info lcdc_info;
 static struct sh_mipi_dsi_info mipidsi0_info = {
        .data_format    = MIPI_RGB888,
-        .lcd_chan       = &lcdc_info.ch[0],
+        .channel        = LCDC_CHAN_MAINLCD,
        .lane           = 2,
        .vsynw_offset   = 17,
        .phyctrl        = 0x6 << 8,
diff --git a/arch/arm/mach-shmobile/board-mackerel.c b/arch/arm/mach-shmobile/board-mackerel.c
index 3f56e70795b7..2fed62f66045 100644
--- a/arch/arm/mach-shmobile/board-mackerel.c
+++ b/arch/arm/mach-shmobile/board-mackerel.c
@@ -370,11 +370,6 @@ static int mackerel_set_brightness(int brightness)
        return 0;
 }
-static int mackerel_get_brightness(void)
-{
-        return gpio_get_value(GPIO_PORT31);
-}
 static const struct sh_mobile_meram_cfg lcd_meram_cfg = {
        .icb[0] = {
                .meram_size     = 0x40,
@@ -403,7 +398,6 @@ static struct sh_mobile_lcdc_info lcdc_info = {
                        .name = "sh_mobile_lcdc_bl",
                        .max_brightness = 1,
                        .set_brightness = mackerel_set_brightness,
-                        .get_brightness = mackerel_get_brightness,
                },
                .meram_cfg = &lcd_meram_cfg,
        }
diff --git a/arch/arm/mach-tegra/common.c b/arch/arm/mach-tegra/common.c
index 0816562725f6..d54cfc54b9fe 100644
--- a/arch/arm/mach-tegra/common.c
+++ b/arch/arm/mach-tegra/common.c
@@ -104,7 +104,7 @@ static __initdata struct tegra_clk_init_table tegra20_clk_init_table[] = {
 static __initdata struct tegra_clk_init_table tegra30_clk_init_table[] = {
        /* name         parent          rate            enabled */
        { "clk_m",      NULL,           0,              true },
-        { "pll_p",      "clk_m",        408000000,      true },
+        { "pll_p",      "pll_ref",      408000000,      true },
        { "pll_p_out1", "pll_p",        9600000,        true },
        { "pll_p_out4", "pll_p",        102000000,      true },
        { "sclk",       "pll_p_out4",   102000000,      true },
diff --git a/arch/arm/mach-tegra/tegra30_clocks.c b/arch/arm/mach-tegra/tegra30_clocks.c
index efc000e32e1c..d7147779f8ea 100644
--- a/arch/arm/mach-tegra/tegra30_clocks.c
+++ b/arch/arm/mach-tegra/tegra30_clocks.c
@@ -2045,9 +2045,7 @@ struct clk_ops tegra30_periph_clk_ops = {
 static int tegra30_dsib_clk_set_parent(struct clk_hw *hw, u8 index)
 {
        struct clk *d = clk_get_sys(NULL, "pll_d");
-        /* The DSIB parent selection bit is in PLLD base
+        /* The DSIB parent selection bit is in PLLD base register */
-           register - can not do direct r-m-w, must be
-           protected by PLLD lock */
        tegra_clk_cfg_ex(
                d, TEGRA_CLK_PLLD_MIPI_MUX_SEL, index);
diff --git a/arch/arm/mach-u300/core.c b/arch/arm/mach-u300/core.c
index 12f3994c43db..0374b9863e9b 100644
--- a/arch/arm/mach-u300/core.c
+++ b/arch/arm/mach-u300/core.c
@@ -27,7 +27,6 @@
 #include <linux/mtd/nand.h>
 #include <linux/mtd/fsmc.h>
 #include <linux/pinctrl/machine.h>
-#include <linux/pinctrl/consumer.h>
 #include <linux/pinctrl/pinconf-generic.h>
 #include <linux/dma-mapping.h>
 #include <linux/platform_data/clk-u300.h>
@@ -1543,39 +1542,6 @@ static struct pinctrl_map __initdata u300_pinmux_map[] = {
                                    pin_highz_conf),
 };
-struct u300_mux_hog {
-        struct device *dev;
-        struct pinctrl *p;
-};
-static struct u300_mux_hog u300_mux_hogs[] = {
-        {
-                .dev = &uart0_device.dev,
-        },
-        {
-                .dev = &mmcsd_device.dev,
-        },
-};
-static int __init u300_pinctrl_fetch(void)
-{
-        int i;
-        for (i = 0; i < ARRAY_SIZE(u300_mux_hogs); i++) {
-                struct pinctrl *p;
-                p = pinctrl_get_select_default(u300_mux_hogs[i].dev);
-                if (IS_ERR(p)) {
-                        pr_err("u300: could not get pinmux hog for dev %s\n",
-                               dev_name(u300_mux_hogs[i].dev));
-                        continue;
-                }
-                u300_mux_hogs[i].p = p;
-        }
-        return 0;
-}
-subsys_initcall(u300_pinctrl_fetch);
 /*
 * Notice that AMBA devices are initialized before platform devices.
 *
diff --git a/arch/arm/mach-ux500/devices-db8500.h b/arch/arm/mach-ux500/devices-db8500.h
index 4b24c9992654..a5e05f6e256f 100644
--- a/arch/arm/mach-ux500/devices-db8500.h
+++ b/arch/arm/mach-ux500/devices-db8500.h
@@ -8,6 +8,7 @@
 #ifndef __DEVICES_DB8500_H
 #define __DEVICES_DB8500_H
+#include <linux/platform_data/usb-musb-ux500.h>
 #include <mach/irqs.h>
 #include "devices-common.h"
diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c
index 5383bc018571..6b2fb87c8698 100644
--- a/arch/arm/mm/dma-mapping.c
+++ b/arch/arm/mm/dma-mapping.c
@@ -1034,7 +1034,8 @@ static inline void __free_iova(struct dma_iommu_mapping *mapping,
        spin_unlock_irqrestore(&mapping->lock, flags);
 }
-static struct page **__iommu_alloc_buffer(struct device *dev, size_t size, gfp_t gfp)
+static struct page **__iommu_alloc_buffer(struct device *dev, size_t size,
+                                          gfp_t gfp, struct dma_attrs *attrs)
 {
        struct page **pages;
        int count = size >> PAGE_SHIFT;
@@ -1048,6 +1049,23 @@ static struct page **__iommu_alloc_buffer(struct device *dev, size_t size, gfp_t
        if (!pages)
                return NULL;
+        if (dma_get_attr(DMA_ATTR_FORCE_CONTIGUOUS, attrs))
+        {
+                unsigned long order = get_order(size);
+                struct page *page;
+                page = dma_alloc_from_contiguous(dev, count, order);
+                if (!page)
+                        goto error;
+                __dma_clear_buffer(page, size);
+                for (i = 0; i < count; i++)
+                        pages[i] = page + i;
+                return pages;
+        }
        while (count) {
                int j, order = __fls(count);
@@ -1081,14 +1099,21 @@ error:
        return NULL;
 }
-static int __iommu_free_buffer(struct device *dev, struct page **pages, size_t size)
+static int __iommu_free_buffer(struct device *dev, struct page **pages,
+                               size_t size, struct dma_attrs *attrs)
 {
        int count = size >> PAGE_SHIFT;
        int array_size = count * sizeof(struct page *);
        int i;
-        for (i = 0; i < count; i++)
-                if (pages[i])
+        if (dma_get_attr(DMA_ATTR_FORCE_CONTIGUOUS, attrs)) {
-                        __free_pages(pages[i], 0);
+                dma_release_from_contiguous(dev, pages[0], count);
+        } else {
+                for (i = 0; i < count; i++)
+                        if (pages[i])
+                                __free_pages(pages[i], 0);
+        }
        if (array_size <= PAGE_SIZE)
                kfree(pages);
        else
@@ -1250,7 +1275,7 @@ static void *arm_iommu_alloc_attrs(struct device *dev, size_t size,
        if (gfp & GFP_ATOMIC)
                return __iommu_alloc_atomic(dev, size, handle);
-        pages = __iommu_alloc_buffer(dev, size, gfp);
+        pages = __iommu_alloc_buffer(dev, size, gfp, attrs);
        if (!pages)
                return NULL;
@@ -1271,7 +1296,7 @@ static void *arm_iommu_alloc_attrs(struct device *dev, size_t size,
 err_mapping:
        __iommu_remove_mapping(dev, *handle, size);
 err_buffer:
-        __iommu_free_buffer(dev, pages, size);
+        __iommu_free_buffer(dev, pages, size, attrs);
        return NULL;
 }
@@ -1327,7 +1352,7 @@ void arm_iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr,
        }
        __iommu_remove_mapping(dev, handle, size);
-        __iommu_free_buffer(dev, pages, size);
+        __iommu_free_buffer(dev, pages, size, attrs);
 }
 static int arm_iommu_get_sgtable(struct device *dev, struct sg_table *sgt,
diff --git a/arch/arm/plat-omap/fb.c b/arch/arm/plat-omap/fb.c
index 3a77b30f53d4..a3367b783fc7 100644
--- a/arch/arm/plat-omap/fb.c
+++ b/arch/arm/plat-omap/fb.c
@@ -29,6 +29,7 @@
 #include <linux/memblock.h>
 #include <linux/io.h>
 #include <linux/omapfb.h>
+#include <linux/dma-mapping.h>
 #include <asm/mach/map.h>
@@ -105,7 +106,7 @@ static struct platform_device omap_fb_device = {
        .id             = -1,
        .dev = {
                .dma_mask               = &omap_fb_dma_mask,
-                .coherent_dma_mask      = ~(u32)0,
+                .coherent_dma_mask      = DMA_BIT_MASK(32),
                .platform_data          = &omapfb_config,
        },
        .num_resources = 0,
@@ -141,7 +142,7 @@ static struct platform_device omap_fb_device = {
        .id             = -1,
        .dev = {
                .dma_mask               = &omap_fb_dma_mask,
-                .coherent_dma_mask      = ~(u32)0,
+                .coherent_dma_mask      = DMA_BIT_MASK(32),
                .platform_data          = &omapfb_config,
        },
        .num_resources = 0,
diff --git a/arch/arm/plat-omap/include/plat/vram.h b/arch/arm/plat-omap/include/plat/vram.h
deleted file mode 100644
index 4d65b7d06e6c..000000000000
--- a/arch/arm/plat-omap/include/plat/vram.h
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * VRAM manager for OMAP
- *
- * Copyright (C) 2009 Nokia Corporation
- * Author: Tomi Valkeinen <tomi.valkeinen@nokia.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
- */
-#ifndef __OMAP_VRAM_H__
-#define __OMAP_VRAM_H__
-#include <linux/types.h>
-extern int omap_vram_add_region(unsigned long paddr, size_t size);
-extern int omap_vram_free(unsigned long paddr, size_t size);
-extern int omap_vram_alloc(size_t size, unsigned long *paddr);
-extern int omap_vram_reserve(unsigned long paddr, size_t size);
-extern void omap_vram_get_info(unsigned long *vram, unsigned long *free_vram,
-                unsigned long *largest_free_block);
-#ifdef CONFIG_OMAP2_VRAM
-extern void omap_vram_set_sdram_vram(u32 size, u32 start);
-extern void omap_vram_reserve_sdram_memblock(void);
-#else
-static inline void omap_vram_set_sdram_vram(u32 size, u32 start) { }
-static inline void omap_vram_reserve_sdram_memblock(void) { }
-#endif
-#endif
diff --git a/arch/m68k/Kconfig.cpu b/arch/m68k/Kconfig.cpu
index 2f2d87b40341..b1cfff832fb5 100644
--- a/arch/m68k/Kconfig.cpu
+++ b/arch/m68k/Kconfig.cpu
@@ -35,7 +35,8 @@ endchoice
 if M68KCLASSIC
 config M68000
-        bool
+        bool "MC68000"
+        depends on !MMU
        select CPU_HAS_NO_BITFIELDS
        select CPU_HAS_NO_MULDIV64
        select CPU_HAS_NO_UNALIGNED
diff --git a/arch/m68k/Makefile b/arch/m68k/Makefile
index 7636751f2f87..2f02acfb8edf 100644
--- a/arch/m68k/Makefile
+++ b/arch/m68k/Makefile
@@ -92,7 +92,7 @@ endif
 head-y                          := arch/m68k/kernel/head.o
 head-$(CONFIG_SUN3)             := arch/m68k/kernel/sun3-head.o
 head-$(CONFIG_M68360)           := arch/m68k/platform/68360/head.o
-head-$(CONFIG_M68000)           := arch/m68k/platform/68328/head.o
+head-$(CONFIG_M68000)           := arch/m68k/platform/68000/head.o
 head-$(CONFIG_COLDFIRE)         := arch/m68k/platform/coldfire/head.o
 core-y                          += arch/m68k/kernel/    arch/m68k/mm/
@@ -114,9 +114,7 @@ core-$(CONFIG_M68040)		+= arch/m68k/fpsp040/
 core-$(CONFIG_M68060)           += arch/m68k/ifpsp060/
 core-$(CONFIG_M68KFPU_EMU)      += arch/m68k/math-emu/
 core-$(CONFIG_M68360)           += arch/m68k/platform/68360/
-core-$(CONFIG_M68000)           += arch/m68k/platform/68328/
+core-$(CONFIG_M68000)           += arch/m68k/platform/68000/
-core-$(CONFIG_M68EZ328)         += arch/m68k/platform/68EZ328/
-core-$(CONFIG_M68VZ328)         += arch/m68k/platform/68VZ328/
 core-$(CONFIG_COLDFIRE)         += arch/m68k/platform/coldfire/
diff --git a/arch/m68k/include/asm/m5249sim.h b/arch/m68k/include/asm/m5249sim.h
deleted file mode 100644
index fdf45e6807c9..000000000000
--- a/arch/m68k/include/asm/m5249sim.h
+++ /dev/null
@@ -1,269 +0,0 @@
-/****************************************************************************/
-/*
- *      m5249sim.h -- ColdFire 5249 System Integration Module support.
- *
- *      (C) Copyright 2002, Greg Ungerer (gerg@snapgear.com)
- */
-/****************************************************************************/
-#ifndef m5249sim_h
-#define m5249sim_h
-/****************************************************************************/
-#define CPU_NAME                "COLDFIRE(m5249)"
-#define CPU_INSTR_PER_JIFFY     3
-#define MCF_BUSCLK              (MCF_CLK / 2)
-#include <asm/m52xxacr.h>
-/*
- *      The 5249 has a second MBAR region, define its address.
- */
-#define MCF_MBAR2               0x80000000
-/*
- *      Define the 5249 SIM register set addresses.
- */
-#define MCFSIM_RSR              (MCF_MBAR + 0x00)       /* Reset Status */
-#define MCFSIM_SYPCR            (MCF_MBAR + 0x01)       /* System Protection */
-#define MCFSIM_SWIVR            (MCF_MBAR + 0x02)       /* SW Watchdog intr */
-#define MCFSIM_SWSR             (MCF_MBAR + 0x03)       /* SW Watchdog srv */
-#define MCFSIM_PAR              (MCF_MBAR + 0x04)       /* Pin Assignment */
-#define MCFSIM_IRQPAR           (MCF_MBAR + 0x06)       /* Intr Assignment */
-#define MCFSIM_MPARK            (MCF_MBAR + 0x0C)       /* BUS Master Ctrl */
-#define MCFSIM_IPR              (MCF_MBAR + 0x40)       /* Interrupt Pending */
-#define MCFSIM_IMR              (MCF_MBAR + 0x44)       /* Interrupt Mask */
-#define MCFSIM_AVR              (MCF_MBAR + 0x4b)       /* Autovector Ctrl */
-#define MCFSIM_ICR0             (MCF_MBAR + 0x4c)       /* Intr Ctrl reg 0 */
-#define MCFSIM_ICR1             (MCF_MBAR + 0x4d)       /* Intr Ctrl reg 1 */
-#define MCFSIM_ICR2             (MCF_MBAR + 0x4e)       /* Intr Ctrl reg 2 */
-#define MCFSIM_ICR3             (MCF_MBAR + 0x4f)       /* Intr Ctrl reg 3 */
-#define MCFSIM_ICR4             (MCF_MBAR + 0x50)       /* Intr Ctrl reg 4 */
-#define MCFSIM_ICR5             (MCF_MBAR + 0x51)       /* Intr Ctrl reg 5 */
-#define MCFSIM_ICR6             (MCF_MBAR + 0x52)       /* Intr Ctrl reg 6 */
-#define MCFSIM_ICR7             (MCF_MBAR + 0x53)       /* Intr Ctrl reg 7 */
-#define MCFSIM_ICR8             (MCF_MBAR + 0x54)       /* Intr Ctrl reg 8 */
-#define MCFSIM_ICR9             (MCF_MBAR + 0x55)       /* Intr Ctrl reg 9 */
-#define MCFSIM_ICR10            (MCF_MBAR + 0x56)       /* Intr Ctrl reg 10 */
-#define MCFSIM_ICR11            (MCF_MBAR + 0x57)       /* Intr Ctrl reg 11 */
-#define MCFSIM_CSAR0            (MCF_MBAR + 0x80)       /* CS 0 Address reg */
-#define MCFSIM_CSMR0            (MCF_MBAR + 0x84)       /* CS 0 Mask reg */
-#define MCFSIM_CSCR0            (MCF_MBAR + 0x8a)       /* CS 0 Control reg */
-#define MCFSIM_CSAR1            (MCF_MBAR + 0x8c)       /* CS 1 Address reg */
-#define MCFSIM_CSMR1            (MCF_MBAR + 0x90)       /* CS 1 Mask reg */
-#define MCFSIM_CSCR1            (MCF_MBAR + 0x96)       /* CS 1 Control reg */
-#define MCFSIM_CSAR2            (MCF_MBAR + 0x98)       /* CS 2 Address reg */
-#define MCFSIM_CSMR2            (MCF_MBAR + 0x9c)       /* CS 2 Mask reg */
-#define MCFSIM_CSCR2            (MCF_MBAR + 0xa2)       /* CS 2 Control reg */
-#define MCFSIM_CSAR3            (MCF_MBAR + 0xa4)       /* CS 3 Address reg */
-#define MCFSIM_CSMR3            (MCF_MBAR + 0xa8)       /* CS 3 Mask reg */
-#define MCFSIM_CSCR3            (MCF_MBAR + 0xae)       /* CS 3 Control reg */
-#define MCFSIM_DCR              (MCF_MBAR + 0x100)      /* DRAM Control */
-#define MCFSIM_DACR0            (MCF_MBAR + 0x108)      /* DRAM 0 Addr/Ctrl */
-#define MCFSIM_DMR0             (MCF_MBAR + 0x10c)      /* DRAM 0 Mask */
-#define MCFSIM_DACR1            (MCF_MBAR + 0x110)      /* DRAM 1 Addr/Ctrl */
-#define MCFSIM_DMR1             (MCF_MBAR + 0x114)      /* DRAM 1 Mask */
-/*
- *      Timer module.
- */
-#define MCFTIMER_BASE1          (MCF_MBAR + 0x140)      /* Base of TIMER1 */
-#define MCFTIMER_BASE2          (MCF_MBAR + 0x180)      /* Base of TIMER2 */
-/*
- *      UART module.
- */
-#define MCFUART_BASE0           (MCF_MBAR + 0x1c0)      /* Base address UART0 */
-#define MCFUART_BASE1           (MCF_MBAR + 0x200)      /* Base address UART1 */
-/*
- *      QSPI module.
- */
-#define MCFQSPI_BASE            (MCF_MBAR + 0x300)      /* Base address QSPI */
-#define MCFQSPI_SIZE            0x40                    /* Register set size */
-#define MCFQSPI_CS0             29
-#define MCFQSPI_CS1             24
-#define MCFQSPI_CS2             21
-#define MCFQSPI_CS3             22
-/*
- *      DMA unit base addresses.
- */
-#define MCFDMA_BASE0            (MCF_MBAR + 0x300)      /* Base address DMA 0 */
-#define MCFDMA_BASE1            (MCF_MBAR + 0x340)      /* Base address DMA 1 */
-#define MCFDMA_BASE2            (MCF_MBAR + 0x380)      /* Base address DMA 2 */
-#define MCFDMA_BASE3            (MCF_MBAR + 0x3C0)      /* Base address DMA 3 */
-/*
- *      Some symbol defines for the above...
- */
-#define MCFSIM_SWDICR           MCFSIM_ICR0     /* Watchdog timer ICR */
-#define MCFSIM_TIMER1ICR        MCFSIM_ICR1     /* Timer 1 ICR */
-#define MCFSIM_TIMER2ICR        MCFSIM_ICR2     /* Timer 2 ICR */
-#define MCFSIM_UART1ICR         MCFSIM_ICR4     /* UART 1 ICR */
-#define MCFSIM_UART2ICR         MCFSIM_ICR5     /* UART 2 ICR */
-#define MCFSIM_DMA0ICR          MCFSIM_ICR6     /* DMA 0 ICR */
-#define MCFSIM_DMA1ICR          MCFSIM_ICR7     /* DMA 1 ICR */
-#define MCFSIM_DMA2ICR          MCFSIM_ICR8     /* DMA 2 ICR */
-#define MCFSIM_DMA3ICR          MCFSIM_ICR9     /* DMA 3 ICR */
-#define MCFSIM_QSPIICR          MCFSIM_ICR10    /* QSPI ICR */
-/*
- *      Define system peripheral IRQ usage.
- */
-#define MCF_IRQ_QSPI            28              /* QSPI, Level 4 */
-#define MCF_IRQ_TIMER           30              /* Timer0, Level 6 */
-#define MCF_IRQ_PROFILER        31              /* Timer1, Level 7 */
-#define MCF_IRQ_UART0           73              /* UART0 */
-#define MCF_IRQ_UART1           74              /* UART1 */
-/*
- *      General purpose IO registers (in MBAR2).
- */
-#define MCFSIM2_GPIOREAD        (MCF_MBAR2 + 0x000)     /* GPIO read values */
-#define MCFSIM2_GPIOWRITE       (MCF_MBAR2 + 0x004)     /* GPIO write values */
-#define MCFSIM2_GPIOENABLE      (MCF_MBAR2 + 0x008)     /* GPIO enabled */
-#define MCFSIM2_GPIOFUNC        (MCF_MBAR2 + 0x00C)     /* GPIO function */
-#define MCFSIM2_GPIO1READ       (MCF_MBAR2 + 0x0B0)     /* GPIO1 read values */
-#define MCFSIM2_GPIO1WRITE      (MCF_MBAR2 + 0x0B4)     /* GPIO1 write values */
-#define MCFSIM2_GPIO1ENABLE     (MCF_MBAR2 + 0x0B8)     /* GPIO1 enabled */
-#define MCFSIM2_GPIO1FUNC       (MCF_MBAR2 + 0x0BC)     /* GPIO1 function */
-#define MCFSIM2_GPIOINTSTAT     (MCF_MBAR2 + 0xc0)      /* GPIO intr status */
-#define MCFSIM2_GPIOINTCLEAR    (MCF_MBAR2 + 0xc0)      /* GPIO intr clear */
-#define MCFSIM2_GPIOINTENABLE   (MCF_MBAR2 + 0xc4)      /* GPIO intr enable */
-#define MCFSIM2_INTLEVEL1       (MCF_MBAR2 + 0x140)     /* Intr level reg 1 */
-#define MCFSIM2_INTLEVEL2       (MCF_MBAR2 + 0x144)     /* Intr level reg 2 */
-#define MCFSIM2_INTLEVEL3       (MCF_MBAR2 + 0x148)     /* Intr level reg 3 */
-#define MCFSIM2_INTLEVEL4       (MCF_MBAR2 + 0x14c)     /* Intr level reg 4 */
-#define MCFSIM2_INTLEVEL5       (MCF_MBAR2 + 0x150)     /* Intr level reg 5 */
-#define MCFSIM2_INTLEVEL6       (MCF_MBAR2 + 0x154)     /* Intr level reg 6 */
-#define MCFSIM2_INTLEVEL7       (MCF_MBAR2 + 0x158)     /* Intr level reg 7 */
-#define MCFSIM2_INTLEVEL8       (MCF_MBAR2 + 0x15c)     /* Intr level reg 8 */
-#define MCFSIM2_DMAROUTE        (MCF_MBAR2 + 0x188)     /* DMA routing */
-#define MCFSIM2_IDECONFIG1      (MCF_MBAR2 + 0x18c)     /* IDEconfig1 */
-#define MCFSIM2_IDECONFIG2      (MCF_MBAR2 + 0x190)     /* IDEconfig2 */
-/*
- * Define the base interrupt for the second interrupt controller.
- * We set it to 128, out of the way of the base interrupts, and plenty
- * of room for its 64 interrupts.
- */
-#define MCFINTC2_VECBASE        128
-#define MCFINTC2_GPIOIRQ0       (MCFINTC2_VECBASE + 32)
-#define MCFINTC2_GPIOIRQ1       (MCFINTC2_VECBASE + 33)
-#define MCFINTC2_GPIOIRQ2       (MCFINTC2_VECBASE + 34)
-#define MCFINTC2_GPIOIRQ3       (MCFINTC2_VECBASE + 35)
-#define MCFINTC2_GPIOIRQ4       (MCFINTC2_VECBASE + 36)
-#define MCFINTC2_GPIOIRQ5       (MCFINTC2_VECBASE + 37)
-#define MCFINTC2_GPIOIRQ6       (MCFINTC2_VECBASE + 38)
-#define MCFINTC2_GPIOIRQ7       (MCFINTC2_VECBASE + 39)
-/*
- * Generic GPIO support
- */
-#define MCFGPIO_PIN_MAX         64
-#define MCFGPIO_IRQ_MAX         -1
-#define MCFGPIO_IRQ_VECBASE     -1
-/****************************************************************************/
-#ifdef __ASSEMBLER__
-/*
- *      The M5249C3 board needs a little help getting all its SIM devices
- *      initialized at kernel start time. dBUG doesn't set much up, so
- *      we need to do it manually.
- */
-.macro m5249c3_setup
-        /*
-         *      Set MBAR1 and MBAR2, just incase they are not set.
-         */
-        movel   #0x10000001,%a0
-        movec   %a0,%MBAR                       /* map MBAR region */
-        subql   #1,%a0                          /* get MBAR address in a0 */
-        movel   #0x80000001,%a1
-        movec   %a1,#3086                       /* map MBAR2 region */
-        subql   #1,%a1                          /* get MBAR2 address in a1 */
-        /*
-         *      Move secondary interrupts to their base (128).
-         */
-        moveb   #MCFINTC2_VECBASE,%d0
-        moveb   %d0,0x16b(%a1)                  /* interrupt base register */
-        /*
-         *      Work around broken CSMR0/DRAM vector problem.
-         */
-        movel   #0x001F0021,%d0                 /* disable C/I bit */
-        movel   %d0,0x84(%a0)                   /* set CSMR0 */
-        /*
-         *      Disable the PLL firstly. (Who knows what state it is
-         *      in here!).
-         */
-        movel   0x180(%a1),%d0                  /* get current PLL value */
-        andl    #0xfffffffe,%d0                 /* PLL bypass first */
-        movel   %d0,0x180(%a1)                  /* set PLL register */
-        nop
-#if CONFIG_CLOCK_FREQ == 140000000
-        /*
-         *      Set initial clock frequency. This assumes M5249C3 board
-         *      is fitted with 11.2896MHz crystal. It will program the
-         *      PLL for 140MHz. Lets go fast :-)
-         */
-        movel   #0x125a40f0,%d0                 /* set for 140MHz */
-        movel   %d0,0x180(%a1)                  /* set PLL register */
-        orl     #0x1,%d0
-        movel   %d0,0x180(%a1)                  /* set PLL register */
-#endif
-        /*
-         *      Setup CS1 for ethernet controller.
-         *      (Setup as per M5249C3 doco).
-         */
-        movel  #0xe0000000,%d0                  /* CS1 mapped at 0xe0000000 */
-        movel  %d0,0x8c(%a0)
-        movel  #0x001f0021,%d0                  /* CS1 size of 1Mb */
-        movel  %d0,0x90(%a0)
-        movew  #0x0080,%d0                      /* CS1 = 16bit port, AA */
-        movew  %d0,0x96(%a0)
-        /*
-         *      Setup CS2 for IDE interface.
-         */
-        movel   #0x50000000,%d0                 /* CS2 mapped at 0x50000000 */
-        movel   %d0,0x98(%a0)
-        movel   #0x001f0001,%d0                 /* CS2 size of 1MB */
-        movel   %d0,0x9c(%a0)
-        movew   #0x0080,%d0                     /* CS2 = 16bit, TA */
-        movew   %d0,0xa2(%a0)
-        movel   #0x00107000,%d0                 /* IDEconfig1 */
-        movel   %d0,0x18c(%a1)
-        movel   #0x000c0400,%d0                 /* IDEconfig2 */
-        movel   %d0,0x190(%a1)
-        movel   #0x00080000,%d0                 /* GPIO19, IDE reset bit */
-        orl     %d0,0xc(%a1)                    /* function GPIO19 */
-        orl     %d0,0x8(%a1)                    /* enable GPIO19 as output */
-        orl     %d0,0x4(%a1)                    /* de-assert IDE reset */
-.endm
-#define PLATFORM_SETUP  m5249c3_setup
-#endif /* __ASSEMBLER__ */
-/****************************************************************************/
-#endif  /* m5249sim_h */
diff --git a/arch/m68k/include/asm/m525xsim.h b/arch/m68k/include/asm/m525xsim.h
index acab61cb91ed..e33f5bb6aca8 100644
--- a/arch/m68k/include/asm/m525xsim.h
+++ b/arch/m68k/include/asm/m525xsim.h
@@ -12,6 +12,11 @@
 #define m525xsim_h
 /****************************************************************************/
+/*
+ *      This header supports ColdFire 5249, 5251 and 5253. There are a few
+ *      little differences between them, but most of the peripheral support
+ *      can be used by all of them.
+ */
 #define CPU_NAME                "COLDFIRE(m525x)"
 #define CPU_INSTR_PER_JIFFY     3
 #define MCF_BUSCLK              (MCF_CLK / 2)
@@ -65,6 +70,8 @@
 #define MCFSIM_DCR              (MCF_MBAR + 0x100)      /* DRAM Control */
 #define MCFSIM_DACR0            (MCF_MBAR + 0x108)      /* DRAM 0 Addr/Ctrl */
 #define MCFSIM_DMR0             (MCF_MBAR + 0x10c)      /* DRAM 0 Mask */
+#define MCFSIM_DACR1            (MCF_MBAR + 0x110)      /* DRAM 1 Addr/Ctrl */
+#define MCFSIM_DMR1             (MCF_MBAR + 0x114)      /* DRAM 1 Mask */
 /*
 * Secondary Interrupt Controller (in MBAR2)
@@ -101,11 +108,17 @@
 #define MCFQSPI_BASE            (MCF_MBAR + 0x300)      /* Base address QSPI */
 #define MCFQSPI_SIZE            0x40                    /* Register set size */
+#ifdef CONFIG_M5249
+#define MCFQSPI_CS0             29
+#define MCFQSPI_CS1             24
+#define MCFQSPI_CS2             21
+#define MCFQSPI_CS3             22
+#else
 #define MCFQSPI_CS0             15
 #define MCFQSPI_CS1             16
 #define MCFQSPI_CS2             24
 #define MCFQSPI_CS3             28
+#endif
 /*
 *      I2C module.
@@ -115,6 +128,7 @@
 #define MCFI2C_BASE1            (MCF_MBAR2 + 0x440)     /* Base addreess I2C1 */
 #define MCFI2C_SIZE1            0x20                    /* Register set size */
 /*
 *      DMA unit base addresses.
 */
@@ -163,6 +177,7 @@
 #define MCF_IRQ_GPIO4           (MCFINTC2_VECBASE + 36)
 #define MCF_IRQ_GPIO5           (MCFINTC2_VECBASE + 37)
 #define MCF_IRQ_GPIO6           (MCFINTC2_VECBASE + 38)
+#define MCF_IRQ_GPIO7           (MCFINTC2_VECBASE + 39)
 #define MCF_IRQ_USBWUP          (MCFINTC2_VECBASE + 40)
 #define MCF_IRQ_I2C1            (MCFINTC2_VECBASE + 62)
@@ -183,12 +198,111 @@
 #define MCFSIM2_GPIOINTCLEAR    (MCF_MBAR2 + 0xc0)      /* GPIO intr clear */
 #define MCFSIM2_GPIOINTENABLE   (MCF_MBAR2 + 0xc4)      /* GPIO intr enable */
+#define MCFSIM2_DMAROUTE        (MCF_MBAR2 + 0x188)     /* DMA routing */
+#define MCFSIM2_IDECONFIG1      (MCF_MBAR2 + 0x18c)     /* IDEconfig1 */
+#define MCFSIM2_IDECONFIG2      (MCF_MBAR2 + 0x190)     /* IDEconfig2 */
 /*
 * Generic GPIO support
 */
 #define MCFGPIO_PIN_MAX         64
+#ifdef CONFIG_M5249
+#define MCFGPIO_IRQ_MAX         -1
+#define MCFGPIO_IRQ_VECBASE     -1
+#else
 #define MCFGPIO_IRQ_MAX         7
 #define MCFGPIO_IRQ_VECBASE     MCF_IRQ_GPIO0
+#endif
+/****************************************************************************/
+#ifdef __ASSEMBLER__
+#ifdef CONFIG_M5249C3
+/*
+ *      The M5249C3 board needs a little help getting all its SIM devices
+ *      initialized at kernel start time. dBUG doesn't set much up, so
+ *      we need to do it manually.
+ */
+.macro m5249c3_setup
+        /*
+         *      Set MBAR1 and MBAR2, just incase they are not set.
+         */
+        movel   #0x10000001,%a0
+        movec   %a0,%MBAR                       /* map MBAR region */
+        subql   #1,%a0                          /* get MBAR address in a0 */
+        movel   #0x80000001,%a1
+        movec   %a1,#3086                       /* map MBAR2 region */
+        subql   #1,%a1                          /* get MBAR2 address in a1 */
+        /*
+         *      Move secondary interrupts to their base (128).
+         */
+        moveb   #MCFINTC2_VECBASE,%d0
+        moveb   %d0,0x16b(%a1)                  /* interrupt base register */
+        /*
+         *      Work around broken CSMR0/DRAM vector problem.
+         */
+        movel   #0x001F0021,%d0                 /* disable C/I bit */
+        movel   %d0,0x84(%a0)                   /* set CSMR0 */
+        /*
+         *      Disable the PLL firstly. (Who knows what state it is
+         *      in here!).
+         */
+        movel   0x180(%a1),%d0                  /* get current PLL value */
+        andl    #0xfffffffe,%d0                 /* PLL bypass first */
+        movel   %d0,0x180(%a1)                  /* set PLL register */
+        nop
+#if CONFIG_CLOCK_FREQ == 140000000
+        /*
+         *      Set initial clock frequency. This assumes M5249C3 board
+         *      is fitted with 11.2896MHz crystal. It will program the
+         *      PLL for 140MHz. Lets go fast :-)
+         */
+        movel   #0x125a40f0,%d0                 /* set for 140MHz */
+        movel   %d0,0x180(%a1)                  /* set PLL register */
+        orl     #0x1,%d0
+        movel   %d0,0x180(%a1)                  /* set PLL register */
+#endif
+        /*
+         *      Setup CS1 for ethernet controller.
+         *      (Setup as per M5249C3 doco).
+         */
+        movel  #0xe0000000,%d0                  /* CS1 mapped at 0xe0000000 */
+        movel  %d0,0x8c(%a0)
+        movel  #0x001f0021,%d0                  /* CS1 size of 1Mb */
+        movel  %d0,0x90(%a0)
+        movew  #0x0080,%d0                      /* CS1 = 16bit port, AA */
+        movew  %d0,0x96(%a0)
+        /*
+         *      Setup CS2 for IDE interface.
+         */
+        movel   #0x50000000,%d0                 /* CS2 mapped at 0x50000000 */
+        movel   %d0,0x98(%a0)
+        movel   #0x001f0001,%d0                 /* CS2 size of 1MB */
+        movel   %d0,0x9c(%a0)
+        movew   #0x0080,%d0                     /* CS2 = 16bit, TA */
+        movew   %d0,0xa2(%a0)
+        movel   #0x00107000,%d0                 /* IDEconfig1 */
+        movel   %d0,0x18c(%a1)
+        movel   #0x000c0400,%d0                 /* IDEconfig2 */
+        movel   %d0,0x190(%a1)
+        movel   #0x00080000,%d0                 /* GPIO19, IDE reset bit */
+        orl     %d0,0xc(%a1)                    /* function GPIO19 */
+        orl     %d0,0x8(%a1)                    /* enable GPIO19 as output */
+        orl     %d0,0x4(%a1)                    /* de-assert IDE reset */
+.endm
+#define PLATFORM_SETUP  m5249c3_setup
+#endif /* CONFIG_M5249C3 */
+#endif /* __ASSEMBLER__ */
 /****************************************************************************/
 #endif  /* m525xsim_h */
diff --git a/arch/m68k/include/asm/mcfclk.h b/arch/m68k/include/asm/mcfclk.h
index b676a02bb392..ea4791e3a557 100644
--- a/arch/m68k/include/asm/mcfclk.h
+++ b/arch/m68k/include/asm/mcfclk.h
@@ -8,7 +8,6 @@
 struct clk;
-#ifdef MCFPM_PPMCR0
 struct clk_ops {
        void (*enable)(struct clk *);
        void (*disable)(struct clk *);
@@ -23,6 +22,8 @@ struct clk {
 };
 extern struct clk *mcf_clks[];
+#ifdef MCFPM_PPMCR0
 extern struct clk_ops clk_ops0;
 #ifdef MCFPM_PPMCR1
 extern struct clk_ops clk_ops1;
@@ -38,6 +39,12 @@ static struct clk __clk_##clk_bank##_##clk_slot = { \
 void __clk_init_enabled(struct clk *);
 void __clk_init_disabled(struct clk *);
+#else
+#define DEFINE_CLK(clk_ref, clk_name, clk_rate) \
+        static struct clk clk_##clk_ref = { \
+                .name = clk_name, \
+                .rate = clk_rate, \
+        }
 #endif /* MCFPM_PPMCR0 */
 #endif /* mcfclk_h */
diff --git a/arch/m68k/include/asm/mcfsim.h b/arch/m68k/include/asm/mcfsim.h
index 7a83e619e73b..a04fd9b2714c 100644
--- a/arch/m68k/include/asm/mcfsim.h
+++ b/arch/m68k/include/asm/mcfsim.h
@@ -24,10 +24,7 @@
 #elif defined(CONFIG_M523x)
 #include <asm/m523xsim.h>
 #include <asm/mcfintc.h>
-#elif defined(CONFIG_M5249)
+#elif defined(CONFIG_M5249) || defined(CONFIG_M525x)
-#include <asm/m5249sim.h>
-#include <asm/mcfintc.h>
-#elif defined(CONFIG_M525x)
 #include <asm/m525xsim.h>
 #include <asm/mcfintc.h>
 #elif defined(CONFIG_M527x)
diff --git a/arch/m68k/include/asm/page_no.h b/arch/m68k/include/asm/page_no.h
index 90595721185f..ef209169579a 100644
--- a/arch/m68k/include/asm/page_no.h
+++ b/arch/m68k/include/asm/page_no.h
@@ -26,7 +26,7 @@ extern unsigned long memory_end;
 #define pfn_to_virt(pfn)        __va((pfn) << PAGE_SHIFT)
 #define virt_to_page(addr)      (mem_map + (((unsigned long)(addr)-PAGE_OFFSET) >> PAGE_SHIFT))
-#define page_to_virt(page)      ((((page) - mem_map) << PAGE_SHIFT) + PAGE_OFFSET)
+#define page_to_virt(page)      __va(((((page) - mem_map) << PAGE_SHIFT) + PAGE_OFFSET))
 #define pfn_to_page(pfn)        virt_to_page(pfn_to_virt(pfn))
 #define page_to_pfn(page)       virt_to_pfn(page_to_virt(page))
diff --git a/arch/m68k/lib/memcpy.c b/arch/m68k/lib/memcpy.c
index 10ca051d56b8..c1e2dfb206f3 100644
--- a/arch/m68k/lib/memcpy.c
+++ b/arch/m68k/lib/memcpy.c
@@ -10,7 +10,7 @@
 void *memcpy(void *to, const void *from, size_t n)
 {
        void *xto = to;
-        size_t temp, temp1;
+        size_t temp;
        if (!n)
                return xto;
@@ -47,6 +47,7 @@ void *memcpy(void *to, const void *from, size_t n)
                for (; temp; temp--)
                        *lto++ = *lfrom++;
 #else
+                size_t temp1;
                asm volatile (
                        "       movel %2,%3\n"
                        "       andw  #7,%3\n"
diff --git a/arch/m68k/platform/68000/Makefile b/arch/m68k/platform/68000/Makefile
new file mode 100644
index 000000000000..1eab70c7194b
--- /dev/null
+++ b/arch/m68k/platform/68000/Makefile
@@ -0,0 +1,18 @@
+##################################################
+#
+# Makefile for 68000 core based cpus
+#
+# 2012.10.21, Luis Alves <ljalvs@gmail.com>
+#             Merged all 68000 based cpu's config
+#             files into a single directory.
+#
+# 68328, 68EZ328, 68VZ328
+obj-y                   += entry.o ints.o timers.o
+obj-$(CONFIG_M68328)    += m68328.o
+obj-$(CONFIG_M68EZ328)  += m68EZ328.o
+obj-$(CONFIG_M68VZ328)  += m68VZ328.o
+obj-$(CONFIG_ROM)       += romvec.o
+extra-y                 := head.o
diff --git a/arch/m68k/platform/68VZ328/bootlogo.h b/arch/m68k/platform/68000/bootlogo-vz.h
index b38e2b255142..b38e2b255142 100644
--- a/arch/m68k/platform/68VZ328/bootlogo.h
+++ b/arch/m68k/platform/68000/bootlogo-vz.h
diff --git a/arch/m68k/platform/68328/bootlogo.h b/arch/m68k/platform/68000/bootlogo.h
index b896c933fafc..b896c933fafc 100644
--- a/arch/m68k/platform/68328/bootlogo.h
+++ b/arch/m68k/platform/68000/bootlogo.h
diff --git a/arch/m68k/platform/68328/entry.S b/arch/m68k/platform/68000/entry.S
index 7f91c2fde509..7f91c2fde509 100644
--- a/arch/m68k/platform/68328/entry.S
+++ b/arch/m68k/platform/68000/entry.S
diff --git a/arch/m68k/platform/68000/head.S b/arch/m68k/platform/68000/head.S
new file mode 100644
index 000000000000..536ef9616dad
--- /dev/null
+++ b/arch/m68k/platform/68000/head.S
@@ -0,0 +1,240 @@
+/*
+ * head.S - Common startup code for 68000 core based CPU's
+ *
+ * 2012.10.21, Luis Alves <ljalvs@gmail.com>, Single head.S file for all
+ *             68000 core based CPU's. Based on the sources from:
+ *             Coldfire by Greg Ungerer <gerg@snapgear.com>
+ *             68328 by D. Jeff Dionne <jeff@ryeham.ee.ryerson.ca>,
+ *                      Kenneth Albanowski <kjahds@kjahds.com>,
+ *                      The Silver Hammer Group, Ltd.
+ *
+ */
+#include <linux/linkage.h>
+#include <linux/init.h>
+#include <asm/asm-offsets.h>
+#include <asm/thread_info.h>
+/*****************************************************************************
+ * UCSIMM and UCDIMM use CONFIG_MEMORY_RESERVE to reserve some RAM
+ *****************************************************************************/
+#ifdef CONFIG_MEMORY_RESERVE
+#define RAMEND (CONFIG_RAMBASE+CONFIG_RAMSIZE)-(CONFIG_MEMORY_RESERVE*0x100000)
+#else
+#define RAMEND (CONFIG_RAMBASE+CONFIG_RAMSIZE)
+#endif
+/*****************************************************************************/
+.global _start
+.global _rambase
+.global _ramvec
+.global _ramstart
+.global _ramend
+#if defined(CONFIG_PILOT) || defined(CONFIG_INIT_LCD)
+.global bootlogo_bits
+#endif
+/* Defining DEBUG_HEAD_CODE, serial port in 68x328 is inited */
+/* #define DEBUG_HEAD_CODE */
+#undef DEBUG_HEAD_CODE
+.data
+/*****************************************************************************
+ * RAM setup pointers. Used by the kernel to determine RAM location and size.
+ *****************************************************************************/
+_rambase:
+        .long   0
+_ramvec:
+        .long   0
+_ramstart:
+        .long   0
+_ramend:
+        .long   0
+__HEAD
+/*****************************************************************************
+ * Entry point, where all begins!
+ *****************************************************************************/
+_start:
+/* Pilot need this specific signature at the start of ROM */
+#ifdef CONFIG_PILOT
+        .byte   0x4e, 0xfa, 0x00, 0x0a          /* bra opcode (jmp 10 bytes) */
+        .byte   'b', 'o', 'o', 't'
+        .word   10000
+        nop
+        moveq   #0, %d0
+        movew   %d0, 0xfffff618                 /* Watchdog off */
+        movel   #0x00011f07, 0xfffff114         /* CS A1 Mask */
+#endif /* CONFIG_PILOT */
+        movew   #0x2700, %sr                    /* disable all interrupts */
+/*****************************************************************************
+ * Setup PLL and wait for it to settle (in 68x328 cpu's).
+ * Also, if enabled, init serial port.
+ *****************************************************************************/
+#if defined(CONFIG_M68328) || \
+    defined(CONFIG_M68EZ328) || \
+    defined(CONFIG_M68VZ328)
+/* Serial port setup. Should only be needed if debugging this startup code. */
+#ifdef DEBUG_HEAD_CODE
+        movew   #0x0800, 0xfffff906             /* Ignore CTS */
+        movew   #0x010b, 0xfffff902             /* BAUD to 9600 */
+        movew   #0xe100, 0xfffff900             /* enable */
+#endif /* DEBUG_HEAD */
+#ifdef CONFIG_PILOT
+        movew   #0x2410, 0xfffff200             /* PLLCR */
+#else
+        movew   #0x2400, 0xfffff200             /* PLLCR */
+#endif
+        movew   #0x0123, 0xfffff202             /* PLLFSR */
+        moveq   #0, %d0
+        movew   #16384, %d0                     /* PLL settle wait loop */
+_pll_settle:
+        subw    #1, %d0
+        bne     _pll_settle
+#endif /* CONFIG_M68x328 */
+/*****************************************************************************
+ * If running kernel from ROM some specific initialization has to be done.
+ * (Assuming that everything is already init'ed when running from RAM)
+ *****************************************************************************/
+#ifdef CONFIG_ROMKERNEL
+/*****************************************************************************
+ * Init chip registers (uCsimm specific)
+ *****************************************************************************/
+#ifdef CONFIG_UCSIMM
+        moveb   #0x00, 0xfffffb0b       /* Watchdog off */
+        moveb   #0x10, 0xfffff000       /* SCR */
+        moveb   #0x00, 0xfffff40b       /* enable chip select */
+        moveb   #0x00, 0xfffff423       /* enable /DWE */
+        moveb   #0x08, 0xfffffd0d       /* disable hardmap */
+        moveb   #0x07, 0xfffffd0e       /* level 7 interrupt clear */
+        movew   #0x8600, 0xfffff100     /* FLASH at 0x10c00000 */
+        movew   #0x018b, 0xfffff110     /* 2Meg, enable, 0ws */
+        movew   #0x8f00, 0xfffffc00     /* DRAM configuration */
+        movew   #0x9667, 0xfffffc02     /* DRAM control */
+        movew   #0x0000, 0xfffff106     /* DRAM at 0x00000000 */
+        movew   #0x068f, 0xfffff116     /* 8Meg, enable, 0ws */
+        moveb   #0x40, 0xfffff300       /* IVR */
+        movel   #0x007FFFFF, %d0        /* IMR */
+        movel   %d0, 0xfffff304
+        moveb   0xfffff42b, %d0
+        andb    #0xe0, %d0
+        moveb   %d0, 0xfffff42b
+#endif
+/*****************************************************************************
+ * Init LCD controller.
+ * (Assuming that LCD controller is already init'ed when running from RAM)
+ *****************************************************************************/
+#ifdef CONFIG_INIT_LCD
+#ifdef CONFIG_PILOT
+        moveb   #0, 0xfffffA27                  /* LCKCON */
+        movel   #_start, 0xfffffA00             /* LSSA */
+        moveb   #0xa, 0xfffffA05                /* LVPW */
+        movew   #0x9f, 0xFFFFFa08               /* LXMAX */
+        movew   #0x9f, 0xFFFFFa0a               /* LYMAX */
+        moveb   #9, 0xfffffa29                  /* LBAR */
+        moveb   #0, 0xfffffa25                  /* LPXCD */
+        moveb   #0x04, 0xFFFFFa20               /* LPICF */
+        moveb   #0x58, 0xfffffA27               /* LCKCON */
+        moveb   #0x85, 0xfffff429               /* PFDATA */
+        moveb   #0xd8, 0xfffffA27               /* LCKCON */
+        moveb   #0xc5, 0xfffff429               /* PFDATA */
+        moveb   #0xd5, 0xfffff429               /* PFDATA */
+        movel   #bootlogo_bits, 0xFFFFFA00      /* LSSA */
+        moveb   #10, 0xFFFFFA05                 /* LVPW */
+        movew   #160, 0xFFFFFA08                /* LXMAX */
+        movew   #160, 0xFFFFFA0A                /* LYMAX */
+#else /* CONFIG_PILOT */
+        movel   #bootlogo_bits, 0xfffffA00      /* LSSA */
+        moveb   #0x28, 0xfffffA05               /* LVPW */
+        movew   #0x280, 0xFFFFFa08              /* LXMAX */
+        movew   #0x1df, 0xFFFFFa0a              /* LYMAX */
+        moveb   #0, 0xfffffa29                  /* LBAR */
+        moveb   #0, 0xfffffa25                  /* LPXCD */
+        moveb   #0x08, 0xFFFFFa20               /* LPICF */
+        moveb   #0x01, 0xFFFFFA21               /* -ve pol */
+        moveb   #0x81, 0xfffffA27               /* LCKCON */
+        movew   #0xff00, 0xfffff412             /* LCD pins */
+#endif /* CONFIG_PILOT */
+#endif /* CONFIG_INIT_LCD */
+/*****************************************************************************
+ * Kernel is running from FLASH/ROM (XIP)
+ * Copy init text & data to RAM
+ *****************************************************************************/
+        moveal  #_etext, %a0
+        moveal  #_sdata, %a1
+        moveal  #__bss_start, %a2
+_copy_initmem:
+        movel   %a0@+, %a1@+
+        cmpal   %a1, %a2
+        bhi     _copy_initmem
+#endif /* CONFIG_ROMKERNEL */
+/*****************************************************************************
+ * Setup basic memory information for kernel
+ *****************************************************************************/
+        movel   #CONFIG_VECTORBASE,_ramvec      /* set vector base location */
+        movel   #CONFIG_RAMBASE,_rambase        /* set the base of RAM */
+        movel   #RAMEND, _ramend                /* set end ram addr */
+        lea     __bss_stop,%a1
+        movel   %a1,_ramstart
+/*****************************************************************************
+ * If the kernel is in RAM, move romfs to right above bss and
+ * adjust _ramstart to where romfs ends.
+ *
+ * (Do this only if CONFIG_MTD_UCLINUX is true)
+ *****************************************************************************/
+#if defined(CONFIG_ROMFS_FS) && defined(CONFIG_RAMKERNEL) && \
+    defined(CONFIG_MTD_UCLINUX)
+        lea     __bss_start, %a0                /* get start of bss */
+        lea     __bss_stop, %a1                 /* set up destination  */
+        movel   %a0, %a2                        /* copy of bss start */
+        movel   8(%a0), %d0                     /* get size of ROMFS */
+        addql   #8, %d0                         /* allow for rounding */
+        andl    #0xfffffffc, %d0                /* whole words */
+        addl    %d0, %a0                        /* copy from end */
+        addl    %d0, %a1                        /* copy from end */
+        movel   %a1, _ramstart                  /* set start of ram */
+_copy_romfs:
+        movel   -(%a0), -(%a1)                  /* copy dword */
+        cmpl    %a0, %a2                        /* check if at end */
+        bne     _copy_romfs
+#endif /* CONFIG_ROMFS_FS && CONFIG_RAMKERNEL && CONFIG_MTD_UCLINUX */
+/*****************************************************************************
+ * Clear bss region
+ *****************************************************************************/
+        lea     __bss_start, %a0                /* get start of bss */
+        lea     __bss_stop, %a1                 /* get end of bss */
+_clear_bss:
+        movel   #0, (%a0)+                      /* clear each word */
+        cmpl    %a0, %a1                        /* check if at end */
+        bne     _clear_bss
+/*****************************************************************************
+ * Load the current task pointer and stack.
+ *****************************************************************************/
+        lea     init_thread_union,%a0
+        lea     THREAD_SIZE(%a0),%sp
+        jsr     start_kernel                    /* start Linux kernel */
+_exit:
+        jmp     _exit                           /* should never get here */
diff --git a/arch/m68k/platform/68328/ints.c b/arch/m68k/platform/68000/ints.c
index b3810febb3e3..cda49b12d7be 100644
--- a/arch/m68k/platform/68328/ints.c
+++ b/arch/m68k/platform/68000/ints.c
@@ -1,5 +1,5 @@
 /*
- * linux/arch/m68knommu/platform/68328/ints.c
+ * ints.c - Generic interrupt controller support
 *
 * This file is subject to the terms and conditions of the GNU General Public
 * License.  See the file COPYING in the main directory of this archive
diff --git a/arch/m68k/platform/68328/config.c b/arch/m68k/platform/68000/m68328.c
index 8c20e891e981..a86eb66835aa 100644
--- a/arch/m68k/platform/68328/config.c
+++ b/arch/m68k/platform/68000/m68328.c
@@ -1,7 +1,7 @@
 /***************************************************************************/
 /*
- *  linux/arch/m68knommu/platform/68328/config.c
+ *  m68328.c - 68328 specific config
 *
 *  Copyright (C) 1993 Hamish Macdonald
 *  Copyright (C) 1999 D. Jeff Dionne
diff --git a/arch/m68k/platform/68EZ328/config.c b/arch/m68k/platform/68000/m68EZ328.c
index 4f158d551f02..a6eb72d75008 100644
--- a/arch/m68k/platform/68EZ328/config.c
+++ b/arch/m68k/platform/68000/m68EZ328.c
@@ -1,7 +1,7 @@
 /***************************************************************************/
 /*
- *  linux/arch/m68knommu/platform/68EZ328/config.c
+ *  m68EZ328.c - 68EZ328 specific config
 *
 *  Copyright (C) 1993 Hamish Macdonald
 *  Copyright (C) 1999 D. Jeff Dionne
diff --git a/arch/m68k/platform/68VZ328/config.c b/arch/m68k/platform/68000/m68VZ328.c
index 2ed8dc305e42..eb6964fbec09 100644
--- a/arch/m68k/platform/68VZ328/config.c
+++ b/arch/m68k/platform/68000/m68VZ328.c
@@ -1,7 +1,7 @@
 /***************************************************************************/
 /*
- *  linux/arch/m68knommu/platform/68VZ328/config.c
+ *  m68VZ328.c - 68VZ328 specific config
 *
 *  Copyright (C) 1993 Hamish Macdonald
 *  Copyright (C) 1999 D. Jeff Dionne
@@ -28,7 +28,7 @@
 #include <asm/bootstd.h>
 #ifdef CONFIG_INIT_LCD
-#include "bootlogo.h"
+#include "bootlogo-vz.h"
 #endif
 /***************************************************************************/
diff --git a/arch/m68k/platform/68328/romvec.S b/arch/m68k/platform/68000/romvec.S
index 31084466eae8..15c70cd6453f 100644
--- a/arch/m68k/platform/68328/romvec.S
+++ b/arch/m68k/platform/68000/romvec.S
@@ -1,5 +1,5 @@
 /*
- * linux/arch/m68knommu/platform/68328/romvec.S
+ * romvec.S - Vector table for 68000 cpus
 *
 * This file is subject to the terms and conditions of the GNU General Public
 * License.  See the file COPYING in the main directory of this archive
diff --git a/arch/m68k/platform/68328/timers.c b/arch/m68k/platform/68000/timers.c
index f4dc9b295609..ec30acbfe6db 100644
--- a/arch/m68k/platform/68328/timers.c
+++ b/arch/m68k/platform/68000/timers.c
@@ -1,7 +1,7 @@
 /***************************************************************************/
 /*
- *  linux/arch/m68knommu/platform/68328/timers.c
+ *  timers.c - Generic hardware timer support.
 *
 *  Copyright (C) 1993 Hamish Macdonald
 *  Copyright (C) 1999 D. Jeff Dionne
diff --git a/arch/m68k/platform/68328/Makefile b/arch/m68k/platform/68328/Makefile
deleted file mode 100644
index ee61bf84d4a0..000000000000
--- a/arch/m68k/platform/68328/Makefile
+++ /dev/null
@@ -1,21 +0,0 @@
-#
-# Makefile for arch/m68knommu/platform/68328.
-#
-model-y                   := ram
-model-$(CONFIG_ROMKERNEL) := rom
-head-y                  = head-$(model-y).o
-head-$(CONFIG_PILOT)    = head-pilot.o
-head-$(CONFIG_DRAGEN2)  = head-de2.o
-obj-y                   += entry.o ints.o timers.o
-obj-$(CONFIG_M68328)    += config.o
-obj-$(CONFIG_ROM)       += romvec.o
-extra-y                 := head.o
-$(obj)/head.o: $(obj)/$(head-y)
-        ln -sf $(head-y) $(obj)/head.o
-clean-files := $(obj)/head.o $(head-y)
diff --git a/arch/m68k/platform/68328/head-de2.S b/arch/m68k/platform/68328/head-de2.S
deleted file mode 100644
index 537d3245b539..000000000000
--- a/arch/m68k/platform/68328/head-de2.S
+++ /dev/null
@@ -1,128 +0,0 @@
-#define MEM_END 0x00800000      /* Memory size 8Mb */
-#undef CRT_DEBUG
-.macro  PUTC CHAR
-#ifdef CRT_DEBUG
-        moveq   #\CHAR, %d7
-        jsr     putc
-#endif
-.endm
-        .global _start
-        .global _rambase
-        .global _ramvec
-        .global _ramstart
-        .global _ramend
-        
-        .data
-/*
- *      Set up the usable of RAM stuff
- */
-_rambase:
-        .long   0
-_ramvec:
-        .long   0
-_ramstart:
-        .long   0
-_ramend:
-        .long   0
-        .text
-_start:
-/*
- * Setup initial stack
- */
-        /* disable all interrupts */
-        movew   #0x2700, %sr
-        movel   #-1, 0xfffff304
-        movel   #MEM_END-4, %sp
-        PUTC    '\r'
-        PUTC    '\n'
-        PUTC    'A'
-        PUTC    'B'
-/*
- *      Determine end of RAM
- */
-        movel   #MEM_END, %a0
-        movel   %a0, _ramend
-        PUTC    'C'
-/*
- *      Move ROM filesystem above bss :-)
- */
-        moveal  #__bss_start, %a0               /* romfs at the start of bss */
-        moveal  #__bss_stop, %a1                /* Set up destination  */
-        movel   %a0, %a2                        /* Copy of bss start */
-        movel   8(%a0), %d1                     /* Get size of ROMFS */
-        addql   #8, %d1                         /* Allow for rounding */
-        andl    #0xfffffffc, %d1        /* Whole words */
-        addl    %d1, %a0                        /* Copy from end */
-        addl    %d1, %a1                        /* Copy from end */
-        movel   %a1, _ramstart          /* Set start of ram */
-1:
-        movel   -(%a0), %d0                     /* Copy dword */
-        movel   %d0, -(%a1)
-        cmpl    %a0, %a2                        /* Check if at end */
-        bne     1b
-        PUTC    'D'
-/*
- * Initialize BSS segment to 0
- */
-        lea     __bss_start, %a0
-        lea     __bss_stop, %a1
-        /* Copy 0 to %a0 until %a0 == %a1 */
-2:      cmpal   %a0, %a1
-        beq     1f
-        clrl    (%a0)+
-        bra     2b
-1:
-        PUTC    'E'
-/*
- * Load the current task pointer and stack
- */
-        lea     init_thread_union, %a0
-        lea     0x2000(%a0), %sp
-        PUTC    'F'
-        PUTC    '\r'
-        PUTC    '\n'
-/*
- * Go
- */
-        jmp     start_kernel
-/*
- * Local functions
- */
- 
-#ifdef CRT_DEBUG
-putc:
-        moveb   %d7, 0xfffff907
-1:
-        movew   0xfffff906, %d7
-        andw    #0x2000, %d7
-        beq     1b
-        rts
-#endif
diff --git a/arch/m68k/platform/68328/head-pilot.S b/arch/m68k/platform/68328/head-pilot.S
deleted file mode 100644
index 45a9dad29e3d..000000000000
--- a/arch/m68k/platform/68328/head-pilot.S
+++ /dev/null
@@ -1,207 +0,0 @@
-/*
- * linux/arch/m68knommu/platform/68328/head-pilot.S
- * - A startup file for the MC68328
- *
- * Copyright (C) 1998  D. Jeff Dionne <jeff@ryeham.ee.ryerson.ca>,
- *                     Kenneth Albanowski <kjahds@kjahds.com>,
- *                     The Silver Hammer Group, Ltd.
- *
- * (c) 1995, Dionne & Associates
- * (c) 1995, DKG Display Tech.
- */
-#define ASSEMBLY
-#define IMMED #
-#define DBG_PUTC(x)     moveb IMMED x, 0xfffff907
-.global _stext
-.global _start
-.global _rambase
-.global _ramvec
-.global _ramstart
-.global _ramend
-.global bootlogo_bits
-/*****************************************************************************/
-.data
-/*
- *      Set up the usable of RAM stuff. Size of RAM is determined then
- *      an initial stack set up at the end.
- */
-.align 4
-_ramvec:
-.long   0
-_rambase:
-.long   0
-_ramstart:
-.long   0
-_ramend:
-.long   0
-.text
-        
-_start:
-_stext:
-#ifdef CONFIG_M68328
-#ifdef CONFIG_PILOT
-        .byte 0x4e, 0xfa, 0x00, 0x0a /* Jmp +X bytes */
-        .byte 'b', 'o', 'o', 't'
-        .word 10000
-        nop
-#endif
-        moveq   #0, %d0
-        movew   %d0, 0xfffff618 /* Watchdog off */
-        movel   #0x00011f07, 0xfffff114 /* CS A1 Mask */
-        movew   #0x0800, 0xfffff906 /* Ignore CTS */
-        movew   #0x010b, 0xfffff902 /* BAUD to 9600 */
-        movew   #0x2410, 0xfffff200 /* PLLCR */
-        movew   #0x123, 0xfffff202 /* PLLFSR */
-#ifdef CONFIG_PILOT
-        moveb   #0, 0xfffffA27 /* LCKCON */
-        movel   #_start, 0xfffffA00 /* LSSA */
-        moveb   #0xa, 0xfffffA05 /* LVPW */
-        movew   #0x9f, 0xFFFFFa08 /* LXMAX */
-        movew   #0x9f, 0xFFFFFa0a /* LYMAX */
-        moveb   #9, 0xfffffa29 /* LBAR */
-        moveb   #0, 0xfffffa25 /* LPXCD */
-        moveb   #0x04, 0xFFFFFa20 /* LPICF */
-        moveb   #0x58, 0xfffffA27 /* LCKCON */
-        moveb   #0x85, 0xfffff429 /* PFDATA */
-        moveb   #0xd8, 0xfffffA27 /* LCKCON */
-        moveb   #0xc5, 0xfffff429 /* PFDATA */
-        moveb   #0xd5, 0xfffff429 /* PFDATA */
-        moveal  #0x00100000, %a3
-        moveal  #0x100ffc00, %a4
-#endif /* CONFIG_PILOT */
-#endif /* CONFIG_M68328 */
-        movew   #0x2700, %sr
-        lea     %a4@(-4), %sp
-        DBG_PUTC('\r')
-        DBG_PUTC('\n')
-        DBG_PUTC('A')
-        moveq   #0,%d0
-        movew   #16384, %d0  /* PLL settle wait loop */
-L0:
-        subw    #1, %d0
-        bne     L0
-        DBG_PUTC('B')
-        /* Copy command line from beginning of RAM (+16) to end of bss */
-        movel   #CONFIG_VECTORBASE, %d7
-        addl    #16, %d7
-        moveal  %d7, %a0
-        moveal  #__bss_stop, %a1
-        lea     %a1@(512), %a2
-        DBG_PUTC('C')
-        /* Copy %a0 to %a1 until %a1 == %a2 */
-L2:
-        movel   %a0@+, %d0
-        movel   %d0, %a1@+
-        cmpal   %a1, %a2
-        bhi     L2
-        /* Copy data+init segment from ROM to RAM */
-        moveal  #_etext, %a0
-        moveal  #_sdata, %a1
-        moveal  #__init_end, %a2
-        DBG_PUTC('D')
-        /* Copy %a0 to %a1 until %a1 == %a2 */
-LD1:
-        movel   %a0@+, %d0
-        movel   %d0, %a1@+
-        cmpal   %a1, %a2
-        bhi     LD1
-        DBG_PUTC('E')
-        moveal  #__bss_start, %a0
-        moveal  #__bss_stop, %a1
-        /* Copy 0 to %a0 until %a0 == %a1 */
-L1:
-        movel   #0, %a0@+
-        cmpal   %a0, %a1
-        bhi     L1
-        DBG_PUTC('F')
-        /* Copy command line from end of bss to command line */
-        moveal  #__bss_stop, %a0
-        moveal  #command_line, %a1
-        lea     %a1@(512), %a2
-        DBG_PUTC('G')
-        /* Copy %a0 to %a1 until %a1 == %a2 */
-L3:
-        movel   %a0@+, %d0
-        movel   %d0, %a1@+
-        cmpal   %a1, %a2
-        bhi     L3
-        movel   #_sdata, %d0    
-        movel   %d0, _rambase   
-        movel   #__bss_stop, %d0
-        movel   %d0, _ramstart
-        movel   %a4, %d0
-        subl    #4096, %d0      /* Reserve 4K of stack */
-        moveq   #79, %d7
-        movel   %d0, _ramend
-        pea     0
-        pea     env
-        pea     %sp@(4)
-        pea     0
-        DBG_PUTC('H')
-#ifdef CONFIG_PILOT
-        movel   #bootlogo_bits, 0xFFFFFA00
-        moveb   #10, 0xFFFFFA05
-        movew   #160, 0xFFFFFA08
-        movew   #160, 0xFFFFFA0A
-#endif /* CONFIG_PILOT */
-        DBG_PUTC('I')
-        lea     init_thread_union, %a0
-        lea     0x2000(%a0), %sp
-        DBG_PUTC('J')
-        DBG_PUTC('\r')
-        DBG_PUTC('\n')
-        jsr     start_kernel
-_exit:
-        jmp     _exit
-        .data
-env:
-        .long   0
diff --git a/arch/m68k/platform/68328/head-ram.S b/arch/m68k/platform/68328/head-ram.S
deleted file mode 100644
index 5189ef926098..000000000000
--- a/arch/m68k/platform/68328/head-ram.S
+++ /dev/null
@@ -1,141 +0,0 @@
-        .global __main
-        .global __rom_start
-        .global _rambase
-        .global _ramstart
-        
-        .global splash_bits
-        .global _start
-        .global _stext
-        .global _edata
-#define DEBUG
-#define ROM_OFFSET 0x10C00000
-#define STACK_GAURD 0x10
-        .text
-        
-_start:
-_stext:
-        movew   #0x2700, %sr            /* Exceptions off! */
-#if 0
-        /* Init chip registers.  uCsimm specific */
-        moveb   #0x00,   0xfffffb0b     /* Watchdog off */
-        moveb   #0x10,   0xfffff000     /* SCR */
-        movew   #0x2400, 0xfffff200     /* PLLCR */
-        movew   #0x0123, 0xfffff202     /* PLLFSR */
-        moveb   #0x00,   0xfffff40b     /* enable chip select */
-        moveb   #0x00,   0xfffff423     /* enable /DWE */
-        moveb   #0x08,   0xfffffd0d     /* disable hardmap */
-        moveb   #0x07,   0xfffffd0e     /* level 7 interrupt clear */
-        movew   #0x8600, 0xfffff100     /* FLASH at 0x10c00000 */
-        movew   #0x018b, 0xfffff110     /* 2Meg, enable, 0ws */
-        movew   #0x8f00, 0xfffffc00     /* DRAM configuration */
-        movew   #0x9667, 0xfffffc02     /* DRAM control */
-        movew   #0x0000, 0xfffff106     /* DRAM at 0x00000000 */
-        movew   #0x068f, 0xfffff116     /* 8Meg, enable, 0ws */
-        moveb   #0x40,   0xfffff300     /* IVR */
-        movel   #0x007FFFFF, %d0        /* IMR */
-        movel   %d0,     0xfffff304
-        moveb   0xfffff42b, %d0
-        andb    #0xe0,   %d0
-        moveb   %d0,     0xfffff42b
-        moveb   #0x08,   0xfffff907     /* Ignore CTS */
-        movew   #0x010b, 0xfffff902     /* BAUD to 9600 */
-        movew   #0xe100, 0xfffff900     /* enable */
-#endif
-        movew   #16384, %d0  /* PLL settle wait loop */
-L0:
-        subw    #1, %d0
-        bne     L0
-#ifdef DEBUG
-        moveq   #70, %d7                /* 'F' */
-        moveb   %d7,0xfffff907          /* No absolute addresses */
-pclp1:
-        movew   0xfffff906, %d7
-        andw    #0x2000, %d7
-        beq     pclp1
-#endif /* DEBUG */
-#ifdef DEBUG
-        moveq   #82, %d7                /* 'R' */
-        moveb   %d7,0xfffff907          /* No absolute addresses */
-pclp3:
-        movew   0xfffff906, %d7
-        andw    #0x2000, %d7
-        beq     pclp3
-#endif /* DEBUG */
-        moveal  #0x007ffff0, %ssp
-        moveal  #__bss_start, %a0
-        moveal  #__bss_stop, %a1
-        /* Copy 0 to %a0 until %a0 >= %a1 */
-L1:
-        movel   #0, %a0@+
-        cmpal   %a0, %a1
-        bhi     L1
-#ifdef DEBUG
-        moveq   #67, %d7                /* 'C' */
-        jsr     putc
-#endif /* DEBUG */
-        pea     0
-        pea     env
-        pea     %sp@(4)
-        pea     0
-#ifdef DEBUG
-        moveq   #70, %d7                /* 'F' */
-        jsr     putc
-#endif /* DEBUG */
-lp:
-        jsr     start_kernel
-        jmp lp
-_exit:
-        jmp     _exit
-__main:
-        /* nothing */
-        rts
-#ifdef DEBUG
-putc:
-        moveb   %d7,0xfffff907
-pclp:
-        movew   0xfffff906, %d7
-        andw    #0x2000, %d7
-        beq     pclp
-        rts
-#endif /* DEBUG */
-        .data
-/*
- *      Set up the usable of RAM stuff. Size of RAM is determined then
- *      an initial stack set up at the end.
- */
-.align 4
-_ramvec:
-.long   0
-_rambase:
-.long   0
-_ramstart:
-.long   0
-_ramend:
-.long   0
-env:
-        .long   0
diff --git a/arch/m68k/platform/68328/head-rom.S b/arch/m68k/platform/68328/head-rom.S
deleted file mode 100644
index 3dff98ba2e97..000000000000
--- a/arch/m68k/platform/68328/head-rom.S
+++ /dev/null
@@ -1,105 +0,0 @@
-        
-        .global _start
-        .global _stext
-        .global _rambase
-        .global _ramvec
-        .global _ramstart
-        .global _ramend
-#ifdef CONFIG_INIT_LCD
-        .global bootlogo_bits
-#endif
-        .data
-/*
- *      Set up the usable of RAM stuff. Size of RAM is determined then
- *      an initial stack set up at the end.
- */
-.align 4
-_ramvec:
-.long   0
-_rambase:
-.long   0
-_ramstart:
-.long   0
-_ramend:
-.long   0
-#define RAMEND  (CONFIG_RAMBASE + CONFIG_RAMSIZE)
-        .text
-_start:
-_stext: movew   #0x2700,%sr
-#ifdef CONFIG_INIT_LCD
-        movel   #bootlogo_bits, 0xfffffA00 /* LSSA */
-        moveb   #0x28,   0xfffffA05     /* LVPW */
-        movew   #0x280,  0xFFFFFa08     /* LXMAX */
-        movew   #0x1df,  0xFFFFFa0a     /* LYMAX */
-        moveb   #0,      0xfffffa29     /* LBAR */
-        moveb   #0,      0xfffffa25     /* LPXCD */
-        moveb   #0x08,   0xFFFFFa20     /* LPICF */
-        moveb   #0x01,   0xFFFFFA21     /* -ve pol */
-        moveb   #0x81,   0xfffffA27     /* LCKCON */
-        movew   #0xff00, 0xfffff412     /* LCD pins */
-#endif
-        moveal  #RAMEND-CONFIG_MEMORY_RESERVE*0x100000 - 0x10, %sp
-        movew   #32767, %d0  /* PLL settle wait loop */
-1:      subq    #1, %d0
-        bne     1b
-        /* Copy data segment from ROM to RAM */
-        moveal  #_etext, %a0
-        moveal  #_sdata, %a1
-        moveal  #_edata, %a2
-        /* Copy %a0 to %a1 until %a1 == %a2 */
-1:      movel   %a0@+, %a1@+
-        cmpal   %a1, %a2
-        bhi     1b
-        moveal  #__bss_start, %a0
-        moveal  #__bss_stop, %a1
-        /* Copy 0 to %a0 until %a0 == %a1 */
-        
-1:
-        clrl    %a0@+
-        cmpal   %a0, %a1
-        bhi     1b
-        movel   #_sdata, %d0    
-        movel   %d0, _rambase        
-        movel   #__bss_stop, %d0
-        movel   %d0, _ramstart
-        movel   #RAMEND-CONFIG_MEMORY_RESERVE*0x100000, %d0
-        movel   %d0, _ramend
-        movel   #CONFIG_VECTORBASE,     %d0
-        movel   %d0, _ramvec
-        
-/*
- * load the current task pointer and stack
- */
-        lea     init_thread_union, %a0
-        lea     0x2000(%a0), %sp
-1:      jsr     start_kernel
-        bra 1b
-_exit:
-        jmp     _exit
-putc:
-        moveb   %d7,0xfffff907
-1:
-        movew   0xfffff906, %d7
-        andw    #0x2000, %d7
-        beq     1b
-        rts
-        .data
-env:
-        .long   0
-        .text
diff --git a/arch/m68k/platform/68EZ328/Makefile b/arch/m68k/platform/68EZ328/Makefile
deleted file mode 100644
index b44d799b1115..000000000000
--- a/arch/m68k/platform/68EZ328/Makefile
+++ /dev/null
@@ -1,5 +0,0 @@
-#
-# Makefile for arch/m68knommu/platform/68EZ328.
-#
-obj-y := config.o
diff --git a/arch/m68k/platform/68VZ328/Makefile b/arch/m68k/platform/68VZ328/Makefile
deleted file mode 100644
index 816674164682..000000000000
--- a/arch/m68k/platform/68VZ328/Makefile
+++ /dev/null
@@ -1,5 +0,0 @@
-#
-# Makefile for arch/m68k/platform/68VZ328.
-#
-obj-y           := config.o
diff --git a/arch/m68k/platform/coldfire/clk.c b/arch/m68k/platform/coldfire/clk.c
index 9cd13b4ce42b..fddfdccae63b 100644
--- a/arch/m68k/platform/coldfire/clk.c
+++ b/arch/m68k/platform/coldfire/clk.c
@@ -19,37 +19,58 @@
 #include <asm/mcfsim.h>
 #include <asm/mcfclk.h>
-/***************************************************************************/
+static DEFINE_SPINLOCK(clk_lock);
-#ifndef MCFPM_PPMCR0
-struct clk *clk_get(struct device *dev, const char *id)
+#ifdef MCFPM_PPMCR0
+/*
+ *      For more advanced ColdFire parts that have clocks that can be enabled
+ *      we supply enable/disable functions. These must properly define their
+ *      clocks in their platform specific code.
+ */
+void __clk_init_enabled(struct clk *clk)
 {
-        return NULL;
+        clk->enabled = 1;
+        clk->clk_ops->enable(clk);
 }
-EXPORT_SYMBOL(clk_get);
-int clk_enable(struct clk *clk)
+void __clk_init_disabled(struct clk *clk)
 {
-        return 0;
+        clk->enabled = 0;
+        clk->clk_ops->disable(clk);
 }
-EXPORT_SYMBOL(clk_enable);
-void clk_disable(struct clk *clk)
+static void __clk_enable0(struct clk *clk)
 {
+        __raw_writeb(clk->slot, MCFPM_PPMCR0);
 }
-EXPORT_SYMBOL(clk_disable);
-void clk_put(struct clk *clk)
+static void __clk_disable0(struct clk *clk)
+{
+        __raw_writeb(clk->slot, MCFPM_PPMSR0);
+}
+struct clk_ops clk_ops0 = {
+        .enable         = __clk_enable0,
+        .disable        = __clk_disable0,
+};
+#ifdef MCFPM_PPMCR1
+static void __clk_enable1(struct clk *clk)
 {
+        __raw_writeb(clk->slot, MCFPM_PPMCR1);
 }
-EXPORT_SYMBOL(clk_put);
-unsigned long clk_get_rate(struct clk *clk)
+static void __clk_disable1(struct clk *clk)
 {
-        return MCF_CLK;
+        __raw_writeb(clk->slot, MCFPM_PPMSR1);
 }
-EXPORT_SYMBOL(clk_get_rate);
-#else
+struct clk_ops clk_ops1 = {
-static DEFINE_SPINLOCK(clk_lock);
+        .enable         = __clk_enable1,
+        .disable        = __clk_disable1,
+};
+#endif /* MCFPM_PPMCR1 */
+#endif /* MCFPM_PPMCR0 */
 struct clk *clk_get(struct device *dev, const char *id)
 {
@@ -101,48 +122,3 @@ unsigned long clk_get_rate(struct clk *clk)
 EXPORT_SYMBOL(clk_get_rate);
 /***************************************************************************/
-void __clk_init_enabled(struct clk *clk)
-{
-        clk->enabled = 1;
-        clk->clk_ops->enable(clk);
-}
-void __clk_init_disabled(struct clk *clk)
-{
-        clk->enabled = 0;
-        clk->clk_ops->disable(clk);
-}
-static void __clk_enable0(struct clk *clk)
-{
-        __raw_writeb(clk->slot, MCFPM_PPMCR0);
-}
-static void __clk_disable0(struct clk *clk)
-{
-        __raw_writeb(clk->slot, MCFPM_PPMSR0);
-}
-struct clk_ops clk_ops0 = {
-        .enable         = __clk_enable0,
-        .disable        = __clk_disable0,
-};
-#ifdef MCFPM_PPMCR1
-static void __clk_enable1(struct clk *clk)
-{
-        __raw_writeb(clk->slot, MCFPM_PPMCR1);
-}
-static void __clk_disable1(struct clk *clk)
-{
-        __raw_writeb(clk->slot, MCFPM_PPMSR1);
-}
-struct clk_ops clk_ops1 = {
-        .enable         = __clk_enable1,
-        .disable        = __clk_disable1,
-};
-#endif /* MCFPM_PPMCR1 */
-#endif /* MCFPM_PPMCR0 */
diff --git a/arch/m68k/platform/coldfire/intc-5249.c b/arch/m68k/platform/coldfire/intc-5249.c
index 0864b836699a..b0d1641053e4 100644
--- a/arch/m68k/platform/coldfire/intc-5249.c
+++ b/arch/m68k/platform/coldfire/intc-5249.c
@@ -21,7 +21,7 @@ static void intc2_irq_gpio_mask(struct irq_data *d)
 {
        u32 imr;
        imr = readl(MCFSIM2_GPIOINTENABLE);
-        imr &= ~(0x1 << (d->irq - MCFINTC2_GPIOIRQ0));
+        imr &= ~(0x1 << (d->irq - MCF_IRQ_GPIO0));
        writel(imr, MCFSIM2_GPIOINTENABLE);
 }
@@ -29,13 +29,13 @@ static void intc2_irq_gpio_unmask(struct irq_data *d)
 {
        u32 imr;
        imr = readl(MCFSIM2_GPIOINTENABLE);
-        imr |= (0x1 << (d->irq - MCFINTC2_GPIOIRQ0));
+        imr |= (0x1 << (d->irq - MCF_IRQ_GPIO0));
        writel(imr, MCFSIM2_GPIOINTENABLE);
 }
 static void intc2_irq_gpio_ack(struct irq_data *d)
 {
-        writel(0x1 << (d->irq - MCFINTC2_GPIOIRQ0), MCFSIM2_GPIOINTCLEAR);
+        writel(0x1 << (d->irq - MCF_IRQ_GPIO0), MCFSIM2_GPIOINTCLEAR);
 }
 static struct irq_chip intc2_irq_gpio_chip = {
@@ -50,7 +50,7 @@ static int __init mcf_intc2_init(void)
        int irq;
        /* GPIO interrupt sources */
-        for (irq = MCFINTC2_GPIOIRQ0; (irq <= MCFINTC2_GPIOIRQ7); irq++) {
+        for (irq = MCF_IRQ_GPIO0; (irq <= MCF_IRQ_GPIO7); irq++) {
                irq_set_chip(irq, &intc2_irq_gpio_chip);
                irq_set_handler(irq, handle_edge_irq);
        }
diff --git a/arch/m68k/platform/coldfire/m5206.c b/arch/m68k/platform/coldfire/m5206.c
index 6bfbeebd231b..0e55f449a88c 100644
--- a/arch/m68k/platform/coldfire/m5206.c
+++ b/arch/m68k/platform/coldfire/m5206.c
@@ -16,6 +16,26 @@
 #include <asm/machdep.h>
 #include <asm/coldfire.h>
 #include <asm/mcfsim.h>
+#include <asm/mcfclk.h>
+/***************************************************************************/
+DEFINE_CLK(pll, "pll.0", MCF_CLK);
+DEFINE_CLK(sys, "sys.0", MCF_BUSCLK);
+DEFINE_CLK(mcftmr0, "mcftmr.0", MCF_BUSCLK);
+DEFINE_CLK(mcftmr1, "mcftmr.1", MCF_BUSCLK);
+DEFINE_CLK(mcfuart0, "mcfuart.0", MCF_BUSCLK);
+DEFINE_CLK(mcfuart1, "mcfuart.1", MCF_BUSCLK);
+struct clk *mcf_clks[] = {
+        &clk_pll,
+        &clk_sys,
+        &clk_mcftmr0,
+        &clk_mcftmr1,
+        &clk_mcfuart0,
+        &clk_mcfuart1,
+        NULL
+};
 /***************************************************************************/
diff --git a/arch/m68k/platform/coldfire/m523x.c b/arch/m68k/platform/coldfire/m523x.c
index ff37fe9553ea..2b10e9f198cd 100644
--- a/arch/m68k/platform/coldfire/m523x.c
+++ b/arch/m68k/platform/coldfire/m523x.c
@@ -19,6 +19,34 @@
 #include <asm/machdep.h>
 #include <asm/coldfire.h>
 #include <asm/mcfsim.h>
+#include <asm/mcfclk.h>
+/***************************************************************************/
+DEFINE_CLK(pll, "pll.0", MCF_CLK);
+DEFINE_CLK(sys, "sys.0", MCF_BUSCLK);
+DEFINE_CLK(mcfpit0, "mcfpit.0", MCF_CLK);
+DEFINE_CLK(mcfpit1, "mcfpit.1", MCF_CLK);
+DEFINE_CLK(mcfpit2, "mcfpit.2", MCF_CLK);
+DEFINE_CLK(mcfpit3, "mcfpit.3", MCF_CLK);
+DEFINE_CLK(mcfuart0, "mcfuart.0", MCF_BUSCLK);
+DEFINE_CLK(mcfuart1, "mcfuart.1", MCF_BUSCLK);
+DEFINE_CLK(mcfuart2, "mcfuart.2", MCF_BUSCLK);
+DEFINE_CLK(fec0, "fec.0", MCF_BUSCLK);
+struct clk *mcf_clks[] = {
+        &clk_pll,
+        &clk_sys,
+        &clk_mcfpit0,
+        &clk_mcfpit1,
+        &clk_mcfpit2,
+        &clk_mcfpit3,
+        &clk_mcfuart0,
+        &clk_mcfuart1,
+        &clk_mcfuart2,
+        &clk_fec0,
+        NULL
+};
 /***************************************************************************/
diff --git a/arch/m68k/platform/coldfire/m5249.c b/arch/m68k/platform/coldfire/m5249.c
index 23b19cb7ab50..c80b5e51d29a 100644
--- a/arch/m68k/platform/coldfire/m5249.c
+++ b/arch/m68k/platform/coldfire/m5249.c
@@ -16,6 +16,26 @@
 #include <asm/machdep.h>
 #include <asm/coldfire.h>
 #include <asm/mcfsim.h>
+#include <asm/mcfclk.h>
+/***************************************************************************/
+DEFINE_CLK(pll, "pll.0", MCF_CLK);
+DEFINE_CLK(sys, "sys.0", MCF_BUSCLK);
+DEFINE_CLK(mcftmr0, "mcftmr.0", MCF_BUSCLK);
+DEFINE_CLK(mcftmr1, "mcftmr.1", MCF_BUSCLK);
+DEFINE_CLK(mcfuart0, "mcfuart.0", MCF_BUSCLK);
+DEFINE_CLK(mcfuart1, "mcfuart.1", MCF_BUSCLK);
+struct clk *mcf_clks[] = {
+        &clk_pll,
+        &clk_sys,
+        &clk_mcftmr0,
+        &clk_mcftmr1,
+        &clk_mcfuart0,
+        &clk_mcfuart1,
+        NULL
+};
 /***************************************************************************/
@@ -28,8 +48,8 @@ static struct resource m5249_smc91x_resources[] = {
                .flags          = IORESOURCE_MEM,
        },
        {
-                .start          = MCFINTC2_GPIOIRQ6,
+                .start          = MCF_IRQ_GPIO6,
-                .end            = MCFINTC2_GPIOIRQ6,
+                .end            = MCF_IRQ_GPIO6,
                .flags          = IORESOURCE_IRQ,
        },
 };
@@ -75,8 +95,8 @@ static void __init m5249_smc91x_init(void)
        gpio = readl(MCFSIM2_GPIOINTENABLE);
        writel(gpio | 0x40, MCFSIM2_GPIOINTENABLE);
-        gpio = readl(MCFSIM2_INTLEVEL5);
+        gpio = readl(MCFINTC2_INTPRI5);
-        writel(gpio | 0x04000000, MCFSIM2_INTLEVEL5);
+        writel(gpio | 0x04000000, MCFINTC2_INTPRI5);
 }
 #endif /* CONFIG_M5249C3 */
diff --git a/arch/m68k/platform/coldfire/m525x.c b/arch/m68k/platform/coldfire/m525x.c
index fce8f8a45bf0..5b9f657b2df0 100644
--- a/arch/m68k/platform/coldfire/m525x.c
+++ b/arch/m68k/platform/coldfire/m525x.c
@@ -16,6 +16,26 @@
 #include <asm/machdep.h>
 #include <asm/coldfire.h>
 #include <asm/mcfsim.h>
+#include <asm/mcfclk.h>
+/***************************************************************************/
+DEFINE_CLK(pll, "pll.0", MCF_CLK);
+DEFINE_CLK(sys, "sys.0", MCF_BUSCLK);
+DEFINE_CLK(mcftmr0, "mcftmr.0", MCF_BUSCLK);
+DEFINE_CLK(mcftmr1, "mcftmr.1", MCF_BUSCLK);
+DEFINE_CLK(mcfuart0, "mcfuart.0", MCF_BUSCLK);
+DEFINE_CLK(mcfuart1, "mcfuart.1", MCF_BUSCLK);
+struct clk *mcf_clks[] = {
+        &clk_pll,
+        &clk_sys,
+        &clk_mcftmr0,
+        &clk_mcftmr1,
+        &clk_mcfuart0,
+        &clk_mcfuart1,
+        NULL
+};
 /***************************************************************************/
diff --git a/arch/m68k/platform/coldfire/m5272.c b/arch/m68k/platform/coldfire/m5272.c
index 45b246d052ef..a8c5856fe5ec 100644
--- a/arch/m68k/platform/coldfire/m5272.c
+++ b/arch/m68k/platform/coldfire/m5272.c
@@ -19,6 +19,7 @@
 #include <asm/coldfire.h>
 #include <asm/mcfsim.h>
 #include <asm/mcfuart.h>
+#include <asm/mcfclk.h>
 /***************************************************************************/
@@ -30,6 +31,31 @@ unsigned char ledbank = 0xff;
 /***************************************************************************/
+DEFINE_CLK(pll, "pll.0", MCF_CLK);
+DEFINE_CLK(sys, "sys.0", MCF_BUSCLK);
+DEFINE_CLK(mcftmr0, "mcftmr.0", MCF_BUSCLK);
+DEFINE_CLK(mcftmr1, "mcftmr.1", MCF_BUSCLK);
+DEFINE_CLK(mcftmr2, "mcftmr.2", MCF_BUSCLK);
+DEFINE_CLK(mcftmr3, "mcftmr.3", MCF_BUSCLK);
+DEFINE_CLK(mcfuart0, "mcfuart.0", MCF_BUSCLK);
+DEFINE_CLK(mcfuart1, "mcfuart.1", MCF_BUSCLK);
+DEFINE_CLK(fec0, "fec.0", MCF_BUSCLK);
+struct clk *mcf_clks[] = {
+        &clk_pll,
+        &clk_sys,
+        &clk_mcftmr0,
+        &clk_mcftmr1,
+        &clk_mcftmr2,
+        &clk_mcftmr3,
+        &clk_mcfuart0,
+        &clk_mcfuart1,
+        &clk_fec0,
+        NULL
+};
+/***************************************************************************/
 static void __init m5272_uarts_init(void)
 {
        u32 v;
diff --git a/arch/m68k/platform/coldfire/m527x.c b/arch/m68k/platform/coldfire/m527x.c
index 1431ba03c602..6fbfe9096c3e 100644
--- a/arch/m68k/platform/coldfire/m527x.c
+++ b/arch/m68k/platform/coldfire/m527x.c
@@ -20,6 +20,36 @@
 #include <asm/coldfire.h>
 #include <asm/mcfsim.h>
 #include <asm/mcfuart.h>
+#include <asm/mcfclk.h>
+/***************************************************************************/
+DEFINE_CLK(pll, "pll.0", MCF_CLK);
+DEFINE_CLK(sys, "sys.0", MCF_BUSCLK);
+DEFINE_CLK(mcfpit0, "mcfpit.0", MCF_CLK);
+DEFINE_CLK(mcfpit1, "mcfpit.1", MCF_CLK);
+DEFINE_CLK(mcfpit2, "mcfpit.2", MCF_CLK);
+DEFINE_CLK(mcfpit3, "mcfpit.3", MCF_CLK);
+DEFINE_CLK(mcfuart0, "mcfuart.0", MCF_BUSCLK);
+DEFINE_CLK(mcfuart1, "mcfuart.1", MCF_BUSCLK);
+DEFINE_CLK(mcfuart2, "mcfuart.2", MCF_BUSCLK);
+DEFINE_CLK(fec0, "fec.0", MCF_BUSCLK);
+DEFINE_CLK(fec1, "fec.1", MCF_BUSCLK);
+struct clk *mcf_clks[] = {
+        &clk_pll,
+        &clk_sys,
+        &clk_mcfpit0,
+        &clk_mcfpit1,
+        &clk_mcfpit2,
+        &clk_mcfpit3,
+        &clk_mcfuart0,
+        &clk_mcfuart1,
+        &clk_mcfuart2,
+        &clk_fec0,
+        &clk_fec1,
+        NULL
+};
 /***************************************************************************/
diff --git a/arch/m68k/platform/coldfire/m528x.c b/arch/m68k/platform/coldfire/m528x.c
index f9f7e6a13d04..83b7dad7a84e 100644
--- a/arch/m68k/platform/coldfire/m528x.c
+++ b/arch/m68k/platform/coldfire/m528x.c
@@ -21,6 +21,34 @@
 #include <asm/coldfire.h>
 #include <asm/mcfsim.h>
 #include <asm/mcfuart.h>
+#include <asm/mcfclk.h>
+/***************************************************************************/
+DEFINE_CLK(pll, "pll.0", MCF_CLK);
+DEFINE_CLK(sys, "sys.0", MCF_BUSCLK);
+DEFINE_CLK(mcfpit0, "mcfpit.0", MCF_CLK);
+DEFINE_CLK(mcfpit1, "mcfpit.1", MCF_CLK);
+DEFINE_CLK(mcfpit2, "mcfpit.2", MCF_CLK);
+DEFINE_CLK(mcfpit3, "mcfpit.3", MCF_CLK);
+DEFINE_CLK(mcfuart0, "mcfuart.0", MCF_BUSCLK);
+DEFINE_CLK(mcfuart1, "mcfuart.1", MCF_BUSCLK);
+DEFINE_CLK(mcfuart2, "mcfuart.2", MCF_BUSCLK);
+DEFINE_CLK(fec0, "fec.0", MCF_BUSCLK);
+struct clk *mcf_clks[] = {
+        &clk_pll,
+        &clk_sys,
+        &clk_mcfpit0,
+        &clk_mcfpit1,
+        &clk_mcfpit2,
+        &clk_mcfpit3,
+        &clk_mcfuart0,
+        &clk_mcfuart1,
+        &clk_mcfuart2,
+        &clk_fec0,
+        NULL
+};
 /***************************************************************************/
diff --git a/arch/m68k/platform/coldfire/m5307.c b/arch/m68k/platform/coldfire/m5307.c
index a568d2870d15..887435361386 100644
--- a/arch/m68k/platform/coldfire/m5307.c
+++ b/arch/m68k/platform/coldfire/m5307.c
@@ -17,6 +17,7 @@
 #include <asm/coldfire.h>
 #include <asm/mcfsim.h>
 #include <asm/mcfwdebug.h>
+#include <asm/mcfclk.h>
 /***************************************************************************/
@@ -28,6 +29,25 @@ unsigned char ledbank = 0xff;
 /***************************************************************************/
+DEFINE_CLK(pll, "pll.0", MCF_CLK);
+DEFINE_CLK(sys, "sys.0", MCF_BUSCLK);
+DEFINE_CLK(mcftmr0, "mcftmr.0", MCF_BUSCLK);
+DEFINE_CLK(mcftmr1, "mcftmr.1", MCF_BUSCLK);
+DEFINE_CLK(mcfuart0, "mcfuart.0", MCF_BUSCLK);
+DEFINE_CLK(mcfuart1, "mcfuart.1", MCF_BUSCLK);
+struct clk *mcf_clks[] = {
+        &clk_pll,
+        &clk_sys,
+        &clk_mcftmr0,
+        &clk_mcftmr1,
+        &clk_mcfuart0,
+        &clk_mcfuart1,
+        NULL
+};
+/***************************************************************************/
 void __init config_BSP(char *commandp, int size)
 {
 #if defined(CONFIG_NETtel) || \
diff --git a/arch/m68k/platform/coldfire/m5407.c b/arch/m68k/platform/coldfire/m5407.c
index bb6c746ae819..2fb3cdbfde30 100644
--- a/arch/m68k/platform/coldfire/m5407.c
+++ b/arch/m68k/platform/coldfire/m5407.c
@@ -16,6 +16,26 @@
 #include <asm/machdep.h>
 #include <asm/coldfire.h>
 #include <asm/mcfsim.h>
+#include <asm/mcfclk.h>
+/***************************************************************************/
+DEFINE_CLK(pll, "pll.0", MCF_CLK);
+DEFINE_CLK(sys, "sys.0", MCF_BUSCLK);
+DEFINE_CLK(mcftmr0, "mcftmr.0", MCF_BUSCLK);
+DEFINE_CLK(mcftmr1, "mcftmr.1", MCF_BUSCLK);
+DEFINE_CLK(mcfuart0, "mcfuart.0", MCF_BUSCLK);
+DEFINE_CLK(mcfuart1, "mcfuart.1", MCF_BUSCLK);
+struct clk *mcf_clks[] = {
+        &clk_pll,
+        &clk_sys,
+        &clk_mcftmr0,
+        &clk_mcftmr1,
+        &clk_mcfuart0,
+        &clk_mcfuart1,
+        NULL
+};
 /***************************************************************************/
diff --git a/arch/m68k/platform/coldfire/m54xx.c b/arch/m68k/platform/coldfire/m54xx.c
index b587bf35175b..952da53aa0bc 100644
--- a/arch/m68k/platform/coldfire/m54xx.c
+++ b/arch/m68k/platform/coldfire/m54xx.c
@@ -14,19 +14,45 @@
 #include <linux/interrupt.h>
 #include <linux/io.h>
 #include <linux/mm.h>
+#include <linux/clk.h>
 #include <linux/bootmem.h>
 #include <asm/pgalloc.h>
 #include <asm/machdep.h>
 #include <asm/coldfire.h>
 #include <asm/m54xxsim.h>
 #include <asm/mcfuart.h>
+#include <asm/mcfclk.h>
 #include <asm/m54xxgpt.h>
+#include <asm/mcfclk.h>
 #ifdef CONFIG_MMU
 #include <asm/mmu_context.h>
 #endif
 /***************************************************************************/
+DEFINE_CLK(pll, "pll.0", MCF_CLK);
+DEFINE_CLK(sys, "sys.0", MCF_BUSCLK);
+DEFINE_CLK(mcfslt0, "mcfslt.0", MCF_BUSCLK);
+DEFINE_CLK(mcfslt1, "mcfslt.1", MCF_BUSCLK);
+DEFINE_CLK(mcfuart0, "mcfuart.0", MCF_BUSCLK);
+DEFINE_CLK(mcfuart1, "mcfuart.1", MCF_BUSCLK);
+DEFINE_CLK(mcfuart2, "mcfuart.2", MCF_BUSCLK);
+DEFINE_CLK(mcfuart3, "mcfuart.3", MCF_BUSCLK);
+struct clk *mcf_clks[] = {
+        &clk_pll,
+        &clk_sys,
+        &clk_mcfslt0,
+        &clk_mcfslt1,
+        &clk_mcfuart0,
+        &clk_mcfuart1,
+        &clk_mcfuart2,
+        &clk_mcfuart3,
+        NULL
+};
+/***************************************************************************/
 static void __init m54xx_uarts_init(void)
 {
        /* enable io pins */
diff --git a/arch/microblaze/include/asm/Kbuild b/arch/microblaze/include/asm/Kbuild
index eb3a46c096fe..d3c51a6a601d 100644
--- a/arch/microblaze/include/asm/Kbuild
+++ b/arch/microblaze/include/asm/Kbuild
@@ -1,6 +1,4 @@
-include include/asm-generic/Kbuild.asm
-header-y  += elf.h
 generic-y += clkdev.h
 generic-y += exec.h
 generic-y += trace_clock.h
diff --git a/arch/microblaze/include/asm/elf.h b/arch/microblaze/include/asm/elf.h
index 640ddd4b6a9b..659024449064 100644
--- a/arch/microblaze/include/asm/elf.h
+++ b/arch/microblaze/include/asm/elf.h
@@ -7,119 +7,24 @@
 * License. See the file "COPYING" in the main directory of this archive
 * for more details.
 */
 #ifndef _ASM_MICROBLAZE_ELF_H
 #define _ASM_MICROBLAZE_ELF_H
-/*
+#include <uapi/asm/elf.h>
- * Note there is no "official" ELF designation for Microblaze.
- * I've snaffled the value from the microblaze binutils source code
- * /binutils/microblaze/include/elf/microblaze.h
- */
-#define EM_MICROBLAZE           189
-#define EM_MICROBLAZE_OLD       0xbaab
-#define ELF_ARCH                EM_MICROBLAZE
-/*
- * This is used to ensure we don't load something for the wrong architecture.
- */
-#define elf_check_arch(x)       ((x)->e_machine == EM_MICROBLAZE \
-                                 || (x)->e_machine == EM_MICROBLAZE_OLD)
-/*
- * These are used to set parameters in the core dumps.
- */
-#define ELF_CLASS       ELFCLASS32
 #ifndef __uClinux__
-/*
- * ELF register definitions..
- */
-#include <asm/ptrace.h>
-#include <asm/byteorder.h>
 #ifndef ELF_GREG_T
-#define ELF_GREG_T
-typedef unsigned long elf_greg_t;
 #endif
 #ifndef ELF_NGREG
-#define ELF_NGREG (sizeof(struct pt_regs) / sizeof(elf_greg_t))
 #endif
 #ifndef ELF_GREGSET_T
-#define ELF_GREGSET_T
-typedef elf_greg_t elf_gregset_t[ELF_NGREG];
 #endif
 #ifndef ELF_FPREGSET_T
-#define ELF_FPREGSET_T
-/* TBD */
-#define ELF_NFPREG      33      /* includes fsr */
-typedef unsigned long elf_fpreg_t;
-typedef elf_fpreg_t elf_fpregset_t[ELF_NFPREG];
-/* typedef struct user_fpu_struct elf_fpregset_t; */
 #endif
-/* This is the location that an ET_DYN program is loaded if exec'ed.  Typical
- * use of this is to invoke "./ld.so someprog" to test out a new version of
- * the loader.  We need to make sure that it is out of the way of the program
- * that it will "exec", and that there is sufficient room for the brk.
- */
-#define ELF_ET_DYN_BASE         (0x08000000)
 #ifdef __MICROBLAZEEL__
-#define ELF_DATA        ELFDATA2LSB
 #else
-#define ELF_DATA        ELFDATA2MSB
 #endif
-#define ELF_EXEC_PAGESIZE       PAGE_SIZE
-#define ELF_CORE_COPY_REGS(_dest, _regs)                        \
-        memcpy((char *) &_dest, (char *) _regs,         \
-        sizeof(struct pt_regs));
-/* This yields a mask that user programs can use to figure out what
- * instruction set this CPU supports.  This could be done in user space,
- * but it's not easy, and we've already done it here.
- */
-#define ELF_HWCAP       (0)
-/* This yields a string that ld.so will use to load implementation
- * specific libraries for optimization.  This is more specific in
- * intent than poking at uname or /proc/cpuinfo.
- * For the moment, we have only optimizations for the Intel generations,
- * but that could change...
- */
-#define ELF_PLATFORM  (NULL)
-/* Added _f parameter. Is this definition correct: TBD */
-#define ELF_PLAT_INIT(_r, _f)                           \
-do {                                                    \
-        _r->r1 =  _r->r1 =  _r->r2 =  _r->r3 =          \
-        _r->r4 =  _r->r5 =  _r->r6 =  _r->r7 =          \
-        _r->r8 =  _r->r9 =  _r->r10 = _r->r11 =         \
-        _r->r12 = _r->r13 = _r->r14 = _r->r15 =         \
-        _r->r16 = _r->r17 = _r->r18 = _r->r19 =         \
-        _r->r20 = _r->r21 = _r->r22 = _r->r23 =         \
-        _r->r24 = _r->r25 = _r->r26 = _r->r27 =         \
-        _r->r28 = _r->r29 = _r->r30 = _r->r31 =         \
-        0;                                              \
-} while (0)
-#ifdef __KERNEL__
 #define SET_PERSONALITY(ex) \
        set_personality(PER_LINUX_32BIT | (current->personality & (~PER_MASK)))
-#endif
 #endif /* __uClinux__ */
 #endif /* _ASM_MICROBLAZE_ELF_H */
diff --git a/arch/microblaze/include/asm/entry.h b/arch/microblaze/include/asm/entry.h
index af0144b91b79..b4a4cb150aa9 100644
--- a/arch/microblaze/include/asm/entry.h
+++ b/arch/microblaze/include/asm/entry.h
@@ -29,6 +29,8 @@ DECLARE_PER_CPU(unsigned int, KM); /* Kernel/user mode */
 DECLARE_PER_CPU(unsigned int, ENTRY_SP); /* Saved SP on kernel entry */
 DECLARE_PER_CPU(unsigned int, R11_SAVE); /* Temp variable for entry */
 DECLARE_PER_CPU(unsigned int, CURRENT_SAVE); /* Saved current pointer */
+extern asmlinkage void do_notify_resume(struct pt_regs *regs, int in_syscall);
 # endif /* __ASSEMBLY__ */
 #endif /* _ASM_MICROBLAZE_ENTRY_H */
diff --git a/arch/microblaze/include/asm/ptrace.h b/arch/microblaze/include/asm/ptrace.h
index 94e92c805859..3732bcf186fd 100644
--- a/arch/microblaze/include/asm/ptrace.h
+++ b/arch/microblaze/include/asm/ptrace.h
@@ -5,56 +5,12 @@
 * License. See the file "COPYING" in the main directory of this archive
 * for more details.
 */
 #ifndef _ASM_MICROBLAZE_PTRACE_H
 #define _ASM_MICROBLAZE_PTRACE_H
-#ifndef __ASSEMBLY__
+#include <uapi/asm/ptrace.h>
-typedef unsigned long microblaze_reg_t;
-struct pt_regs {
+#ifndef __ASSEMBLY__
-        microblaze_reg_t r0;
-        microblaze_reg_t r1;
-        microblaze_reg_t r2;
-        microblaze_reg_t r3;
-        microblaze_reg_t r4;
-        microblaze_reg_t r5;
-        microblaze_reg_t r6;
-        microblaze_reg_t r7;
-        microblaze_reg_t r8;
-        microblaze_reg_t r9;
-        microblaze_reg_t r10;
-        microblaze_reg_t r11;
-        microblaze_reg_t r12;
-        microblaze_reg_t r13;
-        microblaze_reg_t r14;
-        microblaze_reg_t r15;
-        microblaze_reg_t r16;
-        microblaze_reg_t r17;
-        microblaze_reg_t r18;
-        microblaze_reg_t r19;
-        microblaze_reg_t r20;
-        microblaze_reg_t r21;
-        microblaze_reg_t r22;
-        microblaze_reg_t r23;
-        microblaze_reg_t r24;
-        microblaze_reg_t r25;
-        microblaze_reg_t r26;
-        microblaze_reg_t r27;
-        microblaze_reg_t r28;
-        microblaze_reg_t r29;
-        microblaze_reg_t r30;
-        microblaze_reg_t r31;
-        microblaze_reg_t pc;
-        microblaze_reg_t msr;
-        microblaze_reg_t ear;
-        microblaze_reg_t esr;
-        microblaze_reg_t fsr;
-        int pt_mode;
-};
-#ifdef __KERNEL__
 #define kernel_mode(regs)               ((regs)->pt_mode)
 #define user_mode(regs)                 (!kernel_mode(regs))
@@ -66,19 +22,5 @@ static inline long regs_return_value(struct pt_regs *regs)
        return regs->r3;
 }
-#else /* __KERNEL__ */
-/* pt_regs offsets used by gdbserver etc in ptrace syscalls */
-#define PT_GPR(n)       ((n) * sizeof(microblaze_reg_t))
-#define PT_PC           (32 * sizeof(microblaze_reg_t))
-#define PT_MSR          (33 * sizeof(microblaze_reg_t))
-#define PT_EAR          (34 * sizeof(microblaze_reg_t))
-#define PT_ESR          (35 * sizeof(microblaze_reg_t))
-#define PT_FSR          (36 * sizeof(microblaze_reg_t))
-#define PT_KERNEL_MODE  (37 * sizeof(microblaze_reg_t))
-#endif /* __KERNEL */
 #endif /* __ASSEMBLY__ */
 #endif /* _ASM_MICROBLAZE_PTRACE_H */
diff --git a/arch/microblaze/include/asm/setup.h b/arch/microblaze/include/asm/setup.h
index 0061aa13a340..0e0b0a5ec756 100644
--- a/arch/microblaze/include/asm/setup.h
+++ b/arch/microblaze/include/asm/setup.h
@@ -7,15 +7,12 @@
 * License. See the file "COPYING" in the main directory of this archive
 * for more details.
 */
 #ifndef _ASM_MICROBLAZE_SETUP_H
 #define _ASM_MICROBLAZE_SETUP_H
-#define COMMAND_LINE_SIZE       256
+#include <uapi/asm/setup.h>
 # ifndef __ASSEMBLY__
-#  ifdef __KERNEL__
 extern unsigned int boot_cpuid; /* move to smp.h */
 extern char cmd_line[COMMAND_LINE_SIZE];
@@ -53,6 +50,5 @@ void free_init_pages(char *what, unsigned long begin, unsigned long end);
 extern void *alloc_maybe_bootmem(size_t size, gfp_t mask);
 extern void *zalloc_maybe_bootmem(size_t size, gfp_t mask);
-#  endif/* __KERNEL__ */
 # endif /* __ASSEMBLY__ */
 #endif /* _ASM_MICROBLAZE_SETUP_H */
diff --git a/arch/microblaze/include/asm/uaccess.h b/arch/microblaze/include/asm/uaccess.h
index ef25f7538d4a..927540d3cb7d 100644
--- a/arch/microblaze/include/asm/uaccess.h
+++ b/arch/microblaze/include/asm/uaccess.h
@@ -298,11 +298,10 @@ extern long __user_bad(void);
 #define __put_user_check(x, ptr, size)                                  \
 ({                                                                      \
-        typeof(*(ptr)) __pu_val;                                        \
+        typeof(*(ptr)) volatile __pu_val = x;                                   \
        typeof(*(ptr)) __user *__pu_addr = (ptr);                       \
        int __pu_err = 0;                                               \
                                                                        \
-        __pu_val = (x);                                                 \
        if (access_ok(VERIFY_WRITE, __pu_addr, size)) {                 \
                switch (size) {                                         \
                case 1:                                                 \
diff --git a/arch/microblaze/include/asm/unistd.h b/arch/microblaze/include/asm/unistd.h
index 94d978986b75..99e23937a31a 100644
--- a/arch/microblaze/include/asm/unistd.h
+++ b/arch/microblaze/include/asm/unistd.h
@@ -6,398 +6,11 @@
 * License. See the file "COPYING" in the main directory of this archive
 * for more details.
 */
 #ifndef _ASM_MICROBLAZE_UNISTD_H
 #define _ASM_MICROBLAZE_UNISTD_H
-#define __NR_restart_syscall    0 /* ok */
+#include <uapi/asm/unistd.h>
-#define __NR_exit               1 /* ok */
-#define __NR_fork               2 /* not for no MMU - weird */
-#define __NR_read               3 /* ok */
-#define __NR_write              4 /* ok */
-#define __NR_open               5 /* openat */
-#define __NR_close              6 /* ok */
-#define __NR_waitpid            7 /* waitid */
-#define __NR_creat              8 /* openat */
-#define __NR_link               9 /* linkat */
-#define __NR_unlink             10 /* unlinkat */
-#define __NR_execve             11 /* ok */
-#define __NR_chdir              12 /* ok */
-#define __NR_time               13 /* obsolete -> sys_gettimeofday */
-#define __NR_mknod              14 /* mknodat */
-#define __NR_chmod              15 /* fchmodat */
-#define __NR_lchown             16 /* ok */
-#define __NR_break              17 /* don't know */
-#define __NR_oldstat            18 /* remove */
-#define __NR_lseek              19 /* ok */
-#define __NR_getpid             20 /* ok */
-#define __NR_mount              21 /* ok */
-#define __NR_umount             22 /* ok */  /* use only umount2 */
-#define __NR_setuid             23 /* ok */
-#define __NR_getuid             24 /* ok */
-#define __NR_stime              25 /* obsolete -> sys_settimeofday */
-#define __NR_ptrace             26 /* ok */
-#define __NR_alarm              27 /* obsolete -> sys_setitimer */
-#define __NR_oldfstat           28 /* remove */
-#define __NR_pause              29 /* obsolete -> sys_rt_sigtimedwait */
-#define __NR_utime              30 /* obsolete -> sys_utimesat */
-#define __NR_stty               31 /* remove */
-#define __NR_gtty               32 /* remove */
-#define __NR_access             33 /* faccessat */
-/* can be implemented by sys_setpriority */
-#define __NR_nice               34
-#define __NR_ftime              35 /* remove */
-#define __NR_sync               36 /* ok */
-#define __NR_kill               37 /* ok */
-#define __NR_rename             38 /* renameat */
-#define __NR_mkdir              39 /* mkdirat */
-#define __NR_rmdir              40 /* unlinkat */
-#define __NR_dup                41 /* ok */
-#define __NR_pipe               42 /* ok */
-#define __NR_times              43 /* ok */
-#define __NR_prof               44 /* remove */
-#define __NR_brk                45 /* ok -mmu, nommu specific */
-#define __NR_setgid             46 /* ok */
-#define __NR_getgid             47 /* ok */
-#define __NR_signal             48 /* obsolete -> sys_rt_sigaction */
-#define __NR_geteuid            49 /* ok */
-#define __NR_getegid            50 /* ok */
-#define __NR_acct               51 /* add it and then I can disable it */
-#define __NR_umount2            52 /* remove */
-#define __NR_lock               53 /* remove */
-#define __NR_ioctl              54 /* ok */
-#define __NR_fcntl              55 /* ok -> 64bit version*/
-#define __NR_mpx                56 /* remove */
-#define __NR_setpgid            57 /* ok */
-#define __NR_ulimit             58 /* remove */
-#define __NR_oldolduname        59 /* remove */
-#define __NR_umask              60 /* ok */
-#define __NR_chroot             61 /* ok */
-#define __NR_ustat              62 /* obsolete -> statfs64 */
-#define __NR_dup2               63 /* ok */
-#define __NR_getppid            64 /* ok */
-#define __NR_getpgrp            65 /* obsolete -> sys_getpgid */
-#define __NR_setsid             66 /* ok */
-#define __NR_sigaction          67 /* obsolete -> rt_sigaction */
-#define __NR_sgetmask           68 /* obsolete -> sys_rt_sigprocmask */
-#define __NR_ssetmask           69 /* obsolete ->sys_rt_sigprocmask */
-#define __NR_setreuid           70 /* ok */
-#define __NR_setregid           71 /* ok */
-#define __NR_sigsuspend         72 /* obsolete -> rt_sigsuspend */
-#define __NR_sigpending         73 /* obsolete -> sys_rt_sigpending */
-#define __NR_sethostname        74 /* ok */
-#define __NR_setrlimit          75 /* ok */
-#define __NR_getrlimit          76 /* ok Back compatible 2G limited rlimit */
-#define __NR_getrusage          77 /* ok */
-#define __NR_gettimeofday       78 /* ok */
-#define __NR_settimeofday       79 /* ok */
-#define __NR_getgroups          80 /* ok */
-#define __NR_setgroups          81 /* ok */
-#define __NR_select             82 /* obsolete -> sys_pselect7 */
-#define __NR_symlink            83 /* symlinkat */
-#define __NR_oldlstat           84 /* remove */
-#define __NR_readlink           85 /* obsolete -> sys_readlinkat */
-#define __NR_uselib             86 /* remove */
-#define __NR_swapon             87 /* ok */
-#define __NR_reboot             88 /* ok */
-#define __NR_readdir            89 /* remove ? */
-#define __NR_mmap               90 /* obsolete -> sys_mmap2 */
-#define __NR_munmap             91 /* ok - mmu and nommu */
-#define __NR_truncate           92 /* ok or truncate64 */
-#define __NR_ftruncate          93 /* ok or ftruncate64 */
-#define __NR_fchmod             94 /* ok */
-#define __NR_fchown             95 /* ok */
-#define __NR_getpriority        96 /* ok */
-#define __NR_setpriority        97 /* ok */
-#define __NR_profil             98 /* remove */
-#define __NR_statfs             99 /* ok or statfs64 */
-#define __NR_fstatfs            100  /* ok or fstatfs64 */
-#define __NR_ioperm             101 /* remove */
-#define __NR_socketcall         102 /* remove */
-#define __NR_syslog             103 /* ok */
-#define __NR_setitimer          104 /* ok */
-#define __NR_getitimer          105 /* ok */
-#define __NR_stat               106 /* remove */
-#define __NR_lstat              107 /* remove */
-#define __NR_fstat              108 /* remove */
-#define __NR_olduname           109 /* remove */
-#define __NR_iopl               110 /* remove */
-#define __NR_vhangup            111 /* ok */
-#define __NR_idle               112 /* remove */
-#define __NR_vm86old            113 /* remove */
-#define __NR_wait4              114 /* obsolete -> waitid */
-#define __NR_swapoff            115 /* ok */
-#define __NR_sysinfo            116 /* ok */
-#define __NR_ipc                117 /* remove - direct call */
-#define __NR_fsync              118 /* ok */
-#define __NR_sigreturn          119 /* obsolete -> sys_rt_sigreturn */
-#define __NR_clone              120 /* ok */
-#define __NR_setdomainname      121 /* ok */
-#define __NR_uname              122 /* remove */
-#define __NR_modify_ldt         123 /* remove */
-#define __NR_adjtimex           124 /* ok */
-#define __NR_mprotect           125 /* remove */
-#define __NR_sigprocmask        126 /* obsolete -> sys_rt_sigprocmask */
-#define __NR_create_module      127 /* remove */
-#define __NR_init_module        128 /* ok */
-#define __NR_delete_module      129 /* ok */
-#define __NR_get_kernel_syms    130 /* remove */
-#define __NR_quotactl           131 /* ok */
-#define __NR_getpgid            132 /* ok */
-#define __NR_fchdir             133 /* ok */
-#define __NR_bdflush            134 /* remove */
-#define __NR_sysfs              135 /* needed for busybox */
-#define __NR_personality        136 /* ok */
-#define __NR_afs_syscall        137 /* Syscall for Andrew File System */
-#define __NR_setfsuid           138 /* ok */
-#define __NR_setfsgid           139 /* ok */
-#define __NR__llseek            140 /* remove only lseek */
-#define __NR_getdents           141 /* ok or getdents64 */
-#define __NR__newselect         142 /* remove */
-#define __NR_flock              143 /* ok */
-#define __NR_msync              144 /* remove */
-#define __NR_readv              145 /* ok */
-#define __NR_writev             146 /* ok */
-#define __NR_getsid             147 /* ok */
-#define __NR_fdatasync          148 /* ok */
-#define __NR__sysctl            149 /* remove */
-#define __NR_mlock              150 /* ok - nommu or mmu */
-#define __NR_munlock            151 /* ok - nommu or mmu */
-#define __NR_mlockall           152 /* ok - nommu or mmu */
-#define __NR_munlockall         153 /* ok - nommu or mmu */
-#define __NR_sched_setparam             154 /* ok */
-#define __NR_sched_getparam             155 /* ok */
-#define __NR_sched_setscheduler         156 /* ok */
-#define __NR_sched_getscheduler         157 /* ok */
-#define __NR_sched_yield                158 /* ok */
-#define __NR_sched_get_priority_max     159 /* ok */
-#define __NR_sched_get_priority_min     160 /* ok */
-#define __NR_sched_rr_get_interval      161 /* ok */
-#define __NR_nanosleep          162 /* ok */
-#define __NR_mremap             163 /* ok - nommu or mmu */
-#define __NR_setresuid          164 /* ok */
-#define __NR_getresuid          165 /* ok */
-#define __NR_vm86               166 /* remove */
-#define __NR_query_module       167 /* ok */
-#define __NR_poll               168 /* obsolete -> sys_ppoll */
-#define __NR_nfsservctl         169 /* ok */
-#define __NR_setresgid          170 /* ok */
-#define __NR_getresgid          171 /* ok */
-#define __NR_prctl              172 /* ok */
-#define __NR_rt_sigreturn       173 /* ok */
-#define __NR_rt_sigaction       174 /* ok */
-#define __NR_rt_sigprocmask     175 /* ok */
-#define __NR_rt_sigpending      176 /* ok */
-#define __NR_rt_sigtimedwait    177 /* ok */
-#define __NR_rt_sigqueueinfo    178 /* ok */
-#define __NR_rt_sigsuspend      179 /* ok */
-#define __NR_pread64            180 /* ok */
-#define __NR_pwrite64           181 /* ok */
-#define __NR_chown              182 /* obsolete -> fchownat */
-#define __NR_getcwd             183 /* ok */
-#define __NR_capget             184 /* ok */
-#define __NR_capset             185 /* ok */
-#define __NR_sigaltstack        186 /* remove */
-#define __NR_sendfile           187 /* ok -> exist 64bit version*/
-#define __NR_getpmsg            188 /* remove */
-/* remove - some people actually want streams */
-#define __NR_putpmsg            189
-/* for noMMU - group with clone -> maybe remove */
-#define __NR_vfork              190
-#define __NR_ugetrlimit         191 /* remove - SuS compliant getrlimit */
-#define __NR_mmap2              192 /* ok */
-#define __NR_truncate64         193 /* ok */
-#define __NR_ftruncate64        194 /* ok */
-#define __NR_stat64             195 /* remove _ARCH_WANT_STAT64 */
-#define __NR_lstat64            196 /* remove _ARCH_WANT_STAT64 */
-#define __NR_fstat64            197 /* remove _ARCH_WANT_STAT64 */
-#define __NR_lchown32           198 /* ok - without 32 */
-#define __NR_getuid32           199 /* ok - without 32 */
-#define __NR_getgid32           200 /* ok - without 32 */
-#define __NR_geteuid32          201 /* ok - without 32 */
-#define __NR_getegid32          202 /* ok - without 32 */
-#define __NR_setreuid32         203 /* ok - without 32 */
-#define __NR_setregid32         204 /* ok - without 32 */
-#define __NR_getgroups32        205 /* ok - without 32 */
-#define __NR_setgroups32        206 /* ok - without 32 */
-#define __NR_fchown32           207 /* ok - without 32 */
-#define __NR_setresuid32        208 /* ok - without 32 */
-#define __NR_getresuid32        209 /* ok - without 32 */
-#define __NR_setresgid32        210 /* ok - without 32 */
-#define __NR_getresgid32        211 /* ok - without 32 */
-#define __NR_chown32            212 /* ok - without 32 -obsolete -> fchownat */
-#define __NR_setuid32           213 /* ok - without 32 */
-#define __NR_setgid32           214 /* ok - without 32 */
-#define __NR_setfsuid32         215 /* ok - without 32 */
-#define __NR_setfsgid32         216 /* ok - without 32 */
-#define __NR_pivot_root         217 /* ok */
-#define __NR_mincore            218 /* ok */
-#define __NR_madvise            219 /* ok */
-#define __NR_getdents64         220 /* ok */
-#define __NR_fcntl64            221 /* ok */
-/* 223 is unused */
-#define __NR_gettid             224 /* ok */
-#define __NR_readahead          225 /* ok */
-#define __NR_setxattr           226 /* ok */
-#define __NR_lsetxattr          227 /* ok */
-#define __NR_fsetxattr          228 /* ok */
-#define __NR_getxattr           229 /* ok */
-#define __NR_lgetxattr          230 /* ok */
-#define __NR_fgetxattr          231 /* ok */
-#define __NR_listxattr          232 /* ok */
-#define __NR_llistxattr         233 /* ok */
-#define __NR_flistxattr         234 /* ok */
-#define __NR_removexattr        235 /* ok */
-#define __NR_lremovexattr       236 /* ok */
-#define __NR_fremovexattr       237 /* ok */
-#define __NR_tkill              238 /* ok */
-#define __NR_sendfile64         239 /* ok */
-#define __NR_futex              240 /* ok */
-#define __NR_sched_setaffinity  241 /* ok */
-#define __NR_sched_getaffinity  242 /* ok */
-#define __NR_set_thread_area    243 /* remove */
-#define __NR_get_thread_area    244 /* remove */
-#define __NR_io_setup           245 /* ok */
-#define __NR_io_destroy         246 /* ok */
-#define __NR_io_getevents       247 /* ok */
-#define __NR_io_submit          248 /* ok */
-#define __NR_io_cancel          249 /* ok */
-#define __NR_fadvise64          250 /* remove -> sys_fadvise64_64 */
-/* 251 is available for reuse (was briefly sys_set_zone_reclaim) */
-#define __NR_exit_group         252 /* ok */
-#define __NR_lookup_dcookie     253 /* ok */
-#define __NR_epoll_create       254 /* ok */
-#define __NR_epoll_ctl          255 /* ok */
-#define __NR_epoll_wait         256 /* obsolete -> sys_epoll_pwait */
-#define __NR_remap_file_pages   257 /* only for mmu */
-#define __NR_set_tid_address    258 /* ok */
-#define __NR_timer_create       259 /* ok */
-#define __NR_timer_settime      (__NR_timer_create+1) /* 260 */ /* ok */
-#define __NR_timer_gettime      (__NR_timer_create+2) /* 261 */ /* ok */
-#define __NR_timer_getoverrun   (__NR_timer_create+3) /* 262 */ /* ok */
-#define __NR_timer_delete       (__NR_timer_create+4) /* 263 */ /* ok */
-#define __NR_clock_settime      (__NR_timer_create+5) /* 264 */ /* ok */
-#define __NR_clock_gettime      (__NR_timer_create+6) /* 265 */ /* ok */
-#define __NR_clock_getres       (__NR_timer_create+7) /* 266 */ /* ok */
-#define __NR_clock_nanosleep    (__NR_timer_create+8) /* 267 */ /* ok */
-#define __NR_statfs64           268 /* ok */
-#define __NR_fstatfs64          269 /* ok */
-#define __NR_tgkill             270 /* ok */
-#define __NR_utimes             271 /* obsolete -> sys_futimesat */
-#define __NR_fadvise64_64       272 /* ok */
-#define __NR_vserver            273 /* ok */
-#define __NR_mbind              274 /* only for mmu */
-#define __NR_get_mempolicy      275 /* only for mmu */
-#define __NR_set_mempolicy      276 /* only for mmu */
-#define __NR_mq_open            277 /* ok */
-#define __NR_mq_unlink          (__NR_mq_open+1) /* 278 */ /* ok */
-#define __NR_mq_timedsend       (__NR_mq_open+2) /* 279 */ /* ok */
-#define __NR_mq_timedreceive    (__NR_mq_open+3) /* 280 */ /* ok */
-#define __NR_mq_notify          (__NR_mq_open+4) /* 281 */ /* ok */
-#define __NR_mq_getsetattr      (__NR_mq_open+5) /* 282 */ /* ok */
-#define __NR_kexec_load         283 /* ok */
-#define __NR_waitid             284 /* ok */
-/* #define __NR_sys_setaltroot  285 */
-#define __NR_add_key            286 /* ok */
-#define __NR_request_key        287 /* ok */
-#define __NR_keyctl             288 /* ok */
-#define __NR_ioprio_set         289 /* ok */
-#define __NR_ioprio_get         290 /* ok */
-#define __NR_inotify_init       291 /* ok */
-#define __NR_inotify_add_watch  292 /* ok */
-#define __NR_inotify_rm_watch   293 /* ok */
-#define __NR_migrate_pages      294 /* mmu */
-#define __NR_openat             295 /* ok */
-#define __NR_mkdirat            296 /* ok */
-#define __NR_mknodat            297 /* ok */
-#define __NR_fchownat           298 /* ok */
-#define __NR_futimesat          299 /* obsolete -> sys_utimesat */
-#define __NR_fstatat64          300 /* stat64 */
-#define __NR_unlinkat           301 /* ok */
-#define __NR_renameat           302 /* ok */
-#define __NR_linkat             303 /* ok */
-#define __NR_symlinkat          304 /* ok */
-#define __NR_readlinkat         305 /* ok */
-#define __NR_fchmodat           306 /* ok */
-#define __NR_faccessat          307 /* ok */
-#define __NR_pselect6           308 /* obsolete -> sys_pselect7 */
-#define __NR_ppoll              309 /* ok */
-#define __NR_unshare            310 /* ok */
-#define __NR_set_robust_list    311 /* ok */
-#define __NR_get_robust_list    312 /* ok */
-#define __NR_splice             313 /* ok */
-#define __NR_sync_file_range    314 /* ok */
-#define __NR_tee                315 /* ok */
-#define __NR_vmsplice           316 /* ok */
-#define __NR_move_pages         317 /* mmu */
-#define __NR_getcpu             318 /* ok */
-#define __NR_epoll_pwait        319 /* ok */
-#define __NR_utimensat          320 /* ok */
-#define __NR_signalfd           321 /* ok */
-#define __NR_timerfd_create     322 /* ok */
-#define __NR_eventfd            323 /* ok */
-#define __NR_fallocate          324 /* ok */
-#define __NR_semtimedop         325 /* ok - semaphore group */
-#define __NR_timerfd_settime    326 /* ok */
-#define __NR_timerfd_gettime    327 /* ok */
-/* sysv ipc syscalls */
-#define __NR_semctl             328 /* ok */
-#define __NR_semget             329 /* ok */
-#define __NR_semop              330 /* ok */
-#define __NR_msgctl             331 /* ok */
-#define __NR_msgget             332 /* ok */
-#define __NR_msgrcv             333 /* ok */
-#define __NR_msgsnd             334 /* ok */
-#define __NR_shmat              335 /* ok */
-#define __NR_shmctl             336 /* ok */
-#define __NR_shmdt              337 /* ok */
-#define __NR_shmget             338 /* ok */
-#define __NR_signalfd4          339 /* new */
-#define __NR_eventfd2           340 /* new */
-#define __NR_epoll_create1      341 /* new */
-#define __NR_dup3               342 /* new */
-#define __NR_pipe2              343 /* new */
-#define __NR_inotify_init1      344 /* new */
-#define __NR_socket             345 /* new */
-#define __NR_socketpair         346 /* new */
-#define __NR_bind               347 /* new */
-#define __NR_listen             348 /* new */
-#define __NR_accept             349 /* new */
-#define __NR_connect            350 /* new */
-#define __NR_getsockname        351 /* new */
-#define __NR_getpeername        352 /* new */
-#define __NR_sendto             353 /* new */
-#define __NR_send               354 /* new */
-#define __NR_recvfrom           355 /* new */
-#define __NR_recv               356 /* new */
-#define __NR_setsockopt         357 /* new */
-#define __NR_getsockopt         358 /* new */
-#define __NR_shutdown           359 /* new */
-#define __NR_sendmsg            360 /* new */
-#define __NR_recvmsg            361 /* new */
-#define __NR_accept4            362 /* new */
-#define __NR_preadv             363 /* new */
-#define __NR_pwritev            364 /* new */
-#define __NR_rt_tgsigqueueinfo  365 /* new */
-#define __NR_perf_event_open    366 /* new */
-#define __NR_recvmmsg           367 /* new */
-#define __NR_fanotify_init      368
-#define __NR_fanotify_mark      369
-#define __NR_prlimit64          370
-#define __NR_name_to_handle_at  371
-#define __NR_open_by_handle_at  372
-#define __NR_clock_adjtime      373
-#define __NR_syncfs             374
-#define __NR_setns              375
-#define __NR_sendmmsg           376
-#define __NR_process_vm_readv   377
-#define __NR_process_vm_writev  378
-#define __NR_syscalls           379
-#ifdef __KERNEL__
 #ifndef __ASSEMBLY__
 /* #define __ARCH_WANT_OLD_READDIR */
@@ -438,5 +51,4 @@
 #define cond_syscall(x) asm(".weak\t" #x "\n\t.set\t" #x ",sys_ni_syscall");
 #endif /* __ASSEMBLY__ */
-#endif /* __KERNEL__ */
 #endif /* _ASM_MICROBLAZE_UNISTD_H */
diff --git a/arch/microblaze/include/uapi/asm/Kbuild b/arch/microblaze/include/uapi/asm/Kbuild
index baebb3da1d44..6d7d7f4aaae8 100644
--- a/arch/microblaze/include/uapi/asm/Kbuild
+++ b/arch/microblaze/include/uapi/asm/Kbuild
@@ -1,3 +1,35 @@
 # UAPI Header export list
 include include/uapi/asm-generic/Kbuild.asm
+header-y += auxvec.h
+header-y += bitsperlong.h
+header-y += byteorder.h
+header-y += elf.h
+header-y += errno.h
+header-y += fcntl.h
+header-y += ioctl.h
+header-y += ioctls.h
+header-y += ipcbuf.h
+header-y += kvm_para.h
+header-y += mman.h
+header-y += msgbuf.h
+header-y += param.h
+header-y += poll.h
+header-y += posix_types.h
+header-y += ptrace.h
+header-y += resource.h
+header-y += sembuf.h
+header-y += setup.h
+header-y += shmbuf.h
+header-y += sigcontext.h
+header-y += siginfo.h
+header-y += signal.h
+header-y += socket.h
+header-y += sockios.h
+header-y += stat.h
+header-y += statfs.h
+header-y += swab.h
+header-y += termbits.h
+header-y += termios.h
+header-y += types.h
+header-y += unistd.h
diff --git a/arch/microblaze/include/asm/auxvec.h b/arch/microblaze/include/uapi/asm/auxvec.h
index 8b137891791f..8b137891791f 100644
--- a/arch/microblaze/include/asm/auxvec.h
+++ b/arch/microblaze/include/uapi/asm/auxvec.h
diff --git a/arch/microblaze/include/asm/bitsperlong.h b/arch/microblaze/include/uapi/asm/bitsperlong.h
index 6dc0bb0c13b2..6dc0bb0c13b2 100644
--- a/arch/microblaze/include/asm/bitsperlong.h
+++ b/arch/microblaze/include/uapi/asm/bitsperlong.h
diff --git a/arch/microblaze/include/asm/byteorder.h b/arch/microblaze/include/uapi/asm/byteorder.h
index 31902762a426..31902762a426 100644
--- a/arch/microblaze/include/asm/byteorder.h
+++ b/arch/microblaze/include/uapi/asm/byteorder.h
diff --git a/arch/microblaze/include/uapi/asm/elf.h b/arch/microblaze/include/uapi/asm/elf.h
new file mode 100644
index 000000000000..be1731d5e2fa
--- /dev/null
+++ b/arch/microblaze/include/uapi/asm/elf.h
@@ -0,0 +1,121 @@
+/*
+ * Copyright (C) 2008-2009 Michal Simek <monstr@monstr.eu>
+ * Copyright (C) 2008-2009 PetaLogix
+ * Copyright (C) 2006 Atmark Techno, Inc.
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License. See the file "COPYING" in the main directory of this archive
+ * for more details.
+ */
+#ifndef _UAPI_ASM_MICROBLAZE_ELF_H
+#define _UAPI_ASM_MICROBLAZE_ELF_H
+/*
+ * Note there is no "official" ELF designation for Microblaze.
+ * I've snaffled the value from the microblaze binutils source code
+ * /binutils/microblaze/include/elf/microblaze.h
+ */
+#define EM_MICROBLAZE           189
+#define EM_MICROBLAZE_OLD       0xbaab
+#define ELF_ARCH                EM_MICROBLAZE
+/*
+ * This is used to ensure we don't load something for the wrong architecture.
+ */
+#define elf_check_arch(x)       ((x)->e_machine == EM_MICROBLAZE \
+                                 || (x)->e_machine == EM_MICROBLAZE_OLD)
+/*
+ * These are used to set parameters in the core dumps.
+ */
+#define ELF_CLASS       ELFCLASS32
+#ifndef __uClinux__
+/*
+ * ELF register definitions..
+ */
+#include <asm/ptrace.h>
+#include <asm/byteorder.h>
+#ifndef ELF_GREG_T
+#define ELF_GREG_T
+typedef unsigned long elf_greg_t;
+#endif
+#ifndef ELF_NGREG
+#define ELF_NGREG (sizeof(struct pt_regs) / sizeof(elf_greg_t))
+#endif
+#ifndef ELF_GREGSET_T
+#define ELF_GREGSET_T
+typedef elf_greg_t elf_gregset_t[ELF_NGREG];
+#endif
+#ifndef ELF_FPREGSET_T
+#define ELF_FPREGSET_T
+/* TBD */
+#define ELF_NFPREG      33      /* includes fsr */
+typedef unsigned long elf_fpreg_t;
+typedef elf_fpreg_t elf_fpregset_t[ELF_NFPREG];
+/* typedef struct user_fpu_struct elf_fpregset_t; */
+#endif
+/* This is the location that an ET_DYN program is loaded if exec'ed.  Typical
+ * use of this is to invoke "./ld.so someprog" to test out a new version of
+ * the loader.  We need to make sure that it is out of the way of the program
+ * that it will "exec", and that there is sufficient room for the brk.
+ */
+#define ELF_ET_DYN_BASE         (0x08000000)
+#ifdef __MICROBLAZEEL__
+#define ELF_DATA        ELFDATA2LSB
+#else
+#define ELF_DATA        ELFDATA2MSB
+#endif
+#define ELF_EXEC_PAGESIZE       PAGE_SIZE
+#define ELF_CORE_COPY_REGS(_dest, _regs)                        \
+        memcpy((char *) &_dest, (char *) _regs,         \
+        sizeof(struct pt_regs));
+/* This yields a mask that user programs can use to figure out what
+ * instruction set this CPU supports.  This could be done in user space,
+ * but it's not easy, and we've already done it here.
+ */
+#define ELF_HWCAP       (0)
+/* This yields a string that ld.so will use to load implementation
+ * specific libraries for optimization.  This is more specific in
+ * intent than poking at uname or /proc/cpuinfo.
+ * For the moment, we have only optimizations for the Intel generations,
+ * but that could change...
+ */
+#define ELF_PLATFORM  (NULL)
+/* Added _f parameter. Is this definition correct: TBD */
+#define ELF_PLAT_INIT(_r, _f)                           \
+do {                                                    \
+        _r->r0 =  _r->r1 =  _r->r2 =  _r->r3 =          \
+        _r->r4 =  _r->r5 =  _r->r6 =  _r->r7 =          \
+        _r->r8 =  _r->r9 =  _r->r10 = _r->r11 =         \
+        _r->r12 = _r->r13 = _r->r14 = _r->r15 =         \
+        _r->r16 = _r->r17 = _r->r18 = _r->r19 =         \
+        _r->r20 = _r->r21 = _r->r22 = _r->r23 =         \
+        _r->r24 = _r->r25 = _r->r26 = _r->r27 =         \
+        _r->r28 = _r->r29 = _r->r30 = _r->r31 =         \
+        0;                                              \
+} while (0)
+#endif /* __uClinux__ */
+#endif /* _UAPI_ASM_MICROBLAZE_ELF_H */
diff --git a/arch/microblaze/include/asm/errno.h b/arch/microblaze/include/uapi/asm/errno.h
index 4c82b503d92f..4c82b503d92f 100644
--- a/arch/microblaze/include/asm/errno.h
+++ b/arch/microblaze/include/uapi/asm/errno.h
diff --git a/arch/microblaze/include/asm/fcntl.h b/arch/microblaze/include/uapi/asm/fcntl.h
index 46ab12db5739..46ab12db5739 100644
--- a/arch/microblaze/include/asm/fcntl.h
+++ b/arch/microblaze/include/uapi/asm/fcntl.h
diff --git a/arch/microblaze/include/asm/ioctl.h b/arch/microblaze/include/uapi/asm/ioctl.h
index b279fe06dfe5..b279fe06dfe5 100644
--- a/arch/microblaze/include/asm/ioctl.h
+++ b/arch/microblaze/include/uapi/asm/ioctl.h
diff --git a/arch/microblaze/include/asm/ioctls.h b/arch/microblaze/include/uapi/asm/ioctls.h
index ec34c760665e..ec34c760665e 100644
--- a/arch/microblaze/include/asm/ioctls.h
+++ b/arch/microblaze/include/uapi/asm/ioctls.h
diff --git a/arch/microblaze/include/asm/ipcbuf.h b/arch/microblaze/include/uapi/asm/ipcbuf.h
index 84c7e51cb6d0..84c7e51cb6d0 100644
--- a/arch/microblaze/include/asm/ipcbuf.h
+++ b/arch/microblaze/include/uapi/asm/ipcbuf.h
diff --git a/arch/microblaze/include/asm/kvm_para.h b/arch/microblaze/include/uapi/asm/kvm_para.h
index 14fab8f0b957..14fab8f0b957 100644
--- a/arch/microblaze/include/asm/kvm_para.h
+++ b/arch/microblaze/include/uapi/asm/kvm_para.h
diff --git a/arch/microblaze/include/asm/mman.h b/arch/microblaze/include/uapi/asm/mman.h
index 8eebf89f5ab1..8eebf89f5ab1 100644
--- a/arch/microblaze/include/asm/mman.h
+++ b/arch/microblaze/include/uapi/asm/mman.h
diff --git a/arch/microblaze/include/asm/msgbuf.h b/arch/microblaze/include/uapi/asm/msgbuf.h
index 809134c644a6..809134c644a6 100644
--- a/arch/microblaze/include/asm/msgbuf.h
+++ b/arch/microblaze/include/uapi/asm/msgbuf.h
diff --git a/arch/microblaze/include/asm/param.h b/arch/microblaze/include/uapi/asm/param.h
index 965d45427975..965d45427975 100644
--- a/arch/microblaze/include/asm/param.h
+++ b/arch/microblaze/include/uapi/asm/param.h
diff --git a/arch/microblaze/include/asm/poll.h b/arch/microblaze/include/uapi/asm/poll.h
index c98509d3149e..c98509d3149e 100644
--- a/arch/microblaze/include/asm/poll.h
+++ b/arch/microblaze/include/uapi/asm/poll.h
diff --git a/arch/microblaze/include/asm/posix_types.h b/arch/microblaze/include/uapi/asm/posix_types.h
index 0e15039673e3..0e15039673e3 100644
--- a/arch/microblaze/include/asm/posix_types.h
+++ b/arch/microblaze/include/uapi/asm/posix_types.h
diff --git a/arch/microblaze/include/uapi/asm/ptrace.h b/arch/microblaze/include/uapi/asm/ptrace.h
new file mode 100644
index 000000000000..d31238a5f946
--- /dev/null
+++ b/arch/microblaze/include/uapi/asm/ptrace.h
@@ -0,0 +1,72 @@
+/*
+ * Copyright (C) 2006 Atmark Techno, Inc.
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License. See the file "COPYING" in the main directory of this archive
+ * for more details.
+ */
+#ifndef _UAPI_ASM_MICROBLAZE_PTRACE_H
+#define _UAPI_ASM_MICROBLAZE_PTRACE_H
+#ifndef __ASSEMBLY__
+typedef unsigned long microblaze_reg_t;
+struct pt_regs {
+        microblaze_reg_t r0;
+        microblaze_reg_t r1;
+        microblaze_reg_t r2;
+        microblaze_reg_t r3;
+        microblaze_reg_t r4;
+        microblaze_reg_t r5;
+        microblaze_reg_t r6;
+        microblaze_reg_t r7;
+        microblaze_reg_t r8;
+        microblaze_reg_t r9;
+        microblaze_reg_t r10;
+        microblaze_reg_t r11;
+        microblaze_reg_t r12;
+        microblaze_reg_t r13;
+        microblaze_reg_t r14;
+        microblaze_reg_t r15;
+        microblaze_reg_t r16;
+        microblaze_reg_t r17;
+        microblaze_reg_t r18;
+        microblaze_reg_t r19;
+        microblaze_reg_t r20;
+        microblaze_reg_t r21;
+        microblaze_reg_t r22;
+        microblaze_reg_t r23;
+        microblaze_reg_t r24;
+        microblaze_reg_t r25;
+        microblaze_reg_t r26;
+        microblaze_reg_t r27;
+        microblaze_reg_t r28;
+        microblaze_reg_t r29;
+        microblaze_reg_t r30;
+        microblaze_reg_t r31;
+        microblaze_reg_t pc;
+        microblaze_reg_t msr;
+        microblaze_reg_t ear;
+        microblaze_reg_t esr;
+        microblaze_reg_t fsr;
+        int pt_mode;
+};
+#ifndef __KERNEL__
+/* pt_regs offsets used by gdbserver etc in ptrace syscalls */
+#define PT_GPR(n)       ((n) * sizeof(microblaze_reg_t))
+#define PT_PC           (32 * sizeof(microblaze_reg_t))
+#define PT_MSR          (33 * sizeof(microblaze_reg_t))
+#define PT_EAR          (34 * sizeof(microblaze_reg_t))
+#define PT_ESR          (35 * sizeof(microblaze_reg_t))
+#define PT_FSR          (36 * sizeof(microblaze_reg_t))
+#define PT_KERNEL_MODE  (37 * sizeof(microblaze_reg_t))
+#endif /* __KERNEL */
+#endif /* __ASSEMBLY__ */
+#endif /* _UAPI_ASM_MICROBLAZE_PTRACE_H */
diff --git a/arch/microblaze/include/asm/resource.h b/arch/microblaze/include/uapi/asm/resource.h
index 04bc4db8921b..04bc4db8921b 100644
--- a/arch/microblaze/include/asm/resource.h
+++ b/arch/microblaze/include/uapi/asm/resource.h
diff --git a/arch/microblaze/include/asm/sembuf.h b/arch/microblaze/include/uapi/asm/sembuf.h
index 7673b83cfef7..7673b83cfef7 100644
--- a/arch/microblaze/include/asm/sembuf.h
+++ b/arch/microblaze/include/uapi/asm/sembuf.h
diff --git a/arch/microblaze/include/uapi/asm/setup.h b/arch/microblaze/include/uapi/asm/setup.h
new file mode 100644
index 000000000000..76bc2acee6af
--- /dev/null
+++ b/arch/microblaze/include/uapi/asm/setup.h
@@ -0,0 +1,19 @@
+/*
+ * Copyright (C) 2007-2009 Michal Simek <monstr@monstr.eu>
+ * Copyright (C) 2007-2009 PetaLogix
+ * Copyright (C) 2006 Atmark Techno, Inc.
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License. See the file "COPYING" in the main directory of this archive
+ * for more details.
+ */
+#ifndef _UAPI_ASM_MICROBLAZE_SETUP_H
+#define _UAPI_ASM_MICROBLAZE_SETUP_H
+#define COMMAND_LINE_SIZE       256
+# ifndef __ASSEMBLY__
+# endif /* __ASSEMBLY__ */
+#endif /* _UAPI_ASM_MICROBLAZE_SETUP_H */
diff --git a/arch/microblaze/include/asm/shmbuf.h b/arch/microblaze/include/uapi/asm/shmbuf.h
index 83c05fc2de38..83c05fc2de38 100644
--- a/arch/microblaze/include/asm/shmbuf.h
+++ b/arch/microblaze/include/uapi/asm/shmbuf.h
diff --git a/arch/microblaze/include/asm/sigcontext.h b/arch/microblaze/include/uapi/asm/sigcontext.h
index 55873c80c917..55873c80c917 100644
--- a/arch/microblaze/include/asm/sigcontext.h
+++ b/arch/microblaze/include/uapi/asm/sigcontext.h
diff --git a/arch/microblaze/include/asm/siginfo.h b/arch/microblaze/include/uapi/asm/siginfo.h
index 0815d29d82e5..0815d29d82e5 100644
--- a/arch/microblaze/include/asm/siginfo.h
+++ b/arch/microblaze/include/uapi/asm/siginfo.h
diff --git a/arch/microblaze/include/asm/signal.h b/arch/microblaze/include/uapi/asm/signal.h
index 7b1573ce19de..7b1573ce19de 100644
--- a/arch/microblaze/include/asm/signal.h
+++ b/arch/microblaze/include/uapi/asm/signal.h
diff --git a/arch/microblaze/include/asm/socket.h b/arch/microblaze/include/uapi/asm/socket.h
index 6b71384b9d8b..6b71384b9d8b 100644
--- a/arch/microblaze/include/asm/socket.h
+++ b/arch/microblaze/include/uapi/asm/socket.h
diff --git a/arch/microblaze/include/asm/sockios.h b/arch/microblaze/include/uapi/asm/sockios.h
index def6d4746ee7..def6d4746ee7 100644
--- a/arch/microblaze/include/asm/sockios.h
+++ b/arch/microblaze/include/uapi/asm/sockios.h
diff --git a/arch/microblaze/include/asm/stat.h b/arch/microblaze/include/uapi/asm/stat.h
index 3dc90fa92c70..3dc90fa92c70 100644
--- a/arch/microblaze/include/asm/stat.h
+++ b/arch/microblaze/include/uapi/asm/stat.h
diff --git a/arch/microblaze/include/asm/statfs.h b/arch/microblaze/include/uapi/asm/statfs.h
index 0b91fe198c20..0b91fe198c20 100644
--- a/arch/microblaze/include/asm/statfs.h
+++ b/arch/microblaze/include/uapi/asm/statfs.h
diff --git a/arch/microblaze/include/asm/swab.h b/arch/microblaze/include/uapi/asm/swab.h
index 7847e563ab66..7847e563ab66 100644
--- a/arch/microblaze/include/asm/swab.h
+++ b/arch/microblaze/include/uapi/asm/swab.h
diff --git a/arch/microblaze/include/asm/termbits.h b/arch/microblaze/include/uapi/asm/termbits.h
index 3935b106de79..3935b106de79 100644
--- a/arch/microblaze/include/asm/termbits.h
+++ b/arch/microblaze/include/uapi/asm/termbits.h
diff --git a/arch/microblaze/include/asm/termios.h b/arch/microblaze/include/uapi/asm/termios.h
index 280d78a9d966..280d78a9d966 100644
--- a/arch/microblaze/include/asm/termios.h
+++ b/arch/microblaze/include/uapi/asm/termios.h
diff --git a/arch/microblaze/include/asm/types.h b/arch/microblaze/include/uapi/asm/types.h
index b9e79bc580dd..b9e79bc580dd 100644
--- a/arch/microblaze/include/asm/types.h
+++ b/arch/microblaze/include/uapi/asm/types.h
diff --git a/arch/microblaze/include/uapi/asm/unistd.h b/arch/microblaze/include/uapi/asm/unistd.h
new file mode 100644
index 000000000000..ccb6920f3b33
--- /dev/null
+++ b/arch/microblaze/include/uapi/asm/unistd.h
@@ -0,0 +1,401 @@
+/*
+ * Copyright (C) 2007-2008 Michal Simek <monstr@monstr.eu>
+ * Copyright (C) 2006 Atmark Techno, Inc.
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License. See the file "COPYING" in the main directory of this archive
+ * for more details.
+ */
+#ifndef _UAPI_ASM_MICROBLAZE_UNISTD_H
+#define _UAPI_ASM_MICROBLAZE_UNISTD_H
+#define __NR_restart_syscall    0 /* ok */
+#define __NR_exit               1 /* ok */
+#define __NR_fork               2 /* not for no MMU - weird */
+#define __NR_read               3 /* ok */
+#define __NR_write              4 /* ok */
+#define __NR_open               5 /* openat */
+#define __NR_close              6 /* ok */
+#define __NR_waitpid            7 /* waitid */
+#define __NR_creat              8 /* openat */
+#define __NR_link               9 /* linkat */
+#define __NR_unlink             10 /* unlinkat */
+#define __NR_execve             11 /* ok */
+#define __NR_chdir              12 /* ok */
+#define __NR_time               13 /* obsolete -> sys_gettimeofday */
+#define __NR_mknod              14 /* mknodat */
+#define __NR_chmod              15 /* fchmodat */
+#define __NR_lchown             16 /* ok */
+#define __NR_break              17 /* don't know */
+#define __NR_oldstat            18 /* remove */
+#define __NR_lseek              19 /* ok */
+#define __NR_getpid             20 /* ok */
+#define __NR_mount              21 /* ok */
+#define __NR_umount             22 /* ok */  /* use only umount2 */
+#define __NR_setuid             23 /* ok */
+#define __NR_getuid             24 /* ok */
+#define __NR_stime              25 /* obsolete -> sys_settimeofday */
+#define __NR_ptrace             26 /* ok */
+#define __NR_alarm              27 /* obsolete -> sys_setitimer */
+#define __NR_oldfstat           28 /* remove */
+#define __NR_pause              29 /* obsolete -> sys_rt_sigtimedwait */
+#define __NR_utime              30 /* obsolete -> sys_utimesat */
+#define __NR_stty               31 /* remove */
+#define __NR_gtty               32 /* remove */
+#define __NR_access             33 /* faccessat */
+/* can be implemented by sys_setpriority */
+#define __NR_nice               34
+#define __NR_ftime              35 /* remove */
+#define __NR_sync               36 /* ok */
+#define __NR_kill               37 /* ok */
+#define __NR_rename             38 /* renameat */
+#define __NR_mkdir              39 /* mkdirat */
+#define __NR_rmdir              40 /* unlinkat */
+#define __NR_dup                41 /* ok */
+#define __NR_pipe               42 /* ok */
+#define __NR_times              43 /* ok */
+#define __NR_prof               44 /* remove */
+#define __NR_brk                45 /* ok -mmu, nommu specific */
+#define __NR_setgid             46 /* ok */
+#define __NR_getgid             47 /* ok */
+#define __NR_signal             48 /* obsolete -> sys_rt_sigaction */
+#define __NR_geteuid            49 /* ok */
+#define __NR_getegid            50 /* ok */
+#define __NR_acct               51 /* add it and then I can disable it */
+#define __NR_umount2            52 /* remove */
+#define __NR_lock               53 /* remove */
+#define __NR_ioctl              54 /* ok */
+#define __NR_fcntl              55 /* ok -> 64bit version*/
+#define __NR_mpx                56 /* remove */
+#define __NR_setpgid            57 /* ok */
+#define __NR_ulimit             58 /* remove */
+#define __NR_oldolduname        59 /* remove */
+#define __NR_umask              60 /* ok */
+#define __NR_chroot             61 /* ok */
+#define __NR_ustat              62 /* obsolete -> statfs64 */
+#define __NR_dup2               63 /* ok */
+#define __NR_getppid            64 /* ok */
+#define __NR_getpgrp            65 /* obsolete -> sys_getpgid */
+#define __NR_setsid             66 /* ok */
+#define __NR_sigaction          67 /* obsolete -> rt_sigaction */
+#define __NR_sgetmask           68 /* obsolete -> sys_rt_sigprocmask */
+#define __NR_ssetmask           69 /* obsolete ->sys_rt_sigprocmask */
+#define __NR_setreuid           70 /* ok */
+#define __NR_setregid           71 /* ok */
+#define __NR_sigsuspend         72 /* obsolete -> rt_sigsuspend */
+#define __NR_sigpending         73 /* obsolete -> sys_rt_sigpending */
+#define __NR_sethostname        74 /* ok */
+#define __NR_setrlimit          75 /* ok */
+#define __NR_getrlimit          76 /* ok Back compatible 2G limited rlimit */
+#define __NR_getrusage          77 /* ok */
+#define __NR_gettimeofday       78 /* ok */
+#define __NR_settimeofday       79 /* ok */
+#define __NR_getgroups          80 /* ok */
+#define __NR_setgroups          81 /* ok */
+#define __NR_select             82 /* obsolete -> sys_pselect7 */
+#define __NR_symlink            83 /* symlinkat */
+#define __NR_oldlstat           84 /* remove */
+#define __NR_readlink           85 /* obsolete -> sys_readlinkat */
+#define __NR_uselib             86 /* remove */
+#define __NR_swapon             87 /* ok */
+#define __NR_reboot             88 /* ok */
+#define __NR_readdir            89 /* remove ? */
+#define __NR_mmap               90 /* obsolete -> sys_mmap2 */
+#define __NR_munmap             91 /* ok - mmu and nommu */
+#define __NR_truncate           92 /* ok or truncate64 */
+#define __NR_ftruncate          93 /* ok or ftruncate64 */
+#define __NR_fchmod             94 /* ok */
+#define __NR_fchown             95 /* ok */
+#define __NR_getpriority        96 /* ok */
+#define __NR_setpriority        97 /* ok */
+#define __NR_profil             98 /* remove */
+#define __NR_statfs             99 /* ok or statfs64 */
+#define __NR_fstatfs            100  /* ok or fstatfs64 */
+#define __NR_ioperm             101 /* remove */
+#define __NR_socketcall         102 /* remove */
+#define __NR_syslog             103 /* ok */
+#define __NR_setitimer          104 /* ok */
+#define __NR_getitimer          105 /* ok */
+#define __NR_stat               106 /* remove */
+#define __NR_lstat              107 /* remove */
+#define __NR_fstat              108 /* remove */
+#define __NR_olduname           109 /* remove */
+#define __NR_iopl               110 /* remove */
+#define __NR_vhangup            111 /* ok */
+#define __NR_idle               112 /* remove */
+#define __NR_vm86old            113 /* remove */
+#define __NR_wait4              114 /* obsolete -> waitid */
+#define __NR_swapoff            115 /* ok */
+#define __NR_sysinfo            116 /* ok */
+#define __NR_ipc                117 /* remove - direct call */
+#define __NR_fsync              118 /* ok */
+#define __NR_sigreturn          119 /* obsolete -> sys_rt_sigreturn */
+#define __NR_clone              120 /* ok */
+#define __NR_setdomainname      121 /* ok */
+#define __NR_uname              122 /* remove */
+#define __NR_modify_ldt         123 /* remove */
+#define __NR_adjtimex           124 /* ok */
+#define __NR_mprotect           125 /* remove */
+#define __NR_sigprocmask        126 /* obsolete -> sys_rt_sigprocmask */
+#define __NR_create_module      127 /* remove */
+#define __NR_init_module        128 /* ok */
+#define __NR_delete_module      129 /* ok */
+#define __NR_get_kernel_syms    130 /* remove */
+#define __NR_quotactl           131 /* ok */
+#define __NR_getpgid            132 /* ok */
+#define __NR_fchdir             133 /* ok */
+#define __NR_bdflush            134 /* remove */
+#define __NR_sysfs              135 /* needed for busybox */
+#define __NR_personality        136 /* ok */
+#define __NR_afs_syscall        137 /* Syscall for Andrew File System */
+#define __NR_setfsuid           138 /* ok */
+#define __NR_setfsgid           139 /* ok */
+#define __NR__llseek            140 /* remove only lseek */
+#define __NR_getdents           141 /* ok or getdents64 */
+#define __NR__newselect         142 /* remove */
+#define __NR_flock              143 /* ok */
+#define __NR_msync              144 /* remove */
+#define __NR_readv              145 /* ok */
+#define __NR_writev             146 /* ok */
+#define __NR_getsid             147 /* ok */
+#define __NR_fdatasync          148 /* ok */
+#define __NR__sysctl            149 /* remove */
+#define __NR_mlock              150 /* ok - nommu or mmu */
+#define __NR_munlock            151 /* ok - nommu or mmu */
+#define __NR_mlockall           152 /* ok - nommu or mmu */
+#define __NR_munlockall         153 /* ok - nommu or mmu */
+#define __NR_sched_setparam             154 /* ok */
+#define __NR_sched_getparam             155 /* ok */
+#define __NR_sched_setscheduler         156 /* ok */
+#define __NR_sched_getscheduler         157 /* ok */
+#define __NR_sched_yield                158 /* ok */
+#define __NR_sched_get_priority_max     159 /* ok */
+#define __NR_sched_get_priority_min     160 /* ok */
+#define __NR_sched_rr_get_interval      161 /* ok */
+#define __NR_nanosleep          162 /* ok */
+#define __NR_mremap             163 /* ok - nommu or mmu */
+#define __NR_setresuid          164 /* ok */
+#define __NR_getresuid          165 /* ok */
+#define __NR_vm86               166 /* remove */
+#define __NR_query_module       167 /* ok */
+#define __NR_poll               168 /* obsolete -> sys_ppoll */
+#define __NR_nfsservctl         169 /* ok */
+#define __NR_setresgid          170 /* ok */
+#define __NR_getresgid          171 /* ok */
+#define __NR_prctl              172 /* ok */
+#define __NR_rt_sigreturn       173 /* ok */
+#define __NR_rt_sigaction       174 /* ok */
+#define __NR_rt_sigprocmask     175 /* ok */
+#define __NR_rt_sigpending      176 /* ok */
+#define __NR_rt_sigtimedwait    177 /* ok */
+#define __NR_rt_sigqueueinfo    178 /* ok */
+#define __NR_rt_sigsuspend      179 /* ok */
+#define __NR_pread64            180 /* ok */
+#define __NR_pwrite64           181 /* ok */
+#define __NR_chown              182 /* obsolete -> fchownat */
+#define __NR_getcwd             183 /* ok */
+#define __NR_capget             184 /* ok */
+#define __NR_capset             185 /* ok */
+#define __NR_sigaltstack        186 /* remove */
+#define __NR_sendfile           187 /* ok -> exist 64bit version*/
+#define __NR_getpmsg            188 /* remove */
+/* remove - some people actually want streams */
+#define __NR_putpmsg            189
+/* for noMMU - group with clone -> maybe remove */
+#define __NR_vfork              190
+#define __NR_ugetrlimit         191 /* remove - SuS compliant getrlimit */
+#define __NR_mmap2              192 /* ok */
+#define __NR_truncate64         193 /* ok */
+#define __NR_ftruncate64        194 /* ok */
+#define __NR_stat64             195 /* remove _ARCH_WANT_STAT64 */
+#define __NR_lstat64            196 /* remove _ARCH_WANT_STAT64 */
+#define __NR_fstat64            197 /* remove _ARCH_WANT_STAT64 */
+#define __NR_lchown32           198 /* ok - without 32 */
+#define __NR_getuid32           199 /* ok - without 32 */
+#define __NR_getgid32           200 /* ok - without 32 */
+#define __NR_geteuid32          201 /* ok - without 32 */
+#define __NR_getegid32          202 /* ok - without 32 */
+#define __NR_setreuid32         203 /* ok - without 32 */
+#define __NR_setregid32         204 /* ok - without 32 */
+#define __NR_getgroups32        205 /* ok - without 32 */
+#define __NR_setgroups32        206 /* ok - without 32 */
+#define __NR_fchown32           207 /* ok - without 32 */
+#define __NR_setresuid32        208 /* ok - without 32 */
+#define __NR_getresuid32        209 /* ok - without 32 */
+#define __NR_setresgid32        210 /* ok - without 32 */
+#define __NR_getresgid32        211 /* ok - without 32 */
+#define __NR_chown32            212 /* ok - without 32 -obsolete -> fchownat */
+#define __NR_setuid32           213 /* ok - without 32 */
+#define __NR_setgid32           214 /* ok - without 32 */
+#define __NR_setfsuid32         215 /* ok - without 32 */
+#define __NR_setfsgid32         216 /* ok - without 32 */
+#define __NR_pivot_root         217 /* ok */
+#define __NR_mincore            218 /* ok */
+#define __NR_madvise            219 /* ok */
+#define __NR_getdents64         220 /* ok */
+#define __NR_fcntl64            221 /* ok */
+/* 223 is unused */
+#define __NR_gettid             224 /* ok */
+#define __NR_readahead          225 /* ok */
+#define __NR_setxattr           226 /* ok */
+#define __NR_lsetxattr          227 /* ok */
+#define __NR_fsetxattr          228 /* ok */
+#define __NR_getxattr           229 /* ok */
+#define __NR_lgetxattr          230 /* ok */
+#define __NR_fgetxattr          231 /* ok */
+#define __NR_listxattr          232 /* ok */
+#define __NR_llistxattr         233 /* ok */
+#define __NR_flistxattr         234 /* ok */
+#define __NR_removexattr        235 /* ok */
+#define __NR_lremovexattr       236 /* ok */
+#define __NR_fremovexattr       237 /* ok */
+#define __NR_tkill              238 /* ok */
+#define __NR_sendfile64         239 /* ok */
+#define __NR_futex              240 /* ok */
+#define __NR_sched_setaffinity  241 /* ok */
+#define __NR_sched_getaffinity  242 /* ok */
+#define __NR_set_thread_area    243 /* remove */
+#define __NR_get_thread_area    244 /* remove */
+#define __NR_io_setup           245 /* ok */
+#define __NR_io_destroy         246 /* ok */
+#define __NR_io_getevents       247 /* ok */
+#define __NR_io_submit          248 /* ok */
+#define __NR_io_cancel          249 /* ok */
+#define __NR_fadvise64          250 /* remove -> sys_fadvise64_64 */
+/* 251 is available for reuse (was briefly sys_set_zone_reclaim) */
+#define __NR_exit_group         252 /* ok */
+#define __NR_lookup_dcookie     253 /* ok */
+#define __NR_epoll_create       254 /* ok */
+#define __NR_epoll_ctl          255 /* ok */
+#define __NR_epoll_wait         256 /* obsolete -> sys_epoll_pwait */
+#define __NR_remap_file_pages   257 /* only for mmu */
+#define __NR_set_tid_address    258 /* ok */
+#define __NR_timer_create       259 /* ok */
+#define __NR_timer_settime      (__NR_timer_create+1) /* 260 */ /* ok */
+#define __NR_timer_gettime      (__NR_timer_create+2) /* 261 */ /* ok */
+#define __NR_timer_getoverrun   (__NR_timer_create+3) /* 262 */ /* ok */
+#define __NR_timer_delete       (__NR_timer_create+4) /* 263 */ /* ok */
+#define __NR_clock_settime      (__NR_timer_create+5) /* 264 */ /* ok */
+#define __NR_clock_gettime      (__NR_timer_create+6) /* 265 */ /* ok */
+#define __NR_clock_getres       (__NR_timer_create+7) /* 266 */ /* ok */
+#define __NR_clock_nanosleep    (__NR_timer_create+8) /* 267 */ /* ok */
+#define __NR_statfs64           268 /* ok */
+#define __NR_fstatfs64          269 /* ok */
+#define __NR_tgkill             270 /* ok */
+#define __NR_utimes             271 /* obsolete -> sys_futimesat */
+#define __NR_fadvise64_64       272 /* ok */
+#define __NR_vserver            273 /* ok */
+#define __NR_mbind              274 /* only for mmu */
+#define __NR_get_mempolicy      275 /* only for mmu */
+#define __NR_set_mempolicy      276 /* only for mmu */
+#define __NR_mq_open            277 /* ok */
+#define __NR_mq_unlink          (__NR_mq_open+1) /* 278 */ /* ok */
+#define __NR_mq_timedsend       (__NR_mq_open+2) /* 279 */ /* ok */
+#define __NR_mq_timedreceive    (__NR_mq_open+3) /* 280 */ /* ok */
+#define __NR_mq_notify          (__NR_mq_open+4) /* 281 */ /* ok */
+#define __NR_mq_getsetattr      (__NR_mq_open+5) /* 282 */ /* ok */
+#define __NR_kexec_load         283 /* ok */
+#define __NR_waitid             284 /* ok */
+/* #define __NR_sys_setaltroot  285 */
+#define __NR_add_key            286 /* ok */
+#define __NR_request_key        287 /* ok */
+#define __NR_keyctl             288 /* ok */
+#define __NR_ioprio_set         289 /* ok */
+#define __NR_ioprio_get         290 /* ok */
+#define __NR_inotify_init       291 /* ok */
+#define __NR_inotify_add_watch  292 /* ok */
+#define __NR_inotify_rm_watch   293 /* ok */
+#define __NR_migrate_pages      294 /* mmu */
+#define __NR_openat             295 /* ok */
+#define __NR_mkdirat            296 /* ok */
+#define __NR_mknodat            297 /* ok */
+#define __NR_fchownat           298 /* ok */
+#define __NR_futimesat          299 /* obsolete -> sys_utimesat */
+#define __NR_fstatat64          300 /* stat64 */
+#define __NR_unlinkat           301 /* ok */
+#define __NR_renameat           302 /* ok */
+#define __NR_linkat             303 /* ok */
+#define __NR_symlinkat          304 /* ok */
+#define __NR_readlinkat         305 /* ok */
+#define __NR_fchmodat           306 /* ok */
+#define __NR_faccessat          307 /* ok */
+#define __NR_pselect6           308 /* obsolete -> sys_pselect7 */
+#define __NR_ppoll              309 /* ok */
+#define __NR_unshare            310 /* ok */
+#define __NR_set_robust_list    311 /* ok */
+#define __NR_get_robust_list    312 /* ok */
+#define __NR_splice             313 /* ok */
+#define __NR_sync_file_range    314 /* ok */
+#define __NR_tee                315 /* ok */
+#define __NR_vmsplice           316 /* ok */
+#define __NR_move_pages         317 /* mmu */
+#define __NR_getcpu             318 /* ok */
+#define __NR_epoll_pwait        319 /* ok */
+#define __NR_utimensat          320 /* ok */
+#define __NR_signalfd           321 /* ok */
+#define __NR_timerfd_create     322 /* ok */
+#define __NR_eventfd            323 /* ok */
+#define __NR_fallocate          324 /* ok */
+#define __NR_semtimedop         325 /* ok - semaphore group */
+#define __NR_timerfd_settime    326 /* ok */
+#define __NR_timerfd_gettime    327 /* ok */
+/* sysv ipc syscalls */
+#define __NR_semctl             328 /* ok */
+#define __NR_semget             329 /* ok */
+#define __NR_semop              330 /* ok */
+#define __NR_msgctl             331 /* ok */
+#define __NR_msgget             332 /* ok */
+#define __NR_msgrcv             333 /* ok */
+#define __NR_msgsnd             334 /* ok */
+#define __NR_shmat              335 /* ok */
+#define __NR_shmctl             336 /* ok */
+#define __NR_shmdt              337 /* ok */
+#define __NR_shmget             338 /* ok */
+#define __NR_signalfd4          339 /* new */
+#define __NR_eventfd2           340 /* new */
+#define __NR_epoll_create1      341 /* new */
+#define __NR_dup3               342 /* new */
+#define __NR_pipe2              343 /* new */
+#define __NR_inotify_init1      344 /* new */
+#define __NR_socket             345 /* new */
+#define __NR_socketpair         346 /* new */
+#define __NR_bind               347 /* new */
+#define __NR_listen             348 /* new */
+#define __NR_accept             349 /* new */
+#define __NR_connect            350 /* new */
+#define __NR_getsockname        351 /* new */
+#define __NR_getpeername        352 /* new */
+#define __NR_sendto             353 /* new */
+#define __NR_send               354 /* new */
+#define __NR_recvfrom           355 /* new */
+#define __NR_recv               356 /* new */
+#define __NR_setsockopt         357 /* new */
+#define __NR_getsockopt         358 /* new */
+#define __NR_shutdown           359 /* new */
+#define __NR_sendmsg            360 /* new */
+#define __NR_recvmsg            361 /* new */
+#define __NR_accept4            362 /* new */
+#define __NR_preadv             363 /* new */
+#define __NR_pwritev            364 /* new */
+#define __NR_rt_tgsigqueueinfo  365 /* new */
+#define __NR_perf_event_open    366 /* new */
+#define __NR_recvmmsg           367 /* new */
+#define __NR_fanotify_init      368
+#define __NR_fanotify_mark      369
+#define __NR_prlimit64          370
+#define __NR_name_to_handle_at  371
+#define __NR_open_by_handle_at  372
+#define __NR_clock_adjtime      373
+#define __NR_syncfs             374
+#define __NR_setns              375
+#define __NR_sendmmsg           376
+#define __NR_process_vm_readv   377
+#define __NR_process_vm_writev  378
+#define __NR_kcmp               379
+#define __NR_syscalls           380
+#endif /* _UAPI_ASM_MICROBLAZE_UNISTD_H */
diff --git a/arch/microblaze/kernel/entry-nommu.S b/arch/microblaze/kernel/entry-nommu.S
index cb0327f204ab..70da83a49670 100644
--- a/arch/microblaze/kernel/entry-nommu.S
+++ b/arch/microblaze/kernel/entry-nommu.S
@@ -465,7 +465,6 @@ ENTRY(_switch_to)
 ENTRY(ret_from_fork)
        addk    r5, r0, r3
-        addk    r6, r0, r1
        brlid   r15, schedule_tail
        nop
        swi     r31, r1, PT_R31         /* save r31 in user context. */
diff --git a/arch/microblaze/kernel/intc.c b/arch/microblaze/kernel/intc.c
index 6c54d4dcdec3..7a1a8d4354fe 100644
--- a/arch/microblaze/kernel/intc.c
+++ b/arch/microblaze/kernel/intc.c
@@ -44,7 +44,6 @@ static void intc_enable_or_unmask(struct irq_data *d)
        unsigned long mask = 1 << d->hwirq;
        pr_debug("enable_or_unmask: %ld\n", d->hwirq);
-        out_be32(INTC_BASE + SIE, mask);
        /* ack level irqs because they can't be acked during
         * ack function since the handle_level_irq function
@@ -52,6 +51,8 @@ static void intc_enable_or_unmask(struct irq_data *d)
         */
        if (irqd_is_level_type(d))
                out_be32(INTC_BASE + IAR, mask);
+        out_be32(INTC_BASE + SIE, mask);
 }
 static void intc_disable_or_mask(struct irq_data *d)
@@ -98,7 +99,7 @@ unsigned int get_irq(void)
        return irq;
 }
-int xintc_map(struct irq_domain *d, unsigned int irq, irq_hw_number_t hw)
+static int xintc_map(struct irq_domain *d, unsigned int irq, irq_hw_number_t hw)
 {
        u32 intr_mask = (u32)d->host_data;
diff --git a/arch/microblaze/kernel/process.c b/arch/microblaze/kernel/process.c
index 40823fd1db0b..a5b74f729e5b 100644
--- a/arch/microblaze/kernel/process.c
+++ b/arch/microblaze/kernel/process.c
@@ -162,7 +162,6 @@ int copy_thread(unsigned long clone_flags, unsigned long usp,
         * excepting for VM and UMS
         * don't touch UMS , CARRY and cache bits
         * right now MSR is a copy of parent one */
-        childregs->msr |= MSR_BIP;
        childregs->msr &= ~MSR_EIP;
        childregs->msr |= MSR_IE;
        childregs->msr &= ~MSR_VM;
diff --git a/arch/microblaze/kernel/prom.c b/arch/microblaze/kernel/prom.c
index 4a764ccb9f26..a744e3f18883 100644
--- a/arch/microblaze/kernel/prom.c
+++ b/arch/microblaze/kernel/prom.c
@@ -52,9 +52,9 @@ void * __init early_init_dt_alloc_memory_arch(u64 size, u64 align)
 }
 #ifdef CONFIG_EARLY_PRINTK
-char *stdout;
+static char *stdout;
-int __init early_init_dt_scan_chosen_serial(unsigned long node,
+static int __init early_init_dt_scan_chosen_serial(unsigned long node,
                                const char *uname, int depth, void *data)
 {
        unsigned long l;
diff --git a/arch/microblaze/kernel/signal.c b/arch/microblaze/kernel/signal.c
index 3903e3d11f5a..ac3d0a0f4814 100644
--- a/arch/microblaze/kernel/signal.c
+++ b/arch/microblaze/kernel/signal.c
@@ -354,7 +354,7 @@ static void do_signal(struct pt_regs *regs, int in_syscall)
        restore_saved_sigmask();
 }
-void do_notify_resume(struct pt_regs *regs, int in_syscall)
+asmlinkage void do_notify_resume(struct pt_regs *regs, int in_syscall)
 {
        /*
         * We want the common case to go fast, which
diff --git a/arch/microblaze/kernel/syscall_table.S b/arch/microblaze/kernel/syscall_table.S
index ff6431e54680..1cbace29b5e2 100644
--- a/arch/microblaze/kernel/syscall_table.S
+++ b/arch/microblaze/kernel/syscall_table.S
@@ -379,3 +379,4 @@ ENTRY(sys_call_table)
        .long sys_sendmmsg
        .long sys_process_vm_readv
        .long sys_process_vm_writev
+        .long sys_kcmp
diff --git a/arch/microblaze/lib/libgcc.h b/arch/microblaze/lib/libgcc.h
index 05909d58e2fe..ab077ef7e14b 100644
--- a/arch/microblaze/lib/libgcc.h
+++ b/arch/microblaze/lib/libgcc.h
@@ -22,4 +22,11 @@ typedef union {
        long long ll;
 } DWunion;
+extern long long __ashldi3(long long u, word_type b);
+extern long long __ashrdi3(long long u, word_type b);
+extern word_type __cmpdi2(long long a, long long b);
+extern long long __lshrdi3(long long u, word_type b);
+extern long long __muldi3(long long u, long long v);
+extern word_type __ucmpdi2(unsigned long long a, unsigned long long b);
 #endif /* __ASM_LIBGCC_H */
diff --git a/arch/microblaze/lib/muldi3.c b/arch/microblaze/lib/muldi3.c
index 0585bccb7fad..d3659244ab6f 100644
--- a/arch/microblaze/lib/muldi3.c
+++ b/arch/microblaze/lib/muldi3.c
@@ -2,32 +2,28 @@
 #include "libgcc.h"
-#define DWtype long long
-#define UWtype unsigned long
-#define UHWtype unsigned short
 #define W_TYPE_SIZE 32
-#define __ll_B ((UWtype) 1 << (W_TYPE_SIZE / 2))
+#define __ll_B ((unsigned long) 1 << (W_TYPE_SIZE / 2))
-#define __ll_lowpart(t) ((UWtype) (t) & (__ll_B - 1))
+#define __ll_lowpart(t) ((unsigned long) (t) & (__ll_B - 1))
-#define __ll_highpart(t) ((UWtype) (t) >> (W_TYPE_SIZE / 2))
+#define __ll_highpart(t) ((unsigned long) (t) >> (W_TYPE_SIZE / 2))
 /* If we still don't have umul_ppmm, define it using plain C.  */
 #if !defined(umul_ppmm)
 #define umul_ppmm(w1, w0, u, v)                                         \
        do {                                                            \
-                UWtype __x0, __x1, __x2, __x3;                          \
+                unsigned long __x0, __x1, __x2, __x3;                   \
-                UHWtype __ul, __vl, __uh, __vh;                         \
+                unsigned short __ul, __vl, __uh, __vh;                  \
                                                                        \
                __ul = __ll_lowpart(u);                                 \
                __uh = __ll_highpart(u);                                \
                __vl = __ll_lowpart(v);                                 \
                __vh = __ll_highpart(v);                                \
                                                                        \
-                __x0 = (UWtype) __ul * __vl;                            \
+                __x0 = (unsigned long) __ul * __vl;                     \
-                __x1 = (UWtype) __ul * __vh;                            \
+                __x1 = (unsigned long) __ul * __vh;                     \
-                __x2 = (UWtype) __uh * __vl;                            \
+                __x2 = (unsigned long) __uh * __vl;                     \
-                __x3 = (UWtype) __uh * __vh;                            \
+                __x3 = (unsigned long) __uh * __vh;                     \
                                                                        \
                __x1 += __ll_highpart(__x0); /* this can't give carry */\
                __x1 += __x2; /* but this indeed can */                 \
@@ -47,14 +43,14 @@
        })
 #endif
-DWtype __muldi3(DWtype u, DWtype v)
+long long __muldi3(long long u, long long v)
 {
        const DWunion uu = {.ll = u};
        const DWunion vv = {.ll = v};
        DWunion w = {.ll = __umulsidi3(uu.s.low, vv.s.low)};
-        w.s.high += ((UWtype) uu.s.low * (UWtype) vv.s.high
+        w.s.high += ((unsigned long) uu.s.low * (unsigned long) vv.s.high
-                + (UWtype) uu.s.high * (UWtype) vv.s.low);
+                + (unsigned long) uu.s.high * (unsigned long) vv.s.low);
        return w.ll;
 }
diff --git a/arch/sh/boards/mach-ap325rxa/setup.c b/arch/sh/boards/mach-ap325rxa/setup.c
index 9e963c1d1447..5620e33c18a0 100644
--- a/arch/sh/boards/mach-ap325rxa/setup.c
+++ b/arch/sh/boards/mach-ap325rxa/setup.c
@@ -179,11 +179,6 @@ static int ap320_wvga_set_brightness(int brightness)
        return 0;
 }
-static int ap320_wvga_get_brightness(void)
-{
-        return gpio_get_value(GPIO_PTS3);
-}
 static void ap320_wvga_power_on(void)
 {
        msleep(100);
@@ -232,7 +227,6 @@ static struct sh_mobile_lcdc_info lcdc_info = {
                        .name = "sh_mobile_lcdc_bl",
                        .max_brightness = 1,
                        .set_brightness = ap320_wvga_set_brightness,
-                        .get_brightness = ap320_wvga_get_brightness,
                },
        }
 };
diff --git a/arch/sh/boards/mach-ecovec24/setup.c b/arch/sh/boards/mach-ecovec24/setup.c
index 64559e8af14b..3fede4556c91 100644
--- a/arch/sh/boards/mach-ecovec24/setup.c
+++ b/arch/sh/boards/mach-ecovec24/setup.c
@@ -329,11 +329,6 @@ static int ecovec24_set_brightness(int brightness)
        return 0;
 }
-static int ecovec24_get_brightness(void)
-{
-        return gpio_get_value(GPIO_PTR1);
-}
 static struct sh_mobile_lcdc_info lcdc_info = {
        .ch[0] = {
                .interface_type = RGB18,
@@ -347,7 +342,6 @@ static struct sh_mobile_lcdc_info lcdc_info = {
                        .name = "sh_mobile_lcdc_bl",
                        .max_brightness = 1,
                        .set_brightness = ecovec24_set_brightness,
-                        .get_brightness = ecovec24_get_brightness,
                },
        }
 };
diff --git a/arch/sh/boards/mach-kfr2r09/lcd_wqvga.c b/arch/sh/boards/mach-kfr2r09/lcd_wqvga.c
index c148b36ecb65..c62050332629 100644
--- a/arch/sh/boards/mach-kfr2r09/lcd_wqvga.c
+++ b/arch/sh/boards/mach-kfr2r09/lcd_wqvga.c
@@ -283,7 +283,7 @@ void kfr2r09_lcd_start(void *sohandle, struct sh_mobile_lcdc_sys_bus_ops *so)
 #define MAIN_MLED4      0x40
 #define MAIN_MSW        0x80
-static int kfr2r09_lcd_backlight(int on)
+int kfr2r09_lcd_set_brightness(int brightness)
 {
        struct i2c_adapter *a;
        struct i2c_msg msg;
@@ -295,7 +295,7 @@ static int kfr2r09_lcd_backlight(int on)
                return -ENODEV;
        buf[0] = 0x00;
-        if (on)
+        if (brightness)
                buf[1] = CTRL_CPSW | CTRL_C10 | CTRL_CKSW;
        else
                buf[1] = 0;
@@ -309,7 +309,7 @@ static int kfr2r09_lcd_backlight(int on)
                return -ENODEV;
        buf[0] = 0x01;
-        if (on)
+        if (brightness)
                buf[1] = MAIN_MSW | MAIN_MLED4 | 0x0c;
        else
                buf[1] = 0;
@@ -324,13 +324,3 @@ static int kfr2r09_lcd_backlight(int on)
        return 0;
 }
-void kfr2r09_lcd_on(void)
-{
-        kfr2r09_lcd_backlight(1);
-}
-void kfr2r09_lcd_off(void)
-{
-        kfr2r09_lcd_backlight(0);
-}
diff --git a/arch/sh/boards/mach-kfr2r09/setup.c b/arch/sh/boards/mach-kfr2r09/setup.c
index f2a4304fbe23..ab502f12ef57 100644
--- a/arch/sh/boards/mach-kfr2r09/setup.c
+++ b/arch/sh/boards/mach-kfr2r09/setup.c
@@ -158,8 +158,11 @@ static struct sh_mobile_lcdc_info kfr2r09_sh_lcdc_info = {
                        .height = 58,
                        .setup_sys = kfr2r09_lcd_setup,
                        .start_transfer = kfr2r09_lcd_start,
-                        .display_on = kfr2r09_lcd_on,
+                },
-                        .display_off = kfr2r09_lcd_off,
+                .bl_info = {
+                        .name = "sh_mobile_lcdc_bl",
+                        .max_brightness = 1,
+                        .set_brightness = kfr2r09_lcd_set_brightness,
                },
                .sys_bus_cfg = {
                        .ldmt2r = 0x07010904,
diff --git a/arch/sh/include/mach-kfr2r09/mach/kfr2r09.h b/arch/sh/include/mach-kfr2r09/mach/kfr2r09.h
index ba3d93d333f8..c20c9e5f5eab 100644
--- a/arch/sh/include/mach-kfr2r09/mach/kfr2r09.h
+++ b/arch/sh/include/mach-kfr2r09/mach/kfr2r09.h
@@ -4,15 +4,13 @@
 #include <video/sh_mobile_lcdc.h>
 #if defined(CONFIG_FB_SH_MOBILE_LCDC) || defined(CONFIG_FB_SH_MOBILE_LCDC_MODULE)
-void kfr2r09_lcd_on(void);
+int kfr2r09_lcd_set_brightness(int brightness);
-void kfr2r09_lcd_off(void);
 int kfr2r09_lcd_setup(void *sys_ops_handle,
                      struct sh_mobile_lcdc_sys_bus_ops *sys_ops);
 void kfr2r09_lcd_start(void *sys_ops_handle,
                       struct sh_mobile_lcdc_sys_bus_ops *sys_ops);
 #else
-static void kfr2r09_lcd_on(void) {}
+static int kfr2r09_lcd_set_brightness(int brightness) {}
-static void kfr2r09_lcd_off(void) {}
 static int kfr2r09_lcd_setup(void *sys_ops_handle,
                                struct sh_mobile_lcdc_sys_bus_ops *sys_ops)
 {
diff --git a/arch/sh/mm/Kconfig b/arch/sh/mm/Kconfig
index cb8f9920f4dd..0f7c852f355c 100644
--- a/arch/sh/mm/Kconfig
+++ b/arch/sh/mm/Kconfig
@@ -111,6 +111,7 @@ config VSYSCALL
 config NUMA
        bool "Non Uniform Memory Access (NUMA) Support"
        depends on MMU && SYS_SUPPORTS_NUMA && EXPERIMENTAL
+        select ARCH_WANT_NUMA_VARIABLE_LOCALITY
        default n
        help
          Some SH systems have many various memories scattered around
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 65a872bf72f9..97f8c5ad8c2d 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -22,6 +22,8 @@ config X86
        def_bool y
        select HAVE_AOUT if X86_32
        select HAVE_UNSTABLE_SCHED_CLOCK
+        select ARCH_SUPPORTS_NUMA_BALANCING
+        select ARCH_WANTS_PROT_NUMA_PROT_NONE
        select HAVE_IDE
        select HAVE_OPROFILE
        select HAVE_PCSPKR_PLATFORM
diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile
index 5bacb4a226ac..e0ca7c9ac383 100644
--- a/arch/x86/crypto/Makefile
+++ b/arch/x86/crypto/Makefile
@@ -12,6 +12,7 @@ obj-$(CONFIG_CRYPTO_SERPENT_SSE2_586) += serpent-sse2-i586.o
 obj-$(CONFIG_CRYPTO_AES_X86_64) += aes-x86_64.o
 obj-$(CONFIG_CRYPTO_CAMELLIA_X86_64) += camellia-x86_64.o
+obj-$(CONFIG_CRYPTO_CAMELLIA_AESNI_AVX_X86_64) += camellia-aesni-avx-x86_64.o
 obj-$(CONFIG_CRYPTO_CAST5_AVX_X86_64) += cast5-avx-x86_64.o
 obj-$(CONFIG_CRYPTO_CAST6_AVX_X86_64) += cast6-avx-x86_64.o
 obj-$(CONFIG_CRYPTO_BLOWFISH_X86_64) += blowfish-x86_64.o
@@ -34,6 +35,8 @@ serpent-sse2-i586-y := serpent-sse2-i586-asm_32.o serpent_sse2_glue.o
 aes-x86_64-y := aes-x86_64-asm_64.o aes_glue.o
 camellia-x86_64-y := camellia-x86_64-asm_64.o camellia_glue.o
+camellia-aesni-avx-x86_64-y := camellia-aesni-avx-asm_64.o \
+                               camellia_aesni_avx_glue.o
 cast5-avx-x86_64-y := cast5-avx-x86_64-asm_64.o cast5_avx_glue.o
 cast6-avx-x86_64-y := cast6-avx-x86_64-asm_64.o cast6_avx_glue.o
 blowfish-x86_64-y := blowfish-x86_64-asm_64.o blowfish_glue.o
@@ -47,3 +50,5 @@ serpent-avx-x86_64-y := serpent-avx-x86_64-asm_64.o serpent_avx_glue.o
 aesni-intel-y := aesni-intel_asm.o aesni-intel_glue.o fpu.o
 ghash-clmulni-intel-y := ghash-clmulni-intel_asm.o ghash-clmulni-intel_glue.o
 sha1-ssse3-y := sha1_ssse3_asm.o sha1_ssse3_glue.o
+crc32c-intel-y := crc32c-intel_glue.o
+crc32c-intel-$(CONFIG_CRYPTO_CRC32C_X86_64) += crc32c-pcl-intel-asm_64.o
diff --git a/arch/x86/crypto/camellia-aesni-avx-asm_64.S b/arch/x86/crypto/camellia-aesni-avx-asm_64.S
new file mode 100644
index 000000000000..2306d2e4816f
--- /dev/null
+++ b/arch/x86/crypto/camellia-aesni-avx-asm_64.S
@@ -0,0 +1,1102 @@
+/*
+ * x86_64/AVX/AES-NI assembler implementation of Camellia
+ *
+ * Copyright © 2012 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ */
+/*
+ * Version licensed under 2-clause BSD License is available at:
+ *      http://koti.mbnet.fi/axh/crypto/camellia-BSD-1.2.0-aesni1.tar.xz
+ */
+#define CAMELLIA_TABLE_BYTE_LEN 272
+/* struct camellia_ctx: */
+#define key_table 0
+#define key_length CAMELLIA_TABLE_BYTE_LEN
+/* register macros */
+#define CTX %rdi
+/**********************************************************************
+  16-way camellia
+ **********************************************************************/
+#define filter_8bit(x, lo_t, hi_t, mask4bit, tmp0) \
+        vpand x, mask4bit, tmp0; \
+        vpandn x, mask4bit, x; \
+        vpsrld $4, x, x; \
+        \
+        vpshufb tmp0, lo_t, tmp0; \
+        vpshufb x, hi_t, x; \
+        vpxor tmp0, x, x;
+/*
+ * IN:
+ *   x0..x7: byte-sliced AB state
+ *   mem_cd: register pointer storing CD state
+ *   key: index for key material
+ * OUT:
+ *   x0..x7: new byte-sliced CD state
+ */
+#define roundsm16(x0, x1, x2, x3, x4, x5, x6, x7, t0, t1, t2, t3, t4, t5, t6, \
+                  t7, mem_cd, key) \
+        /* \
+         * S-function with AES subbytes \
+         */ \
+        vmovdqa .Linv_shift_row, t4; \
+        vbroadcastss .L0f0f0f0f, t7; \
+        vmovdqa .Lpre_tf_lo_s1, t0; \
+        vmovdqa .Lpre_tf_hi_s1, t1; \
+        \
+        /* AES inverse shift rows */ \
+        vpshufb t4, x0, x0; \
+        vpshufb t4, x7, x7; \
+        vpshufb t4, x1, x1; \
+        vpshufb t4, x4, x4; \
+        vpshufb t4, x2, x2; \
+        vpshufb t4, x5, x5; \
+        vpshufb t4, x3, x3; \
+        vpshufb t4, x6, x6; \
+        \
+        /* prefilter sboxes 1, 2 and 3 */ \
+        vmovdqa .Lpre_tf_lo_s4, t2; \
+        vmovdqa .Lpre_tf_hi_s4, t3; \
+        filter_8bit(x0, t0, t1, t7, t6); \
+        filter_8bit(x7, t0, t1, t7, t6); \
+        filter_8bit(x1, t0, t1, t7, t6); \
+        filter_8bit(x4, t0, t1, t7, t6); \
+        filter_8bit(x2, t0, t1, t7, t6); \
+        filter_8bit(x5, t0, t1, t7, t6); \
+        \
+        /* prefilter sbox 4 */ \
+        vpxor t4, t4, t4; \
+        filter_8bit(x3, t2, t3, t7, t6); \
+        filter_8bit(x6, t2, t3, t7, t6); \
+        \
+        /* AES subbytes + AES shift rows */ \
+        vmovdqa .Lpost_tf_lo_s1, t0; \
+        vmovdqa .Lpost_tf_hi_s1, t1; \
+        vaesenclast t4, x0, x0; \
+        vaesenclast t4, x7, x7; \
+        vaesenclast t4, x1, x1; \
+        vaesenclast t4, x4, x4; \
+        vaesenclast t4, x2, x2; \
+        vaesenclast t4, x5, x5; \
+        vaesenclast t4, x3, x3; \
+        vaesenclast t4, x6, x6; \
+        \
+        /* postfilter sboxes 1 and 4 */ \
+        vmovdqa .Lpost_tf_lo_s3, t2; \
+        vmovdqa .Lpost_tf_hi_s3, t3; \
+        filter_8bit(x0, t0, t1, t7, t6); \
+        filter_8bit(x7, t0, t1, t7, t6); \
+        filter_8bit(x3, t0, t1, t7, t6); \
+        filter_8bit(x6, t0, t1, t7, t6); \
+        \
+        /* postfilter sbox 3 */ \
+        vmovdqa .Lpost_tf_lo_s2, t4; \
+        vmovdqa .Lpost_tf_hi_s2, t5; \
+        filter_8bit(x2, t2, t3, t7, t6); \
+        filter_8bit(x5, t2, t3, t7, t6); \
+        \
+        vpxor t6, t6, t6; \
+        vmovq key, t0; \
+        \
+        /* postfilter sbox 2 */ \
+        filter_8bit(x1, t4, t5, t7, t2); \
+        filter_8bit(x4, t4, t5, t7, t2); \
+        \
+        vpsrldq $5, t0, t5; \
+        vpsrldq $1, t0, t1; \
+        vpsrldq $2, t0, t2; \
+        vpsrldq $3, t0, t3; \
+        vpsrldq $4, t0, t4; \
+        vpshufb t6, t0, t0; \
+        vpshufb t6, t1, t1; \
+        vpshufb t6, t2, t2; \
+        vpshufb t6, t3, t3; \
+        vpshufb t6, t4, t4; \
+        vpsrldq $2, t5, t7; \
+        vpshufb t6, t7, t7; \
+        \
+        /* \
+         * P-function \
+         */ \
+        vpxor x5, x0, x0; \
+        vpxor x6, x1, x1; \
+        vpxor x7, x2, x2; \
+        vpxor x4, x3, x3; \
+        \
+        vpxor x2, x4, x4; \
+        vpxor x3, x5, x5; \
+        vpxor x0, x6, x6; \
+        vpxor x1, x7, x7; \
+        \
+        vpxor x7, x0, x0; \
+        vpxor x4, x1, x1; \
+        vpxor x5, x2, x2; \
+        vpxor x6, x3, x3; \
+        \
+        vpxor x3, x4, x4; \
+        vpxor x0, x5, x5; \
+        vpxor x1, x6, x6; \
+        vpxor x2, x7, x7; /* note: high and low parts swapped */ \
+        \
+        /* \
+         * Add key material and result to CD (x becomes new CD) \
+         */ \
+        \
+        vpxor t3, x4, x4; \
+        vpxor 0 * 16(mem_cd), x4, x4; \
+        \
+        vpxor t2, x5, x5; \
+        vpxor 1 * 16(mem_cd), x5, x5; \
+        \
+        vpsrldq $1, t5, t3; \
+        vpshufb t6, t5, t5; \
+        vpshufb t6, t3, t6; \
+        \
+        vpxor t1, x6, x6; \
+        vpxor 2 * 16(mem_cd), x6, x6; \
+        \
+        vpxor t0, x7, x7; \
+        vpxor 3 * 16(mem_cd), x7, x7; \
+        \
+        vpxor t7, x0, x0; \
+        vpxor 4 * 16(mem_cd), x0, x0; \
+        \
+        vpxor t6, x1, x1; \
+        vpxor 5 * 16(mem_cd), x1, x1; \
+        \
+        vpxor t5, x2, x2; \
+        vpxor 6 * 16(mem_cd), x2, x2; \
+        \
+        vpxor t4, x3, x3; \
+        vpxor 7 * 16(mem_cd), x3, x3;
+/*
+ * Size optimization... with inlined roundsm16, binary would be over 5 times
+ * larger and would only be 0.5% faster (on sandy-bridge).
+ */
+.align 8
+roundsm16_x0_x1_x2_x3_x4_x5_x6_x7_y0_y1_y2_y3_y4_y5_y6_y7_cd:
+        roundsm16(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
+                  %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, %xmm15,
+                  %rcx, (%r9));
+        ret;
+.align 8
+roundsm16_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_ab:
+        roundsm16(%xmm4, %xmm5, %xmm6, %xmm7, %xmm0, %xmm1, %xmm2, %xmm3,
+                  %xmm12, %xmm13, %xmm14, %xmm15, %xmm8, %xmm9, %xmm10, %xmm11,
+                  %rax, (%r9));
+        ret;
+/*
+ * IN/OUT:
+ *  x0..x7: byte-sliced AB state preloaded
+ *  mem_ab: byte-sliced AB state in memory
+ *  mem_cb: byte-sliced CD state in memory
+ */
+#define two_roundsm16(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \
+                      y6, y7, mem_ab, mem_cd, i, dir, store_ab) \
+        leaq (key_table + (i) * 8)(CTX), %r9; \
+        call roundsm16_x0_x1_x2_x3_x4_x5_x6_x7_y0_y1_y2_y3_y4_y5_y6_y7_cd; \
+        \
+        vmovdqu x4, 0 * 16(mem_cd); \
+        vmovdqu x5, 1 * 16(mem_cd); \
+        vmovdqu x6, 2 * 16(mem_cd); \
+        vmovdqu x7, 3 * 16(mem_cd); \
+        vmovdqu x0, 4 * 16(mem_cd); \
+        vmovdqu x1, 5 * 16(mem_cd); \
+        vmovdqu x2, 6 * 16(mem_cd); \
+        vmovdqu x3, 7 * 16(mem_cd); \
+        \
+        leaq (key_table + ((i) + (dir)) * 8)(CTX), %r9; \
+        call roundsm16_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_ab; \
+        \
+        store_ab(x0, x1, x2, x3, x4, x5, x6, x7, mem_ab);
+#define dummy_store(x0, x1, x2, x3, x4, x5, x6, x7, mem_ab) /* do nothing */
+#define store_ab_state(x0, x1, x2, x3, x4, x5, x6, x7, mem_ab) \
+        /* Store new AB state */ \
+        vmovdqu x0, 0 * 16(mem_ab); \
+        vmovdqu x1, 1 * 16(mem_ab); \
+        vmovdqu x2, 2 * 16(mem_ab); \
+        vmovdqu x3, 3 * 16(mem_ab); \
+        vmovdqu x4, 4 * 16(mem_ab); \
+        vmovdqu x5, 5 * 16(mem_ab); \
+        vmovdqu x6, 6 * 16(mem_ab); \
+        vmovdqu x7, 7 * 16(mem_ab);
+#define enc_rounds16(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \
+                      y6, y7, mem_ab, mem_cd, i) \
+        two_roundsm16(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \
+                      y6, y7, mem_ab, mem_cd, (i) + 2, 1, store_ab_state); \
+        two_roundsm16(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \
+                      y6, y7, mem_ab, mem_cd, (i) + 4, 1, store_ab_state); \
+        two_roundsm16(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \
+                      y6, y7, mem_ab, mem_cd, (i) + 6, 1, dummy_store);
+#define dec_rounds16(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \
+                      y6, y7, mem_ab, mem_cd, i) \
+        two_roundsm16(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \
+                      y6, y7, mem_ab, mem_cd, (i) + 7, -1, store_ab_state); \
+        two_roundsm16(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \
+                      y6, y7, mem_ab, mem_cd, (i) + 5, -1, store_ab_state); \
+        two_roundsm16(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \
+                      y6, y7, mem_ab, mem_cd, (i) + 3, -1, dummy_store);
+/*
+ * IN:
+ *  v0..3: byte-sliced 32-bit integers
+ * OUT:
+ *  v0..3: (IN <<< 1)
+ */
+#define rol32_1_16(v0, v1, v2, v3, t0, t1, t2, zero) \
+        vpcmpgtb v0, zero, t0; \
+        vpaddb v0, v0, v0; \
+        vpabsb t0, t0; \
+        \
+        vpcmpgtb v1, zero, t1; \
+        vpaddb v1, v1, v1; \
+        vpabsb t1, t1; \
+        \
+        vpcmpgtb v2, zero, t2; \
+        vpaddb v2, v2, v2; \
+        vpabsb t2, t2; \
+        \
+        vpor t0, v1, v1; \
+        \
+        vpcmpgtb v3, zero, t0; \
+        vpaddb v3, v3, v3; \
+        vpabsb t0, t0; \
+        \
+        vpor t1, v2, v2; \
+        vpor t2, v3, v3; \
+        vpor t0, v0, v0;
+/*
+ * IN:
+ *   r: byte-sliced AB state in memory
+ *   l: byte-sliced CD state in memory
+ * OUT:
+ *   x0..x7: new byte-sliced CD state
+ */
+#define fls16(l, l0, l1, l2, l3, l4, l5, l6, l7, r, t0, t1, t2, t3, tt0, \
+              tt1, tt2, tt3, kll, klr, krl, krr) \
+        /* \
+         * t0 = kll; \
+         * t0 &= ll; \
+         * lr ^= rol32(t0, 1); \
+         */ \
+        vpxor tt0, tt0, tt0; \
+        vmovd kll, t0; \
+        vpshufb tt0, t0, t3; \
+        vpsrldq $1, t0, t0; \
+        vpshufb tt0, t0, t2; \
+        vpsrldq $1, t0, t0; \
+        vpshufb tt0, t0, t1; \
+        vpsrldq $1, t0, t0; \
+        vpshufb tt0, t0, t0; \
+        \
+        vpand l0, t0, t0; \
+        vpand l1, t1, t1; \
+        vpand l2, t2, t2; \
+        vpand l3, t3, t3; \
+        \
+        rol32_1_16(t3, t2, t1, t0, tt1, tt2, tt3, tt0); \
+        \
+        vpxor l4, t0, l4; \
+        vmovdqu l4, 4 * 16(l); \
+        vpxor l5, t1, l5; \
+        vmovdqu l5, 5 * 16(l); \
+        vpxor l6, t2, l6; \
+        vmovdqu l6, 6 * 16(l); \
+        vpxor l7, t3, l7; \
+        vmovdqu l7, 7 * 16(l); \
+        \
+        /* \
+         * t2 = krr; \
+         * t2 |= rr; \
+         * rl ^= t2; \
+         */ \
+        \
+        vmovd krr, t0; \
+        vpshufb tt0, t0, t3; \
+        vpsrldq $1, t0, t0; \
+        vpshufb tt0, t0, t2; \
+        vpsrldq $1, t0, t0; \
+        vpshufb tt0, t0, t1; \
+        vpsrldq $1, t0, t0; \
+        vpshufb tt0, t0, t0; \
+        \
+        vpor 4 * 16(r), t0, t0; \
+        vpor 5 * 16(r), t1, t1; \
+        vpor 6 * 16(r), t2, t2; \
+        vpor 7 * 16(r), t3, t3; \
+        \
+        vpxor 0 * 16(r), t0, t0; \
+        vpxor 1 * 16(r), t1, t1; \
+        vpxor 2 * 16(r), t2, t2; \
+        vpxor 3 * 16(r), t3, t3; \
+        vmovdqu t0, 0 * 16(r); \
+        vmovdqu t1, 1 * 16(r); \
+        vmovdqu t2, 2 * 16(r); \
+        vmovdqu t3, 3 * 16(r); \
+        \
+        /* \
+         * t2 = krl; \
+         * t2 &= rl; \
+         * rr ^= rol32(t2, 1); \
+         */ \
+        vmovd krl, t0; \
+        vpshufb tt0, t0, t3; \
+        vpsrldq $1, t0, t0; \
+        vpshufb tt0, t0, t2; \
+        vpsrldq $1, t0, t0; \
+        vpshufb tt0, t0, t1; \
+        vpsrldq $1, t0, t0; \
+        vpshufb tt0, t0, t0; \
+        \
+        vpand 0 * 16(r), t0, t0; \
+        vpand 1 * 16(r), t1, t1; \
+        vpand 2 * 16(r), t2, t2; \
+        vpand 3 * 16(r), t3, t3; \
+        \
+        rol32_1_16(t3, t2, t1, t0, tt1, tt2, tt3, tt0); \
+        \
+        vpxor 4 * 16(r), t0, t0; \
+        vpxor 5 * 16(r), t1, t1; \
+        vpxor 6 * 16(r), t2, t2; \
+        vpxor 7 * 16(r), t3, t3; \
+        vmovdqu t0, 4 * 16(r); \
+        vmovdqu t1, 5 * 16(r); \
+        vmovdqu t2, 6 * 16(r); \
+        vmovdqu t3, 7 * 16(r); \
+        \
+        /* \
+         * t0 = klr; \
+         * t0 |= lr; \
+         * ll ^= t0; \
+         */ \
+        \
+        vmovd klr, t0; \
+        vpshufb tt0, t0, t3; \
+        vpsrldq $1, t0, t0; \
+        vpshufb tt0, t0, t2; \
+        vpsrldq $1, t0, t0; \
+        vpshufb tt0, t0, t1; \
+        vpsrldq $1, t0, t0; \
+        vpshufb tt0, t0, t0; \
+        \
+        vpor l4, t0, t0; \
+        vpor l5, t1, t1; \
+        vpor l6, t2, t2; \
+        vpor l7, t3, t3; \
+        \
+        vpxor l0, t0, l0; \
+        vmovdqu l0, 0 * 16(l); \
+        vpxor l1, t1, l1; \
+        vmovdqu l1, 1 * 16(l); \
+        vpxor l2, t2, l2; \
+        vmovdqu l2, 2 * 16(l); \
+        vpxor l3, t3, l3; \
+        vmovdqu l3, 3 * 16(l);
+#define transpose_4x4(x0, x1, x2, x3, t1, t2) \
+        vpunpckhdq x1, x0, t2; \
+        vpunpckldq x1, x0, x0; \
+        \
+        vpunpckldq x3, x2, t1; \
+        vpunpckhdq x3, x2, x2; \
+        \
+        vpunpckhqdq t1, x0, x1; \
+        vpunpcklqdq t1, x0, x0; \
+        \
+        vpunpckhqdq x2, t2, x3; \
+        vpunpcklqdq x2, t2, x2;
+#define byteslice_16x16b(a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, \
+                         b3, c3, d3, st0, st1) \
+        vmovdqu d2, st0; \
+        vmovdqu d3, st1; \
+        transpose_4x4(a0, a1, a2, a3, d2, d3); \
+        transpose_4x4(b0, b1, b2, b3, d2, d3); \
+        vmovdqu st0, d2; \
+        vmovdqu st1, d3; \
+        \
+        vmovdqu a0, st0; \
+        vmovdqu a1, st1; \
+        transpose_4x4(c0, c1, c2, c3, a0, a1); \
+        transpose_4x4(d0, d1, d2, d3, a0, a1); \
+        \
+        vmovdqu .Lshufb_16x16b, a0; \
+        vmovdqu st1, a1; \
+        vpshufb a0, a2, a2; \
+        vpshufb a0, a3, a3; \
+        vpshufb a0, b0, b0; \
+        vpshufb a0, b1, b1; \
+        vpshufb a0, b2, b2; \
+        vpshufb a0, b3, b3; \
+        vpshufb a0, a1, a1; \
+        vpshufb a0, c0, c0; \
+        vpshufb a0, c1, c1; \
+        vpshufb a0, c2, c2; \
+        vpshufb a0, c3, c3; \
+        vpshufb a0, d0, d0; \
+        vpshufb a0, d1, d1; \
+        vpshufb a0, d2, d2; \
+        vpshufb a0, d3, d3; \
+        vmovdqu d3, st1; \
+        vmovdqu st0, d3; \
+        vpshufb a0, d3, a0; \
+        vmovdqu d2, st0; \
+        \
+        transpose_4x4(a0, b0, c0, d0, d2, d3); \
+        transpose_4x4(a1, b1, c1, d1, d2, d3); \
+        vmovdqu st0, d2; \
+        vmovdqu st1, d3; \
+        \
+        vmovdqu b0, st0; \
+        vmovdqu b1, st1; \
+        transpose_4x4(a2, b2, c2, d2, b0, b1); \
+        transpose_4x4(a3, b3, c3, d3, b0, b1); \
+        vmovdqu st0, b0; \
+        vmovdqu st1, b1; \
+        /* does not adjust output bytes inside vectors */
+/* load blocks to registers and apply pre-whitening */
+#define inpack16_pre(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \
+                     y6, y7, rio, key) \
+        vmovq key, x0; \
+        vpshufb .Lpack_bswap, x0, x0; \
+        \
+        vpxor 0 * 16(rio), x0, y7; \
+        vpxor 1 * 16(rio), x0, y6; \
+        vpxor 2 * 16(rio), x0, y5; \
+        vpxor 3 * 16(rio), x0, y4; \
+        vpxor 4 * 16(rio), x0, y3; \
+        vpxor 5 * 16(rio), x0, y2; \
+        vpxor 6 * 16(rio), x0, y1; \
+        vpxor 7 * 16(rio), x0, y0; \
+        vpxor 8 * 16(rio), x0, x7; \
+        vpxor 9 * 16(rio), x0, x6; \
+        vpxor 10 * 16(rio), x0, x5; \
+        vpxor 11 * 16(rio), x0, x4; \
+        vpxor 12 * 16(rio), x0, x3; \
+        vpxor 13 * 16(rio), x0, x2; \
+        vpxor 14 * 16(rio), x0, x1; \
+        vpxor 15 * 16(rio), x0, x0;
+/* byteslice pre-whitened blocks and store to temporary memory */
+#define inpack16_post(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \
+                      y6, y7, mem_ab, mem_cd) \
+        byteslice_16x16b(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, \
+                         y5, y6, y7, (mem_ab), (mem_cd)); \
+        \
+        vmovdqu x0, 0 * 16(mem_ab); \
+        vmovdqu x1, 1 * 16(mem_ab); \
+        vmovdqu x2, 2 * 16(mem_ab); \
+        vmovdqu x3, 3 * 16(mem_ab); \
+        vmovdqu x4, 4 * 16(mem_ab); \
+        vmovdqu x5, 5 * 16(mem_ab); \
+        vmovdqu x6, 6 * 16(mem_ab); \
+        vmovdqu x7, 7 * 16(mem_ab); \
+        vmovdqu y0, 0 * 16(mem_cd); \
+        vmovdqu y1, 1 * 16(mem_cd); \
+        vmovdqu y2, 2 * 16(mem_cd); \
+        vmovdqu y3, 3 * 16(mem_cd); \
+        vmovdqu y4, 4 * 16(mem_cd); \
+        vmovdqu y5, 5 * 16(mem_cd); \
+        vmovdqu y6, 6 * 16(mem_cd); \
+        vmovdqu y7, 7 * 16(mem_cd);
+/* de-byteslice, apply post-whitening and store blocks */
+#define outunpack16(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, \
+                    y5, y6, y7, key, stack_tmp0, stack_tmp1) \
+        byteslice_16x16b(y0, y4, x0, x4, y1, y5, x1, x5, y2, y6, x2, x6, y3, \
+                         y7, x3, x7, stack_tmp0, stack_tmp1); \
+        \
+        vmovdqu x0, stack_tmp0; \
+        \
+        vmovq key, x0; \
+        vpshufb .Lpack_bswap, x0, x0; \
+        \
+        vpxor x0, y7, y7; \
+        vpxor x0, y6, y6; \
+        vpxor x0, y5, y5; \
+        vpxor x0, y4, y4; \
+        vpxor x0, y3, y3; \
+        vpxor x0, y2, y2; \
+        vpxor x0, y1, y1; \
+        vpxor x0, y0, y0; \
+        vpxor x0, x7, x7; \
+        vpxor x0, x6, x6; \
+        vpxor x0, x5, x5; \
+        vpxor x0, x4, x4; \
+        vpxor x0, x3, x3; \
+        vpxor x0, x2, x2; \
+        vpxor x0, x1, x1; \
+        vpxor stack_tmp0, x0, x0;
+#define write_output(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \
+                     y6, y7, rio) \
+        vmovdqu x0, 0 * 16(rio); \
+        vmovdqu x1, 1 * 16(rio); \
+        vmovdqu x2, 2 * 16(rio); \
+        vmovdqu x3, 3 * 16(rio); \
+        vmovdqu x4, 4 * 16(rio); \
+        vmovdqu x5, 5 * 16(rio); \
+        vmovdqu x6, 6 * 16(rio); \
+        vmovdqu x7, 7 * 16(rio); \
+        vmovdqu y0, 8 * 16(rio); \
+        vmovdqu y1, 9 * 16(rio); \
+        vmovdqu y2, 10 * 16(rio); \
+        vmovdqu y3, 11 * 16(rio); \
+        vmovdqu y4, 12 * 16(rio); \
+        vmovdqu y5, 13 * 16(rio); \
+        vmovdqu y6, 14 * 16(rio); \
+        vmovdqu y7, 15 * 16(rio);
+.data
+.align 16
+#define SHUFB_BYTES(idx) \
+        0 + (idx), 4 + (idx), 8 + (idx), 12 + (idx)
+.Lshufb_16x16b:
+        .byte SHUFB_BYTES(0), SHUFB_BYTES(1), SHUFB_BYTES(2), SHUFB_BYTES(3);
+.Lpack_bswap:
+        .long 0x00010203
+        .long 0x04050607
+        .long 0x80808080
+        .long 0x80808080
+/* For CTR-mode IV byteswap */
+.Lbswap128_mask:
+        .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
+/*
+ * pre-SubByte transform
+ *
+ * pre-lookup for sbox1, sbox2, sbox3:
+ *   swap_bitendianness(
+ *       isom_map_camellia_to_aes(
+ *           camellia_f(
+ *               swap_bitendianess(in)
+ *           )
+ *       )
+ *   )
+ *
+ * (note: '⊕ 0xc5' inside camellia_f())
+ */
+.Lpre_tf_lo_s1:
+        .byte 0x45, 0xe8, 0x40, 0xed, 0x2e, 0x83, 0x2b, 0x86
+        .byte 0x4b, 0xe6, 0x4e, 0xe3, 0x20, 0x8d, 0x25, 0x88
+.Lpre_tf_hi_s1:
+        .byte 0x00, 0x51, 0xf1, 0xa0, 0x8a, 0xdb, 0x7b, 0x2a
+        .byte 0x09, 0x58, 0xf8, 0xa9, 0x83, 0xd2, 0x72, 0x23
+/*
+ * pre-SubByte transform
+ *
+ * pre-lookup for sbox4:
+ *   swap_bitendianness(
+ *       isom_map_camellia_to_aes(
+ *           camellia_f(
+ *               swap_bitendianess(in <<< 1)
+ *           )
+ *       )
+ *   )
+ *
+ * (note: '⊕ 0xc5' inside camellia_f())
+ */
+.Lpre_tf_lo_s4:
+        .byte 0x45, 0x40, 0x2e, 0x2b, 0x4b, 0x4e, 0x20, 0x25
+        .byte 0x14, 0x11, 0x7f, 0x7a, 0x1a, 0x1f, 0x71, 0x74
+.Lpre_tf_hi_s4:
+        .byte 0x00, 0xf1, 0x8a, 0x7b, 0x09, 0xf8, 0x83, 0x72
+        .byte 0xad, 0x5c, 0x27, 0xd6, 0xa4, 0x55, 0x2e, 0xdf
+/*
+ * post-SubByte transform
+ *
+ * post-lookup for sbox1, sbox4:
+ *  swap_bitendianness(
+ *      camellia_h(
+ *          isom_map_aes_to_camellia(
+ *              swap_bitendianness(
+ *                  aes_inverse_affine_transform(in)
+ *              )
+ *          )
+ *      )
+ *  )
+ *
+ * (note: '⊕ 0x6e' inside camellia_h())
+ */
+.Lpost_tf_lo_s1:
+        .byte 0x3c, 0xcc, 0xcf, 0x3f, 0x32, 0xc2, 0xc1, 0x31
+        .byte 0xdc, 0x2c, 0x2f, 0xdf, 0xd2, 0x22, 0x21, 0xd1
+.Lpost_tf_hi_s1:
+        .byte 0x00, 0xf9, 0x86, 0x7f, 0xd7, 0x2e, 0x51, 0xa8
+        .byte 0xa4, 0x5d, 0x22, 0xdb, 0x73, 0x8a, 0xf5, 0x0c
+/*
+ * post-SubByte transform
+ *
+ * post-lookup for sbox2:
+ *  swap_bitendianness(
+ *      camellia_h(
+ *          isom_map_aes_to_camellia(
+ *              swap_bitendianness(
+ *                  aes_inverse_affine_transform(in)
+ *              )
+ *          )
+ *      )
+ *  ) <<< 1
+ *
+ * (note: '⊕ 0x6e' inside camellia_h())
+ */
+.Lpost_tf_lo_s2:
+        .byte 0x78, 0x99, 0x9f, 0x7e, 0x64, 0x85, 0x83, 0x62
+        .byte 0xb9, 0x58, 0x5e, 0xbf, 0xa5, 0x44, 0x42, 0xa3
+.Lpost_tf_hi_s2:
+        .byte 0x00, 0xf3, 0x0d, 0xfe, 0xaf, 0x5c, 0xa2, 0x51
+        .byte 0x49, 0xba, 0x44, 0xb7, 0xe6, 0x15, 0xeb, 0x18
+/*
+ * post-SubByte transform
+ *
+ * post-lookup for sbox3:
+ *  swap_bitendianness(
+ *      camellia_h(
+ *          isom_map_aes_to_camellia(
+ *              swap_bitendianness(
+ *                  aes_inverse_affine_transform(in)
+ *              )
+ *          )
+ *      )
+ *  ) >>> 1
+ *
+ * (note: '⊕ 0x6e' inside camellia_h())
+ */
+.Lpost_tf_lo_s3:
+        .byte 0x1e, 0x66, 0xe7, 0x9f, 0x19, 0x61, 0xe0, 0x98
+        .byte 0x6e, 0x16, 0x97, 0xef, 0x69, 0x11, 0x90, 0xe8
+.Lpost_tf_hi_s3:
+        .byte 0x00, 0xfc, 0x43, 0xbf, 0xeb, 0x17, 0xa8, 0x54
+        .byte 0x52, 0xae, 0x11, 0xed, 0xb9, 0x45, 0xfa, 0x06
+/* For isolating SubBytes from AESENCLAST, inverse shift row */
+.Linv_shift_row:
+        .byte 0x00, 0x0d, 0x0a, 0x07, 0x04, 0x01, 0x0e, 0x0b
+        .byte 0x08, 0x05, 0x02, 0x0f, 0x0c, 0x09, 0x06, 0x03
+/* 4-bit mask */
+.align 4
+.L0f0f0f0f:
+        .long 0x0f0f0f0f
+.text
+.align 8
+.type   __camellia_enc_blk16,@function;
+__camellia_enc_blk16:
+        /* input:
+         *      %rdi: ctx, CTX
+         *      %rax: temporary storage, 256 bytes
+         *      %xmm0..%xmm15: 16 plaintext blocks
+         * output:
+         *      %xmm0..%xmm15: 16 encrypted blocks, order swapped:
+         *       7, 8, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8
+         */
+        leaq 8 * 16(%rax), %rcx;
+        inpack16_post(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
+                      %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
+                      %xmm15, %rax, %rcx);
+        enc_rounds16(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
+                     %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
+                     %xmm15, %rax, %rcx, 0);
+        fls16(%rax, %xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
+              %rcx, %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
+              %xmm15,
+              ((key_table + (8) * 8) + 0)(CTX),
+              ((key_table + (8) * 8) + 4)(CTX),
+              ((key_table + (8) * 8) + 8)(CTX),
+              ((key_table + (8) * 8) + 12)(CTX));
+        enc_rounds16(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
+                     %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
+                     %xmm15, %rax, %rcx, 8);
+        fls16(%rax, %xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
+              %rcx, %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
+              %xmm15,
+              ((key_table + (16) * 8) + 0)(CTX),
+              ((key_table + (16) * 8) + 4)(CTX),
+              ((key_table + (16) * 8) + 8)(CTX),
+              ((key_table + (16) * 8) + 12)(CTX));
+        enc_rounds16(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
+                     %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
+                     %xmm15, %rax, %rcx, 16);
+        movl $24, %r8d;
+        cmpl $16, key_length(CTX);
+        jne .Lenc_max32;
+.Lenc_done:
+        /* load CD for output */
+        vmovdqu 0 * 16(%rcx), %xmm8;
+        vmovdqu 1 * 16(%rcx), %xmm9;
+        vmovdqu 2 * 16(%rcx), %xmm10;
+        vmovdqu 3 * 16(%rcx), %xmm11;
+        vmovdqu 4 * 16(%rcx), %xmm12;
+        vmovdqu 5 * 16(%rcx), %xmm13;
+        vmovdqu 6 * 16(%rcx), %xmm14;
+        vmovdqu 7 * 16(%rcx), %xmm15;
+        outunpack16(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
+                    %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
+                    %xmm15, (key_table)(CTX, %r8, 8), (%rax), 1 * 16(%rax));
+        ret;
+.align 8
+.Lenc_max32:
+        movl $32, %r8d;
+        fls16(%rax, %xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
+              %rcx, %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
+              %xmm15,
+              ((key_table + (24) * 8) + 0)(CTX),
+              ((key_table + (24) * 8) + 4)(CTX),
+              ((key_table + (24) * 8) + 8)(CTX),
+              ((key_table + (24) * 8) + 12)(CTX));
+        enc_rounds16(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
+                     %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
+                     %xmm15, %rax, %rcx, 24);
+        jmp .Lenc_done;
+.align 8
+.type   __camellia_dec_blk16,@function;
+__camellia_dec_blk16:
+        /* input:
+         *      %rdi: ctx, CTX
+         *      %rax: temporary storage, 256 bytes
+         *      %r8d: 24 for 16 byte key, 32 for larger
+         *      %xmm0..%xmm15: 16 encrypted blocks
+         * output:
+         *      %xmm0..%xmm15: 16 plaintext blocks, order swapped:
+         *       7, 8, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8
+         */
+        leaq 8 * 16(%rax), %rcx;
+        inpack16_post(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
+                      %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
+                      %xmm15, %rax, %rcx);
+        cmpl $32, %r8d;
+        je .Ldec_max32;
+.Ldec_max24:
+        dec_rounds16(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
+                     %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
+                     %xmm15, %rax, %rcx, 16);
+        fls16(%rax, %xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
+              %rcx, %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
+              %xmm15,
+              ((key_table + (16) * 8) + 8)(CTX),
+              ((key_table + (16) * 8) + 12)(CTX),
+              ((key_table + (16) * 8) + 0)(CTX),
+              ((key_table + (16) * 8) + 4)(CTX));
+        dec_rounds16(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
+                     %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
+                     %xmm15, %rax, %rcx, 8);
+        fls16(%rax, %xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
+              %rcx, %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
+              %xmm15,
+              ((key_table + (8) * 8) + 8)(CTX),
+              ((key_table + (8) * 8) + 12)(CTX),
+              ((key_table + (8) * 8) + 0)(CTX),
+              ((key_table + (8) * 8) + 4)(CTX));
+        dec_rounds16(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
+                     %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
+                     %xmm15, %rax, %rcx, 0);
+        /* load CD for output */
+        vmovdqu 0 * 16(%rcx), %xmm8;
+        vmovdqu 1 * 16(%rcx), %xmm9;
+        vmovdqu 2 * 16(%rcx), %xmm10;
+        vmovdqu 3 * 16(%rcx), %xmm11;
+        vmovdqu 4 * 16(%rcx), %xmm12;
+        vmovdqu 5 * 16(%rcx), %xmm13;
+        vmovdqu 6 * 16(%rcx), %xmm14;
+        vmovdqu 7 * 16(%rcx), %xmm15;
+        outunpack16(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
+                    %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
+                    %xmm15, (key_table)(CTX), (%rax), 1 * 16(%rax));
+        ret;
+.align 8
+.Ldec_max32:
+        dec_rounds16(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
+                     %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
+                     %xmm15, %rax, %rcx, 24);
+        fls16(%rax, %xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
+              %rcx, %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
+              %xmm15,
+              ((key_table + (24) * 8) + 8)(CTX),
+              ((key_table + (24) * 8) + 12)(CTX),
+              ((key_table + (24) * 8) + 0)(CTX),
+              ((key_table + (24) * 8) + 4)(CTX));
+        jmp .Ldec_max24;
+.align 8
+.global camellia_ecb_enc_16way
+.type   camellia_ecb_enc_16way,@function;
+camellia_ecb_enc_16way:
+        /* input:
+         *      %rdi: ctx, CTX
+         *      %rsi: dst (16 blocks)
+         *      %rdx: src (16 blocks)
+         */
+        inpack16_pre(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
+                     %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
+                     %xmm15, %rdx, (key_table)(CTX));
+        /* now dst can be used as temporary buffer (even in src == dst case) */
+        movq    %rsi, %rax;
+        call __camellia_enc_blk16;
+        write_output(%xmm7, %xmm6, %xmm5, %xmm4, %xmm3, %xmm2, %xmm1, %xmm0,
+                     %xmm15, %xmm14, %xmm13, %xmm12, %xmm11, %xmm10, %xmm9,
+                     %xmm8, %rsi);
+        ret;
+.align 8
+.global camellia_ecb_dec_16way
+.type   camellia_ecb_dec_16way,@function;
+camellia_ecb_dec_16way:
+        /* input:
+         *      %rdi: ctx, CTX
+         *      %rsi: dst (16 blocks)
+         *      %rdx: src (16 blocks)
+         */
+        cmpl $16, key_length(CTX);
+        movl $32, %r8d;
+        movl $24, %eax;
+        cmovel %eax, %r8d; /* max */
+        inpack16_pre(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
+                     %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
+                     %xmm15, %rdx, (key_table)(CTX, %r8, 8));
+        /* now dst can be used as temporary buffer (even in src == dst case) */
+        movq    %rsi, %rax;
+        call __camellia_dec_blk16;
+        write_output(%xmm7, %xmm6, %xmm5, %xmm4, %xmm3, %xmm2, %xmm1, %xmm0,
+                     %xmm15, %xmm14, %xmm13, %xmm12, %xmm11, %xmm10, %xmm9,
+                     %xmm8, %rsi);
+        ret;
+.align 8
+.global camellia_cbc_dec_16way
+.type   camellia_cbc_dec_16way,@function;
+camellia_cbc_dec_16way:
+        /* input:
+         *      %rdi: ctx, CTX
+         *      %rsi: dst (16 blocks)
+         *      %rdx: src (16 blocks)
+         */
+        cmpl $16, key_length(CTX);
+        movl $32, %r8d;
+        movl $24, %eax;
+        cmovel %eax, %r8d; /* max */
+        inpack16_pre(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
+                     %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
+                     %xmm15, %rdx, (key_table)(CTX, %r8, 8));
+        /*
+         * dst might still be in-use (in case dst == src), so use stack for
+         * temporary storage.
+         */
+        subq $(16 * 16), %rsp;
+        movq %rsp, %rax;
+        call __camellia_dec_blk16;
+        addq $(16 * 16), %rsp;
+        vpxor (0 * 16)(%rdx), %xmm6, %xmm6;
+        vpxor (1 * 16)(%rdx), %xmm5, %xmm5;
+        vpxor (2 * 16)(%rdx), %xmm4, %xmm4;
+        vpxor (3 * 16)(%rdx), %xmm3, %xmm3;
+        vpxor (4 * 16)(%rdx), %xmm2, %xmm2;
+        vpxor (5 * 16)(%rdx), %xmm1, %xmm1;
+        vpxor (6 * 16)(%rdx), %xmm0, %xmm0;
+        vpxor (7 * 16)(%rdx), %xmm15, %xmm15;
+        vpxor (8 * 16)(%rdx), %xmm14, %xmm14;
+        vpxor (9 * 16)(%rdx), %xmm13, %xmm13;
+        vpxor (10 * 16)(%rdx), %xmm12, %xmm12;
+        vpxor (11 * 16)(%rdx), %xmm11, %xmm11;
+        vpxor (12 * 16)(%rdx), %xmm10, %xmm10;
+        vpxor (13 * 16)(%rdx), %xmm9, %xmm9;
+        vpxor (14 * 16)(%rdx), %xmm8, %xmm8;
+        write_output(%xmm7, %xmm6, %xmm5, %xmm4, %xmm3, %xmm2, %xmm1, %xmm0,
+                     %xmm15, %xmm14, %xmm13, %xmm12, %xmm11, %xmm10, %xmm9,
+                     %xmm8, %rsi);
+        ret;
+#define inc_le128(x, minus_one, tmp) \
+        vpcmpeqq minus_one, x, tmp; \
+        vpsubq minus_one, x, x; \
+        vpslldq $8, tmp, tmp; \
+        vpsubq tmp, x, x;
+.align 8
+.global camellia_ctr_16way
+.type   camellia_ctr_16way,@function;
+camellia_ctr_16way:
+        /* input:
+         *      %rdi: ctx, CTX
+         *      %rsi: dst (16 blocks)
+         *      %rdx: src (16 blocks)
+         *      %rcx: iv (little endian, 128bit)
+         */
+        subq $(16 * 16), %rsp;
+        movq %rsp, %rax;
+        vmovdqa .Lbswap128_mask, %xmm14;
+        /* load IV and byteswap */
+        vmovdqu (%rcx), %xmm0;
+        vpshufb %xmm14, %xmm0, %xmm15;
+        vmovdqu %xmm15, 15 * 16(%rax);
+        vpcmpeqd %xmm15, %xmm15, %xmm15;
+        vpsrldq $8, %xmm15, %xmm15; /* low: -1, high: 0 */
+        /* construct IVs */
+        inc_le128(%xmm0, %xmm15, %xmm13);
+        vpshufb %xmm14, %xmm0, %xmm13;
+        vmovdqu %xmm13, 14 * 16(%rax);
+        inc_le128(%xmm0, %xmm15, %xmm13);
+        vpshufb %xmm14, %xmm0, %xmm13;
+        vmovdqu %xmm13, 13 * 16(%rax);
+        inc_le128(%xmm0, %xmm15, %xmm13);
+        vpshufb %xmm14, %xmm0, %xmm12;
+        inc_le128(%xmm0, %xmm15, %xmm13);
+        vpshufb %xmm14, %xmm0, %xmm11;
+        inc_le128(%xmm0, %xmm15, %xmm13);
+        vpshufb %xmm14, %xmm0, %xmm10;
+        inc_le128(%xmm0, %xmm15, %xmm13);
+        vpshufb %xmm14, %xmm0, %xmm9;
+        inc_le128(%xmm0, %xmm15, %xmm13);
+        vpshufb %xmm14, %xmm0, %xmm8;
+        inc_le128(%xmm0, %xmm15, %xmm13);
+        vpshufb %xmm14, %xmm0, %xmm7;
+        inc_le128(%xmm0, %xmm15, %xmm13);
+        vpshufb %xmm14, %xmm0, %xmm6;
+        inc_le128(%xmm0, %xmm15, %xmm13);
+        vpshufb %xmm14, %xmm0, %xmm5;
+        inc_le128(%xmm0, %xmm15, %xmm13);
+        vpshufb %xmm14, %xmm0, %xmm4;
+        inc_le128(%xmm0, %xmm15, %xmm13);
+        vpshufb %xmm14, %xmm0, %xmm3;
+        inc_le128(%xmm0, %xmm15, %xmm13);
+        vpshufb %xmm14, %xmm0, %xmm2;
+        inc_le128(%xmm0, %xmm15, %xmm13);
+        vpshufb %xmm14, %xmm0, %xmm1;
+        inc_le128(%xmm0, %xmm15, %xmm13);
+        vmovdqa %xmm0, %xmm13;
+        vpshufb %xmm14, %xmm0, %xmm0;
+        inc_le128(%xmm13, %xmm15, %xmm14);
+        vmovdqu %xmm13, (%rcx);
+        /* inpack16_pre: */
+        vmovq (key_table)(CTX), %xmm15;
+        vpshufb .Lpack_bswap, %xmm15, %xmm15;
+        vpxor %xmm0, %xmm15, %xmm0;
+        vpxor %xmm1, %xmm15, %xmm1;
+        vpxor %xmm2, %xmm15, %xmm2;
+        vpxor %xmm3, %xmm15, %xmm3;
+        vpxor %xmm4, %xmm15, %xmm4;
+        vpxor %xmm5, %xmm15, %xmm5;
+        vpxor %xmm6, %xmm15, %xmm6;
+        vpxor %xmm7, %xmm15, %xmm7;
+        vpxor %xmm8, %xmm15, %xmm8;
+        vpxor %xmm9, %xmm15, %xmm9;
+        vpxor %xmm10, %xmm15, %xmm10;
+        vpxor %xmm11, %xmm15, %xmm11;
+        vpxor %xmm12, %xmm15, %xmm12;
+        vpxor 13 * 16(%rax), %xmm15, %xmm13;
+        vpxor 14 * 16(%rax), %xmm15, %xmm14;
+        vpxor 15 * 16(%rax), %xmm15, %xmm15;
+        call __camellia_enc_blk16;
+        addq $(16 * 16), %rsp;
+        vpxor 0 * 16(%rdx), %xmm7, %xmm7;
+        vpxor 1 * 16(%rdx), %xmm6, %xmm6;
+        vpxor 2 * 16(%rdx), %xmm5, %xmm5;
+        vpxor 3 * 16(%rdx), %xmm4, %xmm4;
+        vpxor 4 * 16(%rdx), %xmm3, %xmm3;
+        vpxor 5 * 16(%rdx), %xmm2, %xmm2;
+        vpxor 6 * 16(%rdx), %xmm1, %xmm1;
+        vpxor 7 * 16(%rdx), %xmm0, %xmm0;
+        vpxor 8 * 16(%rdx), %xmm15, %xmm15;
+        vpxor 9 * 16(%rdx), %xmm14, %xmm14;
+        vpxor 10 * 16(%rdx), %xmm13, %xmm13;
+        vpxor 11 * 16(%rdx), %xmm12, %xmm12;
+        vpxor 12 * 16(%rdx), %xmm11, %xmm11;
+        vpxor 13 * 16(%rdx), %xmm10, %xmm10;
+        vpxor 14 * 16(%rdx), %xmm9, %xmm9;
+        vpxor 15 * 16(%rdx), %xmm8, %xmm8;
+        write_output(%xmm7, %xmm6, %xmm5, %xmm4, %xmm3, %xmm2, %xmm1, %xmm0,
+                     %xmm15, %xmm14, %xmm13, %xmm12, %xmm11, %xmm10, %xmm9,
+                     %xmm8, %rsi);
+        ret;
diff --git a/arch/x86/crypto/camellia_aesni_avx_glue.c b/arch/x86/crypto/camellia_aesni_avx_glue.c
new file mode 100644
index 000000000000..96cbb6068fce
--- /dev/null
+++ b/arch/x86/crypto/camellia_aesni_avx_glue.c
@@ -0,0 +1,558 @@
+/*
+ * Glue Code for x86_64/AVX/AES-NI assembler optimized version of Camellia
+ *
+ * Copyright © 2012 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ */
+#include <linux/module.h>
+#include <linux/types.h>
+#include <linux/crypto.h>
+#include <linux/err.h>
+#include <crypto/algapi.h>
+#include <crypto/ctr.h>
+#include <crypto/lrw.h>
+#include <crypto/xts.h>
+#include <asm/xcr.h>
+#include <asm/xsave.h>
+#include <asm/crypto/camellia.h>
+#include <asm/crypto/ablk_helper.h>
+#include <asm/crypto/glue_helper.h>
+#define CAMELLIA_AESNI_PARALLEL_BLOCKS 16
+/* 16-way AES-NI parallel cipher functions */
+asmlinkage void camellia_ecb_enc_16way(struct camellia_ctx *ctx, u8 *dst,
+                                       const u8 *src);
+asmlinkage void camellia_ecb_dec_16way(struct camellia_ctx *ctx, u8 *dst,
+                                       const u8 *src);
+asmlinkage void camellia_cbc_dec_16way(struct camellia_ctx *ctx, u8 *dst,
+                                       const u8 *src);
+asmlinkage void camellia_ctr_16way(struct camellia_ctx *ctx, u8 *dst,
+                                   const u8 *src, le128 *iv);
+static const struct common_glue_ctx camellia_enc = {
+        .num_funcs = 3,
+        .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
+        .funcs = { {
+                .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
+                .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_ecb_enc_16way) }
+        }, {
+                .num_blocks = 2,
+                .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_enc_blk_2way) }
+        }, {
+                .num_blocks = 1,
+                .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_enc_blk) }
+        } }
+};
+static const struct common_glue_ctx camellia_ctr = {
+        .num_funcs = 3,
+        .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
+        .funcs = { {
+                .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
+                .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_ctr_16way) }
+        }, {
+                .num_blocks = 2,
+                .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_crypt_ctr_2way) }
+        }, {
+                .num_blocks = 1,
+                .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_crypt_ctr) }
+        } }
+};
+static const struct common_glue_ctx camellia_dec = {
+        .num_funcs = 3,
+        .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
+        .funcs = { {
+                .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
+                .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_ecb_dec_16way) }
+        }, {
+                .num_blocks = 2,
+                .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_dec_blk_2way) }
+        }, {
+                .num_blocks = 1,
+                .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_dec_blk) }
+        } }
+};
+static const struct common_glue_ctx camellia_dec_cbc = {
+        .num_funcs = 3,
+        .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
+        .funcs = { {
+                .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
+                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_cbc_dec_16way) }
+        }, {
+                .num_blocks = 2,
+                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_decrypt_cbc_2way) }
+        }, {
+                .num_blocks = 1,
+                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_dec_blk) }
+        } }
+};
+static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+                       struct scatterlist *src, unsigned int nbytes)
+{
+        return glue_ecb_crypt_128bit(&camellia_enc, desc, dst, src, nbytes);
+}
+static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+                       struct scatterlist *src, unsigned int nbytes)
+{
+        return glue_ecb_crypt_128bit(&camellia_dec, desc, dst, src, nbytes);
+}
+static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+                       struct scatterlist *src, unsigned int nbytes)
+{
+        return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(camellia_enc_blk), desc,
+                                       dst, src, nbytes);
+}
+static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+                       struct scatterlist *src, unsigned int nbytes)
+{
+        return glue_cbc_decrypt_128bit(&camellia_dec_cbc, desc, dst, src,
+                                       nbytes);
+}
+static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+                     struct scatterlist *src, unsigned int nbytes)
+{
+        return glue_ctr_crypt_128bit(&camellia_ctr, desc, dst, src, nbytes);
+}
+static inline bool camellia_fpu_begin(bool fpu_enabled, unsigned int nbytes)
+{
+        return glue_fpu_begin(CAMELLIA_BLOCK_SIZE,
+                              CAMELLIA_AESNI_PARALLEL_BLOCKS, NULL, fpu_enabled,
+                              nbytes);
+}
+static inline void camellia_fpu_end(bool fpu_enabled)
+{
+        glue_fpu_end(fpu_enabled);
+}
+static int camellia_setkey(struct crypto_tfm *tfm, const u8 *in_key,
+                           unsigned int key_len)
+{
+        return __camellia_setkey(crypto_tfm_ctx(tfm), in_key, key_len,
+                                 &tfm->crt_flags);
+}
+struct crypt_priv {
+        struct camellia_ctx *ctx;
+        bool fpu_enabled;
+};
+static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
+{
+        const unsigned int bsize = CAMELLIA_BLOCK_SIZE;
+        struct crypt_priv *ctx = priv;
+        int i;
+        ctx->fpu_enabled = camellia_fpu_begin(ctx->fpu_enabled, nbytes);
+        if (nbytes >= CAMELLIA_AESNI_PARALLEL_BLOCKS * bsize) {
+                camellia_ecb_enc_16way(ctx->ctx, srcdst, srcdst);
+                srcdst += bsize * CAMELLIA_AESNI_PARALLEL_BLOCKS;
+                nbytes -= bsize * CAMELLIA_AESNI_PARALLEL_BLOCKS;
+        }
+        while (nbytes >= CAMELLIA_PARALLEL_BLOCKS * bsize) {
+                camellia_enc_blk_2way(ctx->ctx, srcdst, srcdst);
+                srcdst += bsize * CAMELLIA_PARALLEL_BLOCKS;
+                nbytes -= bsize * CAMELLIA_PARALLEL_BLOCKS;
+        }
+        for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
+                camellia_enc_blk(ctx->ctx, srcdst, srcdst);
+}
+static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
+{
+        const unsigned int bsize = CAMELLIA_BLOCK_SIZE;
+        struct crypt_priv *ctx = priv;
+        int i;
+        ctx->fpu_enabled = camellia_fpu_begin(ctx->fpu_enabled, nbytes);
+        if (nbytes >= CAMELLIA_AESNI_PARALLEL_BLOCKS * bsize) {
+                camellia_ecb_dec_16way(ctx->ctx, srcdst, srcdst);
+                srcdst += bsize * CAMELLIA_AESNI_PARALLEL_BLOCKS;
+                nbytes -= bsize * CAMELLIA_AESNI_PARALLEL_BLOCKS;
+        }
+        while (nbytes >= CAMELLIA_PARALLEL_BLOCKS * bsize) {
+                camellia_dec_blk_2way(ctx->ctx, srcdst, srcdst);
+                srcdst += bsize * CAMELLIA_PARALLEL_BLOCKS;
+                nbytes -= bsize * CAMELLIA_PARALLEL_BLOCKS;
+        }
+        for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
+                camellia_dec_blk(ctx->ctx, srcdst, srcdst);
+}
+static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+                       struct scatterlist *src, unsigned int nbytes)
+{
+        struct camellia_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
+        be128 buf[CAMELLIA_AESNI_PARALLEL_BLOCKS];
+        struct crypt_priv crypt_ctx = {
+                .ctx = &ctx->camellia_ctx,
+                .fpu_enabled = false,
+        };
+        struct lrw_crypt_req req = {
+                .tbuf = buf,
+                .tbuflen = sizeof(buf),
+                .table_ctx = &ctx->lrw_table,
+                .crypt_ctx = &crypt_ctx,
+                .crypt_fn = encrypt_callback,
+        };
+        int ret;
+        desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
+        ret = lrw_crypt(desc, dst, src, nbytes, &req);
+        camellia_fpu_end(crypt_ctx.fpu_enabled);
+        return ret;
+}
+static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+                       struct scatterlist *src, unsigned int nbytes)
+{
+        struct camellia_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
+        be128 buf[CAMELLIA_AESNI_PARALLEL_BLOCKS];
+        struct crypt_priv crypt_ctx = {
+                .ctx = &ctx->camellia_ctx,
+                .fpu_enabled = false,
+        };
+        struct lrw_crypt_req req = {
+                .tbuf = buf,
+                .tbuflen = sizeof(buf),
+                .table_ctx = &ctx->lrw_table,
+                .crypt_ctx = &crypt_ctx,
+                .crypt_fn = decrypt_callback,
+        };
+        int ret;
+        desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
+        ret = lrw_crypt(desc, dst, src, nbytes, &req);
+        camellia_fpu_end(crypt_ctx.fpu_enabled);
+        return ret;
+}
+static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+                       struct scatterlist *src, unsigned int nbytes)
+{
+        struct camellia_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
+        be128 buf[CAMELLIA_AESNI_PARALLEL_BLOCKS];
+        struct crypt_priv crypt_ctx = {
+                .ctx = &ctx->crypt_ctx,
+                .fpu_enabled = false,
+        };
+        struct xts_crypt_req req = {
+                .tbuf = buf,
+                .tbuflen = sizeof(buf),
+                .tweak_ctx = &ctx->tweak_ctx,
+                .tweak_fn = XTS_TWEAK_CAST(camellia_enc_blk),
+                .crypt_ctx = &crypt_ctx,
+                .crypt_fn = encrypt_callback,
+        };
+        int ret;
+        desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
+        ret = xts_crypt(desc, dst, src, nbytes, &req);
+        camellia_fpu_end(crypt_ctx.fpu_enabled);
+        return ret;
+}
+static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+                       struct scatterlist *src, unsigned int nbytes)
+{
+        struct camellia_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
+        be128 buf[CAMELLIA_AESNI_PARALLEL_BLOCKS];
+        struct crypt_priv crypt_ctx = {
+                .ctx = &ctx->crypt_ctx,
+                .fpu_enabled = false,
+        };
+        struct xts_crypt_req req = {
+                .tbuf = buf,
+                .tbuflen = sizeof(buf),
+                .tweak_ctx = &ctx->tweak_ctx,
+                .tweak_fn = XTS_TWEAK_CAST(camellia_enc_blk),
+                .crypt_ctx = &crypt_ctx,
+                .crypt_fn = decrypt_callback,
+        };
+        int ret;
+        desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
+        ret = xts_crypt(desc, dst, src, nbytes, &req);
+        camellia_fpu_end(crypt_ctx.fpu_enabled);
+        return ret;
+}
+static struct crypto_alg cmll_algs[10] = { {
+        .cra_name               = "__ecb-camellia-aesni",
+        .cra_driver_name        = "__driver-ecb-camellia-aesni",
+        .cra_priority           = 0,
+        .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
+        .cra_blocksize          = CAMELLIA_BLOCK_SIZE,
+        .cra_ctxsize            = sizeof(struct camellia_ctx),
+        .cra_alignmask          = 0,
+        .cra_type               = &crypto_blkcipher_type,
+        .cra_module             = THIS_MODULE,
+        .cra_u = {
+                .blkcipher = {
+                        .min_keysize    = CAMELLIA_MIN_KEY_SIZE,
+                        .max_keysize    = CAMELLIA_MAX_KEY_SIZE,
+                        .setkey         = camellia_setkey,
+                        .encrypt        = ecb_encrypt,
+                        .decrypt        = ecb_decrypt,
+                },
+        },
+}, {
+        .cra_name               = "__cbc-camellia-aesni",
+        .cra_driver_name        = "__driver-cbc-camellia-aesni",
+        .cra_priority           = 0,
+        .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
+        .cra_blocksize          = CAMELLIA_BLOCK_SIZE,
+        .cra_ctxsize            = sizeof(struct camellia_ctx),
+        .cra_alignmask          = 0,
+        .cra_type               = &crypto_blkcipher_type,
+        .cra_module             = THIS_MODULE,
+        .cra_u = {
+                .blkcipher = {
+                        .min_keysize    = CAMELLIA_MIN_KEY_SIZE,
+                        .max_keysize    = CAMELLIA_MAX_KEY_SIZE,
+                        .setkey         = camellia_setkey,
+                        .encrypt        = cbc_encrypt,
+                        .decrypt        = cbc_decrypt,
+                },
+        },
+}, {
+        .cra_name               = "__ctr-camellia-aesni",
+        .cra_driver_name        = "__driver-ctr-camellia-aesni",
+        .cra_priority           = 0,
+        .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
+        .cra_blocksize          = 1,
+        .cra_ctxsize            = sizeof(struct camellia_ctx),
+        .cra_alignmask          = 0,
+        .cra_type               = &crypto_blkcipher_type,
+        .cra_module             = THIS_MODULE,
+        .cra_u = {
+                .blkcipher = {
+                        .min_keysize    = CAMELLIA_MIN_KEY_SIZE,
+                        .max_keysize    = CAMELLIA_MAX_KEY_SIZE,
+                        .ivsize         = CAMELLIA_BLOCK_SIZE,
+                        .setkey         = camellia_setkey,
+                        .encrypt        = ctr_crypt,
+                        .decrypt        = ctr_crypt,
+                },
+        },
+}, {
+        .cra_name               = "__lrw-camellia-aesni",
+        .cra_driver_name        = "__driver-lrw-camellia-aesni",
+        .cra_priority           = 0,
+        .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
+        .cra_blocksize          = CAMELLIA_BLOCK_SIZE,
+        .cra_ctxsize            = sizeof(struct camellia_lrw_ctx),
+        .cra_alignmask          = 0,
+        .cra_type               = &crypto_blkcipher_type,
+        .cra_module             = THIS_MODULE,
+        .cra_exit               = lrw_camellia_exit_tfm,
+        .cra_u = {
+                .blkcipher = {
+                        .min_keysize    = CAMELLIA_MIN_KEY_SIZE +
+                                          CAMELLIA_BLOCK_SIZE,
+                        .max_keysize    = CAMELLIA_MAX_KEY_SIZE +
+                                          CAMELLIA_BLOCK_SIZE,
+                        .ivsize         = CAMELLIA_BLOCK_SIZE,
+                        .setkey         = lrw_camellia_setkey,
+                        .encrypt        = lrw_encrypt,
+                        .decrypt        = lrw_decrypt,
+                },
+        },
+}, {
+        .cra_name               = "__xts-camellia-aesni",
+        .cra_driver_name        = "__driver-xts-camellia-aesni",
+        .cra_priority           = 0,
+        .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
+        .cra_blocksize          = CAMELLIA_BLOCK_SIZE,
+        .cra_ctxsize            = sizeof(struct camellia_xts_ctx),
+        .cra_alignmask          = 0,
+        .cra_type               = &crypto_blkcipher_type,
+        .cra_module             = THIS_MODULE,
+        .cra_u = {
+                .blkcipher = {
+                        .min_keysize    = CAMELLIA_MIN_KEY_SIZE * 2,
+                        .max_keysize    = CAMELLIA_MAX_KEY_SIZE * 2,
+                        .ivsize         = CAMELLIA_BLOCK_SIZE,
+                        .setkey         = xts_camellia_setkey,
+                        .encrypt        = xts_encrypt,
+                        .decrypt        = xts_decrypt,
+                },
+        },
+}, {
+        .cra_name               = "ecb(camellia)",
+        .cra_driver_name        = "ecb-camellia-aesni",
+        .cra_priority           = 400,
+        .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
+        .cra_blocksize          = CAMELLIA_BLOCK_SIZE,
+        .cra_ctxsize            = sizeof(struct async_helper_ctx),
+        .cra_alignmask          = 0,
+        .cra_type               = &crypto_ablkcipher_type,
+        .cra_module             = THIS_MODULE,
+        .cra_init               = ablk_init,
+        .cra_exit               = ablk_exit,
+        .cra_u = {
+                .ablkcipher = {
+                        .min_keysize    = CAMELLIA_MIN_KEY_SIZE,
+                        .max_keysize    = CAMELLIA_MAX_KEY_SIZE,
+                        .setkey         = ablk_set_key,
+                        .encrypt        = ablk_encrypt,
+                        .decrypt        = ablk_decrypt,
+                },
+        },
+}, {
+        .cra_name               = "cbc(camellia)",
+        .cra_driver_name        = "cbc-camellia-aesni",
+        .cra_priority           = 400,
+        .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
+        .cra_blocksize          = CAMELLIA_BLOCK_SIZE,
+        .cra_ctxsize            = sizeof(struct async_helper_ctx),
+        .cra_alignmask          = 0,
+        .cra_type               = &crypto_ablkcipher_type,
+        .cra_module             = THIS_MODULE,
+        .cra_init               = ablk_init,
+        .cra_exit               = ablk_exit,
+        .cra_u = {
+                .ablkcipher = {
+                        .min_keysize    = CAMELLIA_MIN_KEY_SIZE,
+                        .max_keysize    = CAMELLIA_MAX_KEY_SIZE,
+                        .ivsize         = CAMELLIA_BLOCK_SIZE,
+                        .setkey         = ablk_set_key,
+                        .encrypt        = __ablk_encrypt,
+                        .decrypt        = ablk_decrypt,
+                },
+        },
+}, {
+        .cra_name               = "ctr(camellia)",
+        .cra_driver_name        = "ctr-camellia-aesni",
+        .cra_priority           = 400,
+        .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
+        .cra_blocksize          = 1,
+        .cra_ctxsize            = sizeof(struct async_helper_ctx),
+        .cra_alignmask          = 0,
+        .cra_type               = &crypto_ablkcipher_type,
+        .cra_module             = THIS_MODULE,
+        .cra_init               = ablk_init,
+        .cra_exit               = ablk_exit,
+        .cra_u = {
+                .ablkcipher = {
+                        .min_keysize    = CAMELLIA_MIN_KEY_SIZE,
+                        .max_keysize    = CAMELLIA_MAX_KEY_SIZE,
+                        .ivsize         = CAMELLIA_BLOCK_SIZE,
+                        .setkey         = ablk_set_key,
+                        .encrypt        = ablk_encrypt,
+                        .decrypt        = ablk_encrypt,
+                        .geniv          = "chainiv",
+                },
+        },
+}, {
+        .cra_name               = "lrw(camellia)",
+        .cra_driver_name        = "lrw-camellia-aesni",
+        .cra_priority           = 400,
+        .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
+        .cra_blocksize          = CAMELLIA_BLOCK_SIZE,
+        .cra_ctxsize            = sizeof(struct async_helper_ctx),
+        .cra_alignmask          = 0,
+        .cra_type               = &crypto_ablkcipher_type,
+        .cra_module             = THIS_MODULE,
+        .cra_init               = ablk_init,
+        .cra_exit               = ablk_exit,
+        .cra_u = {
+                .ablkcipher = {
+                        .min_keysize    = CAMELLIA_MIN_KEY_SIZE +
+                                          CAMELLIA_BLOCK_SIZE,
+                        .max_keysize    = CAMELLIA_MAX_KEY_SIZE +
+                                          CAMELLIA_BLOCK_SIZE,
+                        .ivsize         = CAMELLIA_BLOCK_SIZE,
+                        .setkey         = ablk_set_key,
+                        .encrypt        = ablk_encrypt,
+                        .decrypt        = ablk_decrypt,
+                },
+        },
+}, {
+        .cra_name               = "xts(camellia)",
+        .cra_driver_name        = "xts-camellia-aesni",
+        .cra_priority           = 400,
+        .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
+        .cra_blocksize          = CAMELLIA_BLOCK_SIZE,
+        .cra_ctxsize            = sizeof(struct async_helper_ctx),
+        .cra_alignmask          = 0,
+        .cra_type               = &crypto_ablkcipher_type,
+        .cra_module             = THIS_MODULE,
+        .cra_init               = ablk_init,
+        .cra_exit               = ablk_exit,
+        .cra_u = {
+                .ablkcipher = {
+                        .min_keysize    = CAMELLIA_MIN_KEY_SIZE * 2,
+                        .max_keysize    = CAMELLIA_MAX_KEY_SIZE * 2,
+                        .ivsize         = CAMELLIA_BLOCK_SIZE,
+                        .setkey         = ablk_set_key,
+                        .encrypt        = ablk_encrypt,
+                        .decrypt        = ablk_decrypt,
+                },
+        },
+} };
+static int __init camellia_aesni_init(void)
+{
+        u64 xcr0;
+        if (!cpu_has_avx || !cpu_has_aes || !cpu_has_osxsave) {
+                pr_info("AVX or AES-NI instructions are not detected.\n");
+                return -ENODEV;
+        }
+        xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
+        if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) {
+                pr_info("AVX detected but unusable.\n");
+                return -ENODEV;
+        }
+        return crypto_register_algs(cmll_algs, ARRAY_SIZE(cmll_algs));
+}
+static void __exit camellia_aesni_fini(void)
+{
+        crypto_unregister_algs(cmll_algs, ARRAY_SIZE(cmll_algs));
+}
+module_init(camellia_aesni_init);
+module_exit(camellia_aesni_fini);
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("Camellia Cipher Algorithm, AES-NI/AVX optimized");
+MODULE_ALIAS("camellia");
+MODULE_ALIAS("camellia-asm");
diff --git a/arch/x86/crypto/camellia_glue.c b/arch/x86/crypto/camellia_glue.c
index 42ffd2bbab5b..5cb86ccd4acb 100644
--- a/arch/x86/crypto/camellia_glue.c
+++ b/arch/x86/crypto/camellia_glue.c
@@ -32,53 +32,24 @@
 #include <crypto/algapi.h>
 #include <crypto/lrw.h>
 #include <crypto/xts.h>
+#include <asm/crypto/camellia.h>
 #include <asm/crypto/glue_helper.h>
-#define CAMELLIA_MIN_KEY_SIZE   16
-#define CAMELLIA_MAX_KEY_SIZE   32
-#define CAMELLIA_BLOCK_SIZE     16
-#define CAMELLIA_TABLE_BYTE_LEN 272
-struct camellia_ctx {
-        u64 key_table[CAMELLIA_TABLE_BYTE_LEN / sizeof(u64)];
-        u32 key_length;
-};
 /* regular block cipher functions */
 asmlinkage void __camellia_enc_blk(struct camellia_ctx *ctx, u8 *dst,
                                   const u8 *src, bool xor);
+EXPORT_SYMBOL_GPL(__camellia_enc_blk);
 asmlinkage void camellia_dec_blk(struct camellia_ctx *ctx, u8 *dst,
                                 const u8 *src);
+EXPORT_SYMBOL_GPL(camellia_dec_blk);
 /* 2-way parallel cipher functions */
 asmlinkage void __camellia_enc_blk_2way(struct camellia_ctx *ctx, u8 *dst,
                                        const u8 *src, bool xor);
+EXPORT_SYMBOL_GPL(__camellia_enc_blk_2way);
 asmlinkage void camellia_dec_blk_2way(struct camellia_ctx *ctx, u8 *dst,
                                      const u8 *src);
+EXPORT_SYMBOL_GPL(camellia_dec_blk_2way);
-static inline void camellia_enc_blk(struct camellia_ctx *ctx, u8 *dst,
-                                    const u8 *src)
-{
-        __camellia_enc_blk(ctx, dst, src, false);
-}
-static inline void camellia_enc_blk_xor(struct camellia_ctx *ctx, u8 *dst,
-                                        const u8 *src)
-{
-        __camellia_enc_blk(ctx, dst, src, true);
-}
-static inline void camellia_enc_blk_2way(struct camellia_ctx *ctx, u8 *dst,
-                                         const u8 *src)
-{
-        __camellia_enc_blk_2way(ctx, dst, src, false);
-}
-static inline void camellia_enc_blk_xor_2way(struct camellia_ctx *ctx, u8 *dst,
-                                             const u8 *src)
-{
-        __camellia_enc_blk_2way(ctx, dst, src, true);
-}
 static void camellia_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 {
@@ -1275,9 +1246,8 @@ static void camellia_setup192(const unsigned char *key, u64 *subkey)
        camellia_setup256(kk, subkey);
 }
-static int __camellia_setkey(struct camellia_ctx *cctx,
+int __camellia_setkey(struct camellia_ctx *cctx, const unsigned char *key,
-                             const unsigned char *key,
+                      unsigned int key_len, u32 *flags)
-                             unsigned int key_len, u32 *flags)
 {
        if (key_len != 16 && key_len != 24 && key_len != 32) {
                *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
@@ -1300,6 +1270,7 @@ static int __camellia_setkey(struct camellia_ctx *cctx,
        return 0;
 }
+EXPORT_SYMBOL_GPL(__camellia_setkey);
 static int camellia_setkey(struct crypto_tfm *tfm, const u8 *in_key,
                           unsigned int key_len)
@@ -1308,7 +1279,7 @@ static int camellia_setkey(struct crypto_tfm *tfm, const u8 *in_key,
                                 &tfm->crt_flags);
 }
-static void camellia_decrypt_cbc_2way(void *ctx, u128 *dst, const u128 *src)
+void camellia_decrypt_cbc_2way(void *ctx, u128 *dst, const u128 *src)
 {
        u128 iv = *src;
@@ -1316,22 +1287,23 @@ static void camellia_decrypt_cbc_2way(void *ctx, u128 *dst, const u128 *src)
        u128_xor(&dst[1], &dst[1], &iv);
 }
+EXPORT_SYMBOL_GPL(camellia_decrypt_cbc_2way);
-static void camellia_crypt_ctr(void *ctx, u128 *dst, const u128 *src, u128 *iv)
+void camellia_crypt_ctr(void *ctx, u128 *dst, const u128 *src, le128 *iv)
 {
        be128 ctrblk;
        if (dst != src)
                *dst = *src;
-        u128_to_be128(&ctrblk, iv);
+        le128_to_be128(&ctrblk, iv);
-        u128_inc(iv);
+        le128_inc(iv);
        camellia_enc_blk_xor(ctx, (u8 *)dst, (u8 *)&ctrblk);
 }
+EXPORT_SYMBOL_GPL(camellia_crypt_ctr);
-static void camellia_crypt_ctr_2way(void *ctx, u128 *dst, const u128 *src,
+void camellia_crypt_ctr_2way(void *ctx, u128 *dst, const u128 *src, le128 *iv)
-                                    u128 *iv)
 {
        be128 ctrblks[2];
@@ -1340,13 +1312,14 @@ static void camellia_crypt_ctr_2way(void *ctx, u128 *dst, const u128 *src,
                dst[1] = src[1];
        }
-        u128_to_be128(&ctrblks[0], iv);
+        le128_to_be128(&ctrblks[0], iv);
-        u128_inc(iv);
+        le128_inc(iv);
-        u128_to_be128(&ctrblks[1], iv);
+        le128_to_be128(&ctrblks[1], iv);
-        u128_inc(iv);
+        le128_inc(iv);
        camellia_enc_blk_xor_2way(ctx, (u8 *)dst, (u8 *)ctrblks);
 }
+EXPORT_SYMBOL_GPL(camellia_crypt_ctr_2way);
 static const struct common_glue_ctx camellia_enc = {
        .num_funcs = 2,
@@ -1464,13 +1437,8 @@ static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
                camellia_dec_blk(ctx, srcdst, srcdst);
 }
-struct camellia_lrw_ctx {
+int lrw_camellia_setkey(struct crypto_tfm *tfm, const u8 *key,
-        struct lrw_table_ctx lrw_table;
+                        unsigned int keylen)
-        struct camellia_ctx camellia_ctx;
-};
-static int lrw_camellia_setkey(struct crypto_tfm *tfm, const u8 *key,
-                              unsigned int keylen)
 {
        struct camellia_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
        int err;
@@ -1484,6 +1452,7 @@ static int lrw_camellia_setkey(struct crypto_tfm *tfm, const u8 *key,
        return lrw_init_table(&ctx->lrw_table,
                              key + keylen - CAMELLIA_BLOCK_SIZE);
 }
+EXPORT_SYMBOL_GPL(lrw_camellia_setkey);
 static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
                       struct scatterlist *src, unsigned int nbytes)
@@ -1519,20 +1488,16 @@ static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
        return lrw_crypt(desc, dst, src, nbytes, &req);
 }
-static void lrw_exit_tfm(struct crypto_tfm *tfm)
+void lrw_camellia_exit_tfm(struct crypto_tfm *tfm)
 {
        struct camellia_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
        lrw_free_table(&ctx->lrw_table);
 }
+EXPORT_SYMBOL_GPL(lrw_camellia_exit_tfm);
-struct camellia_xts_ctx {
+int xts_camellia_setkey(struct crypto_tfm *tfm, const u8 *key,
-        struct camellia_ctx tweak_ctx;
+                        unsigned int keylen)
-        struct camellia_ctx crypt_ctx;
-};
-static int xts_camellia_setkey(struct crypto_tfm *tfm, const u8 *key,
-                              unsigned int keylen)
 {
        struct camellia_xts_ctx *ctx = crypto_tfm_ctx(tfm);
        u32 *flags = &tfm->crt_flags;
@@ -1555,6 +1520,7 @@ static int xts_camellia_setkey(struct crypto_tfm *tfm, const u8 *key,
        return __camellia_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2,
                                flags);
 }
+EXPORT_SYMBOL_GPL(xts_camellia_setkey);
 static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
                       struct scatterlist *src, unsigned int nbytes)
@@ -1679,7 +1645,7 @@ static struct crypto_alg camellia_algs[6] = { {
        .cra_alignmask          = 0,
        .cra_type               = &crypto_blkcipher_type,
        .cra_module             = THIS_MODULE,
-        .cra_exit               = lrw_exit_tfm,
+        .cra_exit               = lrw_camellia_exit_tfm,
        .cra_u = {
                .blkcipher = {
                        .min_keysize    = CAMELLIA_MIN_KEY_SIZE +
diff --git a/arch/x86/crypto/cast5-avx-x86_64-asm_64.S b/arch/x86/crypto/cast5-avx-x86_64-asm_64.S
index a41a3aaba220..15b00ac7cbd3 100644
--- a/arch/x86/crypto/cast5-avx-x86_64-asm_64.S
+++ b/arch/x86/crypto/cast5-avx-x86_64-asm_64.S
@@ -25,10 +25,10 @@
 .file "cast5-avx-x86_64-asm_64.S"
-.extern cast5_s1
+.extern cast_s1
-.extern cast5_s2
+.extern cast_s2
-.extern cast5_s3
+.extern cast_s3
-.extern cast5_s4
+.extern cast_s4
 /* structure of crypto context */
 #define km      0
@@ -36,10 +36,10 @@
 #define rr      ((16*4)+16)
 /* s-boxes */
-#define s1      cast5_s1
+#define s1      cast_s1
-#define s2      cast5_s2
+#define s2      cast_s2
-#define s3      cast5_s3
+#define s3      cast_s3
-#define s4      cast5_s4
+#define s4      cast_s4
 /**********************************************************************
  16-way AVX cast5
@@ -180,31 +180,17 @@
        vpunpcklqdq             t1, t0, x0; \
        vpunpckhqdq             t1, t0, x1;
-#define inpack_blocks(in, x0, x1, t0, t1, rmask) \
+#define inpack_blocks(x0, x1, t0, t1, rmask) \
-        vmovdqu (0*4*4)(in),    x0; \
-        vmovdqu (1*4*4)(in),    x1; \
        vpshufb rmask,  x0,     x0; \
        vpshufb rmask,  x1,     x1; \
        \
        transpose_2x4(x0, x1, t0, t1)
-#define outunpack_blocks(out, x0, x1, t0, t1, rmask) \
+#define outunpack_blocks(x0, x1, t0, t1, rmask) \
        transpose_2x4(x0, x1, t0, t1) \
        \
        vpshufb rmask,  x0, x0;           \
-        vpshufb rmask,  x1, x1;           \
+        vpshufb rmask,  x1, x1;
-        vmovdqu         x0, (0*4*4)(out); \
-        vmovdqu         x1, (1*4*4)(out);
-#define outunpack_xor_blocks(out, x0, x1, t0, t1, rmask) \
-        transpose_2x4(x0, x1, t0, t1) \
-        \
-        vpshufb rmask,  x0, x0;               \
-        vpshufb rmask,  x1, x1;               \
-        vpxor           (0*4*4)(out), x0, x0; \
-        vmovdqu         x0, (0*4*4)(out);     \
-        vpxor           (1*4*4)(out), x1, x1; \
-        vmovdqu         x1, (1*4*4)(out);
 .data
@@ -213,6 +199,8 @@
        .byte 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12
 .Lbswap128_mask:
        .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
+.Lbswap_iv_mask:
+        .byte 7, 6, 5, 4, 3, 2, 1, 0, 7, 6, 5, 4, 3, 2, 1, 0
 .L16_mask:
        .byte 16, 16, 16, 16
 .L32_mask:
@@ -223,35 +211,42 @@
 .text
 .align 16
-.global __cast5_enc_blk_16way
+.type   __cast5_enc_blk16,@function;
-.type   __cast5_enc_blk_16way,@function;
-__cast5_enc_blk_16way:
+__cast5_enc_blk16:
        /* input:
         *      %rdi: ctx, CTX
-         *      %rsi: dst
+         *      RL1: blocks 1 and 2
-         *      %rdx: src
+         *      RR1: blocks 3 and 4
-         *      %rcx: bool, if true: xor output
+         *      RL2: blocks 5 and 6
+         *      RR2: blocks 7 and 8
+         *      RL3: blocks 9 and 10
+         *      RR3: blocks 11 and 12
+         *      RL4: blocks 13 and 14
+         *      RR4: blocks 15 and 16
+         * output:
+         *      RL1: encrypted blocks 1 and 2
+         *      RR1: encrypted blocks 3 and 4
+         *      RL2: encrypted blocks 5 and 6
+         *      RR2: encrypted blocks 7 and 8
+         *      RL3: encrypted blocks 9 and 10
+         *      RR3: encrypted blocks 11 and 12
+         *      RL4: encrypted blocks 13 and 14
+         *      RR4: encrypted blocks 15 and 16
         */
        pushq %rbp;
        pushq %rbx;
-        pushq %rcx;
        vmovdqa .Lbswap_mask, RKM;
        vmovd .Lfirst_mask, R1ST;
        vmovd .L32_mask, R32;
        enc_preload_rkr();
-        leaq 1*(2*4*4)(%rdx), %rax;
+        inpack_blocks(RL1, RR1, RTMP, RX, RKM);
-        inpack_blocks(%rdx, RL1, RR1, RTMP, RX, RKM);
+        inpack_blocks(RL2, RR2, RTMP, RX, RKM);
-        inpack_blocks(%rax, RL2, RR2, RTMP, RX, RKM);
+        inpack_blocks(RL3, RR3, RTMP, RX, RKM);
-        leaq 2*(2*4*4)(%rdx), %rax;
+        inpack_blocks(RL4, RR4, RTMP, RX, RKM);
-        inpack_blocks(%rax, RL3, RR3, RTMP, RX, RKM);
-        leaq 3*(2*4*4)(%rdx), %rax;
-        inpack_blocks(%rax, RL4, RR4, RTMP, RX, RKM);
-        movq %rsi, %r11;
        round(RL, RR, 0, 1);
        round(RR, RL, 1, 2);
@@ -276,44 +271,41 @@ __cast5_enc_blk_16way:
        round(RR, RL, 15, 1);
 __skip_enc:
-        popq %rcx;
        popq %rbx;
        popq %rbp;
        vmovdqa .Lbswap_mask, RKM;
-        leaq 1*(2*4*4)(%r11), %rax;
-        testb %cl, %cl;
+        outunpack_blocks(RR1, RL1, RTMP, RX, RKM);
-        jnz __enc_xor16;
+        outunpack_blocks(RR2, RL2, RTMP, RX, RKM);
+        outunpack_blocks(RR3, RL3, RTMP, RX, RKM);
-        outunpack_blocks(%r11, RR1, RL1, RTMP, RX, RKM);
+        outunpack_blocks(RR4, RL4, RTMP, RX, RKM);
-        outunpack_blocks(%rax, RR2, RL2, RTMP, RX, RKM);
-        leaq 2*(2*4*4)(%r11), %rax;
-        outunpack_blocks(%rax, RR3, RL3, RTMP, RX, RKM);
-        leaq 3*(2*4*4)(%r11), %rax;
-        outunpack_blocks(%rax, RR4, RL4, RTMP, RX, RKM);
-        ret;
-__enc_xor16:
-        outunpack_xor_blocks(%r11, RR1, RL1, RTMP, RX, RKM);
-        outunpack_xor_blocks(%rax, RR2, RL2, RTMP, RX, RKM);
-        leaq 2*(2*4*4)(%r11), %rax;
-        outunpack_xor_blocks(%rax, RR3, RL3, RTMP, RX, RKM);
-        leaq 3*(2*4*4)(%r11), %rax;
-        outunpack_xor_blocks(%rax, RR4, RL4, RTMP, RX, RKM);
        ret;
 .align 16
-.global cast5_dec_blk_16way
+.type   __cast5_dec_blk16,@function;
-.type   cast5_dec_blk_16way,@function;
-cast5_dec_blk_16way:
+__cast5_dec_blk16:
        /* input:
         *      %rdi: ctx, CTX
-         *      %rsi: dst
+         *      RL1: encrypted blocks 1 and 2
-         *      %rdx: src
+         *      RR1: encrypted blocks 3 and 4
+         *      RL2: encrypted blocks 5 and 6
+         *      RR2: encrypted blocks 7 and 8
+         *      RL3: encrypted blocks 9 and 10
+         *      RR3: encrypted blocks 11 and 12
+         *      RL4: encrypted blocks 13 and 14
+         *      RR4: encrypted blocks 15 and 16
+         * output:
+         *      RL1: decrypted blocks 1 and 2
+         *      RR1: decrypted blocks 3 and 4
+         *      RL2: decrypted blocks 5 and 6
+         *      RR2: decrypted blocks 7 and 8
+         *      RL3: decrypted blocks 9 and 10
+         *      RR3: decrypted blocks 11 and 12
+         *      RL4: decrypted blocks 13 and 14
+         *      RR4: decrypted blocks 15 and 16
         */
        pushq %rbp;
@@ -324,15 +316,10 @@ cast5_dec_blk_16way:
        vmovd .L32_mask, R32;
        dec_preload_rkr();
-        leaq 1*(2*4*4)(%rdx), %rax;
+        inpack_blocks(RL1, RR1, RTMP, RX, RKM);
-        inpack_blocks(%rdx, RL1, RR1, RTMP, RX, RKM);
+        inpack_blocks(RL2, RR2, RTMP, RX, RKM);
-        inpack_blocks(%rax, RL2, RR2, RTMP, RX, RKM);
+        inpack_blocks(RL3, RR3, RTMP, RX, RKM);
-        leaq 2*(2*4*4)(%rdx), %rax;
+        inpack_blocks(RL4, RR4, RTMP, RX, RKM);
-        inpack_blocks(%rax, RL3, RR3, RTMP, RX, RKM);
-        leaq 3*(2*4*4)(%rdx), %rax;
-        inpack_blocks(%rax, RL4, RR4, RTMP, RX, RKM);
-        movq %rsi, %r11;
        movzbl rr(CTX), %eax;
        testl %eax, %eax;
@@ -361,16 +348,211 @@ __dec_tail:
        popq %rbx;
        popq %rbp;
-        leaq 1*(2*4*4)(%r11), %rax;
+        outunpack_blocks(RR1, RL1, RTMP, RX, RKM);
-        outunpack_blocks(%r11, RR1, RL1, RTMP, RX, RKM);
+        outunpack_blocks(RR2, RL2, RTMP, RX, RKM);
-        outunpack_blocks(%rax, RR2, RL2, RTMP, RX, RKM);
+        outunpack_blocks(RR3, RL3, RTMP, RX, RKM);
-        leaq 2*(2*4*4)(%r11), %rax;
+        outunpack_blocks(RR4, RL4, RTMP, RX, RKM);
-        outunpack_blocks(%rax, RR3, RL3, RTMP, RX, RKM);
-        leaq 3*(2*4*4)(%r11), %rax;
-        outunpack_blocks(%rax, RR4, RL4, RTMP, RX, RKM);
        ret;
 __skip_dec:
        vpsrldq $4, RKR, RKR;
        jmp __dec_tail;
+.align 16
+.global cast5_ecb_enc_16way
+.type   cast5_ecb_enc_16way,@function;
+cast5_ecb_enc_16way:
+        /* input:
+         *      %rdi: ctx, CTX
+         *      %rsi: dst
+         *      %rdx: src
+         */
+        movq %rsi, %r11;
+        vmovdqu (0*4*4)(%rdx), RL1;
+        vmovdqu (1*4*4)(%rdx), RR1;
+        vmovdqu (2*4*4)(%rdx), RL2;
+        vmovdqu (3*4*4)(%rdx), RR2;
+        vmovdqu (4*4*4)(%rdx), RL3;
+        vmovdqu (5*4*4)(%rdx), RR3;
+        vmovdqu (6*4*4)(%rdx), RL4;
+        vmovdqu (7*4*4)(%rdx), RR4;
+        call __cast5_enc_blk16;
+        vmovdqu RR1, (0*4*4)(%r11);
+        vmovdqu RL1, (1*4*4)(%r11);
+        vmovdqu RR2, (2*4*4)(%r11);
+        vmovdqu RL2, (3*4*4)(%r11);
+        vmovdqu RR3, (4*4*4)(%r11);
+        vmovdqu RL3, (5*4*4)(%r11);
+        vmovdqu RR4, (6*4*4)(%r11);
+        vmovdqu RL4, (7*4*4)(%r11);
+        ret;
+.align 16
+.global cast5_ecb_dec_16way
+.type   cast5_ecb_dec_16way,@function;
+cast5_ecb_dec_16way:
+        /* input:
+         *      %rdi: ctx, CTX
+         *      %rsi: dst
+         *      %rdx: src
+         */
+        movq %rsi, %r11;
+        vmovdqu (0*4*4)(%rdx), RL1;
+        vmovdqu (1*4*4)(%rdx), RR1;
+        vmovdqu (2*4*4)(%rdx), RL2;
+        vmovdqu (3*4*4)(%rdx), RR2;
+        vmovdqu (4*4*4)(%rdx), RL3;
+        vmovdqu (5*4*4)(%rdx), RR3;
+        vmovdqu (6*4*4)(%rdx), RL4;
+        vmovdqu (7*4*4)(%rdx), RR4;
+        call __cast5_dec_blk16;
+        vmovdqu RR1, (0*4*4)(%r11);
+        vmovdqu RL1, (1*4*4)(%r11);
+        vmovdqu RR2, (2*4*4)(%r11);
+        vmovdqu RL2, (3*4*4)(%r11);
+        vmovdqu RR3, (4*4*4)(%r11);
+        vmovdqu RL3, (5*4*4)(%r11);
+        vmovdqu RR4, (6*4*4)(%r11);
+        vmovdqu RL4, (7*4*4)(%r11);
+        ret;
+.align 16
+.global cast5_cbc_dec_16way
+.type   cast5_cbc_dec_16way,@function;
+cast5_cbc_dec_16way:
+        /* input:
+         *      %rdi: ctx, CTX
+         *      %rsi: dst
+         *      %rdx: src
+         */
+        pushq %r12;
+        movq %rsi, %r11;
+        movq %rdx, %r12;
+        vmovdqu (0*16)(%rdx), RL1;
+        vmovdqu (1*16)(%rdx), RR1;
+        vmovdqu (2*16)(%rdx), RL2;
+        vmovdqu (3*16)(%rdx), RR2;
+        vmovdqu (4*16)(%rdx), RL3;
+        vmovdqu (5*16)(%rdx), RR3;
+        vmovdqu (6*16)(%rdx), RL4;
+        vmovdqu (7*16)(%rdx), RR4;
+        call __cast5_dec_blk16;
+        /* xor with src */
+        vmovq (%r12), RX;
+        vpshufd $0x4f, RX, RX;
+        vpxor RX, RR1, RR1;
+        vpxor 0*16+8(%r12), RL1, RL1;
+        vpxor 1*16+8(%r12), RR2, RR2;
+        vpxor 2*16+8(%r12), RL2, RL2;
+        vpxor 3*16+8(%r12), RR3, RR3;
+        vpxor 4*16+8(%r12), RL3, RL3;
+        vpxor 5*16+8(%r12), RR4, RR4;
+        vpxor 6*16+8(%r12), RL4, RL4;
+        vmovdqu RR1, (0*16)(%r11);
+        vmovdqu RL1, (1*16)(%r11);
+        vmovdqu RR2, (2*16)(%r11);
+        vmovdqu RL2, (3*16)(%r11);
+        vmovdqu RR3, (4*16)(%r11);
+        vmovdqu RL3, (5*16)(%r11);
+        vmovdqu RR4, (6*16)(%r11);
+        vmovdqu RL4, (7*16)(%r11);
+        popq %r12;
+        ret;
+.align 16
+.global cast5_ctr_16way
+.type   cast5_ctr_16way,@function;
+cast5_ctr_16way:
+        /* input:
+         *      %rdi: ctx, CTX
+         *      %rsi: dst
+         *      %rdx: src
+         *      %rcx: iv (big endian, 64bit)
+         */
+        pushq %r12;
+        movq %rsi, %r11;
+        movq %rdx, %r12;
+        vpcmpeqd RTMP, RTMP, RTMP;
+        vpsrldq $8, RTMP, RTMP; /* low: -1, high: 0 */
+        vpcmpeqd RKR, RKR, RKR;
+        vpaddq RKR, RKR, RKR; /* low: -2, high: -2 */
+        vmovdqa .Lbswap_iv_mask, R1ST;
+        vmovdqa .Lbswap128_mask, RKM;
+        /* load IV and byteswap */
+        vmovq (%rcx), RX;
+        vpshufb R1ST, RX, RX;
+        /* construct IVs */
+        vpsubq RTMP, RX, RX;  /* le: IV1, IV0 */
+        vpshufb RKM, RX, RL1; /* be: IV0, IV1 */
+        vpsubq RKR, RX, RX;
+        vpshufb RKM, RX, RR1; /* be: IV2, IV3 */
+        vpsubq RKR, RX, RX;
+        vpshufb RKM, RX, RL2; /* be: IV4, IV5 */
+        vpsubq RKR, RX, RX;
+        vpshufb RKM, RX, RR2; /* be: IV6, IV7 */
+        vpsubq RKR, RX, RX;
+        vpshufb RKM, RX, RL3; /* be: IV8, IV9 */
+        vpsubq RKR, RX, RX;
+        vpshufb RKM, RX, RR3; /* be: IV10, IV11 */
+        vpsubq RKR, RX, RX;
+        vpshufb RKM, RX, RL4; /* be: IV12, IV13 */
+        vpsubq RKR, RX, RX;
+        vpshufb RKM, RX, RR4; /* be: IV14, IV15 */
+        /* store last IV */
+        vpsubq RTMP, RX, RX; /* le: IV16, IV14 */
+        vpshufb R1ST, RX, RX; /* be: IV16, IV16 */
+        vmovq RX, (%rcx);
+        call __cast5_enc_blk16;
+        /* dst = src ^ iv */
+        vpxor (0*16)(%r12), RR1, RR1;
+        vpxor (1*16)(%r12), RL1, RL1;
+        vpxor (2*16)(%r12), RR2, RR2;
+        vpxor (3*16)(%r12), RL2, RL2;
+        vpxor (4*16)(%r12), RR3, RR3;
+        vpxor (5*16)(%r12), RL3, RL3;
+        vpxor (6*16)(%r12), RR4, RR4;
+        vpxor (7*16)(%r12), RL4, RL4;
+        vmovdqu RR1, (0*16)(%r11);
+        vmovdqu RL1, (1*16)(%r11);
+        vmovdqu RR2, (2*16)(%r11);
+        vmovdqu RL2, (3*16)(%r11);
+        vmovdqu RR3, (4*16)(%r11);
+        vmovdqu RL3, (5*16)(%r11);
+        vmovdqu RR4, (6*16)(%r11);
+        vmovdqu RL4, (7*16)(%r11);
+        popq %r12;
+        ret;
diff --git a/arch/x86/crypto/cast5_avx_glue.c b/arch/x86/crypto/cast5_avx_glue.c
index e0ea14f9547f..c6631813dc11 100644
--- a/arch/x86/crypto/cast5_avx_glue.c
+++ b/arch/x86/crypto/cast5_avx_glue.c
@@ -37,29 +37,14 @@
 #define CAST5_PARALLEL_BLOCKS 16
-asmlinkage void __cast5_enc_blk_16way(struct cast5_ctx *ctx, u8 *dst,
+asmlinkage void cast5_ecb_enc_16way(struct cast5_ctx *ctx, u8 *dst,
-                                      const u8 *src, bool xor);
-asmlinkage void cast5_dec_blk_16way(struct cast5_ctx *ctx, u8 *dst,
                                    const u8 *src);
+asmlinkage void cast5_ecb_dec_16way(struct cast5_ctx *ctx, u8 *dst,
-static inline void cast5_enc_blk_xway(struct cast5_ctx *ctx, u8 *dst,
+                                    const u8 *src);
-                                      const u8 *src)
+asmlinkage void cast5_cbc_dec_16way(struct cast5_ctx *ctx, u8 *dst,
-{
+                                    const u8 *src);
-        __cast5_enc_blk_16way(ctx, dst, src, false);
+asmlinkage void cast5_ctr_16way(struct cast5_ctx *ctx, u8 *dst, const u8 *src,
-}
+                                __be64 *iv);
-static inline void cast5_enc_blk_xway_xor(struct cast5_ctx *ctx, u8 *dst,
-                                          const u8 *src)
-{
-        __cast5_enc_blk_16way(ctx, dst, src, true);
-}
-static inline void cast5_dec_blk_xway(struct cast5_ctx *ctx, u8 *dst,
-                                      const u8 *src)
-{
-        cast5_dec_blk_16way(ctx, dst, src);
-}
 static inline bool cast5_fpu_begin(bool fpu_enabled, unsigned int nbytes)
 {
@@ -79,8 +64,11 @@ static int ecb_crypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk,
        struct cast5_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
        const unsigned int bsize = CAST5_BLOCK_SIZE;
        unsigned int nbytes;
+        void (*fn)(struct cast5_ctx *ctx, u8 *dst, const u8 *src);
        int err;
+        fn = (enc) ? cast5_ecb_enc_16way : cast5_ecb_dec_16way;
        err = blkcipher_walk_virt(desc, walk);
        desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
@@ -93,10 +81,7 @@ static int ecb_crypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk,
                /* Process multi-block batch */
                if (nbytes >= bsize * CAST5_PARALLEL_BLOCKS) {
                        do {
-                                if (enc)
+                                fn(ctx, wdst, wsrc);
-                                        cast5_enc_blk_xway(ctx, wdst, wsrc);
-                                else
-                                        cast5_dec_blk_xway(ctx, wdst, wsrc);
                                wsrc += bsize * CAST5_PARALLEL_BLOCKS;
                                wdst += bsize * CAST5_PARALLEL_BLOCKS;
@@ -107,12 +92,11 @@ static int ecb_crypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk,
                                goto done;
                }
+                fn = (enc) ? __cast5_encrypt : __cast5_decrypt;
                /* Handle leftovers */
                do {
-                        if (enc)
+                        fn(ctx, wdst, wsrc);
-                                __cast5_encrypt(ctx, wdst, wsrc);
-                        else
-                                __cast5_decrypt(ctx, wdst, wsrc);
                        wsrc += bsize;
                        wdst += bsize;
@@ -194,9 +178,7 @@ static unsigned int __cbc_decrypt(struct blkcipher_desc *desc,
        unsigned int nbytes = walk->nbytes;
        u64 *src = (u64 *)walk->src.virt.addr;
        u64 *dst = (u64 *)walk->dst.virt.addr;
-        u64 ivs[CAST5_PARALLEL_BLOCKS - 1];
        u64 last_iv;
-        int i;
        /* Start of the last block. */
        src += nbytes / bsize - 1;
@@ -211,13 +193,7 @@ static unsigned int __cbc_decrypt(struct blkcipher_desc *desc,
                        src -= CAST5_PARALLEL_BLOCKS - 1;
                        dst -= CAST5_PARALLEL_BLOCKS - 1;
-                        for (i = 0; i < CAST5_PARALLEL_BLOCKS - 1; i++)
+                        cast5_cbc_dec_16way(ctx, (u8 *)dst, (u8 *)src);
-                                ivs[i] = src[i];
-                        cast5_dec_blk_xway(ctx, (u8 *)dst, (u8 *)src);
-                        for (i = 0; i < CAST5_PARALLEL_BLOCKS - 1; i++)
-                                *(dst + (i + 1)) ^= *(ivs + i);
                        nbytes -= bsize;
                        if (nbytes < bsize)
@@ -298,23 +274,12 @@ static unsigned int __ctr_crypt(struct blkcipher_desc *desc,
        unsigned int nbytes = walk->nbytes;
        u64 *src = (u64 *)walk->src.virt.addr;
        u64 *dst = (u64 *)walk->dst.virt.addr;
-        u64 ctrblk = be64_to_cpu(*(__be64 *)walk->iv);
-        __be64 ctrblocks[CAST5_PARALLEL_BLOCKS];
-        int i;
        /* Process multi-block batch */
        if (nbytes >= bsize * CAST5_PARALLEL_BLOCKS) {
                do {
-                        /* create ctrblks for parallel encrypt */
+                        cast5_ctr_16way(ctx, (u8 *)dst, (u8 *)src,
-                        for (i = 0; i < CAST5_PARALLEL_BLOCKS; i++) {
+                                        (__be64 *)walk->iv);
-                                if (dst != src)
-                                        dst[i] = src[i];
-                                ctrblocks[i] = cpu_to_be64(ctrblk++);
-                        }
-                        cast5_enc_blk_xway_xor(ctx, (u8 *)dst,
-                                               (u8 *)ctrblocks);
                        src += CAST5_PARALLEL_BLOCKS;
                        dst += CAST5_PARALLEL_BLOCKS;
@@ -327,13 +292,16 @@ static unsigned int __ctr_crypt(struct blkcipher_desc *desc,
        /* Handle leftovers */
        do {
+                u64 ctrblk;
                if (dst != src)
                        *dst = *src;
-                ctrblocks[0] = cpu_to_be64(ctrblk++);
+                ctrblk = *(u64 *)walk->iv;
+                be64_add_cpu((__be64 *)walk->iv, 1);
-                __cast5_encrypt(ctx, (u8 *)ctrblocks, (u8 *)ctrblocks);
+                __cast5_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk);
-                *dst ^= ctrblocks[0];
+                *dst ^= ctrblk;
                src += 1;
                dst += 1;
@@ -341,7 +309,6 @@ static unsigned int __ctr_crypt(struct blkcipher_desc *desc,
        } while (nbytes >= bsize);
 done:
-        *(__be64 *)walk->iv = cpu_to_be64(ctrblk);
        return nbytes;
 }
diff --git a/arch/x86/crypto/cast6-avx-x86_64-asm_64.S b/arch/x86/crypto/cast6-avx-x86_64-asm_64.S
index 218d283772f4..2569d0da841f 100644
--- a/arch/x86/crypto/cast6-avx-x86_64-asm_64.S
+++ b/arch/x86/crypto/cast6-avx-x86_64-asm_64.S
@@ -23,22 +23,24 @@
 *
 */
+#include "glue_helper-asm-avx.S"
 .file "cast6-avx-x86_64-asm_64.S"
-.extern cast6_s1
+.extern cast_s1
-.extern cast6_s2
+.extern cast_s2
-.extern cast6_s3
+.extern cast_s3
-.extern cast6_s4
+.extern cast_s4
 /* structure of crypto context */
 #define km      0
 #define kr      (12*4*4)
 /* s-boxes */
-#define s1      cast6_s1
+#define s1      cast_s1
-#define s2      cast6_s2
+#define s2      cast_s2
-#define s3      cast6_s3
+#define s3      cast_s3
-#define s4      cast6_s4
+#define s4      cast_s4
 /**********************************************************************
  8-way AVX cast6
@@ -205,11 +207,7 @@
        vpunpcklqdq             x3, t2, x2; \
        vpunpckhqdq             x3, t2, x3;
-#define inpack_blocks(in, x0, x1, x2, x3, t0, t1, t2, rmask) \
+#define inpack_blocks(x0, x1, x2, x3, t0, t1, t2, rmask) \
-        vmovdqu (0*4*4)(in),    x0; \
-        vmovdqu (1*4*4)(in),    x1; \
-        vmovdqu (2*4*4)(in),    x2; \
-        vmovdqu (3*4*4)(in),    x3; \
        vpshufb rmask, x0,      x0; \
        vpshufb rmask, x1,      x1; \
        vpshufb rmask, x2,      x2; \
@@ -217,39 +215,21 @@
        \
        transpose_4x4(x0, x1, x2, x3, t0, t1, t2)
-#define outunpack_blocks(out, x0, x1, x2, x3, t0, t1, t2, rmask) \
+#define outunpack_blocks(x0, x1, x2, x3, t0, t1, t2, rmask) \
        transpose_4x4(x0, x1, x2, x3, t0, t1, t2) \
        \
        vpshufb rmask,          x0, x0;       \
        vpshufb rmask,          x1, x1;       \
        vpshufb rmask,          x2, x2;       \
-        vpshufb rmask,          x3, x3;       \
+        vpshufb rmask,          x3, x3;
-        vmovdqu x0,             (0*4*4)(out); \
-        vmovdqu x1,             (1*4*4)(out); \
-        vmovdqu x2,             (2*4*4)(out); \
-        vmovdqu x3,             (3*4*4)(out);
-#define outunpack_xor_blocks(out, x0, x1, x2, x3, t0, t1, t2, rmask) \
-        transpose_4x4(x0, x1, x2, x3, t0, t1, t2) \
-        \
-        vpshufb rmask,          x0, x0;       \
-        vpshufb rmask,          x1, x1;       \
-        vpshufb rmask,          x2, x2;       \
-        vpshufb rmask,          x3, x3;       \
-        vpxor (0*4*4)(out),     x0, x0;       \
-        vmovdqu x0,             (0*4*4)(out); \
-        vpxor (1*4*4)(out),     x1, x1;       \
-        vmovdqu x1,             (1*4*4)(out); \
-        vpxor (2*4*4)(out),     x2, x2;       \
-        vmovdqu x2,             (2*4*4)(out); \
-        vpxor (3*4*4)(out),     x3, x3;       \
-        vmovdqu x3,             (3*4*4)(out);
 .data
 .align 16
 .Lbswap_mask:
        .byte 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12
+.Lbswap128_mask:
+        .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
 .Lrkr_enc_Q_Q_QBAR_QBAR:
        .byte 0, 1, 2, 3, 4, 5, 6, 7, 11, 10, 9, 8, 15, 14, 13, 12
 .Lrkr_enc_QBAR_QBAR_QBAR_QBAR:
@@ -269,31 +249,26 @@
 .text
-.align 16
+.align 8
-.global __cast6_enc_blk_8way
+.type   __cast6_enc_blk8,@function;
-.type   __cast6_enc_blk_8way,@function;
-__cast6_enc_blk_8way:
+__cast6_enc_blk8:
        /* input:
         *      %rdi: ctx, CTX
-         *      %rsi: dst
+         *      RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2: blocks
-         *      %rdx: src
+         * output:
-         *      %rcx: bool, if true: xor output
+         *      RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2: encrypted blocks
         */
        pushq %rbp;
        pushq %rbx;
-        pushq %rcx;
        vmovdqa .Lbswap_mask, RKM;
        vmovd .Lfirst_mask, R1ST;
        vmovd .L32_mask, R32;
-        leaq (4*4*4)(%rdx), %rax;
+        inpack_blocks(RA1, RB1, RC1, RD1, RTMP, RX, RKRF, RKM);
-        inpack_blocks(%rdx, RA1, RB1, RC1, RD1, RTMP, RX, RKRF, RKM);
+        inpack_blocks(RA2, RB2, RC2, RD2, RTMP, RX, RKRF, RKM);
-        inpack_blocks(%rax, RA2, RB2, RC2, RD2, RTMP, RX, RKRF, RKM);
-        movq %rsi, %r11;
        preload_rkr(0, dummy, none);
        Q(0);
@@ -311,36 +286,25 @@ __cast6_enc_blk_8way:
        QBAR(10);
        QBAR(11);
-        popq %rcx;
        popq %rbx;
        popq %rbp;
        vmovdqa .Lbswap_mask, RKM;
-        leaq (4*4*4)(%r11), %rax;
-        testb %cl, %cl;
-        jnz __enc_xor8;
-        outunpack_blocks(%r11, RA1, RB1, RC1, RD1, RTMP, RX, RKRF, RKM);
-        outunpack_blocks(%rax, RA2, RB2, RC2, RD2, RTMP, RX, RKRF, RKM);
-        ret;
-__enc_xor8:
+        outunpack_blocks(RA1, RB1, RC1, RD1, RTMP, RX, RKRF, RKM);
-        outunpack_xor_blocks(%r11, RA1, RB1, RC1, RD1, RTMP, RX, RKRF, RKM);
+        outunpack_blocks(RA2, RB2, RC2, RD2, RTMP, RX, RKRF, RKM);
-        outunpack_xor_blocks(%rax, RA2, RB2, RC2, RD2, RTMP, RX, RKRF, RKM);
        ret;
-.align 16
+.align 8
-.global cast6_dec_blk_8way
+.type   __cast6_dec_blk8,@function;
-.type   cast6_dec_blk_8way,@function;
-cast6_dec_blk_8way:
+__cast6_dec_blk8:
        /* input:
         *      %rdi: ctx, CTX
-         *      %rsi: dst
+         *      RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2: encrypted blocks
-         *      %rdx: src
+         * output:
+         *      RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2: decrypted blocks
         */
        pushq %rbp;
@@ -350,11 +314,8 @@ cast6_dec_blk_8way:
        vmovd .Lfirst_mask, R1ST;
        vmovd .L32_mask, R32;
-        leaq (4*4*4)(%rdx), %rax;
+        inpack_blocks(RA1, RB1, RC1, RD1, RTMP, RX, RKRF, RKM);
-        inpack_blocks(%rdx, RA1, RB1, RC1, RD1, RTMP, RX, RKRF, RKM);
+        inpack_blocks(RA2, RB2, RC2, RD2, RTMP, RX, RKRF, RKM);
-        inpack_blocks(%rax, RA2, RB2, RC2, RD2, RTMP, RX, RKRF, RKM);
-        movq %rsi, %r11;
        preload_rkr(2, shuffle, .Lrkr_dec_Q_Q_Q_Q);
        Q(11);
@@ -376,8 +337,103 @@ cast6_dec_blk_8way:
        popq %rbp;
        vmovdqa .Lbswap_mask, RKM;
-        leaq (4*4*4)(%r11), %rax;
+        outunpack_blocks(RA1, RB1, RC1, RD1, RTMP, RX, RKRF, RKM);
-        outunpack_blocks(%r11, RA1, RB1, RC1, RD1, RTMP, RX, RKRF, RKM);
+        outunpack_blocks(RA2, RB2, RC2, RD2, RTMP, RX, RKRF, RKM);
-        outunpack_blocks(%rax, RA2, RB2, RC2, RD2, RTMP, RX, RKRF, RKM);
+        ret;
+.align 8
+.global cast6_ecb_enc_8way
+.type   cast6_ecb_enc_8way,@function;
+cast6_ecb_enc_8way:
+        /* input:
+         *      %rdi: ctx, CTX
+         *      %rsi: dst
+         *      %rdx: src
+         */
+        movq %rsi, %r11;
+        load_8way(%rdx, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
+        call __cast6_enc_blk8;
+        store_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
+        ret;
+.align 8
+.global cast6_ecb_dec_8way
+.type   cast6_ecb_dec_8way,@function;
+cast6_ecb_dec_8way:
+        /* input:
+         *      %rdi: ctx, CTX
+         *      %rsi: dst
+         *      %rdx: src
+         */
+        movq %rsi, %r11;
+        load_8way(%rdx, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
+        call __cast6_dec_blk8;
+        store_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
+        ret;
+.align 8
+.global cast6_cbc_dec_8way
+.type   cast6_cbc_dec_8way,@function;
+cast6_cbc_dec_8way:
+        /* input:
+         *      %rdi: ctx, CTX
+         *      %rsi: dst
+         *      %rdx: src
+         */
+        pushq %r12;
+        movq %rsi, %r11;
+        movq %rdx, %r12;
+        load_8way(%rdx, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
+        call __cast6_dec_blk8;
+        store_cbc_8way(%r12, %r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
+        popq %r12;
+        ret;
+.align 8
+.global cast6_ctr_8way
+.type   cast6_ctr_8way,@function;
+cast6_ctr_8way:
+        /* input:
+         *      %rdi: ctx, CTX
+         *      %rsi: dst
+         *      %rdx: src
+         *      %rcx: iv (little endian, 128bit)
+         */
+        pushq %r12;
+        movq %rsi, %r11;
+        movq %rdx, %r12;
+        load_ctr_8way(%rcx, .Lbswap128_mask, RA1, RB1, RC1, RD1, RA2, RB2, RC2,
+                      RD2, RX, RKR, RKM);
+        call __cast6_enc_blk8;
+        store_ctr_8way(%r12, %r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
+        popq %r12;
        ret;
diff --git a/arch/x86/crypto/cast6_avx_glue.c b/arch/x86/crypto/cast6_avx_glue.c
index 15e5f85a5011..92f7ca24790a 100644
--- a/arch/x86/crypto/cast6_avx_glue.c
+++ b/arch/x86/crypto/cast6_avx_glue.c
@@ -40,79 +40,34 @@
 #define CAST6_PARALLEL_BLOCKS 8
-asmlinkage void __cast6_enc_blk_8way(struct cast6_ctx *ctx, u8 *dst,
+asmlinkage void cast6_ecb_enc_8way(struct cast6_ctx *ctx, u8 *dst,
-                                     const u8 *src, bool xor);
+                                   const u8 *src);
-asmlinkage void cast6_dec_blk_8way(struct cast6_ctx *ctx, u8 *dst,
+asmlinkage void cast6_ecb_dec_8way(struct cast6_ctx *ctx, u8 *dst,
                                   const u8 *src);
-static inline void cast6_enc_blk_xway(struct cast6_ctx *ctx, u8 *dst,
+asmlinkage void cast6_cbc_dec_8way(struct cast6_ctx *ctx, u8 *dst,
-                                      const u8 *src)
+                                   const u8 *src);
-{
+asmlinkage void cast6_ctr_8way(struct cast6_ctx *ctx, u8 *dst, const u8 *src,
-        __cast6_enc_blk_8way(ctx, dst, src, false);
+                               le128 *iv);
-}
-static inline void cast6_enc_blk_xway_xor(struct cast6_ctx *ctx, u8 *dst,
-                                          const u8 *src)
-{
-        __cast6_enc_blk_8way(ctx, dst, src, true);
-}
-static inline void cast6_dec_blk_xway(struct cast6_ctx *ctx, u8 *dst,
-                                      const u8 *src)
-{
-        cast6_dec_blk_8way(ctx, dst, src);
-}
-static void cast6_decrypt_cbc_xway(void *ctx, u128 *dst, const u128 *src)
-{
-        u128 ivs[CAST6_PARALLEL_BLOCKS - 1];
-        unsigned int j;
-        for (j = 0; j < CAST6_PARALLEL_BLOCKS - 1; j++)
-                ivs[j] = src[j];
-        cast6_dec_blk_xway(ctx, (u8 *)dst, (u8 *)src);
-        for (j = 0; j < CAST6_PARALLEL_BLOCKS - 1; j++)
-                u128_xor(dst + (j + 1), dst + (j + 1), ivs + j);
-}
-static void cast6_crypt_ctr(void *ctx, u128 *dst, const u128 *src, u128 *iv)
+static void cast6_crypt_ctr(void *ctx, u128 *dst, const u128 *src, le128 *iv)
 {
        be128 ctrblk;
-        u128_to_be128(&ctrblk, iv);
+        le128_to_be128(&ctrblk, iv);
-        u128_inc(iv);
+        le128_inc(iv);
        __cast6_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk);
        u128_xor(dst, src, (u128 *)&ctrblk);
 }
-static void cast6_crypt_ctr_xway(void *ctx, u128 *dst, const u128 *src,
-                                   u128 *iv)
-{
-        be128 ctrblks[CAST6_PARALLEL_BLOCKS];
-        unsigned int i;
-        for (i = 0; i < CAST6_PARALLEL_BLOCKS; i++) {
-                if (dst != src)
-                        dst[i] = src[i];
-                u128_to_be128(&ctrblks[i], iv);
-                u128_inc(iv);
-        }
-        cast6_enc_blk_xway_xor(ctx, (u8 *)dst, (u8 *)ctrblks);
-}
 static const struct common_glue_ctx cast6_enc = {
        .num_funcs = 2,
        .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,
        .funcs = { {
                .num_blocks = CAST6_PARALLEL_BLOCKS,
-                .fn_u = { .ecb = GLUE_FUNC_CAST(cast6_enc_blk_xway) }
+                .fn_u = { .ecb = GLUE_FUNC_CAST(cast6_ecb_enc_8way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ecb = GLUE_FUNC_CAST(__cast6_encrypt) }
@@ -125,7 +80,7 @@ static const struct common_glue_ctx cast6_ctr = {
        .funcs = { {
                .num_blocks = CAST6_PARALLEL_BLOCKS,
-                .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(cast6_crypt_ctr_xway) }
+                .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(cast6_ctr_8way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(cast6_crypt_ctr) }
@@ -138,7 +93,7 @@ static const struct common_glue_ctx cast6_dec = {
        .funcs = { {
                .num_blocks = CAST6_PARALLEL_BLOCKS,
-                .fn_u = { .ecb = GLUE_FUNC_CAST(cast6_dec_blk_xway) }
+                .fn_u = { .ecb = GLUE_FUNC_CAST(cast6_ecb_dec_8way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ecb = GLUE_FUNC_CAST(__cast6_decrypt) }
@@ -151,7 +106,7 @@ static const struct common_glue_ctx cast6_dec_cbc = {
        .funcs = { {
                .num_blocks = CAST6_PARALLEL_BLOCKS,
-                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(cast6_decrypt_cbc_xway) }
+                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(cast6_cbc_dec_8way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(__cast6_decrypt) }
@@ -215,7 +170,7 @@ static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
        ctx->fpu_enabled = cast6_fpu_begin(ctx->fpu_enabled, nbytes);
        if (nbytes == bsize * CAST6_PARALLEL_BLOCKS) {
-                cast6_enc_blk_xway(ctx->ctx, srcdst, srcdst);
+                cast6_ecb_enc_8way(ctx->ctx, srcdst, srcdst);
                return;
        }
@@ -232,7 +187,7 @@ static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
        ctx->fpu_enabled = cast6_fpu_begin(ctx->fpu_enabled, nbytes);
        if (nbytes == bsize * CAST6_PARALLEL_BLOCKS) {
-                cast6_dec_blk_xway(ctx->ctx, srcdst, srcdst);
+                cast6_ecb_dec_8way(ctx->ctx, srcdst, srcdst);
                return;
        }
diff --git a/arch/x86/crypto/crc32c-intel.c b/arch/x86/crypto/crc32c-intel_glue.c
index 493f959261f7..6812ad98355c 100644
--- a/arch/x86/crypto/crc32c-intel.c
+++ b/arch/x86/crypto/crc32c-intel_glue.c
@@ -32,6 +32,8 @@
 #include <asm/cpufeature.h>
 #include <asm/cpu_device_id.h>
+#include <asm/i387.h>
+#include <asm/fpu-internal.h>
 #define CHKSUM_BLOCK_SIZE       1
 #define CHKSUM_DIGEST_SIZE      4
@@ -44,6 +46,31 @@
 #define REX_PRE
 #endif
+#ifdef CONFIG_X86_64
+/*
+ * use carryless multiply version of crc32c when buffer
+ * size is >= 512 (when eager fpu is enabled) or
+ * >= 1024 (when eager fpu is disabled) to account
+ * for fpu state save/restore overhead.
+ */
+#define CRC32C_PCL_BREAKEVEN_EAGERFPU   512
+#define CRC32C_PCL_BREAKEVEN_NOEAGERFPU 1024
+asmlinkage unsigned int crc_pcl(const u8 *buffer, int len,
+                                unsigned int crc_init);
+static int crc32c_pcl_breakeven = CRC32C_PCL_BREAKEVEN_EAGERFPU;
+#if defined(X86_FEATURE_EAGER_FPU)
+#define set_pcl_breakeven_point()                                       \
+do {                                                                    \
+        if (!use_eager_fpu())                                           \
+                crc32c_pcl_breakeven = CRC32C_PCL_BREAKEVEN_NOEAGERFPU; \
+} while (0)
+#else
+#define set_pcl_breakeven_point()                                       \
+        (crc32c_pcl_breakeven = CRC32C_PCL_BREAKEVEN_NOEAGERFPU)
+#endif
+#endif /* CONFIG_X86_64 */
 static u32 crc32c_intel_le_hw_byte(u32 crc, unsigned char const *data, size_t length)
 {
        while (length--) {
@@ -154,6 +181,52 @@ static int crc32c_intel_cra_init(struct crypto_tfm *tfm)
        return 0;
 }
+#ifdef CONFIG_X86_64
+static int crc32c_pcl_intel_update(struct shash_desc *desc, const u8 *data,
+                               unsigned int len)
+{
+        u32 *crcp = shash_desc_ctx(desc);
+        /*
+         * use faster PCL version if datasize is large enough to
+         * overcome kernel fpu state save/restore overhead
+         */
+        if (len >= crc32c_pcl_breakeven && irq_fpu_usable()) {
+                kernel_fpu_begin();
+                *crcp = crc_pcl(data, len, *crcp);
+                kernel_fpu_end();
+        } else
+                *crcp = crc32c_intel_le_hw(*crcp, data, len);
+        return 0;
+}
+static int __crc32c_pcl_intel_finup(u32 *crcp, const u8 *data, unsigned int len,
+                                u8 *out)
+{
+        if (len >= crc32c_pcl_breakeven && irq_fpu_usable()) {
+                kernel_fpu_begin();
+                *(__le32 *)out = ~cpu_to_le32(crc_pcl(data, len, *crcp));
+                kernel_fpu_end();
+        } else
+                *(__le32 *)out =
+                        ~cpu_to_le32(crc32c_intel_le_hw(*crcp, data, len));
+        return 0;
+}
+static int crc32c_pcl_intel_finup(struct shash_desc *desc, const u8 *data,
+                              unsigned int len, u8 *out)
+{
+        return __crc32c_pcl_intel_finup(shash_desc_ctx(desc), data, len, out);
+}
+static int crc32c_pcl_intel_digest(struct shash_desc *desc, const u8 *data,
+                               unsigned int len, u8 *out)
+{
+        return __crc32c_pcl_intel_finup(crypto_shash_ctx(desc->tfm), data, len,
+                                    out);
+}
+#endif /* CONFIG_X86_64 */
 static struct shash_alg alg = {
        .setkey                 =       crc32c_intel_setkey,
        .init                   =       crc32c_intel_init,
@@ -184,6 +257,14 @@ static int __init crc32c_intel_mod_init(void)
 {
        if (!x86_match_cpu(crc32c_cpu_id))
                return -ENODEV;
+#ifdef CONFIG_X86_64
+        if (cpu_has_pclmulqdq) {
+                alg.update = crc32c_pcl_intel_update;
+                alg.finup = crc32c_pcl_intel_finup;
+                alg.digest = crc32c_pcl_intel_digest;
+                set_pcl_breakeven_point();
+        }
+#endif
        return crypto_register_shash(&alg);
 }
diff --git a/arch/x86/crypto/crc32c-pcl-intel-asm_64.S b/arch/x86/crypto/crc32c-pcl-intel-asm_64.S
new file mode 100644
index 000000000000..93c6d39237ac
--- /dev/null
+++ b/arch/x86/crypto/crc32c-pcl-intel-asm_64.S
@@ -0,0 +1,460 @@
+/*
+ * Implement fast CRC32C with PCLMULQDQ instructions. (x86_64)
+ *
+ * The white paper on CRC32C calculations with PCLMULQDQ instruction can be
+ * downloaded from:
+ * http://download.intel.com/design/intarch/papers/323405.pdf
+ *
+ * Copyright (C) 2012 Intel Corporation.
+ *
+ * Authors:
+ *      Wajdi Feghali <wajdi.k.feghali@intel.com>
+ *      James Guilford <james.guilford@intel.com>
+ *      David Cote <david.m.cote@intel.com>
+ *      Tim Chen <tim.c.chen@linux.intel.com>
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+## ISCSI CRC 32 Implementation with crc32 and pclmulqdq Instruction
+.macro LABEL prefix n
+\prefix\n\():
+.endm
+.macro JMPTBL_ENTRY i
+.word crc_\i - crc_array
+.endm
+.macro JNC_LESS_THAN j
+        jnc less_than_\j
+.endm
+# Define threshold where buffers are considered "small" and routed to more
+# efficient "by-1" code. This "by-1" code only handles up to 255 bytes, so
+# SMALL_SIZE can be no larger than 255.
+#define SMALL_SIZE 200
+.if (SMALL_SIZE > 255)
+.error "SMALL_ SIZE must be < 256"
+.endif
+# unsigned int crc_pcl(u8 *buffer, int len, unsigned int crc_init);
+.global crc_pcl
+crc_pcl:
+#define    bufp         %rdi
+#define    bufp_dw      %edi
+#define    bufp_w       %di
+#define    bufp_b       %dil
+#define    bufptmp      %rcx
+#define    block_0      %rcx
+#define    block_1      %rdx
+#define    block_2      %r11
+#define    len          %rsi
+#define    len_dw       %esi
+#define    len_w        %si
+#define    len_b        %sil
+#define    crc_init_arg %rdx
+#define    tmp          %rbx
+#define    crc_init     %r8
+#define    crc_init_dw  %r8d
+#define    crc1         %r9
+#define    crc2         %r10
+        pushq   %rbx
+        pushq   %rdi
+        pushq   %rsi
+        ## Move crc_init for Linux to a different
+        mov     crc_init_arg, crc_init
+        ################################################################
+        ## 1) ALIGN:
+        ################################################################
+        mov     bufp, bufptmp           # rdi = *buf
+        neg     bufp
+        and     $7, bufp                # calculate the unalignment amount of
+                                        # the address
+        je      proc_block              # Skip if aligned
+        ## If len is less than 8 and we're unaligned, we need to jump
+        ## to special code to avoid reading beyond the end of the buffer
+        cmp     $8, len
+        jae     do_align
+        # less_than_8 expects length in upper 3 bits of len_dw
+        # less_than_8_post_shl1 expects length = carryflag * 8 + len_dw[31:30]
+        shl     $32-3+1, len_dw
+        jmp     less_than_8_post_shl1
+do_align:
+        #### Calculate CRC of unaligned bytes of the buffer (if any)
+        movq    (bufptmp), tmp          # load a quadward from the buffer
+        add     bufp, bufptmp           # align buffer pointer for quadword
+                                        # processing
+        sub     bufp, len               # update buffer length
+align_loop:
+        crc32b  %bl, crc_init_dw        # compute crc32 of 1-byte
+        shr     $8, tmp                 # get next byte
+        dec     bufp
+        jne     align_loop
+proc_block:
+        ################################################################
+        ## 2) PROCESS  BLOCKS:
+        ################################################################
+        ## compute num of bytes to be processed
+        movq    len, tmp                # save num bytes in tmp
+        cmpq    $128*24, len
+        jae     full_block
+continue_block:
+        cmpq    $SMALL_SIZE, len
+        jb      small
+        ## len < 128*24
+        movq    $2731, %rax             # 2731 = ceil(2^16 / 24)
+        mul     len_dw
+        shrq    $16, %rax
+        ## eax contains floor(bytes / 24) = num 24-byte chunks to do
+        ## process rax 24-byte chunks (128 >= rax >= 0)
+        ## compute end address of each block
+        ## block 0 (base addr + RAX * 8)
+        ## block 1 (base addr + RAX * 16)
+        ## block 2 (base addr + RAX * 24)
+        lea     (bufptmp, %rax, 8), block_0
+        lea     (block_0, %rax, 8), block_1
+        lea     (block_1, %rax, 8), block_2
+        xor     crc1, crc1
+        xor     crc2, crc2
+        ## branch into array
+        lea     jump_table(%rip), bufp
+        movzxw  (bufp, %rax, 2), len
+        offset=crc_array-jump_table
+        lea     offset(bufp, len, 1), bufp
+        jmp     *bufp
+        ################################################################
+        ## 2a) PROCESS FULL BLOCKS:
+        ################################################################
+full_block:
+        movq    $128,%rax
+        lea     128*8*2(block_0), block_1
+        lea     128*8*3(block_0), block_2
+        add     $128*8*1, block_0
+        xor     crc1,crc1
+        xor     crc2,crc2
+        # Fall thruogh into top of crc array (crc_128)
+        ################################################################
+        ## 3) CRC Array:
+        ################################################################
+crc_array:
+        i=128
+.rept 128-1
+.altmacro
+LABEL crc_ %i
+.noaltmacro
+        crc32q   -i*8(block_0), crc_init
+        crc32q   -i*8(block_1), crc1
+        crc32q   -i*8(block_2), crc2
+        i=(i-1)
+.endr
+.altmacro
+LABEL crc_ %i
+.noaltmacro
+        crc32q   -i*8(block_0), crc_init
+        crc32q   -i*8(block_1), crc1
+# SKIP  crc32  -i*8(block_2), crc2 ; Don't do this one yet
+        mov     block_2, block_0
+        ################################################################
+        ## 4) Combine three results:
+        ################################################################
+        lea     (K_table-16)(%rip), bufp        # first entry is for idx 1
+        shlq    $3, %rax                        # rax *= 8
+        subq    %rax, tmp                       # tmp -= rax*8
+        shlq    $1, %rax
+        subq    %rax, tmp                       # tmp -= rax*16
+                                                # (total tmp -= rax*24)
+        addq    %rax, bufp
+        movdqa  (bufp), %xmm0                   # 2 consts: K1:K2
+        movq    crc_init, %xmm1                 # CRC for block 1
+        pclmulqdq $0x00,%xmm0,%xmm1             # Multiply by K2
+        movq    crc1, %xmm2                     # CRC for block 2
+        pclmulqdq $0x10, %xmm0, %xmm2           # Multiply by K1
+        pxor    %xmm2,%xmm1
+        movq    %xmm1, %rax
+        xor     -i*8(block_2), %rax
+        mov     crc2, crc_init
+        crc32   %rax, crc_init
+################################################################
+## 5) Check for end:
+################################################################
+LABEL crc_ 0
+        mov     tmp, len
+        cmp     $128*24, tmp
+        jae     full_block
+        cmp     $24, tmp
+        jae     continue_block
+less_than_24:
+        shl     $32-4, len_dw                   # less_than_16 expects length
+                                                # in upper 4 bits of len_dw
+        jnc     less_than_16
+        crc32q  (bufptmp), crc_init
+        crc32q  8(bufptmp), crc_init
+        jz      do_return
+        add     $16, bufptmp
+        # len is less than 8 if we got here
+        # less_than_8 expects length in upper 3 bits of len_dw
+        # less_than_8_post_shl1 expects length = carryflag * 8 + len_dw[31:30]
+        shl     $2, len_dw
+        jmp     less_than_8_post_shl1
+        #######################################################################
+        ## 6) LESS THAN 256-bytes REMAIN AT THIS POINT (8-bits of len are full)
+        #######################################################################
+small:
+        shl $32-8, len_dw               # Prepare len_dw for less_than_256
+        j=256
+.rept 5                                 # j = {256, 128, 64, 32, 16}
+.altmacro
+LABEL less_than_ %j                     # less_than_j: Length should be in
+                                        # upper lg(j) bits of len_dw
+        j=(j/2)
+        shl     $1, len_dw              # Get next MSB
+        JNC_LESS_THAN %j
+.noaltmacro
+        i=0
+.rept (j/8)
+        crc32q  i(bufptmp), crc_init    # Compute crc32 of 8-byte data
+        i=i+8
+.endr
+        jz      do_return               # Return if remaining length is zero
+        add     $j, bufptmp             # Advance buf
+.endr
+less_than_8:                            # Length should be stored in
+                                        # upper 3 bits of len_dw
+        shl     $1, len_dw
+less_than_8_post_shl1:
+        jnc     less_than_4
+        crc32l  (bufptmp), crc_init_dw  # CRC of 4 bytes
+        jz      do_return               # return if remaining data is zero
+        add     $4, bufptmp
+less_than_4:                            # Length should be stored in
+                                        # upper 2 bits of len_dw
+        shl     $1, len_dw
+        jnc     less_than_2
+        crc32w  (bufptmp), crc_init_dw  # CRC of 2 bytes
+        jz      do_return               # return if remaining data is zero
+        add     $2, bufptmp
+less_than_2:                            # Length should be stored in the MSB
+                                        # of len_dw
+        shl     $1, len_dw
+        jnc     less_than_1
+        crc32b  (bufptmp), crc_init_dw  # CRC of 1 byte
+less_than_1:                            # Length should be zero
+do_return:
+        movq    crc_init, %rax
+        popq    %rsi
+        popq    %rdi
+        popq    %rbx
+        ret
+        ################################################################
+        ## jump table        Table is 129 entries x 2 bytes each
+        ################################################################
+.align 4
+jump_table:
+        i=0
+.rept 129
+.altmacro
+JMPTBL_ENTRY %i
+.noaltmacro
+        i=i+1
+.endr
+        ################################################################
+        ## PCLMULQDQ tables
+        ## Table is 128 entries x 2 quad words each
+        ################################################################
+.data
+.align 64
+K_table:
+        .quad 0x14cd00bd6,0x105ec76f0
+        .quad 0x0ba4fc28e,0x14cd00bd6
+        .quad 0x1d82c63da,0x0f20c0dfe
+        .quad 0x09e4addf8,0x0ba4fc28e
+        .quad 0x039d3b296,0x1384aa63a
+        .quad 0x102f9b8a2,0x1d82c63da
+        .quad 0x14237f5e6,0x01c291d04
+        .quad 0x00d3b6092,0x09e4addf8
+        .quad 0x0c96cfdc0,0x0740eef02
+        .quad 0x18266e456,0x039d3b296
+        .quad 0x0daece73e,0x0083a6eec
+        .quad 0x0ab7aff2a,0x102f9b8a2
+        .quad 0x1248ea574,0x1c1733996
+        .quad 0x083348832,0x14237f5e6
+        .quad 0x12c743124,0x02ad91c30
+        .quad 0x0b9e02b86,0x00d3b6092
+        .quad 0x018b33a4e,0x06992cea2
+        .quad 0x1b331e26a,0x0c96cfdc0
+        .quad 0x17d35ba46,0x07e908048
+        .quad 0x1bf2e8b8a,0x18266e456
+        .quad 0x1a3e0968a,0x11ed1f9d8
+        .quad 0x0ce7f39f4,0x0daece73e
+        .quad 0x061d82e56,0x0f1d0f55e
+        .quad 0x0d270f1a2,0x0ab7aff2a
+        .quad 0x1c3f5f66c,0x0a87ab8a8
+        .quad 0x12ed0daac,0x1248ea574
+        .quad 0x065863b64,0x08462d800
+        .quad 0x11eef4f8e,0x083348832
+        .quad 0x1ee54f54c,0x071d111a8
+        .quad 0x0b3e32c28,0x12c743124
+        .quad 0x0064f7f26,0x0ffd852c6
+        .quad 0x0dd7e3b0c,0x0b9e02b86
+        .quad 0x0f285651c,0x0dcb17aa4
+        .quad 0x010746f3c,0x018b33a4e
+        .quad 0x1c24afea4,0x0f37c5aee
+        .quad 0x0271d9844,0x1b331e26a
+        .quad 0x08e766a0c,0x06051d5a2
+        .quad 0x093a5f730,0x17d35ba46
+        .quad 0x06cb08e5c,0x11d5ca20e
+        .quad 0x06b749fb2,0x1bf2e8b8a
+        .quad 0x1167f94f2,0x021f3d99c
+        .quad 0x0cec3662e,0x1a3e0968a
+        .quad 0x19329634a,0x08f158014
+        .quad 0x0e6fc4e6a,0x0ce7f39f4
+        .quad 0x08227bb8a,0x1a5e82106
+        .quad 0x0b0cd4768,0x061d82e56
+        .quad 0x13c2b89c4,0x188815ab2
+        .quad 0x0d7a4825c,0x0d270f1a2
+        .quad 0x10f5ff2ba,0x105405f3e
+        .quad 0x00167d312,0x1c3f5f66c
+        .quad 0x0f6076544,0x0e9adf796
+        .quad 0x026f6a60a,0x12ed0daac
+        .quad 0x1a2adb74e,0x096638b34
+        .quad 0x19d34af3a,0x065863b64
+        .quad 0x049c3cc9c,0x1e50585a0
+        .quad 0x068bce87a,0x11eef4f8e
+        .quad 0x1524fa6c6,0x19f1c69dc
+        .quad 0x16cba8aca,0x1ee54f54c
+        .quad 0x042d98888,0x12913343e
+        .quad 0x1329d9f7e,0x0b3e32c28
+        .quad 0x1b1c69528,0x088f25a3a
+        .quad 0x02178513a,0x0064f7f26
+        .quad 0x0e0ac139e,0x04e36f0b0
+        .quad 0x0170076fa,0x0dd7e3b0c
+        .quad 0x141a1a2e2,0x0bd6f81f8
+        .quad 0x16ad828b4,0x0f285651c
+        .quad 0x041d17b64,0x19425cbba
+        .quad 0x1fae1cc66,0x010746f3c
+        .quad 0x1a75b4b00,0x18db37e8a
+        .quad 0x0f872e54c,0x1c24afea4
+        .quad 0x01e41e9fc,0x04c144932
+        .quad 0x086d8e4d2,0x0271d9844
+        .quad 0x160f7af7a,0x052148f02
+        .quad 0x05bb8f1bc,0x08e766a0c
+        .quad 0x0a90fd27a,0x0a3c6f37a
+        .quad 0x0b3af077a,0x093a5f730
+        .quad 0x04984d782,0x1d22c238e
+        .quad 0x0ca6ef3ac,0x06cb08e5c
+        .quad 0x0234e0b26,0x063ded06a
+        .quad 0x1d88abd4a,0x06b749fb2
+        .quad 0x04597456a,0x04d56973c
+        .quad 0x0e9e28eb4,0x1167f94f2
+        .quad 0x07b3ff57a,0x19385bf2e
+        .quad 0x0c9c8b782,0x0cec3662e
+        .quad 0x13a9cba9e,0x0e417f38a
+        .quad 0x093e106a4,0x19329634a
+        .quad 0x167001a9c,0x14e727980
+        .quad 0x1ddffc5d4,0x0e6fc4e6a
+        .quad 0x00df04680,0x0d104b8fc
+        .quad 0x02342001e,0x08227bb8a
+        .quad 0x00a2a8d7e,0x05b397730
+        .quad 0x168763fa6,0x0b0cd4768
+        .quad 0x1ed5a407a,0x0e78eb416
+        .quad 0x0d2c3ed1a,0x13c2b89c4
+        .quad 0x0995a5724,0x1641378f0
+        .quad 0x19b1afbc4,0x0d7a4825c
+        .quad 0x109ffedc0,0x08d96551c
+        .quad 0x0f2271e60,0x10f5ff2ba
+        .quad 0x00b0bf8ca,0x00bf80dd2
+        .quad 0x123888b7a,0x00167d312
+        .quad 0x1e888f7dc,0x18dcddd1c
+        .quad 0x002ee03b2,0x0f6076544
+        .quad 0x183e8d8fe,0x06a45d2b2
+        .quad 0x133d7a042,0x026f6a60a
+        .quad 0x116b0f50c,0x1dd3e10e8
+        .quad 0x05fabe670,0x1a2adb74e
+        .quad 0x130004488,0x0de87806c
+        .quad 0x000bcf5f6,0x19d34af3a
+        .quad 0x18f0c7078,0x014338754
+        .quad 0x017f27698,0x049c3cc9c
+        .quad 0x058ca5f00,0x15e3e77ee
+        .quad 0x1af900c24,0x068bce87a
+        .quad 0x0b5cfca28,0x0dd07448e
+        .quad 0x0ded288f8,0x1524fa6c6
+        .quad 0x059f229bc,0x1d8048348
+        .quad 0x06d390dec,0x16cba8aca
+        .quad 0x037170390,0x0a3e3e02c
+        .quad 0x06353c1cc,0x042d98888
+        .quad 0x0c4584f5c,0x0d73c7bea
+        .quad 0x1f16a3418,0x1329d9f7e
+        .quad 0x0531377e2,0x185137662
+        .quad 0x1d8d9ca7c,0x1b1c69528
+        .quad 0x0b25b29f2,0x18a08b5bc
+        .quad 0x19fb2a8b0,0x02178513a
+        .quad 0x1a08fe6ac,0x1da758ae0
+        .quad 0x045cddf4e,0x0e0ac139e
+        .quad 0x1a91647f2,0x169cf9eb0
+        .quad 0x1a0f717c4,0x0170076fa
diff --git a/arch/x86/crypto/glue_helper-asm-avx.S b/arch/x86/crypto/glue_helper-asm-avx.S
new file mode 100644
index 000000000000..f7b6ea2ddfdb
--- /dev/null
+++ b/arch/x86/crypto/glue_helper-asm-avx.S
@@ -0,0 +1,91 @@
+/*
+ * Shared glue code for 128bit block ciphers, AVX assembler macros
+ *
+ * Copyright (c) 2012 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+#define load_8way(src, x0, x1, x2, x3, x4, x5, x6, x7) \
+        vmovdqu (0*16)(src), x0; \
+        vmovdqu (1*16)(src), x1; \
+        vmovdqu (2*16)(src), x2; \
+        vmovdqu (3*16)(src), x3; \
+        vmovdqu (4*16)(src), x4; \
+        vmovdqu (5*16)(src), x5; \
+        vmovdqu (6*16)(src), x6; \
+        vmovdqu (7*16)(src), x7;
+#define store_8way(dst, x0, x1, x2, x3, x4, x5, x6, x7) \
+        vmovdqu x0, (0*16)(dst); \
+        vmovdqu x1, (1*16)(dst); \
+        vmovdqu x2, (2*16)(dst); \
+        vmovdqu x3, (3*16)(dst); \
+        vmovdqu x4, (4*16)(dst); \
+        vmovdqu x5, (5*16)(dst); \
+        vmovdqu x6, (6*16)(dst); \
+        vmovdqu x7, (7*16)(dst);
+#define store_cbc_8way(src, dst, x0, x1, x2, x3, x4, x5, x6, x7) \
+        vpxor (0*16)(src), x1, x1; \
+        vpxor (1*16)(src), x2, x2; \
+        vpxor (2*16)(src), x3, x3; \
+        vpxor (3*16)(src), x4, x4; \
+        vpxor (4*16)(src), x5, x5; \
+        vpxor (5*16)(src), x6, x6; \
+        vpxor (6*16)(src), x7, x7; \
+        store_8way(dst, x0, x1, x2, x3, x4, x5, x6, x7);
+#define inc_le128(x, minus_one, tmp) \
+        vpcmpeqq minus_one, x, tmp; \
+        vpsubq minus_one, x, x; \
+        vpslldq $8, tmp, tmp; \
+        vpsubq tmp, x, x;
+#define load_ctr_8way(iv, bswap, x0, x1, x2, x3, x4, x5, x6, x7, t0, t1, t2) \
+        vpcmpeqd t0, t0, t0; \
+        vpsrldq $8, t0, t0; /* low: -1, high: 0 */ \
+        vmovdqa bswap, t1; \
+        \
+        /* load IV and byteswap */ \
+        vmovdqu (iv), x7; \
+        vpshufb t1, x7, x0; \
+        \
+        /* construct IVs */ \
+        inc_le128(x7, t0, t2); \
+        vpshufb t1, x7, x1; \
+        inc_le128(x7, t0, t2); \
+        vpshufb t1, x7, x2; \
+        inc_le128(x7, t0, t2); \
+        vpshufb t1, x7, x3; \
+        inc_le128(x7, t0, t2); \
+        vpshufb t1, x7, x4; \
+        inc_le128(x7, t0, t2); \
+        vpshufb t1, x7, x5; \
+        inc_le128(x7, t0, t2); \
+        vpshufb t1, x7, x6; \
+        inc_le128(x7, t0, t2); \
+        vmovdqa x7, t2; \
+        vpshufb t1, x7, x7; \
+        inc_le128(t2, t0, t1); \
+        vmovdqu t2, (iv);
+#define store_ctr_8way(src, dst, x0, x1, x2, x3, x4, x5, x6, x7) \
+        vpxor (0*16)(src), x0, x0; \
+        vpxor (1*16)(src), x1, x1; \
+        vpxor (2*16)(src), x2, x2; \
+        vpxor (3*16)(src), x3, x3; \
+        vpxor (4*16)(src), x4, x4; \
+        vpxor (5*16)(src), x5, x5; \
+        vpxor (6*16)(src), x6, x6; \
+        vpxor (7*16)(src), x7, x7; \
+        store_8way(dst, x0, x1, x2, x3, x4, x5, x6, x7);
diff --git a/arch/x86/crypto/glue_helper.c b/arch/x86/crypto/glue_helper.c
index 30b3927bd733..22ce4f683e55 100644
--- a/arch/x86/crypto/glue_helper.c
+++ b/arch/x86/crypto/glue_helper.c
@@ -221,16 +221,16 @@ static void glue_ctr_crypt_final_128bit(const common_glue_ctr_func_t fn_ctr,
        u8 *src = (u8 *)walk->src.virt.addr;
        u8 *dst = (u8 *)walk->dst.virt.addr;
        unsigned int nbytes = walk->nbytes;
-        u128 ctrblk;
+        le128 ctrblk;
        u128 tmp;
-        be128_to_u128(&ctrblk, (be128 *)walk->iv);
+        be128_to_le128(&ctrblk, (be128 *)walk->iv);
        memcpy(&tmp, src, nbytes);
        fn_ctr(ctx, &tmp, &tmp, &ctrblk);
        memcpy(dst, &tmp, nbytes);
-        u128_to_be128((be128 *)walk->iv, &ctrblk);
+        le128_to_be128((be128 *)walk->iv, &ctrblk);
 }
 EXPORT_SYMBOL_GPL(glue_ctr_crypt_final_128bit);
@@ -243,11 +243,11 @@ static unsigned int __glue_ctr_crypt_128bit(const struct common_glue_ctx *gctx,
        unsigned int nbytes = walk->nbytes;
        u128 *src = (u128 *)walk->src.virt.addr;
        u128 *dst = (u128 *)walk->dst.virt.addr;
-        u128 ctrblk;
+        le128 ctrblk;
        unsigned int num_blocks, func_bytes;
        unsigned int i;
-        be128_to_u128(&ctrblk, (be128 *)walk->iv);
+        be128_to_le128(&ctrblk, (be128 *)walk->iv);
        /* Process multi-block batch */
        for (i = 0; i < gctx->num_funcs; i++) {
@@ -269,7 +269,7 @@ static unsigned int __glue_ctr_crypt_128bit(const struct common_glue_ctx *gctx,
        }
 done:
-        u128_to_be128((be128 *)walk->iv, &ctrblk);
+        le128_to_be128((be128 *)walk->iv, &ctrblk);
        return nbytes;
 }
diff --git a/arch/x86/crypto/serpent-avx-x86_64-asm_64.S b/arch/x86/crypto/serpent-avx-x86_64-asm_64.S
index 504106bf04a2..02b0e9fe997c 100644
--- a/arch/x86/crypto/serpent-avx-x86_64-asm_64.S
+++ b/arch/x86/crypto/serpent-avx-x86_64-asm_64.S
@@ -24,7 +24,16 @@
 *
 */
+#include "glue_helper-asm-avx.S"
 .file "serpent-avx-x86_64-asm_64.S"
+.data
+.align 16
+.Lbswap128_mask:
+        .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
 .text
 #define CTX %rdi
@@ -550,51 +559,27 @@
        vpunpcklqdq             x3, t2, x2; \
        vpunpckhqdq             x3, t2, x3;
-#define read_blocks(in, x0, x1, x2, x3, t0, t1, t2) \
+#define read_blocks(x0, x1, x2, x3, t0, t1, t2) \
-        vmovdqu (0*4*4)(in),    x0; \
-        vmovdqu (1*4*4)(in),    x1; \
-        vmovdqu (2*4*4)(in),    x2; \
-        vmovdqu (3*4*4)(in),    x3; \
-        \
        transpose_4x4(x0, x1, x2, x3, t0, t1, t2)
-#define write_blocks(out, x0, x1, x2, x3, t0, t1, t2) \
+#define write_blocks(x0, x1, x2, x3, t0, t1, t2) \
-        transpose_4x4(x0, x1, x2, x3, t0, t1, t2) \
+        transpose_4x4(x0, x1, x2, x3, t0, t1, t2)
-        \
-        vmovdqu x0,             (0*4*4)(out); \
-        vmovdqu x1,             (1*4*4)(out); \
-        vmovdqu x2,             (2*4*4)(out); \
-        vmovdqu x3,             (3*4*4)(out);
-#define xor_blocks(out, x0, x1, x2, x3, t0, t1, t2) \
-        transpose_4x4(x0, x1, x2, x3, t0, t1, t2) \
-        \
-        vpxor (0*4*4)(out),     x0, x0;       \
-        vmovdqu x0,             (0*4*4)(out); \
-        vpxor (1*4*4)(out),     x1, x1;       \
-        vmovdqu x1,             (1*4*4)(out); \
-        vpxor (2*4*4)(out),     x2, x2;       \
-        vmovdqu x2,             (2*4*4)(out); \
-        vpxor (3*4*4)(out),     x3, x3;       \
-        vmovdqu x3,             (3*4*4)(out);
 .align 8
-.global __serpent_enc_blk_8way_avx
+.type   __serpent_enc_blk8_avx,@function;
-.type   __serpent_enc_blk_8way_avx,@function;
-__serpent_enc_blk_8way_avx:
+__serpent_enc_blk8_avx:
        /* input:
         *      %rdi: ctx, CTX
-         *      %rsi: dst
+         *      RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2: blocks
-         *      %rdx: src
+         * output:
-         *      %rcx: bool, if true: xor output
+         *      RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2: encrypted blocks
         */
        vpcmpeqd RNOT, RNOT, RNOT;
-        leaq (4*4*4)(%rdx), %rax;
+        read_blocks(RA1, RB1, RC1, RD1, RK0, RK1, RK2);
-        read_blocks(%rdx, RA1, RB1, RC1, RD1, RK0, RK1, RK2);
+        read_blocks(RA2, RB2, RC2, RD2, RK0, RK1, RK2);
-        read_blocks(%rax, RA2, RB2, RC2, RD2, RK0, RK1, RK2);
                                                 K2(RA, RB, RC, RD, RE, 0);
        S(S0, RA, RB, RC, RD, RE);              LK2(RC, RB, RD, RA, RE, 1);
@@ -630,38 +615,26 @@ __serpent_enc_blk_8way_avx:
        S(S6, RA, RB, RD, RC, RE);              LK2(RD, RE, RB, RC, RA, 31);
        S(S7, RD, RE, RB, RC, RA);               K2(RA, RB, RC, RD, RE, 32);
-        leaq (4*4*4)(%rsi), %rax;
+        write_blocks(RA1, RB1, RC1, RD1, RK0, RK1, RK2);
+        write_blocks(RA2, RB2, RC2, RD2, RK0, RK1, RK2);
-        testb %cl, %cl;
-        jnz __enc_xor8;
-        write_blocks(%rsi, RA1, RB1, RC1, RD1, RK0, RK1, RK2);
-        write_blocks(%rax, RA2, RB2, RC2, RD2, RK0, RK1, RK2);
-        ret;
-__enc_xor8:
-        xor_blocks(%rsi, RA1, RB1, RC1, RD1, RK0, RK1, RK2);
-        xor_blocks(%rax, RA2, RB2, RC2, RD2, RK0, RK1, RK2);
        ret;
 .align 8
-.global serpent_dec_blk_8way_avx
+.type   __serpent_dec_blk8_avx,@function;
-.type   serpent_dec_blk_8way_avx,@function;
-serpent_dec_blk_8way_avx:
+__serpent_dec_blk8_avx:
        /* input:
         *      %rdi: ctx, CTX
-         *      %rsi: dst
+         *      RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2: encrypted blocks
-         *      %rdx: src
+         * output:
+         *      RC1, RD1, RB1, RE1, RC2, RD2, RB2, RE2: decrypted blocks
         */
        vpcmpeqd RNOT, RNOT, RNOT;
-        leaq (4*4*4)(%rdx), %rax;
+        read_blocks(RA1, RB1, RC1, RD1, RK0, RK1, RK2);
-        read_blocks(%rdx, RA1, RB1, RC1, RD1, RK0, RK1, RK2);
+        read_blocks(RA2, RB2, RC2, RD2, RK0, RK1, RK2);
-        read_blocks(%rax, RA2, RB2, RC2, RD2, RK0, RK1, RK2);
                                                 K2(RA, RB, RC, RD, RE, 32);
        SP(SI7, RA, RB, RC, RD, RE, 31);        KL2(RB, RD, RA, RE, RC, 31);
@@ -697,8 +670,85 @@ serpent_dec_blk_8way_avx:
        SP(SI1, RD, RB, RC, RA, RE, 1);         KL2(RE, RB, RC, RA, RD, 1);
        S(SI0, RE, RB, RC, RA, RD);              K2(RC, RD, RB, RE, RA, 0);
-        leaq (4*4*4)(%rsi), %rax;
+        write_blocks(RC1, RD1, RB1, RE1, RK0, RK1, RK2);
-        write_blocks(%rsi, RC1, RD1, RB1, RE1, RK0, RK1, RK2);
+        write_blocks(RC2, RD2, RB2, RE2, RK0, RK1, RK2);
-        write_blocks(%rax, RC2, RD2, RB2, RE2, RK0, RK1, RK2);
+        ret;
+.align 8
+.global serpent_ecb_enc_8way_avx
+.type   serpent_ecb_enc_8way_avx,@function;
+serpent_ecb_enc_8way_avx:
+        /* input:
+         *      %rdi: ctx, CTX
+         *      %rsi: dst
+         *      %rdx: src
+         */
+        load_8way(%rdx, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
+        call __serpent_enc_blk8_avx;
+        store_8way(%rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
+        ret;
+.align 8
+.global serpent_ecb_dec_8way_avx
+.type   serpent_ecb_dec_8way_avx,@function;
+serpent_ecb_dec_8way_avx:
+        /* input:
+         *      %rdi: ctx, CTX
+         *      %rsi: dst
+         *      %rdx: src
+         */
+        load_8way(%rdx, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
+        call __serpent_dec_blk8_avx;
+        store_8way(%rsi, RC1, RD1, RB1, RE1, RC2, RD2, RB2, RE2);
+        ret;
+.align 8
+.global serpent_cbc_dec_8way_avx
+.type   serpent_cbc_dec_8way_avx,@function;
+serpent_cbc_dec_8way_avx:
+        /* input:
+         *      %rdi: ctx, CTX
+         *      %rsi: dst
+         *      %rdx: src
+         */
+        load_8way(%rdx, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
+        call __serpent_dec_blk8_avx;
+        store_cbc_8way(%rdx, %rsi, RC1, RD1, RB1, RE1, RC2, RD2, RB2, RE2);
+        ret;
+.align 8
+.global serpent_ctr_8way_avx
+.type   serpent_ctr_8way_avx,@function;
+serpent_ctr_8way_avx:
+        /* input:
+         *      %rdi: ctx, CTX
+         *      %rsi: dst
+         *      %rdx: src
+         *      %rcx: iv (little endian, 128bit)
+         */
+        load_ctr_8way(%rcx, .Lbswap128_mask, RA1, RB1, RC1, RD1, RA2, RB2, RC2,
+                      RD2, RK0, RK1, RK2);
+        call __serpent_enc_blk8_avx;
+        store_ctr_8way(%rdx, %rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
        ret;
diff --git a/arch/x86/crypto/serpent_avx_glue.c b/arch/x86/crypto/serpent_avx_glue.c
index 3f543a04cf1e..52abaaf28e7f 100644
--- a/arch/x86/crypto/serpent_avx_glue.c
+++ b/arch/x86/crypto/serpent_avx_glue.c
@@ -42,55 +42,24 @@
 #include <asm/crypto/ablk_helper.h>
 #include <asm/crypto/glue_helper.h>
-static void serpent_decrypt_cbc_xway(void *ctx, u128 *dst, const u128 *src)
+static void serpent_crypt_ctr(void *ctx, u128 *dst, const u128 *src, le128 *iv)
-{
-        u128 ivs[SERPENT_PARALLEL_BLOCKS - 1];
-        unsigned int j;
-        for (j = 0; j < SERPENT_PARALLEL_BLOCKS - 1; j++)
-                ivs[j] = src[j];
-        serpent_dec_blk_xway(ctx, (u8 *)dst, (u8 *)src);
-        for (j = 0; j < SERPENT_PARALLEL_BLOCKS - 1; j++)
-                u128_xor(dst + (j + 1), dst + (j + 1), ivs + j);
-}
-static void serpent_crypt_ctr(void *ctx, u128 *dst, const u128 *src, u128 *iv)
 {
        be128 ctrblk;
-        u128_to_be128(&ctrblk, iv);
+        le128_to_be128(&ctrblk, iv);
-        u128_inc(iv);
+        le128_inc(iv);
        __serpent_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk);
        u128_xor(dst, src, (u128 *)&ctrblk);
 }
-static void serpent_crypt_ctr_xway(void *ctx, u128 *dst, const u128 *src,
-                                   u128 *iv)
-{
-        be128 ctrblks[SERPENT_PARALLEL_BLOCKS];
-        unsigned int i;
-        for (i = 0; i < SERPENT_PARALLEL_BLOCKS; i++) {
-                if (dst != src)
-                        dst[i] = src[i];
-                u128_to_be128(&ctrblks[i], iv);
-                u128_inc(iv);
-        }
-        serpent_enc_blk_xway_xor(ctx, (u8 *)dst, (u8 *)ctrblks);
-}
 static const struct common_glue_ctx serpent_enc = {
        .num_funcs = 2,
        .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
        .funcs = { {
                .num_blocks = SERPENT_PARALLEL_BLOCKS,
-                .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_enc_blk_xway) }
+                .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_enc_8way_avx) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_encrypt) }
@@ -103,7 +72,7 @@ static const struct common_glue_ctx serpent_ctr = {
        .funcs = { {
                .num_blocks = SERPENT_PARALLEL_BLOCKS,
-                .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_crypt_ctr_xway) }
+                .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_ctr_8way_avx) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_crypt_ctr) }
@@ -116,7 +85,7 @@ static const struct common_glue_ctx serpent_dec = {
        .funcs = { {
                .num_blocks = SERPENT_PARALLEL_BLOCKS,
-                .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_dec_blk_xway) }
+                .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_dec_8way_avx) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_decrypt) }
@@ -129,7 +98,7 @@ static const struct common_glue_ctx serpent_dec_cbc = {
        .funcs = { {
                .num_blocks = SERPENT_PARALLEL_BLOCKS,
-                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(serpent_decrypt_cbc_xway) }
+                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(serpent_cbc_dec_8way_avx) }
        }, {
                .num_blocks = 1,
                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(__serpent_decrypt) }
@@ -193,7 +162,7 @@ static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
        ctx->fpu_enabled = serpent_fpu_begin(ctx->fpu_enabled, nbytes);
        if (nbytes == bsize * SERPENT_PARALLEL_BLOCKS) {
-                serpent_enc_blk_xway(ctx->ctx, srcdst, srcdst);
+                serpent_ecb_enc_8way_avx(ctx->ctx, srcdst, srcdst);
                return;
        }
@@ -210,7 +179,7 @@ static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
        ctx->fpu_enabled = serpent_fpu_begin(ctx->fpu_enabled, nbytes);
        if (nbytes == bsize * SERPENT_PARALLEL_BLOCKS) {
-                serpent_dec_blk_xway(ctx->ctx, srcdst, srcdst);
+                serpent_ecb_dec_8way_avx(ctx->ctx, srcdst, srcdst);
                return;
        }
diff --git a/arch/x86/crypto/serpent_sse2_glue.c b/arch/x86/crypto/serpent_sse2_glue.c
index 9107a9908c41..97a356ece24d 100644
--- a/arch/x86/crypto/serpent_sse2_glue.c
+++ b/arch/x86/crypto/serpent_sse2_glue.c
@@ -59,19 +59,19 @@ static void serpent_decrypt_cbc_xway(void *ctx, u128 *dst, const u128 *src)
                u128_xor(dst + (j + 1), dst + (j + 1), ivs + j);
 }
-static void serpent_crypt_ctr(void *ctx, u128 *dst, const u128 *src, u128 *iv)
+static void serpent_crypt_ctr(void *ctx, u128 *dst, const u128 *src, le128 *iv)
 {
        be128 ctrblk;
-        u128_to_be128(&ctrblk, iv);
+        le128_to_be128(&ctrblk, iv);
-        u128_inc(iv);
+        le128_inc(iv);
        __serpent_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk);
        u128_xor(dst, src, (u128 *)&ctrblk);
 }
 static void serpent_crypt_ctr_xway(void *ctx, u128 *dst, const u128 *src,
-                                   u128 *iv)
+                                   le128 *iv)
 {
        be128 ctrblks[SERPENT_PARALLEL_BLOCKS];
        unsigned int i;
@@ -80,8 +80,8 @@ static void serpent_crypt_ctr_xway(void *ctx, u128 *dst, const u128 *src,
                if (dst != src)
                        dst[i] = src[i];
-                u128_to_be128(&ctrblks[i], iv);
+                le128_to_be128(&ctrblks[i], iv);
-                u128_inc(iv);
+                le128_inc(iv);
        }
        serpent_enc_blk_xway_xor(ctx, (u8 *)dst, (u8 *)ctrblks);
diff --git a/arch/x86/crypto/twofish-avx-x86_64-asm_64.S b/arch/x86/crypto/twofish-avx-x86_64-asm_64.S
index 1585abb13dde..ebac16bfa830 100644
--- a/arch/x86/crypto/twofish-avx-x86_64-asm_64.S
+++ b/arch/x86/crypto/twofish-avx-x86_64-asm_64.S
@@ -23,7 +23,16 @@
 *
 */
+#include "glue_helper-asm-avx.S"
 .file "twofish-avx-x86_64-asm_64.S"
+.data
+.align 16
+.Lbswap128_mask:
+        .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
 .text
 /* structure of crypto context */
@@ -217,69 +226,45 @@
        vpunpcklqdq             x3, t2, x2; \
        vpunpckhqdq             x3, t2, x3;
-#define inpack_blocks(in, x0, x1, x2, x3, wkey, t0, t1, t2) \
+#define inpack_blocks(x0, x1, x2, x3, wkey, t0, t1, t2) \
-        vpxor (0*4*4)(in),      wkey, x0; \
+        vpxor           x0, wkey, x0; \
-        vpxor (1*4*4)(in),      wkey, x1; \
+        vpxor           x1, wkey, x1; \
-        vpxor (2*4*4)(in),      wkey, x2; \
+        vpxor           x2, wkey, x2; \
-        vpxor (3*4*4)(in),      wkey, x3; \
+        vpxor           x3, wkey, x3; \
        \
        transpose_4x4(x0, x1, x2, x3, t0, t1, t2)
-#define outunpack_blocks(out, x0, x1, x2, x3, wkey, t0, t1, t2) \
+#define outunpack_blocks(x0, x1, x2, x3, wkey, t0, t1, t2) \
-        transpose_4x4(x0, x1, x2, x3, t0, t1, t2) \
-        \
-        vpxor           x0, wkey, x0;     \
-        vmovdqu         x0, (0*4*4)(out); \
-        vpxor           x1, wkey, x1;     \
-        vmovdqu         x1, (1*4*4)(out); \
-        vpxor           x2, wkey, x2;     \
-        vmovdqu         x2, (2*4*4)(out); \
-        vpxor           x3, wkey, x3;     \
-        vmovdqu         x3, (3*4*4)(out);
-#define outunpack_xor_blocks(out, x0, x1, x2, x3, wkey, t0, t1, t2) \
        transpose_4x4(x0, x1, x2, x3, t0, t1, t2) \
        \
-        vpxor           x0, wkey, x0;         \
+        vpxor           x0, wkey, x0; \
-        vpxor           (0*4*4)(out), x0, x0; \
+        vpxor           x1, wkey, x1; \
-        vmovdqu         x0, (0*4*4)(out);     \
+        vpxor           x2, wkey, x2; \
-        vpxor           x1, wkey, x1;         \
+        vpxor           x3, wkey, x3;
-        vpxor           (1*4*4)(out), x1, x1; \
-        vmovdqu         x1, (1*4*4)(out);     \
-        vpxor           x2, wkey, x2;         \
-        vpxor           (2*4*4)(out), x2, x2; \
-        vmovdqu         x2, (2*4*4)(out);     \
-        vpxor           x3, wkey, x3;         \
-        vpxor           (3*4*4)(out), x3, x3; \
-        vmovdqu         x3, (3*4*4)(out);
 .align 8
-.global __twofish_enc_blk_8way
+.type   __twofish_enc_blk8,@function;
-.type   __twofish_enc_blk_8way,@function;
-__twofish_enc_blk_8way:
+__twofish_enc_blk8:
        /* input:
         *      %rdi: ctx, CTX
-         *      %rsi: dst
+         *      RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2: blocks
-         *      %rdx: src
+         * output:
-         *      %rcx: bool, if true: xor output
+         *      RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2: encrypted blocks
         */
+        vmovdqu w(CTX), RK1;
        pushq %rbp;
        pushq %rbx;
        pushq %rcx;
-        vmovdqu w(CTX), RK1;
+        inpack_blocks(RA1, RB1, RC1, RD1, RK1, RX0, RY0, RK2);
-        leaq (4*4*4)(%rdx), %rax;
-        inpack_blocks(%rdx, RA1, RB1, RC1, RD1, RK1, RX0, RY0, RK2);
        preload_rgi(RA1);
        rotate_1l(RD1);
-        inpack_blocks(%rax, RA2, RB2, RC2, RD2, RK1, RX0, RY0, RK2);
+        inpack_blocks(RA2, RB2, RC2, RD2, RK1, RX0, RY0, RK2);
        rotate_1l(RD2);
-        movq %rsi, %r11;
        encrypt_cycle(0);
        encrypt_cycle(1);
        encrypt_cycle(2);
@@ -295,47 +280,33 @@ __twofish_enc_blk_8way:
        popq %rbx;
        popq %rbp;
-        leaq (4*4*4)(%r11), %rax;
+        outunpack_blocks(RC1, RD1, RA1, RB1, RK1, RX0, RY0, RK2);
+        outunpack_blocks(RC2, RD2, RA2, RB2, RK1, RX0, RY0, RK2);
-        testb %cl, %cl;
-        jnz __enc_xor8;
-        outunpack_blocks(%r11, RC1, RD1, RA1, RB1, RK1, RX0, RY0, RK2);
-        outunpack_blocks(%rax, RC2, RD2, RA2, RB2, RK1, RX0, RY0, RK2);
-        ret;
-__enc_xor8:
-        outunpack_xor_blocks(%r11, RC1, RD1, RA1, RB1, RK1, RX0, RY0, RK2);
-        outunpack_xor_blocks(%rax, RC2, RD2, RA2, RB2, RK1, RX0, RY0, RK2);
        ret;
 .align 8
-.global twofish_dec_blk_8way
+.type   __twofish_dec_blk8,@function;
-.type   twofish_dec_blk_8way,@function;
-twofish_dec_blk_8way:
+__twofish_dec_blk8:
        /* input:
         *      %rdi: ctx, CTX
-         *      %rsi: dst
+         *      RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2: encrypted blocks
-         *      %rdx: src
+         * output:
+         *      RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2: decrypted blocks
         */
+        vmovdqu (w+4*4)(CTX), RK1;
        pushq %rbp;
        pushq %rbx;
-        vmovdqu (w+4*4)(CTX), RK1;
+        inpack_blocks(RC1, RD1, RA1, RB1, RK1, RX0, RY0, RK2);
-        leaq (4*4*4)(%rdx), %rax;
-        inpack_blocks(%rdx, RC1, RD1, RA1, RB1, RK1, RX0, RY0, RK2);
        preload_rgi(RC1);
        rotate_1l(RA1);
-        inpack_blocks(%rax, RC2, RD2, RA2, RB2, RK1, RX0, RY0, RK2);
+        inpack_blocks(RC2, RD2, RA2, RB2, RK1, RX0, RY0, RK2);
        rotate_1l(RA2);
-        movq %rsi, %r11;
        decrypt_cycle(7);
        decrypt_cycle(6);
        decrypt_cycle(5);
@@ -350,8 +321,103 @@ twofish_dec_blk_8way:
        popq %rbx;
        popq %rbp;
-        leaq (4*4*4)(%r11), %rax;
+        outunpack_blocks(RA1, RB1, RC1, RD1, RK1, RX0, RY0, RK2);
-        outunpack_blocks(%r11, RA1, RB1, RC1, RD1, RK1, RX0, RY0, RK2);
+        outunpack_blocks(RA2, RB2, RC2, RD2, RK1, RX0, RY0, RK2);
-        outunpack_blocks(%rax, RA2, RB2, RC2, RD2, RK1, RX0, RY0, RK2);
+        ret;
+.align 8
+.global twofish_ecb_enc_8way
+.type   twofish_ecb_enc_8way,@function;
+twofish_ecb_enc_8way:
+        /* input:
+         *      %rdi: ctx, CTX
+         *      %rsi: dst
+         *      %rdx: src
+         */
+        movq %rsi, %r11;
+        load_8way(%rdx, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
+        call __twofish_enc_blk8;
+        store_8way(%r11, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2);
+        ret;
+.align 8
+.global twofish_ecb_dec_8way
+.type   twofish_ecb_dec_8way,@function;
+twofish_ecb_dec_8way:
+        /* input:
+         *      %rdi: ctx, CTX
+         *      %rsi: dst
+         *      %rdx: src
+         */
+        movq %rsi, %r11;
+        load_8way(%rdx, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2);
+        call __twofish_dec_blk8;
+        store_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
+        ret;
+.align 8
+.global twofish_cbc_dec_8way
+.type   twofish_cbc_dec_8way,@function;
+twofish_cbc_dec_8way:
+        /* input:
+         *      %rdi: ctx, CTX
+         *      %rsi: dst
+         *      %rdx: src
+         */
+        pushq %r12;
+        movq %rsi, %r11;
+        movq %rdx, %r12;
+        load_8way(%rdx, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2);
+        call __twofish_dec_blk8;
+        store_cbc_8way(%r12, %r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
+        popq %r12;
+        ret;
+.align 8
+.global twofish_ctr_8way
+.type   twofish_ctr_8way,@function;
+twofish_ctr_8way:
+        /* input:
+         *      %rdi: ctx, CTX
+         *      %rsi: dst
+         *      %rdx: src
+         *      %rcx: iv (little endian, 128bit)
+         */
+        pushq %r12;
+        movq %rsi, %r11;
+        movq %rdx, %r12;
+        load_ctr_8way(%rcx, .Lbswap128_mask, RA1, RB1, RC1, RD1, RA2, RB2, RC2,
+                      RD2, RX0, RX1, RY0);
+        call __twofish_enc_blk8;
+        store_ctr_8way(%r12, %r11, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2);
+        popq %r12;
        ret;
diff --git a/arch/x86/crypto/twofish_avx_glue.c b/arch/x86/crypto/twofish_avx_glue.c
index e7708b5442e0..94ac91d26e47 100644
--- a/arch/x86/crypto/twofish_avx_glue.c
+++ b/arch/x86/crypto/twofish_avx_glue.c
@@ -45,66 +45,23 @@
 #define TWOFISH_PARALLEL_BLOCKS 8
-static inline void twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst,
-                                        const u8 *src)
-{
-        __twofish_enc_blk_3way(ctx, dst, src, false);
-}
 /* 8-way parallel cipher functions */
-asmlinkage void __twofish_enc_blk_8way(struct twofish_ctx *ctx, u8 *dst,
+asmlinkage void twofish_ecb_enc_8way(struct twofish_ctx *ctx, u8 *dst,
-                                       const u8 *src, bool xor);
+                                     const u8 *src);
-asmlinkage void twofish_dec_blk_8way(struct twofish_ctx *ctx, u8 *dst,
+asmlinkage void twofish_ecb_dec_8way(struct twofish_ctx *ctx, u8 *dst,
                                     const u8 *src);
-static inline void twofish_enc_blk_xway(struct twofish_ctx *ctx, u8 *dst,
+asmlinkage void twofish_cbc_dec_8way(struct twofish_ctx *ctx, u8 *dst,
-                                        const u8 *src)
+                                     const u8 *src);
-{
+asmlinkage void twofish_ctr_8way(struct twofish_ctx *ctx, u8 *dst,
-        __twofish_enc_blk_8way(ctx, dst, src, false);
+                                 const u8 *src, le128 *iv);
-}
-static inline void twofish_enc_blk_xway_xor(struct twofish_ctx *ctx, u8 *dst,
-                                            const u8 *src)
-{
-        __twofish_enc_blk_8way(ctx, dst, src, true);
-}
-static inline void twofish_dec_blk_xway(struct twofish_ctx *ctx, u8 *dst,
+static inline void twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst,
                                        const u8 *src)
 {
-        twofish_dec_blk_8way(ctx, dst, src);
+        __twofish_enc_blk_3way(ctx, dst, src, false);
-}
-static void twofish_dec_blk_cbc_xway(void *ctx, u128 *dst, const u128 *src)
-{
-        u128 ivs[TWOFISH_PARALLEL_BLOCKS - 1];
-        unsigned int j;
-        for (j = 0; j < TWOFISH_PARALLEL_BLOCKS - 1; j++)
-                ivs[j] = src[j];
-        twofish_dec_blk_xway(ctx, (u8 *)dst, (u8 *)src);
-        for (j = 0; j < TWOFISH_PARALLEL_BLOCKS - 1; j++)
-                u128_xor(dst + (j + 1), dst + (j + 1), ivs + j);
 }
-static void twofish_enc_blk_ctr_xway(void *ctx, u128 *dst, const u128 *src,
-                                     u128 *iv)
-{
-        be128 ctrblks[TWOFISH_PARALLEL_BLOCKS];
-        unsigned int i;
-        for (i = 0; i < TWOFISH_PARALLEL_BLOCKS; i++) {
-                if (dst != src)
-                        dst[i] = src[i];
-                u128_to_be128(&ctrblks[i], iv);
-                u128_inc(iv);
-        }
-        twofish_enc_blk_xway_xor(ctx, (u8 *)dst, (u8 *)ctrblks);
-}
 static const struct common_glue_ctx twofish_enc = {
        .num_funcs = 3,
@@ -112,7 +69,7 @@ static const struct common_glue_ctx twofish_enc = {
        .funcs = { {
                .num_blocks = TWOFISH_PARALLEL_BLOCKS,
-                .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_xway) }
+                .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_enc_8way) }
        }, {
                .num_blocks = 3,
                .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_3way) }
@@ -128,7 +85,7 @@ static const struct common_glue_ctx twofish_ctr = {
        .funcs = { {
                .num_blocks = TWOFISH_PARALLEL_BLOCKS,
-                .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr_xway) }
+                .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_ctr_8way) }
        }, {
                .num_blocks = 3,
                .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr_3way) }
@@ -144,7 +101,7 @@ static const struct common_glue_ctx twofish_dec = {
        .funcs = { {
                .num_blocks = TWOFISH_PARALLEL_BLOCKS,
-                .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk_xway) }
+                .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_dec_8way) }
        }, {
                .num_blocks = 3,
                .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk_3way) }
@@ -160,7 +117,7 @@ static const struct common_glue_ctx twofish_dec_cbc = {
        .funcs = { {
                .num_blocks = TWOFISH_PARALLEL_BLOCKS,
-                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk_cbc_xway) }
+                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_cbc_dec_8way) }
        }, {
                .num_blocks = 3,
                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk_cbc_3way) }
@@ -227,7 +184,7 @@ static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
        ctx->fpu_enabled = twofish_fpu_begin(ctx->fpu_enabled, nbytes);
        if (nbytes == bsize * TWOFISH_PARALLEL_BLOCKS) {
-                twofish_enc_blk_xway(ctx->ctx, srcdst, srcdst);
+                twofish_ecb_enc_8way(ctx->ctx, srcdst, srcdst);
                return;
        }
@@ -249,7 +206,7 @@ static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
        ctx->fpu_enabled = twofish_fpu_begin(ctx->fpu_enabled, nbytes);
        if (nbytes == bsize * TWOFISH_PARALLEL_BLOCKS) {
-                twofish_dec_blk_xway(ctx->ctx, srcdst, srcdst);
+                twofish_ecb_dec_8way(ctx->ctx, srcdst, srcdst);
                return;
        }
diff --git a/arch/x86/crypto/twofish_glue_3way.c b/arch/x86/crypto/twofish_glue_3way.c
index aa3eb358b7e8..13e63b3e1dfb 100644
--- a/arch/x86/crypto/twofish_glue_3way.c
+++ b/arch/x86/crypto/twofish_glue_3way.c
@@ -62,15 +62,15 @@ void twofish_dec_blk_cbc_3way(void *ctx, u128 *dst, const u128 *src)
 }
 EXPORT_SYMBOL_GPL(twofish_dec_blk_cbc_3way);
-void twofish_enc_blk_ctr(void *ctx, u128 *dst, const u128 *src, u128 *iv)
+void twofish_enc_blk_ctr(void *ctx, u128 *dst, const u128 *src, le128 *iv)
 {
        be128 ctrblk;
        if (dst != src)
                *dst = *src;
-        u128_to_be128(&ctrblk, iv);
+        le128_to_be128(&ctrblk, iv);
-        u128_inc(iv);
+        le128_inc(iv);
        twofish_enc_blk(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk);
        u128_xor(dst, dst, (u128 *)&ctrblk);
@@ -78,7 +78,7 @@ void twofish_enc_blk_ctr(void *ctx, u128 *dst, const u128 *src, u128 *iv)
 EXPORT_SYMBOL_GPL(twofish_enc_blk_ctr);
 void twofish_enc_blk_ctr_3way(void *ctx, u128 *dst, const u128 *src,
-                                     u128 *iv)
+                              le128 *iv)
 {
        be128 ctrblks[3];
@@ -88,12 +88,12 @@ void twofish_enc_blk_ctr_3way(void *ctx, u128 *dst, const u128 *src,
                dst[2] = src[2];
        }
-        u128_to_be128(&ctrblks[0], iv);
+        le128_to_be128(&ctrblks[0], iv);
-        u128_inc(iv);
+        le128_inc(iv);
-        u128_to_be128(&ctrblks[1], iv);
+        le128_to_be128(&ctrblks[1], iv);
-        u128_inc(iv);
+        le128_inc(iv);
-        u128_to_be128(&ctrblks[2], iv);
+        le128_to_be128(&ctrblks[2], iv);
-        u128_inc(iv);
+        le128_inc(iv);
        twofish_enc_blk_xor_3way(ctx, (u8 *)dst, (u8 *)ctrblks);
 }
diff --git a/arch/x86/include/asm/crypto/camellia.h b/arch/x86/include/asm/crypto/camellia.h
new file mode 100644
index 000000000000..98038add801e
--- /dev/null
+++ b/arch/x86/include/asm/crypto/camellia.h
@@ -0,0 +1,82 @@
+#ifndef ASM_X86_CAMELLIA_H
+#define ASM_X86_CAMELLIA_H
+#include <linux/kernel.h>
+#include <linux/crypto.h>
+#define CAMELLIA_MIN_KEY_SIZE   16
+#define CAMELLIA_MAX_KEY_SIZE   32
+#define CAMELLIA_BLOCK_SIZE     16
+#define CAMELLIA_TABLE_BYTE_LEN 272
+#define CAMELLIA_PARALLEL_BLOCKS 2
+struct camellia_ctx {
+        u64 key_table[CAMELLIA_TABLE_BYTE_LEN / sizeof(u64)];
+        u32 key_length;
+};
+struct camellia_lrw_ctx {
+        struct lrw_table_ctx lrw_table;
+        struct camellia_ctx camellia_ctx;
+};
+struct camellia_xts_ctx {
+        struct camellia_ctx tweak_ctx;
+        struct camellia_ctx crypt_ctx;
+};
+extern int __camellia_setkey(struct camellia_ctx *cctx,
+                             const unsigned char *key,
+                             unsigned int key_len, u32 *flags);
+extern int lrw_camellia_setkey(struct crypto_tfm *tfm, const u8 *key,
+                               unsigned int keylen);
+extern void lrw_camellia_exit_tfm(struct crypto_tfm *tfm);
+extern int xts_camellia_setkey(struct crypto_tfm *tfm, const u8 *key,
+                               unsigned int keylen);
+/* regular block cipher functions */
+asmlinkage void __camellia_enc_blk(struct camellia_ctx *ctx, u8 *dst,
+                                   const u8 *src, bool xor);
+asmlinkage void camellia_dec_blk(struct camellia_ctx *ctx, u8 *dst,
+                                 const u8 *src);
+/* 2-way parallel cipher functions */
+asmlinkage void __camellia_enc_blk_2way(struct camellia_ctx *ctx, u8 *dst,
+                                        const u8 *src, bool xor);
+asmlinkage void camellia_dec_blk_2way(struct camellia_ctx *ctx, u8 *dst,
+                                      const u8 *src);
+static inline void camellia_enc_blk(struct camellia_ctx *ctx, u8 *dst,
+                                    const u8 *src)
+{
+        __camellia_enc_blk(ctx, dst, src, false);
+}
+static inline void camellia_enc_blk_xor(struct camellia_ctx *ctx, u8 *dst,
+                                        const u8 *src)
+{
+        __camellia_enc_blk(ctx, dst, src, true);
+}
+static inline void camellia_enc_blk_2way(struct camellia_ctx *ctx, u8 *dst,
+                                         const u8 *src)
+{
+        __camellia_enc_blk_2way(ctx, dst, src, false);
+}
+static inline void camellia_enc_blk_xor_2way(struct camellia_ctx *ctx, u8 *dst,
+                                             const u8 *src)
+{
+        __camellia_enc_blk_2way(ctx, dst, src, true);
+}
+/* glue helpers */
+extern void camellia_decrypt_cbc_2way(void *ctx, u128 *dst, const u128 *src);
+extern void camellia_crypt_ctr(void *ctx, u128 *dst, const u128 *src,
+                               le128 *iv);
+extern void camellia_crypt_ctr_2way(void *ctx, u128 *dst, const u128 *src,
+                                    le128 *iv);
+#endif /* ASM_X86_CAMELLIA_H */
diff --git a/arch/x86/include/asm/crypto/glue_helper.h b/arch/x86/include/asm/crypto/glue_helper.h
index 3e408bddc96f..e2d65b061d27 100644
--- a/arch/x86/include/asm/crypto/glue_helper.h
+++ b/arch/x86/include/asm/crypto/glue_helper.h
@@ -13,7 +13,7 @@
 typedef void (*common_glue_func_t)(void *ctx, u8 *dst, const u8 *src);
 typedef void (*common_glue_cbc_func_t)(void *ctx, u128 *dst, const u128 *src);
 typedef void (*common_glue_ctr_func_t)(void *ctx, u128 *dst, const u128 *src,
-                                       u128 *iv);
+                                       le128 *iv);
 #define GLUE_FUNC_CAST(fn) ((common_glue_func_t)(fn))
 #define GLUE_CBC_FUNC_CAST(fn) ((common_glue_cbc_func_t)(fn))
@@ -71,23 +71,29 @@ static inline void glue_fpu_end(bool fpu_enabled)
                kernel_fpu_end();
 }
-static inline void u128_to_be128(be128 *dst, const u128 *src)
+static inline void le128_to_be128(be128 *dst, const le128 *src)
 {
-        dst->a = cpu_to_be64(src->a);
+        dst->a = cpu_to_be64(le64_to_cpu(src->a));
-        dst->b = cpu_to_be64(src->b);
+        dst->b = cpu_to_be64(le64_to_cpu(src->b));
 }
-static inline void be128_to_u128(u128 *dst, const be128 *src)
+static inline void be128_to_le128(le128 *dst, const be128 *src)
 {
-        dst->a = be64_to_cpu(src->a);
+        dst->a = cpu_to_le64(be64_to_cpu(src->a));
-        dst->b = be64_to_cpu(src->b);
+        dst->b = cpu_to_le64(be64_to_cpu(src->b));
 }
-static inline void u128_inc(u128 *i)
+static inline void le128_inc(le128 *i)
 {
-        i->b++;
+        u64 a = le64_to_cpu(i->a);
-        if (!i->b)
+        u64 b = le64_to_cpu(i->b);
-                i->a++;
+        b++;
+        if (!b)
+                a++;
+        i->a = cpu_to_le64(a);
+        i->b = cpu_to_le64(b);
 }
 extern int glue_ecb_crypt_128bit(const struct common_glue_ctx *gctx,
diff --git a/arch/x86/include/asm/crypto/serpent-avx.h b/arch/x86/include/asm/crypto/serpent-avx.h
index 432deedd2945..0da1d3e2a55c 100644
--- a/arch/x86/include/asm/crypto/serpent-avx.h
+++ b/arch/x86/include/asm/crypto/serpent-avx.h
@@ -6,27 +6,14 @@
 #define SERPENT_PARALLEL_BLOCKS 8
-asmlinkage void __serpent_enc_blk_8way_avx(struct serpent_ctx *ctx, u8 *dst,
+asmlinkage void serpent_ecb_enc_8way_avx(struct serpent_ctx *ctx, u8 *dst,
-                                           const u8 *src, bool xor);
+                                         const u8 *src);
-asmlinkage void serpent_dec_blk_8way_avx(struct serpent_ctx *ctx, u8 *dst,
+asmlinkage void serpent_ecb_dec_8way_avx(struct serpent_ctx *ctx, u8 *dst,
                                         const u8 *src);
-static inline void serpent_enc_blk_xway(struct serpent_ctx *ctx, u8 *dst,
+asmlinkage void serpent_cbc_dec_8way_avx(struct serpent_ctx *ctx, u8 *dst,
-                                   const u8 *src)
+                                         const u8 *src);
-{
+asmlinkage void serpent_ctr_8way_avx(struct serpent_ctx *ctx, u8 *dst,
-        __serpent_enc_blk_8way_avx(ctx, dst, src, false);
+                                     const u8 *src, le128 *iv);
-}
-static inline void serpent_enc_blk_xway_xor(struct serpent_ctx *ctx, u8 *dst,
-                                       const u8 *src)
-{
-        __serpent_enc_blk_8way_avx(ctx, dst, src, true);
-}
-static inline void serpent_dec_blk_xway(struct serpent_ctx *ctx, u8 *dst,
-                                   const u8 *src)
-{
-        serpent_dec_blk_8way_avx(ctx, dst, src);
-}
 #endif
diff --git a/arch/x86/include/asm/crypto/twofish.h b/arch/x86/include/asm/crypto/twofish.h
index 9d2c514bd5f9..878c51ceebb5 100644
--- a/arch/x86/include/asm/crypto/twofish.h
+++ b/arch/x86/include/asm/crypto/twofish.h
@@ -31,9 +31,9 @@ asmlinkage void twofish_dec_blk_3way(struct twofish_ctx *ctx, u8 *dst,
 /* helpers from twofish_x86_64-3way module */
 extern void twofish_dec_blk_cbc_3way(void *ctx, u128 *dst, const u128 *src);
 extern void twofish_enc_blk_ctr(void *ctx, u128 *dst, const u128 *src,
-                                u128 *iv);
+                                le128 *iv);
 extern void twofish_enc_blk_ctr_3way(void *ctx, u128 *dst, const u128 *src,
-                                     u128 *iv);
+                                     le128 *iv);
 extern int lrw_twofish_setkey(struct crypto_tfm *tfm, const u8 *key,
                              unsigned int keylen);
diff --git a/arch/x86/include/asm/efi.h b/arch/x86/include/asm/efi.h
index fd13815fe85c..6e8fdf5ad113 100644
--- a/arch/x86/include/asm/efi.h
+++ b/arch/x86/include/asm/efi.h
@@ -69,37 +69,23 @@ extern u64 efi_call6(void *fp, u64 arg1, u64 arg2, u64 arg3,
        efi_call6((void *)(f), (u64)(a1), (u64)(a2), (u64)(a3),         \
                  (u64)(a4), (u64)(a5), (u64)(a6))
-extern unsigned long efi_call_virt_prelog(void);
-extern void efi_call_virt_epilog(unsigned long);
-#define efi_callx(x, func, ...)                                 \
-        ({                                                      \
-                efi_status_t __status;                          \
-                unsigned long __pgd;                            \
-                                                                \
-                __pgd = efi_call_virt_prelog();                 \
-                __status = efi_call##x(func, __VA_ARGS__);      \
-                efi_call_virt_epilog(__pgd);                    \
-                __status;                                       \
-        })
 #define efi_call_virt0(f)                               \
-        efi_callx(0, (void *)(efi.systab->runtime->f))
+        efi_call0((void *)(efi.systab->runtime->f))
 #define efi_call_virt1(f, a1)                                   \
-        efi_callx(1, (void *)(efi.systab->runtime->f), (u64)(a1))
+        efi_call1((void *)(efi.systab->runtime->f), (u64)(a1))
 #define efi_call_virt2(f, a1, a2)                                       \
-        efi_callx(2, (void *)(efi.systab->runtime->f), (u64)(a1), (u64)(a2))
+        efi_call2((void *)(efi.systab->runtime->f), (u64)(a1), (u64)(a2))
 #define efi_call_virt3(f, a1, a2, a3)                                   \
-        efi_callx(3, (void *)(efi.systab->runtime->f), (u64)(a1), (u64)(a2), \
+        efi_call3((void *)(efi.systab->runtime->f), (u64)(a1), (u64)(a2), \
                  (u64)(a3))
 #define efi_call_virt4(f, a1, a2, a3, a4)                               \
-        efi_callx(4, (void *)(efi.systab->runtime->f), (u64)(a1), (u64)(a2), \
+        efi_call4((void *)(efi.systab->runtime->f), (u64)(a1), (u64)(a2), \
                  (u64)(a3), (u64)(a4))
 #define efi_call_virt5(f, a1, a2, a3, a4, a5)                           \
-        efi_callx(5, (void *)(efi.systab->runtime->f), (u64)(a1), (u64)(a2), \
+        efi_call5((void *)(efi.systab->runtime->f), (u64)(a1), (u64)(a2), \
                  (u64)(a3), (u64)(a4), (u64)(a5))
 #define efi_call_virt6(f, a1, a2, a3, a4, a5, a6)                       \
-        efi_callx(6, (void *)(efi.systab->runtime->f), (u64)(a1), (u64)(a2), \
+        efi_call6((void *)(efi.systab->runtime->f), (u64)(a1), (u64)(a2), \
                  (u64)(a3), (u64)(a4), (u64)(a5), (u64)(a6))
 extern void __iomem *efi_ioremap(unsigned long addr, unsigned long size,
diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h
index a1f780d45f76..5199db2923d3 100644
--- a/arch/x86/include/asm/pgtable.h
+++ b/arch/x86/include/asm/pgtable.h
@@ -404,7 +404,14 @@ static inline int pte_same(pte_t a, pte_t b)
 static inline int pte_present(pte_t a)
 {
-        return pte_flags(a) & (_PAGE_PRESENT | _PAGE_PROTNONE);
+        return pte_flags(a) & (_PAGE_PRESENT | _PAGE_PROTNONE |
+                               _PAGE_NUMA);
+}
+#define pte_accessible pte_accessible
+static inline int pte_accessible(pte_t a)
+{
+        return pte_flags(a) & _PAGE_PRESENT;
 }
 static inline int pte_hidden(pte_t pte)
@@ -420,7 +427,8 @@ static inline int pmd_present(pmd_t pmd)
         * the _PAGE_PSE flag will remain set at all times while the
         * _PAGE_PRESENT bit is clear).
         */
-        return pmd_flags(pmd) & (_PAGE_PRESENT | _PAGE_PROTNONE | _PAGE_PSE);
+        return pmd_flags(pmd) & (_PAGE_PRESENT | _PAGE_PROTNONE | _PAGE_PSE |
+                                 _PAGE_NUMA);
 }
 static inline int pmd_none(pmd_t pmd)
@@ -479,6 +487,11 @@ static inline pte_t *pte_offset_kernel(pmd_t *pmd, unsigned long address)
 static inline int pmd_bad(pmd_t pmd)
 {
+#ifdef CONFIG_NUMA_BALANCING
+        /* pmd_numa check */
+        if ((pmd_flags(pmd) & (_PAGE_NUMA|_PAGE_PRESENT)) == _PAGE_NUMA)
+                return 0;
+#endif
        return (pmd_flags(pmd) & ~_PAGE_USER) != _KERNPG_TABLE;
 }
diff --git a/arch/x86/include/asm/pgtable_types.h b/arch/x86/include/asm/pgtable_types.h
index ec8a1fc9505d..3c32db8c539d 100644
--- a/arch/x86/include/asm/pgtable_types.h
+++ b/arch/x86/include/asm/pgtable_types.h
@@ -64,6 +64,26 @@
 #define _PAGE_FILE      (_AT(pteval_t, 1) << _PAGE_BIT_FILE)
 #define _PAGE_PROTNONE  (_AT(pteval_t, 1) << _PAGE_BIT_PROTNONE)
+/*
+ * _PAGE_NUMA indicates that this page will trigger a numa hinting
+ * minor page fault to gather numa placement statistics (see
+ * pte_numa()). The bit picked (8) is within the range between
+ * _PAGE_FILE (6) and _PAGE_PROTNONE (8) bits. Therefore, it doesn't
+ * require changes to the swp entry format because that bit is always
+ * zero when the pte is not present.
+ *
+ * The bit picked must be always zero when the pmd is present and not
+ * present, so that we don't lose information when we set it while
+ * atomically clearing the present bit.
+ *
+ * Because we shared the same bit (8) with _PAGE_PROTNONE this can be
+ * interpreted as _PAGE_NUMA only in places that _PAGE_PROTNONE
+ * couldn't reach, like handle_mm_fault() (see access_error in
+ * arch/x86/mm/fault.c, the vma protection must not be PROT_NONE for
+ * handle_mm_fault() to be invoked).
+ */
+#define _PAGE_NUMA      _PAGE_PROTNONE
 #define _PAGE_TABLE     (_PAGE_PRESENT | _PAGE_RW | _PAGE_USER |        \
                         _PAGE_ACCESSED | _PAGE_DIRTY)
 #define _KERNPG_TABLE   (_PAGE_PRESENT | _PAGE_RW | _PAGE_ACCESSED |    \
diff --git a/arch/x86/kernel/tboot.c b/arch/x86/kernel/tboot.c
index d4f460f962ee..f84fe00fad48 100644
--- a/arch/x86/kernel/tboot.c
+++ b/arch/x86/kernel/tboot.c
@@ -103,13 +103,71 @@ void __init tboot_probe(void)
        pr_debug("tboot_size: 0x%x\n", tboot->tboot_size);
 }
+static pgd_t *tboot_pg_dir;
+static struct mm_struct tboot_mm = {
+        .mm_rb          = RB_ROOT,
+        .pgd            = swapper_pg_dir,
+        .mm_users       = ATOMIC_INIT(2),
+        .mm_count       = ATOMIC_INIT(1),
+        .mmap_sem       = __RWSEM_INITIALIZER(init_mm.mmap_sem),
+        .page_table_lock =  __SPIN_LOCK_UNLOCKED(init_mm.page_table_lock),
+        .mmlist         = LIST_HEAD_INIT(init_mm.mmlist),
+};
 static inline void switch_to_tboot_pt(void)
 {
-#ifdef CONFIG_X86_32
+        write_cr3(virt_to_phys(tboot_pg_dir));
-        load_cr3(initial_page_table);
+}
-#else
-        write_cr3(real_mode_header->trampoline_pgd);
+static int map_tboot_page(unsigned long vaddr, unsigned long pfn,
-#endif
+                          pgprot_t prot)
+{
+        pgd_t *pgd;
+        pud_t *pud;
+        pmd_t *pmd;
+        pte_t *pte;
+        pgd = pgd_offset(&tboot_mm, vaddr);
+        pud = pud_alloc(&tboot_mm, pgd, vaddr);
+        if (!pud)
+                return -1;
+        pmd = pmd_alloc(&tboot_mm, pud, vaddr);
+        if (!pmd)
+                return -1;
+        pte = pte_alloc_map(&tboot_mm, NULL, pmd, vaddr);
+        if (!pte)
+                return -1;
+        set_pte_at(&tboot_mm, vaddr, pte, pfn_pte(pfn, prot));
+        pte_unmap(pte);
+        return 0;
+}
+static int map_tboot_pages(unsigned long vaddr, unsigned long start_pfn,
+                           unsigned long nr)
+{
+        /* Reuse the original kernel mapping */
+        tboot_pg_dir = pgd_alloc(&tboot_mm);
+        if (!tboot_pg_dir)
+                return -1;
+        for (; nr > 0; nr--, vaddr += PAGE_SIZE, start_pfn++) {
+                if (map_tboot_page(vaddr, start_pfn, PAGE_KERNEL_EXEC))
+                        return -1;
+        }
+        return 0;
+}
+static void tboot_create_trampoline(void)
+{
+        u32 map_base, map_size;
+        /* Create identity map for tboot shutdown code. */
+        map_base = PFN_DOWN(tboot->tboot_base);
+        map_size = PFN_UP(tboot->tboot_size);
+        if (map_tboot_pages(map_base << PAGE_SHIFT, map_base, map_size))
+                panic("tboot: Error mapping tboot pages (mfns) @ 0x%x, 0x%x\n",
+                      map_base, map_size);
 }
 #ifdef CONFIG_ACPI_SLEEP
@@ -167,6 +225,14 @@ void tboot_shutdown(u32 shutdown_type)
        if (!tboot_enabled())
                return;
+        /*
+         * if we're being called before the 1:1 mapping is set up then just
+         * return and let the normal shutdown happen; this should only be
+         * due to very early panic()
+         */
+        if (!tboot_pg_dir)
+                return;
        /* if this is S3 then set regions to MAC */
        if (shutdown_type == TB_SHUTDOWN_S3)
                if (tboot_setup_sleep())
@@ -277,6 +343,8 @@ static __init int tboot_late_init(void)
        if (!tboot_enabled())
                return 0;
+        tboot_create_trampoline();
        atomic_set(&ap_wfs_count, 0);
        register_hotcpu_notifier(&tboot_cpu_notifier);
diff --git a/arch/x86/kernel/vsyscall_64.c b/arch/x86/kernel/vsyscall_64.c
index 3a3e8c9e280d..9a907a67be8f 100644
--- a/arch/x86/kernel/vsyscall_64.c
+++ b/arch/x86/kernel/vsyscall_64.c
@@ -145,19 +145,6 @@ static int addr_to_vsyscall_nr(unsigned long addr)
        return nr;
 }
-#ifdef CONFIG_SECCOMP
-static int vsyscall_seccomp(struct task_struct *tsk, int syscall_nr)
-{
-        if (!seccomp_mode(&tsk->seccomp))
-                return 0;
-        task_pt_regs(tsk)->orig_ax = syscall_nr;
-        task_pt_regs(tsk)->ax = syscall_nr;
-        return __secure_computing(syscall_nr);
-}
-#else
-#define vsyscall_seccomp(_tsk, _nr) 0
-#endif
 static bool write_ok_or_segv(unsigned long ptr, size_t size)
 {
        /*
@@ -190,10 +177,9 @@ bool emulate_vsyscall(struct pt_regs *regs, unsigned long address)
 {
        struct task_struct *tsk;
        unsigned long caller;
-        int vsyscall_nr;
+        int vsyscall_nr, syscall_nr, tmp;
        int prev_sig_on_uaccess_error;
        long ret;
-        int skip;
        /*
         * No point in checking CS -- the only way to get here is a user mode
@@ -225,56 +211,84 @@ bool emulate_vsyscall(struct pt_regs *regs, unsigned long address)
        }
        tsk = current;
-        /*
-         * With a real vsyscall, page faults cause SIGSEGV.  We want to
-         * preserve that behavior to make writing exploits harder.
-         */
-        prev_sig_on_uaccess_error = current_thread_info()->sig_on_uaccess_error;
-        current_thread_info()->sig_on_uaccess_error = 1;
        /*
+         * Check for access_ok violations and find the syscall nr.
+         *
         * NULL is a valid user pointer (in the access_ok sense) on 32-bit and
         * 64-bit, so we don't need to special-case it here.  For all the
         * vsyscalls, NULL means "don't write anything" not "write it at
         * address 0".
         */
-        ret = -EFAULT;
-        skip = 0;
        switch (vsyscall_nr) {
        case 0:
-                skip = vsyscall_seccomp(tsk, __NR_gettimeofday);
-                if (skip)
-                        break;
                if (!write_ok_or_segv(regs->di, sizeof(struct timeval)) ||
-                    !write_ok_or_segv(regs->si, sizeof(struct timezone)))
+                    !write_ok_or_segv(regs->si, sizeof(struct timezone))) {
-                        break;
+                        ret = -EFAULT;
+                        goto check_fault;
+                }
+                syscall_nr = __NR_gettimeofday;
+                break;
+        case 1:
+                if (!write_ok_or_segv(regs->di, sizeof(time_t))) {
+                        ret = -EFAULT;
+                        goto check_fault;
+                }
+                syscall_nr = __NR_time;
+                break;
+        case 2:
+                if (!write_ok_or_segv(regs->di, sizeof(unsigned)) ||
+                    !write_ok_or_segv(regs->si, sizeof(unsigned))) {
+                        ret = -EFAULT;
+                        goto check_fault;
+                }
+                syscall_nr = __NR_getcpu;
+                break;
+        }
+        /*
+         * Handle seccomp.  regs->ip must be the original value.
+         * See seccomp_send_sigsys and Documentation/prctl/seccomp_filter.txt.
+         *
+         * We could optimize the seccomp disabled case, but performance
+         * here doesn't matter.
+         */
+        regs->orig_ax = syscall_nr;
+        regs->ax = -ENOSYS;
+        tmp = secure_computing(syscall_nr);
+        if ((!tmp && regs->orig_ax != syscall_nr) || regs->ip != address) {
+                warn_bad_vsyscall(KERN_DEBUG, regs,
+                                  "seccomp tried to change syscall nr or ip");
+                do_exit(SIGSYS);
+        }
+        if (tmp)
+                goto do_ret;  /* skip requested */
+        /*
+         * With a real vsyscall, page faults cause SIGSEGV.  We want to
+         * preserve that behavior to make writing exploits harder.
+         */
+        prev_sig_on_uaccess_error = current_thread_info()->sig_on_uaccess_error;
+        current_thread_info()->sig_on_uaccess_error = 1;
+        ret = -EFAULT;
+        switch (vsyscall_nr) {
+        case 0:
                ret = sys_gettimeofday(
                        (struct timeval __user *)regs->di,
                        (struct timezone __user *)regs->si);
                break;
        case 1:
-                skip = vsyscall_seccomp(tsk, __NR_time);
-                if (skip)
-                        break;
-                if (!write_ok_or_segv(regs->di, sizeof(time_t)))
-                        break;
                ret = sys_time((time_t __user *)regs->di);
                break;
        case 2:
-                skip = vsyscall_seccomp(tsk, __NR_getcpu);
-                if (skip)
-                        break;
-                if (!write_ok_or_segv(regs->di, sizeof(unsigned)) ||
-                    !write_ok_or_segv(regs->si, sizeof(unsigned)))
-                        break;
                ret = sys_getcpu((unsigned __user *)regs->di,
                                 (unsigned __user *)regs->si,
                                 NULL);
@@ -283,12 +297,7 @@ bool emulate_vsyscall(struct pt_regs *regs, unsigned long address)
        current_thread_info()->sig_on_uaccess_error = prev_sig_on_uaccess_error;
-        if (skip) {
+check_fault:
-                if ((long)regs->ax <= 0L) /* seccomp errno emulation */
-                        goto do_ret;
-                goto done; /* seccomp trace/trap */
-        }
        if (ret == -EFAULT) {
                /* Bad news -- userspace fed a bad pointer to a vsyscall. */
                warn_bad_vsyscall(KERN_INFO, regs,
@@ -311,7 +320,6 @@ do_ret:
        /* Emulate a ret instruction. */
        regs->ip = caller;
        regs->sp += 8;
-done:
        return true;
 sigsegv:
diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
index 07519a120449..2ead3c8a4c84 100644
--- a/arch/x86/mm/init_64.c
+++ b/arch/x86/mm/init_64.c
@@ -108,13 +108,13 @@ void sync_global_pgds(unsigned long start, unsigned long end)
        for (address = start; address <= end; address += PGDIR_SIZE) {
                const pgd_t *pgd_ref = pgd_offset_k(address);
                struct page *page;
-                pgd_t *pgd;
                if (pgd_none(*pgd_ref))
                        continue;
                spin_lock(&pgd_lock);
                list_for_each_entry(page, &pgd_list, lru) {
+                        pgd_t *pgd;
                        spinlock_t *pgt_lock;
                        pgd = (pgd_t *)page_address(page) + pgd_index(address);
@@ -130,13 +130,6 @@ void sync_global_pgds(unsigned long start, unsigned long end)
                        spin_unlock(pgt_lock);
                }
-                pgd = __va(real_mode_header->trampoline_pgd);
-                pgd += pgd_index(address);
-                if (pgd_none(*pgd))
-                        set_pgd(pgd, *pgd_ref);
                spin_unlock(&pgd_lock);
        }
 }
diff --git a/arch/x86/mm/ioremap.c b/arch/x86/mm/ioremap.c
index e190f7b56653..78fe3f1ac49f 100644
--- a/arch/x86/mm/ioremap.c
+++ b/arch/x86/mm/ioremap.c
@@ -50,107 +50,6 @@ int ioremap_change_attr(unsigned long vaddr, unsigned long size,
        return err;
 }
-#ifdef CONFIG_X86_64
-static void ident_pte_range(unsigned long paddr, unsigned long vaddr,
-                            pmd_t *ppmd, pmd_t *vpmd, unsigned long end)
-{
-        pte_t *ppte = pte_offset_kernel(ppmd, paddr);
-        pte_t *vpte = pte_offset_kernel(vpmd, vaddr);
-        do {
-                set_pte(ppte, *vpte);
-        } while (ppte++, vpte++, vaddr += PAGE_SIZE, vaddr != end);
-}
-static int ident_pmd_range(unsigned long paddr, unsigned long vaddr,
-                            pud_t *ppud, pud_t *vpud, unsigned long end)
-{
-        pmd_t *ppmd = pmd_offset(ppud, paddr);
-        pmd_t *vpmd = pmd_offset(vpud, vaddr);
-        unsigned long next;
-        do {
-                next = pmd_addr_end(vaddr, end);
-                if (!pmd_present(*ppmd)) {
-                        pte_t *ppte = (pte_t *)get_zeroed_page(GFP_KERNEL);
-                        if (!ppte)
-                                return 1;
-                        set_pmd(ppmd, __pmd(_KERNPG_TABLE | __pa(ppte)));
-                }
-                ident_pte_range(paddr, vaddr, ppmd, vpmd, next);
-        } while (ppmd++, vpmd++, vaddr = next, vaddr != end);
-        return 0;
-}
-static int ident_pud_range(unsigned long paddr, unsigned long vaddr,
-                            pgd_t *ppgd, pgd_t *vpgd, unsigned long end)
-{
-        pud_t *ppud = pud_offset(ppgd, paddr);
-        pud_t *vpud = pud_offset(vpgd, vaddr);
-        unsigned long next;
-        do {
-                next = pud_addr_end(vaddr, end);
-                if (!pud_present(*ppud)) {
-                        pmd_t *ppmd = (pmd_t *)get_zeroed_page(GFP_KERNEL);
-                        if (!ppmd)
-                                return 1;
-                        set_pud(ppud, __pud(_KERNPG_TABLE | __pa(ppmd)));
-                }
-                if (ident_pmd_range(paddr, vaddr, ppud, vpud, next))
-                        return 1;
-        } while (ppud++, vpud++, vaddr = next, vaddr != end);
-        return 0;
-}
-static int insert_identity_mapping(resource_size_t paddr, unsigned long vaddr,
-                                    unsigned long size)
-{
-        unsigned long end = vaddr + size;
-        unsigned long next;
-        pgd_t *vpgd, *ppgd;
-        /* Don't map over the guard hole. */
-        if (paddr >= 0x800000000000 || paddr + size > 0x800000000000)
-                return 1;
-        ppgd = __va(real_mode_header->trampoline_pgd) + pgd_index(paddr);
-        vpgd = pgd_offset_k(vaddr);
-        do {
-                next = pgd_addr_end(vaddr, end);
-                if (!pgd_present(*ppgd)) {
-                        pud_t *ppud = (pud_t *)get_zeroed_page(GFP_KERNEL);
-                        if (!ppud)
-                                return 1;
-                        set_pgd(ppgd, __pgd(_KERNPG_TABLE | __pa(ppud)));
-                }
-                if (ident_pud_range(paddr, vaddr, ppgd, vpgd, next))
-                        return 1;
-        } while (ppgd++, vpgd++, vaddr = next, vaddr != end);
-        return 0;
-}
-#else
-static inline int insert_identity_mapping(resource_size_t paddr,
-                                          unsigned long vaddr,
-                                          unsigned long size)
-{
-        return 0;
-}
-#endif /* CONFIG_X86_64 */
 /*
 * Remap an arbitrary physical address space into the kernel virtual
 * address space. Needed when the kernel wants to access high addresses
@@ -264,10 +163,6 @@ static void __iomem *__ioremap_caller(resource_size_t phys_addr,
        ret_addr = (void __iomem *) (vaddr + offset);
        mmiotrace_ioremap(unaligned_phys_addr, unaligned_size, ret_addr);
-        if (insert_identity_mapping(phys_addr, vaddr, size))
-                printk(KERN_WARNING "ioremap: unable to map 0x%llx in identity pagetable\n",
-                                        (unsigned long long)phys_addr);
        /*
         * Check if the request spans more than any BAR in the iomem resource
         * tree.
diff --git a/arch/x86/mm/pageattr.c b/arch/x86/mm/pageattr.c
index 931930a96160..a718e0d23503 100644
--- a/arch/x86/mm/pageattr.c
+++ b/arch/x86/mm/pageattr.c
@@ -919,13 +919,11 @@ static int change_page_attr_set_clr(unsigned long *addr, int numpages,
        /*
         * On success we use clflush, when the CPU supports it to
-         * avoid the wbindv. If the CPU does not support it, in the
+         * avoid the wbindv. If the CPU does not support it and in the
-         * error case, and during early boot (for EFI) we fall back
+         * error case we fall back to cpa_flush_all (which uses
-         * to cpa_flush_all (which uses wbinvd):
+         * wbindv):
         */
-        if (early_boot_irqs_disabled)
+        if (!ret && cpu_has_clflush) {
-                __cpa_flush_all((void *)(long)cache);
-        else if (!ret && cpu_has_clflush) {
                if (cpa.flags & (CPA_PAGES_ARRAY | CPA_ARRAY)) {
                        cpa_flush_array(addr, numpages, cache,
                                        cpa.flags, pages);
diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c
index 217eb705fac0..e27fbf887f3b 100644
--- a/arch/x86/mm/pgtable.c
+++ b/arch/x86/mm/pgtable.c
@@ -301,6 +301,13 @@ void pgd_free(struct mm_struct *mm, pgd_t *pgd)
        free_page((unsigned long)pgd);
 }
+/*
+ * Used to set accessed or dirty bits in the page table entries
+ * on other architectures. On x86, the accessed and dirty bits
+ * are tracked by hardware. However, do_wp_page calls this function
+ * to also make the pte writeable at the same time the dirty bit is
+ * set. In that case we do actually need to write the PTE.
+ */
 int ptep_set_access_flags(struct vm_area_struct *vma,
                          unsigned long address, pte_t *ptep,
                          pte_t entry, int dirty)
@@ -310,7 +317,6 @@ int ptep_set_access_flags(struct vm_area_struct *vma,
        if (changed && dirty) {
                *ptep = entry;
                pte_update_defer(vma->vm_mm, address, ptep);
-                flush_tlb_page(vma, address);
        }
        return changed;
diff --git a/arch/x86/platform/efi/efi.c b/arch/x86/platform/efi/efi.c
index 0a34d9e9c263..ad4439145f85 100644
--- a/arch/x86/platform/efi/efi.c
+++ b/arch/x86/platform/efi/efi.c
@@ -239,7 +239,22 @@ static efi_status_t __init phys_efi_set_virtual_address_map(
        return status;
 }
-static int efi_set_rtc_mmss(unsigned long nowtime)
+static efi_status_t __init phys_efi_get_time(efi_time_t *tm,
+                                             efi_time_cap_t *tc)
+{
+        unsigned long flags;
+        efi_status_t status;
+        spin_lock_irqsave(&rtc_lock, flags);
+        efi_call_phys_prelog();
+        status = efi_call_phys2(efi_phys.get_time, virt_to_phys(tm),
+                                virt_to_phys(tc));
+        efi_call_phys_epilog();
+        spin_unlock_irqrestore(&rtc_lock, flags);
+        return status;
+}
+int efi_set_rtc_mmss(unsigned long nowtime)
 {
        int real_seconds, real_minutes;
        efi_status_t    status;
@@ -268,7 +283,7 @@ static int efi_set_rtc_mmss(unsigned long nowtime)
        return 0;
 }
-static unsigned long efi_get_time(void)
+unsigned long efi_get_time(void)
 {
        efi_status_t status;
        efi_time_t eft;
@@ -624,13 +639,18 @@ static int __init efi_runtime_init(void)
        }
        /*
         * We will only need *early* access to the following
-         * EFI runtime service before set_virtual_address_map
+         * two EFI runtime services before set_virtual_address_map
         * is invoked.
         */
+        efi_phys.get_time = (efi_get_time_t *)runtime->get_time;
        efi_phys.set_virtual_address_map =
                (efi_set_virtual_address_map_t *)
                runtime->set_virtual_address_map;
+        /*
+         * Make efi_get_time can be called before entering
+         * virtual mode.
+         */
+        efi.get_time = phys_efi_get_time;
        early_iounmap(runtime, sizeof(efi_runtime_services_t));
        return 0;
@@ -716,10 +736,12 @@ void __init efi_init(void)
                efi_enabled = 0;
                return;
        }
+#ifdef CONFIG_X86_32
        if (efi_is_native()) {
                x86_platform.get_wallclock = efi_get_time;
                x86_platform.set_wallclock = efi_set_rtc_mmss;
        }
+#endif
 #if EFI_DEBUG
        print_efi_memmap();
diff --git a/arch/x86/platform/efi/efi_64.c b/arch/x86/platform/efi/efi_64.c
index 06c8b2e662ab..95fd505dfeb6 100644
--- a/arch/x86/platform/efi/efi_64.c
+++ b/arch/x86/platform/efi/efi_64.c
@@ -58,21 +58,6 @@ static void __init early_code_mapping_set_exec(int executable)
        }
 }
-unsigned long efi_call_virt_prelog(void)
-{
-        unsigned long saved;
-        saved = read_cr3();
-        write_cr3(real_mode_header->trampoline_pgd);
-        return saved;
-}
-void efi_call_virt_epilog(unsigned long saved)
-{
-        write_cr3(saved);
-}
 void __init efi_call_phys_prelog(void)
 {
        unsigned long vaddress;
diff --git a/arch/x86/realmode/init.c b/arch/x86/realmode/init.c
index 8e6ab6137852..cbca565af5bd 100644
--- a/arch/x86/realmode/init.c
+++ b/arch/x86/realmode/init.c
@@ -78,21 +78,8 @@ void __init setup_real_mode(void)
        *trampoline_cr4_features = read_cr4();
        trampoline_pgd = (u64 *) __va(real_mode_header->trampoline_pgd);
+        trampoline_pgd[0] = __pa(level3_ident_pgt) + _KERNPG_TABLE;
-        /*
+        trampoline_pgd[511] = __pa(level3_kernel_pgt) + _KERNPG_TABLE;
-         * Create an identity mapping for all of physical memory.
-         */
-        for (i = 0; i <= pgd_index(max_pfn << PAGE_SHIFT); i++) {
-                int index = pgd_index(PAGE_OFFSET) + i;
-                trampoline_pgd[i] = (u64)pgd_val(swapper_pg_dir[index]);
-        }
-        /*
-         * Copy the upper-half of the kernel pages tables.
-         */
-        for (i = pgd_index(PAGE_OFFSET); i < PTRS_PER_PGD; i++)
-                trampoline_pgd[i] = (u64)pgd_val(swapper_pg_dir[i]);
 #endif
 }