Merge branch 'mpi-master' into wip-k-fmlpwip-k-fmlp

Conflicts:
	litmus/sched_cedf.c

56 files changed, 1806 insertions, 747 deletions

diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig
index a0a2928ae4dd..0074b8dba793 100644
--- a/arch/arm/mm/Kconfig
+++ b/arch/arm/mm/Kconfig
@@ -382,9 +382,15 @@ config CPU_FEROCEON_OLD_ID
           for which the CPU ID is equal to the ARM926 ID.
           Relevant for Feroceon-1850 and early Feroceon-2850.
 
+# Marvell PJ4
+config CPU_PJ4
+        bool
+        select CPU_V7
+        select ARM_THUMBEE
+
 # ARMv6
 config CPU_V6
-        bool "Support ARM V6 processor" if ARCH_INTEGRATOR || MACH_REALVIEW_EB || MACH_REALVIEW_PBX || ARCH_DOVE
+        bool "Support ARM V6 processor" if ARCH_INTEGRATOR || MACH_REALVIEW_EB || MACH_REALVIEW_PBX
         select CPU_32v6
         select CPU_ABRT_EV6
         select CPU_PABRT_V6
@@ -396,21 +402,23 @@ config CPU_V6
         select CPU_TLB_V6 if MMU
 
 # ARMv6k
-config CPU_32v6K
-        bool "Support ARM V6K processor extensions" if !SMP
-        depends on CPU_V6 || CPU_V7
-        default y if SMP && !(ARCH_MX3 || ARCH_OMAP2)
-        help
-          Say Y here if your ARMv6 processor supports the 'K' extension.
-          This enables the kernel to use some instructions not present
-          on previous processors, and as such a kernel build with this
-          enabled will not boot on processors with do not support these
-          instructions.
+config CPU_V6K
+        bool "Support ARM V6K processor" if ARCH_INTEGRATOR || MACH_REALVIEW_EB || MACH_REALVIEW_PBX
+        select CPU_32v6
+        select CPU_32v6K
+        select CPU_ABRT_EV6
+        select CPU_PABRT_V6
+        select CPU_CACHE_V6
+        select CPU_CACHE_VIPT
+        select CPU_CP15_MMU
+        select CPU_HAS_ASID if MMU
+        select CPU_COPY_V6 if MMU
+        select CPU_TLB_V6 if MMU
 
 # ARMv7
 config CPU_V7
         bool "Support ARM V7 processor" if ARCH_INTEGRATOR || MACH_REALVIEW_EB || MACH_REALVIEW_PBX
-        select CPU_32v6K if !ARCH_OMAP2
+        select CPU_32v6K
         select CPU_32v7
         select CPU_ABRT_EV7
         select CPU_PABRT_V7
@@ -427,25 +435,33 @@ config CPU_32v3
         bool
         select TLS_REG_EMUL if SMP || !MMU
         select NEEDS_SYSCALL_FOR_CMPXCHG if SMP
+        select CPU_USE_DOMAINS if MMU
 
 config CPU_32v4
         bool
         select TLS_REG_EMUL if SMP || !MMU
         select NEEDS_SYSCALL_FOR_CMPXCHG if SMP
+        select CPU_USE_DOMAINS if MMU
 
 config CPU_32v4T
         bool
         select TLS_REG_EMUL if SMP || !MMU
         select NEEDS_SYSCALL_FOR_CMPXCHG if SMP
+        select CPU_USE_DOMAINS if MMU
 
 config CPU_32v5
         bool
         select TLS_REG_EMUL if SMP || !MMU
         select NEEDS_SYSCALL_FOR_CMPXCHG if SMP
+        select CPU_USE_DOMAINS if MMU
 
 config CPU_32v6
         bool
         select TLS_REG_EMUL if !CPU_32v6K && !MMU
+        select CPU_USE_DOMAINS if CPU_V6 && MMU
+
+config CPU_32v6K
+        bool
 
 config CPU_32v7
         bool
@@ -599,6 +615,12 @@ config CPU_CP15_MPU
         help
           Processor has the CP15 register, which has MPU related registers.
 
+config CPU_USE_DOMAINS
+        bool
+        help
+          This option enables or disables the use of domain switching
+          via the set_fs() function.
+
 #
 # CPU supports 36-bit I/O
 #
@@ -609,7 +631,7 @@ comment "Processor Features"
 
 config ARM_THUMB
         bool "Support Thumb user binaries"
-        depends on CPU_ARM720T || CPU_ARM740T || CPU_ARM920T || CPU_ARM922T || CPU_ARM925T || CPU_ARM926T || CPU_ARM940T || CPU_ARM946E || CPU_ARM1020 || CPU_ARM1020E || CPU_ARM1022 || CPU_ARM1026 || CPU_XSCALE || CPU_XSC3 || CPU_MOHAWK || CPU_V6 || CPU_V7 || CPU_FEROCEON
+        depends on CPU_ARM720T || CPU_ARM740T || CPU_ARM920T || CPU_ARM922T || CPU_ARM925T || CPU_ARM926T || CPU_ARM940T || CPU_ARM946E || CPU_ARM1020 || CPU_ARM1020E || CPU_ARM1022 || CPU_ARM1026 || CPU_XSCALE || CPU_XSC3 || CPU_MOHAWK || CPU_V6 || CPU_V6K || CPU_V7 || CPU_FEROCEON
         default y
         help
           Say Y if you want to include kernel support for running user space
@@ -628,6 +650,33 @@ config ARM_THUMBEE
           Say Y here if you have a CPU with the ThumbEE extension and code to
           make use of it. Say N for code that can run on CPUs without ThumbEE.
 
+config SWP_EMULATE
+        bool "Emulate SWP/SWPB instructions"
+        depends on !CPU_USE_DOMAINS && CPU_V7
+        select HAVE_PROC_CPU if PROC_FS
+        default y if SMP
+        help
+          ARMv6 architecture deprecates use of the SWP/SWPB instructions.
+          ARMv7 multiprocessing extensions introduce the ability to disable
+          these instructions, triggering an undefined instruction exception
+          when executed. Say Y here to enable software emulation of these
+          instructions for userspace (not kernel) using LDREX/STREX.
+          Also creates /proc/cpu/swp_emulation for statistics.
+
+          In some older versions of glibc [<=2.8] SWP is used during futex
+          trylock() operations with the assumption that the code will not
+          be preempted. This invalid assumption may be more likely to fail
+          with SWP emulation enabled, leading to deadlock of the user
+          application.
+
+          NOTE: when accessing uncached shared regions, LDREX/STREX rely
+          on an external transaction monitoring block called a global
+          monitor to maintain update atomicity. If your system does not
+          implement a global monitor, this option can cause programs that
+          perform SWP operations to uncached memory to deadlock.
+
+          If unsure, say Y.
+
 config CPU_BIG_ENDIAN
         bool "Build big-endian kernel"
         depends on ARCH_SUPPORTS_BIG_ENDIAN
@@ -640,7 +689,7 @@ config CPU_BIG_ENDIAN
 config CPU_ENDIAN_BE8
         bool
         depends on CPU_BIG_ENDIAN
-        default CPU_V6 || CPU_V7
+        default CPU_V6 || CPU_V6K || CPU_V7
         help
           Support for the BE-8 (big-endian) mode on ARMv6 and ARMv7 processors.
 
@@ -706,7 +755,7 @@ config CPU_CACHE_ROUND_ROBIN
 
 config CPU_BPREDICT_DISABLE
         bool "Disable branch prediction"
-        depends on CPU_ARM1020 || CPU_V6 || CPU_MOHAWK || CPU_XSC3 || CPU_V7 || CPU_FA526
+        depends on CPU_ARM1020 || CPU_V6 || CPU_V6K || CPU_MOHAWK || CPU_XSC3 || CPU_V7 || CPU_FA526
         help
           Say Y here to disable branch prediction.  If unsure, say N.
 
@@ -726,7 +775,7 @@ config NEEDS_SYSCALL_FOR_CMPXCHG
 
 config DMA_CACHE_RWFO
         bool "Enable read/write for ownership DMA cache maintenance"
-        depends on CPU_V6 && SMP
+        depends on CPU_V6K && SMP
         default y
         help
           The Snoop Control Unit on ARM11MPCore does not detect the
@@ -770,18 +819,26 @@ config CACHE_FEROCEON_L2_WRITETHROUGH
 config CACHE_L2X0
         bool "Enable the L2x0 outer cache controller"
         depends on REALVIEW_EB_ARM11MP || MACH_REALVIEW_PB11MP || MACH_REALVIEW_PB1176 || \
-                   REALVIEW_EB_A9MP || ARCH_MX35 || ARCH_MX31 || MACH_REALVIEW_PBX || \
-                   ARCH_NOMADIK || ARCH_OMAP4 || ARCH_S5PV310 || ARCH_TEGRA || \
-                   ARCH_U8500 || ARCH_VEXPRESS_CA9X4
+                   REALVIEW_EB_A9MP || SOC_IMX35 || SOC_IMX31 || MACH_REALVIEW_PBX || \
+                   ARCH_NOMADIK || ARCH_OMAP4 || ARCH_EXYNOS4 || ARCH_TEGRA || \
+                   ARCH_U8500 || ARCH_VEXPRESS_CA9X4 || ARCH_SHMOBILE
         default y
         select OUTER_CACHE
         select OUTER_CACHE_SYNC
         help
           This option enables the L2x0 PrimeCell.
 
+config CACHE_PL310
+        bool
+        depends on CACHE_L2X0
+        default y if CPU_V7 && !(CPU_V6 || CPU_V6K)
+        help
+          This option enables optimisations for the PL310 cache
+          controller.
+
 config CACHE_TAUROS2
         bool "Enable the Tauros2 L2 cache controller"
-        depends on (ARCH_DOVE || ARCH_MMP)
+        depends on (ARCH_DOVE || ARCH_MMP || CPU_PJ4)
         default y
         select OUTER_CACHE
         help
@@ -796,16 +853,21 @@ config CACHE_XSC3L2
         help
           This option enables the L2 cache on XScale3.
 
+config ARM_L1_CACHE_SHIFT_6
+        bool
+        help
+          Setting ARM L1 cache line size to 64 Bytes.
+
 config ARM_L1_CACHE_SHIFT
         int
         default 6 if ARM_L1_CACHE_SHIFT_6
         default 5
 
 config ARM_DMA_MEM_BUFFERABLE
-        bool "Use non-cacheable memory for DMA" if CPU_V6 && !CPU_V7
+        bool "Use non-cacheable memory for DMA" if (CPU_V6 || CPU_V6K) && !CPU_V7
         depends on !(MACH_REALVIEW_PB1176 || REALVIEW_EB_ARM11MP || \
                      MACH_REALVIEW_PB11MP)
-        default y if CPU_V6 || CPU_V7
+        default y if CPU_V6 || CPU_V6K || CPU_V7
         help
           Historically, the kernel has used strongly ordered mappings to
           provide DMA coherent memory.  With the advent of ARMv7, mapping
diff --git a/arch/arm/mm/Makefile b/arch/arm/mm/Makefile
index d63b6c413758..bca7e61928c7 100644
--- a/arch/arm/mm/Makefile
+++ b/arch/arm/mm/Makefile
@@ -5,8 +5,8 @@
 obj-y                           := dma-mapping.o extable.o fault.o init.o \
                                    iomap.o
 
-obj-$(CONFIG_MMU)               += fault-armv.o flush.o ioremap.o mmap.o \
-                                   pgd.o mmu.o vmregion.o
+obj-$(CONFIG_MMU)               += fault-armv.o flush.o idmap.o ioremap.o \
+                                   mmap.o pgd.o mmu.o vmregion.o
 
 ifneq ($(CONFIG_MMU),y)
 obj-y                           += nommu.o
@@ -90,6 +90,7 @@ obj-$(CONFIG_CPU_XSC3)		+= proc-xsc3.o
 obj-$(CONFIG_CPU_MOHAWK)        += proc-mohawk.o
 obj-$(CONFIG_CPU_FEROCEON)      += proc-feroceon.o
 obj-$(CONFIG_CPU_V6)            += proc-v6.o
+obj-$(CONFIG_CPU_V6K)           += proc-v6.o
 obj-$(CONFIG_CPU_V7)            += proc-v7.o
 
 AFLAGS_proc-v6.o        :=-Wa,-march=armv6
diff --git a/arch/arm/mm/abort-ev6.S b/arch/arm/mm/abort-ev6.S
index f332df7f0d37..1478aa522144 100644
--- a/arch/arm/mm/abort-ev6.S
+++ b/arch/arm/mm/abort-ev6.S
@@ -20,11 +20,11 @@
  */
         .align  5
 ENTRY(v6_early_abort)
-#ifdef CONFIG_CPU_32v6K
-        clrex
-#else
+#ifdef CONFIG_CPU_V6
         sub     r1, sp, #4                      @ Get unused stack location
         strex   r0, r1, [r1]                    @ Clear the exclusive monitor
+#elif defined(CONFIG_CPU_32v6K)
+        clrex
 #endif
         mrc     p15, 0, r1, c5, c0, 0           @ get FSR
         mrc     p15, 0, r0, c6, c0, 0           @ get FAR
diff --git a/arch/arm/mm/cache-fa.S b/arch/arm/mm/cache-fa.S
index 7148e53e6078..1fa6f71470de 100644
--- a/arch/arm/mm/cache-fa.S
+++ b/arch/arm/mm/cache-fa.S
@@ -38,6 +38,17 @@
 #define CACHE_DLIMIT    (CACHE_DSIZE * 2)
 
 /*
+ *      flush_icache_all()
+ *
+ *      Unconditionally clean and invalidate the entire icache.
+ */
+ENTRY(fa_flush_icache_all)
+        mov     r0, #0
+        mcr     p15, 0, r0, c7, c5, 0           @ invalidate I cache
+        mov     pc, lr
+ENDPROC(fa_flush_icache_all)
+
+/*
  *      flush_user_cache_all()
  *
  *      Clean and invalidate all cache entries in a particular address
@@ -233,6 +244,7 @@ ENDPROC(fa_dma_unmap_area)
 
         .type   fa_cache_fns, #object
 ENTRY(fa_cache_fns)
+        .long   fa_flush_icache_all
         .long   fa_flush_kern_cache_all
         .long   fa_flush_user_cache_all
         .long   fa_flush_user_cache_range
diff --git a/arch/arm/mm/cache-feroceon-l2.c b/arch/arm/mm/cache-feroceon-l2.c
index 6e77c042d8e9..e0b0e7a4ec68 100644
--- a/arch/arm/mm/cache-feroceon-l2.c
+++ b/arch/arm/mm/cache-feroceon-l2.c
@@ -13,13 +13,9 @@
  */
 
 #include <linux/init.h>
+#include <linux/highmem.h>
 #include <asm/cacheflush.h>
-#include <asm/kmap_types.h>
-#include <asm/fixmap.h>
-#include <asm/pgtable.h>
-#include <asm/tlbflush.h>
 #include <plat/cache-feroceon-l2.h>
-#include "mm.h"
 
 /*
  * Low-level cache maintenance operations.
@@ -39,27 +35,30 @@
  * between which we don't want to be preempted.
  */
 
-static inline unsigned long l2_start_va(unsigned long paddr)
+static inline unsigned long l2_get_va(unsigned long paddr)
 {
 #ifdef CONFIG_HIGHMEM
         /*
-         * Let's do our own fixmap stuff in a minimal way here.
          * Because range ops can't be done on physical addresses,
          * we simply install a virtual mapping for it only for the
          * TLB lookup to occur, hence no need to flush the untouched
-         * memory mapping.  This is protected with the disabling of
-         * interrupts by the caller.
+         * memory mapping afterwards (note: a cache flush may happen
+         * in some circumstances depending on the path taken in kunmap_atomic).
          */
-        unsigned long idx = KM_L2_CACHE + KM_TYPE_NR * smp_processor_id();
-        unsigned long vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx);
-        set_pte_ext(TOP_PTE(vaddr), pfn_pte(paddr >> PAGE_SHIFT, PAGE_KERNEL), 0);
-        local_flush_tlb_kernel_page(vaddr);
-        return vaddr + (paddr & ~PAGE_MASK);
+        void *vaddr = kmap_atomic_pfn(paddr >> PAGE_SHIFT);
+        return (unsigned long)vaddr + (paddr & ~PAGE_MASK);
 #else
         return __phys_to_virt(paddr);
 #endif
 }
 
+static inline void l2_put_va(unsigned long vaddr)
+{
+#ifdef CONFIG_HIGHMEM
+        kunmap_atomic((void *)vaddr);
+#endif
+}
+
 static inline void l2_clean_pa(unsigned long addr)
 {
         __asm__("mcr p15, 1, %0, c15, c9, 3" : : "r" (addr));
@@ -76,13 +75,14 @@ static inline void l2_clean_pa_range(unsigned long start, unsigned long end)
          */
         BUG_ON((start ^ end) >> PAGE_SHIFT);
 
-        raw_local_irq_save(flags);
-        va_start = l2_start_va(start);
+        va_start = l2_get_va(start);
         va_end = va_start + (end - start);
+        raw_local_irq_save(flags);
         __asm__("mcr p15, 1, %0, c15, c9, 4\n\t"
                 "mcr p15, 1, %1, c15, c9, 5"
                 : : "r" (va_start), "r" (va_end));
         raw_local_irq_restore(flags);
+        l2_put_va(va_start);
 }
 
 static inline void l2_clean_inv_pa(unsigned long addr)
@@ -106,13 +106,14 @@ static inline void l2_inv_pa_range(unsigned long start, unsigned long end)
          */
         BUG_ON((start ^ end) >> PAGE_SHIFT);
 
-        raw_local_irq_save(flags);
-        va_start = l2_start_va(start);
+        va_start = l2_get_va(start);
         va_end = va_start + (end - start);
+        raw_local_irq_save(flags);
         __asm__("mcr p15, 1, %0, c15, c11, 4\n\t"
                 "mcr p15, 1, %1, c15, c11, 5"
                 : : "r" (va_start), "r" (va_end));
         raw_local_irq_restore(flags);
+        l2_put_va(va_start);
 }
 
 static inline void l2_inv_all(void)
diff --git a/arch/arm/mm/cache-l2x0.c b/arch/arm/mm/cache-l2x0.c
index 9982eb385c0f..44c086710d2b 100644
--- a/arch/arm/mm/cache-l2x0.c
+++ b/arch/arm/mm/cache-l2x0.c
@@ -28,18 +28,34 @@
 static void __iomem *l2x0_base;
 static DEFINE_SPINLOCK(l2x0_lock);
 static uint32_t l2x0_way_mask;  /* Bitmask of active ways */
+static uint32_t l2x0_size;
 
-static inline void cache_wait(void __iomem *reg, unsigned long mask)
+static inline void cache_wait_way(void __iomem *reg, unsigned long mask)
 {
-        /* wait for the operation to complete */
+        /* wait for cache operation by line or way to complete */
         while (readl_relaxed(reg) & mask)
                 ;
 }
 
+#ifdef CONFIG_CACHE_PL310
+static inline void cache_wait(void __iomem *reg, unsigned long mask)
+{
+        /* cache operations by line are atomic on PL310 */
+}
+#else
+#define cache_wait      cache_wait_way
+#endif
+
 static inline void cache_sync(void)
 {
         void __iomem *base = l2x0_base;
+
+#ifdef CONFIG_ARM_ERRATA_753970
+        /* write to an unmmapped register */
+        writel_relaxed(0, base + L2X0_DUMMY_REG);
+#else
         writel_relaxed(0, base + L2X0_CACHE_SYNC);
+#endif
         cache_wait(base + L2X0_CACHE_SYNC, 1);
 }
 
@@ -57,18 +73,24 @@ static inline void l2x0_inv_line(unsigned long addr)
         writel_relaxed(addr, base + L2X0_INV_LINE_PA);
 }
 
-#ifdef CONFIG_PL310_ERRATA_588369
-static void debug_writel(unsigned long val)
-{
-        extern void omap_smc1(u32 fn, u32 arg);
+#if defined(CONFIG_PL310_ERRATA_588369) || defined(CONFIG_PL310_ERRATA_727915)
 
-        /*
-         * Texas Instrument secure monitor api to modify the
-         * PL310 Debug Control Register.
-         */
-        omap_smc1(0x100, val);
+#define debug_writel(val)       outer_cache.set_debug(val)
+
+static void l2x0_set_debug(unsigned long val)
+{
+        writel_relaxed(val, l2x0_base + L2X0_DEBUG_CTRL);
 }
+#else
+/* Optimised out for non-errata case */
+static inline void debug_writel(unsigned long val)
+{
+}
+
+#define l2x0_set_debug  NULL
+#endif
 
+#ifdef CONFIG_PL310_ERRATA_588369
 static inline void l2x0_flush_line(unsigned long addr)
 {
         void __iomem *base = l2x0_base;
@@ -81,11 +103,6 @@ static inline void l2x0_flush_line(unsigned long addr)
 }
 #else
 
-/* Optimised out for non-errata case */
-static inline void debug_writel(unsigned long val)
-{
-}
-
 static inline void l2x0_flush_line(unsigned long addr)
 {
         void __iomem *base = l2x0_base;
@@ -103,14 +120,47 @@ static void l2x0_cache_sync(void)
         spin_unlock_irqrestore(&l2x0_lock, flags);
 }
 
-static inline void l2x0_inv_all(void)
+static void __l2x0_flush_all(void)
+{
+        debug_writel(0x03);
+        writel_relaxed(l2x0_way_mask, l2x0_base + L2X0_CLEAN_INV_WAY);
+        cache_wait_way(l2x0_base + L2X0_CLEAN_INV_WAY, l2x0_way_mask);
+        cache_sync();
+        debug_writel(0x00);
+}
+
+static void l2x0_flush_all(void)
+{
+        unsigned long flags;
+
+        /* clean all ways */
+        spin_lock_irqsave(&l2x0_lock, flags);
+        __l2x0_flush_all();
+        spin_unlock_irqrestore(&l2x0_lock, flags);
+}
+
+static void l2x0_clean_all(void)
+{
+        unsigned long flags;
+
+        /* clean all ways */
+        spin_lock_irqsave(&l2x0_lock, flags);
+        writel_relaxed(l2x0_way_mask, l2x0_base + L2X0_CLEAN_WAY);
+        cache_wait_way(l2x0_base + L2X0_CLEAN_WAY, l2x0_way_mask);
+        cache_sync();
+        spin_unlock_irqrestore(&l2x0_lock, flags);
+}
+
+static void l2x0_inv_all(void)
 {
         unsigned long flags;
 
         /* invalidate all ways */
         spin_lock_irqsave(&l2x0_lock, flags);
+        /* Invalidating when L2 is enabled is a nono */
+        BUG_ON(readl(l2x0_base + L2X0_CTRL) & 1);
         writel_relaxed(l2x0_way_mask, l2x0_base + L2X0_INV_WAY);
-        cache_wait(l2x0_base + L2X0_INV_WAY, l2x0_way_mask);
+        cache_wait_way(l2x0_base + L2X0_INV_WAY, l2x0_way_mask);
         cache_sync();
         spin_unlock_irqrestore(&l2x0_lock, flags);
 }
@@ -159,6 +209,11 @@ static void l2x0_clean_range(unsigned long start, unsigned long end)
         void __iomem *base = l2x0_base;
         unsigned long flags;
 
+        if ((end - start) >= l2x0_size) {
+                l2x0_clean_all();
+                return;
+        }
+
         spin_lock_irqsave(&l2x0_lock, flags);
         start &= ~(CACHE_LINE_SIZE - 1);
         while (start < end) {
@@ -184,6 +239,11 @@ static void l2x0_flush_range(unsigned long start, unsigned long end)
         void __iomem *base = l2x0_base;
         unsigned long flags;
 
+        if ((end - start) >= l2x0_size) {
+                l2x0_flush_all();
+                return;
+        }
+
         spin_lock_irqsave(&l2x0_lock, flags);
         start &= ~(CACHE_LINE_SIZE - 1);
         while (start < end) {
@@ -206,10 +266,22 @@ static void l2x0_flush_range(unsigned long start, unsigned long end)
         spin_unlock_irqrestore(&l2x0_lock, flags);
 }
 
+static void l2x0_disable(void)
+{
+        unsigned long flags;
+
+        spin_lock_irqsave(&l2x0_lock, flags);
+        __l2x0_flush_all();
+        writel_relaxed(0, l2x0_base + L2X0_CTRL);
+        dsb();
+        spin_unlock_irqrestore(&l2x0_lock, flags);
+}
+
 void __init l2x0_init(void __iomem *base, __u32 aux_val, __u32 aux_mask)
 {
         __u32 aux;
         __u32 cache_id;
+        __u32 way_size = 0;
         int ways;
         const char *type;
 
@@ -244,6 +316,13 @@ void __init l2x0_init(void __iomem *base, __u32 aux_val, __u32 aux_mask)
         l2x0_way_mask = (1 << ways) - 1;
 
         /*
+         * L2 cache Size =  Way size * Number of ways
+         */
+        way_size = (aux & L2X0_AUX_CTRL_WAY_SIZE_MASK) >> 17;
+        way_size = 1 << (way_size + 3);
+        l2x0_size = ways * way_size * SZ_1K;
+
+        /*
          * Check if l2x0 controller is already enabled.
          * If you are booting from non-secure mode
          * accessing the below registers will fault.
@@ -263,8 +342,12 @@ void __init l2x0_init(void __iomem *base, __u32 aux_val, __u32 aux_mask)
         outer_cache.clean_range = l2x0_clean_range;
         outer_cache.flush_range = l2x0_flush_range;
         outer_cache.sync = l2x0_cache_sync;
+        outer_cache.flush_all = l2x0_flush_all;
+        outer_cache.inv_all = l2x0_inv_all;
+        outer_cache.disable = l2x0_disable;
+        outer_cache.set_debug = l2x0_set_debug;
 
         printk(KERN_INFO "%s cache controller enabled\n", type);
-        printk(KERN_INFO "l2x0: %d ways, CACHE_ID 0x%08x, AUX_CTRL 0x%08x\n",
-                         ways, cache_id, aux);
+        printk(KERN_INFO "l2x0: %d ways, CACHE_ID 0x%08x, AUX_CTRL 0x%08x, Cache size: %d B\n",
+                        ways, cache_id, aux, l2x0_size);
 }
diff --git a/arch/arm/mm/cache-v3.S b/arch/arm/mm/cache-v3.S
index c2ff3c599fee..2e2bc406a18d 100644
--- a/arch/arm/mm/cache-v3.S
+++ b/arch/arm/mm/cache-v3.S
@@ -13,6 +13,15 @@
 #include "proc-macros.S"
 
 /*
+ *      flush_icache_all()
+ *
+ *      Unconditionally clean and invalidate the entire icache.
+ */
+ENTRY(v3_flush_icache_all)
+        mov     pc, lr
+ENDPROC(v3_flush_icache_all)
+
+/*
  *      flush_user_cache_all()
  *
  *      Invalidate all cache entries in a particular address
@@ -122,6 +131,7 @@ ENDPROC(v3_dma_map_area)
 
         .type   v3_cache_fns, #object
 ENTRY(v3_cache_fns)
+        .long   v3_flush_icache_all
         .long   v3_flush_kern_cache_all
         .long   v3_flush_user_cache_all
         .long   v3_flush_user_cache_range
diff --git a/arch/arm/mm/cache-v4.S b/arch/arm/mm/cache-v4.S
index 4810f7e3e813..a8fefb523f19 100644
--- a/arch/arm/mm/cache-v4.S
+++ b/arch/arm/mm/cache-v4.S
@@ -13,6 +13,15 @@
 #include "proc-macros.S"
 
 /*
+ *      flush_icache_all()
+ *
+ *      Unconditionally clean and invalidate the entire icache.
+ */
+ENTRY(v4_flush_icache_all)
+        mov     pc, lr
+ENDPROC(v4_flush_icache_all)
+
+/*
  *      flush_user_cache_all()
  *
  *      Invalidate all cache entries in a particular address
@@ -134,6 +143,7 @@ ENDPROC(v4_dma_map_area)
 
         .type   v4_cache_fns, #object
 ENTRY(v4_cache_fns)
+        .long   v4_flush_icache_all
         .long   v4_flush_kern_cache_all
         .long   v4_flush_user_cache_all
         .long   v4_flush_user_cache_range
diff --git a/arch/arm/mm/cache-v4wb.S b/arch/arm/mm/cache-v4wb.S
index df8368afa102..f40c69656d8d 100644
--- a/arch/arm/mm/cache-v4wb.S
+++ b/arch/arm/mm/cache-v4wb.S
@@ -32,7 +32,7 @@
 /*
  * This is the size at which it becomes more efficient to
  * clean the whole cache, rather than using the individual
- * cache line maintainence instructions.
+ * cache line maintenance instructions.
  *
  *  Size  Clean (ticks) Dirty (ticks)
  *   4096   21  20  21    53  55  54
@@ -51,6 +51,17 @@ flush_base:
         .text
 
 /*
+ *      flush_icache_all()
+ *
+ *      Unconditionally clean and invalidate the entire icache.
+ */
+ENTRY(v4wb_flush_icache_all)
+        mov     r0, #0
+        mcr     p15, 0, r0, c7, c5, 0           @ invalidate I cache
+        mov     pc, lr
+ENDPROC(v4wb_flush_icache_all)
+
+/*
  *      flush_user_cache_all()
  *
  *      Clean and invalidate all cache entries in a particular address
@@ -244,6 +255,7 @@ ENDPROC(v4wb_dma_unmap_area)
 
         .type   v4wb_cache_fns, #object
 ENTRY(v4wb_cache_fns)
+        .long   v4wb_flush_icache_all
         .long   v4wb_flush_kern_cache_all
         .long   v4wb_flush_user_cache_all
         .long   v4wb_flush_user_cache_range
diff --git a/arch/arm/mm/cache-v4wt.S b/arch/arm/mm/cache-v4wt.S
index 45c70312f43b..a7b276dbda11 100644
--- a/arch/arm/mm/cache-v4wt.S
+++ b/arch/arm/mm/cache-v4wt.S
@@ -34,13 +34,24 @@
 /*
  * This is the size at which it becomes more efficient to
  * clean the whole cache, rather than using the individual
- * cache line maintainence instructions.
+ * cache line maintenance instructions.
  *
  * *** This needs benchmarking
  */
 #define CACHE_DLIMIT    16384
 
 /*
+ *      flush_icache_all()
+ *
+ *      Unconditionally clean and invalidate the entire icache.
+ */
+ENTRY(v4wt_flush_icache_all)
+        mov     r0, #0
+        mcr     p15, 0, r0, c7, c5, 0           @ invalidate I cache
+        mov     pc, lr
+ENDPROC(v4wt_flush_icache_all)
+
+/*
  *      flush_user_cache_all()
  *
  *      Invalidate all cache entries in a particular address
@@ -188,6 +199,7 @@ ENDPROC(v4wt_dma_map_area)
 
         .type   v4wt_cache_fns, #object
 ENTRY(v4wt_cache_fns)
+        .long   v4wt_flush_icache_all
         .long   v4wt_flush_kern_cache_all
         .long   v4wt_flush_user_cache_all
         .long   v4wt_flush_user_cache_range
diff --git a/arch/arm/mm/cache-v6.S b/arch/arm/mm/cache-v6.S
index 86aa689ef1aa..73b4a8b66a57 100644
--- a/arch/arm/mm/cache-v6.S
+++ b/arch/arm/mm/cache-v6.S
@@ -21,18 +21,22 @@
 #define D_CACHE_LINE_SIZE       32
 #define BTB_FLUSH_SIZE          8
 
-#ifdef CONFIG_ARM_ERRATA_411920
 /*
- * Invalidate the entire I cache (this code is a workaround for the ARM1136
- * erratum 411920 - Invalidate Instruction Cache operation can fail. This
- * erratum is present in 1136, 1156 and 1176. It does not affect the MPCore.
+ *      v6_flush_icache_all()
+ *
+ *      Flush the whole I-cache.
+ *
+ *      ARM1136 erratum 411920 - Invalidate Instruction Cache operation can fail.
+ *      This erratum is present in 1136, 1156 and 1176. It does not affect the
+ *      MPCore.
  *
- * Registers:
- *   r0 - set to 0
- *   r1 - corrupted
+ *      Registers:
+ *      r0 - set to 0
+ *      r1 - corrupted
  */
-ENTRY(v6_icache_inval_all)
+ENTRY(v6_flush_icache_all)
         mov     r0, #0
+#ifdef CONFIG_ARM_ERRATA_411920
         mrs     r1, cpsr
         cpsid   ifa                             @ disable interrupts
         mcr     p15, 0, r0, c7, c5, 0           @ invalidate entire I-cache
@@ -43,8 +47,11 @@ ENTRY(v6_icache_inval_all)
         .rept   11                              @ ARM Ltd recommends at least
         nop                                     @ 11 NOPs
         .endr
-        mov     pc, lr
+#else
+        mcr     p15, 0, r0, c7, c5, 0           @ invalidate I-cache
 #endif
+        mov     pc, lr
+ENDPROC(v6_flush_icache_all)
 
 /*
  *      v6_flush_cache_all()
@@ -60,7 +67,7 @@ ENTRY(v6_flush_kern_cache_all)
 #ifndef CONFIG_ARM_ERRATA_411920
         mcr     p15, 0, r0, c7, c5, 0           @ I+BTB cache invalidate
 #else
-        b       v6_icache_inval_all
+        b       v6_flush_icache_all
 #endif
 #else
         mcr     p15, 0, r0, c7, c15, 0          @ Cache clean+invalidate
@@ -138,7 +145,7 @@ ENTRY(v6_coherent_user_range)
 #ifndef CONFIG_ARM_ERRATA_411920
         mcr     p15, 0, r0, c7, c5, 0           @ I+BTB cache invalidate
 #else
-        b       v6_icache_inval_all
+        b       v6_flush_icache_all
 #endif
 #else
         mcr     p15, 0, r0, c7, c5, 6           @ invalidate BTB
@@ -169,6 +176,7 @@ ENDPROC(v6_coherent_kern_range)
  */
 ENTRY(v6_flush_kern_dcache_area)
         add     r1, r0, r1
+        bic     r0, r0, #D_CACHE_LINE_SIZE - 1
 1:
 #ifdef HARVARD_CACHE
         mcr     p15, 0, r0, c7, c14, 1          @ clean & invalidate D line
@@ -196,6 +204,10 @@ ENTRY(v6_flush_kern_dcache_area)
  *      - end     - virtual end address of region
  */
 v6_dma_inv_range:
+#ifdef CONFIG_DMA_CACHE_RWFO
+        ldrb    r2, [r0]                        @ read for ownership
+        strb    r2, [r0]                        @ write for ownership
+#endif
         tst     r0, #D_CACHE_LINE_SIZE - 1
         bic     r0, r0, #D_CACHE_LINE_SIZE - 1
 #ifdef HARVARD_CACHE
@@ -204,6 +216,10 @@ v6_dma_inv_range:
         mcrne   p15, 0, r0, c7, c11, 1          @ clean unified line
 #endif
         tst     r1, #D_CACHE_LINE_SIZE - 1
+#ifdef CONFIG_DMA_CACHE_RWFO
+        ldrneb  r2, [r1, #-1]                   @ read for ownership
+        strneb  r2, [r1, #-1]                   @ write for ownership
+#endif
         bic     r1, r1, #D_CACHE_LINE_SIZE - 1
 #ifdef HARVARD_CACHE
         mcrne   p15, 0, r1, c7, c14, 1          @ clean & invalidate D line
@@ -211,10 +227,6 @@ v6_dma_inv_range:
         mcrne   p15, 0, r1, c7, c15, 1          @ clean & invalidate unified line
 #endif
 1:
-#ifdef CONFIG_DMA_CACHE_RWFO
-        ldr     r2, [r0]                        @ read for ownership
-        str     r2, [r0]                        @ write for ownership
-#endif
 #ifdef HARVARD_CACHE
         mcr     p15, 0, r0, c7, c6, 1           @ invalidate D line
 #else
@@ -222,6 +234,10 @@ v6_dma_inv_range:
 #endif
         add     r0, r0, #D_CACHE_LINE_SIZE
         cmp     r0, r1
+#ifdef CONFIG_DMA_CACHE_RWFO
+        ldrlo   r2, [r0]                        @ read for ownership
+        strlo   r2, [r0]                        @ write for ownership
+#endif
         blo     1b
         mov     r0, #0
         mcr     p15, 0, r0, c7, c10, 4          @ drain write buffer
@@ -256,12 +272,12 @@ v6_dma_clean_range:
  *      - end     - virtual end address of region
  */
 ENTRY(v6_dma_flush_range)
-        bic     r0, r0, #D_CACHE_LINE_SIZE - 1
-1:
 #ifdef CONFIG_DMA_CACHE_RWFO
-        ldr     r2, [r0]                        @ read for ownership
-        str     r2, [r0]                        @ write for ownership
+        ldrb    r2, [r0]                @ read for ownership
+        strb    r2, [r0]                @ write for ownership
 #endif
+        bic     r0, r0, #D_CACHE_LINE_SIZE - 1
+1:
 #ifdef HARVARD_CACHE
         mcr     p15, 0, r0, c7, c14, 1          @ clean & invalidate D line
 #else
@@ -269,6 +285,10 @@ ENTRY(v6_dma_flush_range)
 #endif
         add     r0, r0, #D_CACHE_LINE_SIZE
         cmp     r0, r1
+#ifdef CONFIG_DMA_CACHE_RWFO
+        ldrlob  r2, [r0]                        @ read for ownership
+        strlob  r2, [r0]                        @ write for ownership
+#endif
         blo     1b
         mov     r0, #0
         mcr     p15, 0, r0, c7, c10, 4          @ drain write buffer
@@ -312,6 +332,7 @@ ENDPROC(v6_dma_unmap_area)
 
         .type   v6_cache_fns, #object
 ENTRY(v6_cache_fns)
+        .long   v6_flush_icache_all
         .long   v6_flush_kern_cache_all
         .long   v6_flush_user_cache_all
         .long   v6_flush_user_cache_range
diff --git a/arch/arm/mm/cache-v7.S b/arch/arm/mm/cache-v7.S
index 37c8157e116e..d32f02b61866 100644
--- a/arch/arm/mm/cache-v7.S
+++ b/arch/arm/mm/cache-v7.S
@@ -18,6 +18,21 @@
 #include "proc-macros.S"
 
 /*
+ *      v7_flush_icache_all()
+ *
+ *      Flush the whole I-cache.
+ *
+ *      Registers:
+ *      r0 - set to 0
+ */
+ENTRY(v7_flush_icache_all)
+        mov     r0, #0
+        ALT_SMP(mcr     p15, 0, r0, c7, c1, 0)          @ invalidate I-cache inner shareable
+        ALT_UP(mcr      p15, 0, r0, c7, c5, 0)          @ I+BTB cache invalidate
+        mov     pc, lr
+ENDPROC(v7_flush_icache_all)
+
+/*
  *      v7_flush_dcache_all()
  *
  *      Flush the whole D-cache.
@@ -81,7 +96,7 @@ ENDPROC(v7_flush_dcache_all)
  *      Flush the entire cache system.
  *  The data cache flush is now achieved using atomic clean / invalidates
  *  working outwards from L1 cache. This is done using Set/Way based cache
- *  maintainance instructions.
+ *  maintenance instructions.
  *  The instruction cache can still be invalidated back to the point of
  *  unification in a single instruction.
  *
@@ -91,11 +106,8 @@ ENTRY(v7_flush_kern_cache_all)
  THUMB( stmfd   sp!, {r4-r7, r9-r11, lr}        )
         bl      v7_flush_dcache_all
         mov     r0, #0
-#ifdef CONFIG_SMP
-        mcr     p15, 0, r0, c7, c1, 0           @ invalidate I-cache inner shareable
-#else
-        mcr     p15, 0, r0, c7, c5, 0           @ I+BTB cache invalidate
-#endif
+        ALT_SMP(mcr     p15, 0, r0, c7, c1, 0)  @ invalidate I-cache inner shareable
+        ALT_UP(mcr      p15, 0, r0, c7, c5, 0)  @ I+BTB cache invalidate
  ARM(   ldmfd   sp!, {r4-r5, r7, r9-r11, lr}    )
  THUMB( ldmfd   sp!, {r4-r7, r9-r11, lr}        )
         mov     pc, lr
@@ -161,21 +173,25 @@ ENTRY(v7_coherent_user_range)
  UNWIND(.fnstart                )
         dcache_line_size r2, r3
         sub     r3, r2, #1
-        bic     r0, r0, r3
+        bic     r12, r0, r3
 1:
- USER(  mcr     p15, 0, r0, c7, c11, 1  )       @ clean D line to the point of unification
+ USER(  mcr     p15, 0, r12, c7, c11, 1 )       @ clean D line to the point of unification
+        add     r12, r12, r2
+        cmp     r12, r1
+        blo     1b
         dsb
- USER(  mcr     p15, 0, r0, c7, c5, 1   )       @ invalidate I line
-        add     r0, r0, r2
+        icache_line_size r2, r3
+        sub     r3, r2, #1
+        bic     r12, r0, r3
 2:
-        cmp     r0, r1
-        blo     1b
+ USER(  mcr     p15, 0, r12, c7, c5, 1  )       @ invalidate I line
+        add     r12, r12, r2
+        cmp     r12, r1
+        blo     2b
+3:
         mov     r0, #0
-#ifdef CONFIG_SMP
-        mcr     p15, 0, r0, c7, c1, 6           @ invalidate BTB Inner Shareable
-#else
-        mcr     p15, 0, r0, c7, c5, 6           @ invalidate BTB
-#endif
+        ALT_SMP(mcr     p15, 0, r0, c7, c1, 6)  @ invalidate BTB Inner Shareable
+        ALT_UP(mcr      p15, 0, r0, c7, c5, 6)  @ invalidate BTB
         dsb
         isb
         mov     pc, lr
@@ -185,10 +201,10 @@ ENTRY(v7_coherent_user_range)
  * isn't mapped, just try the next page.
  */
 9001:
-        mov     r0, r0, lsr #12
-        mov     r0, r0, lsl #12
-        add     r0, r0, #4096
-        b       2b
+        mov     r12, r12, lsr #12
+        mov     r12, r12, lsl #12
+        add     r12, r12, #4096
+        b       3b
  UNWIND(.fnend          )
 ENDPROC(v7_coherent_kern_range)
 ENDPROC(v7_coherent_user_range)
@@ -205,6 +221,8 @@ ENDPROC(v7_coherent_user_range)
 ENTRY(v7_flush_kern_dcache_area)
         dcache_line_size r2, r3
         add     r1, r0, r1
+        sub     r3, r2, #1
+        bic     r0, r0, r3
 1:
         mcr     p15, 0, r0, c7, c14, 1          @ clean & invalidate D line / unified line
         add     r0, r0, r2
@@ -309,6 +327,7 @@ ENDPROC(v7_dma_unmap_area)
 
         .type   v7_cache_fns, #object
 ENTRY(v7_cache_fns)
+        .long   v7_flush_icache_all
         .long   v7_flush_kern_cache_all
         .long   v7_flush_user_cache_all
         .long   v7_flush_user_cache_range
diff --git a/arch/arm/mm/cache-xsc3l2.c b/arch/arm/mm/cache-xsc3l2.c
index c3154928bccd..5a32020471e3 100644
--- a/arch/arm/mm/cache-xsc3l2.c
+++ b/arch/arm/mm/cache-xsc3l2.c
@@ -17,14 +17,10 @@
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  */
 #include <linux/init.h>
+#include <linux/highmem.h>
 #include <asm/system.h>
 #include <asm/cputype.h>
 #include <asm/cacheflush.h>
-#include <asm/kmap_types.h>
-#include <asm/fixmap.h>
-#include <asm/pgtable.h>
-#include <asm/tlbflush.h>
-#include "mm.h"
 
 #define CR_L2   (1 << 26)
 
@@ -71,16 +67,15 @@ static inline void xsc3_l2_inv_all(void)
         dsb();
 }
 
+static inline void l2_unmap_va(unsigned long va)
+{
 #ifdef CONFIG_HIGHMEM
-#define l2_map_save_flags(x)            raw_local_save_flags(x)
-#define l2_map_restore_flags(x)         raw_local_irq_restore(x)
-#else
-#define l2_map_save_flags(x)            ((x) = 0)
-#define l2_map_restore_flags(x)         ((void)(x))
+        if (va != -1)
+                kunmap_atomic((void *)va);
 #endif
+}
 
-static inline unsigned long l2_map_va(unsigned long pa, unsigned long prev_va,
-                                      unsigned long flags)
+static inline unsigned long l2_map_va(unsigned long pa, unsigned long prev_va)
 {
 #ifdef CONFIG_HIGHMEM
         unsigned long va = prev_va & PAGE_MASK;
@@ -89,17 +84,10 @@ static inline unsigned long l2_map_va(unsigned long pa, unsigned long prev_va,
                 /*
                  * Switching to a new page.  Because cache ops are
                  * using virtual addresses only, we must put a mapping
-                 * in place for it.  We also enable interrupts for a
-                 * short while and disable them again to protect this
-                 * mapping.
+                 * in place for it.
                  */
-                unsigned long idx;
-                raw_local_irq_restore(flags);
-                idx = KM_L2_CACHE + KM_TYPE_NR * smp_processor_id();
-                va = __fix_to_virt(FIX_KMAP_BEGIN + idx);
-                raw_local_irq_restore(flags | PSR_I_BIT);
-                set_pte_ext(TOP_PTE(va), pfn_pte(pa >> PAGE_SHIFT, PAGE_KERNEL), 0);
-                local_flush_tlb_kernel_page(va);
+                l2_unmap_va(prev_va);
+                va = (unsigned long)kmap_atomic_pfn(pa >> PAGE_SHIFT);
         }
         return va + (pa_offset >> (32 - PAGE_SHIFT));
 #else
@@ -109,7 +97,7 @@ static inline unsigned long l2_map_va(unsigned long pa, unsigned long prev_va,
 
 static void xsc3_l2_inv_range(unsigned long start, unsigned long end)
 {
-        unsigned long vaddr, flags;
+        unsigned long vaddr;
 
         if (start == 0 && end == -1ul) {
                 xsc3_l2_inv_all();
@@ -117,13 +105,12 @@ static void xsc3_l2_inv_range(unsigned long start, unsigned long end)
         }
 
         vaddr = -1;  /* to force the first mapping */
-        l2_map_save_flags(flags);
 
         /*
          * Clean and invalidate partial first cache line.
          */
         if (start & (CACHE_LINE_SIZE - 1)) {
-                vaddr = l2_map_va(start & ~(CACHE_LINE_SIZE - 1), vaddr, flags);
+                vaddr = l2_map_va(start & ~(CACHE_LINE_SIZE - 1), vaddr);
                 xsc3_l2_clean_mva(vaddr);
                 xsc3_l2_inv_mva(vaddr);
                 start = (start | (CACHE_LINE_SIZE - 1)) + 1;
@@ -133,7 +120,7 @@ static void xsc3_l2_inv_range(unsigned long start, unsigned long end)
          * Invalidate all full cache lines between 'start' and 'end'.
          */
         while (start < (end & ~(CACHE_LINE_SIZE - 1))) {
-                vaddr = l2_map_va(start, vaddr, flags);
+                vaddr = l2_map_va(start, vaddr);
                 xsc3_l2_inv_mva(vaddr);
                 start += CACHE_LINE_SIZE;
         }
@@ -142,31 +129,30 @@ static void xsc3_l2_inv_range(unsigned long start, unsigned long end)
          * Clean and invalidate partial last cache line.
          */
         if (start < end) {
-                vaddr = l2_map_va(start, vaddr, flags);
+                vaddr = l2_map_va(start, vaddr);
                 xsc3_l2_clean_mva(vaddr);
                 xsc3_l2_inv_mva(vaddr);
         }
 
-        l2_map_restore_flags(flags);
+        l2_unmap_va(vaddr);
 
         dsb();
 }
 
 static void xsc3_l2_clean_range(unsigned long start, unsigned long end)
 {
-        unsigned long vaddr, flags;
+        unsigned long vaddr;
 
         vaddr = -1;  /* to force the first mapping */
-        l2_map_save_flags(flags);
 
         start &= ~(CACHE_LINE_SIZE - 1);
         while (start < end) {
-                vaddr = l2_map_va(start, vaddr, flags);
+                vaddr = l2_map_va(start, vaddr);
                 xsc3_l2_clean_mva(vaddr);
                 start += CACHE_LINE_SIZE;
         }
 
-        l2_map_restore_flags(flags);
+        l2_unmap_va(vaddr);
 
         dsb();
 }
@@ -193,7 +179,7 @@ static inline void xsc3_l2_flush_all(void)
 
 static void xsc3_l2_flush_range(unsigned long start, unsigned long end)
 {
-        unsigned long vaddr, flags;
+        unsigned long vaddr;
 
         if (start == 0 && end == -1ul) {
                 xsc3_l2_flush_all();
@@ -201,17 +187,16 @@ static void xsc3_l2_flush_range(unsigned long start, unsigned long end)
         }
 
         vaddr = -1;  /* to force the first mapping */
-        l2_map_save_flags(flags);
 
         start &= ~(CACHE_LINE_SIZE - 1);
         while (start < end) {
-                vaddr = l2_map_va(start, vaddr, flags);
+                vaddr = l2_map_va(start, vaddr);
                 xsc3_l2_clean_mva(vaddr);
                 xsc3_l2_inv_mva(vaddr);
                 start += CACHE_LINE_SIZE;
         }
 
-        l2_map_restore_flags(flags);
+        l2_unmap_va(vaddr);
 
         dsb();
 }
diff --git a/arch/arm/mm/copypage-v4mc.c b/arch/arm/mm/copypage-v4mc.c
index 598c51ad5071..b8061519ce77 100644
--- a/arch/arm/mm/copypage-v4mc.c
+++ b/arch/arm/mm/copypage-v4mc.c
@@ -73,7 +73,7 @@ void v4_mc_copy_user_highpage(struct page *to, struct page *from,
 {
         void *kto = kmap_atomic(to, KM_USER1);
 
-        if (test_and_clear_bit(PG_dcache_dirty, &from->flags))
+        if (!test_and_set_bit(PG_dcache_clean, &from->flags))
                 __flush_dcache_page(page_mapping(from), from);
 
         spin_lock(&minicache_lock);
diff --git a/arch/arm/mm/copypage-v6.c b/arch/arm/mm/copypage-v6.c
index f55fa1044f72..bdba6c65c901 100644
--- a/arch/arm/mm/copypage-v6.c
+++ b/arch/arm/mm/copypage-v6.c
@@ -79,7 +79,7 @@ static void v6_copy_user_highpage_aliasing(struct page *to,
         unsigned int offset = CACHE_COLOUR(vaddr);
         unsigned long kfrom, kto;
 
-        if (test_and_clear_bit(PG_dcache_dirty, &from->flags))
+        if (!test_and_set_bit(PG_dcache_clean, &from->flags))
                 __flush_dcache_page(page_mapping(from), from);
 
         /* FIXME: not highmem safe */
diff --git a/arch/arm/mm/copypage-xscale.c b/arch/arm/mm/copypage-xscale.c
index 9920c0ae2096..649bbcd325bf 100644
--- a/arch/arm/mm/copypage-xscale.c
+++ b/arch/arm/mm/copypage-xscale.c
@@ -95,7 +95,7 @@ void xscale_mc_copy_user_highpage(struct page *to, struct page *from,
 {
         void *kto = kmap_atomic(to, KM_USER1);
 
-        if (test_and_clear_bit(PG_dcache_dirty, &from->flags))
+        if (!test_and_set_bit(PG_dcache_clean, &from->flags))
                 __flush_dcache_page(page_mapping(from), from);
 
         spin_lock(&minicache_lock);
diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c
index 4bc43e535d3b..82a093cee09a 100644
--- a/arch/arm/mm/dma-mapping.c
+++ b/arch/arm/mm/dma-mapping.c
@@ -17,6 +17,7 @@
 #include <linux/init.h>
 #include <linux/device.h>
 #include <linux/dma-mapping.h>
+#include <linux/highmem.h>
 
 #include <asm/memory.h>
 #include <asm/highmem.h>
@@ -148,6 +149,7 @@ static int __init consistent_init(void)
 {
         int ret = 0;
         pgd_t *pgd;
+        pud_t *pud;
         pmd_t *pmd;
         pte_t *pte;
         int i = 0;
@@ -155,7 +157,15 @@ static int __init consistent_init(void)
 
         do {
                 pgd = pgd_offset(&init_mm, base);
-                pmd = pmd_alloc(&init_mm, pgd, base);
+
+                pud = pud_alloc(&init_mm, pgd, base);
+                if (!pud) {
+                        printk(KERN_ERR "%s: no pud tables\n", __func__);
+                        ret = -ENOMEM;
+                        break;
+                }
+
+                pmd = pmd_alloc(&init_mm, pud, base);
                 if (!pmd) {
                         printk(KERN_ERR "%s: no pmd tables\n", __func__);
                         ret = -ENOMEM;
@@ -198,7 +208,7 @@ __dma_alloc_remap(struct page *page, size_t size, gfp_t gfp, pgprot_t prot)
          * fragmentation of the DMA space, and also prevents allocations
          * smaller than a section from crossing a section boundary.
          */
-        bit = fls(size - 1) + 1;
+        bit = fls(size - 1);
         if (bit > SECTION_SHIFT)
                 bit = SECTION_SHIFT;
         align = 1 << bit;
@@ -311,7 +321,7 @@ __dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp,
                 addr = page_address(page);
 
         if (addr)
-                *handle = page_to_dma(dev, page);
+                *handle = pfn_to_dma(dev, page_to_pfn(page));
 
         return addr;
 }
@@ -406,7 +416,7 @@ void dma_free_coherent(struct device *dev, size_t size, void *cpu_addr, dma_addr
         if (!arch_is_coherent())
                 __dma_free_remap(cpu_addr, size);
 
-        __dma_free_buffer(dma_to_page(dev, handle), size);
+        __dma_free_buffer(pfn_to_page(dma_to_pfn(dev, handle)), size);
 }
 EXPORT_SYMBOL(dma_free_coherent);
 
@@ -480,10 +490,10 @@ static void dma_cache_maint_page(struct page *page, unsigned long offset,
                                 op(vaddr, len, dir);
                                 kunmap_high(page);
                         } else if (cache_is_vipt()) {
-                                pte_t saved_pte;
-                                vaddr = kmap_high_l1_vipt(page, &saved_pte);
+                                /* unmapped pages might still be cached */
+                                vaddr = kmap_atomic(page);
                                 op(vaddr + offset, len, dir);
-                                kunmap_high_l1_vipt(page, saved_pte);
+                                kunmap_atomic(vaddr);
                         }
                 } else {
                         vaddr = page_address(page) + offset;
@@ -523,6 +533,12 @@ void ___dma_page_dev_to_cpu(struct page *page, unsigned long off,
                 outer_inv_range(paddr, paddr + size);
 
         dma_cache_maint_page(page, off, size, dir, dmac_unmap_area);
+
+        /*
+         * Mark the D-cache clean for this page to avoid extra flushing.
+         */
+        if (dir != DMA_TO_DEVICE && off == 0 && size >= PAGE_SIZE)
+                set_bit(PG_dcache_clean, &page->flags);
 }
 EXPORT_SYMBOL(___dma_page_dev_to_cpu);
 
@@ -548,17 +564,20 @@ int dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
         struct scatterlist *s;
         int i, j;
 
+        BUG_ON(!valid_dma_direction(dir));
+
         for_each_sg(sg, s, nents, i) {
-                s->dma_address = dma_map_page(dev, sg_page(s), s->offset,
+                s->dma_address = __dma_map_page(dev, sg_page(s), s->offset,
                                                 s->length, dir);
                 if (dma_mapping_error(dev, s->dma_address))
                         goto bad_mapping;
         }
+        debug_dma_map_sg(dev, sg, nents, nents, dir);
         return nents;
 
  bad_mapping:
         for_each_sg(sg, s, i, j)
-                dma_unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir);
+                __dma_unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir);
         return 0;
 }
 EXPORT_SYMBOL(dma_map_sg);
@@ -567,7 +586,7 @@ EXPORT_SYMBOL(dma_map_sg);
  * dma_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg
  * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
  * @sg: list of buffers
- * @nents: number of buffers to unmap (returned from dma_map_sg)
+ * @nents: number of buffers to unmap (same as was passed to dma_map_sg)
  * @dir: DMA transfer direction (same as was passed to dma_map_sg)
  *
  * Unmap a set of streaming mode DMA translations.  Again, CPU access
@@ -579,8 +598,10 @@ void dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
         struct scatterlist *s;
         int i;
 
+        debug_dma_unmap_sg(dev, sg, nents, dir);
+
         for_each_sg(sg, s, nents, i)
-                dma_unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir);
+                __dma_unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir);
 }
 EXPORT_SYMBOL(dma_unmap_sg);
 
@@ -605,6 +626,8 @@ void dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
                 __dma_page_dev_to_cpu(sg_page(s), s->offset,
                                       s->length, dir);
         }
+
+        debug_dma_sync_sg_for_cpu(dev, sg, nents, dir);
 }
 EXPORT_SYMBOL(dma_sync_sg_for_cpu);
 
@@ -629,5 +652,16 @@ void dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
                 __dma_page_cpu_to_dev(sg_page(s), s->offset,
                                       s->length, dir);
         }
+
+        debug_dma_sync_sg_for_device(dev, sg, nents, dir);
 }
 EXPORT_SYMBOL(dma_sync_sg_for_device);
+
+#define PREALLOC_DMA_DEBUG_ENTRIES      4096
+
+static int __init dma_debug_do_init(void)
+{
+        dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
+        return 0;
+}
+fs_initcall(dma_debug_do_init);
diff --git a/arch/arm/mm/fault-armv.c b/arch/arm/mm/fault-armv.c
index 9b906dec1ca1..7cab79179421 100644
--- a/arch/arm/mm/fault-armv.c
+++ b/arch/arm/mm/fault-armv.c
@@ -26,8 +26,9 @@
 
 #include "mm.h"
 
-static unsigned long shared_pte_mask = L_PTE_MT_BUFFERABLE;
+static pteval_t shared_pte_mask = L_PTE_MT_BUFFERABLE;
 
+#if __LINUX_ARM_ARCH__ < 6
 /*
  * We take the easy way out of this problem - we make the
  * PTE uncacheable.  However, we leave the write buffer on.
@@ -65,11 +66,36 @@ static int do_adjust_pte(struct vm_area_struct *vma, unsigned long address,
         return ret;
 }
 
+#if USE_SPLIT_PTLOCKS
+/*
+ * If we are using split PTE locks, then we need to take the page
+ * lock here.  Otherwise we are using shared mm->page_table_lock
+ * which is already locked, thus cannot take it.
+ */
+static inline void do_pte_lock(spinlock_t *ptl)
+{
+        /*
+         * Use nested version here to indicate that we are already
+         * holding one similar spinlock.
+         */
+        spin_lock_nested(ptl, SINGLE_DEPTH_NESTING);
+}
+
+static inline void do_pte_unlock(spinlock_t *ptl)
+{
+        spin_unlock(ptl);
+}
+#else /* !USE_SPLIT_PTLOCKS */
+static inline void do_pte_lock(spinlock_t *ptl) {}
+static inline void do_pte_unlock(spinlock_t *ptl) {}
+#endif /* USE_SPLIT_PTLOCKS */
+
 static int adjust_pte(struct vm_area_struct *vma, unsigned long address,
         unsigned long pfn)
 {
         spinlock_t *ptl;
         pgd_t *pgd;
+        pud_t *pud;
         pmd_t *pmd;
         pte_t *pte;
         int ret;
@@ -78,7 +104,11 @@ static int adjust_pte(struct vm_area_struct *vma, unsigned long address,
         if (pgd_none_or_clear_bad(pgd))
                 return 0;
 
-        pmd = pmd_offset(pgd, address);
+        pud = pud_offset(pgd, address);
+        if (pud_none_or_clear_bad(pud))
+                return 0;
+
+        pmd = pmd_offset(pud, address);
         if (pmd_none_or_clear_bad(pmd))
                 return 0;
 
@@ -88,13 +118,13 @@ static int adjust_pte(struct vm_area_struct *vma, unsigned long address,
          * open-code the spin-locking.
          */
         ptl = pte_lockptr(vma->vm_mm, pmd);
-        pte = pte_offset_map_nested(pmd, address);
-        spin_lock(ptl);
+        pte = pte_offset_map(pmd, address);
+        do_pte_lock(ptl);
 
         ret = do_adjust_pte(vma, address, pfn, pte);
 
-        spin_unlock(ptl);
-        pte_unmap_nested(pte);
+        do_pte_unlock(ptl);
+        pte_unmap(pte);
 
         return ret;
 }
@@ -141,7 +171,7 @@ make_coherent(struct address_space *mapping, struct vm_area_struct *vma,
  * a page table, or changing an existing PTE.  Basically, there are two
  * things that we need to take care of:
  *
- *  1. If PG_dcache_dirty is set for the page, we need to ensure
+ *  1. If PG_dcache_clean is not set for the page, we need to ensure
  *     that any cache entries for the kernels virtual memory
  *     range are written back to the page.
  *  2. If we have multiple shared mappings of the same space in
@@ -168,10 +198,8 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long addr,
                 return;
 
         mapping = page_mapping(page);
-#ifndef CONFIG_SMP
-        if (test_and_clear_bit(PG_dcache_dirty, &page->flags))
+        if (!test_and_set_bit(PG_dcache_clean, &page->flags))
                 __flush_dcache_page(mapping, page);
-#endif
         if (mapping) {
                 if (cache_is_vivt())
                         make_coherent(mapping, vma, addr, ptep, pfn);
@@ -179,6 +207,7 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long addr,
                         __flush_icache_all();
         }
 }
+#endif  /* __LINUX_ARM_ARCH__ < 6 */
 
 /*
  * Check whether the write buffer has physical address aliasing
diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c
index 23b0b03af5ea..bc0e1d88fd3b 100644
--- a/arch/arm/mm/fault.c
+++ b/arch/arm/mm/fault.c
@@ -76,9 +76,11 @@ void show_pte(struct mm_struct *mm, unsigned long addr)
 
         printk(KERN_ALERT "pgd = %p\n", mm->pgd);
         pgd = pgd_offset(mm, addr);
-        printk(KERN_ALERT "[%08lx] *pgd=%08lx", addr, pgd_val(*pgd));
+        printk(KERN_ALERT "[%08lx] *pgd=%08llx",
+                        addr, (long long)pgd_val(*pgd));
 
         do {
+                pud_t *pud;
                 pmd_t *pmd;
                 pte_t *pte;
 
@@ -90,9 +92,21 @@ void show_pte(struct mm_struct *mm, unsigned long addr)
                         break;
                 }
 
-                pmd = pmd_offset(pgd, addr);
+                pud = pud_offset(pgd, addr);
+                if (PTRS_PER_PUD != 1)
+                        printk(", *pud=%08lx", pud_val(*pud));
+
+                if (pud_none(*pud))
+                        break;
+
+                if (pud_bad(*pud)) {
+                        printk("(bad)");
+                        break;
+                }
+
+                pmd = pmd_offset(pud, addr);
                 if (PTRS_PER_PMD != 1)
-                        printk(", *pmd=%08lx", pmd_val(*pmd));
+                        printk(", *pmd=%08llx", (long long)pmd_val(*pmd));
 
                 if (pmd_none(*pmd))
                         break;
@@ -107,8 +121,9 @@ void show_pte(struct mm_struct *mm, unsigned long addr)
                         break;
 
                 pte = pte_offset_map(pmd, addr);
-                printk(", *pte=%08lx", pte_val(*pte));
-                printk(", *ppte=%08lx", pte_val(pte[-PTRS_PER_PTE]));
+                printk(", *pte=%08llx", (long long)pte_val(*pte));
+                printk(", *ppte=%08llx",
+                       (long long)pte_val(pte[PTE_HWTABLE_PTRS]));
                 pte_unmap(pte);
         } while(0);
 
@@ -388,6 +403,7 @@ do_translation_fault(unsigned long addr, unsigned int fsr,
 {
         unsigned int index;
         pgd_t *pgd, *pgd_k;
+        pud_t *pud, *pud_k;
         pmd_t *pmd, *pmd_k;
 
         if (addr < TASK_SIZE)
@@ -406,12 +422,19 @@ do_translation_fault(unsigned long addr, unsigned int fsr,
 
         if (pgd_none(*pgd_k))
                 goto bad_area;
-
         if (!pgd_present(*pgd))
                 set_pgd(pgd, *pgd_k);
 
-        pmd_k = pmd_offset(pgd_k, addr);
-        pmd   = pmd_offset(pgd, addr);
+        pud = pud_offset(pgd, addr);
+        pud_k = pud_offset(pgd_k, addr);
+
+        if (pud_none(*pud_k))
+                goto bad_area;
+        if (!pud_present(*pud))
+                set_pud(pud, *pud_k);
+
+        pmd = pmd_offset(pud, addr);
+        pmd_k = pmd_offset(pud_k, addr);
 
         /*
          * On ARM one Linux PGD entry contains two hardware entries (see page
@@ -581,6 +604,19 @@ static struct fsr_info ifsr_info[] = {
         { do_bad,               SIGBUS,  0,             "unknown 31"                       },
 };
 
+void __init
+hook_ifault_code(int nr, int (*fn)(unsigned long, unsigned int, struct pt_regs *),
+                 int sig, int code, const char *name)
+{
+        if (nr < 0 || nr >= ARRAY_SIZE(ifsr_info))
+                BUG();
+
+        ifsr_info[nr].fn   = fn;
+        ifsr_info[nr].sig  = sig;
+        ifsr_info[nr].code = code;
+        ifsr_info[nr].name = name;
+}
+
 asmlinkage void __exception
 do_PrefetchAbort(unsigned long addr, unsigned int ifsr, struct pt_regs *regs)
 {
diff --git a/arch/arm/mm/flush.c b/arch/arm/mm/flush.c
index c6844cb9b508..1a8d4aa821be 100644
--- a/arch/arm/mm/flush.c
+++ b/arch/arm/mm/flush.c
@@ -10,6 +10,7 @@
 #include <linux/module.h>
 #include <linux/mm.h>
 #include <linux/pagemap.h>
+#include <linux/highmem.h>
 
 #include <asm/cacheflush.h>
 #include <asm/cachetype.h>
@@ -39,6 +40,18 @@ static void flush_pfn_alias(unsigned long pfn, unsigned long vaddr)
             : "cc");
 }
 
+static void flush_icache_alias(unsigned long pfn, unsigned long vaddr, unsigned long len)
+{
+        unsigned long colour = CACHE_COLOUR(vaddr);
+        unsigned long offset = vaddr & (PAGE_SIZE - 1);
+        unsigned long to;
+
+        set_pte_ext(TOP_PTE(ALIAS_FLUSH_START) + colour, pfn_pte(pfn, PAGE_KERNEL), 0);
+        to = ALIAS_FLUSH_START + (colour << PAGE_SHIFT) + offset;
+        flush_tlb_kernel_page(to);
+        flush_icache_range(to, to + len);
+}
+
 void flush_cache_mm(struct mm_struct *mm)
 {
         if (cache_is_vivt()) {
@@ -89,16 +102,16 @@ void flush_cache_page(struct vm_area_struct *vma, unsigned long user_addr, unsig
         if (vma->vm_flags & VM_EXEC && icache_is_vivt_asid_tagged())
                 __flush_icache_all();
 }
+
 #else
-#define flush_pfn_alias(pfn,vaddr)      do { } while (0)
+#define flush_pfn_alias(pfn,vaddr)              do { } while (0)
+#define flush_icache_alias(pfn,vaddr,len)       do { } while (0)
 #endif
 
-#ifdef CONFIG_SMP
 static void flush_ptrace_access_other(void *args)
 {
         __flush_icache_all();
 }
-#endif
 
 static
 void flush_ptrace_access(struct vm_area_struct *vma, struct page *page,
@@ -118,15 +131,16 @@ void flush_ptrace_access(struct vm_area_struct *vma, struct page *page,
                 return;
         }
 
-        /* VIPT non-aliasing cache */
+        /* VIPT non-aliasing D-cache */
         if (vma->vm_flags & VM_EXEC) {
                 unsigned long addr = (unsigned long)kaddr;
-                __cpuc_coherent_kern_range(addr, addr + len);
-#ifdef CONFIG_SMP
+                if (icache_is_vipt_aliasing())
+                        flush_icache_alias(page_to_pfn(page), uaddr, len);
+                else
+                        __cpuc_coherent_kern_range(addr, addr + len);
                 if (cache_ops_need_broadcast())
                         smp_call_function(flush_ptrace_access_other,
                                           NULL, 1);
-#endif
         }
 }
 
@@ -166,10 +180,10 @@ void __flush_dcache_page(struct address_space *mapping, struct page *page)
                         __cpuc_flush_dcache_area(addr, PAGE_SIZE);
                         kunmap_high(page);
                 } else if (cache_is_vipt()) {
-                        pte_t saved_pte;
-                        addr = kmap_high_l1_vipt(page, &saved_pte);
+                        /* unmapped pages might still be cached */
+                        addr = kmap_atomic(page);
                         __cpuc_flush_dcache_area(addr, PAGE_SIZE);
-                        kunmap_high_l1_vipt(page, saved_pte);
+                        kunmap_atomic(addr);
                 }
         }
 
@@ -215,6 +229,36 @@ static void __flush_dcache_aliases(struct address_space *mapping, struct page *p
         flush_dcache_mmap_unlock(mapping);
 }
 
+#if __LINUX_ARM_ARCH__ >= 6
+void __sync_icache_dcache(pte_t pteval)
+{
+        unsigned long pfn;
+        struct page *page;
+        struct address_space *mapping;
+
+        if (!pte_present_user(pteval))
+                return;
+        if (cache_is_vipt_nonaliasing() && !pte_exec(pteval))
+                /* only flush non-aliasing VIPT caches for exec mappings */
+                return;
+        pfn = pte_pfn(pteval);
+        if (!pfn_valid(pfn))
+                return;
+
+        page = pfn_to_page(pfn);
+        if (cache_is_vipt_aliasing())
+                mapping = page_mapping(page);
+        else
+                mapping = NULL;
+
+        if (!test_and_set_bit(PG_dcache_clean, &page->flags))
+                __flush_dcache_page(mapping, page);
+
+        if (pte_exec(pteval))
+                __flush_icache_all();
+}
+#endif
+
 /*
  * Ensure cache coherency between kernel mapping and userspace mapping
  * of this page.
@@ -231,7 +275,8 @@ static void __flush_dcache_aliases(struct address_space *mapping, struct page *p
  *  kernel cache lines for later.  Otherwise, we assume we have
  *  aliasing mappings.
  *
- * Note that we disable the lazy flush for SMP.
+ * Note that we disable the lazy flush for SMP configurations where
+ * the cache maintenance operations are not automatically broadcasted.
  */
 void flush_dcache_page(struct page *page)
 {
@@ -246,17 +291,16 @@ void flush_dcache_page(struct page *page)
 
         mapping = page_mapping(page);
 
-#ifndef CONFIG_SMP
-        if (!PageHighMem(page) && mapping && !mapping_mapped(mapping))
-                set_bit(PG_dcache_dirty, &page->flags);
-        else
-#endif
-        {
+        if (!cache_ops_need_broadcast() &&
+            mapping && !mapping_mapped(mapping))
+                clear_bit(PG_dcache_clean, &page->flags);
+        else {
                 __flush_dcache_page(mapping, page);
                 if (mapping && cache_is_vivt())
                         __flush_dcache_aliases(mapping, page);
                 else if (mapping)
                         __flush_icache_all();
+                set_bit(PG_dcache_clean, &page->flags);
         }
 }
 EXPORT_SYMBOL(flush_dcache_page);
diff --git a/arch/arm/mm/highmem.c b/arch/arm/mm/highmem.c
index 1fbdb55bfd1b..807c0573abbe 100644
--- a/arch/arm/mm/highmem.c
+++ b/arch/arm/mm/highmem.c
@@ -36,18 +36,17 @@ void kunmap(struct page *page)
 }
 EXPORT_SYMBOL(kunmap);
 
-void *kmap_atomic(struct page *page, enum km_type type)
+void *__kmap_atomic(struct page *page)
 {
         unsigned int idx;
         unsigned long vaddr;
         void *kmap;
+        int type;
 
         pagefault_disable();
         if (!PageHighMem(page))
                 return page_address(page);
 
-        debug_kmap_atomic(type);
-
 #ifdef CONFIG_DEBUG_HIGHMEM
         /*
          * There is no cache coherency issue when non VIVT, so force the
@@ -61,6 +60,8 @@ void *kmap_atomic(struct page *page, enum km_type type)
         if (kmap)
                 return kmap;
 
+        type = kmap_atomic_idx_push();
+
         idx = type + KM_TYPE_NR * smp_processor_id();
         vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx);
 #ifdef CONFIG_DEBUG_HIGHMEM
@@ -80,14 +81,17 @@ void *kmap_atomic(struct page *page, enum km_type type)
 
         return (void *)vaddr;
 }
-EXPORT_SYMBOL(kmap_atomic);
+EXPORT_SYMBOL(__kmap_atomic);
 
-void kunmap_atomic_notypecheck(void *kvaddr, enum km_type type)
+void __kunmap_atomic(void *kvaddr)
 {
         unsigned long vaddr = (unsigned long) kvaddr & PAGE_MASK;
-        unsigned int idx = type + KM_TYPE_NR * smp_processor_id();
+        int idx, type;
 
         if (kvaddr >= (void *)FIXADDR_START) {
+                type = kmap_atomic_idx();
+                idx = type + KM_TYPE_NR * smp_processor_id();
+
                 if (cache_is_vivt())
                         __cpuc_flush_dcache_area((void *)vaddr, PAGE_SIZE);
 #ifdef CONFIG_DEBUG_HIGHMEM
@@ -97,21 +101,23 @@ void kunmap_atomic_notypecheck(void *kvaddr, enum km_type type)
 #else
                 (void) idx;  /* to kill a warning */
 #endif
+                kmap_atomic_idx_pop();
         } else if (vaddr >= PKMAP_ADDR(0) && vaddr < PKMAP_ADDR(LAST_PKMAP)) {
                 /* this address was obtained through kmap_high_get() */
                 kunmap_high(pte_page(pkmap_page_table[PKMAP_NR(vaddr)]));
         }
         pagefault_enable();
 }
-EXPORT_SYMBOL(kunmap_atomic_notypecheck);
+EXPORT_SYMBOL(__kunmap_atomic);
 
-void *kmap_atomic_pfn(unsigned long pfn, enum km_type type)
+void *kmap_atomic_pfn(unsigned long pfn)
 {
-        unsigned int idx;
         unsigned long vaddr;
+        int idx, type;
 
         pagefault_disable();
 
+        type = kmap_atomic_idx_push();
         idx = type + KM_TYPE_NR * smp_processor_id();
         vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx);
 #ifdef CONFIG_DEBUG_HIGHMEM
@@ -134,90 +140,3 @@ struct page *kmap_atomic_to_page(const void *ptr)
         pte = TOP_PTE(vaddr);
         return pte_page(*pte);
 }
-
-#ifdef CONFIG_CPU_CACHE_VIPT
-
-#include <linux/percpu.h>
-
-/*
- * The VIVT cache of a highmem page is always flushed before the page
- * is unmapped. Hence unmapped highmem pages need no cache maintenance
- * in that case.
- *
- * However unmapped pages may still be cached with a VIPT cache, and
- * it is not possible to perform cache maintenance on them using physical
- * addresses unfortunately.  So we have no choice but to set up a temporary
- * virtual mapping for that purpose.
- *
- * Yet this VIPT cache maintenance may be triggered from DMA support
- * functions which are possibly called from interrupt context. As we don't
- * want to keep interrupt disabled all the time when such maintenance is
- * taking place, we therefore allow for some reentrancy by preserving and
- * restoring the previous fixmap entry before the interrupted context is
- * resumed.  If the reentrancy depth is 0 then there is no need to restore
- * the previous fixmap, and leaving the current one in place allow it to
- * be reused the next time without a TLB flush (common with DMA).
- */
-
-static DEFINE_PER_CPU(int, kmap_high_l1_vipt_depth);
-
-void *kmap_high_l1_vipt(struct page *page, pte_t *saved_pte)
-{
-        unsigned int idx, cpu;
-        int *depth;
-        unsigned long vaddr, flags;
-        pte_t pte, *ptep;
-
-        if (!in_interrupt())
-                preempt_disable();
-
-        cpu = smp_processor_id();
-        depth = &per_cpu(kmap_high_l1_vipt_depth, cpu);
-
-        idx = KM_L1_CACHE + KM_TYPE_NR * cpu;
-        vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx);
-        ptep = TOP_PTE(vaddr);
-        pte = mk_pte(page, kmap_prot);
-
-        raw_local_irq_save(flags);
-        (*depth)++;
-        if (pte_val(*ptep) == pte_val(pte)) {
-                *saved_pte = pte;
-        } else {
-                *saved_pte = *ptep;
-                set_pte_ext(ptep, pte, 0);
-                local_flush_tlb_kernel_page(vaddr);
-        }
-        raw_local_irq_restore(flags);
-
-        return (void *)vaddr;
-}
-
-void kunmap_high_l1_vipt(struct page *page, pte_t saved_pte)
-{
-        unsigned int idx, cpu = smp_processor_id();
-        int *depth = &per_cpu(kmap_high_l1_vipt_depth, cpu);
-        unsigned long vaddr, flags;
-        pte_t pte, *ptep;
-
-        idx = KM_L1_CACHE + KM_TYPE_NR * cpu;
-        vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx);
-        ptep = TOP_PTE(vaddr);
-        pte = mk_pte(page, kmap_prot);
-
-        BUG_ON(pte_val(*ptep) != pte_val(pte));
-        BUG_ON(*depth <= 0);
-
-        raw_local_irq_save(flags);
-        (*depth)--;
-        if (*depth != 0 && pte_val(pte) != pte_val(saved_pte)) {
-                set_pte_ext(ptep, saved_pte, 0);
-                local_flush_tlb_kernel_page(vaddr);
-        }
-        raw_local_irq_restore(flags);
-
-        if (!in_interrupt())
-                preempt_enable();
-}
-
-#endif  /* CONFIG_CPU_CACHE_VIPT */
diff --git a/arch/arm/mm/idmap.c b/arch/arm/mm/idmap.c
new file mode 100644
index 000000000000..2be9139a4ef3
--- /dev/null
+++ b/arch/arm/mm/idmap.c
@@ -0,0 +1,90 @@
+#include <linux/kernel.h>
+
+#include <asm/cputype.h>
+#include <asm/pgalloc.h>
+#include <asm/pgtable.h>
+
+static void idmap_add_pmd(pud_t *pud, unsigned long addr, unsigned long end,
+        unsigned long prot)
+{
+        pmd_t *pmd = pmd_offset(pud, addr);
+
+        addr = (addr & PMD_MASK) | prot;
+        pmd[0] = __pmd(addr);
+        addr += SECTION_SIZE;
+        pmd[1] = __pmd(addr);
+        flush_pmd_entry(pmd);
+}
+
+static void idmap_add_pud(pgd_t *pgd, unsigned long addr, unsigned long end,
+        unsigned long prot)
+{
+        pud_t *pud = pud_offset(pgd, addr);
+        unsigned long next;
+
+        do {
+                next = pud_addr_end(addr, end);
+                idmap_add_pmd(pud, addr, next, prot);
+        } while (pud++, addr = next, addr != end);
+}
+
+void identity_mapping_add(pgd_t *pgd, unsigned long addr, unsigned long end)
+{
+        unsigned long prot, next;
+
+        prot = PMD_TYPE_SECT | PMD_SECT_AP_WRITE;
+        if (cpu_architecture() <= CPU_ARCH_ARMv5TEJ && !cpu_is_xscale())
+                prot |= PMD_BIT4;
+
+        pgd += pgd_index(addr);
+        do {
+                next = pgd_addr_end(addr, end);
+                idmap_add_pud(pgd, addr, next, prot);
+        } while (pgd++, addr = next, addr != end);
+}
+
+#ifdef CONFIG_SMP
+static void idmap_del_pmd(pud_t *pud, unsigned long addr, unsigned long end)
+{
+        pmd_t *pmd = pmd_offset(pud, addr);
+        pmd_clear(pmd);
+}
+
+static void idmap_del_pud(pgd_t *pgd, unsigned long addr, unsigned long end)
+{
+        pud_t *pud = pud_offset(pgd, addr);
+        unsigned long next;
+
+        do {
+                next = pud_addr_end(addr, end);
+                idmap_del_pmd(pud, addr, next);
+        } while (pud++, addr = next, addr != end);
+}
+
+void identity_mapping_del(pgd_t *pgd, unsigned long addr, unsigned long end)
+{
+        unsigned long next;
+
+        pgd += pgd_index(addr);
+        do {
+                next = pgd_addr_end(addr, end);
+                idmap_del_pud(pgd, addr, next);
+        } while (pgd++, addr = next, addr != end);
+}
+#endif
+
+/*
+ * In order to soft-boot, we need to insert a 1:1 mapping in place of
+ * the user-mode pages.  This will then ensure that we have predictable
+ * results when turning the mmu off
+ */
+void setup_mm_for_reboot(char mode)
+{
+        /*
+         * We need to access to user-mode page tables here. For kernel threads
+         * we don't have any user-mode mappings so we use the context that we
+         * "borrowed".
+         */
+        identity_mapping_add(current->active_mm->pgd, 0, TASK_SIZE);
+        local_flush_tlb_all();
+}
diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c
index 7185b00650fe..c19571c40a21 100644
--- a/arch/arm/mm/init.c
+++ b/arch/arm/mm/init.c
@@ -15,11 +15,14 @@
 #include <linux/mman.h>
 #include <linux/nodemask.h>
 #include <linux/initrd.h>
+#include <linux/of_fdt.h>
 #include <linux/highmem.h>
 #include <linux/gfp.h>
 #include <linux/memblock.h>
+#include <linux/sort.h>
 
 #include <asm/mach-types.h>
+#include <asm/prom.h>
 #include <asm/sections.h>
 #include <asm/setup.h>
 #include <asm/sizes.h>
@@ -70,6 +73,14 @@ static int __init parse_tag_initrd2(const struct tag *tag)
 
 __tagtable(ATAG_INITRD2, parse_tag_initrd2);
 
+#ifdef CONFIG_OF_FLATTREE
+void __init early_init_dt_setup_initrd_arch(unsigned long start, unsigned long end)
+{
+        phys_initrd_start = start;
+        phys_initrd_size = end - start;
+}
+#endif /* CONFIG_OF_FLATTREE */
+
 /*
  * This keeps memory configuration data used by a couple memory
  * initialization functions, as well as show_mem() for the skipping
@@ -77,14 +88,14 @@ __tagtable(ATAG_INITRD2, parse_tag_initrd2);
  */
 struct meminfo meminfo;
 
-void show_mem(void)
+void show_mem(unsigned int filter)
 {
         int free = 0, total = 0, reserved = 0;
         int shared = 0, cached = 0, slab = 0, i;
         struct meminfo * mi = &meminfo;
 
         printk("Mem-info:\n");
-        show_free_areas();
+        show_free_areas(filter);
 
         for_each_bank (i, mi) {
                 struct membank *bank = &mi->bank[i];
@@ -121,9 +132,10 @@ void show_mem(void)
         printk("%d pages swap cached\n", cached);
 }
 
-static void __init find_limits(struct meminfo *mi,
-        unsigned long *min, unsigned long *max_low, unsigned long *max_high)
+static void __init find_limits(unsigned long *min, unsigned long *max_low,
+        unsigned long *max_high)
 {
+        struct meminfo *mi = &meminfo;
         int i;
 
         *min = -1UL;
@@ -147,13 +159,13 @@ static void __init find_limits(struct meminfo *mi,
         }
 }
 
-static void __init arm_bootmem_init(struct meminfo *mi,
-        unsigned long start_pfn, unsigned long end_pfn)
+static void __init arm_bootmem_init(unsigned long start_pfn,
+        unsigned long end_pfn)
 {
+        struct memblock_region *reg;
         unsigned int boot_pages;
         phys_addr_t bitmap;
         pg_data_t *pgdat;
-        int i;
 
         /*
          * Allocate the bootmem bitmap page.  This must be in a region
@@ -171,30 +183,53 @@ static void __init arm_bootmem_init(struct meminfo *mi,
         pgdat = NODE_DATA(0);
         init_bootmem_node(pgdat, __phys_to_pfn(bitmap), start_pfn, end_pfn);
 
-        for_each_bank(i, mi) {
-                struct membank *bank = &mi->bank[i];
-                if (!bank->highmem)
-                        free_bootmem(bank_phys_start(bank), bank_phys_size(bank));
+        /* Free the lowmem regions from memblock into bootmem. */
+        for_each_memblock(memory, reg) {
+                unsigned long start = memblock_region_memory_base_pfn(reg);
+                unsigned long end = memblock_region_memory_end_pfn(reg);
+
+                if (end >= end_pfn)
+                        end = end_pfn;
+                if (start >= end)
+                        break;
+
+                free_bootmem(__pfn_to_phys(start), (end - start) << PAGE_SHIFT);
         }
 
-        /*
-         * Reserve the memblock reserved regions in bootmem.
-         */
-        for (i = 0; i < memblock.reserved.cnt; i++) {
-                phys_addr_t start = memblock_start_pfn(&memblock.reserved, i);
-                if (start >= start_pfn &&
-                    memblock_end_pfn(&memblock.reserved, i) <= end_pfn)
-                        reserve_bootmem_node(pgdat, __pfn_to_phys(start),
-                                memblock_size_bytes(&memblock.reserved, i),
-                                BOOTMEM_DEFAULT);
+        /* Reserve the lowmem memblock reserved regions in bootmem. */
+        for_each_memblock(reserved, reg) {
+                unsigned long start = memblock_region_reserved_base_pfn(reg);
+                unsigned long end = memblock_region_reserved_end_pfn(reg);
+
+                if (end >= end_pfn)
+                        end = end_pfn;
+                if (start >= end)
+                        break;
+
+                reserve_bootmem(__pfn_to_phys(start),
+                                (end - start) << PAGE_SHIFT, BOOTMEM_DEFAULT);
         }
 }
 
-static void __init arm_bootmem_free(struct meminfo *mi, unsigned long min,
-        unsigned long max_low, unsigned long max_high)
+#ifdef CONFIG_ZONE_DMA
+static void __init arm_adjust_dma_zone(unsigned long *size, unsigned long *hole,
+        unsigned long dma_size)
+{
+        if (size[0] <= dma_size)
+                return;
+
+        size[ZONE_NORMAL] = size[0] - dma_size;
+        size[ZONE_DMA] = dma_size;
+        hole[ZONE_NORMAL] = hole[0];
+        hole[ZONE_DMA] = 0;
+}
+#endif
+
+static void __init arm_bootmem_free(unsigned long min, unsigned long max_low,
+        unsigned long max_high)
 {
         unsigned long zone_size[MAX_NR_ZONES], zhole_size[MAX_NR_ZONES];
-        int i;
+        struct memblock_region *reg;
 
         /*
          * initialise the zones.
@@ -216,72 +251,97 @@ static void __init arm_bootmem_free(struct meminfo *mi, unsigned long min,
          *  holes = node_size - sum(bank_sizes)
          */
         memcpy(zhole_size, zone_size, sizeof(zhole_size));
-        for_each_bank(i, mi) {
-                int idx = 0;
+        for_each_memblock(memory, reg) {
+                unsigned long start = memblock_region_memory_base_pfn(reg);
+                unsigned long end = memblock_region_memory_end_pfn(reg);
+
+                if (start < max_low) {
+                        unsigned long low_end = min(end, max_low);
+                        zhole_size[0] -= low_end - start;
+                }
 #ifdef CONFIG_HIGHMEM
-                if (mi->bank[i].highmem)
-                        idx = ZONE_HIGHMEM;
+                if (end > max_low) {
+                        unsigned long high_start = max(start, max_low);
+                        zhole_size[ZONE_HIGHMEM] -= end - high_start;
+                }
 #endif
-                zhole_size[idx] -= bank_pfn_size(&mi->bank[i]);
         }
 
+#ifdef ARM_DMA_ZONE_SIZE
+#ifndef CONFIG_ZONE_DMA
+#error ARM_DMA_ZONE_SIZE set but no DMA zone to limit allocations
+#endif
+
         /*
          * Adjust the sizes according to any special requirements for
          * this machine type.
          */
-        arch_adjust_zones(zone_size, zhole_size);
+        arm_adjust_dma_zone(zone_size, zhole_size,
+                ARM_DMA_ZONE_SIZE >> PAGE_SHIFT);
+#endif
 
         free_area_init_node(0, zone_size, min, zhole_size);
 }
 
-#ifndef CONFIG_SPARSEMEM
+#ifdef CONFIG_HAVE_ARCH_PFN_VALID
 int pfn_valid(unsigned long pfn)
 {
-        struct memblock_region *mem = &memblock.memory;
-        unsigned int left = 0, right = mem->cnt;
-
-        do {
-                unsigned int mid = (right + left) / 2;
-
-                if (pfn < memblock_start_pfn(mem, mid))
-                        right = mid;
-                else if (pfn >= memblock_end_pfn(mem, mid))
-                        left = mid + 1;
-                else
-                        return 1;
-        } while (left < right);
-        return 0;
+        return memblock_is_memory(pfn << PAGE_SHIFT);
 }
 EXPORT_SYMBOL(pfn_valid);
+#endif
 
+#ifndef CONFIG_SPARSEMEM
 static void arm_memory_present(void)
 {
 }
 #else
 static void arm_memory_present(void)
 {
-        int i;
-        for (i = 0; i < memblock.memory.cnt; i++)
-                memory_present(0, memblock_start_pfn(&memblock.memory, i),
-                                  memblock_end_pfn(&memblock.memory, i));
+        struct memblock_region *reg;
+
+        for_each_memblock(memory, reg)
+                memory_present(0, memblock_region_memory_base_pfn(reg),
+                               memblock_region_memory_end_pfn(reg));
 }
 #endif
 
+static int __init meminfo_cmp(const void *_a, const void *_b)
+{
+        const struct membank *a = _a, *b = _b;
+        long cmp = bank_pfn_start(a) - bank_pfn_start(b);
+        return cmp < 0 ? -1 : cmp > 0 ? 1 : 0;
+}
+
 void __init arm_memblock_init(struct meminfo *mi, struct machine_desc *mdesc)
 {
         int i;
 
+        sort(&meminfo.bank, meminfo.nr_banks, sizeof(meminfo.bank[0]), meminfo_cmp, NULL);
+
         memblock_init();
         for (i = 0; i < mi->nr_banks; i++)
                 memblock_add(mi->bank[i].start, mi->bank[i].size);
 
         /* Register the kernel text, kernel data and initrd with memblock. */
 #ifdef CONFIG_XIP_KERNEL
-        memblock_reserve(__pa(_data), _end - _data);
+        memblock_reserve(__pa(_sdata), _end - _sdata);
 #else
         memblock_reserve(__pa(_stext), _end - _stext);
 #endif
 #ifdef CONFIG_BLK_DEV_INITRD
+        if (phys_initrd_size &&
+            !memblock_is_region_memory(phys_initrd_start, phys_initrd_size)) {
+                pr_err("INITRD: 0x%08lx+0x%08lx is not a memory region - disabling initrd\n",
+                       phys_initrd_start, phys_initrd_size);
+                phys_initrd_start = phys_initrd_size = 0;
+        }
+        if (phys_initrd_size &&
+            memblock_is_region_reserved(phys_initrd_start, phys_initrd_size)) {
+                pr_err("INITRD: 0x%08lx+0x%08lx overlaps in-use memory region - disabling initrd\n",
+                       phys_initrd_start, phys_initrd_size);
+                phys_initrd_start = phys_initrd_size = 0;
+        }
         if (phys_initrd_size) {
                 memblock_reserve(phys_initrd_start, phys_initrd_size);
 
@@ -292,6 +352,7 @@ void __init arm_memblock_init(struct meminfo *mi, struct machine_desc *mdesc)
 #endif
 
         arm_mm_memblock_reserve();
+        arm_dt_memblock_reserve();
 
         /* reserve any platform specific memblock areas */
         if (mdesc->reserve)
@@ -303,14 +364,13 @@ void __init arm_memblock_init(struct meminfo *mi, struct machine_desc *mdesc)
 
 void __init bootmem_init(void)
 {
-        struct meminfo *mi = &meminfo;
         unsigned long min, max_low, max_high;
 
         max_low = max_high = 0;
 
-        find_limits(mi, &min, &max_low, &max_high);
+        find_limits(&min, &max_low, &max_high);
 
-        arm_bootmem_init(mi, min, max_low);
+        arm_bootmem_init(min, max_low);
 
         /*
          * Sparsemem tries to allocate bootmem in memory_present(),
@@ -328,9 +388,9 @@ void __init bootmem_init(void)
          * the sparse mem_map arrays initialized by sparse_init()
          * for memmap_init_zone(), otherwise all PFNs are invalid.
          */
-        arm_bootmem_free(mi, min, max_low, max_high);
+        arm_bootmem_free(min, max_low, max_high);
 
-        high_memory = __va((max_low << PAGE_SHIFT) - 1) + 1;
+        high_memory = __va(((phys_addr_t)max_low << PAGE_SHIFT) - 1) + 1;
 
         /*
          * This doesn't seem to be used by the Linux memory manager any
@@ -372,14 +432,14 @@ free_memmap(unsigned long start_pfn, unsigned long end_pfn)
          * Convert start_pfn/end_pfn to a struct page pointer.
          */
         start_pg = pfn_to_page(start_pfn - 1) + 1;
-        end_pg = pfn_to_page(end_pfn);
+        end_pg = pfn_to_page(end_pfn - 1) + 1;
 
         /*
          * Convert to physical addresses, and
          * round start upwards and end downwards.
          */
-        pg = PAGE_ALIGN(__pa(start_pg));
-        pgend = __pa(end_pg) & PAGE_MASK;
+        pg = (unsigned long)PAGE_ALIGN(__pa(start_pg));
+        pgend = (unsigned long)__pa(end_pg) & PAGE_MASK;
 
         /*
          * If there are free pages between these,
@@ -406,6 +466,14 @@ static void __init free_unused_memmap(struct meminfo *mi)
 
                 bank_start = bank_pfn_start(bank);
 
+#ifdef CONFIG_SPARSEMEM
+                /*
+                 * Take care not to free memmap entries that don't exist
+                 * due to SPARSEMEM sections which aren't present.
+                 */
+                bank_start = min(bank_start,
+                                 ALIGN(prev_bank_end, PAGES_PER_SECTION));
+#endif
                 /*
                  * If we had a previous bank, and there is a space
                  * between the current bank and the previous, free it.
@@ -420,6 +488,62 @@ static void __init free_unused_memmap(struct meminfo *mi)
                  */
                 prev_bank_end = ALIGN(bank_pfn_end(bank), MAX_ORDER_NR_PAGES);
         }
+
+#ifdef CONFIG_SPARSEMEM
+        if (!IS_ALIGNED(prev_bank_end, PAGES_PER_SECTION))
+                free_memmap(prev_bank_end,
+                            ALIGN(prev_bank_end, PAGES_PER_SECTION));
+#endif
+}
+
+static void __init free_highpages(void)
+{
+#ifdef CONFIG_HIGHMEM
+        unsigned long max_low = max_low_pfn + PHYS_PFN_OFFSET;
+        struct memblock_region *mem, *res;
+
+        /* set highmem page free */
+        for_each_memblock(memory, mem) {
+                unsigned long start = memblock_region_memory_base_pfn(mem);
+                unsigned long end = memblock_region_memory_end_pfn(mem);
+
+                /* Ignore complete lowmem entries */
+                if (end <= max_low)
+                        continue;
+
+                /* Truncate partial highmem entries */
+                if (start < max_low)
+                        start = max_low;
+
+                /* Find and exclude any reserved regions */
+                for_each_memblock(reserved, res) {
+                        unsigned long res_start, res_end;
+
+                        res_start = memblock_region_reserved_base_pfn(res);
+                        res_end = memblock_region_reserved_end_pfn(res);
+
+                        if (res_end < start)
+                                continue;
+                        if (res_start < start)
+                                res_start = start;
+                        if (res_start > end)
+                                res_start = end;
+                        if (res_end > end)
+                                res_end = end;
+                        if (res_start != start)
+                                totalhigh_pages += free_area(start, res_start,
+                                                             NULL);
+                        start = res_end;
+                        if (start == end)
+                                break;
+                }
+
+                /* And now free anything which remains */
+                if (start < end)
+                        totalhigh_pages += free_area(start, end, NULL);
+        }
+        totalram_pages += totalhigh_pages;
+#endif
 }
 
 /*
@@ -430,6 +554,7 @@ static void __init free_unused_memmap(struct meminfo *mi)
 void __init mem_init(void)
 {
         unsigned long reserved_pages, free_pages;
+        struct memblock_region *reg;
         int i;
 #ifdef CONFIG_HAVE_TCM
         /* These pointers are filled in on TCM detection */
@@ -450,16 +575,7 @@ void __init mem_init(void)
                                     __phys_to_pfn(__pa(swapper_pg_dir)), NULL);
 #endif
 
-#ifdef CONFIG_HIGHMEM
-        /* set highmem page free */
-        for_each_bank (i, &meminfo) {
-                unsigned long start = bank_pfn_start(&meminfo.bank[i]);
-                unsigned long end = bank_pfn_end(&meminfo.bank[i]);
-                if (start >= max_low_pfn + PHYS_PFN_OFFSET)
-                        totalhigh_pages += free_area(start, end, NULL);
-        }
-        totalram_pages += totalhigh_pages;
-#endif
+        free_highpages();
 
         reserved_pages = free_pages = 0;
 
@@ -489,9 +605,11 @@ void __init mem_init(void)
          */
         printk(KERN_INFO "Memory:");
         num_physpages = 0;
-        for (i = 0; i < meminfo.nr_banks; i++) {
-                num_physpages += bank_pfn_size(&meminfo.bank[i]);
-                printk(" %ldMB", bank_phys_size(&meminfo.bank[i]) >> 20);
+        for_each_memblock(memory, reg) {
+                unsigned long pages = memblock_region_memory_end_pfn(reg) -
+                        memblock_region_memory_base_pfn(reg);
+                num_physpages += pages;
+                printk(" %ldMB", pages >> (20 - PAGE_SHIFT));
         }
         printk(" = %luMB total\n", num_physpages >> (20 - PAGE_SHIFT));
 
@@ -523,7 +641,8 @@ void __init mem_init(void)
                         "    modules : 0x%08lx - 0x%08lx   (%4ld MB)\n"
                         "      .init : 0x%p" " - 0x%p" "   (%4d kB)\n"
                         "      .text : 0x%p" " - 0x%p" "   (%4d kB)\n"
-                        "      .data : 0x%p" " - 0x%p" "   (%4d kB)\n",
+                        "      .data : 0x%p" " - 0x%p" "   (%4d kB)\n"
+                        "       .bss : 0x%p" " - 0x%p" "   (%4d kB)\n",
 
                         MLK(UL(CONFIG_VECTORS_BASE), UL(CONFIG_VECTORS_BASE) +
                                 (PAGE_SIZE)),
@@ -545,7 +664,8 @@ void __init mem_init(void)
 
                         MLK_ROUNDUP(__init_begin, __init_end),
                         MLK_ROUNDUP(_text, _etext),
-                        MLK_ROUNDUP(_data, _edata));
+                        MLK_ROUNDUP(_sdata, _edata),
+                        MLK_ROUNDUP(__bss_start, __bss_stop));
 
 #undef MLK
 #undef MLM
diff --git a/arch/arm/mm/ioremap.c b/arch/arm/mm/ioremap.c
index 17e7b0b57e49..ab506272b2d3 100644
--- a/arch/arm/mm/ioremap.c
+++ b/arch/arm/mm/ioremap.c
@@ -204,12 +204,8 @@ void __iomem * __arm_ioremap_pfn_caller(unsigned long pfn,
         /*
          * Don't allow RAM to be mapped - this causes problems with ARMv6+
          */
-        if (pfn_valid(pfn)) {
-                printk(KERN_WARNING "BUG: Your driver calls ioremap() on system memory.  This leads\n"
-                       KERN_WARNING "to architecturally unpredictable behaviour on ARMv6+, and ioremap()\n"
-                       KERN_WARNING "will fail in the next kernel release.  Please fix your driver.\n");
-                WARN_ON(1);
-        }
+        if (WARN_ON(pfn_valid(pfn)))
+                return NULL;
 
         type = get_mem_type(mtype);
         if (!type)
diff --git a/arch/arm/mm/mm.h b/arch/arm/mm/mm.h
index 6630620380a4..5b3d7d543659 100644
--- a/arch/arm/mm/mm.h
+++ b/arch/arm/mm/mm.h
@@ -5,18 +5,13 @@ extern pmd_t *top_pmd;
 
 #define TOP_PTE(x)      pte_offset_kernel(top_pmd, x)
 
-static inline pmd_t *pmd_off(pgd_t *pgd, unsigned long virt)
-{
-        return pmd_offset(pgd, virt);
-}
-
 static inline pmd_t *pmd_off_k(unsigned long virt)
 {
-        return pmd_off(pgd_offset_k(virt), virt);
+        return pmd_offset(pud_offset(pgd_offset_k(virt), virt), virt);
 }
 
 struct mem_type {
-        unsigned int prot_pte;
+        pteval_t prot_pte;
         unsigned int prot_l1;
         unsigned int prot_sect;
         unsigned int domain;
diff --git a/arch/arm/mm/mmap.c b/arch/arm/mm/mmap.c
index 4f5b39687df5..74be05f3e03a 100644
--- a/arch/arm/mm/mmap.c
+++ b/arch/arm/mm/mmap.c
@@ -7,6 +7,7 @@
 #include <linux/shm.h>
 #include <linux/sched.h>
 #include <linux/io.h>
+#include <linux/personality.h>
 #include <linux/random.h>
 #include <asm/cputype.h>
 #include <asm/system.h>
@@ -31,7 +32,7 @@ arch_get_unmapped_area(struct file *filp, unsigned long addr,
         struct mm_struct *mm = current->mm;
         struct vm_area_struct *vma;
         unsigned long start_addr;
-#ifdef CONFIG_CPU_V6
+#if defined(CONFIG_CPU_V6) || defined(CONFIG_CPU_V6K)
         unsigned int cache_type;
         int do_align = 0, aliasing = 0;
 
@@ -82,7 +83,8 @@ arch_get_unmapped_area(struct file *filp, unsigned long addr,
                 mm->cached_hole_size = 0;
         }
         /* 8 bits of randomness in 20 address space bits */
-        if (current->flags & PF_RANDOMIZE)
+        if ((current->flags & PF_RANDOMIZE) &&
+            !(current->personality & ADDR_NO_RANDOMIZE))
                 addr += (get_random_int() % (1 << 8)) << PAGE_SHIFT;
 
 full_search:
@@ -144,3 +146,25 @@ int valid_mmap_phys_addr_range(unsigned long pfn, size_t size)
 {
         return !(pfn + (size >> PAGE_SHIFT) > 0x00100000);
 }
+
+#ifdef CONFIG_STRICT_DEVMEM
+
+#include <linux/ioport.h>
+
+/*
+ * devmem_is_allowed() checks to see if /dev/mem access to a certain
+ * address is valid. The argument is a physical page number.
+ * We mimic x86 here by disallowing access to system RAM as well as
+ * device-exclusive MMIO regions. This effectively disable read()/write()
+ * on /dev/mem.
+ */
+int devmem_is_allowed(unsigned long pfn)
+{
+        if (iomem_is_exclusive(pfn << PAGE_SHIFT))
+                return 0;
+        if (!page_is_ram(pfn))
+                return 1;
+        return 0;
+}
+
+#endif
diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c
index e8ed9dc461fe..594d677b92c8 100644
--- a/arch/arm/mm/mmu.c
+++ b/arch/arm/mm/mmu.c
@@ -14,7 +14,6 @@
 #include <linux/mman.h>
 #include <linux/nodemask.h>
 #include <linux/memblock.h>
-#include <linux/sort.h>
 #include <linux/fs.h>
 
 #include <asm/cputype.h>
@@ -25,14 +24,13 @@
 #include <asm/smp_plat.h>
 #include <asm/tlb.h>
 #include <asm/highmem.h>
+#include <asm/traps.h>
 
 #include <asm/mach/arch.h>
 #include <asm/mach/map.h>
 
 #include "mm.h"
 
-DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
-
 /*
  * empty_zero_page is a special page that is used for
  * zero-initialized data and COW.
@@ -63,7 +61,7 @@ struct cachepolicy {
         const char      policy[16];
         unsigned int    cr_mask;
         unsigned int    pmd;
-        unsigned int    pte;
+        pteval_t        pte;
 };
 
 static struct cachepolicy cache_policies[] __initdata = {
@@ -191,7 +189,7 @@ void adjust_cr(unsigned long mask, unsigned long set)
 }
 #endif
 
-#define PROT_PTE_DEVICE         L_PTE_PRESENT|L_PTE_YOUNG|L_PTE_DIRTY|L_PTE_WRITE
+#define PROT_PTE_DEVICE         L_PTE_PRESENT|L_PTE_YOUNG|L_PTE_DIRTY|L_PTE_XN
 #define PROT_SECT_DEVICE        PMD_TYPE_SECT|PMD_SECT_AP_WRITE
 
 static struct mem_type mem_types[] = {
@@ -236,19 +234,18 @@ static struct mem_type mem_types[] = {
         },
         [MT_LOW_VECTORS] = {
                 .prot_pte  = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
-                                L_PTE_EXEC,
+                                L_PTE_RDONLY,
                 .prot_l1   = PMD_TYPE_TABLE,
                 .domain    = DOMAIN_USER,
         },
         [MT_HIGH_VECTORS] = {
                 .prot_pte  = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
-                                L_PTE_USER | L_PTE_EXEC,
+                                L_PTE_USER | L_PTE_RDONLY,
                 .prot_l1   = PMD_TYPE_TABLE,
                 .domain    = DOMAIN_USER,
         },
         [MT_MEMORY] = {
-                .prot_pte  = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
-                                L_PTE_WRITE | L_PTE_EXEC,
+                .prot_pte  = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY,
                 .prot_l1   = PMD_TYPE_TABLE,
                 .prot_sect = PMD_TYPE_SECT | PMD_SECT_AP_WRITE,
                 .domain    = DOMAIN_KERNEL,
@@ -259,23 +256,22 @@ static struct mem_type mem_types[] = {
         },
         [MT_MEMORY_NONCACHED] = {
                 .prot_pte  = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
-                                L_PTE_WRITE | L_PTE_EXEC | L_PTE_MT_BUFFERABLE,
+                                L_PTE_MT_BUFFERABLE,
                 .prot_l1   = PMD_TYPE_TABLE,
                 .prot_sect = PMD_TYPE_SECT | PMD_SECT_AP_WRITE,
                 .domain    = DOMAIN_KERNEL,
         },
         [MT_MEMORY_DTCM] = {
-                .prot_pte       = L_PTE_PRESENT | L_PTE_YOUNG |
-                                  L_PTE_DIRTY | L_PTE_WRITE,
-                .prot_l1        = PMD_TYPE_TABLE,
-                .prot_sect      = PMD_TYPE_SECT | PMD_SECT_XN,
-                .domain         = DOMAIN_KERNEL,
+                .prot_pte  = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
+                                L_PTE_XN,
+                .prot_l1   = PMD_TYPE_TABLE,
+                .prot_sect = PMD_TYPE_SECT | PMD_SECT_XN,
+                .domain    = DOMAIN_KERNEL,
         },
         [MT_MEMORY_ITCM] = {
-                .prot_pte  = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
-                                L_PTE_USER | L_PTE_EXEC,
+                .prot_pte  = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY,
                 .prot_l1   = PMD_TYPE_TABLE,
-                .domain    = DOMAIN_IO,
+                .domain    = DOMAIN_KERNEL,
         },
 };
 
@@ -310,9 +306,8 @@ static void __init build_mem_type_table(void)
                         cachepolicy = CPOLICY_WRITEBACK;
                 ecc_mask = 0;
         }
-#ifdef CONFIG_SMP
-        cachepolicy = CPOLICY_WRITEALLOC;
-#endif
+        if (is_smp())
+                cachepolicy = CPOLICY_WRITEALLOC;
 
         /*
          * Strip out features not present on earlier architectures.
@@ -406,13 +401,11 @@ static void __init build_mem_type_table(void)
         cp = &cache_policies[cachepolicy];
         vecs_pgprot = kern_pgprot = user_pgprot = cp->pte;
 
-#ifndef CONFIG_SMP
         /*
          * Only use write-through for non-SMP systems
          */
-        if (cpu_arch >= CPU_ARCH_ARMv5 && cachepolicy > CPOLICY_WRITETHROUGH)
+        if (!is_smp() && cpu_arch >= CPU_ARCH_ARMv5 && cachepolicy > CPOLICY_WRITETHROUGH)
                 vecs_pgprot = cache_policies[CPOLICY_WRITETHROUGH].pte;
-#endif
 
         /*
          * Enable CPU-specific coherency if supported.
@@ -436,22 +429,23 @@ static void __init build_mem_type_table(void)
                 mem_types[MT_MINICLEAN].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE;
                 mem_types[MT_CACHECLEAN].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE;
 
-#ifdef CONFIG_SMP
-                /*
-                 * Mark memory with the "shared" attribute for SMP systems
-                 */
-                user_pgprot |= L_PTE_SHARED;
-                kern_pgprot |= L_PTE_SHARED;
-                vecs_pgprot |= L_PTE_SHARED;
-                mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_S;
-                mem_types[MT_DEVICE_WC].prot_pte |= L_PTE_SHARED;
-                mem_types[MT_DEVICE_CACHED].prot_sect |= PMD_SECT_S;
-                mem_types[MT_DEVICE_CACHED].prot_pte |= L_PTE_SHARED;
-                mem_types[MT_MEMORY].prot_sect |= PMD_SECT_S;
-                mem_types[MT_MEMORY].prot_pte |= L_PTE_SHARED;
-                mem_types[MT_MEMORY_NONCACHED].prot_sect |= PMD_SECT_S;
-                mem_types[MT_MEMORY_NONCACHED].prot_pte |= L_PTE_SHARED;
-#endif
+                if (is_smp()) {
+                        /*
+                         * Mark memory with the "shared" attribute
+                         * for SMP systems
+                         */
+                        user_pgprot |= L_PTE_SHARED;
+                        kern_pgprot |= L_PTE_SHARED;
+                        vecs_pgprot |= L_PTE_SHARED;
+                        mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_S;
+                        mem_types[MT_DEVICE_WC].prot_pte |= L_PTE_SHARED;
+                        mem_types[MT_DEVICE_CACHED].prot_sect |= PMD_SECT_S;
+                        mem_types[MT_DEVICE_CACHED].prot_pte |= L_PTE_SHARED;
+                        mem_types[MT_MEMORY].prot_sect |= PMD_SECT_S;
+                        mem_types[MT_MEMORY].prot_pte |= L_PTE_SHARED;
+                        mem_types[MT_MEMORY_NONCACHED].prot_sect |= PMD_SECT_S;
+                        mem_types[MT_MEMORY_NONCACHED].prot_pte |= L_PTE_SHARED;
+                }
         }
 
         /*
@@ -482,7 +476,7 @@ static void __init build_mem_type_table(void)
 
         pgprot_user   = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG | user_pgprot);
         pgprot_kernel = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG |
-                                 L_PTE_DIRTY | L_PTE_WRITE | kern_pgprot);
+                                 L_PTE_DIRTY | kern_pgprot);
 
         mem_types[MT_LOW_VECTORS].prot_l1 |= ecc_mask;
         mem_types[MT_HIGH_VECTORS].prot_l1 |= ecc_mask;
@@ -537,8 +531,8 @@ static void __init *early_alloc(unsigned long sz)
 static pte_t * __init early_pte_alloc(pmd_t *pmd, unsigned long addr, unsigned long prot)
 {
         if (pmd_none(*pmd)) {
-                pte_t *pte = early_alloc(2 * PTRS_PER_PTE * sizeof(pte_t));
-                __pmd_populate(pmd, __pa(pte) | prot);
+                pte_t *pte = early_alloc(PTE_HWTABLE_OFF + PTE_HWTABLE_SIZE);
+                __pmd_populate(pmd, __pa(pte), prot);
         }
         BUG_ON(pmd_bad(*pmd));
         return pte_offset_kernel(pmd, addr);
@@ -555,11 +549,11 @@ static void __init alloc_init_pte(pmd_t *pmd, unsigned long addr,
         } while (pte++, addr += PAGE_SIZE, addr != end);
 }
 
-static void __init alloc_init_section(pgd_t *pgd, unsigned long addr,
-                                      unsigned long end, unsigned long phys,
+static void __init alloc_init_section(pud_t *pud, unsigned long addr,
+                                      unsigned long end, phys_addr_t phys,
                                       const struct mem_type *type)
 {
-        pmd_t *pmd = pmd_offset(pgd, addr);
+        pmd_t *pmd = pmd_offset(pud, addr);
 
         /*
          * Try a section mapping - end, addr and phys must all be aligned
@@ -588,20 +582,34 @@ static void __init alloc_init_section(pgd_t *pgd, unsigned long addr,
         }
 }
 
+static void alloc_init_pud(pgd_t *pgd, unsigned long addr, unsigned long end,
+        unsigned long phys, const struct mem_type *type)
+{
+        pud_t *pud = pud_offset(pgd, addr);
+        unsigned long next;
+
+        do {
+                next = pud_addr_end(addr, end);
+                alloc_init_section(pud, addr, next, phys, type);
+                phys += next - addr;
+        } while (pud++, addr = next, addr != end);
+}
+
 static void __init create_36bit_mapping(struct map_desc *md,
                                         const struct mem_type *type)
 {
-        unsigned long phys, addr, length, end;
+        unsigned long addr, length, end;
+        phys_addr_t phys;
         pgd_t *pgd;
 
         addr = md->virtual;
-        phys = (unsigned long)__pfn_to_phys(md->pfn);
+        phys = __pfn_to_phys(md->pfn);
         length = PAGE_ALIGN(md->length);
 
         if (!(cpu_architecture() >= CPU_ARCH_ARMv6 || cpu_is_xsc3())) {
                 printk(KERN_ERR "MM: CPU does not support supersection "
                        "mapping for 0x%08llx at 0x%08lx\n",
-                       __pfn_to_phys((u64)md->pfn), addr);
+                       (long long)__pfn_to_phys((u64)md->pfn), addr);
                 return;
         }
 
@@ -614,14 +622,14 @@ static void __init create_36bit_mapping(struct map_desc *md,
         if (type->domain) {
                 printk(KERN_ERR "MM: invalid domain in supersection "
                        "mapping for 0x%08llx at 0x%08lx\n",
-                       __pfn_to_phys((u64)md->pfn), addr);
+                       (long long)__pfn_to_phys((u64)md->pfn), addr);
                 return;
         }
 
         if ((addr | length | __pfn_to_phys(md->pfn)) & ~SUPERSECTION_MASK) {
-                printk(KERN_ERR "MM: cannot create mapping for "
-                       "0x%08llx at 0x%08lx invalid alignment\n",
-                       __pfn_to_phys((u64)md->pfn), addr);
+                printk(KERN_ERR "MM: cannot create mapping for 0x%08llx"
+                       " at 0x%08lx invalid alignment\n",
+                       (long long)__pfn_to_phys((u64)md->pfn), addr);
                 return;
         }
 
@@ -634,7 +642,8 @@ static void __init create_36bit_mapping(struct map_desc *md,
         pgd = pgd_offset_k(addr);
         end = addr + length;
         do {
-                pmd_t *pmd = pmd_offset(pgd, addr);
+                pud_t *pud = pud_offset(pgd, addr);
+                pmd_t *pmd = pmd_offset(pud, addr);
                 int i;
 
                 for (i = 0; i < 16; i++)
@@ -655,22 +664,23 @@ static void __init create_36bit_mapping(struct map_desc *md,
  */
 static void __init create_mapping(struct map_desc *md)
 {
-        unsigned long phys, addr, length, end;
+        unsigned long addr, length, end;
+        phys_addr_t phys;
         const struct mem_type *type;
         pgd_t *pgd;
 
         if (md->virtual != vectors_base() && md->virtual < TASK_SIZE) {
-                printk(KERN_WARNING "BUG: not creating mapping for "
-                       "0x%08llx at 0x%08lx in user region\n",
-                       __pfn_to_phys((u64)md->pfn), md->virtual);
+                printk(KERN_WARNING "BUG: not creating mapping for 0x%08llx"
+                       " at 0x%08lx in user region\n",
+                       (long long)__pfn_to_phys((u64)md->pfn), md->virtual);
                 return;
         }
 
         if ((md->type == MT_DEVICE || md->type == MT_ROM) &&
             md->virtual >= PAGE_OFFSET && md->virtual < VMALLOC_END) {
-                printk(KERN_WARNING "BUG: mapping for 0x%08llx at 0x%08lx "
-                       "overlaps vmalloc space\n",
-                       __pfn_to_phys((u64)md->pfn), md->virtual);
+                printk(KERN_WARNING "BUG: mapping for 0x%08llx"
+                       " at 0x%08lx overlaps vmalloc space\n",
+                       (long long)__pfn_to_phys((u64)md->pfn), md->virtual);
         }
 
         type = &mem_types[md->type];
@@ -684,13 +694,13 @@ static void __init create_mapping(struct map_desc *md)
         }
 
         addr = md->virtual & PAGE_MASK;
-        phys = (unsigned long)__pfn_to_phys(md->pfn);
+        phys = __pfn_to_phys(md->pfn);
         length = PAGE_ALIGN(md->length + (md->virtual & ~PAGE_MASK));
 
         if (type->prot_l1 == 0 && ((addr | phys | length) & ~SECTION_MASK)) {
-                printk(KERN_WARNING "BUG: map for 0x%08lx at 0x%08lx can not "
+                printk(KERN_WARNING "BUG: map for 0x%08llx at 0x%08lx can not "
                        "be mapped using pages, ignoring.\n",
-                       __pfn_to_phys(md->pfn), addr);
+                       (long long)__pfn_to_phys(md->pfn), addr);
                 return;
         }
 
@@ -699,7 +709,7 @@ static void __init create_mapping(struct map_desc *md)
         do {
                 unsigned long next = pgd_addr_end(addr, end);
 
-                alloc_init_section(pgd, addr, next, phys, type);
+                alloc_init_pud(pgd, addr, next, phys, type);
 
                 phys += next - addr;
                 addr = next;
@@ -747,20 +757,18 @@ static int __init early_vmalloc(char *arg)
 }
 early_param("vmalloc", early_vmalloc);
 
-phys_addr_t lowmem_end_addr;
+static phys_addr_t lowmem_limit __initdata = 0;
 
-static void __init sanity_check_meminfo(void)
+void __init sanity_check_meminfo(void)
 {
         int i, j, highmem = 0;
 
-        lowmem_end_addr = __pa(vmalloc_min - 1) + 1;
-
         for (i = 0, j = 0; i < meminfo.nr_banks; i++) {
                 struct membank *bank = &meminfo.bank[j];
                 *bank = meminfo.bank[i];
 
 #ifdef CONFIG_HIGHMEM
-                if (__va(bank->start) > vmalloc_min ||
+                if (__va(bank->start) >= vmalloc_min ||
                     __va(bank->start) < (void *)PAGE_OFFSET)
                         highmem = 1;
 
@@ -796,9 +804,10 @@ static void __init sanity_check_meminfo(void)
                  */
                 if (__va(bank->start) >= vmalloc_min ||
                     __va(bank->start) < (void *)PAGE_OFFSET) {
-                        printk(KERN_NOTICE "Ignoring RAM at %.8lx-%.8lx "
+                        printk(KERN_NOTICE "Ignoring RAM at %.8llx-%.8llx "
                                "(vmalloc region overlap).\n",
-                               bank->start, bank->start + bank->size - 1);
+                               (unsigned long long)bank->start,
+                               (unsigned long long)bank->start + bank->size - 1);
                         continue;
                 }
 
@@ -809,13 +818,17 @@ static void __init sanity_check_meminfo(void)
                 if (__va(bank->start + bank->size) > vmalloc_min ||
                     __va(bank->start + bank->size) < __va(bank->start)) {
                         unsigned long newsize = vmalloc_min - __va(bank->start);
-                        printk(KERN_NOTICE "Truncating RAM at %.8lx-%.8lx "
-                               "to -%.8lx (vmalloc region overlap).\n",
-                               bank->start, bank->start + bank->size - 1,
-                               bank->start + newsize - 1);
+                        printk(KERN_NOTICE "Truncating RAM at %.8llx-%.8llx "
+                               "to -%.8llx (vmalloc region overlap).\n",
+                               (unsigned long long)bank->start,
+                               (unsigned long long)bank->start + bank->size - 1,
+                               (unsigned long long)bank->start + newsize - 1);
                         bank->size = newsize;
                 }
 #endif
+                if (!bank->highmem && bank->start + bank->size > lowmem_limit)
+                        lowmem_limit = bank->start + bank->size;
+
                 j++;
         }
 #ifdef CONFIG_HIGHMEM
@@ -829,18 +842,6 @@ static void __init sanity_check_meminfo(void)
                          * rather difficult.
                          */
                         reason = "with VIPT aliasing cache";
-#ifdef CONFIG_SMP
-                } else if (tlb_ops_need_broadcast()) {
-                        /*
-                         * kmap_high needs to occasionally flush TLB entries,
-                         * however, if the TLB entries need to be broadcast
-                         * we may deadlock:
-                         *  kmap_high(irqs off)->flush_all_zero_pkmaps->
-                         *  flush_tlb_kernel_range->smp_call_function_many
-                         *   (must not be called with irqs off)
-                         */
-                        reason = "without hardware TLB ops broadcasting";
-#endif
                 }
                 if (reason) {
                         printk(KERN_CRIT "HIGHMEM is not supported %s, ignoring high memory\n",
@@ -851,11 +852,13 @@ static void __init sanity_check_meminfo(void)
         }
 #endif
         meminfo.nr_banks = j;
+        memblock_set_current_limit(lowmem_limit);
 }
 
 static inline void prepare_page_table(void)
 {
         unsigned long addr;
+        phys_addr_t end;
 
         /*
          * Clear out all the mappings below the kernel image.
@@ -871,10 +874,17 @@ static inline void prepare_page_table(void)
                 pmd_clear(pmd_off_k(addr));
 
         /*
+         * Find the end of the first block of lowmem.
+         */
+        end = memblock.memory.regions[0].base + memblock.memory.regions[0].size;
+        if (end >= lowmem_limit)
+                end = lowmem_limit;
+
+        /*
          * Clear out all the kernel space mappings, except for the first
          * memory bank, up to the end of the vmalloc region.
          */
-        for (addr = __phys_to_virt(bank_phys_end(&meminfo.bank[0]));
+        for (addr = __phys_to_virt(end);
              addr < VMALLOC_END; addr += PGDIR_SIZE)
                 pmd_clear(pmd_off_k(addr));
 }
@@ -910,12 +920,11 @@ static void __init devicemaps_init(struct machine_desc *mdesc)
 {
         struct map_desc map;
         unsigned long addr;
-        void *vectors;
 
         /*
          * Allocate the vector page early.
          */
-        vectors = early_alloc(PAGE_SIZE);
+        vectors_page = early_alloc(PAGE_SIZE);
 
         for (addr = VMALLOC_END; addr; addr += PGDIR_SIZE)
                 pmd_clear(pmd_off_k(addr));
@@ -955,7 +964,7 @@ static void __init devicemaps_init(struct machine_desc *mdesc)
          * location (0xffff0000).  If we aren't using high-vectors, also
          * create a mapping at the low-vectors virtual address.
          */
-        map.pfn = __phys_to_pfn(virt_to_phys(vectors));
+        map.pfn = __phys_to_pfn(virt_to_phys(vectors_page));
         map.virtual = 0xffff0000;
         map.length = PAGE_SIZE;
         map.type = MT_HIGH_VECTORS;
@@ -991,37 +1000,28 @@ static void __init kmap_init(void)
 #endif
 }
 
-static inline void map_memory_bank(struct membank *bank)
-{
-        struct map_desc map;
-
-        map.pfn = bank_pfn_start(bank);
-        map.virtual = __phys_to_virt(bank_phys_start(bank));
-        map.length = bank_phys_size(bank);
-        map.type = MT_MEMORY;
-
-        create_mapping(&map);
-}
-
 static void __init map_lowmem(void)
 {
-        struct meminfo *mi = &meminfo;
-        int i;
+        struct memblock_region *reg;
 
         /* Map all the lowmem memory banks. */
-        for (i = 0; i < mi->nr_banks; i++) {
-                struct membank *bank = &mi->bank[i];
+        for_each_memblock(memory, reg) {
+                phys_addr_t start = reg->base;
+                phys_addr_t end = start + reg->size;
+                struct map_desc map;
+
+                if (end > lowmem_limit)
+                        end = lowmem_limit;
+                if (start >= end)
+                        break;
 
-                if (!bank->highmem)
-                        map_memory_bank(bank);
-        }
-}
+                map.pfn = __phys_to_pfn(start);
+                map.virtual = __phys_to_virt(start);
+                map.length = end - start;
+                map.type = MT_MEMORY;
 
-static int __init meminfo_cmp(const void *_a, const void *_b)
-{
-        const struct membank *a = _a, *b = _b;
-        long cmp = bank_pfn_start(a) - bank_pfn_start(b);
-        return cmp < 0 ? -1 : cmp > 0 ? 1 : 0;
+                create_mapping(&map);
+        }
 }
 
 /*
@@ -1032,10 +1032,9 @@ void __init paging_init(struct machine_desc *mdesc)
 {
         void *zero_page;
 
-        sort(&meminfo.bank, meminfo.nr_banks, sizeof(meminfo.bank[0]), meminfo_cmp, NULL);
+        memblock_set_current_limit(lowmem_limit);
 
         build_mem_type_table();
-        sanity_check_meminfo();
         prepare_page_table();
         map_lowmem();
         devicemaps_init(mdesc);
@@ -1051,38 +1050,3 @@ void __init paging_init(struct machine_desc *mdesc)
         empty_zero_page = virt_to_page(zero_page);
         __flush_dcache_page(NULL, empty_zero_page);
 }
-
-/*
- * In order to soft-boot, we need to insert a 1:1 mapping in place of
- * the user-mode pages.  This will then ensure that we have predictable
- * results when turning the mmu off
- */
-void setup_mm_for_reboot(char mode)
-{
-        unsigned long base_pmdval;
-        pgd_t *pgd;
-        int i;
-
-        /*
-         * We need to access to user-mode page tables here. For kernel threads
-         * we don't have any user-mode mappings so we use the context that we
-         * "borrowed".
-         */
-        pgd = current->active_mm->pgd;
-
-        base_pmdval = PMD_SECT_AP_WRITE | PMD_SECT_AP_READ | PMD_TYPE_SECT;
-        if (cpu_architecture() <= CPU_ARCH_ARMv5TEJ && !cpu_is_xscale())
-                base_pmdval |= PMD_BIT4;
-
-        for (i = 0; i < FIRST_USER_PGD_NR + USER_PTRS_PER_PGD; i++, pgd++) {
-                unsigned long pmdval = (i << PGDIR_SHIFT) | base_pmdval;
-                pmd_t *pmd;
-
-                pmd = pmd_off(pgd, i << PGDIR_SHIFT);
-                pmd[0] = __pmd(pmdval);
-                pmd[1] = __pmd(pmdval + (1 << (PGDIR_SHIFT - 1)));
-                flush_pmd_entry(pmd);
-        }
-
-        local_flush_tlb_all();
-}
diff --git a/arch/arm/mm/nommu.c b/arch/arm/mm/nommu.c
index 687d02319a41..941a98c9e8aa 100644
--- a/arch/arm/mm/nommu.c
+++ b/arch/arm/mm/nommu.c
@@ -27,6 +27,10 @@ void __init arm_mm_memblock_reserve(void)
         memblock_reserve(CONFIG_VECTORS_BASE, PAGE_SIZE);
 }
 
+void __init sanity_check_meminfo(void)
+{
+}
+
 /*
  * paging_init() sets up the page tables, initialises the zone memory
  * maps, and sets up the zero page, bad page and bad page tables.
diff --git a/arch/arm/mm/pgd.c b/arch/arm/mm/pgd.c
index be5f58e153bf..b2027c154b2a 100644
--- a/arch/arm/mm/pgd.c
+++ b/arch/arm/mm/pgd.c
@@ -17,14 +17,13 @@
 
 #include "mm.h"
 
-#define FIRST_KERNEL_PGD_NR     (FIRST_USER_PGD_NR + USER_PTRS_PER_PGD)
-
 /*
  * need to get a 16k page for level 1
  */
-pgd_t *get_pgd_slow(struct mm_struct *mm)
+pgd_t *pgd_alloc(struct mm_struct *mm)
 {
         pgd_t *new_pgd, *init_pgd;
+        pud_t *new_pud, *init_pud;
         pmd_t *new_pmd, *init_pmd;
         pte_t *new_pte, *init_pte;
 
@@ -32,14 +31,14 @@ pgd_t *get_pgd_slow(struct mm_struct *mm)
         if (!new_pgd)
                 goto no_pgd;
 
-        memset(new_pgd, 0, FIRST_KERNEL_PGD_NR * sizeof(pgd_t));
+        memset(new_pgd, 0, USER_PTRS_PER_PGD * sizeof(pgd_t));
 
         /*
          * Copy over the kernel and IO PGD entries
          */
         init_pgd = pgd_offset_k(0);
-        memcpy(new_pgd + FIRST_KERNEL_PGD_NR, init_pgd + FIRST_KERNEL_PGD_NR,
-                       (PTRS_PER_PGD - FIRST_KERNEL_PGD_NR) * sizeof(pgd_t));
+        memcpy(new_pgd + USER_PTRS_PER_PGD, init_pgd + USER_PTRS_PER_PGD,
+                       (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t));
 
         clean_dcache_area(new_pgd, PTRS_PER_PGD * sizeof(pgd_t));
 
@@ -48,18 +47,23 @@ pgd_t *get_pgd_slow(struct mm_struct *mm)
                  * On ARM, first page must always be allocated since it
                  * contains the machine vectors.
                  */
-                new_pmd = pmd_alloc(mm, new_pgd, 0);
+                new_pud = pud_alloc(mm, new_pgd, 0);
+                if (!new_pud)
+                        goto no_pud;
+
+                new_pmd = pmd_alloc(mm, new_pud, 0);
                 if (!new_pmd)
                         goto no_pmd;
 
-                new_pte = pte_alloc_map(mm, new_pmd, 0);
+                new_pte = pte_alloc_map(mm, NULL, new_pmd, 0);
                 if (!new_pte)
                         goto no_pte;
 
-                init_pmd = pmd_offset(init_pgd, 0);
-                init_pte = pte_offset_map_nested(init_pmd, 0);
+                init_pud = pud_offset(init_pgd, 0);
+                init_pmd = pmd_offset(init_pud, 0);
+                init_pte = pte_offset_map(init_pmd, 0);
                 set_pte_ext(new_pte, *init_pte, 0);
-                pte_unmap_nested(init_pte);
+                pte_unmap(init_pte);
                 pte_unmap(new_pte);
         }
 
@@ -68,33 +72,44 @@ pgd_t *get_pgd_slow(struct mm_struct *mm)
 no_pte:
         pmd_free(mm, new_pmd);
 no_pmd:
+        pud_free(mm, new_pud);
+no_pud:
         free_pages((unsigned long)new_pgd, 2);
 no_pgd:
         return NULL;
 }
 
-void free_pgd_slow(struct mm_struct *mm, pgd_t *pgd)
+void pgd_free(struct mm_struct *mm, pgd_t *pgd_base)
 {
+        pgd_t *pgd;
+        pud_t *pud;
         pmd_t *pmd;
         pgtable_t pte;
 
-        if (!pgd)
+        if (!pgd_base)
                 return;
 
-        /* pgd is always present and good */
-        pmd = pmd_off(pgd, 0);
-        if (pmd_none(*pmd))
-                goto free;
-        if (pmd_bad(*pmd)) {
-                pmd_ERROR(*pmd);
-                pmd_clear(pmd);
-                goto free;
-        }
+        pgd = pgd_base + pgd_index(0);
+        if (pgd_none_or_clear_bad(pgd))
+                goto no_pgd;
+
+        pud = pud_offset(pgd, 0);
+        if (pud_none_or_clear_bad(pud))
+                goto no_pud;
+
+        pmd = pmd_offset(pud, 0);
+        if (pmd_none_or_clear_bad(pmd))
+                goto no_pmd;
 
         pte = pmd_pgtable(*pmd);
         pmd_clear(pmd);
         pte_free(mm, pte);
+no_pmd:
+        pud_clear(pud);
         pmd_free(mm, pmd);
-free:
-        free_pages((unsigned long) pgd, 2);
+no_pud:
+        pgd_clear(pgd);
+        pud_free(mm, pud);
+no_pgd:
+        free_pages((unsigned long) pgd_base, 2);
 }
diff --git a/arch/arm/mm/proc-arm1020.S b/arch/arm/mm/proc-arm1020.S
index 203a4e944d9e..6c4e7fd6c8af 100644
--- a/arch/arm/mm/proc-arm1020.S
+++ b/arch/arm/mm/proc-arm1020.S
@@ -64,7 +64,7 @@
 /*
  * This is the size at which it becomes more efficient to
  * clean the whole cache, rather than using the individual
- * cache line maintainence instructions.
+ * cache line maintenance instructions.
  */
 #define CACHE_DLIMIT    32768
 
@@ -119,6 +119,20 @@ ENTRY(cpu_arm1020_do_idle)
 /* ================================= CACHE ================================ */
 
         .align  5
+
+/*
+ *      flush_icache_all()
+ *
+ *      Unconditionally clean and invalidate the entire icache.
+ */
+ENTRY(arm1020_flush_icache_all)
+#ifndef CONFIG_CPU_ICACHE_DISABLE
+        mov     r0, #0
+        mcr     p15, 0, r0, c7, c5, 0           @ invalidate I cache
+#endif
+        mov     pc, lr
+ENDPROC(arm1020_flush_icache_all)
+
 /*
  *      flush_user_cache_all()
  *
@@ -351,6 +365,7 @@ ENTRY(arm1020_dma_unmap_area)
 ENDPROC(arm1020_dma_unmap_area)
 
 ENTRY(arm1020_cache_fns)
+        .long   arm1020_flush_icache_all
         .long   arm1020_flush_kern_cache_all
         .long   arm1020_flush_user_cache_all
         .long   arm1020_flush_user_cache_range
@@ -430,7 +445,7 @@ ENTRY(cpu_arm1020_set_pte_ext)
 #endif /* CONFIG_MMU */
         mov     pc, lr
 
-        __INIT
+        __CPUINIT
 
         .type   __arm1020_setup, #function
 __arm1020_setup:
@@ -478,6 +493,9 @@ arm1020_processor_functions:
         .word   cpu_arm1020_dcache_clean_area
         .word   cpu_arm1020_switch_mm
         .word   cpu_arm1020_set_pte_ext
+        .word   0
+        .word   0
+        .word   0
         .size   arm1020_processor_functions, . - arm1020_processor_functions
 
         .section ".rodata"
diff --git a/arch/arm/mm/proc-arm1020e.S b/arch/arm/mm/proc-arm1020e.S
index 1a511e765909..4ce947c19623 100644
--- a/arch/arm/mm/proc-arm1020e.S
+++ b/arch/arm/mm/proc-arm1020e.S
@@ -64,7 +64,7 @@
 /*
  * This is the size at which it becomes more efficient to
  * clean the whole cache, rather than using the individual
- * cache line maintainence instructions.
+ * cache line maintenance instructions.
  */
 #define CACHE_DLIMIT    32768
 
@@ -119,6 +119,20 @@ ENTRY(cpu_arm1020e_do_idle)
 /* ================================= CACHE ================================ */
 
         .align  5
+
+/*
+ *      flush_icache_all()
+ *
+ *      Unconditionally clean and invalidate the entire icache.
+ */
+ENTRY(arm1020e_flush_icache_all)
+#ifndef CONFIG_CPU_ICACHE_DISABLE
+        mov     r0, #0
+        mcr     p15, 0, r0, c7, c5, 0           @ invalidate I cache
+#endif
+        mov     pc, lr
+ENDPROC(arm1020e_flush_icache_all)
+
 /*
  *      flush_user_cache_all()
  *
@@ -337,6 +351,7 @@ ENTRY(arm1020e_dma_unmap_area)
 ENDPROC(arm1020e_dma_unmap_area)
 
 ENTRY(arm1020e_cache_fns)
+        .long   arm1020e_flush_icache_all
         .long   arm1020e_flush_kern_cache_all
         .long   arm1020e_flush_user_cache_all
         .long   arm1020e_flush_user_cache_range
@@ -412,7 +427,7 @@ ENTRY(cpu_arm1020e_set_pte_ext)
 #endif /* CONFIG_MMU */
         mov     pc, lr
 
-        __INIT
+        __CPUINIT
 
         .type   __arm1020e_setup, #function
 __arm1020e_setup:
@@ -459,6 +474,9 @@ arm1020e_processor_functions:
         .word   cpu_arm1020e_dcache_clean_area
         .word   cpu_arm1020e_switch_mm
         .word   cpu_arm1020e_set_pte_ext
+        .word   0
+        .word   0
+        .word   0
         .size   arm1020e_processor_functions, . - arm1020e_processor_functions
 
         .section ".rodata"
diff --git a/arch/arm/mm/proc-arm1022.S b/arch/arm/mm/proc-arm1022.S
index 1ffa4eb9c34f..c8884c5413a2 100644
--- a/arch/arm/mm/proc-arm1022.S
+++ b/arch/arm/mm/proc-arm1022.S
@@ -53,7 +53,7 @@
 /*
  * This is the size at which it becomes more efficient to
  * clean the whole cache, rather than using the individual
- * cache line maintainence instructions.
+ * cache line maintenance instructions.
  */
 #define CACHE_DLIMIT    32768
 
@@ -108,6 +108,20 @@ ENTRY(cpu_arm1022_do_idle)
 /* ================================= CACHE ================================ */
 
         .align  5
+
+/*
+ *      flush_icache_all()
+ *
+ *      Unconditionally clean and invalidate the entire icache.
+ */
+ENTRY(arm1022_flush_icache_all)
+#ifndef CONFIG_CPU_ICACHE_DISABLE
+        mov     r0, #0
+        mcr     p15, 0, r0, c7, c5, 0           @ invalidate I cache
+#endif
+        mov     pc, lr
+ENDPROC(arm1022_flush_icache_all)
+
 /*
  *      flush_user_cache_all()
  *
@@ -326,6 +340,7 @@ ENTRY(arm1022_dma_unmap_area)
 ENDPROC(arm1022_dma_unmap_area)
 
 ENTRY(arm1022_cache_fns)
+        .long   arm1022_flush_icache_all
         .long   arm1022_flush_kern_cache_all
         .long   arm1022_flush_user_cache_all
         .long   arm1022_flush_user_cache_range
@@ -394,7 +409,7 @@ ENTRY(cpu_arm1022_set_pte_ext)
 #endif /* CONFIG_MMU */
         mov     pc, lr
 
-        __INIT
+        __CPUINIT
 
         .type   __arm1022_setup, #function
 __arm1022_setup:
@@ -442,6 +457,9 @@ arm1022_processor_functions:
         .word   cpu_arm1022_dcache_clean_area
         .word   cpu_arm1022_switch_mm
         .word   cpu_arm1022_set_pte_ext
+        .word   0
+        .word   0
+        .word   0
         .size   arm1022_processor_functions, . - arm1022_processor_functions
 
         .section ".rodata"
diff --git a/arch/arm/mm/proc-arm1026.S b/arch/arm/mm/proc-arm1026.S
index 5697c34b95b0..413684660aad 100644
--- a/arch/arm/mm/proc-arm1026.S
+++ b/arch/arm/mm/proc-arm1026.S
@@ -53,7 +53,7 @@
 /*
  * This is the size at which it becomes more efficient to
  * clean the whole cache, rather than using the individual
- * cache line maintainence instructions.
+ * cache line maintenance instructions.
  */
 #define CACHE_DLIMIT    32768
 
@@ -108,6 +108,20 @@ ENTRY(cpu_arm1026_do_idle)
 /* ================================= CACHE ================================ */
 
         .align  5
+
+/*
+ *      flush_icache_all()
+ *
+ *      Unconditionally clean and invalidate the entire icache.
+ */
+ENTRY(arm1026_flush_icache_all)
+#ifndef CONFIG_CPU_ICACHE_DISABLE
+        mov     r0, #0
+        mcr     p15, 0, r0, c7, c5, 0           @ invalidate I cache
+#endif
+        mov     pc, lr
+ENDPROC(arm1026_flush_icache_all)
+
 /*
  *      flush_user_cache_all()
  *
@@ -320,6 +334,7 @@ ENTRY(arm1026_dma_unmap_area)
 ENDPROC(arm1026_dma_unmap_area)
 
 ENTRY(arm1026_cache_fns)
+        .long   arm1026_flush_icache_all
         .long   arm1026_flush_kern_cache_all
         .long   arm1026_flush_user_cache_all
         .long   arm1026_flush_user_cache_range
@@ -384,7 +399,7 @@ ENTRY(cpu_arm1026_set_pte_ext)
         mov     pc, lr
 
 
-        __INIT
+        __CPUINIT
 
         .type   __arm1026_setup, #function
 __arm1026_setup:
@@ -437,6 +452,9 @@ arm1026_processor_functions:
         .word   cpu_arm1026_dcache_clean_area
         .word   cpu_arm1026_switch_mm
         .word   cpu_arm1026_set_pte_ext
+        .word   0
+        .word   0
+        .word   0
         .size   arm1026_processor_functions, . - arm1026_processor_functions
 
         .section .rodata
diff --git a/arch/arm/mm/proc-arm6_7.S b/arch/arm/mm/proc-arm6_7.S
index 64e0b327c7c5..5f79dc4ce3fb 100644
--- a/arch/arm/mm/proc-arm6_7.S
+++ b/arch/arm/mm/proc-arm6_7.S
@@ -238,7 +238,7 @@ ENTRY(cpu_arm7_reset)
                 mcr     p15, 0, r1, c1, c0, 0           @ turn off MMU etc
                 mov     pc, r0
 
-                __INIT
+                __CPUINIT
 
                 .type   __arm6_setup, #function
 __arm6_setup:   mov     r0, #0
@@ -284,6 +284,9 @@ ENTRY(arm6_processor_functions)
                 .word   cpu_arm6_dcache_clean_area
                 .word   cpu_arm6_switch_mm
                 .word   cpu_arm6_set_pte_ext
+                .word   0
+                .word   0
+                .word   0
                 .size   arm6_processor_functions, . - arm6_processor_functions
 
 /*
@@ -301,6 +304,9 @@ ENTRY(arm7_processor_functions)
                 .word   cpu_arm7_dcache_clean_area
                 .word   cpu_arm7_switch_mm
                 .word   cpu_arm7_set_pte_ext
+                .word   0
+                .word   0
+                .word   0
                 .size   arm7_processor_functions, . - arm7_processor_functions
 
                 .section ".rodata"
diff --git a/arch/arm/mm/proc-arm720.S b/arch/arm/mm/proc-arm720.S
index 9d96824134fc..7a06e5964f59 100644
--- a/arch/arm/mm/proc-arm720.S
+++ b/arch/arm/mm/proc-arm720.S
@@ -63,7 +63,7 @@ ENTRY(cpu_arm720_proc_fin)
 /*
  * Function: arm720_proc_do_idle(void)
  * Params  : r0 = unused
- * Purpose : put the processer in proper idle mode
+ * Purpose : put the processor in proper idle mode
  */
 ENTRY(cpu_arm720_do_idle)
                 mov     pc, lr
@@ -113,7 +113,7 @@ ENTRY(cpu_arm720_reset)
                 mcr     p15, 0, ip, c1, c0, 0           @ ctrl register
                 mov     pc, r0
 
-        __INIT
+        __CPUINIT
 
         .type   __arm710_setup, #function
 __arm710_setup:
@@ -185,6 +185,9 @@ ENTRY(arm720_processor_functions)
                 .word   cpu_arm720_dcache_clean_area
                 .word   cpu_arm720_switch_mm
                 .word   cpu_arm720_set_pte_ext
+                .word   0
+                .word   0
+                .word   0
                 .size   arm720_processor_functions, . - arm720_processor_functions
 
                 .section ".rodata"
diff --git a/arch/arm/mm/proc-arm740.S b/arch/arm/mm/proc-arm740.S
index 6c1a9ab059ae..6f9d12effee1 100644
--- a/arch/arm/mm/proc-arm740.S
+++ b/arch/arm/mm/proc-arm740.S
@@ -55,7 +55,7 @@ ENTRY(cpu_arm740_reset)
         mcr     p15, 0, ip, c1, c0, 0           @ ctrl register
         mov     pc, r0
 
-        __INIT
+        __CPUINIT
 
         .type   __arm740_setup, #function
 __arm740_setup:
@@ -130,6 +130,9 @@ ENTRY(arm740_processor_functions)
         .word   cpu_arm740_dcache_clean_area
         .word   cpu_arm740_switch_mm
         .word   0                       @ cpu_*_set_pte
+        .word   0
+        .word   0
+        .word   0
         .size   arm740_processor_functions, . - arm740_processor_functions
 
         .section ".rodata"
diff --git a/arch/arm/mm/proc-arm7tdmi.S b/arch/arm/mm/proc-arm7tdmi.S
index 6a850dbba22e..537ffcb0646d 100644
--- a/arch/arm/mm/proc-arm7tdmi.S
+++ b/arch/arm/mm/proc-arm7tdmi.S
@@ -46,7 +46,7 @@ ENTRY(cpu_arm7tdmi_proc_fin)
 ENTRY(cpu_arm7tdmi_reset)
                 mov     pc, r0
 
-                __INIT
+                __CPUINIT
 
                 .type   __arm7tdmi_setup, #function
 __arm7tdmi_setup:
@@ -70,6 +70,9 @@ ENTRY(arm7tdmi_processor_functions)
                 .word   cpu_arm7tdmi_dcache_clean_area
                 .word   cpu_arm7tdmi_switch_mm
                 .word   0               @ cpu_*_set_pte
+                .word   0
+                .word   0
+                .word   0
                 .size   arm7tdmi_processor_functions, . - arm7tdmi_processor_functions
 
                 .section ".rodata"
@@ -143,7 +146,7 @@ __arm7tdmi_proc_info:
                 .long   0
                 .long   0
                 .long   v4_cache_fns
-                .size   __arm7tdmi_proc_info, . - __arm7dmi_proc_info
+                .size   __arm7tdmi_proc_info, . - __arm7tdmi_proc_info
 
                 .type   __triscenda7_proc_info, #object
 __triscenda7_proc_info:
diff --git a/arch/arm/mm/proc-arm920.S b/arch/arm/mm/proc-arm920.S
index 86f80aa56216..bf8a1d1cccb6 100644
--- a/arch/arm/mm/proc-arm920.S
+++ b/arch/arm/mm/proc-arm920.S
@@ -53,7 +53,7 @@
 /*
  * This is the size at which it becomes more efficient to
  * clean the whole cache, rather than using the individual
- * cache line maintainence instructions.
+ * cache line maintenance instructions.
  */
 #define CACHE_DLIMIT    65536
 
@@ -110,6 +110,17 @@ ENTRY(cpu_arm920_do_idle)
 #ifndef CONFIG_CPU_DCACHE_WRITETHROUGH
 
 /*
+ *      flush_icache_all()
+ *
+ *      Unconditionally clean and invalidate the entire icache.
+ */
+ENTRY(arm920_flush_icache_all)
+        mov     r0, #0
+        mcr     p15, 0, r0, c7, c5, 0           @ invalidate I cache
+        mov     pc, lr
+ENDPROC(arm920_flush_icache_all)
+
+/*
  *      flush_user_cache_all()
  *
  *      Invalidate all cache entries in a particular address
@@ -305,6 +316,7 @@ ENTRY(arm920_dma_unmap_area)
 ENDPROC(arm920_dma_unmap_area)
 
 ENTRY(arm920_cache_fns)
+        .long   arm920_flush_icache_all
         .long   arm920_flush_kern_cache_all
         .long   arm920_flush_user_cache_all
         .long   arm920_flush_user_cache_range
@@ -375,7 +387,41 @@ ENTRY(cpu_arm920_set_pte_ext)
 #endif
         mov     pc, lr
 
-        __INIT
+/* Suspend/resume support: taken from arch/arm/plat-s3c24xx/sleep.S */
+.globl  cpu_arm920_suspend_size
+.equ    cpu_arm920_suspend_size, 4 * 3
+#ifdef CONFIG_PM_SLEEP
+ENTRY(cpu_arm920_do_suspend)
+        stmfd   sp!, {r4 - r7, lr}
+        mrc     p15, 0, r4, c13, c0, 0  @ PID
+        mrc     p15, 0, r5, c3, c0, 0   @ Domain ID
+        mrc     p15, 0, r6, c2, c0, 0   @ TTB address
+        mrc     p15, 0, r7, c1, c0, 0   @ Control register
+        stmia   r0, {r4 - r7}
+        ldmfd   sp!, {r4 - r7, pc}
+ENDPROC(cpu_arm920_do_suspend)
+
+ENTRY(cpu_arm920_do_resume)
+        mov     ip, #0
+        mcr     p15, 0, ip, c8, c7, 0   @ invalidate I+D TLBs
+        mcr     p15, 0, ip, c7, c7, 0   @ invalidate I+D caches
+        ldmia   r0, {r4 - r7}
+        mcr     p15, 0, r4, c13, c0, 0  @ PID
+        mcr     p15, 0, r5, c3, c0, 0   @ Domain ID
+        mcr     p15, 0, r6, c2, c0, 0   @ TTB address
+        mov     r0, r7                  @ control register
+        mov     r2, r6, lsr #14         @ get TTB0 base
+        mov     r2, r2, lsl #14
+        ldr     r3, =PMD_TYPE_SECT | PMD_SECT_BUFFERABLE | \
+                     PMD_SECT_CACHEABLE | PMD_BIT4 | PMD_SECT_AP_WRITE
+        b       cpu_resume_mmu
+ENDPROC(cpu_arm920_do_resume)
+#else
+#define cpu_arm920_do_suspend   0
+#define cpu_arm920_do_resume    0
+#endif
+
+        __CPUINIT
 
         .type   __arm920_setup, #function
 __arm920_setup:
@@ -420,6 +466,9 @@ arm920_processor_functions:
         .word   cpu_arm920_dcache_clean_area
         .word   cpu_arm920_switch_mm
         .word   cpu_arm920_set_pte_ext
+        .word   cpu_arm920_suspend_size
+        .word   cpu_arm920_do_suspend
+        .word   cpu_arm920_do_resume
         .size   arm920_processor_functions, . - arm920_processor_functions
 
         .section ".rodata"
diff --git a/arch/arm/mm/proc-arm922.S b/arch/arm/mm/proc-arm922.S
index f76ce9b62883..95ba1fc56e4d 100644
--- a/arch/arm/mm/proc-arm922.S
+++ b/arch/arm/mm/proc-arm922.S
@@ -54,7 +54,7 @@
 /*
  * This is the size at which it becomes more efficient to
  * clean the whole cache, rather than using the individual
- * cache line maintainence instructions.  (I think this should
+ * cache line maintenance instructions.  (I think this should
  * be 32768).
  */
 #define CACHE_DLIMIT    8192
@@ -112,6 +112,17 @@ ENTRY(cpu_arm922_do_idle)
 #ifndef CONFIG_CPU_DCACHE_WRITETHROUGH
 
 /*
+ *      flush_icache_all()
+ *
+ *      Unconditionally clean and invalidate the entire icache.
+ */
+ENTRY(arm922_flush_icache_all)
+        mov     r0, #0
+        mcr     p15, 0, r0, c7, c5, 0           @ invalidate I cache
+        mov     pc, lr
+ENDPROC(arm922_flush_icache_all)
+
+/*
  *      flush_user_cache_all()
  *
  *      Clean and invalidate all cache entries in a particular
@@ -307,6 +318,7 @@ ENTRY(arm922_dma_unmap_area)
 ENDPROC(arm922_dma_unmap_area)
 
 ENTRY(arm922_cache_fns)
+        .long   arm922_flush_icache_all
         .long   arm922_flush_kern_cache_all
         .long   arm922_flush_user_cache_all
         .long   arm922_flush_user_cache_range
@@ -379,7 +391,7 @@ ENTRY(cpu_arm922_set_pte_ext)
 #endif /* CONFIG_MMU */
         mov     pc, lr
 
-        __INIT
+        __CPUINIT
 
         .type   __arm922_setup, #function
 __arm922_setup:
@@ -424,6 +436,9 @@ arm922_processor_functions:
         .word   cpu_arm922_dcache_clean_area
         .word   cpu_arm922_switch_mm
         .word   cpu_arm922_set_pte_ext
+        .word   0
+        .word   0
+        .word   0
         .size   arm922_processor_functions, . - arm922_processor_functions
 
         .section ".rodata"
diff --git a/arch/arm/mm/proc-arm925.S b/arch/arm/mm/proc-arm925.S
index 657bd3f7c153..541e4774eea1 100644
--- a/arch/arm/mm/proc-arm925.S
+++ b/arch/arm/mm/proc-arm925.S
@@ -77,7 +77,7 @@
 /*
  * This is the size at which it becomes more efficient to
  * clean the whole cache, rather than using the individual
- * cache line maintainence instructions.
+ * cache line maintenance instructions.
  */
 #define CACHE_DLIMIT    8192
 
@@ -145,6 +145,17 @@ ENTRY(cpu_arm925_do_idle)
         mov     pc, lr
 
 /*
+ *      flush_icache_all()
+ *
+ *      Unconditionally clean and invalidate the entire icache.
+ */
+ENTRY(arm925_flush_icache_all)
+        mov     r0, #0
+        mcr     p15, 0, r0, c7, c5, 0           @ invalidate I cache
+        mov     pc, lr
+ENDPROC(arm925_flush_icache_all)
+
+/*
  *      flush_user_cache_all()
  *
  *      Clean and invalidate all cache entries in a particular
@@ -362,6 +373,7 @@ ENTRY(arm925_dma_unmap_area)
 ENDPROC(arm925_dma_unmap_area)
 
 ENTRY(arm925_cache_fns)
+        .long   arm925_flush_icache_all
         .long   arm925_flush_kern_cache_all
         .long   arm925_flush_user_cache_all
         .long   arm925_flush_user_cache_range
@@ -428,7 +440,7 @@ ENTRY(cpu_arm925_set_pte_ext)
 #endif /* CONFIG_MMU */
         mov     pc, lr
 
-        __INIT
+        __CPUINIT
 
         .type   __arm925_setup, #function
 __arm925_setup:
@@ -491,6 +503,9 @@ arm925_processor_functions:
         .word   cpu_arm925_dcache_clean_area
         .word   cpu_arm925_switch_mm
         .word   cpu_arm925_set_pte_ext
+        .word   0
+        .word   0
+        .word   0
         .size   arm925_processor_functions, . - arm925_processor_functions
 
         .section ".rodata"
diff --git a/arch/arm/mm/proc-arm926.S b/arch/arm/mm/proc-arm926.S
index 73f1f3c68910..0ed85d930c09 100644
--- a/arch/arm/mm/proc-arm926.S
+++ b/arch/arm/mm/proc-arm926.S
@@ -111,6 +111,17 @@ ENTRY(cpu_arm926_do_idle)
         mov     pc, lr
 
 /*
+ *      flush_icache_all()
+ *
+ *      Unconditionally clean and invalidate the entire icache.
+ */
+ENTRY(arm926_flush_icache_all)
+        mov     r0, #0
+        mcr     p15, 0, r0, c7, c5, 0           @ invalidate I cache
+        mov     pc, lr
+ENDPROC(arm926_flush_icache_all)
+
+/*
  *      flush_user_cache_all()
  *
  *      Clean and invalidate all cache entries in a particular
@@ -325,6 +336,7 @@ ENTRY(arm926_dma_unmap_area)
 ENDPROC(arm926_dma_unmap_area)
 
 ENTRY(arm926_cache_fns)
+        .long   arm926_flush_icache_all
         .long   arm926_flush_kern_cache_all
         .long   arm926_flush_user_cache_all
         .long   arm926_flush_user_cache_range
@@ -389,7 +401,41 @@ ENTRY(cpu_arm926_set_pte_ext)
 #endif
         mov     pc, lr
 
-        __INIT
+/* Suspend/resume support: taken from arch/arm/plat-s3c24xx/sleep.S */
+.globl  cpu_arm926_suspend_size
+.equ    cpu_arm926_suspend_size, 4 * 3
+#ifdef CONFIG_PM_SLEEP
+ENTRY(cpu_arm926_do_suspend)
+        stmfd   sp!, {r4 - r7, lr}
+        mrc     p15, 0, r4, c13, c0, 0  @ PID
+        mrc     p15, 0, r5, c3, c0, 0   @ Domain ID
+        mrc     p15, 0, r6, c2, c0, 0   @ TTB address
+        mrc     p15, 0, r7, c1, c0, 0   @ Control register
+        stmia   r0, {r4 - r7}
+        ldmfd   sp!, {r4 - r7, pc}
+ENDPROC(cpu_arm926_do_suspend)
+
+ENTRY(cpu_arm926_do_resume)
+        mov     ip, #0
+        mcr     p15, 0, ip, c8, c7, 0   @ invalidate I+D TLBs
+        mcr     p15, 0, ip, c7, c7, 0   @ invalidate I+D caches
+        ldmia   r0, {r4 - r7}
+        mcr     p15, 0, r4, c13, c0, 0  @ PID
+        mcr     p15, 0, r5, c3, c0, 0   @ Domain ID
+        mcr     p15, 0, r6, c2, c0, 0   @ TTB address
+        mov     r0, r7                  @ control register
+        mov     r2, r6, lsr #14         @ get TTB0 base
+        mov     r2, r2, lsl #14
+        ldr     r3, =PMD_TYPE_SECT | PMD_SECT_BUFFERABLE | \
+                     PMD_SECT_CACHEABLE | PMD_BIT4 | PMD_SECT_AP_WRITE
+        b       cpu_resume_mmu
+ENDPROC(cpu_arm926_do_resume)
+#else
+#define cpu_arm926_do_suspend   0
+#define cpu_arm926_do_resume    0
+#endif
+
+        __CPUINIT
 
         .type   __arm926_setup, #function
 __arm926_setup:
@@ -444,6 +490,9 @@ arm926_processor_functions:
         .word   cpu_arm926_dcache_clean_area
         .word   cpu_arm926_switch_mm
         .word   cpu_arm926_set_pte_ext
+        .word   cpu_arm926_suspend_size
+        .word   cpu_arm926_do_suspend
+        .word   cpu_arm926_do_resume
         .size   arm926_processor_functions, . - arm926_processor_functions
 
         .section ".rodata"
diff --git a/arch/arm/mm/proc-arm940.S b/arch/arm/mm/proc-arm940.S
index fffb061a45a5..26aea3f71c26 100644
--- a/arch/arm/mm/proc-arm940.S
+++ b/arch/arm/mm/proc-arm940.S
@@ -68,6 +68,17 @@ ENTRY(cpu_arm940_do_idle)
         mov     pc, lr
 
 /*
+ *      flush_icache_all()
+ *
+ *      Unconditionally clean and invalidate the entire icache.
+ */
+ENTRY(arm940_flush_icache_all)
+        mov     r0, #0
+        mcr     p15, 0, r0, c7, c5, 0           @ invalidate I cache
+        mov     pc, lr
+ENDPROC(arm940_flush_icache_all)
+
+/*
  *      flush_user_cache_all()
  */
 ENTRY(arm940_flush_user_cache_all)
@@ -254,6 +265,7 @@ ENTRY(arm940_dma_unmap_area)
 ENDPROC(arm940_dma_unmap_area)
 
 ENTRY(arm940_cache_fns)
+        .long   arm940_flush_icache_all
         .long   arm940_flush_kern_cache_all
         .long   arm940_flush_user_cache_all
         .long   arm940_flush_user_cache_range
@@ -264,7 +276,7 @@ ENTRY(arm940_cache_fns)
         .long   arm940_dma_unmap_area
         .long   arm940_dma_flush_range
 
-        __INIT
+        __CPUINIT
 
         .type   __arm940_setup, #function
 __arm940_setup:
@@ -351,6 +363,9 @@ ENTRY(arm940_processor_functions)
         .word   cpu_arm940_dcache_clean_area
         .word   cpu_arm940_switch_mm
         .word   0               @ cpu_*_set_pte
+        .word   0
+        .word   0
+        .word   0
         .size   arm940_processor_functions, . - arm940_processor_functions
 
         .section ".rodata"
diff --git a/arch/arm/mm/proc-arm946.S b/arch/arm/mm/proc-arm946.S
index 249a6053760a..8063345406fe 100644
--- a/arch/arm/mm/proc-arm946.S
+++ b/arch/arm/mm/proc-arm946.S
@@ -75,6 +75,17 @@ ENTRY(cpu_arm946_do_idle)
         mov     pc, lr
 
 /*
+ *      flush_icache_all()
+ *
+ *      Unconditionally clean and invalidate the entire icache.
+ */
+ENTRY(arm946_flush_icache_all)
+        mov     r0, #0
+        mcr     p15, 0, r0, c7, c5, 0           @ invalidate I cache
+        mov     pc, lr
+ENDPROC(arm946_flush_icache_all)
+
+/*
  *      flush_user_cache_all()
  */
 ENTRY(arm946_flush_user_cache_all)
@@ -296,6 +307,7 @@ ENTRY(arm946_dma_unmap_area)
 ENDPROC(arm946_dma_unmap_area)
 
 ENTRY(arm946_cache_fns)
+        .long   arm946_flush_icache_all
         .long   arm946_flush_kern_cache_all
         .long   arm946_flush_user_cache_all
         .long   arm946_flush_user_cache_range
@@ -317,7 +329,7 @@ ENTRY(cpu_arm946_dcache_clean_area)
         mcr     p15, 0, r0, c7, c10, 4          @ drain WB
         mov     pc, lr
 
-        __INIT
+        __CPUINIT
 
         .type   __arm946_setup, #function
 __arm946_setup:
@@ -407,6 +419,9 @@ ENTRY(arm946_processor_functions)
         .word   cpu_arm946_dcache_clean_area
         .word   cpu_arm946_switch_mm
         .word   0               @ cpu_*_set_pte
+        .word   0
+        .word   0
+        .word   0
         .size   arm946_processor_functions, . - arm946_processor_functions
 
         .section ".rodata"
diff --git a/arch/arm/mm/proc-arm9tdmi.S b/arch/arm/mm/proc-arm9tdmi.S
index db475667fac2..546b54da1005 100644
--- a/arch/arm/mm/proc-arm9tdmi.S
+++ b/arch/arm/mm/proc-arm9tdmi.S
@@ -46,7 +46,7 @@ ENTRY(cpu_arm9tdmi_proc_fin)
 ENTRY(cpu_arm9tdmi_reset)
                 mov     pc, r0
 
-                __INIT
+                __CPUINIT
 
                 .type   __arm9tdmi_setup, #function
 __arm9tdmi_setup:
@@ -70,6 +70,9 @@ ENTRY(arm9tdmi_processor_functions)
                 .word   cpu_arm9tdmi_dcache_clean_area
                 .word   cpu_arm9tdmi_switch_mm
                 .word   0               @ cpu_*_set_pte
+                .word   0
+                .word   0
+                .word   0
                 .size   arm9tdmi_processor_functions, . - arm9tdmi_processor_functions
 
                 .section ".rodata"
@@ -113,7 +116,7 @@ __arm9tdmi_proc_info:
                 .long   0
                 .long   0
                 .long   v4_cache_fns
-                .size   __arm9tdmi_proc_info, . - __arm9dmi_proc_info
+                .size   __arm9tdmi_proc_info, . - __arm9tdmi_proc_info
 
                 .type   __p2001_proc_info, #object
 __p2001_proc_info:
diff --git a/arch/arm/mm/proc-fa526.S b/arch/arm/mm/proc-fa526.S
index 7803fdf70029..fc2a4ae15cf4 100644
--- a/arch/arm/mm/proc-fa526.S
+++ b/arch/arm/mm/proc-fa526.S
@@ -134,7 +134,7 @@ ENTRY(cpu_fa526_set_pte_ext)
 #endif
         mov     pc, lr
 
-        __INIT
+        __CPUINIT
 
         .type   __fa526_setup, #function
 __fa526_setup:
@@ -195,6 +195,9 @@ fa526_processor_functions:
         .word   cpu_fa526_dcache_clean_area
         .word   cpu_fa526_switch_mm
         .word   cpu_fa526_set_pte_ext
+        .word   0
+        .word   0
+        .word   0
         .size   fa526_processor_functions, . - fa526_processor_functions
 
         .section ".rodata"
diff --git a/arch/arm/mm/proc-feroceon.S b/arch/arm/mm/proc-feroceon.S
index b304d0104a4e..d3883eed7a4a 100644
--- a/arch/arm/mm/proc-feroceon.S
+++ b/arch/arm/mm/proc-feroceon.S
@@ -124,6 +124,17 @@ ENTRY(cpu_feroceon_do_idle)
         mov     pc, lr
 
 /*
+ *      flush_icache_all()
+ *
+ *      Unconditionally clean and invalidate the entire icache.
+ */
+ENTRY(feroceon_flush_icache_all)
+        mov     r0, #0
+        mcr     p15, 0, r0, c7, c5, 0           @ invalidate I cache
+        mov     pc, lr
+ENDPROC(feroceon_flush_icache_all)
+
+/*
  *      flush_user_cache_all()
  *
  *      Clean and invalidate all cache entries in a particular
@@ -401,6 +412,7 @@ ENTRY(feroceon_dma_unmap_area)
 ENDPROC(feroceon_dma_unmap_area)
 
 ENTRY(feroceon_cache_fns)
+        .long   feroceon_flush_icache_all
         .long   feroceon_flush_kern_cache_all
         .long   feroceon_flush_user_cache_all
         .long   feroceon_flush_user_cache_range
@@ -412,6 +424,7 @@ ENTRY(feroceon_cache_fns)
         .long   feroceon_dma_flush_range
 
 ENTRY(feroceon_range_cache_fns)
+        .long   feroceon_flush_icache_all
         .long   feroceon_flush_kern_cache_all
         .long   feroceon_flush_user_cache_all
         .long   feroceon_flush_user_cache_range
@@ -494,7 +507,7 @@ ENTRY(cpu_feroceon_set_pte_ext)
 #endif
         mov     pc, lr
 
-        __INIT
+        __CPUINIT
 
         .type   __feroceon_setup, #function
 __feroceon_setup:
@@ -541,6 +554,9 @@ feroceon_processor_functions:
         .word   cpu_feroceon_dcache_clean_area
         .word   cpu_feroceon_switch_mm
         .word   cpu_feroceon_set_pte_ext
+        .word   0
+        .word   0
+        .word   0
         .size   feroceon_processor_functions, . - feroceon_processor_functions
 
         .section ".rodata"
diff --git a/arch/arm/mm/proc-macros.S b/arch/arm/mm/proc-macros.S
index 7d63beaf9745..34261f9486b9 100644
--- a/arch/arm/mm/proc-macros.S
+++ b/arch/arm/mm/proc-macros.S
@@ -61,27 +61,37 @@
         .endm
 
 /*
- * cache_line_size - get the cache line size from the CSIDR register
- * (available on ARMv7+). It assumes that the CSSR register was configured
- * to access the L1 data cache CSIDR.
+ * dcache_line_size - get the minimum D-cache line size from the CTR register
+ * on ARMv7.
  */
         .macro  dcache_line_size, reg, tmp
-        mrc     p15, 1, \tmp, c0, c0, 0         @ read CSIDR
-        and     \tmp, \tmp, #7                  @ cache line size encoding
-        mov     \reg, #16                       @ size offset
+        mrc     p15, 0, \tmp, c0, c0, 1         @ read ctr
+        lsr     \tmp, \tmp, #16
+        and     \tmp, \tmp, #0xf                @ cache line size encoding
+        mov     \reg, #4                        @ bytes per word
         mov     \reg, \reg, lsl \tmp            @ actual cache line size
         .endm
 
+/*
+ * icache_line_size - get the minimum I-cache line size from the CTR register
+ * on ARMv7.
+ */
+        .macro  icache_line_size, reg, tmp
+        mrc     p15, 0, \tmp, c0, c0, 1         @ read ctr
+        and     \tmp, \tmp, #0xf                @ cache line size encoding
+        mov     \reg, #4                        @ bytes per word
+        mov     \reg, \reg, lsl \tmp            @ actual cache line size
+        .endm
 
 /*
  * Sanity check the PTE configuration for the code below - which makes
- * certain assumptions about how these bits are layed out.
+ * certain assumptions about how these bits are laid out.
  */
 #ifdef CONFIG_MMU
 #if L_PTE_SHARED != PTE_EXT_SHARED
 #error PTE shared bit mismatch
 #endif
-#if (L_PTE_EXEC+L_PTE_USER+L_PTE_WRITE+L_PTE_DIRTY+L_PTE_YOUNG+\
+#if (L_PTE_XN+L_PTE_USER+L_PTE_RDONLY+L_PTE_DIRTY+L_PTE_YOUNG+\
      L_PTE_FILE+L_PTE_PRESENT) > L_PTE_SHARED
 #error Invalid Linux PTE bit settings
 #endif
@@ -99,6 +109,10 @@
  *  110x   0   1   0    r/w     r/o
  *  11x0   0   1   0    r/w     r/o
  *  1111   0   1   1    r/w     r/w
+ *
+ * If !CONFIG_CPU_USE_DOMAINS, the following permissions are changed:
+ *  110x   1   1   1    r/o     r/o
+ *  11x0   1   1   1    r/o     r/o
  */
         .macro  armv6_mt_table pfx
 \pfx\()_mt_table:
@@ -121,7 +135,7 @@
         .endm
 
         .macro  armv6_set_pte_ext pfx
-        str     r1, [r0], #-2048                @ linux version
+        str     r1, [r0], #2048                 @ linux version
 
         bic     r3, r1, #0x000003fc
         bic     r3, r3, #PTE_TYPE_MASK
@@ -132,17 +146,20 @@
         and     r2, r1, #L_PTE_MT_MASK
         ldr     r2, [ip, r2]
 
-        tst     r1, #L_PTE_WRITE
-        tstne   r1, #L_PTE_DIRTY
-        orreq   r3, r3, #PTE_EXT_APX
+        eor     r1, r1, #L_PTE_DIRTY
+        tst     r1, #L_PTE_DIRTY|L_PTE_RDONLY
+        orrne   r3, r3, #PTE_EXT_APX
 
         tst     r1, #L_PTE_USER
         orrne   r3, r3, #PTE_EXT_AP1
+#ifdef CONFIG_CPU_USE_DOMAINS
+        @ allow kernel read/write access to read-only user pages
         tstne   r3, #PTE_EXT_APX
         bicne   r3, r3, #PTE_EXT_APX | PTE_EXT_AP0
+#endif
 
-        tst     r1, #L_PTE_EXEC
-        orreq   r3, r3, #PTE_EXT_XN
+        tst     r1, #L_PTE_XN
+        orrne   r3, r3, #PTE_EXT_XN
 
         orr     r3, r3, r2
 
@@ -170,9 +187,9 @@
  *  1111  0xff  r/w     r/w
  */
         .macro  armv3_set_pte_ext wc_disable=1
-        str     r1, [r0], #-2048                @ linux version
+        str     r1, [r0], #2048                 @ linux version
 
-        eor     r3, r1, #L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_WRITE | L_PTE_DIRTY
+        eor     r3, r1, #L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY
 
         bic     r2, r1, #PTE_SMALL_AP_MASK      @ keep C, B bits
         bic     r2, r2, #PTE_TYPE_MASK
@@ -181,7 +198,7 @@
         tst     r3, #L_PTE_USER                 @ user?
         orrne   r2, r2, #PTE_SMALL_AP_URO_SRW
 
-        tst     r3, #L_PTE_WRITE | L_PTE_DIRTY  @ write and dirty?
+        tst     r3, #L_PTE_RDONLY | L_PTE_DIRTY @ write and dirty?
         orreq   r2, r2, #PTE_SMALL_AP_UNO_SRW
 
         tst     r3, #L_PTE_PRESENT | L_PTE_YOUNG        @ present and young?
@@ -193,7 +210,7 @@
         bicne   r2, r2, #PTE_BUFFERABLE
 #endif
         .endif
-        str     r2, [r0]                        @ hardware version
+        str     r2, [r0]                @ hardware version
         .endm
 
 
@@ -213,9 +230,9 @@
  *  1111  11    r/w     r/w
  */
         .macro  xscale_set_pte_ext_prologue
-        str     r1, [r0], #-2048                @ linux version
+        str     r1, [r0]                        @ linux version
 
-        eor     r3, r1, #L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_WRITE | L_PTE_DIRTY
+        eor     r3, r1, #L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY
 
         bic     r2, r1, #PTE_SMALL_AP_MASK      @ keep C, B bits
         orr     r2, r2, #PTE_TYPE_EXT           @ extended page
@@ -223,7 +240,7 @@
         tst     r3, #L_PTE_USER                 @ user?
         orrne   r2, r2, #PTE_EXT_AP_URO_SRW     @ yes -> user r/o, system r/w
 
-        tst     r3, #L_PTE_WRITE | L_PTE_DIRTY  @ write and dirty?
+        tst     r3, #L_PTE_RDONLY | L_PTE_DIRTY @ write and dirty?
         orreq   r2, r2, #PTE_EXT_AP_UNO_SRW     @ yes -> user n/a, system r/w
                                                 @ combined with user -> user r/w
         .endm
@@ -232,7 +249,7 @@
         tst     r3, #L_PTE_PRESENT | L_PTE_YOUNG        @ present and young?
         movne   r2, #0                          @ no -> fault
 
-        str     r2, [r0]                        @ hardware version
+        str     r2, [r0, #2048]!                @ hardware version
         mov     ip, #0
         mcr     p15, 0, r0, c7, c10, 1          @ clean L1 D line
         mcr     p15, 0, ip, c7, c10, 4          @ data write barrier
diff --git a/arch/arm/mm/proc-mohawk.S b/arch/arm/mm/proc-mohawk.S
index 5f6892fcc167..9d4f2ae63370 100644
--- a/arch/arm/mm/proc-mohawk.S
+++ b/arch/arm/mm/proc-mohawk.S
@@ -338,7 +338,7 @@ ENTRY(cpu_mohawk_set_pte_ext)
         mcr     p15, 0, r0, c7, c10, 4          @ drain WB
         mov     pc, lr
 
-        __INIT
+        __CPUINIT
 
         .type   __mohawk_setup, #function
 __mohawk_setup:
@@ -388,6 +388,9 @@ mohawk_processor_functions:
         .word   cpu_mohawk_dcache_clean_area
         .word   cpu_mohawk_switch_mm
         .word   cpu_mohawk_set_pte_ext
+        .word   0
+        .word   0
+        .word   0
         .size   mohawk_processor_functions, . - mohawk_processor_functions
 
         .section ".rodata"
diff --git a/arch/arm/mm/proc-sa110.S b/arch/arm/mm/proc-sa110.S
index a201eb04b5e1..46f09ed16b98 100644
--- a/arch/arm/mm/proc-sa110.S
+++ b/arch/arm/mm/proc-sa110.S
@@ -156,7 +156,7 @@ ENTRY(cpu_sa110_set_pte_ext)
 #endif
         mov     pc, lr
 
-        __INIT
+        __CPUINIT
 
         .type   __sa110_setup, #function
 __sa110_setup:
@@ -203,6 +203,9 @@ ENTRY(sa110_processor_functions)
         .word   cpu_sa110_dcache_clean_area
         .word   cpu_sa110_switch_mm
         .word   cpu_sa110_set_pte_ext
+        .word   0
+        .word   0
+        .word   0
         .size   sa110_processor_functions, . - sa110_processor_functions
 
         .section ".rodata"
diff --git a/arch/arm/mm/proc-sa1100.S b/arch/arm/mm/proc-sa1100.S
index 7ddc4805bf97..184a9c997e36 100644
--- a/arch/arm/mm/proc-sa1100.S
+++ b/arch/arm/mm/proc-sa1100.S
@@ -169,7 +169,43 @@ ENTRY(cpu_sa1100_set_pte_ext)
 #endif
         mov     pc, lr
 
-        __INIT
+.globl  cpu_sa1100_suspend_size
+.equ    cpu_sa1100_suspend_size, 4*4
+#ifdef CONFIG_PM_SLEEP
+ENTRY(cpu_sa1100_do_suspend)
+        stmfd   sp!, {r4 - r7, lr}
+        mrc     p15, 0, r4, c3, c0, 0           @ domain ID
+        mrc     p15, 0, r5, c2, c0, 0           @ translation table base addr
+        mrc     p15, 0, r6, c13, c0, 0          @ PID
+        mrc     p15, 0, r7, c1, c0, 0           @ control reg
+        stmia   r0, {r4 - r7}                   @ store cp regs
+        ldmfd   sp!, {r4 - r7, pc}
+ENDPROC(cpu_sa1100_do_suspend)
+
+ENTRY(cpu_sa1100_do_resume)
+        ldmia   r0, {r4 - r7}                   @ load cp regs
+        mov     r1, #0
+        mcr     p15, 0, r1, c8, c7, 0           @ flush I+D TLBs
+        mcr     p15, 0, r1, c7, c7, 0           @ flush I&D cache
+        mcr     p15, 0, r1, c9, c0, 0           @ invalidate RB
+        mcr     p15, 0, r1, c9, c0, 5           @ allow user space to use RB
+
+        mcr     p15, 0, r4, c3, c0, 0           @ domain ID
+        mcr     p15, 0, r5, c2, c0, 0           @ translation table base addr
+        mcr     p15, 0, r6, c13, c0, 0          @ PID
+        mov     r0, r7                          @ control register
+        mov     r2, r5, lsr #14                 @ get TTB0 base
+        mov     r2, r2, lsl #14
+        ldr     r3, =PMD_TYPE_SECT | PMD_SECT_BUFFERABLE | \
+                     PMD_SECT_CACHEABLE | PMD_SECT_AP_WRITE
+        b       cpu_resume_mmu
+ENDPROC(cpu_sa1100_do_resume)
+#else
+#define cpu_sa1100_do_suspend   0
+#define cpu_sa1100_do_resume    0
+#endif
+
+        __CPUINIT
 
         .type   __sa1100_setup, #function
 __sa1100_setup:
@@ -218,6 +254,9 @@ ENTRY(sa1100_processor_functions)
         .word   cpu_sa1100_dcache_clean_area
         .word   cpu_sa1100_switch_mm
         .word   cpu_sa1100_set_pte_ext
+        .word   cpu_sa1100_suspend_size
+        .word   cpu_sa1100_do_suspend
+        .word   cpu_sa1100_do_resume
         .size   sa1100_processor_functions, . - sa1100_processor_functions
 
         .section ".rodata"
diff --git a/arch/arm/mm/proc-v6.S b/arch/arm/mm/proc-v6.S
index 22aac8515196..1d2b8451bf25 100644
--- a/arch/arm/mm/proc-v6.S
+++ b/arch/arm/mm/proc-v6.S
@@ -30,13 +30,10 @@
 #define TTB_RGN_WT      (2 << 3)
 #define TTB_RGN_WB      (3 << 3)
 
-#ifndef CONFIG_SMP
-#define TTB_FLAGS       TTB_RGN_WBWA
-#define PMD_FLAGS       PMD_SECT_WB
-#else
-#define TTB_FLAGS       TTB_RGN_WBWA|TTB_S
-#define PMD_FLAGS       PMD_SECT_WBWA|PMD_SECT_S
-#endif
+#define TTB_FLAGS_UP    TTB_RGN_WBWA
+#define PMD_FLAGS_UP    PMD_SECT_WB
+#define TTB_FLAGS_SMP   TTB_RGN_WBWA|TTB_S
+#define PMD_FLAGS_SMP   PMD_SECT_WBWA|PMD_SECT_S
 
 ENTRY(cpu_v6_proc_init)
         mov     pc, lr
@@ -97,7 +94,8 @@ ENTRY(cpu_v6_switch_mm)
 #ifdef CONFIG_MMU
         mov     r2, #0
         ldr     r1, [r1, #MM_CONTEXT_ID]        @ get mm->context.id
-        orr     r0, r0, #TTB_FLAGS
+        ALT_SMP(orr     r0, r0, #TTB_FLAGS_SMP)
+        ALT_UP(orr      r0, r0, #TTB_FLAGS_UP)
         mcr     p15, 0, r2, c7, c5, 6           @ flush BTAC/BTB
         mcr     p15, 0, r2, c7, c10, 4          @ drain write buffer
         mcr     p15, 0, r0, c2, c0, 0           @ set TTB 0
@@ -123,6 +121,53 @@ ENTRY(cpu_v6_set_pte_ext)
 #endif
         mov     pc, lr
 
+/* Suspend/resume support: taken from arch/arm/mach-s3c64xx/sleep.S */
+.globl  cpu_v6_suspend_size
+.equ    cpu_v6_suspend_size, 4 * 8
+#ifdef CONFIG_PM_SLEEP
+ENTRY(cpu_v6_do_suspend)
+        stmfd   sp!, {r4 - r11, lr}
+        mrc     p15, 0, r4, c13, c0, 0  @ FCSE/PID
+        mrc     p15, 0, r5, c13, c0, 1  @ Context ID
+        mrc     p15, 0, r6, c3, c0, 0   @ Domain ID
+        mrc     p15, 0, r7, c2, c0, 0   @ Translation table base 0
+        mrc     p15, 0, r8, c2, c0, 1   @ Translation table base 1
+        mrc     p15, 0, r9, c1, c0, 1   @ auxiliary control register
+        mrc     p15, 0, r10, c1, c0, 2  @ co-processor access control
+        mrc     p15, 0, r11, c1, c0, 0  @ control register
+        stmia   r0, {r4 - r11}
+        ldmfd   sp!, {r4- r11, pc}
+ENDPROC(cpu_v6_do_suspend)
+
+ENTRY(cpu_v6_do_resume)
+        mov     ip, #0
+        mcr     p15, 0, ip, c7, c14, 0  @ clean+invalidate D cache
+        mcr     p15, 0, ip, c7, c5, 0   @ invalidate I cache
+        mcr     p15, 0, ip, c7, c15, 0  @ clean+invalidate cache
+        mcr     p15, 0, ip, c7, c10, 4  @ drain write buffer
+        ldmia   r0, {r4 - r11}
+        mcr     p15, 0, r4, c13, c0, 0  @ FCSE/PID
+        mcr     p15, 0, r5, c13, c0, 1  @ Context ID
+        mcr     p15, 0, r6, c3, c0, 0   @ Domain ID
+        mcr     p15, 0, r7, c2, c0, 0   @ Translation table base 0
+        mcr     p15, 0, r8, c2, c0, 1   @ Translation table base 1
+        mcr     p15, 0, r9, c1, c0, 1   @ auxiliary control register
+        mcr     p15, 0, r10, c1, c0, 2  @ co-processor access control
+        mcr     p15, 0, ip, c2, c0, 2   @ TTB control register
+        mcr     p15, 0, ip, c7, c5, 4   @ ISB
+        mov     r0, r11                 @ control register
+        mov     r2, r7, lsr #14         @ get TTB0 base
+        mov     r2, r2, lsl #14
+        ldr     r3, cpu_resume_l1_flags
+        b       cpu_resume_mmu
+ENDPROC(cpu_v6_do_resume)
+cpu_resume_l1_flags:
+        ALT_SMP(.long PMD_TYPE_SECT | PMD_SECT_AP_WRITE | PMD_FLAGS_SMP)
+        ALT_UP(.long  PMD_TYPE_SECT | PMD_SECT_AP_WRITE | PMD_FLAGS_UP)
+#else
+#define cpu_v6_do_suspend 0
+#define cpu_v6_do_resume 0
+#endif
 
 
         .type   cpu_v6_name, #object
@@ -130,14 +175,9 @@ cpu_v6_name:
         .asciz  "ARMv6-compatible processor"
         .size   cpu_v6_name, . - cpu_v6_name
 
-        .type   cpu_pj4_name, #object
-cpu_pj4_name:
-        .asciz  "Marvell PJ4 processor"
-        .size   cpu_pj4_name, . - cpu_pj4_name
-
         .align
 
-        __INIT
+        __CPUINIT
 
 /*
  *      __v6_setup
@@ -156,9 +196,11 @@ cpu_pj4_name:
  */
 __v6_setup:
 #ifdef CONFIG_SMP
-        mrc     p15, 0, r0, c1, c0, 1           @ Enable SMP/nAMP mode
+        ALT_SMP(mrc     p15, 0, r0, c1, c0, 1)  @ Enable SMP/nAMP mode
+        ALT_UP(nop)
         orr     r0, r0, #0x20
-        mcr     p15, 0, r0, c1, c0, 1
+        ALT_SMP(mcr     p15, 0, r0, c1, c0, 1)
+        ALT_UP(nop)
 #endif
 
         mov     r0, #0
@@ -169,8 +211,11 @@ __v6_setup:
 #ifdef CONFIG_MMU
         mcr     p15, 0, r0, c8, c7, 0           @ invalidate I + D TLBs
         mcr     p15, 0, r0, c2, c0, 2           @ TTB control register
-        orr     r4, r4, #TTB_FLAGS
-        mcr     p15, 0, r4, c2, c0, 1           @ load TTB1
+        ALT_SMP(orr     r4, r4, #TTB_FLAGS_SMP)
+        ALT_UP(orr      r4, r4, #TTB_FLAGS_UP)
+        ALT_SMP(orr     r8, r8, #TTB_FLAGS_SMP)
+        ALT_UP(orr      r8, r8, #TTB_FLAGS_UP)
+        mcr     p15, 0, r8, c2, c0, 1           @ load TTB1
 #endif /* CONFIG_MMU */
         adr     r5, v6_crval
         ldmia   r5, {r5, r6}
@@ -192,6 +237,8 @@ __v6_setup:
 v6_crval:
         crval   clear=0x01e0fb7f, mmuset=0x00c0387d, ucset=0x00c0187c
 
+        __INITDATA
+
         .type   v6_processor_functions, #object
 ENTRY(v6_processor_functions)
         .word   v6_early_abort
@@ -203,8 +250,13 @@ ENTRY(v6_processor_functions)
         .word   cpu_v6_dcache_clean_area
         .word   cpu_v6_switch_mm
         .word   cpu_v6_set_pte_ext
+        .word   cpu_v6_suspend_size
+        .word   cpu_v6_do_suspend
+        .word   cpu_v6_do_resume
         .size   v6_processor_functions, . - v6_processor_functions
 
+        .section ".rodata"
+
         .type   cpu_arch_name, #object
 cpu_arch_name:
         .asciz  "armv6"
@@ -225,10 +277,16 @@ cpu_elf_name:
 __v6_proc_info:
         .long   0x0007b000
         .long   0x0007f000
-        .long   PMD_TYPE_SECT | \
+        ALT_SMP(.long \
+                PMD_TYPE_SECT | \
+                PMD_SECT_AP_WRITE | \
+                PMD_SECT_AP_READ | \
+                PMD_FLAGS_SMP)
+        ALT_UP(.long \
+                PMD_TYPE_SECT | \
                 PMD_SECT_AP_WRITE | \
                 PMD_SECT_AP_READ | \
-                PMD_FLAGS
+                PMD_FLAGS_UP)
         .long   PMD_TYPE_SECT | \
                 PMD_SECT_XN | \
                 PMD_SECT_AP_WRITE | \
@@ -244,26 +302,3 @@ __v6_proc_info:
         .long   v6_user_fns
         .long   v6_cache_fns
         .size   __v6_proc_info, . - __v6_proc_info
-
-        .type   __pj4_v6_proc_info, #object
-__pj4_v6_proc_info:
-        .long   0x560f5810
-        .long   0xff0ffff0
-        .long   PMD_TYPE_SECT | \
-                PMD_SECT_AP_WRITE | \
-                PMD_SECT_AP_READ | \
-                PMD_FLAGS
-        .long   PMD_TYPE_SECT | \
-                PMD_SECT_XN | \
-                PMD_SECT_AP_WRITE | \
-                PMD_SECT_AP_READ
-        b       __v6_setup
-        .long   cpu_arch_name
-        .long   cpu_elf_name
-        .long   HWCAP_SWP|HWCAP_HALF|HWCAP_THUMB|HWCAP_FAST_MULT|HWCAP_EDSP|HWCAP_TLS
-        .long   cpu_pj4_name
-        .long   v6_processor_functions
-        .long   v6wbi_tlb_fns
-        .long   v6_user_fns
-        .long   v6_cache_fns
-        .size   __pj4_v6_proc_info, . - __pj4_v6_proc_info
diff --git a/arch/arm/mm/proc-v7.S b/arch/arm/mm/proc-v7.S
index 197f21bed5e9..089c0b5e454f 100644
--- a/arch/arm/mm/proc-v7.S
+++ b/arch/arm/mm/proc-v7.S
@@ -30,15 +30,13 @@
 #define TTB_IRGN_WT     ((1 << 0) | (0 << 6))
 #define TTB_IRGN_WB     ((1 << 0) | (1 << 6))
 
-#ifndef CONFIG_SMP
 /* PTWs cacheable, inner WB not shareable, outer WB not shareable */
-#define TTB_FLAGS       TTB_IRGN_WB|TTB_RGN_OC_WB
-#define PMD_FLAGS       PMD_SECT_WB
-#else
+#define TTB_FLAGS_UP    TTB_IRGN_WB|TTB_RGN_OC_WB
+#define PMD_FLAGS_UP    PMD_SECT_WB
+
 /* PTWs cacheable, inner WBWA shareable, outer WBWA not shareable */
-#define TTB_FLAGS       TTB_IRGN_WBWA|TTB_S|TTB_NOS|TTB_RGN_OC_WBWA
-#define PMD_FLAGS       PMD_SECT_WBWA|PMD_SECT_S
-#endif
+#define TTB_FLAGS_SMP   TTB_IRGN_WBWA|TTB_S|TTB_NOS|TTB_RGN_OC_WBWA
+#define PMD_FLAGS_SMP   PMD_SECT_WBWA|PMD_SECT_S
 
 ENTRY(cpu_v7_proc_init)
         mov     pc, lr
@@ -105,14 +103,21 @@ ENTRY(cpu_v7_switch_mm)
 #ifdef CONFIG_MMU
         mov     r2, #0
         ldr     r1, [r1, #MM_CONTEXT_ID]        @ get mm->context.id
-        orr     r0, r0, #TTB_FLAGS
+        ALT_SMP(orr     r0, r0, #TTB_FLAGS_SMP)
+        ALT_UP(orr      r0, r0, #TTB_FLAGS_UP)
 #ifdef CONFIG_ARM_ERRATA_430973
         mcr     p15, 0, r2, c7, c5, 6           @ flush BTAC/BTB
 #endif
+#ifdef CONFIG_ARM_ERRATA_754322
+        dsb
+#endif
         mcr     p15, 0, r2, c13, c0, 1          @ set reserved context ID
         isb
 1:      mcr     p15, 0, r0, c2, c0, 0           @ set TTB 0
         isb
+#ifdef CONFIG_ARM_ERRATA_754322
+        dsb
+#endif
         mcr     p15, 0, r1, c13, c0, 1          @ set context ID
         isb
 #endif
@@ -125,15 +130,13 @@ ENDPROC(cpu_v7_switch_mm)
  *      Set a level 2 translation table entry.
  *
  *      - ptep  - pointer to level 2 translation table entry
- *                (hardware version is stored at -1024 bytes)
+ *                (hardware version is stored at +2048 bytes)
  *      - pte   - PTE value to store
  *      - ext   - value for extended PTE bits
  */
 ENTRY(cpu_v7_set_pte_ext)
 #ifdef CONFIG_MMU
- ARM(   str     r1, [r0], #-2048        )       @ linux version
- THUMB( str     r1, [r0]                )       @ linux version
- THUMB( sub     r0, r0, #2048           )
+        str     r1, [r0]                        @ linux version
 
         bic     r3, r1, #0x000003f0
         bic     r3, r3, #PTE_TYPE_MASK
@@ -143,23 +146,28 @@ ENTRY(cpu_v7_set_pte_ext)
         tst     r1, #1 << 4
         orrne   r3, r3, #PTE_EXT_TEX(1)
 
-        tst     r1, #L_PTE_WRITE
-        tstne   r1, #L_PTE_DIRTY
-        orreq   r3, r3, #PTE_EXT_APX
+        eor     r1, r1, #L_PTE_DIRTY
+        tst     r1, #L_PTE_RDONLY | L_PTE_DIRTY
+        orrne   r3, r3, #PTE_EXT_APX
 
         tst     r1, #L_PTE_USER
         orrne   r3, r3, #PTE_EXT_AP1
+#ifdef CONFIG_CPU_USE_DOMAINS
+        @ allow kernel read/write access to read-only user pages
         tstne   r3, #PTE_EXT_APX
         bicne   r3, r3, #PTE_EXT_APX | PTE_EXT_AP0
+#endif
 
-        tst     r1, #L_PTE_EXEC
-        orreq   r3, r3, #PTE_EXT_XN
+        tst     r1, #L_PTE_XN
+        orrne   r3, r3, #PTE_EXT_XN
 
         tst     r1, #L_PTE_YOUNG
         tstne   r1, #L_PTE_PRESENT
         moveq   r3, #0
 
-        str     r3, [r0]
+ ARM(   str     r3, [r0, #2048]! )
+ THUMB( add     r0, r0, #2048 )
+ THUMB( str     r3, [r0] )
         mcr     p15, 0, r0, c7, c10, 1          @ flush_pte
 #endif
         mov     pc, lr
@@ -169,7 +177,92 @@ cpu_v7_name:
         .ascii  "ARMv7 Processor"
         .align
 
-        __INIT
+        /*
+         * Memory region attributes with SCTLR.TRE=1
+         *
+         *   n = TEX[0],C,B
+         *   TR = PRRR[2n+1:2n]         - memory type
+         *   IR = NMRR[2n+1:2n]         - inner cacheable property
+         *   OR = NMRR[2n+17:2n+16]     - outer cacheable property
+         *
+         *                      n       TR      IR      OR
+         *   UNCACHED           000     00
+         *   BUFFERABLE         001     10      00      00
+         *   WRITETHROUGH       010     10      10      10
+         *   WRITEBACK          011     10      11      11
+         *   reserved           110
+         *   WRITEALLOC         111     10      01      01
+         *   DEV_SHARED         100     01
+         *   DEV_NONSHARED      100     01
+         *   DEV_WC             001     10
+         *   DEV_CACHED         011     10
+         *
+         * Other attributes:
+         *
+         *   DS0 = PRRR[16] = 0         - device shareable property
+         *   DS1 = PRRR[17] = 1         - device shareable property
+         *   NS0 = PRRR[18] = 0         - normal shareable property
+         *   NS1 = PRRR[19] = 1         - normal shareable property
+         *   NOS = PRRR[24+n] = 1       - not outer shareable
+         */
+.equ    PRRR,   0xff0a81a8
+.equ    NMRR,   0x40e040e0
+
+/* Suspend/resume support: derived from arch/arm/mach-s5pv210/sleep.S */
+.globl  cpu_v7_suspend_size
+.equ    cpu_v7_suspend_size, 4 * 9
+#ifdef CONFIG_PM_SLEEP
+ENTRY(cpu_v7_do_suspend)
+        stmfd   sp!, {r4 - r11, lr}
+        mrc     p15, 0, r4, c13, c0, 0  @ FCSE/PID
+        mrc     p15, 0, r5, c13, c0, 1  @ Context ID
+        mrc     p15, 0, r6, c13, c0, 3  @ User r/o thread ID
+        stmia   r0!, {r4 - r6}
+        mrc     p15, 0, r6, c3, c0, 0   @ Domain ID
+        mrc     p15, 0, r7, c2, c0, 0   @ TTB 0
+        mrc     p15, 0, r8, c2, c0, 1   @ TTB 1
+        mrc     p15, 0, r9, c1, c0, 0   @ Control register
+        mrc     p15, 0, r10, c1, c0, 1  @ Auxiliary control register
+        mrc     p15, 0, r11, c1, c0, 2  @ Co-processor access control
+        stmia   r0, {r6 - r11}
+        ldmfd   sp!, {r4 - r11, pc}
+ENDPROC(cpu_v7_do_suspend)
+
+ENTRY(cpu_v7_do_resume)
+        mov     ip, #0
+        mcr     p15, 0, ip, c8, c7, 0   @ invalidate TLBs
+        mcr     p15, 0, ip, c7, c5, 0   @ invalidate I cache
+        ldmia   r0!, {r4 - r6}
+        mcr     p15, 0, r4, c13, c0, 0  @ FCSE/PID
+        mcr     p15, 0, r5, c13, c0, 1  @ Context ID
+        mcr     p15, 0, r6, c13, c0, 3  @ User r/o thread ID
+        ldmia   r0, {r6 - r11}
+        mcr     p15, 0, r6, c3, c0, 0   @ Domain ID
+        mcr     p15, 0, r7, c2, c0, 0   @ TTB 0
+        mcr     p15, 0, r8, c2, c0, 1   @ TTB 1
+        mcr     p15, 0, ip, c2, c0, 2   @ TTB control register
+        mcr     p15, 0, r10, c1, c0, 1  @ Auxiliary control register
+        mcr     p15, 0, r11, c1, c0, 2  @ Co-processor access control
+        ldr     r4, =PRRR               @ PRRR
+        ldr     r5, =NMRR               @ NMRR
+        mcr     p15, 0, r4, c10, c2, 0  @ write PRRR
+        mcr     p15, 0, r5, c10, c2, 1  @ write NMRR
+        isb
+        mov     r0, r9                  @ control register
+        mov     r2, r7, lsr #14         @ get TTB0 base
+        mov     r2, r2, lsl #14
+        ldr     r3, cpu_resume_l1_flags
+        b       cpu_resume_mmu
+ENDPROC(cpu_v7_do_resume)
+cpu_resume_l1_flags:
+        ALT_SMP(.long PMD_TYPE_SECT | PMD_SECT_AP_WRITE | PMD_FLAGS_SMP)
+        ALT_UP(.long  PMD_TYPE_SECT | PMD_SECT_AP_WRITE | PMD_FLAGS_UP)
+#else
+#define cpu_v7_do_suspend       0
+#define cpu_v7_do_resume        0
+#endif
+
+        __CPUINIT
 
 /*
  *      __v7_setup
@@ -188,7 +281,8 @@ cpu_v7_name:
  */
 __v7_ca9mp_setup:
 #ifdef CONFIG_SMP
-        mrc     p15, 0, r0, c1, c0, 1
+        ALT_SMP(mrc     p15, 0, r0, c1, c0, 1)
+        ALT_UP(mov      r0, #(1 << 6))          @ fake it for UP
         tst     r0, #(1 << 6)                   @ SMP/nAMP mode enabled?
         orreq   r0, r0, #(1 << 6) | (1 << 0)    @ Enable SMP/nAMP mode and
         mcreq   p15, 0, r0, c1, c0, 1           @ TLB ops broadcasting
@@ -261,6 +355,12 @@ __v7_setup:
         orreq   r10, r10, #1 << 6               @ set bit #6
         mcreq   p15, 0, r10, c15, c0, 1         @ write diagnostic register
 #endif
+#ifdef CONFIG_ARM_ERRATA_751472
+        cmp     r6, #0x30                       @ present prior to r3p0
+        mrclt   p15, 0, r10, c15, c0, 1         @ read diagnostic register
+        orrlt   r10, r10, #1 << 11              @ set bit #11
+        mcrlt   p15, 0, r10, c15, c0, 1         @ write diagnostic register
+#endif
 
 3:      mov     r10, #0
 #ifdef HARVARD_CACHE
@@ -270,40 +370,13 @@ __v7_setup:
 #ifdef CONFIG_MMU
         mcr     p15, 0, r10, c8, c7, 0          @ invalidate I + D TLBs
         mcr     p15, 0, r10, c2, c0, 2          @ TTB control register
-        orr     r4, r4, #TTB_FLAGS
-        mcr     p15, 0, r4, c2, c0, 1           @ load TTB1
-        mov     r10, #0x1f                      @ domains 0, 1 = manager
-        mcr     p15, 0, r10, c3, c0, 0          @ load domain access register
-        /*
-         * Memory region attributes with SCTLR.TRE=1
-         *
-         *   n = TEX[0],C,B
-         *   TR = PRRR[2n+1:2n]         - memory type
-         *   IR = NMRR[2n+1:2n]         - inner cacheable property
-         *   OR = NMRR[2n+17:2n+16]     - outer cacheable property
-         *
-         *                      n       TR      IR      OR
-         *   UNCACHED           000     00
-         *   BUFFERABLE         001     10      00      00
-         *   WRITETHROUGH       010     10      10      10
-         *   WRITEBACK          011     10      11      11
-         *   reserved           110
-         *   WRITEALLOC         111     10      01      01
-         *   DEV_SHARED         100     01
-         *   DEV_NONSHARED      100     01
-         *   DEV_WC             001     10
-         *   DEV_CACHED         011     10
-         *
-         * Other attributes:
-         *
-         *   DS0 = PRRR[16] = 0         - device shareable property
-         *   DS1 = PRRR[17] = 1         - device shareable property
-         *   NS0 = PRRR[18] = 0         - normal shareable property
-         *   NS1 = PRRR[19] = 1         - normal shareable property
-         *   NOS = PRRR[24+n] = 1       - not outer shareable
-         */
-        ldr     r5, =0xff0a81a8                 @ PRRR
-        ldr     r6, =0x40e040e0                 @ NMRR
+        ALT_SMP(orr     r4, r4, #TTB_FLAGS_SMP)
+        ALT_UP(orr      r4, r4, #TTB_FLAGS_UP)
+        ALT_SMP(orr     r8, r8, #TTB_FLAGS_SMP)
+        ALT_UP(orr      r8, r8, #TTB_FLAGS_UP)
+        mcr     p15, 0, r8, c2, c0, 1           @ load TTB1
+        ldr     r5, =PRRR                       @ PRRR
+        ldr     r6, =NMRR                       @ NMRR
         mcr     p15, 0, r5, c10, c2, 0          @ write PRRR
         mcr     p15, 0, r6, c10, c2, 1          @ write NMRR
 #endif
@@ -312,6 +385,10 @@ __v7_setup:
 #ifdef CONFIG_CPU_ENDIAN_BE8
         orr     r6, r6, #1 << 25                @ big-endian page tables
 #endif
+#ifdef CONFIG_SWP_EMULATE
+        orr     r5, r5, #(1 << 10)              @ set SW bit in "clear"
+        bic     r6, r6, #(1 << 10)              @ clear it in "mmuset"
+#endif
         mrc     p15, 0, r0, c1, c0, 0           @ read control register
         bic     r0, r0, r5                      @ clear bits them
         orr     r0, r0, r6                      @ set them
@@ -332,6 +409,8 @@ v7_crval:
 __v7_setup_stack:
         .space  4 * 11                          @ 11 registers
 
+        __INITDATA
+
         .type   v7_processor_functions, #object
 ENTRY(v7_processor_functions)
         .word   v7_early_abort
@@ -343,8 +422,13 @@ ENTRY(v7_processor_functions)
         .word   cpu_v7_dcache_clean_area
         .word   cpu_v7_switch_mm
         .word   cpu_v7_set_pte_ext
+        .word   cpu_v7_suspend_size
+        .word   cpu_v7_do_suspend
+        .word   cpu_v7_do_resume
         .size   v7_processor_functions, . - v7_processor_functions
 
+        .section ".rodata"
+
         .type   cpu_arch_name, #object
 cpu_arch_name:
         .asciz  "armv7"
@@ -362,15 +446,21 @@ cpu_elf_name:
 __v7_ca9mp_proc_info:
         .long   0x410fc090              @ Required ID value
         .long   0xff0ffff0              @ Mask for ID
-        .long   PMD_TYPE_SECT | \
+        ALT_SMP(.long \
+                PMD_TYPE_SECT | \
                 PMD_SECT_AP_WRITE | \
                 PMD_SECT_AP_READ | \
-                PMD_FLAGS
+                PMD_FLAGS_SMP)
+        ALT_UP(.long \
+                PMD_TYPE_SECT | \
+                PMD_SECT_AP_WRITE | \
+                PMD_SECT_AP_READ | \
+                PMD_FLAGS_UP)
         .long   PMD_TYPE_SECT | \
                 PMD_SECT_XN | \
                 PMD_SECT_AP_WRITE | \
                 PMD_SECT_AP_READ
-        b       __v7_ca9mp_setup
+        W(b)    __v7_ca9mp_setup
         .long   cpu_arch_name
         .long   cpu_elf_name
         .long   HWCAP_SWP|HWCAP_HALF|HWCAP_THUMB|HWCAP_FAST_MULT|HWCAP_EDSP|HWCAP_TLS
@@ -388,15 +478,21 @@ __v7_ca9mp_proc_info:
 __v7_proc_info:
         .long   0x000f0000              @ Required ID value
         .long   0x000f0000              @ Mask for ID
-        .long   PMD_TYPE_SECT | \
+        ALT_SMP(.long \
+                PMD_TYPE_SECT | \
+                PMD_SECT_AP_WRITE | \
+                PMD_SECT_AP_READ | \
+                PMD_FLAGS_SMP)
+        ALT_UP(.long \
+                PMD_TYPE_SECT | \
                 PMD_SECT_AP_WRITE | \
                 PMD_SECT_AP_READ | \
-                PMD_FLAGS
+                PMD_FLAGS_UP)
         .long   PMD_TYPE_SECT | \
                 PMD_SECT_XN | \
                 PMD_SECT_AP_WRITE | \
                 PMD_SECT_AP_READ
-        b       __v7_setup
+        W(b)    __v7_setup
         .long   cpu_arch_name
         .long   cpu_elf_name
         .long   HWCAP_SWP|HWCAP_HALF|HWCAP_THUMB|HWCAP_FAST_MULT|HWCAP_EDSP|HWCAP_TLS
diff --git a/arch/arm/mm/proc-xsc3.S b/arch/arm/mm/proc-xsc3.S
index 361a51e49030..596213699f37 100644
--- a/arch/arm/mm/proc-xsc3.S
+++ b/arch/arm/mm/proc-xsc3.S
@@ -141,6 +141,17 @@ ENTRY(cpu_xsc3_do_idle)
 /* ================================= CACHE ================================ */
 
 /*
+ *      flush_icache_all()
+ *
+ *      Unconditionally clean and invalidate the entire icache.
+ */
+ENTRY(xsc3_flush_icache_all)
+        mov     r0, #0
+        mcr     p15, 0, r0, c7, c5, 0           @ invalidate I cache
+        mov     pc, lr
+ENDPROC(xsc3_flush_icache_all)
+
+/*
  *      flush_user_cache_all()
  *
  *      Invalidate all cache entries in a particular address
@@ -325,6 +336,7 @@ ENTRY(xsc3_dma_unmap_area)
 ENDPROC(xsc3_dma_unmap_area)
 
 ENTRY(xsc3_cache_fns)
+        .long   xsc3_flush_icache_all
         .long   xsc3_flush_kern_cache_all
         .long   xsc3_flush_user_cache_all
         .long   xsc3_flush_user_cache_range
@@ -401,10 +413,53 @@ ENTRY(cpu_xsc3_set_pte_ext)
         mov     pc, lr
 
         .ltorg
-
         .align
 
-        __INIT
+.globl  cpu_xsc3_suspend_size
+.equ    cpu_xsc3_suspend_size, 4 * 8
+#ifdef CONFIG_PM_SLEEP
+ENTRY(cpu_xsc3_do_suspend)
+        stmfd   sp!, {r4 - r10, lr}
+        mrc     p14, 0, r4, c6, c0, 0   @ clock configuration, for turbo mode
+        mrc     p15, 0, r5, c15, c1, 0  @ CP access reg
+        mrc     p15, 0, r6, c13, c0, 0  @ PID
+        mrc     p15, 0, r7, c3, c0, 0   @ domain ID
+        mrc     p15, 0, r8, c2, c0, 0   @ translation table base addr
+        mrc     p15, 0, r9, c1, c0, 1   @ auxiliary control reg
+        mrc     p15, 0, r10, c1, c0, 0  @ control reg
+        bic     r4, r4, #2              @ clear frequency change bit
+        stmia   r0, {r1, r4 - r10}      @ store v:p offset + cp regs
+        ldmia   sp!, {r4 - r10, pc}
+ENDPROC(cpu_xsc3_do_suspend)
+
+ENTRY(cpu_xsc3_do_resume)
+        ldmia   r0, {r1, r4 - r10}      @ load v:p offset + cp regs
+        mov     ip, #0
+        mcr     p15, 0, ip, c7, c7, 0   @ invalidate I & D caches, BTB
+        mcr     p15, 0, ip, c7, c10, 4  @ drain write (&fill) buffer
+        mcr     p15, 0, ip, c7, c5, 4   @ flush prefetch buffer
+        mcr     p15, 0, ip, c8, c7, 0   @ invalidate I & D TLBs
+        mcr     p14, 0, r4, c6, c0, 0   @ clock configuration, turbo mode.
+        mcr     p15, 0, r5, c15, c1, 0  @ CP access reg
+        mcr     p15, 0, r6, c13, c0, 0  @ PID
+        mcr     p15, 0, r7, c3, c0, 0   @ domain ID
+        mcr     p15, 0, r8, c2, c0, 0   @ translation table base addr
+        mcr     p15, 0, r9, c1, c0, 1   @ auxiliary control reg
+
+        @ temporarily map resume_turn_on_mmu into the page table,
+        @ otherwise prefetch abort occurs after MMU is turned on
+        mov     r0, r10                 @ control register
+        mov     r2, r8, lsr #14         @ get TTB0 base
+        mov     r2, r2, lsl #14
+        ldr     r3, =0x542e             @ section flags
+        b       cpu_resume_mmu
+ENDPROC(cpu_xsc3_do_resume)
+#else
+#define cpu_xsc3_do_suspend     0
+#define cpu_xsc3_do_resume      0
+#endif
+
+        __CPUINIT
 
         .type   __xsc3_setup, #function
 __xsc3_setup:
@@ -464,6 +519,9 @@ ENTRY(xsc3_processor_functions)
         .word   cpu_xsc3_dcache_clean_area
         .word   cpu_xsc3_switch_mm
         .word   cpu_xsc3_set_pte_ext
+        .word   cpu_xsc3_suspend_size
+        .word   cpu_xsc3_do_suspend
+        .word   cpu_xsc3_do_resume
         .size   xsc3_processor_functions, . - xsc3_processor_functions
 
         .section ".rodata"
diff --git a/arch/arm/mm/proc-xscale.S b/arch/arm/mm/proc-xscale.S
index 14075979bcba..42af97664c9d 100644
--- a/arch/arm/mm/proc-xscale.S
+++ b/arch/arm/mm/proc-xscale.S
@@ -181,6 +181,17 @@ ENTRY(cpu_xscale_do_idle)
 /* ================================= CACHE ================================ */
 
 /*
+ *      flush_icache_all()
+ *
+ *      Unconditionally clean and invalidate the entire icache.
+ */
+ENTRY(xscale_flush_icache_all)
+        mov     r0, #0
+        mcr     p15, 0, r0, c7, c5, 0           @ invalidate I cache
+        mov     pc, lr
+ENDPROC(xscale_flush_icache_all)
+
+/*
  *      flush_user_cache_all()
  *
  *      Invalidate all cache entries in a particular address
@@ -384,7 +395,7 @@ ENTRY(xscale_dma_a0_map_area)
         teq     r2, #DMA_TO_DEVICE
         beq     xscale_dma_clean_range
         b       xscale_dma_flush_range
-ENDPROC(xscsale_dma_a0_map_area)
+ENDPROC(xscale_dma_a0_map_area)
 
 /*
  *      dma_unmap_area(start, size, dir)
@@ -397,6 +408,7 @@ ENTRY(xscale_dma_unmap_area)
 ENDPROC(xscale_dma_unmap_area)
 
 ENTRY(xscale_cache_fns)
+        .long   xscale_flush_icache_all
         .long   xscale_flush_kern_cache_all
         .long   xscale_flush_user_cache_all
         .long   xscale_flush_user_cache_range
@@ -488,8 +500,8 @@ ENTRY(cpu_xscale_set_pte_ext)
         @
         @ Erratum 40: must set memory to write-through for user read-only pages
         @
-        and     ip, r1, #(L_PTE_MT_MASK | L_PTE_USER | L_PTE_WRITE) & ~(4 << 2)
-        teq     ip, #L_PTE_MT_WRITEBACK | L_PTE_USER
+        and     ip, r1, #(L_PTE_MT_MASK | L_PTE_USER | L_PTE_RDONLY) & ~(4 << 2)
+        teq     ip, #L_PTE_MT_WRITEBACK | L_PTE_USER | L_PTE_RDONLY
 
         moveq   r1, #L_PTE_MT_WRITETHROUGH
         and     r1, r1, #L_PTE_MT_MASK
@@ -501,12 +513,50 @@ ENTRY(cpu_xscale_set_pte_ext)
         xscale_set_pte_ext_epilogue
         mov     pc, lr
 
-
         .ltorg
-
         .align
 
-        __INIT
+.globl  cpu_xscale_suspend_size
+.equ    cpu_xscale_suspend_size, 4 * 7
+#ifdef CONFIG_PM_SLEEP
+ENTRY(cpu_xscale_do_suspend)
+        stmfd   sp!, {r4 - r10, lr}
+        mrc     p14, 0, r4, c6, c0, 0   @ clock configuration, for turbo mode
+        mrc     p15, 0, r5, c15, c1, 0  @ CP access reg
+        mrc     p15, 0, r6, c13, c0, 0  @ PID
+        mrc     p15, 0, r7, c3, c0, 0   @ domain ID
+        mrc     p15, 0, r8, c2, c0, 0   @ translation table base addr
+        mrc     p15, 0, r9, c1, c1, 0   @ auxiliary control reg
+        mrc     p15, 0, r10, c1, c0, 0  @ control reg
+        bic     r4, r4, #2              @ clear frequency change bit
+        stmia   r0, {r4 - r10}          @ store cp regs
+        ldmfd   sp!, {r4 - r10, pc}
+ENDPROC(cpu_xscale_do_suspend)
+
+ENTRY(cpu_xscale_do_resume)
+        ldmia   r0, {r4 - r10}          @ load cp regs
+        mov     ip, #0
+        mcr     p15, 0, ip, c8, c7, 0   @ invalidate I & D TLBs
+        mcr     p15, 0, ip, c7, c7, 0   @ invalidate I & D caches, BTB
+        mcr     p14, 0, r4, c6, c0, 0   @ clock configuration, turbo mode.
+        mcr     p15, 0, r5, c15, c1, 0  @ CP access reg
+        mcr     p15, 0, r6, c13, c0, 0  @ PID
+        mcr     p15, 0, r7, c3, c0, 0   @ domain ID
+        mcr     p15, 0, r8, c2, c0, 0   @ translation table base addr
+        mcr     p15, 0, r9, c1, c1, 0   @ auxiliary control reg
+        mov     r0, r10                 @ control register
+        mov     r2, r8, lsr #14         @ get TTB0 base
+        mov     r2, r2, lsl #14
+        ldr     r3, =PMD_TYPE_SECT | PMD_SECT_BUFFERABLE | \
+                     PMD_SECT_CACHEABLE | PMD_SECT_AP_WRITE
+        b       cpu_resume_mmu
+ENDPROC(cpu_xscale_do_resume)
+#else
+#define cpu_xscale_do_suspend   0
+#define cpu_xscale_do_resume    0
+#endif
+
+        __CPUINIT
 
         .type   __xscale_setup, #function
 __xscale_setup:
@@ -553,6 +603,9 @@ ENTRY(xscale_processor_functions)
         .word   cpu_xscale_dcache_clean_area
         .word   cpu_xscale_switch_mm
         .word   cpu_xscale_set_pte_ext
+        .word   cpu_xscale_suspend_size
+        .word   cpu_xscale_do_suspend
+        .word   cpu_xscale_do_resume
         .size   xscale_processor_functions, . - xscale_processor_functions
 
         .section ".rodata"
diff --git a/arch/arm/mm/tlb-v7.S b/arch/arm/mm/tlb-v7.S
index f3f288a9546d..53cd5b454673 100644
--- a/arch/arm/mm/tlb-v7.S
+++ b/arch/arm/mm/tlb-v7.S
@@ -13,6 +13,7 @@
  */
 #include <linux/init.h>
 #include <linux/linkage.h>
+#include <asm/assembler.h>
 #include <asm/asm-offsets.h>
 #include <asm/page.h>
 #include <asm/tlbflush.h>
@@ -41,20 +42,15 @@ ENTRY(v7wbi_flush_user_tlb_range)
         orr     r0, r3, r0, lsl #PAGE_SHIFT     @ Create initial MVA
         mov     r1, r1, lsl #PAGE_SHIFT
 1:
-#ifdef CONFIG_SMP
-        mcr     p15, 0, r0, c8, c3, 1           @ TLB invalidate U MVA (shareable) 
-#else
-        mcr     p15, 0, r0, c8, c7, 1           @ TLB invalidate U MVA
-#endif
+        ALT_SMP(mcr     p15, 0, r0, c8, c3, 1)  @ TLB invalidate U MVA (shareable)
+        ALT_UP(mcr      p15, 0, r0, c8, c7, 1)  @ TLB invalidate U MVA
+
         add     r0, r0, #PAGE_SZ
         cmp     r0, r1
         blo     1b
         mov     ip, #0
-#ifdef CONFIG_SMP
-        mcr     p15, 0, ip, c7, c1, 6           @ flush BTAC/BTB Inner Shareable
-#else
-        mcr     p15, 0, ip, c7, c5, 6           @ flush BTAC/BTB
-#endif
+        ALT_SMP(mcr     p15, 0, ip, c7, c1, 6)  @ flush BTAC/BTB Inner Shareable
+        ALT_UP(mcr      p15, 0, ip, c7, c5, 6)  @ flush BTAC/BTB
         dsb
         mov     pc, lr
 ENDPROC(v7wbi_flush_user_tlb_range)
@@ -74,20 +70,14 @@ ENTRY(v7wbi_flush_kern_tlb_range)
         mov     r0, r0, lsl #PAGE_SHIFT
         mov     r1, r1, lsl #PAGE_SHIFT
 1:
-#ifdef CONFIG_SMP
-        mcr     p15, 0, r0, c8, c3, 1           @ TLB invalidate U MVA (shareable)
-#else
-        mcr     p15, 0, r0, c8, c7, 1           @ TLB invalidate U MVA
-#endif
+        ALT_SMP(mcr     p15, 0, r0, c8, c3, 1)  @ TLB invalidate U MVA (shareable)
+        ALT_UP(mcr      p15, 0, r0, c8, c7, 1)  @ TLB invalidate U MVA
         add     r0, r0, #PAGE_SZ
         cmp     r0, r1
         blo     1b
         mov     r2, #0
-#ifdef CONFIG_SMP
-        mcr     p15, 0, r2, c7, c1, 6           @ flush BTAC/BTB Inner Shareable
-#else
-        mcr     p15, 0, r2, c7, c5, 6           @ flush BTAC/BTB
-#endif
+        ALT_SMP(mcr     p15, 0, r2, c7, c1, 6)  @ flush BTAC/BTB Inner Shareable
+        ALT_UP(mcr      p15, 0, r2, c7, c5, 6)  @ flush BTAC/BTB
         dsb
         isb
         mov     pc, lr
@@ -99,5 +89,6 @@ ENDPROC(v7wbi_flush_kern_tlb_range)
 ENTRY(v7wbi_tlb_fns)
         .long   v7wbi_flush_user_tlb_range
         .long   v7wbi_flush_kern_tlb_range
-        .long   v7wbi_tlb_flags
+        ALT_SMP(.long   v7wbi_tlb_flags_smp)
+        ALT_UP(.long    v7wbi_tlb_flags_up)
         .size   v7wbi_tlb_fns, . - v7wbi_tlb_fns
diff --git a/arch/arm/mm/vmregion.c b/arch/arm/mm/vmregion.c
index 935993e1b1ef..036fdbfdd62f 100644
--- a/arch/arm/mm/vmregion.c
+++ b/arch/arm/mm/vmregion.c
@@ -38,7 +38,7 @@ struct arm_vmregion *
 arm_vmregion_alloc(struct arm_vmregion_head *head, size_t align,
                    size_t size, gfp_t gfp)
 {
-        unsigned long addr = head->vm_start, end = head->vm_end - size;
+        unsigned long start = head->vm_start, addr = head->vm_end;
         unsigned long flags;
         struct arm_vmregion *c, *new;
 
@@ -54,21 +54,20 @@ arm_vmregion_alloc(struct arm_vmregion_head *head, size_t align,
 
         spin_lock_irqsave(&head->vm_lock, flags);
 
-        list_for_each_entry(c, &head->vm_list, vm_list) {
-                if ((addr + size) < addr)
-                        goto nospc;
-                if ((addr + size) <= c->vm_start)
+        addr = rounddown(addr - size, align);
+        list_for_each_entry_reverse(c, &head->vm_list, vm_list) {
+                if (addr >= c->vm_end)
                         goto found;
-                addr = ALIGN(c->vm_end, align);
-                if (addr > end)
+                addr = rounddown(c->vm_start - size, align);
+                if (addr < start)
                         goto nospc;
         }
 
  found:
         /*
-         * Insert this entry _before_ the one we found.
+         * Insert this entry after the one we found.
          */
-        list_add_tail(&new->vm_list, &c->vm_list);
+        list_add(&new->vm_list, &c->vm_list);
         new->vm_start = addr;
         new->vm_end = addr + size;
         new->vm_active = 1;