Merge branch 'wip-2.6.34' into old-private-masterarchived-private-master

56 files changed, 2261 insertions, 882 deletions

diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig
index 9264d814cd7a..5bd7c89a6045 100644
--- a/arch/arm/mm/Kconfig
+++ b/arch/arm/mm/Kconfig
@@ -1,9 +1,5 @@
 comment "Processor Type"
 
-config CPU_32
-        bool
-        default y
-
 # Select CPU types depending on the architecture selected.  This selects
 # which CPUs we support in the kernel image, and the compiler instruction
 # optimiser behaviour.
@@ -388,7 +384,7 @@ config CPU_FEROCEON_OLD_ID
 
 # ARMv6
 config CPU_V6
-        bool "Support ARM V6 processor" if ARCH_INTEGRATOR || MACH_REALVIEW_EB || MACH_REALVIEW_PBX
+        bool "Support ARM V6 processor" if ARCH_INTEGRATOR || MACH_REALVIEW_EB || MACH_REALVIEW_PBX || ARCH_DOVE
         select CPU_32v6
         select CPU_ABRT_EV6
         select CPU_PABRT_V6
@@ -403,7 +399,7 @@ config CPU_V6
 config CPU_32v6K
         bool "Support ARM V6K processor extensions" if !SMP
         depends on CPU_V6
-        default y if SMP && !ARCH_MX3
+        default y if SMP && !(ARCH_MX3 || ARCH_OMAP2)
         help
           Say Y here if your ARMv6 processor supports the 'K' extension.
           This enables the kernel to use some instructions not present
@@ -414,7 +410,7 @@ config CPU_32v6K
 # ARMv7
 config CPU_V7
         bool "Support ARM V7 processor" if ARCH_INTEGRATOR || MACH_REALVIEW_EB || MACH_REALVIEW_PBX
-        select CPU_32v6K
+        select CPU_32v6K if !ARCH_OMAP2
         select CPU_32v7
         select CPU_ABRT_EV7
         select CPU_PABRT_V7
@@ -740,6 +736,12 @@ config NEEDS_SYSCALL_FOR_CMPXCHG
 config OUTER_CACHE
         bool
 
+config OUTER_CACHE_SYNC
+        bool
+        help
+          The outer cache has a outer_cache_fns.sync function pointer
+          that can be used to drain the write buffer of the outer cache.
+
 config CACHE_FEROCEON_L2
         bool "Enable the Feroceon L2 cache controller"
         depends on ARCH_KIRKWOOD || ARCH_MV78XX0
@@ -758,12 +760,22 @@ config CACHE_FEROCEON_L2_WRITETHROUGH
 config CACHE_L2X0
         bool "Enable the L2x0 outer cache controller"
         depends on REALVIEW_EB_ARM11MP || MACH_REALVIEW_PB11MP || MACH_REALVIEW_PB1176 || \
-                   REALVIEW_EB_A9MP || ARCH_MX35 || ARCH_MX31 || MACH_REALVIEW_PBX || ARCH_NOMADIK
+                   REALVIEW_EB_A9MP || ARCH_MX35 || ARCH_MX31 || MACH_REALVIEW_PBX || ARCH_NOMADIK || ARCH_OMAP4
         default y
         select OUTER_CACHE
+        select OUTER_CACHE_SYNC
         help
           This option enables the L2x0 PrimeCell.
 
+config CACHE_TAUROS2
+        bool "Enable the Tauros2 L2 cache controller"
+        depends on ARCH_DOVE
+        default y
+        select OUTER_CACHE
+        help
+          This option enables the Tauros2 L2 cache controller (as
+          found on PJ1/PJ4).
+
 config CACHE_XSC3L2
         bool "Enable the L2 cache on XScale3"
         depends on CPU_XSC3
@@ -774,5 +786,11 @@ config CACHE_XSC3L2
 
 config ARM_L1_CACHE_SHIFT
         int
-        default 6 if ARCH_OMAP3
+        default 6 if ARM_L1_CACHE_SHIFT_6
         default 5
+
+config ARCH_HAS_BARRIERS
+        bool
+        help
+          This option allows the use of custom mandatory barriers
+          included via the mach/barriers.h file.
diff --git a/arch/arm/mm/Makefile b/arch/arm/mm/Makefile
index 055cb2aa8134..e8d34a80851c 100644
--- a/arch/arm/mm/Makefile
+++ b/arch/arm/mm/Makefile
@@ -6,7 +6,7 @@ obj-y				:= dma-mapping.o extable.o fault.o init.o \
                                    iomap.o
 
 obj-$(CONFIG_MMU)               += fault-armv.o flush.o ioremap.o mmap.o \
-                                   pgd.o mmu.o
+                                   pgd.o mmu.o vmregion.o
 
 ifneq ($(CONFIG_MMU),y)
 obj-y                           += nommu.o
@@ -27,6 +27,9 @@ obj-$(CONFIG_CPU_ABRT_EV5TJ)	+= abort-ev5tj.o
 obj-$(CONFIG_CPU_ABRT_EV6)      += abort-ev6.o
 obj-$(CONFIG_CPU_ABRT_EV7)      += abort-ev7.o
 
+AFLAGS_abort-ev6.o      :=-Wa,-march=armv6k
+AFLAGS_abort-ev7.o      :=-Wa,-march=armv7-a
+
 obj-$(CONFIG_CPU_PABRT_LEGACY)  += pabort-legacy.o
 obj-$(CONFIG_CPU_PABRT_V6)      += pabort-v6.o
 obj-$(CONFIG_CPU_PABRT_V7)      += pabort-v7.o
@@ -39,6 +42,9 @@ obj-$(CONFIG_CPU_CACHE_V6)	+= cache-v6.o
 obj-$(CONFIG_CPU_CACHE_V7)      += cache-v7.o
 obj-$(CONFIG_CPU_CACHE_FA)      += cache-fa.o
 
+AFLAGS_cache-v6.o       :=-Wa,-march=armv6
+AFLAGS_cache-v7.o       :=-Wa,-march=armv7-a
+
 obj-$(CONFIG_CPU_COPY_V3)       += copypage-v3.o
 obj-$(CONFIG_CPU_COPY_V4WT)     += copypage-v4wt.o
 obj-$(CONFIG_CPU_COPY_V4WB)     += copypage-v4wb.o
@@ -58,6 +64,9 @@ obj-$(CONFIG_CPU_TLB_V6)	+= tlb-v6.o
 obj-$(CONFIG_CPU_TLB_V7)        += tlb-v7.o
 obj-$(CONFIG_CPU_TLB_FA)        += tlb-fa.o
 
+AFLAGS_tlb-v6.o         :=-Wa,-march=armv6
+AFLAGS_tlb-v7.o         :=-Wa,-march=armv7-a
+
 obj-$(CONFIG_CPU_ARM610)        += proc-arm6_7.o
 obj-$(CONFIG_CPU_ARM710)        += proc-arm6_7.o
 obj-$(CONFIG_CPU_ARM7TDMI)      += proc-arm7tdmi.o
@@ -84,7 +93,10 @@ obj-$(CONFIG_CPU_FEROCEON)	+= proc-feroceon.o
 obj-$(CONFIG_CPU_V6)            += proc-v6.o
 obj-$(CONFIG_CPU_V7)            += proc-v7.o
 
+AFLAGS_proc-v6.o        :=-Wa,-march=armv6
+AFLAGS_proc-v7.o        :=-Wa,-march=armv7-a
+
 obj-$(CONFIG_CACHE_FEROCEON_L2) += cache-feroceon-l2.o
 obj-$(CONFIG_CACHE_L2X0)        += cache-l2x0.o
 obj-$(CONFIG_CACHE_XSC3L2)      += cache-xsc3l2.o
-
+obj-$(CONFIG_CACHE_TAUROS2)     += cache-tauros2.o
diff --git a/arch/arm/mm/alignment.c b/arch/arm/mm/alignment.c
index b270d6228fe2..a2ab51fa73e2 100644
--- a/arch/arm/mm/alignment.c
+++ b/arch/arm/mm/alignment.c
@@ -11,6 +11,7 @@
  * it under the terms of the GNU General Public License version 2 as
  * published by the Free Software Foundation.
  */
+#include <linux/moduleparam.h>
 #include <linux/compiler.h>
 #include <linux/kernel.h>
 #include <linux/errno.h>
@@ -77,6 +78,8 @@ static unsigned long ai_dword;
 static unsigned long ai_multi;
 static int ai_usermode;
 
+core_param(alignment, ai_usermode, int, 0600);
+
 #define UM_WARN         (1 << 0)
 #define UM_FIXUP        (1 << 1)
 #define UM_SIGNAL       (1 << 2)
@@ -163,15 +166,15 @@ union offset_union {
  THUMB( "1:     "ins"   %1, [%2]\n"     )               \
  THUMB( "       add     %2, %2, #1\n"   )               \
         "2:\n"                                          \
-        "       .section .fixup,\"ax\"\n"               \
+        "       .pushsection .fixup,\"ax\"\n"           \
         "       .align  2\n"                            \
         "3:     mov     %0, #1\n"                       \
         "       b       2b\n"                           \
-        "       .previous\n"                            \
-        "       .section __ex_table,\"a\"\n"            \
+        "       .popsection\n"                          \
+        "       .pushsection __ex_table,\"a\"\n"        \
         "       .align  3\n"                            \
         "       .long   1b, 3b\n"                       \
-        "       .previous\n"                            \
+        "       .popsection\n"                          \
         : "=r" (err), "=&r" (val), "=r" (addr)          \
         : "0" (err), "2" (addr))
 
@@ -223,16 +226,16 @@ union offset_union {
                 "       mov     %1, %1, "NEXT_BYTE"\n"          \
                 "2:     "ins"   %1, [%2]\n"                     \
                 "3:\n"                                          \
-                "       .section .fixup,\"ax\"\n"               \
+                "       .pushsection .fixup,\"ax\"\n"           \
                 "       .align  2\n"                            \
                 "4:     mov     %0, #1\n"                       \
                 "       b       3b\n"                           \
-                "       .previous\n"                            \
-                "       .section __ex_table,\"a\"\n"            \
+                "       .popsection\n"                          \
+                "       .pushsection __ex_table,\"a\"\n"        \
                 "       .align  3\n"                            \
                 "       .long   1b, 4b\n"                       \
                 "       .long   2b, 4b\n"                       \
-                "       .previous\n"                            \
+                "       .popsection\n"                          \
                 : "=r" (err), "=&r" (v), "=&r" (a)              \
                 : "0" (err), "1" (v), "2" (a));                 \
                 if (err)                                        \
@@ -263,18 +266,18 @@ union offset_union {
                 "       mov     %1, %1, "NEXT_BYTE"\n"          \
                 "4:     "ins"   %1, [%2]\n"                     \
                 "5:\n"                                          \
-                "       .section .fixup,\"ax\"\n"               \
+                "       .pushsection .fixup,\"ax\"\n"           \
                 "       .align  2\n"                            \
                 "6:     mov     %0, #1\n"                       \
                 "       b       5b\n"                           \
-                "       .previous\n"                            \
-                "       .section __ex_table,\"a\"\n"            \
+                "       .popsection\n"                          \
+                "       .pushsection __ex_table,\"a\"\n"        \
                 "       .align  3\n"                            \
                 "       .long   1b, 6b\n"                       \
                 "       .long   2b, 6b\n"                       \
                 "       .long   3b, 6b\n"                       \
                 "       .long   4b, 6b\n"                       \
-                "       .previous\n"                            \
+                "       .popsection\n"                          \
                 : "=r" (err), "=&r" (v), "=&r" (a)              \
                 : "0" (err), "1" (v), "2" (a));                 \
                 if (err)                                        \
@@ -898,11 +901,7 @@ static int __init alignment_init(void)
 #ifdef CONFIG_PROC_FS
         struct proc_dir_entry *res;
 
-        res = proc_mkdir("cpu", NULL);
-        if (!res)
-                return -ENOMEM;
-
-        res = create_proc_entry("alignment", S_IWUSR | S_IRUGO, res);
+        res = create_proc_entry("cpu/alignment", S_IWUSR | S_IRUGO, NULL);
         if (!res)
                 return -ENOMEM;
 
diff --git a/arch/arm/mm/cache-fa.S b/arch/arm/mm/cache-fa.S
index b63a8f7b95cf..7148e53e6078 100644
--- a/arch/arm/mm/cache-fa.S
+++ b/arch/arm/mm/cache-fa.S
@@ -127,15 +127,16 @@ ENTRY(fa_coherent_user_range)
         mov     pc, lr
 
 /*
- *      flush_kern_dcache_page(kaddr)
+ *      flush_kern_dcache_area(void *addr, size_t size)
  *
  *      Ensure that the data held in the page kaddr is written back
  *      to the page in question.
  *
- *      - kaddr   - kernel address (guaranteed to be page aligned)
+ *      - addr  - kernel address
+ *      - size  - size of region
  */
-ENTRY(fa_flush_kern_dcache_page)
-        add     r1, r0, #PAGE_SZ
+ENTRY(fa_flush_kern_dcache_area)
+        add     r1, r0, r1
 1:      mcr     p15, 0, r0, c7, c14, 1          @ clean & invalidate D line
         add     r0, r0, #CACHE_DLINESIZE
         cmp     r0, r1
@@ -156,7 +157,7 @@ ENTRY(fa_flush_kern_dcache_page)
  *      - start  - virtual start address
  *      - end    - virtual end address
  */
-ENTRY(fa_dma_inv_range)
+fa_dma_inv_range:
         tst     r0, #CACHE_DLINESIZE - 1
         bic     r0, r0, #CACHE_DLINESIZE - 1
         mcrne   p15, 0, r0, c7, c14, 1          @ clean & invalidate D entry
@@ -179,7 +180,7 @@ ENTRY(fa_dma_inv_range)
  *      - start  - virtual start address
  *      - end    - virtual end address
  */
-ENTRY(fa_dma_clean_range)
+fa_dma_clean_range:
         bic     r0, r0, #CACHE_DLINESIZE - 1
 1:      mcr     p15, 0, r0, c7, c10, 1          @ clean D entry
         add     r0, r0, #CACHE_DLINESIZE
@@ -204,6 +205,30 @@ ENTRY(fa_dma_flush_range)
         mcr     p15, 0, r0, c7, c10, 4          @ drain write buffer
         mov     pc, lr
 
+/*
+ *      dma_map_area(start, size, dir)
+ *      - start - kernel virtual start address
+ *      - size  - size of region
+ *      - dir   - DMA direction
+ */
+ENTRY(fa_dma_map_area)
+        add     r1, r1, r0
+        cmp     r2, #DMA_TO_DEVICE
+        beq     fa_dma_clean_range
+        bcs     fa_dma_inv_range
+        b       fa_dma_flush_range
+ENDPROC(fa_dma_map_area)
+
+/*
+ *      dma_unmap_area(start, size, dir)
+ *      - start - kernel virtual start address
+ *      - size  - size of region
+ *      - dir   - DMA direction
+ */
+ENTRY(fa_dma_unmap_area)
+        mov     pc, lr
+ENDPROC(fa_dma_unmap_area)
+
         __INITDATA
 
         .type   fa_cache_fns, #object
@@ -213,8 +238,8 @@ ENTRY(fa_cache_fns)
         .long   fa_flush_user_cache_range
         .long   fa_coherent_kern_range
         .long   fa_coherent_user_range
-        .long   fa_flush_kern_dcache_page
-        .long   fa_dma_inv_range
-        .long   fa_dma_clean_range
+        .long   fa_flush_kern_dcache_area
+        .long   fa_dma_map_area
+        .long   fa_dma_unmap_area
         .long   fa_dma_flush_range
         .size   fa_cache_fns, . - fa_cache_fns
diff --git a/arch/arm/mm/cache-l2x0.c b/arch/arm/mm/cache-l2x0.c
index b480f1d3591f..21ad68ba22ba 100644
--- a/arch/arm/mm/cache-l2x0.c
+++ b/arch/arm/mm/cache-l2x0.c
@@ -28,69 +28,181 @@
 static void __iomem *l2x0_base;
 static DEFINE_SPINLOCK(l2x0_lock);
 
-static inline void sync_writel(unsigned long val, unsigned long reg,
-                               unsigned long complete_mask)
+static inline void cache_wait(void __iomem *reg, unsigned long mask)
 {
-        unsigned long flags;
-
-        spin_lock_irqsave(&l2x0_lock, flags);
-        writel(val, l2x0_base + reg);
         /* wait for the operation to complete */
-        while (readl(l2x0_base + reg) & complete_mask)
+        while (readl(reg) & mask)
                 ;
-        spin_unlock_irqrestore(&l2x0_lock, flags);
 }
 
 static inline void cache_sync(void)
 {
-        sync_writel(0, L2X0_CACHE_SYNC, 1);
+        void __iomem *base = l2x0_base;
+        writel(0, base + L2X0_CACHE_SYNC);
+        cache_wait(base + L2X0_CACHE_SYNC, 1);
+}
+
+static inline void l2x0_clean_line(unsigned long addr)
+{
+        void __iomem *base = l2x0_base;
+        cache_wait(base + L2X0_CLEAN_LINE_PA, 1);
+        writel(addr, base + L2X0_CLEAN_LINE_PA);
+}
+
+static inline void l2x0_inv_line(unsigned long addr)
+{
+        void __iomem *base = l2x0_base;
+        cache_wait(base + L2X0_INV_LINE_PA, 1);
+        writel(addr, base + L2X0_INV_LINE_PA);
+}
+
+#ifdef CONFIG_PL310_ERRATA_588369
+static void debug_writel(unsigned long val)
+{
+        extern void omap_smc1(u32 fn, u32 arg);
+
+        /*
+         * Texas Instrument secure monitor api to modify the
+         * PL310 Debug Control Register.
+         */
+        omap_smc1(0x100, val);
+}
+
+static inline void l2x0_flush_line(unsigned long addr)
+{
+        void __iomem *base = l2x0_base;
+
+        /* Clean by PA followed by Invalidate by PA */
+        cache_wait(base + L2X0_CLEAN_LINE_PA, 1);
+        writel(addr, base + L2X0_CLEAN_LINE_PA);
+        cache_wait(base + L2X0_INV_LINE_PA, 1);
+        writel(addr, base + L2X0_INV_LINE_PA);
+}
+#else
+
+/* Optimised out for non-errata case */
+static inline void debug_writel(unsigned long val)
+{
+}
+
+static inline void l2x0_flush_line(unsigned long addr)
+{
+        void __iomem *base = l2x0_base;
+        cache_wait(base + L2X0_CLEAN_INV_LINE_PA, 1);
+        writel(addr, base + L2X0_CLEAN_INV_LINE_PA);
+}
+#endif
+
+static void l2x0_cache_sync(void)
+{
+        unsigned long flags;
+
+        spin_lock_irqsave(&l2x0_lock, flags);
+        cache_sync();
+        spin_unlock_irqrestore(&l2x0_lock, flags);
 }
 
 static inline void l2x0_inv_all(void)
 {
+        unsigned long flags;
+
         /* invalidate all ways */
-        sync_writel(0xff, L2X0_INV_WAY, 0xff);
+        spin_lock_irqsave(&l2x0_lock, flags);
+        writel(0xff, l2x0_base + L2X0_INV_WAY);
+        cache_wait(l2x0_base + L2X0_INV_WAY, 0xff);
         cache_sync();
+        spin_unlock_irqrestore(&l2x0_lock, flags);
 }
 
 static void l2x0_inv_range(unsigned long start, unsigned long end)
 {
-        unsigned long addr;
+        void __iomem *base = l2x0_base;
+        unsigned long flags;
 
+        spin_lock_irqsave(&l2x0_lock, flags);
         if (start & (CACHE_LINE_SIZE - 1)) {
                 start &= ~(CACHE_LINE_SIZE - 1);
-                sync_writel(start, L2X0_CLEAN_INV_LINE_PA, 1);
+                debug_writel(0x03);
+                l2x0_flush_line(start);
+                debug_writel(0x00);
                 start += CACHE_LINE_SIZE;
         }
 
         if (end & (CACHE_LINE_SIZE - 1)) {
                 end &= ~(CACHE_LINE_SIZE - 1);
-                sync_writel(end, L2X0_CLEAN_INV_LINE_PA, 1);
+                debug_writel(0x03);
+                l2x0_flush_line(end);
+                debug_writel(0x00);
         }
 
-        for (addr = start; addr < end; addr += CACHE_LINE_SIZE)
-                sync_writel(addr, L2X0_INV_LINE_PA, 1);
+        while (start < end) {
+                unsigned long blk_end = start + min(end - start, 4096UL);
+
+                while (start < blk_end) {
+                        l2x0_inv_line(start);
+                        start += CACHE_LINE_SIZE;
+                }
+
+                if (blk_end < end) {
+                        spin_unlock_irqrestore(&l2x0_lock, flags);
+                        spin_lock_irqsave(&l2x0_lock, flags);
+                }
+        }
+        cache_wait(base + L2X0_INV_LINE_PA, 1);
         cache_sync();
+        spin_unlock_irqrestore(&l2x0_lock, flags);
 }
 
 static void l2x0_clean_range(unsigned long start, unsigned long end)
 {
-        unsigned long addr;
+        void __iomem *base = l2x0_base;
+        unsigned long flags;
 
+        spin_lock_irqsave(&l2x0_lock, flags);
         start &= ~(CACHE_LINE_SIZE - 1);
-        for (addr = start; addr < end; addr += CACHE_LINE_SIZE)
-                sync_writel(addr, L2X0_CLEAN_LINE_PA, 1);
+        while (start < end) {
+                unsigned long blk_end = start + min(end - start, 4096UL);
+
+                while (start < blk_end) {
+                        l2x0_clean_line(start);
+                        start += CACHE_LINE_SIZE;
+                }
+
+                if (blk_end < end) {
+                        spin_unlock_irqrestore(&l2x0_lock, flags);
+                        spin_lock_irqsave(&l2x0_lock, flags);
+                }
+        }
+        cache_wait(base + L2X0_CLEAN_LINE_PA, 1);
         cache_sync();
+        spin_unlock_irqrestore(&l2x0_lock, flags);
 }
 
 static void l2x0_flush_range(unsigned long start, unsigned long end)
 {
-        unsigned long addr;
+        void __iomem *base = l2x0_base;
+        unsigned long flags;
 
+        spin_lock_irqsave(&l2x0_lock, flags);
         start &= ~(CACHE_LINE_SIZE - 1);
-        for (addr = start; addr < end; addr += CACHE_LINE_SIZE)
-                sync_writel(addr, L2X0_CLEAN_INV_LINE_PA, 1);
+        while (start < end) {
+                unsigned long blk_end = start + min(end - start, 4096UL);
+
+                debug_writel(0x03);
+                while (start < blk_end) {
+                        l2x0_flush_line(start);
+                        start += CACHE_LINE_SIZE;
+                }
+                debug_writel(0x00);
+
+                if (blk_end < end) {
+                        spin_unlock_irqrestore(&l2x0_lock, flags);
+                        spin_lock_irqsave(&l2x0_lock, flags);
+                }
+        }
+        cache_wait(base + L2X0_CLEAN_INV_LINE_PA, 1);
         cache_sync();
+        spin_unlock_irqrestore(&l2x0_lock, flags);
 }
 
 void __init l2x0_init(void __iomem *base, __u32 aux_val, __u32 aux_mask)
@@ -99,22 +211,30 @@ void __init l2x0_init(void __iomem *base, __u32 aux_val, __u32 aux_mask)
 
         l2x0_base = base;
 
-        /* disable L2X0 */
-        writel(0, l2x0_base + L2X0_CTRL);
+        /*
+         * Check if l2x0 controller is already enabled.
+         * If you are booting from non-secure mode
+         * accessing the below registers will fault.
+         */
+        if (!(readl(l2x0_base + L2X0_CTRL) & 1)) {
+
+                /* l2x0 controller is disabled */
 
-        aux = readl(l2x0_base + L2X0_AUX_CTRL);
-        aux &= aux_mask;
-        aux |= aux_val;
-        writel(aux, l2x0_base + L2X0_AUX_CTRL);
+                aux = readl(l2x0_base + L2X0_AUX_CTRL);
+                aux &= aux_mask;
+                aux |= aux_val;
+                writel(aux, l2x0_base + L2X0_AUX_CTRL);
 
-        l2x0_inv_all();
+                l2x0_inv_all();
 
-        /* enable L2X0 */
-        writel(1, l2x0_base + L2X0_CTRL);
+                /* enable L2X0 */
+                writel(1, l2x0_base + L2X0_CTRL);
+        }
 
         outer_cache.inv_range = l2x0_inv_range;
         outer_cache.clean_range = l2x0_clean_range;
         outer_cache.flush_range = l2x0_flush_range;
+        outer_cache.sync = l2x0_cache_sync;
 
         printk(KERN_INFO "L2X0 cache controller enabled\n");
 }
diff --git a/arch/arm/mm/cache-tauros2.c b/arch/arm/mm/cache-tauros2.c
new file mode 100644
index 000000000000..50868651890f
--- /dev/null
+++ b/arch/arm/mm/cache-tauros2.c
@@ -0,0 +1,263 @@
+/*
+ * arch/arm/mm/cache-tauros2.c - Tauros2 L2 cache controller support
+ *
+ * Copyright (C) 2008 Marvell Semiconductor
+ *
+ * This file is licensed under the terms of the GNU General Public
+ * License version 2.  This program is licensed "as is" without any
+ * warranty of any kind, whether express or implied.
+ *
+ * References:
+ * - PJ1 CPU Core Datasheet,
+ *   Document ID MV-S104837-01, Rev 0.7, January 24 2008.
+ * - PJ4 CPU Core Datasheet,
+ *   Document ID MV-S105190-00, Rev 0.7, March 14 2008.
+ */
+
+#include <linux/init.h>
+#include <asm/cacheflush.h>
+#include <asm/hardware/cache-tauros2.h>
+
+
+/*
+ * When Tauros2 is used on a CPU that supports the v7 hierarchical
+ * cache operations, the cache handling code in proc-v7.S takes care
+ * of everything, including handling DMA coherency.
+ *
+ * So, we only need to register outer cache operations here if we're
+ * being used on a pre-v7 CPU, and we only need to build support for
+ * outer cache operations into the kernel image if the kernel has been
+ * configured to support a pre-v7 CPU.
+ */
+#if __LINUX_ARM_ARCH__ < 7
+/*
+ * Low-level cache maintenance operations.
+ */
+static inline void tauros2_clean_pa(unsigned long addr)
+{
+        __asm__("mcr p15, 1, %0, c7, c11, 3" : : "r" (addr));
+}
+
+static inline void tauros2_clean_inv_pa(unsigned long addr)
+{
+        __asm__("mcr p15, 1, %0, c7, c15, 3" : : "r" (addr));
+}
+
+static inline void tauros2_inv_pa(unsigned long addr)
+{
+        __asm__("mcr p15, 1, %0, c7, c7, 3" : : "r" (addr));
+}
+
+
+/*
+ * Linux primitives.
+ *
+ * Note that the end addresses passed to Linux primitives are
+ * noninclusive.
+ */
+#define CACHE_LINE_SIZE         32
+
+static void tauros2_inv_range(unsigned long start, unsigned long end)
+{
+        /*
+         * Clean and invalidate partial first cache line.
+         */
+        if (start & (CACHE_LINE_SIZE - 1)) {
+                tauros2_clean_inv_pa(start & ~(CACHE_LINE_SIZE - 1));
+                start = (start | (CACHE_LINE_SIZE - 1)) + 1;
+        }
+
+        /*
+         * Clean and invalidate partial last cache line.
+         */
+        if (end & (CACHE_LINE_SIZE - 1)) {
+                tauros2_clean_inv_pa(end & ~(CACHE_LINE_SIZE - 1));
+                end &= ~(CACHE_LINE_SIZE - 1);
+        }
+
+        /*
+         * Invalidate all full cache lines between 'start' and 'end'.
+         */
+        while (start < end) {
+                tauros2_inv_pa(start);
+                start += CACHE_LINE_SIZE;
+        }
+
+        dsb();
+}
+
+static void tauros2_clean_range(unsigned long start, unsigned long end)
+{
+        start &= ~(CACHE_LINE_SIZE - 1);
+        while (start < end) {
+                tauros2_clean_pa(start);
+                start += CACHE_LINE_SIZE;
+        }
+
+        dsb();
+}
+
+static void tauros2_flush_range(unsigned long start, unsigned long end)
+{
+        start &= ~(CACHE_LINE_SIZE - 1);
+        while (start < end) {
+                tauros2_clean_inv_pa(start);
+                start += CACHE_LINE_SIZE;
+        }
+
+        dsb();
+}
+#endif
+
+static inline u32 __init read_extra_features(void)
+{
+        u32 u;
+
+        __asm__("mrc p15, 1, %0, c15, c1, 0" : "=r" (u));
+
+        return u;
+}
+
+static inline void __init write_extra_features(u32 u)
+{
+        __asm__("mcr p15, 1, %0, c15, c1, 0" : : "r" (u));
+}
+
+static void __init disable_l2_prefetch(void)
+{
+        u32 u;
+
+        /*
+         * Read the CPU Extra Features register and verify that the
+         * Disable L2 Prefetch bit is set.
+         */
+        u = read_extra_features();
+        if (!(u & 0x01000000)) {
+                printk(KERN_INFO "Tauros2: Disabling L2 prefetch.\n");
+                write_extra_features(u | 0x01000000);
+        }
+}
+
+static inline int __init cpuid_scheme(void)
+{
+        extern int processor_id;
+
+        return !!((processor_id & 0x000f0000) == 0x000f0000);
+}
+
+static inline u32 __init read_mmfr3(void)
+{
+        u32 mmfr3;
+
+        __asm__("mrc p15, 0, %0, c0, c1, 7\n" : "=r" (mmfr3));
+
+        return mmfr3;
+}
+
+static inline u32 __init read_actlr(void)
+{
+        u32 actlr;
+
+        __asm__("mrc p15, 0, %0, c1, c0, 1\n" : "=r" (actlr));
+
+        return actlr;
+}
+
+static inline void __init write_actlr(u32 actlr)
+{
+        __asm__("mcr p15, 0, %0, c1, c0, 1\n" : : "r" (actlr));
+}
+
+void __init tauros2_init(void)
+{
+        extern int processor_id;
+        char *mode;
+
+        disable_l2_prefetch();
+
+#ifdef CONFIG_CPU_32v5
+        if ((processor_id & 0xff0f0000) == 0x56050000) {
+                u32 feat;
+
+                /*
+                 * v5 CPUs with Tauros2 have the L2 cache enable bit
+                 * located in the CPU Extra Features register.
+                 */
+                feat = read_extra_features();
+                if (!(feat & 0x00400000)) {
+                        printk(KERN_INFO "Tauros2: Enabling L2 cache.\n");
+                        write_extra_features(feat | 0x00400000);
+                }
+
+                mode = "ARMv5";
+                outer_cache.inv_range = tauros2_inv_range;
+                outer_cache.clean_range = tauros2_clean_range;
+                outer_cache.flush_range = tauros2_flush_range;
+        }
+#endif
+
+#ifdef CONFIG_CPU_32v6
+        /*
+         * Check whether this CPU lacks support for the v7 hierarchical
+         * cache ops.  (PJ4 is in its v6 personality mode if the MMFR3
+         * register indicates no support for the v7 hierarchical cache
+         * ops.)
+         */
+        if (cpuid_scheme() && (read_mmfr3() & 0xf) == 0) {
+                /*
+                 * When Tauros2 is used in an ARMv6 system, the L2
+                 * enable bit is in the ARMv6 ARM-mandated position
+                 * (bit [26] of the System Control Register).
+                 */
+                if (!(get_cr() & 0x04000000)) {
+                        printk(KERN_INFO "Tauros2: Enabling L2 cache.\n");
+                        adjust_cr(0x04000000, 0x04000000);
+                }
+
+                mode = "ARMv6";
+                outer_cache.inv_range = tauros2_inv_range;
+                outer_cache.clean_range = tauros2_clean_range;
+                outer_cache.flush_range = tauros2_flush_range;
+        }
+#endif
+
+#ifdef CONFIG_CPU_32v7
+        /*
+         * Check whether this CPU has support for the v7 hierarchical
+         * cache ops.  (PJ4 is in its v7 personality mode if the MMFR3
+         * register indicates support for the v7 hierarchical cache
+         * ops.)
+         *
+         * (Although strictly speaking there may exist CPUs that
+         * implement the v7 cache ops but are only ARMv6 CPUs (due to
+         * not complying with all of the other ARMv7 requirements),
+         * there are no real-life examples of Tauros2 being used on
+         * such CPUs as of yet.)
+         */
+        if (cpuid_scheme() && (read_mmfr3() & 0xf) == 1) {
+                u32 actlr;
+
+                /*
+                 * When Tauros2 is used in an ARMv7 system, the L2
+                 * enable bit is located in the Auxiliary System Control
+                 * Register (which is the only register allowed by the
+                 * ARMv7 spec to contain fine-grained cache control bits).
+                 */
+                actlr = read_actlr();
+                if (!(actlr & 0x00000002)) {
+                        printk(KERN_INFO "Tauros2: Enabling L2 cache.\n");
+                        write_actlr(actlr | 0x00000002);
+                }
+
+                mode = "ARMv7";
+        }
+#endif
+
+        if (mode == NULL) {
+                printk(KERN_CRIT "Tauros2: Unable to detect CPU mode.\n");
+                return;
+        }
+
+        printk(KERN_INFO "Tauros2: L2 cache support initialised "
+                         "in %s mode.\n", mode);
+}
diff --git a/arch/arm/mm/cache-v3.S b/arch/arm/mm/cache-v3.S
index 8a4abebc478a..c2ff3c599fee 100644
--- a/arch/arm/mm/cache-v3.S
+++ b/arch/arm/mm/cache-v3.S
@@ -72,28 +72,15 @@ ENTRY(v3_coherent_user_range)
         mov     pc, lr
 
 /*
- *      flush_kern_dcache_page(void *page)
+ *      flush_kern_dcache_area(void *page, size_t size)
  *
  *      Ensure no D cache aliasing occurs, either with itself or
  *      the I cache
  *
- *      - addr  - page aligned address
+ *      - addr  - kernel address
+ *      - size  - region size
  */
-ENTRY(v3_flush_kern_dcache_page)
-        /* FALLTHROUGH */
-
-/*
- *      dma_inv_range(start, end)
- *
- *      Invalidate (discard) the specified virtual address range.
- *      May not write back any entries.  If 'start' or 'end'
- *      are not cache line aligned, those lines must be written
- *      back.
- *
- *      - start  - virtual start address
- *      - end    - virtual end address
- */
-ENTRY(v3_dma_inv_range)
+ENTRY(v3_flush_kern_dcache_area)
         /* FALLTHROUGH */
 
 /*
@@ -107,18 +94,29 @@ ENTRY(v3_dma_inv_range)
 ENTRY(v3_dma_flush_range)
         mov     r0, #0
         mcr     p15, 0, r0, c7, c0, 0           @ flush ID cache
+        mov     pc, lr
+
+/*
+ *      dma_unmap_area(start, size, dir)
+ *      - start - kernel virtual start address
+ *      - size  - size of region
+ *      - dir   - DMA direction
+ */
+ENTRY(v3_dma_unmap_area)
+        teq     r2, #DMA_TO_DEVICE
+        bne     v3_dma_flush_range
         /* FALLTHROUGH */
 
 /*
- *      dma_clean_range(start, end)
- *
- *      Clean (write back) the specified virtual address range.
- *
- *      - start  - virtual start address
- *      - end    - virtual end address
+ *      dma_map_area(start, size, dir)
+ *      - start - kernel virtual start address
+ *      - size  - size of region
+ *      - dir   - DMA direction
  */
-ENTRY(v3_dma_clean_range)
+ENTRY(v3_dma_map_area)
         mov     pc, lr
+ENDPROC(v3_dma_unmap_area)
+ENDPROC(v3_dma_map_area)
 
         __INITDATA
 
@@ -129,8 +127,8 @@ ENTRY(v3_cache_fns)
         .long   v3_flush_user_cache_range
         .long   v3_coherent_kern_range
         .long   v3_coherent_user_range
-        .long   v3_flush_kern_dcache_page
-        .long   v3_dma_inv_range
-        .long   v3_dma_clean_range
+        .long   v3_flush_kern_dcache_area
+        .long   v3_dma_map_area
+        .long   v3_dma_unmap_area
         .long   v3_dma_flush_range
         .size   v3_cache_fns, . - v3_cache_fns
diff --git a/arch/arm/mm/cache-v4.S b/arch/arm/mm/cache-v4.S
index 3668611cb400..4810f7e3e813 100644
--- a/arch/arm/mm/cache-v4.S
+++ b/arch/arm/mm/cache-v4.S
@@ -82,28 +82,15 @@ ENTRY(v4_coherent_user_range)
         mov     pc, lr
 
 /*
- *      flush_kern_dcache_page(void *page)
+ *      flush_kern_dcache_area(void *addr, size_t size)
  *
  *      Ensure no D cache aliasing occurs, either with itself or
  *      the I cache
  *
- *      - addr  - page aligned address
+ *      - addr  - kernel address
+ *      - size  - region size
  */
-ENTRY(v4_flush_kern_dcache_page)
-        /* FALLTHROUGH */
-
-/*
- *      dma_inv_range(start, end)
- *
- *      Invalidate (discard) the specified virtual address range.
- *      May not write back any entries.  If 'start' or 'end'
- *      are not cache line aligned, those lines must be written
- *      back.
- *
- *      - start  - virtual start address
- *      - end    - virtual end address
- */
-ENTRY(v4_dma_inv_range)
+ENTRY(v4_flush_kern_dcache_area)
         /* FALLTHROUGH */
 
 /*
@@ -119,18 +106,29 @@ ENTRY(v4_dma_flush_range)
         mov     r0, #0
         mcr     p15, 0, r0, c7, c7, 0           @ flush ID cache
 #endif
+        mov     pc, lr
+
+/*
+ *      dma_unmap_area(start, size, dir)
+ *      - start - kernel virtual start address
+ *      - size  - size of region
+ *      - dir   - DMA direction
+ */
+ENTRY(v4_dma_unmap_area)
+        teq     r2, #DMA_TO_DEVICE
+        bne     v4_dma_flush_range
         /* FALLTHROUGH */
 
 /*
- *      dma_clean_range(start, end)
- *
- *      Clean (write back) the specified virtual address range.
- *
- *      - start  - virtual start address
- *      - end    - virtual end address
+ *      dma_map_area(start, size, dir)
+ *      - start - kernel virtual start address
+ *      - size  - size of region
+ *      - dir   - DMA direction
  */
-ENTRY(v4_dma_clean_range)
+ENTRY(v4_dma_map_area)
         mov     pc, lr
+ENDPROC(v4_dma_unmap_area)
+ENDPROC(v4_dma_map_area)
 
         __INITDATA
 
@@ -141,8 +139,8 @@ ENTRY(v4_cache_fns)
         .long   v4_flush_user_cache_range
         .long   v4_coherent_kern_range
         .long   v4_coherent_user_range
-        .long   v4_flush_kern_dcache_page
-        .long   v4_dma_inv_range
-        .long   v4_dma_clean_range
+        .long   v4_flush_kern_dcache_area
+        .long   v4_dma_map_area
+        .long   v4_dma_unmap_area
         .long   v4_dma_flush_range
         .size   v4_cache_fns, . - v4_cache_fns
diff --git a/arch/arm/mm/cache-v4wb.S b/arch/arm/mm/cache-v4wb.S
index 2ebc1b3bf856..df8368afa102 100644
--- a/arch/arm/mm/cache-v4wb.S
+++ b/arch/arm/mm/cache-v4wb.S
@@ -114,15 +114,16 @@ ENTRY(v4wb_flush_user_cache_range)
         mov     pc, lr
 
 /*
- *      flush_kern_dcache_page(void *page)
+ *      flush_kern_dcache_area(void *addr, size_t size)
  *
  *      Ensure no D cache aliasing occurs, either with itself or
  *      the I cache
  *
- *      - addr  - page aligned address
+ *      - addr  - kernel address
+ *      - size  - region size
  */
-ENTRY(v4wb_flush_kern_dcache_page)
-        add     r1, r0, #PAGE_SZ
+ENTRY(v4wb_flush_kern_dcache_area)
+        add     r1, r0, r1
         /* fall through */
 
 /*
@@ -172,7 +173,7 @@ ENTRY(v4wb_coherent_user_range)
  *      - start  - virtual start address
  *      - end    - virtual end address
  */
-ENTRY(v4wb_dma_inv_range)
+v4wb_dma_inv_range:
         tst     r0, #CACHE_DLINESIZE - 1
         bic     r0, r0, #CACHE_DLINESIZE - 1
         mcrne   p15, 0, r0, c7, c10, 1          @ clean D entry
@@ -193,7 +194,7 @@ ENTRY(v4wb_dma_inv_range)
  *      - start  - virtual start address
  *      - end    - virtual end address
  */
-ENTRY(v4wb_dma_clean_range)
+v4wb_dma_clean_range:
         bic     r0, r0, #CACHE_DLINESIZE - 1
 1:      mcr     p15, 0, r0, c7, c10, 1          @ clean D entry
         add     r0, r0, #CACHE_DLINESIZE
@@ -215,6 +216,30 @@ ENTRY(v4wb_dma_clean_range)
         .globl  v4wb_dma_flush_range
         .set    v4wb_dma_flush_range, v4wb_coherent_kern_range
 
+/*
+ *      dma_map_area(start, size, dir)
+ *      - start - kernel virtual start address
+ *      - size  - size of region
+ *      - dir   - DMA direction
+ */
+ENTRY(v4wb_dma_map_area)
+        add     r1, r1, r0
+        cmp     r2, #DMA_TO_DEVICE
+        beq     v4wb_dma_clean_range
+        bcs     v4wb_dma_inv_range
+        b       v4wb_dma_flush_range
+ENDPROC(v4wb_dma_map_area)
+
+/*
+ *      dma_unmap_area(start, size, dir)
+ *      - start - kernel virtual start address
+ *      - size  - size of region
+ *      - dir   - DMA direction
+ */
+ENTRY(v4wb_dma_unmap_area)
+        mov     pc, lr
+ENDPROC(v4wb_dma_unmap_area)
+
         __INITDATA
 
         .type   v4wb_cache_fns, #object
@@ -224,8 +249,8 @@ ENTRY(v4wb_cache_fns)
         .long   v4wb_flush_user_cache_range
         .long   v4wb_coherent_kern_range
         .long   v4wb_coherent_user_range
-        .long   v4wb_flush_kern_dcache_page
-        .long   v4wb_dma_inv_range
-        .long   v4wb_dma_clean_range
+        .long   v4wb_flush_kern_dcache_area
+        .long   v4wb_dma_map_area
+        .long   v4wb_dma_unmap_area
         .long   v4wb_dma_flush_range
         .size   v4wb_cache_fns, . - v4wb_cache_fns
diff --git a/arch/arm/mm/cache-v4wt.S b/arch/arm/mm/cache-v4wt.S
index c54fa2cc40e6..45c70312f43b 100644
--- a/arch/arm/mm/cache-v4wt.S
+++ b/arch/arm/mm/cache-v4wt.S
@@ -117,17 +117,18 @@ ENTRY(v4wt_coherent_user_range)
         mov     pc, lr
 
 /*
- *      flush_kern_dcache_page(void *page)
+ *      flush_kern_dcache_area(void *addr, size_t size)
  *
  *      Ensure no D cache aliasing occurs, either with itself or
  *      the I cache
  *
- *      - addr  - page aligned address
+ *      - addr  - kernel address
+ *      - size  - region size
  */
-ENTRY(v4wt_flush_kern_dcache_page)
+ENTRY(v4wt_flush_kern_dcache_area)
         mov     r2, #0
         mcr     p15, 0, r2, c7, c5, 0           @ invalidate I cache
-        add     r1, r0, #PAGE_SZ
+        add     r1, r0, r1
         /* fallthrough */
 
 /*
@@ -141,23 +142,12 @@ ENTRY(v4wt_flush_kern_dcache_page)
  *      - start  - virtual start address
  *      - end    - virtual end address
  */
-ENTRY(v4wt_dma_inv_range)
+v4wt_dma_inv_range:
         bic     r0, r0, #CACHE_DLINESIZE - 1
 1:      mcr     p15, 0, r0, c7, c6, 1           @ invalidate D entry
         add     r0, r0, #CACHE_DLINESIZE
         cmp     r0, r1
         blo     1b
-        /* FALLTHROUGH */
-
-/*
- *      dma_clean_range(start, end)
- *
- *      Clean the specified virtual address range.
- *
- *      - start  - virtual start address
- *      - end    - virtual end address
- */
-ENTRY(v4wt_dma_clean_range)
         mov     pc, lr
 
 /*
@@ -171,6 +161,29 @@ ENTRY(v4wt_dma_clean_range)
         .globl  v4wt_dma_flush_range
         .equ    v4wt_dma_flush_range, v4wt_dma_inv_range
 
+/*
+ *      dma_unmap_area(start, size, dir)
+ *      - start - kernel virtual start address
+ *      - size  - size of region
+ *      - dir   - DMA direction
+ */
+ENTRY(v4wt_dma_unmap_area)
+        add     r1, r1, r0
+        teq     r2, #DMA_TO_DEVICE
+        bne     v4wt_dma_inv_range
+        /* FALLTHROUGH */
+
+/*
+ *      dma_map_area(start, size, dir)
+ *      - start - kernel virtual start address
+ *      - size  - size of region
+ *      - dir   - DMA direction
+ */
+ENTRY(v4wt_dma_map_area)
+        mov     pc, lr
+ENDPROC(v4wt_dma_unmap_area)
+ENDPROC(v4wt_dma_map_area)
+
         __INITDATA
 
         .type   v4wt_cache_fns, #object
@@ -180,8 +193,8 @@ ENTRY(v4wt_cache_fns)
         .long   v4wt_flush_user_cache_range
         .long   v4wt_coherent_kern_range
         .long   v4wt_coherent_user_range
-        .long   v4wt_flush_kern_dcache_page
-        .long   v4wt_dma_inv_range
-        .long   v4wt_dma_clean_range
+        .long   v4wt_flush_kern_dcache_area
+        .long   v4wt_dma_map_area
+        .long   v4wt_dma_unmap_area
         .long   v4wt_dma_flush_range
         .size   v4wt_cache_fns, . - v4wt_cache_fns
diff --git a/arch/arm/mm/cache-v6.S b/arch/arm/mm/cache-v6.S
index 295e25dd6381..e46ecd847138 100644
--- a/arch/arm/mm/cache-v6.S
+++ b/arch/arm/mm/cache-v6.S
@@ -159,15 +159,16 @@ ENDPROC(v6_coherent_user_range)
 ENDPROC(v6_coherent_kern_range)
 
 /*
- *      v6_flush_kern_dcache_page(kaddr)
+ *      v6_flush_kern_dcache_area(void *addr, size_t size)
  *
  *      Ensure that the data held in the page kaddr is written back
  *      to the page in question.
  *
- *      - kaddr   - kernel address (guaranteed to be page aligned)
+ *      - addr  - kernel address
+ *      - size  - region size
  */
-ENTRY(v6_flush_kern_dcache_page)
-        add     r1, r0, #PAGE_SZ
+ENTRY(v6_flush_kern_dcache_area)
+        add     r1, r0, r1
 1:
 #ifdef HARVARD_CACHE
         mcr     p15, 0, r0, c7, c14, 1          @ clean & invalidate D line
@@ -194,7 +195,7 @@ ENTRY(v6_flush_kern_dcache_page)
  *      - start   - virtual start address of region
  *      - end     - virtual end address of region
  */
-ENTRY(v6_dma_inv_range)
+v6_dma_inv_range:
         tst     r0, #D_CACHE_LINE_SIZE - 1
         bic     r0, r0, #D_CACHE_LINE_SIZE - 1
 #ifdef HARVARD_CACHE
@@ -210,6 +211,9 @@ ENTRY(v6_dma_inv_range)
         mcrne   p15, 0, r1, c7, c15, 1          @ clean & invalidate unified line
 #endif
 1:
+#ifdef CONFIG_SMP
+        str     r0, [r0]                        @ write for ownership
+#endif
 #ifdef HARVARD_CACHE
         mcr     p15, 0, r0, c7, c6, 1           @ invalidate D line
 #else
@@ -227,9 +231,12 @@ ENTRY(v6_dma_inv_range)
  *      - start   - virtual start address of region
  *      - end     - virtual end address of region
  */
-ENTRY(v6_dma_clean_range)
+v6_dma_clean_range:
         bic     r0, r0, #D_CACHE_LINE_SIZE - 1
 1:
+#ifdef CONFIG_SMP
+        ldr     r2, [r0]                        @ read for ownership
+#endif
 #ifdef HARVARD_CACHE
         mcr     p15, 0, r0, c7, c10, 1          @ clean D line
 #else
@@ -250,6 +257,10 @@ ENTRY(v6_dma_clean_range)
 ENTRY(v6_dma_flush_range)
         bic     r0, r0, #D_CACHE_LINE_SIZE - 1
 1:
+#ifdef CONFIG_SMP
+        ldr     r2, [r0]                        @ read for ownership
+        str     r2, [r0]                        @ write for ownership
+#endif
 #ifdef HARVARD_CACHE
         mcr     p15, 0, r0, c7, c14, 1          @ clean & invalidate D line
 #else
@@ -262,6 +273,31 @@ ENTRY(v6_dma_flush_range)
         mcr     p15, 0, r0, c7, c10, 4          @ drain write buffer
         mov     pc, lr
 
+/*
+ *      dma_map_area(start, size, dir)
+ *      - start - kernel virtual start address
+ *      - size  - size of region
+ *      - dir   - DMA direction
+ */
+ENTRY(v6_dma_map_area)
+        add     r1, r1, r0
+        teq     r2, #DMA_FROM_DEVICE
+        beq     v6_dma_inv_range
+        teq     r2, #DMA_TO_DEVICE
+        beq     v6_dma_clean_range
+        b       v6_dma_flush_range
+ENDPROC(v6_dma_map_area)
+
+/*
+ *      dma_unmap_area(start, size, dir)
+ *      - start - kernel virtual start address
+ *      - size  - size of region
+ *      - dir   - DMA direction
+ */
+ENTRY(v6_dma_unmap_area)
+        mov     pc, lr
+ENDPROC(v6_dma_unmap_area)
+
         __INITDATA
 
         .type   v6_cache_fns, #object
@@ -271,8 +307,8 @@ ENTRY(v6_cache_fns)
         .long   v6_flush_user_cache_range
         .long   v6_coherent_kern_range
         .long   v6_coherent_user_range
-        .long   v6_flush_kern_dcache_page
-        .long   v6_dma_inv_range
-        .long   v6_dma_clean_range
+        .long   v6_flush_kern_dcache_area
+        .long   v6_dma_map_area
+        .long   v6_dma_unmap_area
         .long   v6_dma_flush_range
         .size   v6_cache_fns, . - v6_cache_fns
diff --git a/arch/arm/mm/cache-v7.S b/arch/arm/mm/cache-v7.S
index e1bd9759617f..06a90dcfc60a 100644
--- a/arch/arm/mm/cache-v7.S
+++ b/arch/arm/mm/cache-v7.S
@@ -167,7 +167,11 @@ ENTRY(v7_coherent_user_range)
         cmp     r0, r1
         blo     1b
         mov     r0, #0
+#ifdef CONFIG_SMP
+        mcr     p15, 0, r0, c7, c1, 6           @ invalidate BTB Inner Shareable
+#else
         mcr     p15, 0, r0, c7, c5, 6           @ invalidate BTB
+#endif
         dsb
         isb
         mov     pc, lr
@@ -186,16 +190,17 @@ ENDPROC(v7_coherent_kern_range)
 ENDPROC(v7_coherent_user_range)
 
 /*
- *      v7_flush_kern_dcache_page(kaddr)
+ *      v7_flush_kern_dcache_area(void *addr, size_t size)
  *
  *      Ensure that the data held in the page kaddr is written back
  *      to the page in question.
  *
- *      - kaddr   - kernel address (guaranteed to be page aligned)
+ *      - addr  - kernel address
+ *      - size  - region size
  */
-ENTRY(v7_flush_kern_dcache_page)
+ENTRY(v7_flush_kern_dcache_area)
         dcache_line_size r2, r3
-        add     r1, r0, #PAGE_SZ
+        add     r1, r0, r1
 1:
         mcr     p15, 0, r0, c7, c14, 1          @ clean & invalidate D line / unified line
         add     r0, r0, r2
@@ -203,7 +208,7 @@ ENTRY(v7_flush_kern_dcache_page)
         blo     1b
         dsb
         mov     pc, lr
-ENDPROC(v7_flush_kern_dcache_page)
+ENDPROC(v7_flush_kern_dcache_area)
 
 /*
  *      v7_dma_inv_range(start,end)
@@ -215,7 +220,7 @@ ENDPROC(v7_flush_kern_dcache_page)
  *      - start   - virtual start address of region
  *      - end     - virtual end address of region
  */
-ENTRY(v7_dma_inv_range)
+v7_dma_inv_range:
         dcache_line_size r2, r3
         sub     r3, r2, #1
         tst     r0, r3
@@ -239,7 +244,7 @@ ENDPROC(v7_dma_inv_range)
  *      - start   - virtual start address of region
  *      - end     - virtual end address of region
  */
-ENTRY(v7_dma_clean_range)
+v7_dma_clean_range:
         dcache_line_size r2, r3
         sub     r3, r2, #1
         bic     r0, r0, r3
@@ -270,6 +275,32 @@ ENTRY(v7_dma_flush_range)
         mov     pc, lr
 ENDPROC(v7_dma_flush_range)
 
+/*
+ *      dma_map_area(start, size, dir)
+ *      - start - kernel virtual start address
+ *      - size  - size of region
+ *      - dir   - DMA direction
+ */
+ENTRY(v7_dma_map_area)
+        add     r1, r1, r0
+        teq     r2, #DMA_FROM_DEVICE
+        beq     v7_dma_inv_range
+        b       v7_dma_clean_range
+ENDPROC(v7_dma_map_area)
+
+/*
+ *      dma_unmap_area(start, size, dir)
+ *      - start - kernel virtual start address
+ *      - size  - size of region
+ *      - dir   - DMA direction
+ */
+ENTRY(v7_dma_unmap_area)
+        add     r1, r1, r0
+        teq     r2, #DMA_TO_DEVICE
+        bne     v7_dma_inv_range
+        mov     pc, lr
+ENDPROC(v7_dma_unmap_area)
+
         __INITDATA
 
         .type   v7_cache_fns, #object
@@ -279,8 +310,8 @@ ENTRY(v7_cache_fns)
         .long   v7_flush_user_cache_range
         .long   v7_coherent_kern_range
         .long   v7_coherent_user_range
-        .long   v7_flush_kern_dcache_page
-        .long   v7_dma_inv_range
-        .long   v7_dma_clean_range
+        .long   v7_flush_kern_dcache_area
+        .long   v7_dma_map_area
+        .long   v7_dma_unmap_area
         .long   v7_dma_flush_range
         .size   v7_cache_fns, . - v7_cache_fns
diff --git a/arch/arm/mm/cache-xsc3l2.c b/arch/arm/mm/cache-xsc3l2.c
index 5d180cb0bd94..c3154928bccd 100644
--- a/arch/arm/mm/cache-xsc3l2.c
+++ b/arch/arm/mm/cache-xsc3l2.c
@@ -221,15 +221,14 @@ static int __init xsc3_l2_init(void)
         if (!cpu_is_xsc3() || !xsc3_l2_present())
                 return 0;
 
-        if (!(get_cr() & CR_L2)) {
+        if (get_cr() & CR_L2) {
                 pr_info("XScale3 L2 cache enabled.\n");
-                adjust_cr(CR_L2, CR_L2);
                 xsc3_l2_inv_all();
-        }
 
-        outer_cache.inv_range = xsc3_l2_inv_range;
-        outer_cache.clean_range = xsc3_l2_clean_range;
-        outer_cache.flush_range = xsc3_l2_flush_range;
+                outer_cache.inv_range = xsc3_l2_inv_range;
+                outer_cache.clean_range = xsc3_l2_clean_range;
+                outer_cache.flush_range = xsc3_l2_flush_range;
+        }
 
         return 0;
 }
diff --git a/arch/arm/mm/context.c b/arch/arm/mm/context.c
index a9e22e31eaa1..b0ee9ba3cfab 100644
--- a/arch/arm/mm/context.c
+++ b/arch/arm/mm/context.c
@@ -10,12 +10,17 @@
 #include <linux/init.h>
 #include <linux/sched.h>
 #include <linux/mm.h>
+#include <linux/smp.h>
+#include <linux/percpu.h>
 
 #include <asm/mmu_context.h>
 #include <asm/tlbflush.h>
 
 static DEFINE_SPINLOCK(cpu_asid_lock);
 unsigned int cpu_last_asid = ASID_FIRST_VERSION;
+#ifdef CONFIG_SMP
+DEFINE_PER_CPU(struct mm_struct *, current_mm);
+#endif
 
 /*
  * We fork()ed a process, and we need a new context for the child
@@ -26,13 +31,109 @@ unsigned int cpu_last_asid = ASID_FIRST_VERSION;
 void __init_new_context(struct task_struct *tsk, struct mm_struct *mm)
 {
         mm->context.id = 0;
+        spin_lock_init(&mm->context.id_lock);
 }
 
+static void flush_context(void)
+{
+        /* set the reserved ASID before flushing the TLB */
+        asm("mcr        p15, 0, %0, c13, c0, 1\n" : : "r" (0));
+        isb();
+        local_flush_tlb_all();
+        if (icache_is_vivt_asid_tagged()) {
+                __flush_icache_all();
+                dsb();
+        }
+}
+
+#ifdef CONFIG_SMP
+
+static void set_mm_context(struct mm_struct *mm, unsigned int asid)
+{
+        unsigned long flags;
+
+        /*
+         * Locking needed for multi-threaded applications where the
+         * same mm->context.id could be set from different CPUs during
+         * the broadcast. This function is also called via IPI so the
+         * mm->context.id_lock has to be IRQ-safe.
+         */
+        spin_lock_irqsave(&mm->context.id_lock, flags);
+        if (likely((mm->context.id ^ cpu_last_asid) >> ASID_BITS)) {
+                /*
+                 * Old version of ASID found. Set the new one and
+                 * reset mm_cpumask(mm).
+                 */
+                mm->context.id = asid;
+                cpumask_clear(mm_cpumask(mm));
+        }
+        spin_unlock_irqrestore(&mm->context.id_lock, flags);
+
+        /*
+         * Set the mm_cpumask(mm) bit for the current CPU.
+         */
+        cpumask_set_cpu(smp_processor_id(), mm_cpumask(mm));
+}
+
+/*
+ * Reset the ASID on the current CPU. This function call is broadcast
+ * from the CPU handling the ASID rollover and holding cpu_asid_lock.
+ */
+static void reset_context(void *info)
+{
+        unsigned int asid;
+        unsigned int cpu = smp_processor_id();
+        struct mm_struct *mm = per_cpu(current_mm, cpu);
+
+        /*
+         * Check if a current_mm was set on this CPU as it might still
+         * be in the early booting stages and using the reserved ASID.
+         */
+        if (!mm)
+                return;
+
+        smp_rmb();
+        asid = cpu_last_asid + cpu + 1;
+
+        flush_context();
+        set_mm_context(mm, asid);
+
+        /* set the new ASID */
+        asm("mcr        p15, 0, %0, c13, c0, 1\n" : : "r" (mm->context.id));
+        isb();
+}
+
+#else
+
+static inline void set_mm_context(struct mm_struct *mm, unsigned int asid)
+{
+        mm->context.id = asid;
+        cpumask_copy(mm_cpumask(mm), cpumask_of(smp_processor_id()));
+}
+
+#endif
+
 void __new_context(struct mm_struct *mm)
 {
         unsigned int asid;
 
         spin_lock(&cpu_asid_lock);
+#ifdef CONFIG_SMP
+        /*
+         * Check the ASID again, in case the change was broadcast from
+         * another CPU before we acquired the lock.
+         */
+        if (unlikely(((mm->context.id ^ cpu_last_asid) >> ASID_BITS) == 0)) {
+                cpumask_set_cpu(smp_processor_id(), mm_cpumask(mm));
+                spin_unlock(&cpu_asid_lock);
+                return;
+        }
+#endif
+        /*
+         * At this point, it is guaranteed that the current mm (with
+         * an old ASID) isn't active on any other CPU since the ASIDs
+         * are changed simultaneously via IPI.
+         */
         asid = ++cpu_last_asid;
         if (asid == 0)
                 asid = cpu_last_asid = ASID_FIRST_VERSION;
@@ -42,20 +143,15 @@ void __new_context(struct mm_struct *mm)
          * to start a new version and flush the TLB.
          */
         if (unlikely((asid & ~ASID_MASK) == 0)) {
-                asid = ++cpu_last_asid;
-                /* set the reserved ASID before flushing the TLB */
-                asm("mcr        p15, 0, %0, c13, c0, 1  @ set reserved context ID\n"
-                    :
-                    : "r" (0));
-                isb();
-                flush_tlb_all();
-                if (icache_is_vivt_asid_tagged()) {
-                        __flush_icache_all();
-                        dsb();
-                }
+                asid = cpu_last_asid + smp_processor_id() + 1;
+                flush_context();
+#ifdef CONFIG_SMP
+                smp_wmb();
+                smp_call_function(reset_context, NULL, 1);
+#endif
+                cpu_last_asid += NR_CPUS;
         }
-        spin_unlock(&cpu_asid_lock);
 
-        cpumask_copy(mm_cpumask(mm), cpumask_of(smp_processor_id()));
-        mm->context.id = asid;
+        set_mm_context(mm, asid);
+        spin_unlock(&cpu_asid_lock);
 }
diff --git a/arch/arm/mm/copypage-feroceon.c b/arch/arm/mm/copypage-feroceon.c
index 70997d5bee2d..5eb4fd93893d 100644
--- a/arch/arm/mm/copypage-feroceon.c
+++ b/arch/arm/mm/copypage-feroceon.c
@@ -68,12 +68,13 @@ feroceon_copy_user_page(void *kto, const void *kfrom)
 }
 
 void feroceon_copy_user_highpage(struct page *to, struct page *from,
-        unsigned long vaddr)
+        unsigned long vaddr, struct vm_area_struct *vma)
 {
         void *kto, *kfrom;
 
         kto = kmap_atomic(to, KM_USER0);
         kfrom = kmap_atomic(from, KM_USER1);
+        flush_cache_page(vma, vaddr, page_to_pfn(from));
         feroceon_copy_user_page(kto, kfrom);
         kunmap_atomic(kfrom, KM_USER1);
         kunmap_atomic(kto, KM_USER0);
diff --git a/arch/arm/mm/copypage-v3.c b/arch/arm/mm/copypage-v3.c
index de9c06854ad7..f72303e1d804 100644
--- a/arch/arm/mm/copypage-v3.c
+++ b/arch/arm/mm/copypage-v3.c
@@ -38,7 +38,7 @@ v3_copy_user_page(void *kto, const void *kfrom)
 }
 
 void v3_copy_user_highpage(struct page *to, struct page *from,
-        unsigned long vaddr)
+        unsigned long vaddr, struct vm_area_struct *vma)
 {
         void *kto, *kfrom;
 
diff --git a/arch/arm/mm/copypage-v4mc.c b/arch/arm/mm/copypage-v4mc.c
index 7370a7142b04..598c51ad5071 100644
--- a/arch/arm/mm/copypage-v4mc.c
+++ b/arch/arm/mm/copypage-v4mc.c
@@ -69,7 +69,7 @@ mc_copy_user_page(void *from, void *to)
 }
 
 void v4_mc_copy_user_highpage(struct page *to, struct page *from,
-        unsigned long vaddr)
+        unsigned long vaddr, struct vm_area_struct *vma)
 {
         void *kto = kmap_atomic(to, KM_USER1);
 
diff --git a/arch/arm/mm/copypage-v4wb.c b/arch/arm/mm/copypage-v4wb.c
index 9ab098414227..7c2eb55cd4a9 100644
--- a/arch/arm/mm/copypage-v4wb.c
+++ b/arch/arm/mm/copypage-v4wb.c
@@ -48,12 +48,13 @@ v4wb_copy_user_page(void *kto, const void *kfrom)
 }
 
 void v4wb_copy_user_highpage(struct page *to, struct page *from,
-        unsigned long vaddr)
+        unsigned long vaddr, struct vm_area_struct *vma)
 {
         void *kto, *kfrom;
 
         kto = kmap_atomic(to, KM_USER0);
         kfrom = kmap_atomic(from, KM_USER1);
+        flush_cache_page(vma, vaddr, page_to_pfn(from));
         v4wb_copy_user_page(kto, kfrom);
         kunmap_atomic(kfrom, KM_USER1);
         kunmap_atomic(kto, KM_USER0);
diff --git a/arch/arm/mm/copypage-v4wt.c b/arch/arm/mm/copypage-v4wt.c
index 300efafd6643..172e6a55458e 100644
--- a/arch/arm/mm/copypage-v4wt.c
+++ b/arch/arm/mm/copypage-v4wt.c
@@ -44,7 +44,7 @@ v4wt_copy_user_page(void *kto, const void *kfrom)
 }
 
 void v4wt_copy_user_highpage(struct page *to, struct page *from,
-        unsigned long vaddr)
+        unsigned long vaddr, struct vm_area_struct *vma)
 {
         void *kto, *kfrom;
 
diff --git a/arch/arm/mm/copypage-v6.c b/arch/arm/mm/copypage-v6.c
index 4127a7bddfe5..f55fa1044f72 100644
--- a/arch/arm/mm/copypage-v6.c
+++ b/arch/arm/mm/copypage-v6.c
@@ -34,13 +34,14 @@ static DEFINE_SPINLOCK(v6_lock);
  * attack the kernel's existing mapping of these pages.
  */
 static void v6_copy_user_highpage_nonaliasing(struct page *to,
-        struct page *from, unsigned long vaddr)
+        struct page *from, unsigned long vaddr, struct vm_area_struct *vma)
 {
         void *kto, *kfrom;
 
         kfrom = kmap_atomic(from, KM_USER0);
         kto = kmap_atomic(to, KM_USER1);
         copy_page(kto, kfrom);
+        __cpuc_flush_dcache_area(kto, PAGE_SIZE);
         kunmap_atomic(kto, KM_USER1);
         kunmap_atomic(kfrom, KM_USER0);
 }
@@ -73,7 +74,7 @@ static void discard_old_kernel_data(void *kto)
  * Copy the page, taking account of the cache colour.
  */
 static void v6_copy_user_highpage_aliasing(struct page *to,
-        struct page *from, unsigned long vaddr)
+        struct page *from, unsigned long vaddr, struct vm_area_struct *vma)
 {
         unsigned int offset = CACHE_COLOUR(vaddr);
         unsigned long kfrom, kto;
diff --git a/arch/arm/mm/copypage-xsc3.c b/arch/arm/mm/copypage-xsc3.c
index bc4525f5ab23..747ad4140fc7 100644
--- a/arch/arm/mm/copypage-xsc3.c
+++ b/arch/arm/mm/copypage-xsc3.c
@@ -71,12 +71,13 @@ xsc3_mc_copy_user_page(void *kto, const void *kfrom)
 }
 
 void xsc3_mc_copy_user_highpage(struct page *to, struct page *from,
-        unsigned long vaddr)
+        unsigned long vaddr, struct vm_area_struct *vma)
 {
         void *kto, *kfrom;
 
         kto = kmap_atomic(to, KM_USER0);
         kfrom = kmap_atomic(from, KM_USER1);
+        flush_cache_page(vma, vaddr, page_to_pfn(from));
         xsc3_mc_copy_user_page(kto, kfrom);
         kunmap_atomic(kfrom, KM_USER1);
         kunmap_atomic(kto, KM_USER0);
diff --git a/arch/arm/mm/copypage-xscale.c b/arch/arm/mm/copypage-xscale.c
index 76824d3e966a..9920c0ae2096 100644
--- a/arch/arm/mm/copypage-xscale.c
+++ b/arch/arm/mm/copypage-xscale.c
@@ -91,7 +91,7 @@ mc_copy_user_page(void *from, void *to)
 }
 
 void xscale_mc_copy_user_highpage(struct page *to, struct page *from,
-        unsigned long vaddr)
+        unsigned long vaddr, struct vm_area_struct *vma)
 {
         void *kto = kmap_atomic(to, KM_USER1);
 
diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c
index b9590a7085ca..13fa536d82e6 100644
--- a/arch/arm/mm/dma-mapping.c
+++ b/arch/arm/mm/dma-mapping.c
@@ -11,7 +11,7 @@
  */
 #include <linux/module.h>
 #include <linux/mm.h>
-#include <linux/slab.h>
+#include <linux/gfp.h>
 #include <linux/errno.h>
 #include <linux/list.h>
 #include <linux/init.h>
@@ -29,9 +29,6 @@
 #error "CONSISTENT_DMA_SIZE must be multiple of 2MiB"
 #endif
 
-#define CONSISTENT_END  (0xffe00000)
-#define CONSISTENT_BASE (CONSISTENT_END - CONSISTENT_DMA_SIZE)
-
 #define CONSISTENT_OFFSET(x)    (((unsigned long)(x) - CONSISTENT_BASE) >> PAGE_SHIFT)
 #define CONSISTENT_PTE_INDEX(x) (((unsigned long)(x) - CONSISTENT_BASE) >> PGDIR_SHIFT)
 #define NUM_CONSISTENT_PTES (CONSISTENT_DMA_SIZE >> PGDIR_SHIFT)
@@ -63,194 +60,152 @@ static u64 get_coherent_dma_mask(struct device *dev)
         return mask;
 }
 
-#ifdef CONFIG_MMU
 /*
- * These are the page tables (2MB each) covering uncached, DMA consistent allocations
+ * Allocate a DMA buffer for 'dev' of size 'size' using the
+ * specified gfp mask.  Note that 'size' must be page aligned.
  */
-static pte_t *consistent_pte[NUM_CONSISTENT_PTES];
-static DEFINE_SPINLOCK(consistent_lock);
+static struct page *__dma_alloc_buffer(struct device *dev, size_t size, gfp_t gfp)
+{
+        unsigned long order = get_order(size);
+        struct page *page, *p, *e;
+        void *ptr;
+        u64 mask = get_coherent_dma_mask(dev);
 
-/*
- * VM region handling support.
- *
- * This should become something generic, handling VM region allocations for
- * vmalloc and similar (ioremap, module space, etc).
- *
- * I envisage vmalloc()'s supporting vm_struct becoming:
- *
- *  struct vm_struct {
- *    struct vm_region  region;
- *    unsigned long     flags;
- *    struct page       **pages;
- *    unsigned int      nr_pages;
- *    unsigned long     phys_addr;
- *  };
- *
- * get_vm_area() would then call vm_region_alloc with an appropriate
- * struct vm_region head (eg):
- *
- *  struct vm_region vmalloc_head = {
- *      .vm_list        = LIST_HEAD_INIT(vmalloc_head.vm_list),
- *      .vm_start       = VMALLOC_START,
- *      .vm_end         = VMALLOC_END,
- *  };
- *
- * However, vmalloc_head.vm_start is variable (typically, it is dependent on
- * the amount of RAM found at boot time.)  I would imagine that get_vm_area()
- * would have to initialise this each time prior to calling vm_region_alloc().
- */
-struct arm_vm_region {
-        struct list_head        vm_list;
-        unsigned long           vm_start;
-        unsigned long           vm_end;
-        struct page             *vm_pages;
-        int                     vm_active;
-};
+#ifdef CONFIG_DMA_API_DEBUG
+        u64 limit = (mask + 1) & ~mask;
+        if (limit && size >= limit) {
+                dev_warn(dev, "coherent allocation too big (requested %#x mask %#llx)\n",
+                        size, mask);
+                return NULL;
+        }
+#endif
 
-static struct arm_vm_region consistent_head = {
-        .vm_list        = LIST_HEAD_INIT(consistent_head.vm_list),
-        .vm_start       = CONSISTENT_BASE,
-        .vm_end         = CONSISTENT_END,
-};
+        if (!mask)
+                return NULL;
 
-static struct arm_vm_region *
-arm_vm_region_alloc(struct arm_vm_region *head, size_t size, gfp_t gfp)
-{
-        unsigned long addr = head->vm_start, end = head->vm_end - size;
-        unsigned long flags;
-        struct arm_vm_region *c, *new;
-
-        new = kmalloc(sizeof(struct arm_vm_region), gfp);
-        if (!new)
-                goto out;
-
-        spin_lock_irqsave(&consistent_lock, flags);
-
-        list_for_each_entry(c, &head->vm_list, vm_list) {
-                if ((addr + size) < addr)
-                        goto nospc;
-                if ((addr + size) <= c->vm_start)
-                        goto found;
-                addr = c->vm_end;
-                if (addr > end)
-                        goto nospc;
-        }
+        if (mask < 0xffffffffULL)
+                gfp |= GFP_DMA;
+
+        page = alloc_pages(gfp, order);
+        if (!page)
+                return NULL;
 
- found:
         /*
-         * Insert this entry _before_ the one we found.
+         * Now split the huge page and free the excess pages
          */
-        list_add_tail(&new->vm_list, &c->vm_list);
-        new->vm_start = addr;
-        new->vm_end = addr + size;
-        new->vm_active = 1;
-
-        spin_unlock_irqrestore(&consistent_lock, flags);
-        return new;
-
- nospc:
-        spin_unlock_irqrestore(&consistent_lock, flags);
-        kfree(new);
- out:
-        return NULL;
+        split_page(page, order);
+        for (p = page + (size >> PAGE_SHIFT), e = page + (1 << order); p < e; p++)
+                __free_page(p);
+
+        /*
+         * Ensure that the allocated pages are zeroed, and that any data
+         * lurking in the kernel direct-mapped region is invalidated.
+         */
+        ptr = page_address(page);
+        memset(ptr, 0, size);
+        dmac_flush_range(ptr, ptr + size);
+        outer_flush_range(__pa(ptr), __pa(ptr) + size);
+
+        return page;
 }
 
-static struct arm_vm_region *arm_vm_region_find(struct arm_vm_region *head, unsigned long addr)
+/*
+ * Free a DMA buffer.  'size' must be page aligned.
+ */
+static void __dma_free_buffer(struct page *page, size_t size)
 {
-        struct arm_vm_region *c;
-        
-        list_for_each_entry(c, &head->vm_list, vm_list) {
-                if (c->vm_active && c->vm_start == addr)
-                        goto out;
+        struct page *e = page + (size >> PAGE_SHIFT);
+
+        while (page < e) {
+                __free_page(page);
+                page++;
         }
-        c = NULL;
- out:
-        return c;
 }
 
+#ifdef CONFIG_MMU
+/*
+ * These are the page tables (2MB each) covering uncached, DMA consistent allocations
+ */
+static pte_t *consistent_pte[NUM_CONSISTENT_PTES];
+
+#include "vmregion.h"
+
+static struct arm_vmregion_head consistent_head = {
+        .vm_lock        = __SPIN_LOCK_UNLOCKED(&consistent_head.vm_lock),
+        .vm_list        = LIST_HEAD_INIT(consistent_head.vm_list),
+        .vm_start       = CONSISTENT_BASE,
+        .vm_end         = CONSISTENT_END,
+};
+
 #ifdef CONFIG_HUGETLB_PAGE
 #error ARM Coherent DMA allocator does not (yet) support huge TLB
 #endif
 
-static void *
-__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp,
-            pgprot_t prot)
+/*
+ * Initialise the consistent memory allocation.
+ */
+static int __init consistent_init(void)
 {
-        struct page *page;
-        struct arm_vm_region *c;
-        unsigned long order;
-        u64 mask = get_coherent_dma_mask(dev);
-        u64 limit;
+        int ret = 0;
+        pgd_t *pgd;
+        pmd_t *pmd;
+        pte_t *pte;
+        int i = 0;
+        u32 base = CONSISTENT_BASE;
 
-        if (!consistent_pte[0]) {
-                printk(KERN_ERR "%s: not initialised\n", __func__);
-                dump_stack();
-                return NULL;
-        }
+        do {
+                pgd = pgd_offset(&init_mm, base);
+                pmd = pmd_alloc(&init_mm, pgd, base);
+                if (!pmd) {
+                        printk(KERN_ERR "%s: no pmd tables\n", __func__);
+                        ret = -ENOMEM;
+                        break;
+                }
+                WARN_ON(!pmd_none(*pmd));
 
-        if (!mask)
-                goto no_page;
+                pte = pte_alloc_kernel(pmd, base);
+                if (!pte) {
+                        printk(KERN_ERR "%s: no pte tables\n", __func__);
+                        ret = -ENOMEM;
+                        break;
+                }
 
-        /*
-         * Sanity check the allocation size.
-         */
-        size = PAGE_ALIGN(size);
-        limit = (mask + 1) & ~mask;
-        if ((limit && size >= limit) ||
-            size >= (CONSISTENT_END - CONSISTENT_BASE)) {
-                printk(KERN_WARNING "coherent allocation too big "
-                       "(requested %#x mask %#llx)\n", size, mask);
-                goto no_page;
-        }
+                consistent_pte[i++] = pte;
+                base += (1 << PGDIR_SHIFT);
+        } while (base < CONSISTENT_END);
 
-        order = get_order(size);
+        return ret;
+}
 
-        if (mask < 0xffffffffULL)
-                gfp |= GFP_DMA;
+core_initcall(consistent_init);
 
-        page = alloc_pages(gfp, order);
-        if (!page)
-                goto no_page;
+static void *
+__dma_alloc_remap(struct page *page, size_t size, gfp_t gfp, pgprot_t prot)
+{
+        struct arm_vmregion *c;
 
-        /*
-         * Invalidate any data that might be lurking in the
-         * kernel direct-mapped region for device DMA.
-         */
-        {
-                void *ptr = page_address(page);
-                memset(ptr, 0, size);
-                dmac_flush_range(ptr, ptr + size);
-                outer_flush_range(__pa(ptr), __pa(ptr) + size);
+        if (!consistent_pte[0]) {
+                printk(KERN_ERR "%s: not initialised\n", __func__);
+                dump_stack();
+                return NULL;
         }
 
         /*
          * Allocate a virtual address in the consistent mapping region.
          */
-        c = arm_vm_region_alloc(&consistent_head, size,
+        c = arm_vmregion_alloc(&consistent_head, size,
                             gfp & ~(__GFP_DMA | __GFP_HIGHMEM));
         if (c) {
                 pte_t *pte;
-                struct page *end = page + (1 << order);
                 int idx = CONSISTENT_PTE_INDEX(c->vm_start);
                 u32 off = CONSISTENT_OFFSET(c->vm_start) & (PTRS_PER_PTE-1);
 
                 pte = consistent_pte[idx] + off;
                 c->vm_pages = page;
 
-                split_page(page, order);
-
-                /*
-                 * Set the "dma handle"
-                 */
-                *handle = page_to_dma(dev, page);
-
                 do {
                         BUG_ON(!pte_none(*pte));
 
-                        /*
-                         * x86 does not mark the pages reserved...
-                         */
-                        SetPageReserved(page);
                         set_pte_ext(pte, mk_pte(page, prot), 0);
                         page++;
                         pte++;
@@ -261,48 +216,90 @@ __dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp,
                         }
                 } while (size -= PAGE_SIZE);
 
-                /*
-                 * Free the otherwise unused pages.
-                 */
-                while (page < end) {
-                        __free_page(page);
-                        page++;
-                }
-
                 return (void *)c->vm_start;
         }
-
-        if (page)
-                __free_pages(page, order);
- no_page:
-        *handle = ~0;
         return NULL;
 }
+
+static void __dma_free_remap(void *cpu_addr, size_t size)
+{
+        struct arm_vmregion *c;
+        unsigned long addr;
+        pte_t *ptep;
+        int idx;
+        u32 off;
+
+        c = arm_vmregion_find_remove(&consistent_head, (unsigned long)cpu_addr);
+        if (!c) {
+                printk(KERN_ERR "%s: trying to free invalid coherent area: %p\n",
+                       __func__, cpu_addr);
+                dump_stack();
+                return;
+        }
+
+        if ((c->vm_end - c->vm_start) != size) {
+                printk(KERN_ERR "%s: freeing wrong coherent size (%ld != %d)\n",
+                       __func__, c->vm_end - c->vm_start, size);
+                dump_stack();
+                size = c->vm_end - c->vm_start;
+        }
+
+        idx = CONSISTENT_PTE_INDEX(c->vm_start);
+        off = CONSISTENT_OFFSET(c->vm_start) & (PTRS_PER_PTE-1);
+        ptep = consistent_pte[idx] + off;
+        addr = c->vm_start;
+        do {
+                pte_t pte = ptep_get_and_clear(&init_mm, addr, ptep);
+
+                ptep++;
+                addr += PAGE_SIZE;
+                off++;
+                if (off >= PTRS_PER_PTE) {
+                        off = 0;
+                        ptep = consistent_pte[++idx];
+                }
+
+                if (pte_none(pte) || !pte_present(pte))
+                        printk(KERN_CRIT "%s: bad page in kernel page table\n",
+                               __func__);
+        } while (size -= PAGE_SIZE);
+
+        flush_tlb_kernel_range(c->vm_start, c->vm_end);
+
+        arm_vmregion_free(&consistent_head, c);
+}
+
 #else   /* !CONFIG_MMU */
+
+#define __dma_alloc_remap(page, size, gfp, prot)        page_address(page)
+#define __dma_free_remap(addr, size)                    do { } while (0)
+
+#endif  /* CONFIG_MMU */
+
 static void *
 __dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp,
             pgprot_t prot)
 {
-        void *virt;
-        u64 mask = get_coherent_dma_mask(dev);
+        struct page *page;
+        void *addr;
 
-        if (!mask)
-                goto error;
+        *handle = ~0;
+        size = PAGE_ALIGN(size);
 
-        if (mask < 0xffffffffULL)
-                gfp |= GFP_DMA;
-        virt = kmalloc(size, gfp);
-        if (!virt)
-                goto error;
+        page = __dma_alloc_buffer(dev, size, gfp);
+        if (!page)
+                return NULL;
 
-        *handle =  virt_to_dma(dev, virt);
-        return virt;
+        if (!arch_is_coherent())
+                addr = __dma_alloc_remap(page, size, gfp, prot);
+        else
+                addr = page_address(page);
 
-error:
-        *handle = ~0;
-        return NULL;
+        if (addr)
+                *handle = page_to_dma(dev, page);
+
+        return addr;
 }
-#endif  /* CONFIG_MMU */
 
 /*
  * Allocate DMA-coherent memory space and return both the kernel remapped
@@ -316,19 +313,8 @@ dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gf
         if (dma_alloc_from_coherent(dev, size, handle, &memory))
                 return memory;
 
-        if (arch_is_coherent()) {
-                void *virt;
-
-                virt = kmalloc(size, gfp);
-                if (!virt)
-                        return NULL;
-                *handle =  virt_to_dma(dev, virt);
-
-                return virt;
-        }
-
         return __dma_alloc(dev, size, handle, gfp,
-                           pgprot_noncached(pgprot_kernel));
+                           pgprot_dmacoherent(pgprot_kernel));
 }
 EXPORT_SYMBOL(dma_alloc_coherent);
 
@@ -349,15 +335,12 @@ static int dma_mmap(struct device *dev, struct vm_area_struct *vma,
 {
         int ret = -ENXIO;
 #ifdef CONFIG_MMU
-        unsigned long flags, user_size, kern_size;
-        struct arm_vm_region *c;
+        unsigned long user_size, kern_size;
+        struct arm_vmregion *c;
 
         user_size = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
 
-        spin_lock_irqsave(&consistent_lock, flags);
-        c = arm_vm_region_find(&consistent_head, (unsigned long)cpu_addr);
-        spin_unlock_irqrestore(&consistent_lock, flags);
-
+        c = arm_vmregion_find(&consistent_head, (unsigned long)cpu_addr);
         if (c) {
                 unsigned long off = vma->vm_pgoff;
 
@@ -379,7 +362,7 @@ static int dma_mmap(struct device *dev, struct vm_area_struct *vma,
 int dma_mmap_coherent(struct device *dev, struct vm_area_struct *vma,
                       void *cpu_addr, dma_addr_t dma_addr, size_t size)
 {
-        vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+        vma->vm_page_prot = pgprot_dmacoherent(vma->vm_page_prot);
         return dma_mmap(dev, vma, cpu_addr, dma_addr, size);
 }
 EXPORT_SYMBOL(dma_mmap_coherent);
@@ -396,221 +379,66 @@ EXPORT_SYMBOL(dma_mmap_writecombine);
  * free a page as defined by the above mapping.
  * Must not be called with IRQs disabled.
  */
-#ifdef CONFIG_MMU
 void dma_free_coherent(struct device *dev, size_t size, void *cpu_addr, dma_addr_t handle)
 {
-        struct arm_vm_region *c;
-        unsigned long flags, addr;
-        pte_t *ptep;
-        int idx;
-        u32 off;
-
         WARN_ON(irqs_disabled());
 
         if (dma_release_from_coherent(dev, get_order(size), cpu_addr))
                 return;
 
-        if (arch_is_coherent()) {
-                kfree(cpu_addr);
-                return;
-        }
-
         size = PAGE_ALIGN(size);
 
-        spin_lock_irqsave(&consistent_lock, flags);
-        c = arm_vm_region_find(&consistent_head, (unsigned long)cpu_addr);
-        if (!c)
-                goto no_area;
-
-        c->vm_active = 0;
-        spin_unlock_irqrestore(&consistent_lock, flags);
-
-        if ((c->vm_end - c->vm_start) != size) {
-                printk(KERN_ERR "%s: freeing wrong coherent size (%ld != %d)\n",
-                       __func__, c->vm_end - c->vm_start, size);
-                dump_stack();
-                size = c->vm_end - c->vm_start;
-        }
-
-        idx = CONSISTENT_PTE_INDEX(c->vm_start);
-        off = CONSISTENT_OFFSET(c->vm_start) & (PTRS_PER_PTE-1);
-        ptep = consistent_pte[idx] + off;
-        addr = c->vm_start;
-        do {
-                pte_t pte = ptep_get_and_clear(&init_mm, addr, ptep);
-                unsigned long pfn;
-
-                ptep++;
-                addr += PAGE_SIZE;
-                off++;
-                if (off >= PTRS_PER_PTE) {
-                        off = 0;
-                        ptep = consistent_pte[++idx];
-                }
-
-                if (!pte_none(pte) && pte_present(pte)) {
-                        pfn = pte_pfn(pte);
-
-                        if (pfn_valid(pfn)) {
-                                struct page *page = pfn_to_page(pfn);
-
-                                /*
-                                 * x86 does not mark the pages reserved...
-                                 */
-                                ClearPageReserved(page);
-
-                                __free_page(page);
-                                continue;
-                        }
-                }
-
-                printk(KERN_CRIT "%s: bad page in kernel page table\n",
-                       __func__);
-        } while (size -= PAGE_SIZE);
-
-        flush_tlb_kernel_range(c->vm_start, c->vm_end);
+        if (!arch_is_coherent())
+                __dma_free_remap(cpu_addr, size);
 
-        spin_lock_irqsave(&consistent_lock, flags);
-        list_del(&c->vm_list);
-        spin_unlock_irqrestore(&consistent_lock, flags);
-
-        kfree(c);
-        return;
-
- no_area:
-        spin_unlock_irqrestore(&consistent_lock, flags);
-        printk(KERN_ERR "%s: trying to free invalid coherent area: %p\n",
-               __func__, cpu_addr);
-        dump_stack();
-}
-#else   /* !CONFIG_MMU */
-void dma_free_coherent(struct device *dev, size_t size, void *cpu_addr, dma_addr_t handle)
-{
-        if (dma_release_from_coherent(dev, get_order(size), cpu_addr))
-                return;
-        kfree(cpu_addr);
+        __dma_free_buffer(dma_to_page(dev, handle), size);
 }
-#endif  /* CONFIG_MMU */
 EXPORT_SYMBOL(dma_free_coherent);
 
 /*
- * Initialise the consistent memory allocation.
- */
-static int __init consistent_init(void)
-{
-        int ret = 0;
-#ifdef CONFIG_MMU
-        pgd_t *pgd;
-        pmd_t *pmd;
-        pte_t *pte;
-        int i = 0;
-        u32 base = CONSISTENT_BASE;
-
-        do {
-                pgd = pgd_offset(&init_mm, base);
-                pmd = pmd_alloc(&init_mm, pgd, base);
-                if (!pmd) {
-                        printk(KERN_ERR "%s: no pmd tables\n", __func__);
-                        ret = -ENOMEM;
-                        break;
-                }
-                WARN_ON(!pmd_none(*pmd));
-
-                pte = pte_alloc_kernel(pmd, base);
-                if (!pte) {
-                        printk(KERN_ERR "%s: no pte tables\n", __func__);
-                        ret = -ENOMEM;
-                        break;
-                }
-
-                consistent_pte[i++] = pte;
-                base += (1 << PGDIR_SHIFT);
-        } while (base < CONSISTENT_END);
-#endif  /* !CONFIG_MMU */
-
-        return ret;
-}
-
-core_initcall(consistent_init);
-
-/*
  * Make an area consistent for devices.
  * Note: Drivers should NOT use this function directly, as it will break
  * platforms with CONFIG_DMABOUNCE.
  * Use the driver DMA support - see dma-mapping.h (dma_sync_*)
  */
-void dma_cache_maint(const void *start, size_t size, int direction)
+void ___dma_single_cpu_to_dev(const void *kaddr, size_t size,
+        enum dma_data_direction dir)
 {
-        void (*inner_op)(const void *, const void *);
-        void (*outer_op)(unsigned long, unsigned long);
-
-        BUG_ON(!virt_addr_valid(start) || !virt_addr_valid(start + size - 1));
-
-        switch (direction) {
-        case DMA_FROM_DEVICE:           /* invalidate only */
-                inner_op = dmac_inv_range;
-                outer_op = outer_inv_range;
-                break;
-        case DMA_TO_DEVICE:             /* writeback only */
-                inner_op = dmac_clean_range;
-                outer_op = outer_clean_range;
-                break;
-        case DMA_BIDIRECTIONAL:         /* writeback and invalidate */
-                inner_op = dmac_flush_range;
-                outer_op = outer_flush_range;
-                break;
-        default:
-                BUG();
-        }
+        unsigned long paddr;
+
+        BUG_ON(!virt_addr_valid(kaddr) || !virt_addr_valid(kaddr + size - 1));
+
+        dmac_map_area(kaddr, size, dir);
 
-        inner_op(start, start + size);
-        outer_op(__pa(start), __pa(start) + size);
+        paddr = __pa(kaddr);
+        if (dir == DMA_FROM_DEVICE) {
+                outer_inv_range(paddr, paddr + size);
+        } else {
+                outer_clean_range(paddr, paddr + size);
+        }
+        /* FIXME: non-speculating: flush on bidirectional mappings? */
 }
-EXPORT_SYMBOL(dma_cache_maint);
+EXPORT_SYMBOL(___dma_single_cpu_to_dev);
 
-static void dma_cache_maint_contiguous(struct page *page, unsigned long offset,
-                                       size_t size, int direction)
+void ___dma_single_dev_to_cpu(const void *kaddr, size_t size,
+        enum dma_data_direction dir)
 {
-        void *vaddr;
-        unsigned long paddr;
-        void (*inner_op)(const void *, const void *);
-        void (*outer_op)(unsigned long, unsigned long);
-
-        switch (direction) {
-        case DMA_FROM_DEVICE:           /* invalidate only */
-                inner_op = dmac_inv_range;
-                outer_op = outer_inv_range;
-                break;
-        case DMA_TO_DEVICE:             /* writeback only */
-                inner_op = dmac_clean_range;
-                outer_op = outer_clean_range;
-                break;
-        case DMA_BIDIRECTIONAL:         /* writeback and invalidate */
-                inner_op = dmac_flush_range;
-                outer_op = outer_flush_range;
-                break;
-        default:
-                BUG();
-        }
+        BUG_ON(!virt_addr_valid(kaddr) || !virt_addr_valid(kaddr + size - 1));
 
-        if (!PageHighMem(page)) {
-                vaddr = page_address(page) + offset;
-                inner_op(vaddr, vaddr + size);
-        } else {
-                vaddr = kmap_high_get(page);
-                if (vaddr) {
-                        vaddr += offset;
-                        inner_op(vaddr, vaddr + size);
-                        kunmap_high(page);
-                }
+        /* FIXME: non-speculating: not required */
+        /* don't bother invalidating if DMA to device */
+        if (dir != DMA_TO_DEVICE) {
+                unsigned long paddr = __pa(kaddr);
+                outer_inv_range(paddr, paddr + size);
         }
 
-        paddr = page_to_phys(page) + offset;
-        outer_op(paddr, paddr + size);
+        dmac_unmap_area(kaddr, size, dir);
 }
+EXPORT_SYMBOL(___dma_single_dev_to_cpu);
 
-void dma_cache_maint_page(struct page *page, unsigned long offset,
-                          size_t size, int dir)
+static void dma_cache_maint_page(struct page *page, unsigned long offset,
+        size_t size, enum dma_data_direction dir,
+        void (*op)(const void *, size_t, int))
 {
         /*
          * A single sg entry may refer to multiple physically contiguous
@@ -621,20 +449,67 @@ void dma_cache_maint_page(struct page *page, unsigned long offset,
         size_t left = size;
         do {
                 size_t len = left;
-                if (PageHighMem(page) && len + offset > PAGE_SIZE) {
-                        if (offset >= PAGE_SIZE) {
-                                page += offset / PAGE_SIZE;
-                                offset %= PAGE_SIZE;
+                void *vaddr;
+
+                if (PageHighMem(page)) {
+                        if (len + offset > PAGE_SIZE) {
+                                if (offset >= PAGE_SIZE) {
+                                        page += offset / PAGE_SIZE;
+                                        offset %= PAGE_SIZE;
+                                }
+                                len = PAGE_SIZE - offset;
+                        }
+                        vaddr = kmap_high_get(page);
+                        if (vaddr) {
+                                vaddr += offset;
+                                op(vaddr, len, dir);
+                                kunmap_high(page);
+                        } else if (cache_is_vipt()) {
+                                pte_t saved_pte;
+                                vaddr = kmap_high_l1_vipt(page, &saved_pte);
+                                op(vaddr + offset, len, dir);
+                                kunmap_high_l1_vipt(page, saved_pte);
                         }
-                        len = PAGE_SIZE - offset;
+                } else {
+                        vaddr = page_address(page) + offset;
+                        op(vaddr, len, dir);
                 }
-                dma_cache_maint_contiguous(page, offset, len, dir);
                 offset = 0;
                 page++;
                 left -= len;
         } while (left);
 }
-EXPORT_SYMBOL(dma_cache_maint_page);
+
+void ___dma_page_cpu_to_dev(struct page *page, unsigned long off,
+        size_t size, enum dma_data_direction dir)
+{
+        unsigned long paddr;
+
+        dma_cache_maint_page(page, off, size, dir, dmac_map_area);
+
+        paddr = page_to_phys(page) + off;
+        if (dir == DMA_FROM_DEVICE) {
+                outer_inv_range(paddr, paddr + size);
+        } else {
+                outer_clean_range(paddr, paddr + size);
+        }
+        /* FIXME: non-speculating: flush on bidirectional mappings? */
+}
+EXPORT_SYMBOL(___dma_page_cpu_to_dev);
+
+void ___dma_page_dev_to_cpu(struct page *page, unsigned long off,
+        size_t size, enum dma_data_direction dir)
+{
+        unsigned long paddr = page_to_phys(page) + off;
+
+        /* FIXME: non-speculating: not required */
+        /* don't bother invalidating if DMA to device */
+        if (dir != DMA_TO_DEVICE)
+                outer_inv_range(paddr, paddr + size);
+
+        dma_cache_maint_page(page, off, size, dir, dmac_unmap_area);
+}
+EXPORT_SYMBOL(___dma_page_dev_to_cpu);
 
 /**
  * dma_map_sg - map a set of SG buffers for streaming mode DMA
@@ -708,8 +583,12 @@ void dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
         int i;
 
         for_each_sg(sg, s, nents, i) {
-                dmabounce_sync_for_cpu(dev, sg_dma_address(s), 0,
-                                        sg_dma_len(s), dir);
+                if (!dmabounce_sync_for_cpu(dev, sg_dma_address(s), 0,
+                                            sg_dma_len(s), dir))
+                        continue;
+
+                __dma_page_dev_to_cpu(sg_page(s), s->offset,
+                                      s->length, dir);
         }
 }
 EXPORT_SYMBOL(dma_sync_sg_for_cpu);
@@ -732,9 +611,8 @@ void dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
                                         sg_dma_len(s), dir))
                         continue;
 
-                if (!arch_is_coherent())
-                        dma_cache_maint_page(sg_page(s), s->offset,
-                                             s->length, dir);
+                __dma_page_cpu_to_dev(sg_page(s), s->offset,
+                                      s->length, dir);
         }
 }
 EXPORT_SYMBOL(dma_sync_sg_for_device);
diff --git a/arch/arm/mm/fault-armv.c b/arch/arm/mm/fault-armv.c
index d0d17b6a3703..0d414c28eb2c 100644
--- a/arch/arm/mm/fault-armv.c
+++ b/arch/arm/mm/fault-armv.c
@@ -16,6 +16,7 @@
 #include <linux/vmalloc.h>
 #include <linux/init.h>
 #include <linux/pagemap.h>
+#include <linux/gfp.h>
 
 #include <asm/bugs.h>
 #include <asm/cacheflush.h>
@@ -23,6 +24,8 @@
 #include <asm/pgtable.h>
 #include <asm/tlbflush.h>
 
+#include "mm.h"
+
 static unsigned long shared_pte_mask = L_PTE_MT_BUFFERABLE;
 
 /*
@@ -34,28 +37,12 @@ static unsigned long shared_pte_mask = L_PTE_MT_BUFFERABLE;
  * Therefore those configurations which might call adjust_pte (those
  * without CONFIG_CPU_CACHE_VIPT) cannot support split page_table_lock.
  */
-static int adjust_pte(struct vm_area_struct *vma, unsigned long address)
+static int do_adjust_pte(struct vm_area_struct *vma, unsigned long address,
+        unsigned long pfn, pte_t *ptep)
 {
-        pgd_t *pgd;
-        pmd_t *pmd;
-        pte_t *pte, entry;
+        pte_t entry = *ptep;
         int ret;
 
-        pgd = pgd_offset(vma->vm_mm, address);
-        if (pgd_none(*pgd))
-                goto no_pgd;
-        if (pgd_bad(*pgd))
-                goto bad_pgd;
-
-        pmd = pmd_offset(pgd, address);
-        if (pmd_none(*pmd))
-                goto no_pmd;
-        if (pmd_bad(*pmd))
-                goto bad_pmd;
-
-        pte = pte_offset_map(pmd, address);
-        entry = *pte;
-
         /*
          * If this page is present, it's actually being shared.
          */
@@ -66,33 +53,55 @@ static int adjust_pte(struct vm_area_struct *vma, unsigned long address)
          * fault (ie, is old), we can safely ignore any issues.
          */
         if (ret && (pte_val(entry) & L_PTE_MT_MASK) != shared_pte_mask) {
-                unsigned long pfn = pte_pfn(entry);
                 flush_cache_page(vma, address, pfn);
                 outer_flush_range((pfn << PAGE_SHIFT),
                                   (pfn << PAGE_SHIFT) + PAGE_SIZE);
                 pte_val(entry) &= ~L_PTE_MT_MASK;
                 pte_val(entry) |= shared_pte_mask;
-                set_pte_at(vma->vm_mm, address, pte, entry);
+                set_pte_at(vma->vm_mm, address, ptep, entry);
                 flush_tlb_page(vma, address);
         }
-        pte_unmap(pte);
+
         return ret;
+}
+
+static int adjust_pte(struct vm_area_struct *vma, unsigned long address,
+        unsigned long pfn)
+{
+        spinlock_t *ptl;
+        pgd_t *pgd;
+        pmd_t *pmd;
+        pte_t *pte;
+        int ret;
 
-bad_pgd:
-        pgd_ERROR(*pgd);
-        pgd_clear(pgd);
-no_pgd:
-        return 0;
-
-bad_pmd:
-        pmd_ERROR(*pmd);
-        pmd_clear(pmd);
-no_pmd:
-        return 0;
+        pgd = pgd_offset(vma->vm_mm, address);
+        if (pgd_none_or_clear_bad(pgd))
+                return 0;
+
+        pmd = pmd_offset(pgd, address);
+        if (pmd_none_or_clear_bad(pmd))
+                return 0;
+
+        /*
+         * This is called while another page table is mapped, so we
+         * must use the nested version.  This also means we need to
+         * open-code the spin-locking.
+         */
+        ptl = pte_lockptr(vma->vm_mm, pmd);
+        pte = pte_offset_map_nested(pmd, address);
+        spin_lock(ptl);
+
+        ret = do_adjust_pte(vma, address, pfn, pte);
+
+        spin_unlock(ptl);
+        pte_unmap_nested(pte);
+
+        return ret;
 }
 
 static void
-make_coherent(struct address_space *mapping, struct vm_area_struct *vma, unsigned long addr, unsigned long pfn)
+make_coherent(struct address_space *mapping, struct vm_area_struct *vma,
+        unsigned long addr, pte_t *ptep, unsigned long pfn)
 {
         struct mm_struct *mm = vma->vm_mm;
         struct vm_area_struct *mpnt;
@@ -120,11 +129,11 @@ make_coherent(struct address_space *mapping, struct vm_area_struct *vma, unsigne
                 if (!(mpnt->vm_flags & VM_MAYSHARE))
                         continue;
                 offset = (pgoff - mpnt->vm_pgoff) << PAGE_SHIFT;
-                aliases += adjust_pte(mpnt, mpnt->vm_start + offset);
+                aliases += adjust_pte(mpnt, mpnt->vm_start + offset, pfn);
         }
         flush_dcache_mmap_unlock(mapping);
         if (aliases)
-                adjust_pte(vma, addr);
+                do_adjust_pte(vma, addr, pfn, ptep);
         else
                 flush_cache_page(vma, addr, pfn);
 }
@@ -142,16 +151,24 @@ make_coherent(struct address_space *mapping, struct vm_area_struct *vma, unsigne
  *
  * Note that the pte lock will be held.
  */
-void update_mmu_cache(struct vm_area_struct *vma, unsigned long addr, pte_t pte)
+void update_mmu_cache(struct vm_area_struct *vma, unsigned long addr,
+        pte_t *ptep)
 {
-        unsigned long pfn = pte_pfn(pte);
+        unsigned long pfn = pte_pfn(*ptep);
         struct address_space *mapping;
         struct page *page;
 
         if (!pfn_valid(pfn))
                 return;
 
+        /*
+         * The zero page is never written to, so never has any dirty
+         * cache lines, and therefore never needs to be flushed.
+         */
         page = pfn_to_page(pfn);
+        if (page == ZERO_PAGE(0))
+                return;
+
         mapping = page_mapping(page);
 #ifndef CONFIG_SMP
         if (test_and_clear_bit(PG_dcache_dirty, &page->flags))
@@ -159,7 +176,7 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long addr, pte_t pte)
 #endif
         if (mapping) {
                 if (cache_is_vivt())
-                        make_coherent(mapping, vma, addr, pfn);
+                        make_coherent(mapping, vma, addr, ptep, pfn);
                 else if (vma->vm_flags & VM_EXEC)
                         __flush_icache_all();
         }
@@ -198,9 +215,8 @@ void __init check_writebuffer_bugs(void)
         page = alloc_page(GFP_KERNEL);
         if (page) {
                 unsigned long *p1, *p2;
-                pgprot_t prot = __pgprot(L_PTE_PRESENT|L_PTE_YOUNG|
-                                         L_PTE_DIRTY|L_PTE_WRITE|
-                                         L_PTE_MT_BUFFERABLE);
+                pgprot_t prot = __pgprot_modify(PAGE_KERNEL,
+                                        L_PTE_MT_MASK, L_PTE_MT_BUFFERABLE);
 
                 p1 = vmap(&page, 1, VM_IOREMAP, prot);
                 p2 = vmap(&page, 1, VM_IOREMAP, prot);
diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c
index 10e06801afb3..9d40c341e07e 100644
--- a/arch/arm/mm/fault.c
+++ b/arch/arm/mm/fault.c
@@ -18,6 +18,7 @@
 #include <linux/page-flags.h>
 #include <linux/sched.h>
 #include <linux/highmem.h>
+#include <linux/perf_event.h>
 
 #include <asm/system.h>
 #include <asm/pgtable.h>
@@ -302,6 +303,12 @@ do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
         fault = __do_page_fault(mm, addr, fsr, tsk);
         up_read(&mm->mmap_sem);
 
+        perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, 0, regs, addr);
+        if (fault & VM_FAULT_MAJOR)
+                perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, 0, regs, addr);
+        else if (fault & VM_FAULT_MINOR)
+                perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, 0, regs, addr);
+
         /*
          * Handle the "normal" case first - VM_FAULT_MAJOR / VM_FAULT_MINOR
          */
diff --git a/arch/arm/mm/flush.c b/arch/arm/mm/flush.c
index 7f294f307c83..c6844cb9b508 100644
--- a/arch/arm/mm/flush.c
+++ b/arch/arm/mm/flush.c
@@ -13,6 +13,8 @@
 
 #include <asm/cacheflush.h>
 #include <asm/cachetype.h>
+#include <asm/highmem.h>
+#include <asm/smp_plat.h>
 #include <asm/system.h>
 #include <asm/tlbflush.h>
 
@@ -35,14 +37,12 @@ static void flush_pfn_alias(unsigned long pfn, unsigned long vaddr)
             :
             : "r" (to), "r" (to + PAGE_SIZE - L1_CACHE_BYTES), "r" (zero)
             : "cc");
-        __flush_icache_all();
 }
 
 void flush_cache_mm(struct mm_struct *mm)
 {
         if (cache_is_vivt()) {
-                if (cpumask_test_cpu(smp_processor_id(), mm_cpumask(mm)))
-                        __cpuc_flush_user_all();
+                vivt_flush_cache_mm(mm);
                 return;
         }
 
@@ -52,16 +52,13 @@ void flush_cache_mm(struct mm_struct *mm)
                     :
                     : "r" (0)
                     : "cc");
-                __flush_icache_all();
         }
 }
 
 void flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
 {
         if (cache_is_vivt()) {
-                if (cpumask_test_cpu(smp_processor_id(), mm_cpumask(vma->vm_mm)))
-                        __cpuc_flush_user_range(start & PAGE_MASK, PAGE_ALIGN(end),
-                                                vma->vm_flags);
+                vivt_flush_cache_range(vma, start, end);
                 return;
         }
 
@@ -71,27 +68,41 @@ void flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned
                     :
                     : "r" (0)
                     : "cc");
-                __flush_icache_all();
         }
+
+        if (vma->vm_flags & VM_EXEC)
+                __flush_icache_all();
 }
 
 void flush_cache_page(struct vm_area_struct *vma, unsigned long user_addr, unsigned long pfn)
 {
         if (cache_is_vivt()) {
-                if (cpumask_test_cpu(smp_processor_id(), mm_cpumask(vma->vm_mm))) {
-                        unsigned long addr = user_addr & PAGE_MASK;
-                        __cpuc_flush_user_range(addr, addr + PAGE_SIZE, vma->vm_flags);
-                }
+                vivt_flush_cache_page(vma, user_addr, pfn);
                 return;
         }
 
-        if (cache_is_vipt_aliasing())
+        if (cache_is_vipt_aliasing()) {
                 flush_pfn_alias(pfn, user_addr);
+                __flush_icache_all();
+        }
+
+        if (vma->vm_flags & VM_EXEC && icache_is_vivt_asid_tagged())
+                __flush_icache_all();
 }
+#else
+#define flush_pfn_alias(pfn,vaddr)      do { } while (0)
+#endif
 
+#ifdef CONFIG_SMP
+static void flush_ptrace_access_other(void *args)
+{
+        __flush_icache_all();
+}
+#endif
+
+static
 void flush_ptrace_access(struct vm_area_struct *vma, struct page *page,
-                         unsigned long uaddr, void *kaddr,
-                         unsigned long len, int write)
+                         unsigned long uaddr, void *kaddr, unsigned long len)
 {
         if (cache_is_vivt()) {
                 if (cpumask_test_cpu(smp_processor_id(), mm_cpumask(vma->vm_mm))) {
@@ -103,20 +114,42 @@ void flush_ptrace_access(struct vm_area_struct *vma, struct page *page,
 
         if (cache_is_vipt_aliasing()) {
                 flush_pfn_alias(page_to_pfn(page), uaddr);
+                __flush_icache_all();
                 return;
         }
 
         /* VIPT non-aliasing cache */
-        if (cpumask_test_cpu(smp_processor_id(), mm_cpumask(vma->vm_mm)) &&
-            vma->vm_flags & VM_EXEC) {
+        if (vma->vm_flags & VM_EXEC) {
                 unsigned long addr = (unsigned long)kaddr;
-                /* only flushing the kernel mapping on non-aliasing VIPT */
                 __cpuc_coherent_kern_range(addr, addr + len);
+#ifdef CONFIG_SMP
+                if (cache_ops_need_broadcast())
+                        smp_call_function(flush_ptrace_access_other,
+                                          NULL, 1);
+#endif
         }
 }
-#else
-#define flush_pfn_alias(pfn,vaddr)      do { } while (0)
+
+/*
+ * Copy user data from/to a page which is mapped into a different
+ * processes address space.  Really, we want to allow our "user
+ * space" model to handle this.
+ *
+ * Note that this code needs to run on the current CPU.
+ */
+void copy_to_user_page(struct vm_area_struct *vma, struct page *page,
+                       unsigned long uaddr, void *dst, const void *src,
+                       unsigned long len)
+{
+#ifdef CONFIG_SMP
+        preempt_disable();
+#endif
+        memcpy(dst, src, len);
+        flush_ptrace_access(vma, page, uaddr, dst, len);
+#ifdef CONFIG_SMP
+        preempt_enable();
 #endif
+}
 
 void __flush_dcache_page(struct address_space *mapping, struct page *page)
 {
@@ -125,14 +158,20 @@ void __flush_dcache_page(struct address_space *mapping, struct page *page)
          * page.  This ensures that data in the physical page is mutually
          * coherent with the kernels mapping.
          */
-#ifdef CONFIG_HIGHMEM
-        /*
-         * kmap_atomic() doesn't set the page virtual address, and
-         * kunmap_atomic() takes care of cache flushing already.
-         */
-        if (page_address(page))
-#endif
-                __cpuc_flush_dcache_page(page_address(page));
+        if (!PageHighMem(page)) {
+                __cpuc_flush_dcache_area(page_address(page), PAGE_SIZE);
+        } else {
+                void *addr = kmap_high_get(page);
+                if (addr) {
+                        __cpuc_flush_dcache_area(addr, PAGE_SIZE);
+                        kunmap_high(page);
+                } else if (cache_is_vipt()) {
+                        pte_t saved_pte;
+                        addr = kmap_high_l1_vipt(page, &saved_pte);
+                        __cpuc_flush_dcache_area(addr, PAGE_SIZE);
+                        kunmap_high_l1_vipt(page, saved_pte);
+                }
+        }
 
         /*
          * If this is a page cache page, and we have an aliasing VIPT cache,
@@ -196,7 +235,16 @@ static void __flush_dcache_aliases(struct address_space *mapping, struct page *p
  */
 void flush_dcache_page(struct page *page)
 {
-        struct address_space *mapping = page_mapping(page);
+        struct address_space *mapping;
+
+        /*
+         * The zero page is never written to, so never has any dirty
+         * cache lines, and therefore never needs to be flushed.
+         */
+        if (page == ZERO_PAGE(0))
+                return;
+
+        mapping = page_mapping(page);
 
 #ifndef CONFIG_SMP
         if (!PageHighMem(page) && mapping && !mapping_mapped(mapping))
@@ -242,6 +290,7 @@ void __flush_anon_page(struct vm_area_struct *vma, struct page *page, unsigned l
                  * userspace address only.
                  */
                 flush_pfn_alias(pfn, vmaddr);
+                __flush_icache_all();
         }
 
         /*
@@ -249,5 +298,5 @@ void __flush_anon_page(struct vm_area_struct *vma, struct page *page, unsigned l
          * in this mapping of the page.  FIXME: this is overkill
          * since we actually ask for a write-back and invalidate.
          */
-        __cpuc_flush_dcache_page(page_address(page));
+        __cpuc_flush_dcache_area(page_address(page), PAGE_SIZE);
 }
diff --git a/arch/arm/mm/highmem.c b/arch/arm/mm/highmem.c
index 30f82fb5918c..77b030f5ec09 100644
--- a/arch/arm/mm/highmem.c
+++ b/arch/arm/mm/highmem.c
@@ -79,7 +79,8 @@ void kunmap_atomic(void *kvaddr, enum km_type type)
         unsigned int idx = type + KM_TYPE_NR * smp_processor_id();
 
         if (kvaddr >= (void *)FIXADDR_START) {
-                __cpuc_flush_dcache_page((void *)vaddr);
+                if (cache_is_vivt())
+                        __cpuc_flush_dcache_area((void *)vaddr, PAGE_SIZE);
 #ifdef CONFIG_DEBUG_HIGHMEM
                 BUG_ON(vaddr != __fix_to_virt(FIX_KMAP_BEGIN + idx));
                 set_pte_ext(TOP_PTE(vaddr), __pte(0), 0);
@@ -124,3 +125,87 @@ struct page *kmap_atomic_to_page(const void *ptr)
         pte = TOP_PTE(vaddr);
         return pte_page(*pte);
 }
+
+#ifdef CONFIG_CPU_CACHE_VIPT
+
+#include <linux/percpu.h>
+
+/*
+ * The VIVT cache of a highmem page is always flushed before the page
+ * is unmapped. Hence unmapped highmem pages need no cache maintenance
+ * in that case.
+ *
+ * However unmapped pages may still be cached with a VIPT cache, and
+ * it is not possible to perform cache maintenance on them using physical
+ * addresses unfortunately.  So we have no choice but to set up a temporary
+ * virtual mapping for that purpose.
+ *
+ * Yet this VIPT cache maintenance may be triggered from DMA support
+ * functions which are possibly called from interrupt context. As we don't
+ * want to keep interrupt disabled all the time when such maintenance is
+ * taking place, we therefore allow for some reentrancy by preserving and
+ * restoring the previous fixmap entry before the interrupted context is
+ * resumed.  If the reentrancy depth is 0 then there is no need to restore
+ * the previous fixmap, and leaving the current one in place allow it to
+ * be reused the next time without a TLB flush (common with DMA).
+ */
+
+static DEFINE_PER_CPU(int, kmap_high_l1_vipt_depth);
+
+void *kmap_high_l1_vipt(struct page *page, pte_t *saved_pte)
+{
+        unsigned int idx, cpu = smp_processor_id();
+        int *depth = &per_cpu(kmap_high_l1_vipt_depth, cpu);
+        unsigned long vaddr, flags;
+        pte_t pte, *ptep;
+
+        idx = KM_L1_CACHE + KM_TYPE_NR * cpu;
+        vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx);
+        ptep = TOP_PTE(vaddr);
+        pte = mk_pte(page, kmap_prot);
+
+        if (!in_interrupt())
+                preempt_disable();
+
+        raw_local_irq_save(flags);
+        (*depth)++;
+        if (pte_val(*ptep) == pte_val(pte)) {
+                *saved_pte = pte;
+        } else {
+                *saved_pte = *ptep;
+                set_pte_ext(ptep, pte, 0);
+                local_flush_tlb_kernel_page(vaddr);
+        }
+        raw_local_irq_restore(flags);
+
+        return (void *)vaddr;
+}
+
+void kunmap_high_l1_vipt(struct page *page, pte_t saved_pte)
+{
+        unsigned int idx, cpu = smp_processor_id();
+        int *depth = &per_cpu(kmap_high_l1_vipt_depth, cpu);
+        unsigned long vaddr, flags;
+        pte_t pte, *ptep;
+
+        idx = KM_L1_CACHE + KM_TYPE_NR * cpu;
+        vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx);
+        ptep = TOP_PTE(vaddr);
+        pte = mk_pte(page, kmap_prot);
+
+        BUG_ON(pte_val(*ptep) != pte_val(pte));
+        BUG_ON(*depth <= 0);
+
+        raw_local_irq_save(flags);
+        (*depth)--;
+        if (*depth != 0 && pte_val(pte) != pte_val(saved_pte)) {
+                set_pte_ext(ptep, saved_pte, 0);
+                local_flush_tlb_kernel_page(vaddr);
+        }
+        raw_local_irq_restore(flags);
+
+        if (!in_interrupt())
+                preempt_enable();
+}
+
+#endif  /* CONFIG_CPU_CACHE_VIPT */
diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c
index 52c40d155672..0ed29bfeba1c 100644
--- a/arch/arm/mm/init.c
+++ b/arch/arm/mm/init.c
@@ -17,12 +17,14 @@
 #include <linux/initrd.h>
 #include <linux/sort.h>
 #include <linux/highmem.h>
+#include <linux/gfp.h>
 
 #include <asm/mach-types.h>
 #include <asm/sections.h>
 #include <asm/setup.h>
 #include <asm/sizes.h>
 #include <asm/tlb.h>
+#include <asm/fixmap.h>
 
 #include <asm/mach/arch.h>
 #include <asm/mach/map.h>
@@ -32,19 +34,21 @@
 static unsigned long phys_initrd_start __initdata = 0;
 static unsigned long phys_initrd_size __initdata = 0;
 
-static void __init early_initrd(char **p)
+static int __init early_initrd(char *p)
 {
         unsigned long start, size;
+        char *endp;
 
-        start = memparse(*p, p);
-        if (**p == ',') {
-                size = memparse((*p) + 1, p);
+        start = memparse(p, &endp);
+        if (*endp == ',') {
+                size = memparse(endp + 1, NULL);
 
                 phys_initrd_start = start;
                 phys_initrd_size = size;
         }
+        return 0;
 }
-__early_param("initrd=", early_initrd);
+early_param("initrd", early_initrd);
 
 static int __init parse_tag_initrd(const struct tag *tag)
 {
@@ -82,9 +86,6 @@ void show_mem(void)
         printk("Mem-info:\n");
         show_free_areas();
         for_each_online_node(node) {
-                pg_data_t *n = NODE_DATA(node);
-                struct page *map = pgdat_page_nr(n, 0) - n->node_start_pfn;
-
                 for_each_nodebank (i,mi,node) {
                         struct membank *bank = &mi->bank[i];
                         unsigned int pfn1, pfn2;
@@ -93,8 +94,8 @@ void show_mem(void)
                         pfn1 = bank_pfn_start(bank);
                         pfn2 = bank_pfn_end(bank);
 
-                        page = map + pfn1;
-                        end  = map + pfn2;
+                        page = pfn_to_page(pfn1);
+                        end  = pfn_to_page(pfn2 - 1) + 1;
 
                         do {
                                 total++;
@@ -560,7 +561,7 @@ static void __init free_unused_memmap_node(int node, struct meminfo *mi)
  */
 void __init mem_init(void)
 {
-        unsigned int codesize, datasize, initsize;
+        unsigned long reserved_pages, free_pages;
         int i, node;
 
 #ifndef CONFIG_DISCONTIGMEM
@@ -596,6 +597,30 @@ void __init mem_init(void)
         totalram_pages += totalhigh_pages;
 #endif
 
+        reserved_pages = free_pages = 0;
+
+        for_each_online_node(node) {
+                for_each_nodebank(i, &meminfo, node) {
+                        struct membank *bank = &meminfo.bank[i];
+                        unsigned int pfn1, pfn2;
+                        struct page *page, *end;
+
+                        pfn1 = bank_pfn_start(bank);
+                        pfn2 = bank_pfn_end(bank);
+
+                        page = pfn_to_page(pfn1);
+                        end  = pfn_to_page(pfn2 - 1) + 1;
+
+                        do {
+                                if (PageReserved(page))
+                                        reserved_pages++;
+                                else if (!page_count(page))
+                                        free_pages++;
+                                page++;
+                        } while (page < end);
+                }
+        }
+
         /*
          * Since our memory may not be contiguous, calculate the
          * real number of pages we have in this system
@@ -608,15 +633,70 @@ void __init mem_init(void)
         }
         printk(" = %luMB total\n", num_physpages >> (20 - PAGE_SHIFT));
 
-        codesize = _etext - _text;
-        datasize = _end - _data;
-        initsize = __init_end - __init_begin;
+        printk(KERN_NOTICE "Memory: %luk/%luk available, %luk reserved, %luK highmem\n",
+                nr_free_pages() << (PAGE_SHIFT-10),
+                free_pages << (PAGE_SHIFT-10),
+                reserved_pages << (PAGE_SHIFT-10),
+                totalhigh_pages << (PAGE_SHIFT-10));
+
+#define MLK(b, t) b, t, ((t) - (b)) >> 10
+#define MLM(b, t) b, t, ((t) - (b)) >> 20
+#define MLK_ROUNDUP(b, t) b, t, DIV_ROUND_UP(((t) - (b)), SZ_1K)
 
-        printk(KERN_NOTICE "Memory: %luKB available (%dK code, "
-                "%dK data, %dK init, %luK highmem)\n",
-                nr_free_pages() << (PAGE_SHIFT-10), codesize >> 10,
-                datasize >> 10, initsize >> 10,
-                (unsigned long) (totalhigh_pages << (PAGE_SHIFT-10)));
+        printk(KERN_NOTICE "Virtual kernel memory layout:\n"
+                        "    vector  : 0x%08lx - 0x%08lx   (%4ld kB)\n"
+                        "    fixmap  : 0x%08lx - 0x%08lx   (%4ld kB)\n"
+#ifdef CONFIG_MMU
+                        "    DMA     : 0x%08lx - 0x%08lx   (%4ld MB)\n"
+#endif
+                        "    vmalloc : 0x%08lx - 0x%08lx   (%4ld MB)\n"
+                        "    lowmem  : 0x%08lx - 0x%08lx   (%4ld MB)\n"
+#ifdef CONFIG_HIGHMEM
+                        "    pkmap   : 0x%08lx - 0x%08lx   (%4ld MB)\n"
+#endif
+                        "    modules : 0x%08lx - 0x%08lx   (%4ld MB)\n"
+                        "      .init : 0x%p" " - 0x%p" "   (%4d kB)\n"
+                        "      .text : 0x%p" " - 0x%p" "   (%4d kB)\n"
+                        "      .data : 0x%p" " - 0x%p" "   (%4d kB)\n",
+
+                        MLK(UL(CONFIG_VECTORS_BASE), UL(CONFIG_VECTORS_BASE) +
+                                (PAGE_SIZE)),
+                        MLK(FIXADDR_START, FIXADDR_TOP),
+#ifdef CONFIG_MMU
+                        MLM(CONSISTENT_BASE, CONSISTENT_END),
+#endif
+                        MLM(VMALLOC_START, VMALLOC_END),
+                        MLM(PAGE_OFFSET, (unsigned long)high_memory),
+#ifdef CONFIG_HIGHMEM
+                        MLM(PKMAP_BASE, (PKMAP_BASE) + (LAST_PKMAP) *
+                                (PAGE_SIZE)),
+#endif
+                        MLM(MODULES_VADDR, MODULES_END),
+
+                        MLK_ROUNDUP(__init_begin, __init_end),
+                        MLK_ROUNDUP(_text, _etext),
+                        MLK_ROUNDUP(_data, _edata));
+
+#undef MLK
+#undef MLM
+#undef MLK_ROUNDUP
+
+        /*
+         * Check boundaries twice: Some fundamental inconsistencies can
+         * be detected at build time already.
+         */
+#ifdef CONFIG_MMU
+        BUILD_BUG_ON(VMALLOC_END                        > CONSISTENT_BASE);
+        BUG_ON(VMALLOC_END                              > CONSISTENT_BASE);
+
+        BUILD_BUG_ON(TASK_SIZE                          > MODULES_VADDR);
+        BUG_ON(TASK_SIZE                                > MODULES_VADDR);
+#endif
+
+#ifdef CONFIG_HIGHMEM
+        BUILD_BUG_ON(PKMAP_BASE + LAST_PKMAP * PAGE_SIZE > PAGE_OFFSET);
+        BUG_ON(PKMAP_BASE + LAST_PKMAP * PAGE_SIZE      > PAGE_OFFSET);
+#endif
 
         if (PAGE_SIZE >= 16384 && num_physpages <= 128) {
                 extern int sysctl_overcommit_memory;
diff --git a/arch/arm/mm/ioremap.c b/arch/arm/mm/ioremap.c
index 0ab75c60f7cf..28c8b950ef04 100644
--- a/arch/arm/mm/ioremap.c
+++ b/arch/arm/mm/ioremap.c
@@ -139,8 +139,8 @@ void __check_kvm_seq(struct mm_struct *mm)
  * which requires the new ioremap'd region to be referenced, the CPU will
  * reference the _old_ region.
  *
- * Note that get_vm_area() allocates a guard 4K page, so we need to mask
- * the size back to 1MB aligned or we will overflow in the loop below.
+ * Note that get_vm_area_caller() allocates a guard 4K page, so we need to
+ * mask the size back to 1MB aligned or we will overflow in the loop below.
  */
 static void unmap_area_sections(unsigned long virt, unsigned long size)
 {
@@ -254,22 +254,8 @@ remap_area_supersections(unsigned long virt, unsigned long pfn,
 }
 #endif
 
-
-/*
- * Remap an arbitrary physical address space into the kernel virtual
- * address space. Needed when the kernel wants to access high addresses
- * directly.
- *
- * NOTE! We need to allow non-page-aligned mappings too: we will obviously
- * have to convert them into an offset in a page-aligned mapping, but the
- * caller shouldn't need to know that small detail.
- *
- * 'flags' are the extra L_PTE_ flags that you want to specify for this
- * mapping.  See <asm/pgtable.h> for more information.
- */
-void __iomem *
-__arm_ioremap_pfn(unsigned long pfn, unsigned long offset, size_t size,
-                  unsigned int mtype)
+void __iomem * __arm_ioremap_pfn_caller(unsigned long pfn,
+        unsigned long offset, size_t size, unsigned int mtype, void *caller)
 {
         const struct mem_type *type;
         int err;
@@ -291,7 +277,7 @@ __arm_ioremap_pfn(unsigned long pfn, unsigned long offset, size_t size,
          */
         size = PAGE_ALIGN(offset + size);
 
-        area = get_vm_area(size, VM_IOREMAP);
+        area = get_vm_area_caller(size, VM_IOREMAP, caller);
         if (!area)
                 return NULL;
         addr = (unsigned long)area->addr;
@@ -318,10 +304,9 @@ __arm_ioremap_pfn(unsigned long pfn, unsigned long offset, size_t size,
         flush_cache_vmap(addr, addr + size);
         return (void __iomem *) (offset + addr);
 }
-EXPORT_SYMBOL(__arm_ioremap_pfn);
 
-void __iomem *
-__arm_ioremap(unsigned long phys_addr, size_t size, unsigned int mtype)
+void __iomem *__arm_ioremap_caller(unsigned long phys_addr, size_t size,
+        unsigned int mtype, void *caller)
 {
         unsigned long last_addr;
         unsigned long offset = phys_addr & ~PAGE_MASK;
@@ -334,7 +319,33 @@ __arm_ioremap(unsigned long phys_addr, size_t size, unsigned int mtype)
         if (!size || last_addr < phys_addr)
                 return NULL;
 
-        return __arm_ioremap_pfn(pfn, offset, size, mtype);
+        return __arm_ioremap_pfn_caller(pfn, offset, size, mtype,
+                        caller);
+}
+
+/*
+ * Remap an arbitrary physical address space into the kernel virtual
+ * address space. Needed when the kernel wants to access high addresses
+ * directly.
+ *
+ * NOTE! We need to allow non-page-aligned mappings too: we will obviously
+ * have to convert them into an offset in a page-aligned mapping, but the
+ * caller shouldn't need to know that small detail.
+ */
+void __iomem *
+__arm_ioremap_pfn(unsigned long pfn, unsigned long offset, size_t size,
+                  unsigned int mtype)
+{
+        return __arm_ioremap_pfn_caller(pfn, offset, size, mtype,
+                        __builtin_return_address(0));
+}
+EXPORT_SYMBOL(__arm_ioremap_pfn);
+
+void __iomem *
+__arm_ioremap(unsigned long phys_addr, size_t size, unsigned int mtype)
+{
+        return __arm_ioremap_caller(phys_addr, size, mtype,
+                        __builtin_return_address(0));
 }
 EXPORT_SYMBOL(__arm_ioremap);
 
diff --git a/arch/arm/mm/mm.h b/arch/arm/mm/mm.h
index c4f6f05198e0..a888363398f8 100644
--- a/arch/arm/mm/mm.h
+++ b/arch/arm/mm/mm.h
@@ -24,6 +24,8 @@ struct mem_type {
 
 const struct mem_type *get_mem_type(unsigned int type);
 
+extern void __flush_dcache_page(struct address_space *mapping, struct page *page);
+
 #endif
 
 struct map_desc;
diff --git a/arch/arm/mm/mmap.c b/arch/arm/mm/mmap.c
index 2b7996401b0f..f5abc51c5a07 100644
--- a/arch/arm/mm/mmap.c
+++ b/arch/arm/mm/mmap.c
@@ -54,7 +54,8 @@ arch_get_unmapped_area(struct file *filp, unsigned long addr,
          * We enforce the MAP_FIXED case.
          */
         if (flags & MAP_FIXED) {
-                if (aliasing && flags & MAP_SHARED && addr & (SHMLBA - 1))
+                if (aliasing && flags & MAP_SHARED &&
+                    (addr - (pgoff << PAGE_SHIFT)) & (SHMLBA - 1))
                         return -EINVAL;
                 return addr;
         }
diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c
index ea67be0223ac..241c24a1c18f 100644
--- a/arch/arm/mm/mmu.c
+++ b/arch/arm/mm/mmu.c
@@ -100,18 +100,17 @@ static struct cachepolicy cache_policies[] __initdata = {
  * writebuffer to be turned off.  (Note: the write
  * buffer should not be on and the cache off).
  */
-static void __init early_cachepolicy(char **p)
+static int __init early_cachepolicy(char *p)
 {
         int i;
 
         for (i = 0; i < ARRAY_SIZE(cache_policies); i++) {
                 int len = strlen(cache_policies[i].policy);
 
-                if (memcmp(*p, cache_policies[i].policy, len) == 0) {
+                if (memcmp(p, cache_policies[i].policy, len) == 0) {
                         cachepolicy = i;
                         cr_alignment &= ~cache_policies[i].cr_mask;
                         cr_no_alignment &= ~cache_policies[i].cr_mask;
-                        *p += len;
                         break;
                 }
         }
@@ -130,36 +129,37 @@ static void __init early_cachepolicy(char **p)
         }
         flush_cache_all();
         set_cr(cr_alignment);
+        return 0;
 }
-__early_param("cachepolicy=", early_cachepolicy);
+early_param("cachepolicy", early_cachepolicy);
 
-static void __init early_nocache(char **__unused)
+static int __init early_nocache(char *__unused)
 {
         char *p = "buffered";
         printk(KERN_WARNING "nocache is deprecated; use cachepolicy=%s\n", p);
-        early_cachepolicy(&p);
+        early_cachepolicy(p);
+        return 0;
 }
-__early_param("nocache", early_nocache);
+early_param("nocache", early_nocache);
 
-static void __init early_nowrite(char **__unused)
+static int __init early_nowrite(char *__unused)
 {
         char *p = "uncached";
         printk(KERN_WARNING "nowb is deprecated; use cachepolicy=%s\n", p);
-        early_cachepolicy(&p);
+        early_cachepolicy(p);
+        return 0;
 }
-__early_param("nowb", early_nowrite);
+early_param("nowb", early_nowrite);
 
-static void __init early_ecc(char **p)
+static int __init early_ecc(char *p)
 {
-        if (memcmp(*p, "on", 2) == 0) {
+        if (memcmp(p, "on", 2) == 0)
                 ecc_mask = PMD_PROTECTION;
-                *p += 2;
-        } else if (memcmp(*p, "off", 3) == 0) {
+        else if (memcmp(p, "off", 3) == 0)
                 ecc_mask = 0;
-                *p += 3;
-        }
+        return 0;
 }
-__early_param("ecc=", early_ecc);
+early_param("ecc", early_ecc);
 
 static int __init noalign_setup(char *__unused)
 {
@@ -420,6 +420,10 @@ static void __init build_mem_type_table(void)
                 user_pgprot |= L_PTE_SHARED;
                 kern_pgprot |= L_PTE_SHARED;
                 vecs_pgprot |= L_PTE_SHARED;
+                mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_S;
+                mem_types[MT_DEVICE_WC].prot_pte |= L_PTE_SHARED;
+                mem_types[MT_DEVICE_CACHED].prot_sect |= PMD_SECT_S;
+                mem_types[MT_DEVICE_CACHED].prot_pte |= L_PTE_SHARED;
                 mem_types[MT_MEMORY].prot_sect |= PMD_SECT_S;
                 mem_types[MT_MEMORY_NONCACHED].prot_sect |= PMD_SECT_S;
 #endif
@@ -453,8 +457,7 @@ static void __init build_mem_type_table(void)
 
         pgprot_user   = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG | user_pgprot);
         pgprot_kernel = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG |
-                                 L_PTE_DIRTY | L_PTE_WRITE |
-                                 L_PTE_EXEC | kern_pgprot);
+                                 L_PTE_DIRTY | L_PTE_WRITE | kern_pgprot);
 
         mem_types[MT_LOW_VECTORS].prot_l1 |= ecc_mask;
         mem_types[MT_HIGH_VECTORS].prot_l1 |= ecc_mask;
@@ -671,9 +674,9 @@ static unsigned long __initdata vmalloc_reserve = SZ_128M;
  * bytes. This can be used to increase (or decrease) the vmalloc
  * area - the default is 128m.
  */
-static void __init early_vmalloc(char **arg)
+static int __init early_vmalloc(char *arg)
 {
-        vmalloc_reserve = memparse(*arg, arg);
+        vmalloc_reserve = memparse(arg, NULL);
 
         if (vmalloc_reserve < SZ_16M) {
                 vmalloc_reserve = SZ_16M;
@@ -688,8 +691,9 @@ static void __init early_vmalloc(char **arg)
                         "vmalloc area is too big, limiting to %luMB\n",
                         vmalloc_reserve >> 20);
         }
+        return 0;
 }
-__early_param("vmalloc=", early_vmalloc);
+early_param("vmalloc", early_vmalloc);
 
 #define VMALLOC_MIN     (void *)(VMALLOC_END - vmalloc_reserve)
 
@@ -881,7 +885,7 @@ void __init reserve_node_zero(pg_data_t *pgdat)
                                 BOOTMEM_EXCLUSIVE);
         }
 
-        if (machine_is_treo680()) {
+        if (machine_is_treo680() || machine_is_centro()) {
                 reserve_bootmem_node(pgdat, 0xa0000000, 0x1000,
                                 BOOTMEM_EXCLUSIVE);
                 reserve_bootmem_node(pgdat, 0xa2000000, 0x1000,
@@ -1036,7 +1040,7 @@ void __init paging_init(struct machine_desc *mdesc)
          */
         zero_page = alloc_bootmem_low_pages(PAGE_SIZE);
         empty_zero_page = virt_to_page(zero_page);
-        flush_dcache_page(empty_zero_page);
+        __flush_dcache_page(NULL, empty_zero_page);
 }
 
 /*
@@ -1050,10 +1054,12 @@ void setup_mm_for_reboot(char mode)
         pgd_t *pgd;
         int i;
 
-        if (current->mm && current->mm->pgd)
-                pgd = current->mm->pgd;
-        else
-                pgd = init_mm.pgd;
+        /*
+         * We need to access to user-mode page tables here. For kernel threads
+         * we don't have any user-mode mappings so we use the context that we
+         * "borrowed".
+         */
+        pgd = current->active_mm->pgd;
 
         base_pmdval = PMD_SECT_AP_WRITE | PMD_SECT_AP_READ | PMD_TYPE_SECT;
         if (cpu_architecture() <= CPU_ARCH_ARMv5TEJ && !cpu_is_xscale())
@@ -1068,4 +1074,6 @@ void setup_mm_for_reboot(char mode)
                 pmd[1] = __pmd(pmdval + (1 << (PGDIR_SHIFT - 1)));
                 flush_pmd_entry(pmd);
         }
+
+        local_flush_tlb_all();
 }
diff --git a/arch/arm/mm/nommu.c b/arch/arm/mm/nommu.c
index 900811cc9130..33b327379f07 100644
--- a/arch/arm/mm/nommu.c
+++ b/arch/arm/mm/nommu.c
@@ -61,10 +61,19 @@ void setup_mm_for_reboot(char mode)
 
 void flush_dcache_page(struct page *page)
 {
-        __cpuc_flush_dcache_page(page_address(page));
+        __cpuc_flush_dcache_area(page_address(page), PAGE_SIZE);
 }
 EXPORT_SYMBOL(flush_dcache_page);
 
+void copy_to_user_page(struct vm_area_struct *vma, struct page *page,
+                       unsigned long uaddr, void *dst, const void *src,
+                       unsigned long len)
+{
+        memcpy(dst, src, len);
+        if (vma->vm_flags & VM_EXEC)
+                __cpuc_coherent_user_range(uaddr, uaddr + len);
+}
+
 void __iomem *__arm_ioremap_pfn(unsigned long pfn, unsigned long offset,
                                 size_t size, unsigned int mtype)
 {
@@ -74,6 +83,12 @@ void __iomem *__arm_ioremap_pfn(unsigned long pfn, unsigned long offset,
 }
 EXPORT_SYMBOL(__arm_ioremap_pfn);
 
+void __iomem *__arm_ioremap_pfn_caller(unsigned long pfn, unsigned long offset,
+                           size_t size, unsigned int mtype, void *caller)
+{
+        return __arm_ioremap_pfn(pfn, offset, size, mtype);
+}
+
 void __iomem *__arm_ioremap(unsigned long phys_addr, size_t size,
                             unsigned int mtype)
 {
@@ -81,6 +96,12 @@ void __iomem *__arm_ioremap(unsigned long phys_addr, size_t size,
 }
 EXPORT_SYMBOL(__arm_ioremap);
 
+void __iomem *__arm_ioremap_caller(unsigned long phys_addr, size_t size,
+                                   unsigned int mtype, void *caller)
+{
+        return __arm_ioremap(phys_addr, size, mtype);
+}
+
 void __iounmap(volatile void __iomem *addr)
 {
 }
diff --git a/arch/arm/mm/pgd.c b/arch/arm/mm/pgd.c
index 2690146161ba..be5f58e153bf 100644
--- a/arch/arm/mm/pgd.c
+++ b/arch/arm/mm/pgd.c
@@ -8,6 +8,7 @@
  * published by the Free Software Foundation.
  */
 #include <linux/mm.h>
+#include <linux/gfp.h>
 #include <linux/highmem.h>
 
 #include <asm/pgalloc.h>
diff --git a/arch/arm/mm/proc-arm1020.S b/arch/arm/mm/proc-arm1020.S
index d9fb4b98c49f..72507c630ceb 100644
--- a/arch/arm/mm/proc-arm1020.S
+++ b/arch/arm/mm/proc-arm1020.S
@@ -231,17 +231,18 @@ ENTRY(arm1020_coherent_user_range)
         mov     pc, lr
 
 /*
- *      flush_kern_dcache_page(void *page)
+ *      flush_kern_dcache_area(void *addr, size_t size)
  *
  *      Ensure no D cache aliasing occurs, either with itself or
  *      the I cache
  *
- *      - page  - page aligned address
+ *      - addr  - kernel address
+ *      - size  - region size
  */
-ENTRY(arm1020_flush_kern_dcache_page)
+ENTRY(arm1020_flush_kern_dcache_area)
         mov     ip, #0
 #ifndef CONFIG_CPU_DCACHE_DISABLE
-        add     r1, r0, #PAGE_SZ
+        add     r1, r0, r1
 1:      mcr     p15, 0, r0, c7, c14, 1          @ clean+invalidate D entry
         mcr     p15, 0, ip, c7, c10, 4          @ drain WB
         add     r0, r0, #CACHE_DLINESIZE
@@ -264,7 +265,7 @@ ENTRY(arm1020_flush_kern_dcache_page)
  *
  * (same as v4wb)
  */
-ENTRY(arm1020_dma_inv_range)
+arm1020_dma_inv_range:
         mov     ip, #0
 #ifndef CONFIG_CPU_DCACHE_DISABLE
         tst     r0, #CACHE_DLINESIZE - 1
@@ -294,7 +295,7 @@ ENTRY(arm1020_dma_inv_range)
  *
  * (same as v4wb)
  */
-ENTRY(arm1020_dma_clean_range)
+arm1020_dma_clean_range:
         mov     ip, #0
 #ifndef CONFIG_CPU_DCACHE_DISABLE
         bic     r0, r0, #CACHE_DLINESIZE - 1
@@ -329,15 +330,39 @@ ENTRY(arm1020_dma_flush_range)
         mcr     p15, 0, ip, c7, c10, 4          @ drain WB
         mov     pc, lr
 
+/*
+ *      dma_map_area(start, size, dir)
+ *      - start - kernel virtual start address
+ *      - size  - size of region
+ *      - dir   - DMA direction
+ */
+ENTRY(arm1020_dma_map_area)
+        add     r1, r1, r0
+        cmp     r2, #DMA_TO_DEVICE
+        beq     arm1020_dma_clean_range
+        bcs     arm1020_dma_inv_range
+        b       arm1020_dma_flush_range
+ENDPROC(arm1020_dma_map_area)
+
+/*
+ *      dma_unmap_area(start, size, dir)
+ *      - start - kernel virtual start address
+ *      - size  - size of region
+ *      - dir   - DMA direction
+ */
+ENTRY(arm1020_dma_unmap_area)
+        mov     pc, lr
+ENDPROC(arm1020_dma_unmap_area)
+
 ENTRY(arm1020_cache_fns)
         .long   arm1020_flush_kern_cache_all
         .long   arm1020_flush_user_cache_all
         .long   arm1020_flush_user_cache_range
         .long   arm1020_coherent_kern_range
         .long   arm1020_coherent_user_range
-        .long   arm1020_flush_kern_dcache_page
-        .long   arm1020_dma_inv_range
-        .long   arm1020_dma_clean_range
+        .long   arm1020_flush_kern_dcache_area
+        .long   arm1020_dma_map_area
+        .long   arm1020_dma_unmap_area
         .long   arm1020_dma_flush_range
 
         .align  5
diff --git a/arch/arm/mm/proc-arm1020e.S b/arch/arm/mm/proc-arm1020e.S
index 7453b75dcea5..d27829805609 100644
--- a/arch/arm/mm/proc-arm1020e.S
+++ b/arch/arm/mm/proc-arm1020e.S
@@ -225,17 +225,18 @@ ENTRY(arm1020e_coherent_user_range)
         mov     pc, lr
 
 /*
- *      flush_kern_dcache_page(void *page)
+ *      flush_kern_dcache_area(void *addr, size_t size)
  *
  *      Ensure no D cache aliasing occurs, either with itself or
  *      the I cache
  *
- *      - page  - page aligned address
+ *      - addr  - kernel address
+ *      - size  - region size
  */
-ENTRY(arm1020e_flush_kern_dcache_page)
+ENTRY(arm1020e_flush_kern_dcache_area)
         mov     ip, #0
 #ifndef CONFIG_CPU_DCACHE_DISABLE
-        add     r1, r0, #PAGE_SZ
+        add     r1, r0, r1
 1:      mcr     p15, 0, r0, c7, c14, 1          @ clean+invalidate D entry
         add     r0, r0, #CACHE_DLINESIZE
         cmp     r0, r1
@@ -257,7 +258,7 @@ ENTRY(arm1020e_flush_kern_dcache_page)
  *
  * (same as v4wb)
  */
-ENTRY(arm1020e_dma_inv_range)
+arm1020e_dma_inv_range:
         mov     ip, #0
 #ifndef CONFIG_CPU_DCACHE_DISABLE
         tst     r0, #CACHE_DLINESIZE - 1
@@ -283,7 +284,7 @@ ENTRY(arm1020e_dma_inv_range)
  *
  * (same as v4wb)
  */
-ENTRY(arm1020e_dma_clean_range)
+arm1020e_dma_clean_range:
         mov     ip, #0
 #ifndef CONFIG_CPU_DCACHE_DISABLE
         bic     r0, r0, #CACHE_DLINESIZE - 1
@@ -315,15 +316,39 @@ ENTRY(arm1020e_dma_flush_range)
         mcr     p15, 0, ip, c7, c10, 4          @ drain WB
         mov     pc, lr
 
+/*
+ *      dma_map_area(start, size, dir)
+ *      - start - kernel virtual start address
+ *      - size  - size of region
+ *      - dir   - DMA direction
+ */
+ENTRY(arm1020e_dma_map_area)
+        add     r1, r1, r0
+        cmp     r2, #DMA_TO_DEVICE
+        beq     arm1020e_dma_clean_range
+        bcs     arm1020e_dma_inv_range
+        b       arm1020e_dma_flush_range
+ENDPROC(arm1020e_dma_map_area)
+
+/*
+ *      dma_unmap_area(start, size, dir)
+ *      - start - kernel virtual start address
+ *      - size  - size of region
+ *      - dir   - DMA direction
+ */
+ENTRY(arm1020e_dma_unmap_area)
+        mov     pc, lr
+ENDPROC(arm1020e_dma_unmap_area)
+
 ENTRY(arm1020e_cache_fns)
         .long   arm1020e_flush_kern_cache_all
         .long   arm1020e_flush_user_cache_all
         .long   arm1020e_flush_user_cache_range
         .long   arm1020e_coherent_kern_range
         .long   arm1020e_coherent_user_range
-        .long   arm1020e_flush_kern_dcache_page
-        .long   arm1020e_dma_inv_range
-        .long   arm1020e_dma_clean_range
+        .long   arm1020e_flush_kern_dcache_area
+        .long   arm1020e_dma_map_area
+        .long   arm1020e_dma_unmap_area
         .long   arm1020e_dma_flush_range
 
         .align  5
diff --git a/arch/arm/mm/proc-arm1022.S b/arch/arm/mm/proc-arm1022.S
index 8eb72d75a8b6..ce13e4a827de 100644
--- a/arch/arm/mm/proc-arm1022.S
+++ b/arch/arm/mm/proc-arm1022.S
@@ -214,17 +214,18 @@ ENTRY(arm1022_coherent_user_range)
         mov     pc, lr
 
 /*
- *      flush_kern_dcache_page(void *page)
+ *      flush_kern_dcache_area(void *addr, size_t size)
  *
  *      Ensure no D cache aliasing occurs, either with itself or
  *      the I cache
  *
- *      - page  - page aligned address
+ *      - addr  - kernel address
+ *      - size  - region size
  */
-ENTRY(arm1022_flush_kern_dcache_page)
+ENTRY(arm1022_flush_kern_dcache_area)
         mov     ip, #0
 #ifndef CONFIG_CPU_DCACHE_DISABLE
-        add     r1, r0, #PAGE_SZ
+        add     r1, r0, r1
 1:      mcr     p15, 0, r0, c7, c14, 1          @ clean+invalidate D entry
         add     r0, r0, #CACHE_DLINESIZE
         cmp     r0, r1
@@ -246,7 +247,7 @@ ENTRY(arm1022_flush_kern_dcache_page)
  *
  * (same as v4wb)
  */
-ENTRY(arm1022_dma_inv_range)
+arm1022_dma_inv_range:
         mov     ip, #0
 #ifndef CONFIG_CPU_DCACHE_DISABLE
         tst     r0, #CACHE_DLINESIZE - 1
@@ -272,7 +273,7 @@ ENTRY(arm1022_dma_inv_range)
  *
  * (same as v4wb)
  */
-ENTRY(arm1022_dma_clean_range)
+arm1022_dma_clean_range:
         mov     ip, #0
 #ifndef CONFIG_CPU_DCACHE_DISABLE
         bic     r0, r0, #CACHE_DLINESIZE - 1
@@ -304,15 +305,39 @@ ENTRY(arm1022_dma_flush_range)
         mcr     p15, 0, ip, c7, c10, 4          @ drain WB
         mov     pc, lr
 
+/*
+ *      dma_map_area(start, size, dir)
+ *      - start - kernel virtual start address
+ *      - size  - size of region
+ *      - dir   - DMA direction
+ */
+ENTRY(arm1022_dma_map_area)
+        add     r1, r1, r0
+        cmp     r2, #DMA_TO_DEVICE
+        beq     arm1022_dma_clean_range
+        bcs     arm1022_dma_inv_range
+        b       arm1022_dma_flush_range
+ENDPROC(arm1022_dma_map_area)
+
+/*
+ *      dma_unmap_area(start, size, dir)
+ *      - start - kernel virtual start address
+ *      - size  - size of region
+ *      - dir   - DMA direction
+ */
+ENTRY(arm1022_dma_unmap_area)
+        mov     pc, lr
+ENDPROC(arm1022_dma_unmap_area)
+
 ENTRY(arm1022_cache_fns)
         .long   arm1022_flush_kern_cache_all
         .long   arm1022_flush_user_cache_all
         .long   arm1022_flush_user_cache_range
         .long   arm1022_coherent_kern_range
         .long   arm1022_coherent_user_range
-        .long   arm1022_flush_kern_dcache_page
-        .long   arm1022_dma_inv_range
-        .long   arm1022_dma_clean_range
+        .long   arm1022_flush_kern_dcache_area
+        .long   arm1022_dma_map_area
+        .long   arm1022_dma_unmap_area
         .long   arm1022_dma_flush_range
 
         .align  5
diff --git a/arch/arm/mm/proc-arm1026.S b/arch/arm/mm/proc-arm1026.S
index 3b59f0d67139..636672a29c6d 100644
--- a/arch/arm/mm/proc-arm1026.S
+++ b/arch/arm/mm/proc-arm1026.S
@@ -208,17 +208,18 @@ ENTRY(arm1026_coherent_user_range)
         mov     pc, lr
 
 /*
- *      flush_kern_dcache_page(void *page)
+ *      flush_kern_dcache_area(void *addr, size_t size)
  *
  *      Ensure no D cache aliasing occurs, either with itself or
  *      the I cache
  *
- *      - page  - page aligned address
+ *      - addr  - kernel address
+ *      - size  - region size
  */
-ENTRY(arm1026_flush_kern_dcache_page)
+ENTRY(arm1026_flush_kern_dcache_area)
         mov     ip, #0
 #ifndef CONFIG_CPU_DCACHE_DISABLE
-        add     r1, r0, #PAGE_SZ
+        add     r1, r0, r1
 1:      mcr     p15, 0, r0, c7, c14, 1          @ clean+invalidate D entry
         add     r0, r0, #CACHE_DLINESIZE
         cmp     r0, r1
@@ -240,7 +241,7 @@ ENTRY(arm1026_flush_kern_dcache_page)
  *
  * (same as v4wb)
  */
-ENTRY(arm1026_dma_inv_range)
+arm1026_dma_inv_range:
         mov     ip, #0
 #ifndef CONFIG_CPU_DCACHE_DISABLE
         tst     r0, #CACHE_DLINESIZE - 1
@@ -266,7 +267,7 @@ ENTRY(arm1026_dma_inv_range)
  *
  * (same as v4wb)
  */
-ENTRY(arm1026_dma_clean_range)
+arm1026_dma_clean_range:
         mov     ip, #0
 #ifndef CONFIG_CPU_DCACHE_DISABLE
         bic     r0, r0, #CACHE_DLINESIZE - 1
@@ -298,15 +299,39 @@ ENTRY(arm1026_dma_flush_range)
         mcr     p15, 0, ip, c7, c10, 4          @ drain WB
         mov     pc, lr
 
+/*
+ *      dma_map_area(start, size, dir)
+ *      - start - kernel virtual start address
+ *      - size  - size of region
+ *      - dir   - DMA direction
+ */
+ENTRY(arm1026_dma_map_area)
+        add     r1, r1, r0
+        cmp     r2, #DMA_TO_DEVICE
+        beq     arm1026_dma_clean_range
+        bcs     arm1026_dma_inv_range
+        b       arm1026_dma_flush_range
+ENDPROC(arm1026_dma_map_area)
+
+/*
+ *      dma_unmap_area(start, size, dir)
+ *      - start - kernel virtual start address
+ *      - size  - size of region
+ *      - dir   - DMA direction
+ */
+ENTRY(arm1026_dma_unmap_area)
+        mov     pc, lr
+ENDPROC(arm1026_dma_unmap_area)
+
 ENTRY(arm1026_cache_fns)
         .long   arm1026_flush_kern_cache_all
         .long   arm1026_flush_user_cache_all
         .long   arm1026_flush_user_cache_range
         .long   arm1026_coherent_kern_range
         .long   arm1026_coherent_user_range
-        .long   arm1026_flush_kern_dcache_page
-        .long   arm1026_dma_inv_range
-        .long   arm1026_dma_clean_range
+        .long   arm1026_flush_kern_dcache_area
+        .long   arm1026_dma_map_area
+        .long   arm1026_dma_unmap_area
         .long   arm1026_dma_flush_range
 
         .align  5
diff --git a/arch/arm/mm/proc-arm6_7.S b/arch/arm/mm/proc-arm6_7.S
index 3f9cd3d8f6d5..795dc615f43b 100644
--- a/arch/arm/mm/proc-arm6_7.S
+++ b/arch/arm/mm/proc-arm6_7.S
@@ -41,7 +41,7 @@ ENTRY(cpu_arm7_dcache_clean_area)
 ENTRY(cpu_arm7_data_abort)
         mrc     p15, 0, r1, c5, c0, 0           @ get FSR
         mrc     p15, 0, r0, c6, c0, 0           @ get FAR
-        ldr     r8, [r0]                        @ read arm instruction
+        ldr     r8, [r2]                        @ read arm instruction
         tst     r8, #1 << 20                    @ L = 0 -> write?
         orreq   r1, r1, #1 << 11                @ yes.
         and     r7, r8, #15 << 24
diff --git a/arch/arm/mm/proc-arm920.S b/arch/arm/mm/proc-arm920.S
index 2b7c197cc58d..8be81992645d 100644
--- a/arch/arm/mm/proc-arm920.S
+++ b/arch/arm/mm/proc-arm920.S
@@ -207,15 +207,16 @@ ENTRY(arm920_coherent_user_range)
         mov     pc, lr
 
 /*
- *      flush_kern_dcache_page(void *page)
+ *      flush_kern_dcache_area(void *addr, size_t size)
  *
  *      Ensure no D cache aliasing occurs, either with itself or
  *      the I cache
  *
- *      - addr  - page aligned address
+ *      - addr  - kernel address
+ *      - size  - region size
  */
-ENTRY(arm920_flush_kern_dcache_page)
-        add     r1, r0, #PAGE_SZ
+ENTRY(arm920_flush_kern_dcache_area)
+        add     r1, r0, r1
 1:      mcr     p15, 0, r0, c7, c14, 1          @ clean+invalidate D entry
         add     r0, r0, #CACHE_DLINESIZE
         cmp     r0, r1
@@ -238,7 +239,7 @@ ENTRY(arm920_flush_kern_dcache_page)
  *
  * (same as v4wb)
  */
-ENTRY(arm920_dma_inv_range)
+arm920_dma_inv_range:
         tst     r0, #CACHE_DLINESIZE - 1
         bic     r0, r0, #CACHE_DLINESIZE - 1
         mcrne   p15, 0, r0, c7, c10, 1          @ clean D entry
@@ -261,7 +262,7 @@ ENTRY(arm920_dma_inv_range)
  *
  * (same as v4wb)
  */
-ENTRY(arm920_dma_clean_range)
+arm920_dma_clean_range:
         bic     r0, r0, #CACHE_DLINESIZE - 1
 1:      mcr     p15, 0, r0, c7, c10, 1          @ clean D entry
         add     r0, r0, #CACHE_DLINESIZE
@@ -287,15 +288,39 @@ ENTRY(arm920_dma_flush_range)
         mcr     p15, 0, r0, c7, c10, 4          @ drain WB
         mov     pc, lr
 
+/*
+ *      dma_map_area(start, size, dir)
+ *      - start - kernel virtual start address
+ *      - size  - size of region
+ *      - dir   - DMA direction
+ */
+ENTRY(arm920_dma_map_area)
+        add     r1, r1, r0
+        cmp     r2, #DMA_TO_DEVICE
+        beq     arm920_dma_clean_range
+        bcs     arm920_dma_inv_range
+        b       arm920_dma_flush_range
+ENDPROC(arm920_dma_map_area)
+
+/*
+ *      dma_unmap_area(start, size, dir)
+ *      - start - kernel virtual start address
+ *      - size  - size of region
+ *      - dir   - DMA direction
+ */
+ENTRY(arm920_dma_unmap_area)
+        mov     pc, lr
+ENDPROC(arm920_dma_unmap_area)
+
 ENTRY(arm920_cache_fns)
         .long   arm920_flush_kern_cache_all
         .long   arm920_flush_user_cache_all
         .long   arm920_flush_user_cache_range
         .long   arm920_coherent_kern_range
         .long   arm920_coherent_user_range
-        .long   arm920_flush_kern_dcache_page
-        .long   arm920_dma_inv_range
-        .long   arm920_dma_clean_range
+        .long   arm920_flush_kern_dcache_area
+        .long   arm920_dma_map_area
+        .long   arm920_dma_unmap_area
         .long   arm920_dma_flush_range
 
 #endif
diff --git a/arch/arm/mm/proc-arm922.S b/arch/arm/mm/proc-arm922.S
index 06a1aa4e3398..c0ff8e4b1074 100644
--- a/arch/arm/mm/proc-arm922.S
+++ b/arch/arm/mm/proc-arm922.S
@@ -209,15 +209,16 @@ ENTRY(arm922_coherent_user_range)
         mov     pc, lr
 
 /*
- *      flush_kern_dcache_page(void *page)
+ *      flush_kern_dcache_area(void *addr, size_t size)
  *
  *      Ensure no D cache aliasing occurs, either with itself or
  *      the I cache
  *
- *      - addr  - page aligned address
+ *      - addr  - kernel address
+ *      - size  - region size
  */
-ENTRY(arm922_flush_kern_dcache_page)
-        add     r1, r0, #PAGE_SZ
+ENTRY(arm922_flush_kern_dcache_area)
+        add     r1, r0, r1
 1:      mcr     p15, 0, r0, c7, c14, 1          @ clean+invalidate D entry
         add     r0, r0, #CACHE_DLINESIZE
         cmp     r0, r1
@@ -240,7 +241,7 @@ ENTRY(arm922_flush_kern_dcache_page)
  *
  * (same as v4wb)
  */
-ENTRY(arm922_dma_inv_range)
+arm922_dma_inv_range:
         tst     r0, #CACHE_DLINESIZE - 1
         bic     r0, r0, #CACHE_DLINESIZE - 1
         mcrne   p15, 0, r0, c7, c10, 1          @ clean D entry
@@ -263,7 +264,7 @@ ENTRY(arm922_dma_inv_range)
  *
  * (same as v4wb)
  */
-ENTRY(arm922_dma_clean_range)
+arm922_dma_clean_range:
         bic     r0, r0, #CACHE_DLINESIZE - 1
 1:      mcr     p15, 0, r0, c7, c10, 1          @ clean D entry
         add     r0, r0, #CACHE_DLINESIZE
@@ -289,15 +290,39 @@ ENTRY(arm922_dma_flush_range)
         mcr     p15, 0, r0, c7, c10, 4          @ drain WB
         mov     pc, lr
 
+/*
+ *      dma_map_area(start, size, dir)
+ *      - start - kernel virtual start address
+ *      - size  - size of region
+ *      - dir   - DMA direction
+ */
+ENTRY(arm922_dma_map_area)
+        add     r1, r1, r0
+        cmp     r2, #DMA_TO_DEVICE
+        beq     arm922_dma_clean_range
+        bcs     arm922_dma_inv_range
+        b       arm922_dma_flush_range
+ENDPROC(arm922_dma_map_area)
+
+/*
+ *      dma_unmap_area(start, size, dir)
+ *      - start - kernel virtual start address
+ *      - size  - size of region
+ *      - dir   - DMA direction
+ */
+ENTRY(arm922_dma_unmap_area)
+        mov     pc, lr
+ENDPROC(arm922_dma_unmap_area)
+
 ENTRY(arm922_cache_fns)
         .long   arm922_flush_kern_cache_all
         .long   arm922_flush_user_cache_all
         .long   arm922_flush_user_cache_range
         .long   arm922_coherent_kern_range
         .long   arm922_coherent_user_range
-        .long   arm922_flush_kern_dcache_page
-        .long   arm922_dma_inv_range
-        .long   arm922_dma_clean_range
+        .long   arm922_flush_kern_dcache_area
+        .long   arm922_dma_map_area
+        .long   arm922_dma_unmap_area
         .long   arm922_dma_flush_range
 
 #endif
diff --git a/arch/arm/mm/proc-arm925.S b/arch/arm/mm/proc-arm925.S
index cb53435a85ae..3c6cffe400f6 100644
--- a/arch/arm/mm/proc-arm925.S
+++ b/arch/arm/mm/proc-arm925.S
@@ -251,15 +251,16 @@ ENTRY(arm925_coherent_user_range)
         mov     pc, lr
 
 /*
- *      flush_kern_dcache_page(void *page)
+ *      flush_kern_dcache_area(void *addr, size_t size)
  *
  *      Ensure no D cache aliasing occurs, either with itself or
  *      the I cache
  *
- *      - addr  - page aligned address
+ *      - addr  - kernel address
+ *      - size  - region size
  */
-ENTRY(arm925_flush_kern_dcache_page)
-        add     r1, r0, #PAGE_SZ
+ENTRY(arm925_flush_kern_dcache_area)
+        add     r1, r0, r1
 1:      mcr     p15, 0, r0, c7, c14, 1          @ clean+invalidate D entry
         add     r0, r0, #CACHE_DLINESIZE
         cmp     r0, r1
@@ -282,7 +283,7 @@ ENTRY(arm925_flush_kern_dcache_page)
  *
  * (same as v4wb)
  */
-ENTRY(arm925_dma_inv_range)
+arm925_dma_inv_range:
 #ifndef CONFIG_CPU_DCACHE_WRITETHROUGH
         tst     r0, #CACHE_DLINESIZE - 1
         mcrne   p15, 0, r0, c7, c10, 1          @ clean D entry
@@ -307,7 +308,7 @@ ENTRY(arm925_dma_inv_range)
  *
  * (same as v4wb)
  */
-ENTRY(arm925_dma_clean_range)
+arm925_dma_clean_range:
 #ifndef CONFIG_CPU_DCACHE_WRITETHROUGH
         bic     r0, r0, #CACHE_DLINESIZE - 1
 1:      mcr     p15, 0, r0, c7, c10, 1          @ clean D entry
@@ -340,15 +341,39 @@ ENTRY(arm925_dma_flush_range)
         mcr     p15, 0, r0, c7, c10, 4          @ drain WB
         mov     pc, lr
 
+/*
+ *      dma_map_area(start, size, dir)
+ *      - start - kernel virtual start address
+ *      - size  - size of region
+ *      - dir   - DMA direction
+ */
+ENTRY(arm925_dma_map_area)
+        add     r1, r1, r0
+        cmp     r2, #DMA_TO_DEVICE
+        beq     arm925_dma_clean_range
+        bcs     arm925_dma_inv_range
+        b       arm925_dma_flush_range
+ENDPROC(arm925_dma_map_area)
+
+/*
+ *      dma_unmap_area(start, size, dir)
+ *      - start - kernel virtual start address
+ *      - size  - size of region
+ *      - dir   - DMA direction
+ */
+ENTRY(arm925_dma_unmap_area)
+        mov     pc, lr
+ENDPROC(arm925_dma_unmap_area)
+
 ENTRY(arm925_cache_fns)
         .long   arm925_flush_kern_cache_all
         .long   arm925_flush_user_cache_all
         .long   arm925_flush_user_cache_range
         .long   arm925_coherent_kern_range
         .long   arm925_coherent_user_range
-        .long   arm925_flush_kern_dcache_page
-        .long   arm925_dma_inv_range
-        .long   arm925_dma_clean_range
+        .long   arm925_flush_kern_dcache_area
+        .long   arm925_dma_map_area
+        .long   arm925_dma_unmap_area
         .long   arm925_dma_flush_range
 
 ENTRY(cpu_arm925_dcache_clean_area)
diff --git a/arch/arm/mm/proc-arm926.S b/arch/arm/mm/proc-arm926.S
index 1c4848704bb3..75b707c9cce1 100644
--- a/arch/arm/mm/proc-arm926.S
+++ b/arch/arm/mm/proc-arm926.S
@@ -214,15 +214,16 @@ ENTRY(arm926_coherent_user_range)
         mov     pc, lr
 
 /*
- *      flush_kern_dcache_page(void *page)
+ *      flush_kern_dcache_area(void *addr, size_t size)
  *
  *      Ensure no D cache aliasing occurs, either with itself or
  *      the I cache
  *
- *      - addr  - page aligned address
+ *      - addr  - kernel address
+ *      - size  - region size
  */
-ENTRY(arm926_flush_kern_dcache_page)
-        add     r1, r0, #PAGE_SZ
+ENTRY(arm926_flush_kern_dcache_area)
+        add     r1, r0, r1
 1:      mcr     p15, 0, r0, c7, c14, 1          @ clean+invalidate D entry
         add     r0, r0, #CACHE_DLINESIZE
         cmp     r0, r1
@@ -245,7 +246,7 @@ ENTRY(arm926_flush_kern_dcache_page)
  *
  * (same as v4wb)
  */
-ENTRY(arm926_dma_inv_range)
+arm926_dma_inv_range:
 #ifndef CONFIG_CPU_DCACHE_WRITETHROUGH
         tst     r0, #CACHE_DLINESIZE - 1
         mcrne   p15, 0, r0, c7, c10, 1          @ clean D entry
@@ -270,7 +271,7 @@ ENTRY(arm926_dma_inv_range)
  *
  * (same as v4wb)
  */
-ENTRY(arm926_dma_clean_range)
+arm926_dma_clean_range:
 #ifndef CONFIG_CPU_DCACHE_WRITETHROUGH
         bic     r0, r0, #CACHE_DLINESIZE - 1
 1:      mcr     p15, 0, r0, c7, c10, 1          @ clean D entry
@@ -303,15 +304,39 @@ ENTRY(arm926_dma_flush_range)
         mcr     p15, 0, r0, c7, c10, 4          @ drain WB
         mov     pc, lr
 
+/*
+ *      dma_map_area(start, size, dir)
+ *      - start - kernel virtual start address
+ *      - size  - size of region
+ *      - dir   - DMA direction
+ */
+ENTRY(arm926_dma_map_area)
+        add     r1, r1, r0
+        cmp     r2, #DMA_TO_DEVICE
+        beq     arm926_dma_clean_range
+        bcs     arm926_dma_inv_range
+        b       arm926_dma_flush_range
+ENDPROC(arm926_dma_map_area)
+
+/*
+ *      dma_unmap_area(start, size, dir)
+ *      - start - kernel virtual start address
+ *      - size  - size of region
+ *      - dir   - DMA direction
+ */
+ENTRY(arm926_dma_unmap_area)
+        mov     pc, lr
+ENDPROC(arm926_dma_unmap_area)
+
 ENTRY(arm926_cache_fns)
         .long   arm926_flush_kern_cache_all
         .long   arm926_flush_user_cache_all
         .long   arm926_flush_user_cache_range
         .long   arm926_coherent_kern_range
         .long   arm926_coherent_user_range
-        .long   arm926_flush_kern_dcache_page
-        .long   arm926_dma_inv_range
-        .long   arm926_dma_clean_range
+        .long   arm926_flush_kern_dcache_area
+        .long   arm926_dma_map_area
+        .long   arm926_dma_unmap_area
         .long   arm926_dma_flush_range
 
 ENTRY(cpu_arm926_dcache_clean_area)
diff --git a/arch/arm/mm/proc-arm940.S b/arch/arm/mm/proc-arm940.S
index 5b0f8464c8f2..1af1657819eb 100644
--- a/arch/arm/mm/proc-arm940.S
+++ b/arch/arm/mm/proc-arm940.S
@@ -141,14 +141,15 @@ ENTRY(arm940_coherent_user_range)
         /* FALLTHROUGH */
 
 /*
- *      flush_kern_dcache_page(void *page)
+ *      flush_kern_dcache_area(void *addr, size_t size)
  *
  *      Ensure no D cache aliasing occurs, either with itself or
  *      the I cache
  *
- *      - addr  - page aligned address
+ *      - addr  - kernel address
+ *      - size  - region size
  */
-ENTRY(arm940_flush_kern_dcache_page)
+ENTRY(arm940_flush_kern_dcache_area)
         mov     ip, #0
         mov     r1, #(CACHE_DSEGMENTS - 1) << 4 @ 4 segments
 1:      orr     r3, r1, #(CACHE_DENTRIES - 1) << 26 @ 64 entries
@@ -170,7 +171,7 @@ ENTRY(arm940_flush_kern_dcache_page)
  *      - start - virtual start address
  *      - end   - virtual end address
  */
-ENTRY(arm940_dma_inv_range)
+arm940_dma_inv_range:
         mov     ip, #0
         mov     r1, #(CACHE_DSEGMENTS - 1) << 4 @ 4 segments
 1:      orr     r3, r1, #(CACHE_DENTRIES - 1) << 26 @ 64 entries
@@ -191,7 +192,7 @@ ENTRY(arm940_dma_inv_range)
  *      - start - virtual start address
  *      - end   - virtual end address
  */
-ENTRY(arm940_dma_clean_range)
+arm940_dma_clean_range:
 ENTRY(cpu_arm940_dcache_clean_area)
         mov     ip, #0
 #ifndef CONFIG_CPU_DCACHE_WRITETHROUGH
@@ -232,15 +233,39 @@ ENTRY(arm940_dma_flush_range)
         mcr     p15, 0, ip, c7, c10, 4          @ drain WB
         mov     pc, lr
 
+/*
+ *      dma_map_area(start, size, dir)
+ *      - start - kernel virtual start address
+ *      - size  - size of region
+ *      - dir   - DMA direction
+ */
+ENTRY(arm940_dma_map_area)
+        add     r1, r1, r0
+        cmp     r2, #DMA_TO_DEVICE
+        beq     arm940_dma_clean_range
+        bcs     arm940_dma_inv_range
+        b       arm940_dma_flush_range
+ENDPROC(arm940_dma_map_area)
+
+/*
+ *      dma_unmap_area(start, size, dir)
+ *      - start - kernel virtual start address
+ *      - size  - size of region
+ *      - dir   - DMA direction
+ */
+ENTRY(arm940_dma_unmap_area)
+        mov     pc, lr
+ENDPROC(arm940_dma_unmap_area)
+
 ENTRY(arm940_cache_fns)
         .long   arm940_flush_kern_cache_all
         .long   arm940_flush_user_cache_all
         .long   arm940_flush_user_cache_range
         .long   arm940_coherent_kern_range
         .long   arm940_coherent_user_range
-        .long   arm940_flush_kern_dcache_page
-        .long   arm940_dma_inv_range
-        .long   arm940_dma_clean_range
+        .long   arm940_flush_kern_dcache_area
+        .long   arm940_dma_map_area
+        .long   arm940_dma_unmap_area
         .long   arm940_dma_flush_range
 
         __INIT
diff --git a/arch/arm/mm/proc-arm946.S b/arch/arm/mm/proc-arm946.S
index 40c0449a139b..1664b6aaff79 100644
--- a/arch/arm/mm/proc-arm946.S
+++ b/arch/arm/mm/proc-arm946.S
@@ -183,16 +183,17 @@ ENTRY(arm946_coherent_user_range)
         mov     pc, lr
 
 /*
- *      flush_kern_dcache_page(void *page)
+ *      flush_kern_dcache_area(void *addr, size_t size)
  *
  *      Ensure no D cache aliasing occurs, either with itself or
  *      the I cache
  *
- *      - addr  - page aligned address
+ *      - addr  - kernel address
+ *      - size  - region size
  * (same as arm926)
  */
-ENTRY(arm946_flush_kern_dcache_page)
-        add     r1, r0, #PAGE_SZ
+ENTRY(arm946_flush_kern_dcache_area)
+        add     r1, r0, r1
 1:      mcr     p15, 0, r0, c7, c14, 1          @ clean+invalidate D entry
         add     r0, r0, #CACHE_DLINESIZE
         cmp     r0, r1
@@ -214,7 +215,7 @@ ENTRY(arm946_flush_kern_dcache_page)
  *      - end   - virtual end address
  * (same as arm926)
  */
-ENTRY(arm946_dma_inv_range)
+arm946_dma_inv_range:
 #ifndef CONFIG_CPU_DCACHE_WRITETHROUGH
         tst     r0, #CACHE_DLINESIZE - 1
         mcrne   p15, 0, r0, c7, c10, 1          @ clean D entry
@@ -239,7 +240,7 @@ ENTRY(arm946_dma_inv_range)
  *
  * (same as arm926)
  */
-ENTRY(arm946_dma_clean_range)
+arm946_dma_clean_range:
 #ifndef CONFIG_CPU_DCACHE_WRITETHROUGH
         bic     r0, r0, #CACHE_DLINESIZE - 1
 1:      mcr     p15, 0, r0, c7, c10, 1          @ clean D entry
@@ -274,15 +275,39 @@ ENTRY(arm946_dma_flush_range)
         mcr     p15, 0, r0, c7, c10, 4          @ drain WB
         mov     pc, lr
 
+/*
+ *      dma_map_area(start, size, dir)
+ *      - start - kernel virtual start address
+ *      - size  - size of region
+ *      - dir   - DMA direction
+ */
+ENTRY(arm946_dma_map_area)
+        add     r1, r1, r0
+        cmp     r2, #DMA_TO_DEVICE
+        beq     arm946_dma_clean_range
+        bcs     arm946_dma_inv_range
+        b       arm946_dma_flush_range
+ENDPROC(arm946_dma_map_area)
+
+/*
+ *      dma_unmap_area(start, size, dir)
+ *      - start - kernel virtual start address
+ *      - size  - size of region
+ *      - dir   - DMA direction
+ */
+ENTRY(arm946_dma_unmap_area)
+        mov     pc, lr
+ENDPROC(arm946_dma_unmap_area)
+
 ENTRY(arm946_cache_fns)
         .long   arm946_flush_kern_cache_all
         .long   arm946_flush_user_cache_all
         .long   arm946_flush_user_cache_range
         .long   arm946_coherent_kern_range
         .long   arm946_coherent_user_range
-        .long   arm946_flush_kern_dcache_page
-        .long   arm946_dma_inv_range
-        .long   arm946_dma_clean_range
+        .long   arm946_flush_kern_dcache_area
+        .long   arm946_dma_map_area
+        .long   arm946_dma_unmap_area
         .long   arm946_dma_flush_range
 
 
diff --git a/arch/arm/mm/proc-feroceon.S b/arch/arm/mm/proc-feroceon.S
index d0d7795200fc..53e632343849 100644
--- a/arch/arm/mm/proc-feroceon.S
+++ b/arch/arm/mm/proc-feroceon.S
@@ -226,16 +226,17 @@ ENTRY(feroceon_coherent_user_range)
         mov     pc, lr
 
 /*
- *      flush_kern_dcache_page(void *page)
+ *      flush_kern_dcache_area(void *addr, size_t size)
  *
  *      Ensure no D cache aliasing occurs, either with itself or
  *      the I cache
  *
- *      - addr  - page aligned address
+ *      - addr  - kernel address
+ *      - size  - region size
  */
         .align  5
-ENTRY(feroceon_flush_kern_dcache_page)
-        add     r1, r0, #PAGE_SZ
+ENTRY(feroceon_flush_kern_dcache_area)
+        add     r1, r0, r1
 1:      mcr     p15, 0, r0, c7, c14, 1          @ clean+invalidate D entry
         add     r0, r0, #CACHE_DLINESIZE
         cmp     r0, r1
@@ -246,7 +247,7 @@ ENTRY(feroceon_flush_kern_dcache_page)
         mov     pc, lr
 
         .align  5
-ENTRY(feroceon_range_flush_kern_dcache_page)
+ENTRY(feroceon_range_flush_kern_dcache_area)
         mrs     r2, cpsr
         add     r1, r0, #PAGE_SZ - CACHE_DLINESIZE      @ top addr is inclusive
         orr     r3, r2, #PSR_I_BIT
@@ -273,7 +274,7 @@ ENTRY(feroceon_range_flush_kern_dcache_page)
  * (same as v4wb)
  */
         .align  5
-ENTRY(feroceon_dma_inv_range)
+feroceon_dma_inv_range:
         tst     r0, #CACHE_DLINESIZE - 1
         bic     r0, r0, #CACHE_DLINESIZE - 1
         mcrne   p15, 0, r0, c7, c10, 1          @ clean D entry
@@ -287,7 +288,7 @@ ENTRY(feroceon_dma_inv_range)
         mov     pc, lr
 
         .align  5
-ENTRY(feroceon_range_dma_inv_range)
+feroceon_range_dma_inv_range:
         mrs     r2, cpsr
         tst     r0, #CACHE_DLINESIZE - 1
         mcrne   p15, 0, r0, c7, c10, 1          @ clean D entry
@@ -313,7 +314,7 @@ ENTRY(feroceon_range_dma_inv_range)
  * (same as v4wb)
  */
         .align  5
-ENTRY(feroceon_dma_clean_range)
+feroceon_dma_clean_range:
         bic     r0, r0, #CACHE_DLINESIZE - 1
 1:      mcr     p15, 0, r0, c7, c10, 1          @ clean D entry
         add     r0, r0, #CACHE_DLINESIZE
@@ -323,7 +324,7 @@ ENTRY(feroceon_dma_clean_range)
         mov     pc, lr
 
         .align  5
-ENTRY(feroceon_range_dma_clean_range)
+feroceon_range_dma_clean_range:
         mrs     r2, cpsr
         cmp     r1, r0
         subne   r1, r1, #1                      @ top address is inclusive
@@ -366,15 +367,53 @@ ENTRY(feroceon_range_dma_flush_range)
         mcr     p15, 0, r0, c7, c10, 4          @ drain WB
         mov     pc, lr
 
+/*
+ *      dma_map_area(start, size, dir)
+ *      - start - kernel virtual start address
+ *      - size  - size of region
+ *      - dir   - DMA direction
+ */
+ENTRY(feroceon_dma_map_area)
+        add     r1, r1, r0
+        cmp     r2, #DMA_TO_DEVICE
+        beq     feroceon_dma_clean_range
+        bcs     feroceon_dma_inv_range
+        b       feroceon_dma_flush_range
+ENDPROC(feroceon_dma_map_area)
+
+/*
+ *      dma_map_area(start, size, dir)
+ *      - start - kernel virtual start address
+ *      - size  - size of region
+ *      - dir   - DMA direction
+ */
+ENTRY(feroceon_range_dma_map_area)
+        add     r1, r1, r0
+        cmp     r2, #DMA_TO_DEVICE
+        beq     feroceon_range_dma_clean_range
+        bcs     feroceon_range_dma_inv_range
+        b       feroceon_range_dma_flush_range
+ENDPROC(feroceon_range_dma_map_area)
+
+/*
+ *      dma_unmap_area(start, size, dir)
+ *      - start - kernel virtual start address
+ *      - size  - size of region
+ *      - dir   - DMA direction
+ */
+ENTRY(feroceon_dma_unmap_area)
+        mov     pc, lr
+ENDPROC(feroceon_dma_unmap_area)
+
 ENTRY(feroceon_cache_fns)
         .long   feroceon_flush_kern_cache_all
         .long   feroceon_flush_user_cache_all
         .long   feroceon_flush_user_cache_range
         .long   feroceon_coherent_kern_range
         .long   feroceon_coherent_user_range
-        .long   feroceon_flush_kern_dcache_page
-        .long   feroceon_dma_inv_range
-        .long   feroceon_dma_clean_range
+        .long   feroceon_flush_kern_dcache_area
+        .long   feroceon_dma_map_area
+        .long   feroceon_dma_unmap_area
         .long   feroceon_dma_flush_range
 
 ENTRY(feroceon_range_cache_fns)
@@ -383,9 +422,9 @@ ENTRY(feroceon_range_cache_fns)
         .long   feroceon_flush_user_cache_range
         .long   feroceon_coherent_kern_range
         .long   feroceon_coherent_user_range
-        .long   feroceon_range_flush_kern_dcache_page
-        .long   feroceon_range_dma_inv_range
-        .long   feroceon_range_dma_clean_range
+        .long   feroceon_range_flush_kern_dcache_area
+        .long   feroceon_range_dma_map_area
+        .long   feroceon_dma_unmap_area
         .long   feroceon_range_dma_flush_range
 
         .align  5
diff --git a/arch/arm/mm/proc-mohawk.S b/arch/arm/mm/proc-mohawk.S
index 52b5fd74fbb3..caa31154e7db 100644
--- a/arch/arm/mm/proc-mohawk.S
+++ b/arch/arm/mm/proc-mohawk.S
@@ -186,15 +186,16 @@ ENTRY(mohawk_coherent_user_range)
         mov     pc, lr
 
 /*
- *      flush_kern_dcache_page(void *page)
+ *      flush_kern_dcache_area(void *addr, size_t size)
  *
  *      Ensure no D cache aliasing occurs, either with itself or
  *      the I cache
  *
- *      - addr  - page aligned address
+ *      - addr  - kernel address
+ *      - size  - region size
  */
-ENTRY(mohawk_flush_kern_dcache_page)
-        add     r1, r0, #PAGE_SZ
+ENTRY(mohawk_flush_kern_dcache_area)
+        add     r1, r0, r1
 1:      mcr     p15, 0, r0, c7, c14, 1          @ clean+invalidate D entry
         add     r0, r0, #CACHE_DLINESIZE
         cmp     r0, r1
@@ -217,7 +218,7 @@ ENTRY(mohawk_flush_kern_dcache_page)
  *
  * (same as v4wb)
  */
-ENTRY(mohawk_dma_inv_range)
+mohawk_dma_inv_range:
         tst     r0, #CACHE_DLINESIZE - 1
         mcrne   p15, 0, r0, c7, c10, 1          @ clean D entry
         tst     r1, #CACHE_DLINESIZE - 1
@@ -240,7 +241,7 @@ ENTRY(mohawk_dma_inv_range)
  *
  * (same as v4wb)
  */
-ENTRY(mohawk_dma_clean_range)
+mohawk_dma_clean_range:
         bic     r0, r0, #CACHE_DLINESIZE - 1
 1:      mcr     p15, 0, r0, c7, c10, 1          @ clean D entry
         add     r0, r0, #CACHE_DLINESIZE
@@ -267,15 +268,39 @@ ENTRY(mohawk_dma_flush_range)
         mcr     p15, 0, r0, c7, c10, 4          @ drain WB
         mov     pc, lr
 
+/*
+ *      dma_map_area(start, size, dir)
+ *      - start - kernel virtual start address
+ *      - size  - size of region
+ *      - dir   - DMA direction
+ */
+ENTRY(mohawk_dma_map_area)
+        add     r1, r1, r0
+        cmp     r2, #DMA_TO_DEVICE
+        beq     mohawk_dma_clean_range
+        bcs     mohawk_dma_inv_range
+        b       mohawk_dma_flush_range
+ENDPROC(mohawk_dma_map_area)
+
+/*
+ *      dma_unmap_area(start, size, dir)
+ *      - start - kernel virtual start address
+ *      - size  - size of region
+ *      - dir   - DMA direction
+ */
+ENTRY(mohawk_dma_unmap_area)
+        mov     pc, lr
+ENDPROC(mohawk_dma_unmap_area)
+
 ENTRY(mohawk_cache_fns)
         .long   mohawk_flush_kern_cache_all
         .long   mohawk_flush_user_cache_all
         .long   mohawk_flush_user_cache_range
         .long   mohawk_coherent_kern_range
         .long   mohawk_coherent_user_range
-        .long   mohawk_flush_kern_dcache_page
-        .long   mohawk_dma_inv_range
-        .long   mohawk_dma_clean_range
+        .long   mohawk_flush_kern_dcache_area
+        .long   mohawk_dma_map_area
+        .long   mohawk_dma_unmap_area
         .long   mohawk_dma_flush_range
 
 ENTRY(cpu_mohawk_dcache_clean_area)
diff --git a/arch/arm/mm/proc-sa1100.S b/arch/arm/mm/proc-sa1100.S
index ee7700242c19..5c47760c2064 100644
--- a/arch/arm/mm/proc-sa1100.S
+++ b/arch/arm/mm/proc-sa1100.S
@@ -45,7 +45,7 @@ ENTRY(cpu_sa1100_proc_init)
         mcr     p15, 0, r0, c9, c0, 5           @ Allow read-buffer operations from userland
         mov     pc, lr
 
-        .previous
+        .section .text
 
 /*
  * cpu_sa1100_proc_fin()
diff --git a/arch/arm/mm/proc-syms.c b/arch/arm/mm/proc-syms.c
index ac5c80062b70..3e6210b4d6d4 100644
--- a/arch/arm/mm/proc-syms.c
+++ b/arch/arm/mm/proc-syms.c
@@ -27,8 +27,7 @@ EXPORT_SYMBOL(__cpuc_flush_kern_all);
 EXPORT_SYMBOL(__cpuc_flush_user_all);
 EXPORT_SYMBOL(__cpuc_flush_user_range);
 EXPORT_SYMBOL(__cpuc_coherent_kern_range);
-EXPORT_SYMBOL(__cpuc_flush_dcache_page);
-EXPORT_SYMBOL(dmac_inv_range);  /* because of flush_ioremap_region() */
+EXPORT_SYMBOL(__cpuc_flush_dcache_area);
 #else
 EXPORT_SYMBOL(cpu_cache);
 #endif
diff --git a/arch/arm/mm/proc-v6.S b/arch/arm/mm/proc-v6.S
index 70f75d2e3ead..7a5337ed7d68 100644
--- a/arch/arm/mm/proc-v6.S
+++ b/arch/arm/mm/proc-v6.S
@@ -59,8 +59,6 @@ ENTRY(cpu_v6_proc_fin)
  *      to what would be the reset vector.
  *
  *      - loc   - location to jump to for soft reset
- *
- *      It is assumed that:
  */
         .align  5
 ENTRY(cpu_v6_reset)
@@ -130,9 +128,16 @@ ENTRY(cpu_v6_set_pte_ext)
 
 
 
-
+        .type   cpu_v6_name, #object
 cpu_v6_name:
         .asciz  "ARMv6-compatible processor"
+        .size   cpu_v6_name, . - cpu_v6_name
+
+        .type   cpu_pj4_name, #object
+cpu_pj4_name:
+        .asciz  "Marvell PJ4 processor"
+        .size   cpu_pj4_name, . - cpu_pj4_name
+
         .align
 
         __INIT
@@ -241,3 +246,26 @@ __v6_proc_info:
         .long   v6_user_fns
         .long   v6_cache_fns
         .size   __v6_proc_info, . - __v6_proc_info
+
+        .type   __pj4_v6_proc_info, #object
+__pj4_v6_proc_info:
+        .long   0x560f5810
+        .long   0xff0ffff0
+        .long   PMD_TYPE_SECT | \
+                PMD_SECT_AP_WRITE | \
+                PMD_SECT_AP_READ | \
+                PMD_FLAGS
+        .long   PMD_TYPE_SECT | \
+                PMD_SECT_XN | \
+                PMD_SECT_AP_WRITE | \
+                PMD_SECT_AP_READ
+        b       __v6_setup
+        .long   cpu_arch_name
+        .long   cpu_elf_name
+        .long   HWCAP_SWP|HWCAP_HALF|HWCAP_THUMB|HWCAP_FAST_MULT|HWCAP_EDSP
+        .long   cpu_pj4_name
+        .long   v6_processor_functions
+        .long   v6wbi_tlb_fns
+        .long   v6_user_fns
+        .long   v6_cache_fns
+        .size   __pj4_v6_proc_info, . - __pj4_v6_proc_info
diff --git a/arch/arm/mm/proc-v7.S b/arch/arm/mm/proc-v7.S
index 3a285218fd15..7aaf88a3b7aa 100644
--- a/arch/arm/mm/proc-v7.S
+++ b/arch/arm/mm/proc-v7.S
@@ -45,7 +45,14 @@ ENTRY(cpu_v7_proc_init)
 ENDPROC(cpu_v7_proc_init)
 
 ENTRY(cpu_v7_proc_fin)
-        mov     pc, lr
+        stmfd   sp!, {lr}
+        cpsid   if                              @ disable interrupts
+        bl      v7_flush_kern_cache_all
+        mrc     p15, 0, r0, c1, c0, 0           @ ctrl register
+        bic     r0, r0, #0x1000                 @ ...i............
+        bic     r0, r0, #0x0006                 @ .............ca.
+        mcr     p15, 0, r0, c1, c0, 0           @ disable caches
+        ldmfd   sp!, {pc}
 ENDPROC(cpu_v7_proc_fin)
 
 /*
@@ -56,8 +63,6 @@ ENDPROC(cpu_v7_proc_fin)
  *      to what would be the reset vector.
  *
  *      - loc   - location to jump to for soft reset
- *
- *      It is assumed that:
  */
         .align  5
 ENTRY(cpu_v7_reset)
diff --git a/arch/arm/mm/proc-xsc3.S b/arch/arm/mm/proc-xsc3.S
index 2028f3702881..e5797f1c1db7 100644
--- a/arch/arm/mm/proc-xsc3.S
+++ b/arch/arm/mm/proc-xsc3.S
@@ -226,15 +226,16 @@ ENTRY(xsc3_coherent_user_range)
         mov     pc, lr
 
 /*
- *      flush_kern_dcache_page(void *page)
+ *      flush_kern_dcache_area(void *addr, size_t size)
  *
  *      Ensure no D cache aliasing occurs, either with itself or
  *      the I cache.
  *
- *      - addr  - page aligned address
+ *      - addr  - kernel address
+ *      - size  - region size
  */
-ENTRY(xsc3_flush_kern_dcache_page)
-        add     r1, r0, #PAGE_SZ
+ENTRY(xsc3_flush_kern_dcache_area)
+        add     r1, r0, r1
 1:      mcr     p15, 0, r0, c7, c14, 1          @ clean/invalidate L1 D line
         add     r0, r0, #CACHELINESIZE
         cmp     r0, r1
@@ -256,7 +257,7 @@ ENTRY(xsc3_flush_kern_dcache_page)
  *      - start  - virtual start address
  *      - end    - virtual end address
  */
-ENTRY(xsc3_dma_inv_range)
+xsc3_dma_inv_range:
         tst     r0, #CACHELINESIZE - 1
         bic     r0, r0, #CACHELINESIZE - 1
         mcrne   p15, 0, r0, c7, c10, 1          @ clean L1 D line
@@ -277,7 +278,7 @@ ENTRY(xsc3_dma_inv_range)
  *      - start  - virtual start address
  *      - end    - virtual end address
  */
-ENTRY(xsc3_dma_clean_range)
+xsc3_dma_clean_range:
         bic     r0, r0, #CACHELINESIZE - 1
 1:      mcr     p15, 0, r0, c7, c10, 1          @ clean L1 D line
         add     r0, r0, #CACHELINESIZE
@@ -303,15 +304,39 @@ ENTRY(xsc3_dma_flush_range)
         mcr     p15, 0, r0, c7, c10, 4          @ data write barrier
         mov     pc, lr
 
+/*
+ *      dma_map_area(start, size, dir)
+ *      - start - kernel virtual start address
+ *      - size  - size of region
+ *      - dir   - DMA direction
+ */
+ENTRY(xsc3_dma_map_area)
+        add     r1, r1, r0
+        cmp     r2, #DMA_TO_DEVICE
+        beq     xsc3_dma_clean_range
+        bcs     xsc3_dma_inv_range
+        b       xsc3_dma_flush_range
+ENDPROC(xsc3_dma_map_area)
+
+/*
+ *      dma_unmap_area(start, size, dir)
+ *      - start - kernel virtual start address
+ *      - size  - size of region
+ *      - dir   - DMA direction
+ */
+ENTRY(xsc3_dma_unmap_area)
+        mov     pc, lr
+ENDPROC(xsc3_dma_unmap_area)
+
 ENTRY(xsc3_cache_fns)
         .long   xsc3_flush_kern_cache_all
         .long   xsc3_flush_user_cache_all
         .long   xsc3_flush_user_cache_range
         .long   xsc3_coherent_kern_range
         .long   xsc3_coherent_user_range
-        .long   xsc3_flush_kern_dcache_page
-        .long   xsc3_dma_inv_range
-        .long   xsc3_dma_clean_range
+        .long   xsc3_flush_kern_dcache_area
+        .long   xsc3_dma_map_area
+        .long   xsc3_dma_unmap_area
         .long   xsc3_dma_flush_range
 
 ENTRY(cpu_xsc3_dcache_clean_area)
@@ -396,7 +421,7 @@ __xsc3_setup:
         orr     r4, r4, #0x18                   @ cache the page table in L2
         mcr     p15, 0, r4, c2, c0, 0           @ load page table pointer
 
-        mov     r0, #0                          @ don't allow CP access
+        mov     r0, #1 << 6                     @ cp6 access for early sched_clock
         mcr     p15, 0, r0, c15, c1, 0          @ write CP access register
 
         mrc     p15, 0, r0, c1, c0, 1           @ get auxiliary control reg
@@ -406,6 +431,13 @@ __xsc3_setup:
 
         adr     r5, xsc3_crval
         ldmia   r5, {r5, r6}
+
+#ifdef CONFIG_CACHE_XSC3L2
+        mrc     p15, 1, r0, c0, c0, 1           @ get L2 present information
+        ands    r0, r0, #0xf8
+        orrne   r6, r6, #(1 << 26)              @ enable L2 if present
+#endif
+
         mrc     p15, 0, r0, c1, c0, 0           @ get control register
         bic     r0, r0, r5                      @ ..V. ..R. .... ..A.
         orr     r0, r0, r6                      @ ..VI Z..S .... .C.M (mmu)
diff --git a/arch/arm/mm/proc-xscale.S b/arch/arm/mm/proc-xscale.S
index f056c283682d..63037e2162f2 100644
--- a/arch/arm/mm/proc-xscale.S
+++ b/arch/arm/mm/proc-xscale.S
@@ -284,15 +284,16 @@ ENTRY(xscale_coherent_user_range)
         mov     pc, lr
 
 /*
- *      flush_kern_dcache_page(void *page)
+ *      flush_kern_dcache_area(void *addr, size_t size)
  *
  *      Ensure no D cache aliasing occurs, either with itself or
  *      the I cache
  *
- *      - addr  - page aligned address
+ *      - addr  - kernel address
+ *      - size  - region size
  */
-ENTRY(xscale_flush_kern_dcache_page)
-        add     r1, r0, #PAGE_SZ
+ENTRY(xscale_flush_kern_dcache_area)
+        add     r1, r0, r1
 1:      mcr     p15, 0, r0, c7, c10, 1          @ clean D entry
         mcr     p15, 0, r0, c7, c6, 1           @ invalidate D entry
         add     r0, r0, #CACHELINESIZE
@@ -314,7 +315,7 @@ ENTRY(xscale_flush_kern_dcache_page)
  *      - start  - virtual start address
  *      - end    - virtual end address
  */
-ENTRY(xscale_dma_inv_range)
+xscale_dma_inv_range:
         tst     r0, #CACHELINESIZE - 1
         bic     r0, r0, #CACHELINESIZE - 1
         mcrne   p15, 0, r0, c7, c10, 1          @ clean D entry
@@ -335,7 +336,7 @@ ENTRY(xscale_dma_inv_range)
  *      - start  - virtual start address
  *      - end    - virtual end address
  */
-ENTRY(xscale_dma_clean_range)
+xscale_dma_clean_range:
         bic     r0, r0, #CACHELINESIZE - 1
 1:      mcr     p15, 0, r0, c7, c10, 1          @ clean D entry
         add     r0, r0, #CACHELINESIZE
@@ -362,15 +363,52 @@ ENTRY(xscale_dma_flush_range)
         mcr     p15, 0, r0, c7, c10, 4          @ Drain Write (& Fill) Buffer
         mov     pc, lr
 
+/*
+ *      dma_map_area(start, size, dir)
+ *      - start - kernel virtual start address
+ *      - size  - size of region
+ *      - dir   - DMA direction
+ */
+ENTRY(xscale_dma_map_area)
+        add     r1, r1, r0
+        cmp     r2, #DMA_TO_DEVICE
+        beq     xscale_dma_clean_range
+        bcs     xscale_dma_inv_range
+        b       xscale_dma_flush_range
+ENDPROC(xscale_dma_map_area)
+
+/*
+ *      dma_map_area(start, size, dir)
+ *      - start - kernel virtual start address
+ *      - size  - size of region
+ *      - dir   - DMA direction
+ */
+ENTRY(xscale_dma_a0_map_area)
+        add     r1, r1, r0
+        teq     r2, #DMA_TO_DEVICE
+        beq     xscale_dma_clean_range
+        b       xscale_dma_flush_range
+ENDPROC(xscsale_dma_a0_map_area)
+
+/*
+ *      dma_unmap_area(start, size, dir)
+ *      - start - kernel virtual start address
+ *      - size  - size of region
+ *      - dir   - DMA direction
+ */
+ENTRY(xscale_dma_unmap_area)
+        mov     pc, lr
+ENDPROC(xscale_dma_unmap_area)
+
 ENTRY(xscale_cache_fns)
         .long   xscale_flush_kern_cache_all
         .long   xscale_flush_user_cache_all
         .long   xscale_flush_user_cache_range
         .long   xscale_coherent_kern_range
         .long   xscale_coherent_user_range
-        .long   xscale_flush_kern_dcache_page
-        .long   xscale_dma_inv_range
-        .long   xscale_dma_clean_range
+        .long   xscale_flush_kern_dcache_area
+        .long   xscale_dma_map_area
+        .long   xscale_dma_unmap_area
         .long   xscale_dma_flush_range
 
 /*
@@ -392,9 +430,9 @@ ENTRY(xscale_80200_A0_A1_cache_fns)
         .long   xscale_flush_user_cache_range
         .long   xscale_coherent_kern_range
         .long   xscale_coherent_user_range
-        .long   xscale_flush_kern_dcache_page
-        .long   xscale_dma_flush_range
-        .long   xscale_dma_clean_range
+        .long   xscale_flush_kern_dcache_area
+        .long   xscale_dma_a0_map_area
+        .long   xscale_dma_unmap_area
         .long   xscale_dma_flush_range
 
 ENTRY(cpu_xscale_dcache_clean_area)
diff --git a/arch/arm/mm/tlb-v7.S b/arch/arm/mm/tlb-v7.S
index a26a605b73bd..f3f288a9546d 100644
--- a/arch/arm/mm/tlb-v7.S
+++ b/arch/arm/mm/tlb-v7.S
@@ -40,7 +40,6 @@ ENTRY(v7wbi_flush_user_tlb_range)
         asid    r3, r3                          @ mask ASID
         orr     r0, r3, r0, lsl #PAGE_SHIFT     @ Create initial MVA
         mov     r1, r1, lsl #PAGE_SHIFT
-        vma_vm_flags r2, r2                     @ get vma->vm_flags
 1:
 #ifdef CONFIG_SMP
         mcr     p15, 0, r0, c8, c3, 1           @ TLB invalidate U MVA (shareable) 
@@ -51,7 +50,11 @@ ENTRY(v7wbi_flush_user_tlb_range)
         cmp     r0, r1
         blo     1b
         mov     ip, #0
+#ifdef CONFIG_SMP
+        mcr     p15, 0, ip, c7, c1, 6           @ flush BTAC/BTB Inner Shareable
+#else
         mcr     p15, 0, ip, c7, c5, 6           @ flush BTAC/BTB
+#endif
         dsb
         mov     pc, lr
 ENDPROC(v7wbi_flush_user_tlb_range)
@@ -80,7 +83,11 @@ ENTRY(v7wbi_flush_kern_tlb_range)
         cmp     r0, r1
         blo     1b
         mov     r2, #0
+#ifdef CONFIG_SMP
+        mcr     p15, 0, r2, c7, c1, 6           @ flush BTAC/BTB Inner Shareable
+#else
         mcr     p15, 0, r2, c7, c5, 6           @ flush BTAC/BTB
+#endif
         dsb
         isb
         mov     pc, lr
diff --git a/arch/arm/mm/vmregion.c b/arch/arm/mm/vmregion.c
new file mode 100644
index 000000000000..19e09bdb1b8a
--- /dev/null
+++ b/arch/arm/mm/vmregion.c
@@ -0,0 +1,131 @@
+#include <linux/spinlock.h>
+#include <linux/list.h>
+#include <linux/slab.h>
+
+#include "vmregion.h"
+
+/*
+ * VM region handling support.
+ *
+ * This should become something generic, handling VM region allocations for
+ * vmalloc and similar (ioremap, module space, etc).
+ *
+ * I envisage vmalloc()'s supporting vm_struct becoming:
+ *
+ *  struct vm_struct {
+ *    struct vmregion   region;
+ *    unsigned long     flags;
+ *    struct page       **pages;
+ *    unsigned int      nr_pages;
+ *    unsigned long     phys_addr;
+ *  };
+ *
+ * get_vm_area() would then call vmregion_alloc with an appropriate
+ * struct vmregion head (eg):
+ *
+ *  struct vmregion vmalloc_head = {
+ *      .vm_list        = LIST_HEAD_INIT(vmalloc_head.vm_list),
+ *      .vm_start       = VMALLOC_START,
+ *      .vm_end         = VMALLOC_END,
+ *  };
+ *
+ * However, vmalloc_head.vm_start is variable (typically, it is dependent on
+ * the amount of RAM found at boot time.)  I would imagine that get_vm_area()
+ * would have to initialise this each time prior to calling vmregion_alloc().
+ */
+
+struct arm_vmregion *
+arm_vmregion_alloc(struct arm_vmregion_head *head, size_t size, gfp_t gfp)
+{
+        unsigned long addr = head->vm_start, end = head->vm_end - size;
+        unsigned long flags;
+        struct arm_vmregion *c, *new;
+
+        if (head->vm_end - head->vm_start < size) {
+                printk(KERN_WARNING "%s: allocation too big (requested %#x)\n",
+                        __func__, size);
+                goto out;
+        }
+
+        new = kmalloc(sizeof(struct arm_vmregion), gfp);
+        if (!new)
+                goto out;
+
+        spin_lock_irqsave(&head->vm_lock, flags);
+
+        list_for_each_entry(c, &head->vm_list, vm_list) {
+                if ((addr + size) < addr)
+                        goto nospc;
+                if ((addr + size) <= c->vm_start)
+                        goto found;
+                addr = c->vm_end;
+                if (addr > end)
+                        goto nospc;
+        }
+
+ found:
+        /*
+         * Insert this entry _before_ the one we found.
+         */
+        list_add_tail(&new->vm_list, &c->vm_list);
+        new->vm_start = addr;
+        new->vm_end = addr + size;
+        new->vm_active = 1;
+
+        spin_unlock_irqrestore(&head->vm_lock, flags);
+        return new;
+
+ nospc:
+        spin_unlock_irqrestore(&head->vm_lock, flags);
+        kfree(new);
+ out:
+        return NULL;
+}
+
+static struct arm_vmregion *__arm_vmregion_find(struct arm_vmregion_head *head, unsigned long addr)
+{
+        struct arm_vmregion *c;
+
+        list_for_each_entry(c, &head->vm_list, vm_list) {
+                if (c->vm_active && c->vm_start == addr)
+                        goto out;
+        }
+        c = NULL;
+ out:
+        return c;
+}
+
+struct arm_vmregion *arm_vmregion_find(struct arm_vmregion_head *head, unsigned long addr)
+{
+        struct arm_vmregion *c;
+        unsigned long flags;
+
+        spin_lock_irqsave(&head->vm_lock, flags);
+        c = __arm_vmregion_find(head, addr);
+        spin_unlock_irqrestore(&head->vm_lock, flags);
+        return c;
+}
+
+struct arm_vmregion *arm_vmregion_find_remove(struct arm_vmregion_head *head, unsigned long addr)
+{
+        struct arm_vmregion *c;
+        unsigned long flags;
+
+        spin_lock_irqsave(&head->vm_lock, flags);
+        c = __arm_vmregion_find(head, addr);
+        if (c)
+                c->vm_active = 0;
+        spin_unlock_irqrestore(&head->vm_lock, flags);
+        return c;
+}
+
+void arm_vmregion_free(struct arm_vmregion_head *head, struct arm_vmregion *c)
+{
+        unsigned long flags;
+
+        spin_lock_irqsave(&head->vm_lock, flags);
+        list_del(&c->vm_list);
+        spin_unlock_irqrestore(&head->vm_lock, flags);
+
+        kfree(c);
+}
diff --git a/arch/arm/mm/vmregion.h b/arch/arm/mm/vmregion.h
new file mode 100644
index 000000000000..6b2cdbdf3a85
--- /dev/null
+++ b/arch/arm/mm/vmregion.h
@@ -0,0 +1,29 @@
+#ifndef VMREGION_H
+#define VMREGION_H
+
+#include <linux/spinlock.h>
+#include <linux/list.h>
+
+struct page;
+
+struct arm_vmregion_head {
+        spinlock_t              vm_lock;
+        struct list_head        vm_list;
+        unsigned long           vm_start;
+        unsigned long           vm_end;
+};
+
+struct arm_vmregion {
+        struct list_head        vm_list;
+        unsigned long           vm_start;
+        unsigned long           vm_end;
+        struct page             *vm_pages;
+        int                     vm_active;
+};
+
+struct arm_vmregion *arm_vmregion_alloc(struct arm_vmregion_head *, size_t, gfp_t);
+struct arm_vmregion *arm_vmregion_find(struct arm_vmregion_head *, unsigned long);
+struct arm_vmregion *arm_vmregion_find_remove(struct arm_vmregion_head *, unsigned long);
+void arm_vmregion_free(struct arm_vmregion_head *, struct arm_vmregion *);
+
+#endif