Merge branch 'for-linus' of git://git.linaro.org/people/mszyprowski/linux-dma-mapping

Pull CMA and ARM DMA-mapping updates from Marek Szyprowski:
 "These patches contain two major updates for DMA mapping subsystem
  (mainly for ARM architecture).  First one is Contiguous Memory
  Allocator (CMA) which makes it possible for device drivers to allocate
  big contiguous chunks of memory after the system has booted.

  The main difference from the similar frameworks is the fact that CMA
  allows to transparently reuse the memory region reserved for the big
  chunk allocation as a system memory, so no memory is wasted when no
  big chunk is allocated.  Once the alloc request is issued, the
  framework migrates system pages to create space for the required big
  chunk of physically contiguous memory.

  For more information one can refer to nice LWN articles:

   - 'A reworked contiguous memory allocator':
		http://lwn.net/Articles/447405/

   - 'CMA and ARM':
		http://lwn.net/Articles/450286/

   - 'A deep dive into CMA':
		http://lwn.net/Articles/486301/

   - and the following thread with the patches and links to all previous
     versions:
		https://lkml.org/lkml/2012/4/3/204

  The main client for this new framework is ARM DMA-mapping subsystem.

  The second part provides a complete redesign in ARM DMA-mapping
  subsystem.  The core implementation has been changed to use common
  struct dma_map_ops based infrastructure with the recent updates for
  new dma attributes merged in v3.4-rc2.  This allows to use more than
  one implementation of dma-mapping calls and change/select them on the
  struct device basis.  The first client of this new infractructure is
  dmabounce implementation which has been completely cut out of the
  core, common code.

  The last patch of this redesign update introduces a new, experimental
  implementation of dma-mapping calls on top of generic IOMMU framework.
  This lets ARM sub-platform to transparently use IOMMU for DMA-mapping
  calls if one provides required IOMMU hardware.

  For more information please refer to the following thread:
		http://www.spinics.net/lists/arm-kernel/msg175729.html

  The last patch merges changes from both updates and provides a
  resolution for the conflicts which cannot be avoided when patches have
  been applied on the same files (mainly arch/arm/mm/dma-mapping.c)."

Acked by Andrew Morton <akpm@linux-foundation.org>:
 "Yup, this one please.  It's had much work, plenty of review and I
  think even Russell is happy with it."

* 'for-linus' of git://git.linaro.org/people/mszyprowski/linux-dma-mapping: (28 commits)
  ARM: dma-mapping: use PMD size for section unmap
  cma: fix migration mode
  ARM: integrate CMA with DMA-mapping subsystem
  X86: integrate CMA with DMA-mapping subsystem
  drivers: add Contiguous Memory Allocator
  mm: trigger page reclaim in alloc_contig_range() to stabilise watermarks
  mm: extract reclaim code from __alloc_pages_direct_reclaim()
  mm: Serialize access to min_free_kbytes
  mm: page_isolation: MIGRATE_CMA isolation functions added
  mm: mmzone: MIGRATE_CMA migration type added
  mm: page_alloc: change fallbacks array handling
  mm: page_alloc: introduce alloc_contig_range()
  mm: compaction: export some of the functions
  mm: compaction: introduce isolate_freepages_range()
  mm: compaction: introduce map_pages()
  mm: compaction: introduce isolate_migratepages_range()
  mm: page_alloc: remove trailing whitespace
  ARM: dma-mapping: add support for IOMMU mapper
  ARM: dma-mapping: use alloc, mmap, free from dma_ops
  ARM: dma-mapping: remove redundant code and do the cleanup
  ...

Conflicts:
	arch/x86/include/asm/dma-mapping.h

5 files changed, 1192 insertions, 215 deletions

diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c
index db23ae4aaaab..ea6b43154090 100644
--- a/arch/arm/mm/dma-mapping.c
+++ b/arch/arm/mm/dma-mapping.c
@@ -17,8 +17,12 @@
 #include <linux/init.h>
 #include <linux/device.h>
 #include <linux/dma-mapping.h>
+#include <linux/dma-contiguous.h>
 #include <linux/highmem.h>
+#include <linux/memblock.h>
 #include <linux/slab.h>
+#include <linux/iommu.h>
+#include <linux/vmalloc.h>
 
 #include <asm/memory.h>
 #include <asm/highmem.h>
@@ -26,9 +30,112 @@
 #include <asm/tlbflush.h>
 #include <asm/sizes.h>
 #include <asm/mach/arch.h>
+#include <asm/dma-iommu.h>
+#include <asm/mach/map.h>
+#include <asm/system_info.h>
+#include <asm/dma-contiguous.h>
 
 #include "mm.h"
 
+/*
+ * The DMA API is built upon the notion of "buffer ownership".  A buffer
+ * is either exclusively owned by the CPU (and therefore may be accessed
+ * by it) or exclusively owned by the DMA device.  These helper functions
+ * represent the transitions between these two ownership states.
+ *
+ * Note, however, that on later ARMs, this notion does not work due to
+ * speculative prefetches.  We model our approach on the assumption that
+ * the CPU does do speculative prefetches, which means we clean caches
+ * before transfers and delay cache invalidation until transfer completion.
+ *
+ */
+static void __dma_page_cpu_to_dev(struct page *, unsigned long,
+                size_t, enum dma_data_direction);
+static void __dma_page_dev_to_cpu(struct page *, unsigned long,
+                size_t, enum dma_data_direction);
+
+/**
+ * arm_dma_map_page - map a portion of a page for streaming DMA
+ * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
+ * @page: page that buffer resides in
+ * @offset: offset into page for start of buffer
+ * @size: size of buffer to map
+ * @dir: DMA transfer direction
+ *
+ * Ensure that any data held in the cache is appropriately discarded
+ * or written back.
+ *
+ * The device owns this memory once this call has completed.  The CPU
+ * can regain ownership by calling dma_unmap_page().
+ */
+static dma_addr_t arm_dma_map_page(struct device *dev, struct page *page,
+             unsigned long offset, size_t size, enum dma_data_direction dir,
+             struct dma_attrs *attrs)
+{
+        if (!arch_is_coherent())
+                __dma_page_cpu_to_dev(page, offset, size, dir);
+        return pfn_to_dma(dev, page_to_pfn(page)) + offset;
+}
+
+/**
+ * arm_dma_unmap_page - unmap a buffer previously mapped through dma_map_page()
+ * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
+ * @handle: DMA address of buffer
+ * @size: size of buffer (same as passed to dma_map_page)
+ * @dir: DMA transfer direction (same as passed to dma_map_page)
+ *
+ * Unmap a page streaming mode DMA translation.  The handle and size
+ * must match what was provided in the previous dma_map_page() call.
+ * All other usages are undefined.
+ *
+ * After this call, reads by the CPU to the buffer are guaranteed to see
+ * whatever the device wrote there.
+ */
+static void arm_dma_unmap_page(struct device *dev, dma_addr_t handle,
+                size_t size, enum dma_data_direction dir,
+                struct dma_attrs *attrs)
+{
+        if (!arch_is_coherent())
+                __dma_page_dev_to_cpu(pfn_to_page(dma_to_pfn(dev, handle)),
+                                      handle & ~PAGE_MASK, size, dir);
+}
+
+static void arm_dma_sync_single_for_cpu(struct device *dev,
+                dma_addr_t handle, size_t size, enum dma_data_direction dir)
+{
+        unsigned int offset = handle & (PAGE_SIZE - 1);
+        struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset));
+        if (!arch_is_coherent())
+                __dma_page_dev_to_cpu(page, offset, size, dir);
+}
+
+static void arm_dma_sync_single_for_device(struct device *dev,
+                dma_addr_t handle, size_t size, enum dma_data_direction dir)
+{
+        unsigned int offset = handle & (PAGE_SIZE - 1);
+        struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset));
+        if (!arch_is_coherent())
+                __dma_page_cpu_to_dev(page, offset, size, dir);
+}
+
+static int arm_dma_set_mask(struct device *dev, u64 dma_mask);
+
+struct dma_map_ops arm_dma_ops = {
+        .alloc                  = arm_dma_alloc,
+        .free                   = arm_dma_free,
+        .mmap                   = arm_dma_mmap,
+        .map_page               = arm_dma_map_page,
+        .unmap_page             = arm_dma_unmap_page,
+        .map_sg                 = arm_dma_map_sg,
+        .unmap_sg               = arm_dma_unmap_sg,
+        .sync_single_for_cpu    = arm_dma_sync_single_for_cpu,
+        .sync_single_for_device = arm_dma_sync_single_for_device,
+        .sync_sg_for_cpu        = arm_dma_sync_sg_for_cpu,
+        .sync_sg_for_device     = arm_dma_sync_sg_for_device,
+        .set_dma_mask           = arm_dma_set_mask,
+};
+EXPORT_SYMBOL(arm_dma_ops);
+
 static u64 get_coherent_dma_mask(struct device *dev)
 {
         u64 mask = (u64)arm_dma_limit;
@@ -56,6 +163,21 @@ static u64 get_coherent_dma_mask(struct device *dev)
         return mask;
 }
 
+static void __dma_clear_buffer(struct page *page, size_t size)
+{
+        void *ptr;
+        /*
+         * Ensure that the allocated pages are zeroed, and that any data
+         * lurking in the kernel direct-mapped region is invalidated.
+         */
+        ptr = page_address(page);
+        if (ptr) {
+                memset(ptr, 0, size);
+                dmac_flush_range(ptr, ptr + size);
+                outer_flush_range(__pa(ptr), __pa(ptr) + size);
+        }
+}
+
 /*
  * Allocate a DMA buffer for 'dev' of size 'size' using the
  * specified gfp mask.  Note that 'size' must be page aligned.
@@ -64,23 +186,6 @@ static struct page *__dma_alloc_buffer(struct device *dev, size_t size, gfp_t gf
 {
         unsigned long order = get_order(size);
         struct page *page, *p, *e;
-        void *ptr;
-        u64 mask = get_coherent_dma_mask(dev);
-
-#ifdef CONFIG_DMA_API_DEBUG
-        u64 limit = (mask + 1) & ~mask;
-        if (limit && size >= limit) {
-                dev_warn(dev, "coherent allocation too big (requested %#x mask %#llx)\n",
-                        size, mask);
-                return NULL;
-        }
-#endif
-
-        if (!mask)
-                return NULL;
-
-        if (mask < 0xffffffffULL)
-                gfp |= GFP_DMA;
 
         page = alloc_pages(gfp, order);
         if (!page)
@@ -93,14 +198,7 @@ static struct page *__dma_alloc_buffer(struct device *dev, size_t size, gfp_t gf
         for (p = page + (size >> PAGE_SHIFT), e = page + (1 << order); p < e; p++)
                 __free_page(p);
 
-        /*
-         * Ensure that the allocated pages are zeroed, and that any data
-         * lurking in the kernel direct-mapped region is invalidated.
-         */
-        ptr = page_address(page);
-        memset(ptr, 0, size);
-        dmac_flush_range(ptr, ptr + size);
-        outer_flush_range(__pa(ptr), __pa(ptr) + size);
+        __dma_clear_buffer(page, size);
 
         return page;
 }
@@ -170,6 +268,11 @@ static int __init consistent_init(void)
         unsigned long base = consistent_base;
         unsigned long num_ptes = (CONSISTENT_END - base) >> PMD_SHIFT;
 
+#ifndef CONFIG_ARM_DMA_USE_IOMMU
+        if (cpu_architecture() >= CPU_ARCH_ARMv6)
+                return 0;
+#endif
+
         consistent_pte = kmalloc(num_ptes * sizeof(pte_t), GFP_KERNEL);
         if (!consistent_pte) {
                 pr_err("%s: no memory\n", __func__);
@@ -184,14 +287,14 @@ static int __init consistent_init(void)
 
                 pud = pud_alloc(&init_mm, pgd, base);
                 if (!pud) {
-                        printk(KERN_ERR "%s: no pud tables\n", __func__);
+                        pr_err("%s: no pud tables\n", __func__);
                         ret = -ENOMEM;
                         break;
                 }
 
                 pmd = pmd_alloc(&init_mm, pud, base);
                 if (!pmd) {
-                        printk(KERN_ERR "%s: no pmd tables\n", __func__);
+                        pr_err("%s: no pmd tables\n", __func__);
                         ret = -ENOMEM;
                         break;
                 }
@@ -199,7 +302,7 @@ static int __init consistent_init(void)
 
                 pte = pte_alloc_kernel(pmd, base);
                 if (!pte) {
-                        printk(KERN_ERR "%s: no pte tables\n", __func__);
+                        pr_err("%s: no pte tables\n", __func__);
                         ret = -ENOMEM;
                         break;
                 }
@@ -210,9 +313,101 @@ static int __init consistent_init(void)
 
         return ret;
 }
-
 core_initcall(consistent_init);
 
+static void *__alloc_from_contiguous(struct device *dev, size_t size,
+                                     pgprot_t prot, struct page **ret_page);
+
+static struct arm_vmregion_head coherent_head = {
+        .vm_lock        = __SPIN_LOCK_UNLOCKED(&coherent_head.vm_lock),
+        .vm_list        = LIST_HEAD_INIT(coherent_head.vm_list),
+};
+
+size_t coherent_pool_size = DEFAULT_CONSISTENT_DMA_SIZE / 8;
+
+static int __init early_coherent_pool(char *p)
+{
+        coherent_pool_size = memparse(p, &p);
+        return 0;
+}
+early_param("coherent_pool", early_coherent_pool);
+
+/*
+ * Initialise the coherent pool for atomic allocations.
+ */
+static int __init coherent_init(void)
+{
+        pgprot_t prot = pgprot_dmacoherent(pgprot_kernel);
+        size_t size = coherent_pool_size;
+        struct page *page;
+        void *ptr;
+
+        if (cpu_architecture() < CPU_ARCH_ARMv6)
+                return 0;
+
+        ptr = __alloc_from_contiguous(NULL, size, prot, &page);
+        if (ptr) {
+                coherent_head.vm_start = (unsigned long) ptr;
+                coherent_head.vm_end = (unsigned long) ptr + size;
+                printk(KERN_INFO "DMA: preallocated %u KiB pool for atomic coherent allocations\n",
+                       (unsigned)size / 1024);
+                return 0;
+        }
+        printk(KERN_ERR "DMA: failed to allocate %u KiB pool for atomic coherent allocation\n",
+               (unsigned)size / 1024);
+        return -ENOMEM;
+}
+/*
+ * CMA is activated by core_initcall, so we must be called after it.
+ */
+postcore_initcall(coherent_init);
+
+struct dma_contig_early_reserve {
+        phys_addr_t base;
+        unsigned long size;
+};
+
+static struct dma_contig_early_reserve dma_mmu_remap[MAX_CMA_AREAS] __initdata;
+
+static int dma_mmu_remap_num __initdata;
+
+void __init dma_contiguous_early_fixup(phys_addr_t base, unsigned long size)
+{
+        dma_mmu_remap[dma_mmu_remap_num].base = base;
+        dma_mmu_remap[dma_mmu_remap_num].size = size;
+        dma_mmu_remap_num++;
+}
+
+void __init dma_contiguous_remap(void)
+{
+        int i;
+        for (i = 0; i < dma_mmu_remap_num; i++) {
+                phys_addr_t start = dma_mmu_remap[i].base;
+                phys_addr_t end = start + dma_mmu_remap[i].size;
+                struct map_desc map;
+                unsigned long addr;
+
+                if (end > arm_lowmem_limit)
+                        end = arm_lowmem_limit;
+                if (start >= end)
+                        return;
+
+                map.pfn = __phys_to_pfn(start);
+                map.virtual = __phys_to_virt(start);
+                map.length = end - start;
+                map.type = MT_MEMORY_DMA_READY;
+
+                /*
+                 * Clear previous low-memory mapping
+                 */
+                for (addr = __phys_to_virt(start); addr < __phys_to_virt(end);
+                     addr += PMD_SIZE)
+                        pmd_clear(pmd_off_k(addr));
+
+                iotable_init(&map, 1);
+        }
+}
+
 static void *
 __dma_alloc_remap(struct page *page, size_t size, gfp_t gfp, pgprot_t prot,
         const void *caller)
@@ -222,7 +417,7 @@ __dma_alloc_remap(struct page *page, size_t size, gfp_t gfp, pgprot_t prot,
         int bit;
 
         if (!consistent_pte) {
-                printk(KERN_ERR "%s: not initialised\n", __func__);
+                pr_err("%s: not initialised\n", __func__);
                 dump_stack();
                 return NULL;
         }
@@ -249,7 +444,7 @@ __dma_alloc_remap(struct page *page, size_t size, gfp_t gfp, pgprot_t prot,
                 u32 off = CONSISTENT_OFFSET(c->vm_start) & (PTRS_PER_PTE-1);
 
                 pte = consistent_pte[idx] + off;
-                c->vm_pages = page;
+                c->priv = page;
 
                 do {
                         BUG_ON(!pte_none(*pte));
@@ -281,14 +476,14 @@ static void __dma_free_remap(void *cpu_addr, size_t size)
 
         c = arm_vmregion_find_remove(&consistent_head, (unsigned long)cpu_addr);
         if (!c) {
-                printk(KERN_ERR "%s: trying to free invalid coherent area: %p\n",
+                pr_err("%s: trying to free invalid coherent area: %p\n",
                        __func__, cpu_addr);
                 dump_stack();
                 return;
         }
 
         if ((c->vm_end - c->vm_start) != size) {
-                printk(KERN_ERR "%s: freeing wrong coherent size (%ld != %d)\n",
+                pr_err("%s: freeing wrong coherent size (%ld != %d)\n",
                        __func__, c->vm_end - c->vm_start, size);
                 dump_stack();
                 size = c->vm_end - c->vm_start;
@@ -310,8 +505,8 @@ static void __dma_free_remap(void *cpu_addr, size_t size)
                 }
 
                 if (pte_none(pte) || !pte_present(pte))
-                        printk(KERN_CRIT "%s: bad page in kernel page table\n",
-                               __func__);
+                        pr_crit("%s: bad page in kernel page table\n",
+                                __func__);
         } while (size -= PAGE_SIZE);
 
         flush_tlb_kernel_range(c->vm_start, c->vm_end);
@@ -319,20 +514,182 @@ static void __dma_free_remap(void *cpu_addr, size_t size)
         arm_vmregion_free(&consistent_head, c);
 }
 
+static int __dma_update_pte(pte_t *pte, pgtable_t token, unsigned long addr,
+                            void *data)
+{
+        struct page *page = virt_to_page(addr);
+        pgprot_t prot = *(pgprot_t *)data;
+
+        set_pte_ext(pte, mk_pte(page, prot), 0);
+        return 0;
+}
+
+static void __dma_remap(struct page *page, size_t size, pgprot_t prot)
+{
+        unsigned long start = (unsigned long) page_address(page);
+        unsigned end = start + size;
+
+        apply_to_page_range(&init_mm, start, size, __dma_update_pte, &prot);
+        dsb();
+        flush_tlb_kernel_range(start, end);
+}
+
+static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp,
+                                 pgprot_t prot, struct page **ret_page,
+                                 const void *caller)
+{
+        struct page *page;
+        void *ptr;
+        page = __dma_alloc_buffer(dev, size, gfp);
+        if (!page)
+                return NULL;
+
+        ptr = __dma_alloc_remap(page, size, gfp, prot, caller);
+        if (!ptr) {
+                __dma_free_buffer(page, size);
+                return NULL;
+        }
+
+        *ret_page = page;
+        return ptr;
+}
+
+static void *__alloc_from_pool(struct device *dev, size_t size,
+                               struct page **ret_page, const void *caller)
+{
+        struct arm_vmregion *c;
+        size_t align;
+
+        if (!coherent_head.vm_start) {
+                printk(KERN_ERR "%s: coherent pool not initialised!\n",
+                       __func__);
+                dump_stack();
+                return NULL;
+        }
+
+        /*
+         * Align the region allocation - allocations from pool are rather
+         * small, so align them to their order in pages, minimum is a page
+         * size. This helps reduce fragmentation of the DMA space.
+         */
+        align = PAGE_SIZE << get_order(size);
+        c = arm_vmregion_alloc(&coherent_head, align, size, 0, caller);
+        if (c) {
+                void *ptr = (void *)c->vm_start;
+                struct page *page = virt_to_page(ptr);
+                *ret_page = page;
+                return ptr;
+        }
+        return NULL;
+}
+
+static int __free_from_pool(void *cpu_addr, size_t size)
+{
+        unsigned long start = (unsigned long)cpu_addr;
+        unsigned long end = start + size;
+        struct arm_vmregion *c;
+
+        if (start < coherent_head.vm_start || end > coherent_head.vm_end)
+                return 0;
+
+        c = arm_vmregion_find_remove(&coherent_head, (unsigned long)start);
+
+        if ((c->vm_end - c->vm_start) != size) {
+                printk(KERN_ERR "%s: freeing wrong coherent size (%ld != %d)\n",
+                       __func__, c->vm_end - c->vm_start, size);
+                dump_stack();
+                size = c->vm_end - c->vm_start;
+        }
+
+        arm_vmregion_free(&coherent_head, c);
+        return 1;
+}
+
+static void *__alloc_from_contiguous(struct device *dev, size_t size,
+                                     pgprot_t prot, struct page **ret_page)
+{
+        unsigned long order = get_order(size);
+        size_t count = size >> PAGE_SHIFT;
+        struct page *page;
+
+        page = dma_alloc_from_contiguous(dev, count, order);
+        if (!page)
+                return NULL;
+
+        __dma_clear_buffer(page, size);
+        __dma_remap(page, size, prot);
+
+        *ret_page = page;
+        return page_address(page);
+}
+
+static void __free_from_contiguous(struct device *dev, struct page *page,
+                                   size_t size)
+{
+        __dma_remap(page, size, pgprot_kernel);
+        dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT);
+}
+
+static inline pgprot_t __get_dma_pgprot(struct dma_attrs *attrs, pgprot_t prot)
+{
+        prot = dma_get_attr(DMA_ATTR_WRITE_COMBINE, attrs) ?
+                            pgprot_writecombine(prot) :
+                            pgprot_dmacoherent(prot);
+        return prot;
+}
+
+#define nommu() 0
+
 #else   /* !CONFIG_MMU */
 
-#define __dma_alloc_remap(page, size, gfp, prot, c)     page_address(page)
-#define __dma_free_remap(addr, size)                    do { } while (0)
+#define nommu() 1
+
+#define __get_dma_pgprot(attrs, prot)   __pgprot(0)
+#define __alloc_remap_buffer(dev, size, gfp, prot, ret, c)      NULL
+#define __alloc_from_pool(dev, size, ret_page, c)               NULL
+#define __alloc_from_contiguous(dev, size, prot, ret)           NULL
+#define __free_from_pool(cpu_addr, size)                        0
+#define __free_from_contiguous(dev, page, size)                 do { } while (0)
+#define __dma_free_remap(cpu_addr, size)                        do { } while (0)
 
 #endif  /* CONFIG_MMU */
 
-static void *
-__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp,
-            pgprot_t prot, const void *caller)
+static void *__alloc_simple_buffer(struct device *dev, size_t size, gfp_t gfp,
+                                   struct page **ret_page)
+{
+        struct page *page;
+        page = __dma_alloc_buffer(dev, size, gfp);
+        if (!page)
+                return NULL;
+
+        *ret_page = page;
+        return page_address(page);
+}
+
+
+
+static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
+                         gfp_t gfp, pgprot_t prot, const void *caller)
 {
+        u64 mask = get_coherent_dma_mask(dev);
         struct page *page;
         void *addr;
 
+#ifdef CONFIG_DMA_API_DEBUG
+        u64 limit = (mask + 1) & ~mask;
+        if (limit && size >= limit) {
+                dev_warn(dev, "coherent allocation too big (requested %#x mask %#llx)\n",
+                        size, mask);
+                return NULL;
+        }
+#endif
+
+        if (!mask)
+                return NULL;
+
+        if (mask < 0xffffffffULL)
+                gfp |= GFP_DMA;
+
         /*
          * Following is a work-around (a.k.a. hack) to prevent pages
          * with __GFP_COMP being passed to split_page() which cannot
@@ -342,22 +699,20 @@ __dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp,
          */
         gfp &= ~(__GFP_COMP);
 
-        *handle = ~0;
+        *handle = DMA_ERROR_CODE;
         size = PAGE_ALIGN(size);
 
-        page = __dma_alloc_buffer(dev, size, gfp);
-        if (!page)
-                return NULL;
-
-        if (!arch_is_coherent())
-                addr = __dma_alloc_remap(page, size, gfp, prot, caller);
+        if (arch_is_coherent() || nommu())
+                addr = __alloc_simple_buffer(dev, size, gfp, &page);
+        else if (cpu_architecture() < CPU_ARCH_ARMv6)
+                addr = __alloc_remap_buffer(dev, size, gfp, prot, &page, caller);
+        else if (gfp & GFP_ATOMIC)
+                addr = __alloc_from_pool(dev, size, &page, caller);
         else
-                addr = page_address(page);
+                addr = __alloc_from_contiguous(dev, size, prot, &page);
 
         if (addr)
                 *handle = pfn_to_dma(dev, page_to_pfn(page));
-        else
-                __dma_free_buffer(page, size);
 
         return addr;
 }
@@ -366,138 +721,71 @@ __dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp,
  * Allocate DMA-coherent memory space and return both the kernel remapped
  * virtual and bus address for that space.
  */
-void *
-dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp)
+void *arm_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
+                    gfp_t gfp, struct dma_attrs *attrs)
 {
+        pgprot_t prot = __get_dma_pgprot(attrs, pgprot_kernel);
         void *memory;
 
         if (dma_alloc_from_coherent(dev, size, handle, &memory))
                 return memory;
 
-        return __dma_alloc(dev, size, handle, gfp,
-                           pgprot_dmacoherent(pgprot_kernel),
+        return __dma_alloc(dev, size, handle, gfp, prot,
                            __builtin_return_address(0));
 }
-EXPORT_SYMBOL(dma_alloc_coherent);
 
 /*
- * Allocate a writecombining region, in much the same way as
- * dma_alloc_coherent above.
+ * Create userspace mapping for the DMA-coherent memory.
  */
-void *
-dma_alloc_writecombine(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp)
-{
-        return __dma_alloc(dev, size, handle, gfp,
-                           pgprot_writecombine(pgprot_kernel),
-                           __builtin_return_address(0));
-}
-EXPORT_SYMBOL(dma_alloc_writecombine);
-
-static int dma_mmap(struct device *dev, struct vm_area_struct *vma,
-                    void *cpu_addr, dma_addr_t dma_addr, size_t size)
+int arm_dma_mmap(struct device *dev, struct vm_area_struct *vma,
+                 void *cpu_addr, dma_addr_t dma_addr, size_t size,
+                 struct dma_attrs *attrs)
 {
         int ret = -ENXIO;
 #ifdef CONFIG_MMU
-        unsigned long user_size, kern_size;
-        struct arm_vmregion *c;
+        unsigned long pfn = dma_to_pfn(dev, dma_addr);
+        vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot);
 
-        user_size = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
+        if (dma_mmap_from_coherent(dev, vma, cpu_addr, size, &ret))
+                return ret;
 
-        c = arm_vmregion_find(&consistent_head, (unsigned long)cpu_addr);
-        if (c) {
-                unsigned long off = vma->vm_pgoff;
-
-                kern_size = (c->vm_end - c->vm_start) >> PAGE_SHIFT;
-
-                if (off < kern_size &&
-                    user_size <= (kern_size - off)) {
-                        ret = remap_pfn_range(vma, vma->vm_start,
-                                              page_to_pfn(c->vm_pages) + off,
-                                              user_size << PAGE_SHIFT,
-                                              vma->vm_page_prot);
-                }
-        }
+        ret = remap_pfn_range(vma, vma->vm_start,
+                              pfn + vma->vm_pgoff,
+                              vma->vm_end - vma->vm_start,
+                              vma->vm_page_prot);
 #endif  /* CONFIG_MMU */
 
         return ret;
 }
 
-int dma_mmap_coherent(struct device *dev, struct vm_area_struct *vma,
-                      void *cpu_addr, dma_addr_t dma_addr, size_t size)
-{
-        vma->vm_page_prot = pgprot_dmacoherent(vma->vm_page_prot);
-        return dma_mmap(dev, vma, cpu_addr, dma_addr, size);
-}
-EXPORT_SYMBOL(dma_mmap_coherent);
-
-int dma_mmap_writecombine(struct device *dev, struct vm_area_struct *vma,
-                          void *cpu_addr, dma_addr_t dma_addr, size_t size)
-{
-        vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
-        return dma_mmap(dev, vma, cpu_addr, dma_addr, size);
-}
-EXPORT_SYMBOL(dma_mmap_writecombine);
-
 /*
- * free a page as defined by the above mapping.
- * Must not be called with IRQs disabled.
+ * Free a buffer as defined by the above mapping.
  */
-void dma_free_coherent(struct device *dev, size_t size, void *cpu_addr, dma_addr_t handle)
+void arm_dma_free(struct device *dev, size_t size, void *cpu_addr,
+                  dma_addr_t handle, struct dma_attrs *attrs)
 {
-        WARN_ON(irqs_disabled());
+        struct page *page = pfn_to_page(dma_to_pfn(dev, handle));
 
         if (dma_release_from_coherent(dev, get_order(size), cpu_addr))
                 return;
 
         size = PAGE_ALIGN(size);
 
-        if (!arch_is_coherent())
+        if (arch_is_coherent() || nommu()) {
+                __dma_free_buffer(page, size);
+        } else if (cpu_architecture() < CPU_ARCH_ARMv6) {
                 __dma_free_remap(cpu_addr, size);
-
-        __dma_free_buffer(pfn_to_page(dma_to_pfn(dev, handle)), size);
-}
-EXPORT_SYMBOL(dma_free_coherent);
-
-/*
- * Make an area consistent for devices.
- * Note: Drivers should NOT use this function directly, as it will break
- * platforms with CONFIG_DMABOUNCE.
- * Use the driver DMA support - see dma-mapping.h (dma_sync_*)
- */
-void ___dma_single_cpu_to_dev(const void *kaddr, size_t size,
-        enum dma_data_direction dir)
-{
-        unsigned long paddr;
-
-        BUG_ON(!virt_addr_valid(kaddr) || !virt_addr_valid(kaddr + size - 1));
-
-        dmac_map_area(kaddr, size, dir);
-
-        paddr = __pa(kaddr);
-        if (dir == DMA_FROM_DEVICE) {
-                outer_inv_range(paddr, paddr + size);
+                __dma_free_buffer(page, size);
         } else {
-                outer_clean_range(paddr, paddr + size);
-        }
-        /* FIXME: non-speculating: flush on bidirectional mappings? */
-}
-EXPORT_SYMBOL(___dma_single_cpu_to_dev);
-
-void ___dma_single_dev_to_cpu(const void *kaddr, size_t size,
-        enum dma_data_direction dir)
-{
-        BUG_ON(!virt_addr_valid(kaddr) || !virt_addr_valid(kaddr + size - 1));
-
-        /* FIXME: non-speculating: not required */
-        /* don't bother invalidating if DMA to device */
-        if (dir != DMA_TO_DEVICE) {
-                unsigned long paddr = __pa(kaddr);
-                outer_inv_range(paddr, paddr + size);
+                if (__free_from_pool(cpu_addr, size))
+                        return;
+                /*
+                 * Non-atomic allocations cannot be freed with IRQs disabled
+                 */
+                WARN_ON(irqs_disabled());
+                __free_from_contiguous(dev, page, size);
         }
-
-        dmac_unmap_area(kaddr, size, dir);
 }
-EXPORT_SYMBOL(___dma_single_dev_to_cpu);
 
 static void dma_cache_maint_page(struct page *page, unsigned long offset,
         size_t size, enum dma_data_direction dir,
@@ -543,7 +831,13 @@ static void dma_cache_maint_page(struct page *page, unsigned long offset,
         } while (left);
 }
 
-void ___dma_page_cpu_to_dev(struct page *page, unsigned long off,
+/*
+ * Make an area consistent for devices.
+ * Note: Drivers should NOT use this function directly, as it will break
+ * platforms with CONFIG_DMABOUNCE.
+ * Use the driver DMA support - see dma-mapping.h (dma_sync_*)
+ */
+static void __dma_page_cpu_to_dev(struct page *page, unsigned long off,
         size_t size, enum dma_data_direction dir)
 {
         unsigned long paddr;
@@ -558,9 +852,8 @@ void ___dma_page_cpu_to_dev(struct page *page, unsigned long off,
         }
         /* FIXME: non-speculating: flush on bidirectional mappings? */
 }
-EXPORT_SYMBOL(___dma_page_cpu_to_dev);
 
-void ___dma_page_dev_to_cpu(struct page *page, unsigned long off,
+static void __dma_page_dev_to_cpu(struct page *page, unsigned long off,
         size_t size, enum dma_data_direction dir)
 {
         unsigned long paddr = page_to_phys(page) + off;
@@ -578,10 +871,9 @@ void ___dma_page_dev_to_cpu(struct page *page, unsigned long off,
         if (dir != DMA_TO_DEVICE && off == 0 && size >= PAGE_SIZE)
                 set_bit(PG_dcache_clean, &page->flags);
 }
-EXPORT_SYMBOL(___dma_page_dev_to_cpu);
 
 /**
- * dma_map_sg - map a set of SG buffers for streaming mode DMA
+ * arm_dma_map_sg - map a set of SG buffers for streaming mode DMA
  * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
  * @sg: list of buffers
  * @nents: number of buffers to map
@@ -596,32 +888,32 @@ EXPORT_SYMBOL(___dma_page_dev_to_cpu);
  * Device ownership issues as mentioned for dma_map_single are the same
  * here.
  */
-int dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
-                enum dma_data_direction dir)
+int arm_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
+                enum dma_data_direction dir, struct dma_attrs *attrs)
 {
+        struct dma_map_ops *ops = get_dma_ops(dev);
         struct scatterlist *s;
         int i, j;
 
-        BUG_ON(!valid_dma_direction(dir));
-
         for_each_sg(sg, s, nents, i) {
-                s->dma_address = __dma_map_page(dev, sg_page(s), s->offset,
-                                                s->length, dir);
+#ifdef CONFIG_NEED_SG_DMA_LENGTH
+                s->dma_length = s->length;
+#endif
+                s->dma_address = ops->map_page(dev, sg_page(s), s->offset,
+                                                s->length, dir, attrs);
                 if (dma_mapping_error(dev, s->dma_address))
                         goto bad_mapping;
         }
-        debug_dma_map_sg(dev, sg, nents, nents, dir);
         return nents;
 
  bad_mapping:
         for_each_sg(sg, s, i, j)
-                __dma_unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir);
+                ops->unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, attrs);
         return 0;
 }
-EXPORT_SYMBOL(dma_map_sg);
 
 /**
- * dma_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg
+ * arm_dma_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg
  * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
  * @sg: list of buffers
  * @nents: number of buffers to unmap (same as was passed to dma_map_sg)
@@ -630,70 +922,55 @@ EXPORT_SYMBOL(dma_map_sg);
  * Unmap a set of streaming mode DMA translations.  Again, CPU access
  * rules concerning calls here are the same as for dma_unmap_single().
  */
-void dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
-                enum dma_data_direction dir)
+void arm_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
+                enum dma_data_direction dir, struct dma_attrs *attrs)
 {
+        struct dma_map_ops *ops = get_dma_ops(dev);
         struct scatterlist *s;
-        int i;
 
-        debug_dma_unmap_sg(dev, sg, nents, dir);
+        int i;
 
         for_each_sg(sg, s, nents, i)
-                __dma_unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir);
+                ops->unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, attrs);
 }
-EXPORT_SYMBOL(dma_unmap_sg);
 
 /**
- * dma_sync_sg_for_cpu
+ * arm_dma_sync_sg_for_cpu
  * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
  * @sg: list of buffers
  * @nents: number of buffers to map (returned from dma_map_sg)
  * @dir: DMA transfer direction (same as was passed to dma_map_sg)
  */
-void dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
+void arm_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
                         int nents, enum dma_data_direction dir)
 {
+        struct dma_map_ops *ops = get_dma_ops(dev);
         struct scatterlist *s;
         int i;
 
-        for_each_sg(sg, s, nents, i) {
-                if (!dmabounce_sync_for_cpu(dev, sg_dma_address(s), 0,
-                                            sg_dma_len(s), dir))
-                        continue;
-
-                __dma_page_dev_to_cpu(sg_page(s), s->offset,
-                                      s->length, dir);
-        }
-
-        debug_dma_sync_sg_for_cpu(dev, sg, nents, dir);
+        for_each_sg(sg, s, nents, i)
+                ops->sync_single_for_cpu(dev, sg_dma_address(s), s->length,
+                                         dir);
 }
-EXPORT_SYMBOL(dma_sync_sg_for_cpu);
 
 /**
- * dma_sync_sg_for_device
+ * arm_dma_sync_sg_for_device
  * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
  * @sg: list of buffers
  * @nents: number of buffers to map (returned from dma_map_sg)
  * @dir: DMA transfer direction (same as was passed to dma_map_sg)
  */
-void dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
+void arm_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
                         int nents, enum dma_data_direction dir)
 {
+        struct dma_map_ops *ops = get_dma_ops(dev);
         struct scatterlist *s;
         int i;
 
-        for_each_sg(sg, s, nents, i) {
-                if (!dmabounce_sync_for_device(dev, sg_dma_address(s), 0,
-                                        sg_dma_len(s), dir))
-                        continue;
-
-                __dma_page_cpu_to_dev(sg_page(s), s->offset,
-                                      s->length, dir);
-        }
-
-        debug_dma_sync_sg_for_device(dev, sg, nents, dir);
+        for_each_sg(sg, s, nents, i)
+                ops->sync_single_for_device(dev, sg_dma_address(s), s->length,
+                                            dir);
 }
-EXPORT_SYMBOL(dma_sync_sg_for_device);
 
 /*
  * Return whether the given device DMA address mask can be supported
@@ -709,18 +986,15 @@ int dma_supported(struct device *dev, u64 mask)
 }
 EXPORT_SYMBOL(dma_supported);
 
-int dma_set_mask(struct device *dev, u64 dma_mask)
+static int arm_dma_set_mask(struct device *dev, u64 dma_mask)
 {
         if (!dev->dma_mask || !dma_supported(dev, dma_mask))
                 return -EIO;
 
-#ifndef CONFIG_DMABOUNCE
         *dev->dma_mask = dma_mask;
-#endif
 
         return 0;
 }
-EXPORT_SYMBOL(dma_set_mask);
 
 #define PREALLOC_DMA_DEBUG_ENTRIES      4096
 
@@ -733,3 +1007,679 @@ static int __init dma_debug_do_init(void)
         return 0;
 }
 fs_initcall(dma_debug_do_init);
+
+#ifdef CONFIG_ARM_DMA_USE_IOMMU
+
+/* IOMMU */
+
+static inline dma_addr_t __alloc_iova(struct dma_iommu_mapping *mapping,
+                                      size_t size)
+{
+        unsigned int order = get_order(size);
+        unsigned int align = 0;
+        unsigned int count, start;
+        unsigned long flags;
+
+        count = ((PAGE_ALIGN(size) >> PAGE_SHIFT) +
+                 (1 << mapping->order) - 1) >> mapping->order;
+
+        if (order > mapping->order)
+                align = (1 << (order - mapping->order)) - 1;
+
+        spin_lock_irqsave(&mapping->lock, flags);
+        start = bitmap_find_next_zero_area(mapping->bitmap, mapping->bits, 0,
+                                           count, align);
+        if (start > mapping->bits) {
+                spin_unlock_irqrestore(&mapping->lock, flags);
+                return DMA_ERROR_CODE;
+        }
+
+        bitmap_set(mapping->bitmap, start, count);
+        spin_unlock_irqrestore(&mapping->lock, flags);
+
+        return mapping->base + (start << (mapping->order + PAGE_SHIFT));
+}
+
+static inline void __free_iova(struct dma_iommu_mapping *mapping,
+                               dma_addr_t addr, size_t size)
+{
+        unsigned int start = (addr - mapping->base) >>
+                             (mapping->order + PAGE_SHIFT);
+        unsigned int count = ((size >> PAGE_SHIFT) +
+                              (1 << mapping->order) - 1) >> mapping->order;
+        unsigned long flags;
+
+        spin_lock_irqsave(&mapping->lock, flags);
+        bitmap_clear(mapping->bitmap, start, count);
+        spin_unlock_irqrestore(&mapping->lock, flags);
+}
+
+static struct page **__iommu_alloc_buffer(struct device *dev, size_t size, gfp_t gfp)
+{
+        struct page **pages;
+        int count = size >> PAGE_SHIFT;
+        int array_size = count * sizeof(struct page *);
+        int i = 0;
+
+        if (array_size <= PAGE_SIZE)
+                pages = kzalloc(array_size, gfp);
+        else
+                pages = vzalloc(array_size);
+        if (!pages)
+                return NULL;
+
+        while (count) {
+                int j, order = __ffs(count);
+
+                pages[i] = alloc_pages(gfp | __GFP_NOWARN, order);
+                while (!pages[i] && order)
+                        pages[i] = alloc_pages(gfp | __GFP_NOWARN, --order);
+                if (!pages[i])
+                        goto error;
+
+                if (order)
+                        split_page(pages[i], order);
+                j = 1 << order;
+                while (--j)
+                        pages[i + j] = pages[i] + j;
+
+                __dma_clear_buffer(pages[i], PAGE_SIZE << order);
+                i += 1 << order;
+                count -= 1 << order;
+        }
+
+        return pages;
+error:
+        while (--i)
+                if (pages[i])
+                        __free_pages(pages[i], 0);
+        if (array_size < PAGE_SIZE)
+                kfree(pages);
+        else
+                vfree(pages);
+        return NULL;
+}
+
+static int __iommu_free_buffer(struct device *dev, struct page **pages, size_t size)
+{
+        int count = size >> PAGE_SHIFT;
+        int array_size = count * sizeof(struct page *);
+        int i;
+        for (i = 0; i < count; i++)
+                if (pages[i])
+                        __free_pages(pages[i], 0);
+        if (array_size < PAGE_SIZE)
+                kfree(pages);
+        else
+                vfree(pages);
+        return 0;
+}
+
+/*
+ * Create a CPU mapping for a specified pages
+ */
+static void *
+__iommu_alloc_remap(struct page **pages, size_t size, gfp_t gfp, pgprot_t prot)
+{
+        struct arm_vmregion *c;
+        size_t align;
+        size_t count = size >> PAGE_SHIFT;
+        int bit;
+
+        if (!consistent_pte[0]) {
+                pr_err("%s: not initialised\n", __func__);
+                dump_stack();
+                return NULL;
+        }
+
+        /*
+         * Align the virtual region allocation - maximum alignment is
+         * a section size, minimum is a page size.  This helps reduce
+         * fragmentation of the DMA space, and also prevents allocations
+         * smaller than a section from crossing a section boundary.
+         */
+        bit = fls(size - 1);
+        if (bit > SECTION_SHIFT)
+                bit = SECTION_SHIFT;
+        align = 1 << bit;
+
+        /*
+         * Allocate a virtual address in the consistent mapping region.
+         */
+        c = arm_vmregion_alloc(&consistent_head, align, size,
+                            gfp & ~(__GFP_DMA | __GFP_HIGHMEM), NULL);
+        if (c) {
+                pte_t *pte;
+                int idx = CONSISTENT_PTE_INDEX(c->vm_start);
+                int i = 0;
+                u32 off = CONSISTENT_OFFSET(c->vm_start) & (PTRS_PER_PTE-1);
+
+                pte = consistent_pte[idx] + off;
+                c->priv = pages;
+
+                do {
+                        BUG_ON(!pte_none(*pte));
+
+                        set_pte_ext(pte, mk_pte(pages[i], prot), 0);
+                        pte++;
+                        off++;
+                        i++;
+                        if (off >= PTRS_PER_PTE) {
+                                off = 0;
+                                pte = consistent_pte[++idx];
+                        }
+                } while (i < count);
+
+                dsb();
+
+                return (void *)c->vm_start;
+        }
+        return NULL;
+}
+
+/*
+ * Create a mapping in device IO address space for specified pages
+ */
+static dma_addr_t
+__iommu_create_mapping(struct device *dev, struct page **pages, size_t size)
+{
+        struct dma_iommu_mapping *mapping = dev->archdata.mapping;
+        unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
+        dma_addr_t dma_addr, iova;
+        int i, ret = DMA_ERROR_CODE;
+
+        dma_addr = __alloc_iova(mapping, size);
+        if (dma_addr == DMA_ERROR_CODE)
+                return dma_addr;
+
+        iova = dma_addr;
+        for (i = 0; i < count; ) {
+                unsigned int next_pfn = page_to_pfn(pages[i]) + 1;
+                phys_addr_t phys = page_to_phys(pages[i]);
+                unsigned int len, j;
+
+                for (j = i + 1; j < count; j++, next_pfn++)
+                        if (page_to_pfn(pages[j]) != next_pfn)
+                                break;
+
+                len = (j - i) << PAGE_SHIFT;
+                ret = iommu_map(mapping->domain, iova, phys, len, 0);
+                if (ret < 0)
+                        goto fail;
+                iova += len;
+                i = j;
+        }
+        return dma_addr;
+fail:
+        iommu_unmap(mapping->domain, dma_addr, iova-dma_addr);
+        __free_iova(mapping, dma_addr, size);
+        return DMA_ERROR_CODE;
+}
+
+static int __iommu_remove_mapping(struct device *dev, dma_addr_t iova, size_t size)
+{
+        struct dma_iommu_mapping *mapping = dev->archdata.mapping;
+
+        /*
+         * add optional in-page offset from iova to size and align
+         * result to page size
+         */
+        size = PAGE_ALIGN((iova & ~PAGE_MASK) + size);
+        iova &= PAGE_MASK;
+
+        iommu_unmap(mapping->domain, iova, size);
+        __free_iova(mapping, iova, size);
+        return 0;
+}
+
+static void *arm_iommu_alloc_attrs(struct device *dev, size_t size,
+            dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs)
+{
+        pgprot_t prot = __get_dma_pgprot(attrs, pgprot_kernel);
+        struct page **pages;
+        void *addr = NULL;
+
+        *handle = DMA_ERROR_CODE;
+        size = PAGE_ALIGN(size);
+
+        pages = __iommu_alloc_buffer(dev, size, gfp);
+        if (!pages)
+                return NULL;
+
+        *handle = __iommu_create_mapping(dev, pages, size);
+        if (*handle == DMA_ERROR_CODE)
+                goto err_buffer;
+
+        addr = __iommu_alloc_remap(pages, size, gfp, prot);
+        if (!addr)
+                goto err_mapping;
+
+        return addr;
+
+err_mapping:
+        __iommu_remove_mapping(dev, *handle, size);
+err_buffer:
+        __iommu_free_buffer(dev, pages, size);
+        return NULL;
+}
+
+static int arm_iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma,
+                    void *cpu_addr, dma_addr_t dma_addr, size_t size,
+                    struct dma_attrs *attrs)
+{
+        struct arm_vmregion *c;
+
+        vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot);
+        c = arm_vmregion_find(&consistent_head, (unsigned long)cpu_addr);
+
+        if (c) {
+                struct page **pages = c->priv;
+
+                unsigned long uaddr = vma->vm_start;
+                unsigned long usize = vma->vm_end - vma->vm_start;
+                int i = 0;
+
+                do {
+                        int ret;
+
+                        ret = vm_insert_page(vma, uaddr, pages[i++]);
+                        if (ret) {
+                                pr_err("Remapping memory, error: %d\n", ret);
+                                return ret;
+                        }
+
+                        uaddr += PAGE_SIZE;
+                        usize -= PAGE_SIZE;
+                } while (usize > 0);
+        }
+        return 0;
+}
+
+/*
+ * free a page as defined by the above mapping.
+ * Must not be called with IRQs disabled.
+ */
+void arm_iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr,
+                          dma_addr_t handle, struct dma_attrs *attrs)
+{
+        struct arm_vmregion *c;
+        size = PAGE_ALIGN(size);
+
+        c = arm_vmregion_find(&consistent_head, (unsigned long)cpu_addr);
+        if (c) {
+                struct page **pages = c->priv;
+                __dma_free_remap(cpu_addr, size);
+                __iommu_remove_mapping(dev, handle, size);
+                __iommu_free_buffer(dev, pages, size);
+        }
+}
+
+/*
+ * Map a part of the scatter-gather list into contiguous io address space
+ */
+static int __map_sg_chunk(struct device *dev, struct scatterlist *sg,
+                          size_t size, dma_addr_t *handle,
+                          enum dma_data_direction dir)
+{
+        struct dma_iommu_mapping *mapping = dev->archdata.mapping;
+        dma_addr_t iova, iova_base;
+        int ret = 0;
+        unsigned int count;
+        struct scatterlist *s;
+
+        size = PAGE_ALIGN(size);
+        *handle = DMA_ERROR_CODE;
+
+        iova_base = iova = __alloc_iova(mapping, size);
+        if (iova == DMA_ERROR_CODE)
+                return -ENOMEM;
+
+        for (count = 0, s = sg; count < (size >> PAGE_SHIFT); s = sg_next(s)) {
+                phys_addr_t phys = page_to_phys(sg_page(s));
+                unsigned int len = PAGE_ALIGN(s->offset + s->length);
+
+                if (!arch_is_coherent())
+                        __dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir);
+
+                ret = iommu_map(mapping->domain, iova, phys, len, 0);
+                if (ret < 0)
+                        goto fail;
+                count += len >> PAGE_SHIFT;
+                iova += len;
+        }
+        *handle = iova_base;
+
+        return 0;
+fail:
+        iommu_unmap(mapping->domain, iova_base, count * PAGE_SIZE);
+        __free_iova(mapping, iova_base, size);
+        return ret;
+}
+
+/**
+ * arm_iommu_map_sg - map a set of SG buffers for streaming mode DMA
+ * @dev: valid struct device pointer
+ * @sg: list of buffers
+ * @nents: number of buffers to map
+ * @dir: DMA transfer direction
+ *
+ * Map a set of buffers described by scatterlist in streaming mode for DMA.
+ * The scatter gather list elements are merged together (if possible) and
+ * tagged with the appropriate dma address and length. They are obtained via
+ * sg_dma_{address,length}.
+ */
+int arm_iommu_map_sg(struct device *dev, struct scatterlist *sg, int nents,
+                     enum dma_data_direction dir, struct dma_attrs *attrs)
+{
+        struct scatterlist *s = sg, *dma = sg, *start = sg;
+        int i, count = 0;
+        unsigned int offset = s->offset;
+        unsigned int size = s->offset + s->length;
+        unsigned int max = dma_get_max_seg_size(dev);
+
+        for (i = 1; i < nents; i++) {
+                s = sg_next(s);
+
+                s->dma_address = DMA_ERROR_CODE;
+                s->dma_length = 0;
+
+                if (s->offset || (size & ~PAGE_MASK) || size + s->length > max) {
+                        if (__map_sg_chunk(dev, start, size, &dma->dma_address,
+                            dir) < 0)
+                                goto bad_mapping;
+
+                        dma->dma_address += offset;
+                        dma->dma_length = size - offset;
+
+                        size = offset = s->offset;
+                        start = s;
+                        dma = sg_next(dma);
+                        count += 1;
+                }
+                size += s->length;
+        }
+        if (__map_sg_chunk(dev, start, size, &dma->dma_address, dir) < 0)
+                goto bad_mapping;
+
+        dma->dma_address += offset;
+        dma->dma_length = size - offset;
+
+        return count+1;
+
+bad_mapping:
+        for_each_sg(sg, s, count, i)
+                __iommu_remove_mapping(dev, sg_dma_address(s), sg_dma_len(s));
+        return 0;
+}
+
+/**
+ * arm_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg
+ * @dev: valid struct device pointer
+ * @sg: list of buffers
+ * @nents: number of buffers to unmap (same as was passed to dma_map_sg)
+ * @dir: DMA transfer direction (same as was passed to dma_map_sg)
+ *
+ * Unmap a set of streaming mode DMA translations.  Again, CPU access
+ * rules concerning calls here are the same as for dma_unmap_single().
+ */
+void arm_iommu_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
+                        enum dma_data_direction dir, struct dma_attrs *attrs)
+{
+        struct scatterlist *s;
+        int i;
+
+        for_each_sg(sg, s, nents, i) {
+                if (sg_dma_len(s))
+                        __iommu_remove_mapping(dev, sg_dma_address(s),
+                                               sg_dma_len(s));
+                if (!arch_is_coherent())
+                        __dma_page_dev_to_cpu(sg_page(s), s->offset,
+                                              s->length, dir);
+        }
+}
+
+/**
+ * arm_iommu_sync_sg_for_cpu
+ * @dev: valid struct device pointer
+ * @sg: list of buffers
+ * @nents: number of buffers to map (returned from dma_map_sg)
+ * @dir: DMA transfer direction (same as was passed to dma_map_sg)
+ */
+void arm_iommu_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
+                        int nents, enum dma_data_direction dir)
+{
+        struct scatterlist *s;
+        int i;
+
+        for_each_sg(sg, s, nents, i)
+                if (!arch_is_coherent())
+                        __dma_page_dev_to_cpu(sg_page(s), s->offset, s->length, dir);
+
+}
+
+/**
+ * arm_iommu_sync_sg_for_device
+ * @dev: valid struct device pointer
+ * @sg: list of buffers
+ * @nents: number of buffers to map (returned from dma_map_sg)
+ * @dir: DMA transfer direction (same as was passed to dma_map_sg)
+ */
+void arm_iommu_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
+                        int nents, enum dma_data_direction dir)
+{
+        struct scatterlist *s;
+        int i;
+
+        for_each_sg(sg, s, nents, i)
+                if (!arch_is_coherent())
+                        __dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir);
+}
+
+
+/**
+ * arm_iommu_map_page
+ * @dev: valid struct device pointer
+ * @page: page that buffer resides in
+ * @offset: offset into page for start of buffer
+ * @size: size of buffer to map
+ * @dir: DMA transfer direction
+ *
+ * IOMMU aware version of arm_dma_map_page()
+ */
+static dma_addr_t arm_iommu_map_page(struct device *dev, struct page *page,
+             unsigned long offset, size_t size, enum dma_data_direction dir,
+             struct dma_attrs *attrs)
+{
+        struct dma_iommu_mapping *mapping = dev->archdata.mapping;
+        dma_addr_t dma_addr;
+        int ret, len = PAGE_ALIGN(size + offset);
+
+        if (!arch_is_coherent())
+                __dma_page_cpu_to_dev(page, offset, size, dir);
+
+        dma_addr = __alloc_iova(mapping, len);
+        if (dma_addr == DMA_ERROR_CODE)
+                return dma_addr;
+
+        ret = iommu_map(mapping->domain, dma_addr, page_to_phys(page), len, 0);
+        if (ret < 0)
+                goto fail;
+
+        return dma_addr + offset;
+fail:
+        __free_iova(mapping, dma_addr, len);
+        return DMA_ERROR_CODE;
+}
+
+/**
+ * arm_iommu_unmap_page
+ * @dev: valid struct device pointer
+ * @handle: DMA address of buffer
+ * @size: size of buffer (same as passed to dma_map_page)
+ * @dir: DMA transfer direction (same as passed to dma_map_page)
+ *
+ * IOMMU aware version of arm_dma_unmap_page()
+ */
+static void arm_iommu_unmap_page(struct device *dev, dma_addr_t handle,
+                size_t size, enum dma_data_direction dir,
+                struct dma_attrs *attrs)
+{
+        struct dma_iommu_mapping *mapping = dev->archdata.mapping;
+        dma_addr_t iova = handle & PAGE_MASK;
+        struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
+        int offset = handle & ~PAGE_MASK;
+        int len = PAGE_ALIGN(size + offset);
+
+        if (!iova)
+                return;
+
+        if (!arch_is_coherent())
+                __dma_page_dev_to_cpu(page, offset, size, dir);
+
+        iommu_unmap(mapping->domain, iova, len);
+        __free_iova(mapping, iova, len);
+}
+
+static void arm_iommu_sync_single_for_cpu(struct device *dev,
+                dma_addr_t handle, size_t size, enum dma_data_direction dir)
+{
+        struct dma_iommu_mapping *mapping = dev->archdata.mapping;
+        dma_addr_t iova = handle & PAGE_MASK;
+        struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
+        unsigned int offset = handle & ~PAGE_MASK;
+
+        if (!iova)
+                return;
+
+        if (!arch_is_coherent())
+                __dma_page_dev_to_cpu(page, offset, size, dir);
+}
+
+static void arm_iommu_sync_single_for_device(struct device *dev,
+                dma_addr_t handle, size_t size, enum dma_data_direction dir)
+{
+        struct dma_iommu_mapping *mapping = dev->archdata.mapping;
+        dma_addr_t iova = handle & PAGE_MASK;
+        struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
+        unsigned int offset = handle & ~PAGE_MASK;
+
+        if (!iova)
+                return;
+
+        __dma_page_cpu_to_dev(page, offset, size, dir);
+}
+
+struct dma_map_ops iommu_ops = {
+        .alloc          = arm_iommu_alloc_attrs,
+        .free           = arm_iommu_free_attrs,
+        .mmap           = arm_iommu_mmap_attrs,
+
+        .map_page               = arm_iommu_map_page,
+        .unmap_page             = arm_iommu_unmap_page,
+        .sync_single_for_cpu    = arm_iommu_sync_single_for_cpu,
+        .sync_single_for_device = arm_iommu_sync_single_for_device,
+
+        .map_sg                 = arm_iommu_map_sg,
+        .unmap_sg               = arm_iommu_unmap_sg,
+        .sync_sg_for_cpu        = arm_iommu_sync_sg_for_cpu,
+        .sync_sg_for_device     = arm_iommu_sync_sg_for_device,
+};
+
+/**
+ * arm_iommu_create_mapping
+ * @bus: pointer to the bus holding the client device (for IOMMU calls)
+ * @base: start address of the valid IO address space
+ * @size: size of the valid IO address space
+ * @order: accuracy of the IO addresses allocations
+ *
+ * Creates a mapping structure which holds information about used/unused
+ * IO address ranges, which is required to perform memory allocation and
+ * mapping with IOMMU aware functions.
+ *
+ * The client device need to be attached to the mapping with
+ * arm_iommu_attach_device function.
+ */
+struct dma_iommu_mapping *
+arm_iommu_create_mapping(struct bus_type *bus, dma_addr_t base, size_t size,
+                         int order)
+{
+        unsigned int count = size >> (PAGE_SHIFT + order);
+        unsigned int bitmap_size = BITS_TO_LONGS(count) * sizeof(long);
+        struct dma_iommu_mapping *mapping;
+        int err = -ENOMEM;
+
+        if (!count)
+                return ERR_PTR(-EINVAL);
+
+        mapping = kzalloc(sizeof(struct dma_iommu_mapping), GFP_KERNEL);
+        if (!mapping)
+                goto err;
+
+        mapping->bitmap = kzalloc(bitmap_size, GFP_KERNEL);
+        if (!mapping->bitmap)
+                goto err2;
+
+        mapping->base = base;
+        mapping->bits = BITS_PER_BYTE * bitmap_size;
+        mapping->order = order;
+        spin_lock_init(&mapping->lock);
+
+        mapping->domain = iommu_domain_alloc(bus);
+        if (!mapping->domain)
+                goto err3;
+
+        kref_init(&mapping->kref);
+        return mapping;
+err3:
+        kfree(mapping->bitmap);
+err2:
+        kfree(mapping);
+err:
+        return ERR_PTR(err);
+}
+
+static void release_iommu_mapping(struct kref *kref)
+{
+        struct dma_iommu_mapping *mapping =
+                container_of(kref, struct dma_iommu_mapping, kref);
+
+        iommu_domain_free(mapping->domain);
+        kfree(mapping->bitmap);
+        kfree(mapping);
+}
+
+void arm_iommu_release_mapping(struct dma_iommu_mapping *mapping)
+{
+        if (mapping)
+                kref_put(&mapping->kref, release_iommu_mapping);
+}
+
+/**
+ * arm_iommu_attach_device
+ * @dev: valid struct device pointer
+ * @mapping: io address space mapping structure (returned from
+ *      arm_iommu_create_mapping)
+ *
+ * Attaches specified io address space mapping to the provided device,
+ * this replaces the dma operations (dma_map_ops pointer) with the
+ * IOMMU aware version. More than one client might be attached to
+ * the same io address space mapping.
+ */
+int arm_iommu_attach_device(struct device *dev,
+                            struct dma_iommu_mapping *mapping)
+{
+        int err;
+
+        err = iommu_attach_device(mapping->domain, dev);
+        if (err)
+                return err;
+
+        kref_get(&mapping->kref);
+        dev->archdata.mapping = mapping;
+        set_dma_ops(dev, &iommu_ops);
+
+        pr_info("Attached IOMMU controller to %s device.\n", dev_name(dev));
+        return 0;
+}
+
+#endif
diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c
index 8f5813bbffb5..c21d06c7dd7e 100644
--- a/arch/arm/mm/init.c
+++ b/arch/arm/mm/init.c
@@ -20,6 +20,7 @@
 #include <linux/highmem.h>
 #include <linux/gfp.h>
 #include <linux/memblock.h>
+#include <linux/dma-contiguous.h>
 
 #include <asm/mach-types.h>
 #include <asm/memblock.h>
@@ -226,6 +227,17 @@ static void __init arm_adjust_dma_zone(unsigned long *size, unsigned long *hole,
 }
 #endif
 
+void __init setup_dma_zone(struct machine_desc *mdesc)
+{
+#ifdef CONFIG_ZONE_DMA
+        if (mdesc->dma_zone_size) {
+                arm_dma_zone_size = mdesc->dma_zone_size;
+                arm_dma_limit = PHYS_OFFSET + arm_dma_zone_size - 1;
+        } else
+                arm_dma_limit = 0xffffffff;
+#endif
+}
+
 static void __init arm_bootmem_free(unsigned long min, unsigned long max_low,
         unsigned long max_high)
 {
@@ -273,12 +285,9 @@ static void __init arm_bootmem_free(unsigned long min, unsigned long max_low,
          * Adjust the sizes according to any special requirements for
          * this machine type.
          */
-        if (arm_dma_zone_size) {
+        if (arm_dma_zone_size)
                 arm_adjust_dma_zone(zone_size, zhole_size,
                         arm_dma_zone_size >> PAGE_SHIFT);
-                arm_dma_limit = PHYS_OFFSET + arm_dma_zone_size - 1;
-        } else
-                arm_dma_limit = 0xffffffff;
 #endif
 
         free_area_init_node(0, zone_size, min, zhole_size);
@@ -364,6 +373,12 @@ void __init arm_memblock_init(struct meminfo *mi, struct machine_desc *mdesc)
         if (mdesc->reserve)
                 mdesc->reserve();
 
+        /*
+         * reserve memory for DMA contigouos allocations,
+         * must come from DMA area inside low memory
+         */
+        dma_contiguous_reserve(min(arm_dma_limit, arm_lowmem_limit));
+
         arm_memblock_steal_permitted = false;
         memblock_allow_resize();
         memblock_dump_all();
diff --git a/arch/arm/mm/mm.h b/arch/arm/mm/mm.h
index 27f4a619b35d..93dc0c17cdcb 100644
--- a/arch/arm/mm/mm.h
+++ b/arch/arm/mm/mm.h
@@ -67,5 +67,8 @@ extern u32 arm_dma_limit;
 #define arm_dma_limit ((u32)~0)
 #endif
 
+extern phys_addr_t arm_lowmem_limit;
+
 void __init bootmem_init(void);
 void arm_mm_memblock_reserve(void);
+void dma_contiguous_remap(void);
diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c
index aa78de8bfdd3..e5dad60b558b 100644
--- a/arch/arm/mm/mmu.c
+++ b/arch/arm/mm/mmu.c
@@ -288,6 +288,11 @@ static struct mem_type mem_types[] = {
                                 PMD_SECT_UNCACHED | PMD_SECT_XN,
                 .domain    = DOMAIN_KERNEL,
         },
+        [MT_MEMORY_DMA_READY] = {
+                .prot_pte  = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY,
+                .prot_l1   = PMD_TYPE_TABLE,
+                .domain    = DOMAIN_KERNEL,
+        },
 };
 
 const struct mem_type *get_mem_type(unsigned int type)
@@ -429,6 +434,7 @@ static void __init build_mem_type_table(void)
         if (arch_is_coherent() && cpu_is_xsc3()) {
                 mem_types[MT_MEMORY].prot_sect |= PMD_SECT_S;
                 mem_types[MT_MEMORY].prot_pte |= L_PTE_SHARED;
+                mem_types[MT_MEMORY_DMA_READY].prot_pte |= L_PTE_SHARED;
                 mem_types[MT_MEMORY_NONCACHED].prot_sect |= PMD_SECT_S;
                 mem_types[MT_MEMORY_NONCACHED].prot_pte |= L_PTE_SHARED;
         }
@@ -460,6 +466,7 @@ static void __init build_mem_type_table(void)
                         mem_types[MT_DEVICE_CACHED].prot_pte |= L_PTE_SHARED;
                         mem_types[MT_MEMORY].prot_sect |= PMD_SECT_S;
                         mem_types[MT_MEMORY].prot_pte |= L_PTE_SHARED;
+                        mem_types[MT_MEMORY_DMA_READY].prot_pte |= L_PTE_SHARED;
                         mem_types[MT_MEMORY_NONCACHED].prot_sect |= PMD_SECT_S;
                         mem_types[MT_MEMORY_NONCACHED].prot_pte |= L_PTE_SHARED;
                 }
@@ -512,6 +519,7 @@ static void __init build_mem_type_table(void)
         mem_types[MT_HIGH_VECTORS].prot_l1 |= ecc_mask;
         mem_types[MT_MEMORY].prot_sect |= ecc_mask | cp->pmd;
         mem_types[MT_MEMORY].prot_pte |= kern_pgprot;
+        mem_types[MT_MEMORY_DMA_READY].prot_pte |= kern_pgprot;
         mem_types[MT_MEMORY_NONCACHED].prot_sect |= ecc_mask;
         mem_types[MT_ROM].prot_sect |= cp->pmd;
 
@@ -596,7 +604,7 @@ static void __init alloc_init_section(pud_t *pud, unsigned long addr,
          * L1 entries, whereas PGDs refer to a group of L1 entries making
          * up one logical pointer to an L2 table.
          */
-        if (((addr | end | phys) & ~SECTION_MASK) == 0) {
+        if (type->prot_sect && ((addr | end | phys) & ~SECTION_MASK) == 0) {
                 pmd_t *p = pmd;
 
 #ifndef CONFIG_ARM_LPAE
@@ -814,7 +822,7 @@ static int __init early_vmalloc(char *arg)
 }
 early_param("vmalloc", early_vmalloc);
 
-static phys_addr_t lowmem_limit __initdata = 0;
+phys_addr_t arm_lowmem_limit __initdata = 0;
 
 void __init sanity_check_meminfo(void)
 {
@@ -897,8 +905,8 @@ void __init sanity_check_meminfo(void)
                         bank->size = newsize;
                 }
 #endif
-                if (!bank->highmem && bank->start + bank->size > lowmem_limit)
-                        lowmem_limit = bank->start + bank->size;
+                if (!bank->highmem && bank->start + bank->size > arm_lowmem_limit)
+                        arm_lowmem_limit = bank->start + bank->size;
 
                 j++;
         }
@@ -923,8 +931,8 @@ void __init sanity_check_meminfo(void)
         }
 #endif
         meminfo.nr_banks = j;
-        high_memory = __va(lowmem_limit - 1) + 1;
-        memblock_set_current_limit(lowmem_limit);
+        high_memory = __va(arm_lowmem_limit - 1) + 1;
+        memblock_set_current_limit(arm_lowmem_limit);
 }
 
 static inline void prepare_page_table(void)
@@ -949,8 +957,8 @@ static inline void prepare_page_table(void)
          * Find the end of the first block of lowmem.
          */
         end = memblock.memory.regions[0].base + memblock.memory.regions[0].size;
-        if (end >= lowmem_limit)
-                end = lowmem_limit;
+        if (end >= arm_lowmem_limit)
+                end = arm_lowmem_limit;
 
         /*
          * Clear out all the kernel space mappings, except for the first
@@ -1093,8 +1101,8 @@ static void __init map_lowmem(void)
                 phys_addr_t end = start + reg->size;
                 struct map_desc map;
 
-                if (end > lowmem_limit)
-                        end = lowmem_limit;
+                if (end > arm_lowmem_limit)
+                        end = arm_lowmem_limit;
                 if (start >= end)
                         break;
 
@@ -1115,11 +1123,12 @@ void __init paging_init(struct machine_desc *mdesc)
 {
         void *zero_page;
 
-        memblock_set_current_limit(lowmem_limit);
+        memblock_set_current_limit(arm_lowmem_limit);
 
         build_mem_type_table();
         prepare_page_table();
         map_lowmem();
+        dma_contiguous_remap();
         devicemaps_init(mdesc);
         kmap_init();
 
diff --git a/arch/arm/mm/vmregion.h b/arch/arm/mm/vmregion.h
index 162be662c088..bf312c354a21 100644
--- a/arch/arm/mm/vmregion.h
+++ b/arch/arm/mm/vmregion.h
@@ -17,7 +17,7 @@ struct arm_vmregion {
         struct list_head        vm_list;
         unsigned long           vm_start;
         unsigned long           vm_end;
-        struct page             *vm_pages;
+        void                    *priv;
         int                     vm_active;
         const void              *caller;
 };