4 files changed, 334 insertions, 39 deletions

diff --git a/arch/arm64/mm/cache.S b/arch/arm64/mm/cache.S
index 1ea9f26d1b70..c46f48b33c14 100644
--- a/arch/arm64/mm/cache.S
+++ b/arch/arm64/mm/cache.S
@@ -30,7 +30,7 @@
  *
  *      Corrupted registers: x0-x7, x9-x11
  */
-ENTRY(__flush_dcache_all)
+__flush_dcache_all:
         dsb     sy                              // ensure ordering with previous memory accesses
         mrs     x0, clidr_el1                   // read clidr
         and     x3, x0, #0x7000000              // extract loc from clidr
@@ -166,3 +166,81 @@ ENTRY(__flush_dcache_area)
         dsb     sy
         ret
 ENDPROC(__flush_dcache_area)
+
+/*
+ *      __dma_inv_range(start, end)
+ *      - start   - virtual start address of region
+ *      - end     - virtual end address of region
+ */
+__dma_inv_range:
+        dcache_line_size x2, x3
+        sub     x3, x2, #1
+        bic     x0, x0, x3
+        bic     x1, x1, x3
+1:      dc      ivac, x0                        // invalidate D / U line
+        add     x0, x0, x2
+        cmp     x0, x1
+        b.lo    1b
+        dsb     sy
+        ret
+ENDPROC(__dma_inv_range)
+
+/*
+ *      __dma_clean_range(start, end)
+ *      - start   - virtual start address of region
+ *      - end     - virtual end address of region
+ */
+__dma_clean_range:
+        dcache_line_size x2, x3
+        sub     x3, x2, #1
+        bic     x0, x0, x3
+1:      dc      cvac, x0                        // clean D / U line
+        add     x0, x0, x2
+        cmp     x0, x1
+        b.lo    1b
+        dsb     sy
+        ret
+ENDPROC(__dma_clean_range)
+
+/*
+ *      __dma_flush_range(start, end)
+ *      - start   - virtual start address of region
+ *      - end     - virtual end address of region
+ */
+ENTRY(__dma_flush_range)
+        dcache_line_size x2, x3
+        sub     x3, x2, #1
+        bic     x0, x0, x3
+1:      dc      civac, x0                       // clean & invalidate D / U line
+        add     x0, x0, x2
+        cmp     x0, x1
+        b.lo    1b
+        dsb     sy
+        ret
+ENDPROC(__dma_flush_range)
+
+/*
+ *      __dma_map_area(start, size, dir)
+ *      - start - kernel virtual start address
+ *      - size  - size of region
+ *      - dir   - DMA direction
+ */
+ENTRY(__dma_map_area)
+        add     x1, x1, x0
+        cmp     w2, #DMA_FROM_DEVICE
+        b.eq    __dma_inv_range
+        b       __dma_clean_range
+ENDPROC(__dma_map_area)
+
+/*
+ *      __dma_unmap_area(start, size, dir)
+ *      - start - kernel virtual start address
+ *      - size  - size of region
+ *      - dir   - DMA direction
+ */
+ENTRY(__dma_unmap_area)
+        add     x1, x1, x0
+        cmp     w2, #DMA_TO_DEVICE
+        b.ne    __dma_inv_range
+        ret
+ENDPROC(__dma_unmap_area)
diff --git a/arch/arm64/mm/dma-mapping.c b/arch/arm64/mm/dma-mapping.c
index fbd76785c5db..0ba347e59f06 100644
--- a/arch/arm64/mm/dma-mapping.c
+++ b/arch/arm64/mm/dma-mapping.c
@@ -30,18 +30,26 @@
 struct dma_map_ops *dma_ops;
 EXPORT_SYMBOL(dma_ops);
 
-static void *arm64_swiotlb_alloc_coherent(struct device *dev, size_t size,
-                                          dma_addr_t *dma_handle, gfp_t flags,
-                                          struct dma_attrs *attrs)
+static pgprot_t __get_dma_pgprot(struct dma_attrs *attrs, pgprot_t prot,
+                                 bool coherent)
+{
+        if (!coherent || dma_get_attr(DMA_ATTR_WRITE_COMBINE, attrs))
+                return pgprot_writecombine(prot);
+        return prot;
+}
+
+static void *__dma_alloc_coherent(struct device *dev, size_t size,
+                                  dma_addr_t *dma_handle, gfp_t flags,
+                                  struct dma_attrs *attrs)
 {
         if (dev == NULL) {
                 WARN_ONCE(1, "Use an actual device structure for DMA allocation\n");
                 return NULL;
         }
 
-        if (IS_ENABLED(CONFIG_ZONE_DMA32) &&
+        if (IS_ENABLED(CONFIG_ZONE_DMA) &&
             dev->coherent_dma_mask <= DMA_BIT_MASK(32))
-                flags |= GFP_DMA32;
+                flags |= GFP_DMA;
         if (IS_ENABLED(CONFIG_DMA_CMA)) {
                 struct page *page;
 
@@ -58,9 +66,9 @@ static void *arm64_swiotlb_alloc_coherent(struct device *dev, size_t size,
         }
 }
 
-static void arm64_swiotlb_free_coherent(struct device *dev, size_t size,
-                                        void *vaddr, dma_addr_t dma_handle,
-                                        struct dma_attrs *attrs)
+static void __dma_free_coherent(struct device *dev, size_t size,
+                                void *vaddr, dma_addr_t dma_handle,
+                                struct dma_attrs *attrs)
 {
         if (dev == NULL) {
                 WARN_ONCE(1, "Use an actual device structure for DMA allocation\n");
@@ -78,9 +86,212 @@ static void arm64_swiotlb_free_coherent(struct device *dev, size_t size,
         }
 }
 
-static struct dma_map_ops arm64_swiotlb_dma_ops = {
-        .alloc = arm64_swiotlb_alloc_coherent,
-        .free = arm64_swiotlb_free_coherent,
+static void *__dma_alloc_noncoherent(struct device *dev, size_t size,
+                                     dma_addr_t *dma_handle, gfp_t flags,
+                                     struct dma_attrs *attrs)
+{
+        struct page *page, **map;
+        void *ptr, *coherent_ptr;
+        int order, i;
+
+        size = PAGE_ALIGN(size);
+        order = get_order(size);
+
+        ptr = __dma_alloc_coherent(dev, size, dma_handle, flags, attrs);
+        if (!ptr)
+                goto no_mem;
+        map = kmalloc(sizeof(struct page *) << order, flags & ~GFP_DMA);
+        if (!map)
+                goto no_map;
+
+        /* remove any dirty cache lines on the kernel alias */
+        __dma_flush_range(ptr, ptr + size);
+
+        /* create a coherent mapping */
+        page = virt_to_page(ptr);
+        for (i = 0; i < (size >> PAGE_SHIFT); i++)
+                map[i] = page + i;
+        coherent_ptr = vmap(map, size >> PAGE_SHIFT, VM_MAP,
+                            __get_dma_pgprot(attrs, pgprot_default, false));
+        kfree(map);
+        if (!coherent_ptr)
+                goto no_map;
+
+        return coherent_ptr;
+
+no_map:
+        __dma_free_coherent(dev, size, ptr, *dma_handle, attrs);
+no_mem:
+        *dma_handle = ~0;
+        return NULL;
+}
+
+static void __dma_free_noncoherent(struct device *dev, size_t size,
+                                   void *vaddr, dma_addr_t dma_handle,
+                                   struct dma_attrs *attrs)
+{
+        void *swiotlb_addr = phys_to_virt(dma_to_phys(dev, dma_handle));
+
+        vunmap(vaddr);
+        __dma_free_coherent(dev, size, swiotlb_addr, dma_handle, attrs);
+}
+
+static dma_addr_t __swiotlb_map_page(struct device *dev, struct page *page,
+                                     unsigned long offset, size_t size,
+                                     enum dma_data_direction dir,
+                                     struct dma_attrs *attrs)
+{
+        dma_addr_t dev_addr;
+
+        dev_addr = swiotlb_map_page(dev, page, offset, size, dir, attrs);
+        __dma_map_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);
+
+        return dev_addr;
+}
+
+
+static void __swiotlb_unmap_page(struct device *dev, dma_addr_t dev_addr,
+                                 size_t size, enum dma_data_direction dir,
+                                 struct dma_attrs *attrs)
+{
+        __dma_unmap_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);
+        swiotlb_unmap_page(dev, dev_addr, size, dir, attrs);
+}
+
+static int __swiotlb_map_sg_attrs(struct device *dev, struct scatterlist *sgl,
+                                  int nelems, enum dma_data_direction dir,
+                                  struct dma_attrs *attrs)
+{
+        struct scatterlist *sg;
+        int i, ret;
+
+        ret = swiotlb_map_sg_attrs(dev, sgl, nelems, dir, attrs);
+        for_each_sg(sgl, sg, ret, i)
+                __dma_map_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)),
+                               sg->length, dir);
+
+        return ret;
+}
+
+static void __swiotlb_unmap_sg_attrs(struct device *dev,
+                                     struct scatterlist *sgl, int nelems,
+                                     enum dma_data_direction dir,
+                                     struct dma_attrs *attrs)
+{
+        struct scatterlist *sg;
+        int i;
+
+        for_each_sg(sgl, sg, nelems, i)
+                __dma_unmap_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)),
+                                 sg->length, dir);
+        swiotlb_unmap_sg_attrs(dev, sgl, nelems, dir, attrs);
+}
+
+static void __swiotlb_sync_single_for_cpu(struct device *dev,
+                                          dma_addr_t dev_addr, size_t size,
+                                          enum dma_data_direction dir)
+{
+        __dma_unmap_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);
+        swiotlb_sync_single_for_cpu(dev, dev_addr, size, dir);
+}
+
+static void __swiotlb_sync_single_for_device(struct device *dev,
+                                             dma_addr_t dev_addr, size_t size,
+                                             enum dma_data_direction dir)
+{
+        swiotlb_sync_single_for_device(dev, dev_addr, size, dir);
+        __dma_map_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);
+}
+
+static void __swiotlb_sync_sg_for_cpu(struct device *dev,
+                                      struct scatterlist *sgl, int nelems,
+                                      enum dma_data_direction dir)
+{
+        struct scatterlist *sg;
+        int i;
+
+        for_each_sg(sgl, sg, nelems, i)
+                __dma_unmap_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)),
+                                 sg->length, dir);
+        swiotlb_sync_sg_for_cpu(dev, sgl, nelems, dir);
+}
+
+static void __swiotlb_sync_sg_for_device(struct device *dev,
+                                         struct scatterlist *sgl, int nelems,
+                                         enum dma_data_direction dir)
+{
+        struct scatterlist *sg;
+        int i;
+
+        swiotlb_sync_sg_for_device(dev, sgl, nelems, dir);
+        for_each_sg(sgl, sg, nelems, i)
+                __dma_map_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)),
+                               sg->length, dir);
+}
+
+/* vma->vm_page_prot must be set appropriately before calling this function */
+static int __dma_common_mmap(struct device *dev, struct vm_area_struct *vma,
+                             void *cpu_addr, dma_addr_t dma_addr, size_t size)
+{
+        int ret = -ENXIO;
+        unsigned long nr_vma_pages = (vma->vm_end - vma->vm_start) >>
+                                        PAGE_SHIFT;
+        unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
+        unsigned long pfn = dma_to_phys(dev, dma_addr) >> PAGE_SHIFT;
+        unsigned long off = vma->vm_pgoff;
+
+        if (dma_mmap_from_coherent(dev, vma, cpu_addr, size, &ret))
+                return ret;
+
+        if (off < nr_pages && nr_vma_pages <= (nr_pages - off)) {
+                ret = remap_pfn_range(vma, vma->vm_start,
+                                      pfn + off,
+                                      vma->vm_end - vma->vm_start,
+                                      vma->vm_page_prot);
+        }
+
+        return ret;
+}
+
+static int __swiotlb_mmap_noncoherent(struct device *dev,
+                struct vm_area_struct *vma,
+                void *cpu_addr, dma_addr_t dma_addr, size_t size,
+                struct dma_attrs *attrs)
+{
+        vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot, false);
+        return __dma_common_mmap(dev, vma, cpu_addr, dma_addr, size);
+}
+
+static int __swiotlb_mmap_coherent(struct device *dev,
+                struct vm_area_struct *vma,
+                void *cpu_addr, dma_addr_t dma_addr, size_t size,
+                struct dma_attrs *attrs)
+{
+        /* Just use whatever page_prot attributes were specified */
+        return __dma_common_mmap(dev, vma, cpu_addr, dma_addr, size);
+}
+
+struct dma_map_ops noncoherent_swiotlb_dma_ops = {
+        .alloc = __dma_alloc_noncoherent,
+        .free = __dma_free_noncoherent,
+        .mmap = __swiotlb_mmap_noncoherent,
+        .map_page = __swiotlb_map_page,
+        .unmap_page = __swiotlb_unmap_page,
+        .map_sg = __swiotlb_map_sg_attrs,
+        .unmap_sg = __swiotlb_unmap_sg_attrs,
+        .sync_single_for_cpu = __swiotlb_sync_single_for_cpu,
+        .sync_single_for_device = __swiotlb_sync_single_for_device,
+        .sync_sg_for_cpu = __swiotlb_sync_sg_for_cpu,
+        .sync_sg_for_device = __swiotlb_sync_sg_for_device,
+        .dma_supported = swiotlb_dma_supported,
+        .mapping_error = swiotlb_dma_mapping_error,
+};
+EXPORT_SYMBOL(noncoherent_swiotlb_dma_ops);
+
+struct dma_map_ops coherent_swiotlb_dma_ops = {
+        .alloc = __dma_alloc_coherent,
+        .free = __dma_free_coherent,
+        .mmap = __swiotlb_mmap_coherent,
         .map_page = swiotlb_map_page,
         .unmap_page = swiotlb_unmap_page,
         .map_sg = swiotlb_map_sg_attrs,
@@ -92,12 +303,19 @@ static struct dma_map_ops arm64_swiotlb_dma_ops = {
         .dma_supported = swiotlb_dma_supported,
         .mapping_error = swiotlb_dma_mapping_error,
 };
+EXPORT_SYMBOL(coherent_swiotlb_dma_ops);
+
+extern int swiotlb_late_init_with_default_size(size_t default_size);
 
-void __init arm64_swiotlb_init(void)
+static int __init swiotlb_late_init(void)
 {
-        dma_ops = &arm64_swiotlb_dma_ops;
-        swiotlb_init(1);
+        size_t swiotlb_size = min(SZ_64M, MAX_ORDER_NR_PAGES << PAGE_SHIFT);
+
+        dma_ops = &coherent_swiotlb_dma_ops;
+
+        return swiotlb_late_init_with_default_size(swiotlb_size);
 }
+subsys_initcall(swiotlb_late_init);
 
 #define PREALLOC_DMA_DEBUG_ENTRIES      4096
 
diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c
index d0b4c2efda90..88627c450a6c 100644
--- a/arch/arm64/mm/init.c
+++ b/arch/arm64/mm/init.c
@@ -30,6 +30,7 @@
 #include <linux/memblock.h>
 #include <linux/sort.h>
 #include <linux/of_fdt.h>
+#include <linux/dma-mapping.h>
 #include <linux/dma-contiguous.h>
 
 #include <asm/sections.h>
@@ -59,22 +60,22 @@ static int __init early_initrd(char *p)
 early_param("initrd", early_initrd);
 #endif
 
-#define MAX_DMA32_PFN ((4UL * 1024 * 1024 * 1024) >> PAGE_SHIFT)
-
 static void __init zone_sizes_init(unsigned long min, unsigned long max)
 {
         struct memblock_region *reg;
         unsigned long zone_size[MAX_NR_ZONES], zhole_size[MAX_NR_ZONES];
-        unsigned long max_dma32 = min;
+        unsigned long max_dma = min;
 
         memset(zone_size, 0, sizeof(zone_size));
 
-#ifdef CONFIG_ZONE_DMA32
         /* 4GB maximum for 32-bit only capable devices */
-        max_dma32 = max(min, min(max, MAX_DMA32_PFN));
-        zone_size[ZONE_DMA32] = max_dma32 - min;
-#endif
-        zone_size[ZONE_NORMAL] = max - max_dma32;
+        if (IS_ENABLED(CONFIG_ZONE_DMA)) {
+                unsigned long max_dma_phys =
+                        (unsigned long)dma_to_phys(NULL, DMA_BIT_MASK(32) + 1);
+                max_dma = max(min, min(max, max_dma_phys >> PAGE_SHIFT));
+                zone_size[ZONE_DMA] = max_dma - min;
+        }
+        zone_size[ZONE_NORMAL] = max - max_dma;
 
         memcpy(zhole_size, zone_size, sizeof(zhole_size));
 
@@ -84,15 +85,15 @@ static void __init zone_sizes_init(unsigned long min, unsigned long max)
 
                 if (start >= max)
                         continue;
-#ifdef CONFIG_ZONE_DMA32
-                if (start < max_dma32) {
-                        unsigned long dma_end = min(end, max_dma32);
-                        zhole_size[ZONE_DMA32] -= dma_end - start;
+
+                if (IS_ENABLED(CONFIG_ZONE_DMA) && start < max_dma) {
+                        unsigned long dma_end = min(end, max_dma);
+                        zhole_size[ZONE_DMA] -= dma_end - start;
                 }
-#endif
-                if (end > max_dma32) {
+
+                if (end > max_dma) {
                         unsigned long normal_end = min(end, max);
-                        unsigned long normal_start = max(start, max_dma32);
+                        unsigned long normal_start = max(start, max_dma);
                         zhole_size[ZONE_NORMAL] -= normal_end - normal_start;
                 }
         }
@@ -261,8 +262,6 @@ static void __init free_unused_memmap(void)
  */
 void __init mem_init(void)
 {
-        arm64_swiotlb_init();
-
         max_mapnr   = pfn_to_page(max_pfn + PHYS_PFN_OFFSET) - mem_map;
 
 #ifndef CONFIG_SPARSEMEM_VMEMMAP
diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S
index 1333e6f9a8e5..e085ee6ef4e2 100644
--- a/arch/arm64/mm/proc.S
+++ b/arch/arm64/mm/proc.S
@@ -173,12 +173,6 @@ ENDPROC(cpu_do_switch_mm)
  *      value of the SCTLR_EL1 register.
  */
 ENTRY(__cpu_setup)
-        /*
-         * Preserve the link register across the function call.
-         */
-        mov     x28, lr
-        bl      __flush_dcache_all
-        mov     lr, x28
         ic      iallu                           // I+BTB cache invalidate
         tlbi    vmalle1is                       // invalidate I + D TLBs
         dsb     sy
@@ -215,8 +209,14 @@ ENTRY(__cpu_setup)
          * Set/prepare TCR and TTBR. We use 512GB (39-bit) address range for
          * both user and kernel.
          */
-        ldr     x10, =TCR_TxSZ(VA_BITS) | TCR_FLAGS | TCR_IPS_40BIT | \
+        ldr     x10, =TCR_TxSZ(VA_BITS) | TCR_FLAGS | \
                       TCR_ASID16 | TCR_TBI0 | (1 << 31)
+        /*
+         * Read the PARange bits from ID_AA64MMFR0_EL1 and set the IPS bits in
+         * TCR_EL1.
+         */
+        mrs     x9, ID_AA64MMFR0_EL1
+        bfi     x10, x9, #32, #3
 #ifdef CONFIG_ARM64_64K_PAGES
         orr     x10, x10, TCR_TG0_64K
         orr     x10, x10, TCR_TG1_64K