aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2018-10-22 13:16:03 -0400
committerLinus Torvalds <torvalds@linux-foundation.org>2018-10-22 13:16:03 -0400
commitcff229491af5df946781edfbeafd43e9cf66a3b4 (patch)
treedff787191eb78a69c054510a9ea74cc391330b73
parent13775dacca5c158a257320f4b47e1220b82e3b21 (diff)
parentb9fd04262a8abc366f40a9e97598e94591352c26 (diff)
Merge tag 'dma-mapping-4.20' of git://git.infradead.org/users/hch/dma-mapping
Pull dma mapping updates from Christoph Hellwig: "First batch of dma-mapping changes for 4.20. There will be a second PR as some big changes were only applied just before the end of the merge window, and I want to give them a few more days in linux-next. Summary: - mostly more consolidation of the direct mapping code, including converting over hexagon, and merging the coherent and non-coherent code into a single dma_map_ops instance (me) - cleanups for the dma_configure/dma_unconfigure callchains (me) - better handling of dma_masks in odd setups (me, Alexander Duyck) - better debugging of passing vmalloc address to the DMA API (Stephen Boyd) - CMA command line parsing fix (He Zhe)" * tag 'dma-mapping-4.20' of git://git.infradead.org/users/hch/dma-mapping: (27 commits) dma-direct: respect DMA_ATTR_NO_WARN dma-mapping: translate __GFP_NOFAIL to DMA_ATTR_NO_WARN dma-direct: document the zone selection logic dma-debug: Check for drivers mapping invalid addresses in dma_map_single() dma-direct: fix return value of dma_direct_supported dma-mapping: move dma_default_get_required_mask under ifdef dma-direct: always allow dma mask <= physiscal memory size dma-direct: implement complete bus_dma_mask handling dma-direct: refine dma_direct_alloc zone selection dma-direct: add an explicit dma_direct_get_required_mask dma-mapping: make the get_required_mask method available unconditionally unicore32: remove swiotlb support Revert "dma-mapping: clear dev->dma_ops in arch_teardown_dma_ops" dma-mapping: support non-coherent devices in dma_common_get_sgtable dma-mapping: consolidate the dma mmap implementations dma-mapping: merge direct and noncoherent ops dma-mapping: move the dma_coherent flag to struct device MIPS: don't select DMA_MAYBE_COHERENT from DMA_PERDEV_COHERENT dma-mapping: add the missing ARCH_HAS_SYNC_DMA_FOR_CPU_ALL declaration dma-mapping: fix panic caused by passing empty cma command line argument ...
Diffstat
-rw-r--r--arch/arc/Kconfig4
-rw-r--r--arch/arc/mm/dma.c41
-rw-r--r--arch/arm/include/asm/dma-mapping.h2
-rw-r--r--arch/arm/mm/dma-mapping-nommu.c11
-rw-r--r--arch/c6x/Kconfig2
-rw-r--r--arch/hexagon/Kconfig2
-rw-r--r--arch/hexagon/include/asm/Kbuild1
-rw-r--r--arch/hexagon/include/asm/dma-mapping.h40
-rw-r--r--arch/hexagon/kernel/dma.c143
-rw-r--r--arch/ia64/include/asm/dma-mapping.h2
-rw-r--r--arch/ia64/include/asm/machvec.h7
-rw-r--r--arch/ia64/include/asm/machvec_init.h1
-rw-r--r--arch/ia64/include/asm/machvec_sn2.h2
-rw-r--r--arch/ia64/pci/pci.c26
-rw-r--r--arch/ia64/sn/pci/pci_dma.c4
-rw-r--r--arch/m68k/Kconfig2
-rw-r--r--arch/microblaze/Kconfig4
-rw-r--r--arch/microblaze/include/asm/pgtable.h2
-rw-r--r--arch/microblaze/kernel/dma.c22
-rw-r--r--arch/microblaze/mm/consistent.c3
-rw-r--r--arch/mips/Kconfig7
-rw-r--r--arch/mips/include/asm/Kbuild1
-rw-r--r--arch/mips/include/asm/device.h19
-rw-r--r--arch/mips/include/asm/dma-coherence.h6
-rw-r--r--arch/mips/include/asm/dma-mapping.h4
-rw-r--r--arch/mips/jazz/jazzdma.c7
-rw-r--r--arch/mips/kernel/setup.c2
-rw-r--r--arch/mips/mm/c-r4k.c17
-rw-r--r--arch/mips/mm/dma-noncoherent.c79
-rw-r--r--arch/nds32/Kconfig2
-rw-r--r--arch/nios2/Kconfig2
-rw-r--r--arch/openrisc/Kconfig2
-rw-r--r--arch/parisc/Kconfig2
-rw-r--r--arch/parisc/kernel/setup.c2
-rw-r--r--arch/sh/Kconfig3
-rw-r--r--arch/sparc/Kconfig2
-rw-r--r--arch/sparc/include/asm/dma-mapping.h4
-rw-r--r--arch/unicore32/Kconfig2
-rw-r--r--arch/unicore32/include/asm/Kbuild1
-rw-r--r--arch/unicore32/include/asm/dma-mapping.h22
-rw-r--r--arch/unicore32/mm/init.c3
-rw-r--r--arch/x86/kernel/amd_gart_64.c6
-rw-r--r--arch/xtensa/Kconfig2
-rw-r--r--drivers/acpi/arm64/iort.c2
-rw-r--r--drivers/acpi/scan.c10
-rw-r--r--drivers/base/dd.c12
-rw-r--r--drivers/base/platform.c11
-rw-r--r--drivers/of/device.c12
-rw-r--r--drivers/pci/controller/vmd.c4
-rw-r--r--drivers/xen/swiotlb-xen.c4
-rw-r--r--include/acpi/acpi_bus.h1
-rw-r--r--include/asm-generic/dma-mapping.h9
-rw-r--r--include/linux/acpi.h2
-rw-r--r--include/linux/device.h7
-rw-r--r--include/linux/dma-debug.h8
-rw-r--r--include/linux/dma-direct.h8
-rw-r--r--include/linux/dma-mapping.h42
-rw-r--r--include/linux/dma-noncoherent.h27
-rw-r--r--include/linux/of_device.h3
-rw-r--r--kernel/dma/Kconfig16
-rw-r--r--kernel/dma/Makefile1
-rw-r--r--kernel/dma/contiguous.c6
-rw-r--r--kernel/dma/debug.c16
-rw-r--r--kernel/dma/direct.c222
-rw-r--r--kernel/dma/mapping.c71
-rw-r--r--kernel/dma/noncoherent.c106
66 files changed, 420 insertions, 698 deletions
diff --git a/arch/arc/Kconfig b/arch/arc/Kconfig
index a045f3086047..e98c6b8e6186 100644
--- a/arch/arc/Kconfig
+++ b/arch/arc/Kconfig
@@ -9,6 +9,7 @@
9config ARC 9config ARC
10 def_bool y 10 def_bool y
11 select ARC_TIMERS 11 select ARC_TIMERS
12 select ARCH_HAS_DMA_COHERENT_TO_PFN
12 select ARCH_HAS_PTE_SPECIAL 13 select ARCH_HAS_PTE_SPECIAL
13 select ARCH_HAS_SYNC_DMA_FOR_CPU 14 select ARCH_HAS_SYNC_DMA_FOR_CPU
14 select ARCH_HAS_SYNC_DMA_FOR_DEVICE 15 select ARCH_HAS_SYNC_DMA_FOR_DEVICE
@@ -17,8 +18,7 @@ config ARC
17 select BUILDTIME_EXTABLE_SORT 18 select BUILDTIME_EXTABLE_SORT
18 select CLONE_BACKWARDS 19 select CLONE_BACKWARDS
19 select COMMON_CLK 20 select COMMON_CLK
20 select DMA_NONCOHERENT_OPS 21 select DMA_DIRECT_OPS
21 select DMA_NONCOHERENT_MMAP
22 select GENERIC_ATOMIC64 if !ISA_ARCV2 || !(ARC_HAS_LL64 && ARC_HAS_LLSC) 22 select GENERIC_ATOMIC64 if !ISA_ARCV2 || !(ARC_HAS_LL64 && ARC_HAS_LLSC)
23 select GENERIC_CLOCKEVENTS 23 select GENERIC_CLOCKEVENTS
24 select GENERIC_FIND_FIRST_BIT 24 select GENERIC_FIND_FIRST_BIT
diff --git a/arch/arc/mm/dma.c b/arch/arc/mm/dma.c
index c75d5c3470e3..db203ff69ccf 100644
--- a/arch/arc/mm/dma.c
+++ b/arch/arc/mm/dma.c
@@ -84,29 +84,10 @@ void arch_dma_free(struct device *dev, size_t size, void *vaddr,
84 __free_pages(page, get_order(size)); 84 __free_pages(page, get_order(size));
85} 85}
86 86
87int arch_dma_mmap(struct device *dev, struct vm_area_struct *vma, 87long arch_dma_coherent_to_pfn(struct device *dev, void *cpu_addr,
88 void *cpu_addr, dma_addr_t dma_addr, size_t size, 88 dma_addr_t dma_addr)
89 unsigned long attrs)
90{ 89{
91 unsigned long user_count = vma_pages(vma); 90 return __phys_to_pfn(dma_addr);
92 unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT;
93 unsigned long pfn = __phys_to_pfn(dma_addr);
94 unsigned long off = vma->vm_pgoff;
95 int ret = -ENXIO;
96
97 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
98
99 if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret))
100 return ret;
101
102 if (off < count && user_count <= (count - off)) {
103 ret = remap_pfn_range(vma, vma->vm_start,
104 pfn + off,
105 user_count << PAGE_SHIFT,
106 vma->vm_page_prot);
107 }
108
109 return ret;
110} 91}
111 92
112/* 93/*
@@ -167,7 +148,7 @@ void arch_sync_dma_for_cpu(struct device *dev, phys_addr_t paddr,
167} 148}
168 149
169/* 150/*
170 * Plug in coherent or noncoherent dma ops 151 * Plug in direct dma map ops.
171 */ 152 */
172void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, 153void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
173 const struct iommu_ops *iommu, bool coherent) 154 const struct iommu_ops *iommu, bool coherent)
@@ -175,13 +156,11 @@ void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
175 /* 156 /*
176 * IOC hardware snoops all DMA traffic keeping the caches consistent 157 * IOC hardware snoops all DMA traffic keeping the caches consistent
177 * with memory - eliding need for any explicit cache maintenance of 158 * with memory - eliding need for any explicit cache maintenance of
178 * DMA buffers - so we can use dma_direct cache ops. 159 * DMA buffers.
179 */ 160 */
180 if (is_isa_arcv2() && ioc_enable && coherent) { 161 if (is_isa_arcv2() && ioc_enable && coherent)
181 set_dma_ops(dev, &dma_direct_ops); 162 dev->dma_coherent = true;
182 dev_info(dev, "use dma_direct_ops cache ops\n"); 163
183 } else { 164 dev_info(dev, "use %sncoherent DMA ops\n",
184 set_dma_ops(dev, &dma_noncoherent_ops); 165 dev->dma_coherent ? "" : "non");
185 dev_info(dev, "use dma_noncoherent_ops cache ops\n");
186 }
187} 166}
diff --git a/arch/arm/include/asm/dma-mapping.h b/arch/arm/include/asm/dma-mapping.h
index 8436f6ade57d..965b7c846ecb 100644
--- a/arch/arm/include/asm/dma-mapping.h
+++ b/arch/arm/include/asm/dma-mapping.h
@@ -100,8 +100,10 @@ static inline unsigned long dma_max_pfn(struct device *dev)
100extern void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, 100extern void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
101 const struct iommu_ops *iommu, bool coherent); 101 const struct iommu_ops *iommu, bool coherent);
102 102
103#ifdef CONFIG_MMU
103#define arch_teardown_dma_ops arch_teardown_dma_ops 104#define arch_teardown_dma_ops arch_teardown_dma_ops
104extern void arch_teardown_dma_ops(struct device *dev); 105extern void arch_teardown_dma_ops(struct device *dev);
106#endif
105 107
106/* do not use this function in a driver */ 108/* do not use this function in a driver */
107static inline bool is_device_dma_coherent(struct device *dev) 109static inline bool is_device_dma_coherent(struct device *dev)
diff --git a/arch/arm/mm/dma-mapping-nommu.c b/arch/arm/mm/dma-mapping-nommu.c
index f448a0663b10..712416ecd8e6 100644
--- a/arch/arm/mm/dma-mapping-nommu.c
+++ b/arch/arm/mm/dma-mapping-nommu.c
@@ -47,7 +47,8 @@ static void *arm_nommu_dma_alloc(struct device *dev, size_t size,
47 */ 47 */
48 48
49 if (attrs & DMA_ATTR_NON_CONSISTENT) 49 if (attrs & DMA_ATTR_NON_CONSISTENT)
50 return dma_direct_alloc(dev, size, dma_handle, gfp, attrs); 50 return dma_direct_alloc_pages(dev, size, dma_handle, gfp,
51 attrs);
51 52
52 ret = dma_alloc_from_global_coherent(size, dma_handle); 53 ret = dma_alloc_from_global_coherent(size, dma_handle);
53 54
@@ -70,7 +71,7 @@ static void arm_nommu_dma_free(struct device *dev, size_t size,
70 unsigned long attrs) 71 unsigned long attrs)
71{ 72{
72 if (attrs & DMA_ATTR_NON_CONSISTENT) { 73 if (attrs & DMA_ATTR_NON_CONSISTENT) {
73 dma_direct_free(dev, size, cpu_addr, dma_addr, attrs); 74 dma_direct_free_pages(dev, size, cpu_addr, dma_addr, attrs);
74 } else { 75 } else {
75 int ret = dma_release_from_global_coherent(get_order(size), 76 int ret = dma_release_from_global_coherent(get_order(size),
76 cpu_addr); 77 cpu_addr);
@@ -90,7 +91,7 @@ static int arm_nommu_dma_mmap(struct device *dev, struct vm_area_struct *vma,
90 if (dma_mmap_from_global_coherent(vma, cpu_addr, size, &ret)) 91 if (dma_mmap_from_global_coherent(vma, cpu_addr, size, &ret))
91 return ret; 92 return ret;
92 93
93 return dma_common_mmap(dev, vma, cpu_addr, dma_addr, size); 94 return dma_common_mmap(dev, vma, cpu_addr, dma_addr, size, attrs);
94} 95}
95 96
96 97
@@ -237,7 +238,3 @@ void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
237 238
238 set_dma_ops(dev, dma_ops); 239 set_dma_ops(dev, dma_ops);
239} 240}
240
241void arch_teardown_dma_ops(struct device *dev)
242{
243}
diff --git a/arch/c6x/Kconfig b/arch/c6x/Kconfig
index a641b0bf1611..f65a084607fd 100644
--- a/arch/c6x/Kconfig
+++ b/arch/c6x/Kconfig
@@ -9,7 +9,7 @@ config C6X
9 select ARCH_HAS_SYNC_DMA_FOR_CPU 9 select ARCH_HAS_SYNC_DMA_FOR_CPU
10 select ARCH_HAS_SYNC_DMA_FOR_DEVICE 10 select ARCH_HAS_SYNC_DMA_FOR_DEVICE
11 select CLKDEV_LOOKUP 11 select CLKDEV_LOOKUP
12 select DMA_NONCOHERENT_OPS 12 select DMA_DIRECT_OPS
13 select GENERIC_ATOMIC64 13 select GENERIC_ATOMIC64
14 select GENERIC_IRQ_SHOW 14 select GENERIC_IRQ_SHOW
15 select HAVE_ARCH_TRACEHOOK 15 select HAVE_ARCH_TRACEHOOK
diff --git a/arch/hexagon/Kconfig b/arch/hexagon/Kconfig
index 89a4b22f34d9..3ef46522e89f 100644
--- a/arch/hexagon/Kconfig
+++ b/arch/hexagon/Kconfig
@@ -4,6 +4,7 @@ comment "Linux Kernel Configuration for Hexagon"
4 4
5config HEXAGON 5config HEXAGON
6 def_bool y 6 def_bool y
7 select ARCH_HAS_SYNC_DMA_FOR_DEVICE
7 select ARCH_NO_PREEMPT 8 select ARCH_NO_PREEMPT
8 select HAVE_OPROFILE 9 select HAVE_OPROFILE
9 # Other pending projects/to-do items. 10 # Other pending projects/to-do items.
@@ -29,6 +30,7 @@ config HEXAGON
29 select GENERIC_CLOCKEVENTS_BROADCAST 30 select GENERIC_CLOCKEVENTS_BROADCAST
30 select MODULES_USE_ELF_RELA 31 select MODULES_USE_ELF_RELA
31 select GENERIC_CPU_DEVICES 32 select GENERIC_CPU_DEVICES
33 select DMA_DIRECT_OPS
32 ---help--- 34 ---help---
33 Qualcomm Hexagon is a processor architecture designed for high 35 Qualcomm Hexagon is a processor architecture designed for high
34 performance and low power across a wide variety of applications. 36 performance and low power across a wide variety of applications.
diff --git a/arch/hexagon/include/asm/Kbuild b/arch/hexagon/include/asm/Kbuild
index dd2fd9c0d292..47c4da3d64a4 100644
--- a/arch/hexagon/include/asm/Kbuild
+++ b/arch/hexagon/include/asm/Kbuild
@@ -6,6 +6,7 @@ generic-y += compat.h
6generic-y += current.h 6generic-y += current.h
7generic-y += device.h 7generic-y += device.h
8generic-y += div64.h 8generic-y += div64.h
9generic-y += dma-mapping.h
9generic-y += emergency-restart.h 10generic-y += emergency-restart.h
10generic-y += extable.h 11generic-y += extable.h
11generic-y += fb.h 12generic-y += fb.h
diff --git a/arch/hexagon/include/asm/dma-mapping.h b/arch/hexagon/include/asm/dma-mapping.h
deleted file mode 100644
index 263f6acbfb0f..000000000000
--- a/arch/hexagon/include/asm/dma-mapping.h
+++ /dev/null
@@ -1,40 +0,0 @@
1/*
2 * DMA operations for the Hexagon architecture
3 *
4 * Copyright (c) 2010-2011, The Linux Foundation. All rights reserved.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 and
8 * only version 2 as published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
18 * 02110-1301, USA.
19 */
20
21#ifndef _ASM_DMA_MAPPING_H
22#define _ASM_DMA_MAPPING_H
23
24#include <linux/types.h>
25#include <linux/cache.h>
26#include <linux/mm.h>
27#include <linux/scatterlist.h>
28#include <linux/dma-debug.h>
29#include <asm/io.h>
30
31struct device;
32
33extern const struct dma_map_ops *dma_ops;
34
35static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus)
36{
37 return dma_ops;
38}
39
40#endif
diff --git a/arch/hexagon/kernel/dma.c b/arch/hexagon/kernel/dma.c
index 7ebe7ad19d15..706699374444 100644
--- a/arch/hexagon/kernel/dma.c
+++ b/arch/hexagon/kernel/dma.c
@@ -18,32 +18,19 @@
18 * 02110-1301, USA. 18 * 02110-1301, USA.
19 */ 19 */
20 20
21#include <linux/dma-mapping.h> 21#include <linux/dma-noncoherent.h>
22#include <linux/dma-direct.h>
23#include <linux/bootmem.h> 22#include <linux/bootmem.h>
24#include <linux/genalloc.h> 23#include <linux/genalloc.h>
25#include <asm/dma-mapping.h>
26#include <linux/module.h> 24#include <linux/module.h>
27#include <asm/page.h> 25#include <asm/page.h>
28 26
29#define HEXAGON_MAPPING_ERROR 0
30
31const struct dma_map_ops *dma_ops;
32EXPORT_SYMBOL(dma_ops);
33
34static inline void *dma_addr_to_virt(dma_addr_t dma_addr)
35{
36 return phys_to_virt((unsigned long) dma_addr);
37}
38
39static struct gen_pool *coherent_pool; 27static struct gen_pool *coherent_pool;
40 28
41 29
42/* Allocates from a pool of uncached memory that was reserved at boot time */ 30/* Allocates from a pool of uncached memory that was reserved at boot time */
43 31
44static void *hexagon_dma_alloc_coherent(struct device *dev, size_t size, 32void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_addr,
45 dma_addr_t *dma_addr, gfp_t flag, 33 gfp_t flag, unsigned long attrs)
46 unsigned long attrs)
47{ 34{
48 void *ret; 35 void *ret;
49 36
@@ -75,58 +62,17 @@ static void *hexagon_dma_alloc_coherent(struct device *dev, size_t size,
75 return ret; 62 return ret;
76} 63}
77 64
78static void hexagon_free_coherent(struct device *dev, size_t size, void *vaddr, 65void arch_dma_free(struct device *dev, size_t size, void *vaddr,
79 dma_addr_t dma_addr, unsigned long attrs) 66 dma_addr_t dma_addr, unsigned long attrs)
80{ 67{
81 gen_pool_free(coherent_pool, (unsigned long) vaddr, size); 68 gen_pool_free(coherent_pool, (unsigned long) vaddr, size);
82} 69}
83 70
84static int check_addr(const char *name, struct device *hwdev, 71void arch_sync_dma_for_device(struct device *dev, phys_addr_t paddr,
85 dma_addr_t bus, size_t size) 72 size_t size, enum dma_data_direction dir)
86{
87 if (hwdev && hwdev->dma_mask && !dma_capable(hwdev, bus, size)) {
88 if (*hwdev->dma_mask >= DMA_BIT_MASK(32))
89 printk(KERN_ERR
90 "%s: overflow %Lx+%zu of device mask %Lx\n",
91 name, (long long)bus, size,
92 (long long)*hwdev->dma_mask);
93 return 0;
94 }
95 return 1;
96}
97
98static int hexagon_map_sg(struct device *hwdev, struct scatterlist *sg,
99 int nents, enum dma_data_direction dir,
100 unsigned long attrs)
101{ 73{
102 struct scatterlist *s; 74 void *addr = phys_to_virt(paddr);
103 int i;
104
105 WARN_ON(nents == 0 || sg[0].length == 0);
106
107 for_each_sg(sg, s, nents, i) {
108 s->dma_address = sg_phys(s);
109 if (!check_addr("map_sg", hwdev, s->dma_address, s->length))
110 return 0;
111
112 s->dma_length = s->length;
113 75
114 if (attrs & DMA_ATTR_SKIP_CPU_SYNC)
115 continue;
116
117 flush_dcache_range(dma_addr_to_virt(s->dma_address),
118 dma_addr_to_virt(s->dma_address + s->length));
119 }
120
121 return nents;
122}
123
124/*
125 * address is virtual
126 */
127static inline void dma_sync(void *addr, size_t size,
128 enum dma_data_direction dir)
129{
130 switch (dir) { 76 switch (dir) {
131 case DMA_TO_DEVICE: 77 case DMA_TO_DEVICE:
132 hexagon_clean_dcache_range((unsigned long) addr, 78 hexagon_clean_dcache_range((unsigned long) addr,
@@ -144,76 +90,3 @@ static inline void dma_sync(void *addr, size_t size,
144 BUG(); 90 BUG();
145 } 91 }
146} 92}
147
148/**
149 * hexagon_map_page() - maps an address for device DMA
150 * @dev: pointer to DMA device
151 * @page: pointer to page struct of DMA memory
152 * @offset: offset within page
153 * @size: size of memory to map
154 * @dir: transfer direction
155 * @attrs: pointer to DMA attrs (not used)
156 *
157 * Called to map a memory address to a DMA address prior
158 * to accesses to/from device.
159 *
160 * We don't particularly have many hoops to jump through
161 * so far. Straight translation between phys and virtual.
162 *
163 * DMA is not cache coherent so sync is necessary; this
164 * seems to be a convenient place to do it.
165 *
166 */
167static dma_addr_t hexagon_map_page(struct device *dev, struct page *page,
168 unsigned long offset, size_t size,
169 enum dma_data_direction dir,
170 unsigned long attrs)
171{
172 dma_addr_t bus = page_to_phys(page) + offset;
173 WARN_ON(size == 0);
174
175 if (!check_addr("map_single", dev, bus, size))
176 return HEXAGON_MAPPING_ERROR;
177
178 if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
179 dma_sync(dma_addr_to_virt(bus), size, dir);
180
181 return bus;
182}
183
184static void hexagon_sync_single_for_cpu(struct device *dev,
185 dma_addr_t dma_handle, size_t size,
186 enum dma_data_direction dir)
187{
188 dma_sync(dma_addr_to_virt(dma_handle), size, dir);
189}
190
191static void hexagon_sync_single_for_device(struct device *dev,
192 dma_addr_t dma_handle, size_t size,
193 enum dma_data_direction dir)
194{
195 dma_sync(dma_addr_to_virt(dma_handle), size, dir);
196}
197
198static int hexagon_mapping_error(struct device *dev, dma_addr_t dma_addr)
199{
200 return dma_addr == HEXAGON_MAPPING_ERROR;
201}
202
203const struct dma_map_ops hexagon_dma_ops = {
204 .alloc = hexagon_dma_alloc_coherent,
205 .free = hexagon_free_coherent,
206 .map_sg = hexagon_map_sg,
207 .map_page = hexagon_map_page,
208 .sync_single_for_cpu = hexagon_sync_single_for_cpu,
209 .sync_single_for_device = hexagon_sync_single_for_device,
210 .mapping_error = hexagon_mapping_error,
211};
212
213void __init hexagon_dma_init(void)
214{
215 if (dma_ops)
216 return;
217
218 dma_ops = &hexagon_dma_ops;
219}
diff --git a/arch/ia64/include/asm/dma-mapping.h b/arch/ia64/include/asm/dma-mapping.h
index 76e4d6632d68..522745ae67bb 100644
--- a/arch/ia64/include/asm/dma-mapping.h
+++ b/arch/ia64/include/asm/dma-mapping.h
@@ -10,8 +10,6 @@
10#include <linux/scatterlist.h> 10#include <linux/scatterlist.h>
11#include <linux/dma-debug.h> 11#include <linux/dma-debug.h>
12 12
13#define ARCH_HAS_DMA_GET_REQUIRED_MASK
14
15extern const struct dma_map_ops *dma_ops; 13extern const struct dma_map_ops *dma_ops;
16extern struct ia64_machine_vector ia64_mv; 14extern struct ia64_machine_vector ia64_mv;
17extern void set_iommu_machvec(void); 15extern void set_iommu_machvec(void);
diff --git a/arch/ia64/include/asm/machvec.h b/arch/ia64/include/asm/machvec.h
index 267f4f170191..5133739966bc 100644
--- a/arch/ia64/include/asm/machvec.h
+++ b/arch/ia64/include/asm/machvec.h
@@ -44,7 +44,6 @@ typedef void ia64_mv_kernel_launch_event_t(void);
44 44
45/* DMA-mapping interface: */ 45/* DMA-mapping interface: */
46typedef void ia64_mv_dma_init (void); 46typedef void ia64_mv_dma_init (void);
47typedef u64 ia64_mv_dma_get_required_mask (struct device *);
48typedef const struct dma_map_ops *ia64_mv_dma_get_ops(struct device *); 47typedef const struct dma_map_ops *ia64_mv_dma_get_ops(struct device *);
49 48
50/* 49/*
@@ -127,7 +126,6 @@ extern void machvec_tlb_migrate_finish (struct mm_struct *);
127# define platform_global_tlb_purge ia64_mv.global_tlb_purge 126# define platform_global_tlb_purge ia64_mv.global_tlb_purge
128# define platform_tlb_migrate_finish ia64_mv.tlb_migrate_finish 127# define platform_tlb_migrate_finish ia64_mv.tlb_migrate_finish
129# define platform_dma_init ia64_mv.dma_init 128# define platform_dma_init ia64_mv.dma_init
130# define platform_dma_get_required_mask ia64_mv.dma_get_required_mask
131# define platform_dma_get_ops ia64_mv.dma_get_ops 129# define platform_dma_get_ops ia64_mv.dma_get_ops
132# define platform_irq_to_vector ia64_mv.irq_to_vector 130# define platform_irq_to_vector ia64_mv.irq_to_vector
133# define platform_local_vector_to_irq ia64_mv.local_vector_to_irq 131# define platform_local_vector_to_irq ia64_mv.local_vector_to_irq
@@ -171,7 +169,6 @@ struct ia64_machine_vector {
171 ia64_mv_global_tlb_purge_t *global_tlb_purge; 169 ia64_mv_global_tlb_purge_t *global_tlb_purge;
172 ia64_mv_tlb_migrate_finish_t *tlb_migrate_finish; 170 ia64_mv_tlb_migrate_finish_t *tlb_migrate_finish;
173 ia64_mv_dma_init *dma_init; 171 ia64_mv_dma_init *dma_init;
174 ia64_mv_dma_get_required_mask *dma_get_required_mask;
175 ia64_mv_dma_get_ops *dma_get_ops; 172 ia64_mv_dma_get_ops *dma_get_ops;
176 ia64_mv_irq_to_vector *irq_to_vector; 173 ia64_mv_irq_to_vector *irq_to_vector;
177 ia64_mv_local_vector_to_irq *local_vector_to_irq; 174 ia64_mv_local_vector_to_irq *local_vector_to_irq;
@@ -211,7 +208,6 @@ struct ia64_machine_vector {
211 platform_global_tlb_purge, \ 208 platform_global_tlb_purge, \
212 platform_tlb_migrate_finish, \ 209 platform_tlb_migrate_finish, \
213 platform_dma_init, \ 210 platform_dma_init, \
214 platform_dma_get_required_mask, \
215 platform_dma_get_ops, \ 211 platform_dma_get_ops, \
216 platform_irq_to_vector, \ 212 platform_irq_to_vector, \
217 platform_local_vector_to_irq, \ 213 platform_local_vector_to_irq, \
@@ -286,9 +282,6 @@ extern const struct dma_map_ops *dma_get_ops(struct device *);
286#ifndef platform_dma_get_ops 282#ifndef platform_dma_get_ops
287# define platform_dma_get_ops dma_get_ops 283# define platform_dma_get_ops dma_get_ops
288#endif 284#endif
289#ifndef platform_dma_get_required_mask
290# define platform_dma_get_required_mask ia64_dma_get_required_mask
291#endif
292#ifndef platform_irq_to_vector 285#ifndef platform_irq_to_vector
293# define platform_irq_to_vector __ia64_irq_to_vector 286# define platform_irq_to_vector __ia64_irq_to_vector
294#endif 287#endif
diff --git a/arch/ia64/include/asm/machvec_init.h b/arch/ia64/include/asm/machvec_init.h
index 2b32fd06b7c6..2aafb69a3787 100644
--- a/arch/ia64/include/asm/machvec_init.h
+++ b/arch/ia64/include/asm/machvec_init.h
@@ -4,7 +4,6 @@
4 4
5extern ia64_mv_send_ipi_t ia64_send_ipi; 5extern ia64_mv_send_ipi_t ia64_send_ipi;
6extern ia64_mv_global_tlb_purge_t ia64_global_tlb_purge; 6extern ia64_mv_global_tlb_purge_t ia64_global_tlb_purge;
7extern ia64_mv_dma_get_required_mask ia64_dma_get_required_mask;
8extern ia64_mv_irq_to_vector __ia64_irq_to_vector; 7extern ia64_mv_irq_to_vector __ia64_irq_to_vector;
9extern ia64_mv_local_vector_to_irq __ia64_local_vector_to_irq; 8extern ia64_mv_local_vector_to_irq __ia64_local_vector_to_irq;
10extern ia64_mv_pci_get_legacy_mem_t ia64_pci_get_legacy_mem; 9extern ia64_mv_pci_get_legacy_mem_t ia64_pci_get_legacy_mem;
diff --git a/arch/ia64/include/asm/machvec_sn2.h b/arch/ia64/include/asm/machvec_sn2.h
index ece9fa85be88..b5153d300289 100644
--- a/arch/ia64/include/asm/machvec_sn2.h
+++ b/arch/ia64/include/asm/machvec_sn2.h
@@ -55,7 +55,6 @@ extern ia64_mv_readb_t __sn_readb_relaxed;
55extern ia64_mv_readw_t __sn_readw_relaxed; 55extern ia64_mv_readw_t __sn_readw_relaxed;
56extern ia64_mv_readl_t __sn_readl_relaxed; 56extern ia64_mv_readl_t __sn_readl_relaxed;
57extern ia64_mv_readq_t __sn_readq_relaxed; 57extern ia64_mv_readq_t __sn_readq_relaxed;
58extern ia64_mv_dma_get_required_mask sn_dma_get_required_mask;
59extern ia64_mv_dma_init sn_dma_init; 58extern ia64_mv_dma_init sn_dma_init;
60extern ia64_mv_migrate_t sn_migrate; 59extern ia64_mv_migrate_t sn_migrate;
61extern ia64_mv_kernel_launch_event_t sn_kernel_launch_event; 60extern ia64_mv_kernel_launch_event_t sn_kernel_launch_event;
@@ -100,7 +99,6 @@ extern ia64_mv_pci_fixup_bus_t sn_pci_fixup_bus;
100#define platform_pci_get_legacy_mem sn_pci_get_legacy_mem 99#define platform_pci_get_legacy_mem sn_pci_get_legacy_mem
101#define platform_pci_legacy_read sn_pci_legacy_read 100#define platform_pci_legacy_read sn_pci_legacy_read
102#define platform_pci_legacy_write sn_pci_legacy_write 101#define platform_pci_legacy_write sn_pci_legacy_write
103#define platform_dma_get_required_mask sn_dma_get_required_mask
104#define platform_dma_init sn_dma_init 102#define platform_dma_init sn_dma_init
105#define platform_migrate sn_migrate 103#define platform_migrate sn_migrate
106#define platform_kernel_launch_event sn_kernel_launch_event 104#define platform_kernel_launch_event sn_kernel_launch_event
diff --git a/arch/ia64/pci/pci.c b/arch/ia64/pci/pci.c
index 7ccc64d5fe3e..5d71800df431 100644
--- a/arch/ia64/pci/pci.c
+++ b/arch/ia64/pci/pci.c
@@ -568,32 +568,6 @@ static void __init set_pci_dfl_cacheline_size(void)
568 pci_dfl_cache_line_size = (1 << cci.pcci_line_size) / 4; 568 pci_dfl_cache_line_size = (1 << cci.pcci_line_size) / 4;
569} 569}
570 570
571u64 ia64_dma_get_required_mask(struct device *dev)
572{
573 u32 low_totalram = ((max_pfn - 1) << PAGE_SHIFT);
574 u32 high_totalram = ((max_pfn - 1) >> (32 - PAGE_SHIFT));
575 u64 mask;
576
577 if (!high_totalram) {
578 /* convert to mask just covering totalram */
579 low_totalram = (1 << (fls(low_totalram) - 1));
580 low_totalram += low_totalram - 1;
581 mask = low_totalram;
582 } else {
583 high_totalram = (1 << (fls(high_totalram) - 1));
584 high_totalram += high_totalram - 1;
585 mask = (((u64)high_totalram) << 32) + 0xffffffff;
586 }
587 return mask;
588}
589EXPORT_SYMBOL_GPL(ia64_dma_get_required_mask);
590
591u64 dma_get_required_mask(struct device *dev)
592{
593 return platform_dma_get_required_mask(dev);
594}
595EXPORT_SYMBOL_GPL(dma_get_required_mask);
596
597static int __init pcibios_init(void) 571static int __init pcibios_init(void)
598{ 572{
599 set_pci_dfl_cacheline_size(); 573 set_pci_dfl_cacheline_size();
diff --git a/arch/ia64/sn/pci/pci_dma.c b/arch/ia64/sn/pci/pci_dma.c
index 74c934a997bb..96eb2567718a 100644
--- a/arch/ia64/sn/pci/pci_dma.c
+++ b/arch/ia64/sn/pci/pci_dma.c
@@ -344,11 +344,10 @@ static int sn_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
344 return 0; 344 return 0;
345} 345}
346 346
347u64 sn_dma_get_required_mask(struct device *dev) 347static u64 sn_dma_get_required_mask(struct device *dev)
348{ 348{
349 return DMA_BIT_MASK(64); 349 return DMA_BIT_MASK(64);
350} 350}
351EXPORT_SYMBOL_GPL(sn_dma_get_required_mask);
352 351
353char *sn_pci_get_legacy_mem(struct pci_bus *bus) 352char *sn_pci_get_legacy_mem(struct pci_bus *bus)
354{ 353{
@@ -473,6 +472,7 @@ static struct dma_map_ops sn_dma_ops = {
473 .sync_sg_for_device = sn_dma_sync_sg_for_device, 472 .sync_sg_for_device = sn_dma_sync_sg_for_device,
474 .mapping_error = sn_dma_mapping_error, 473 .mapping_error = sn_dma_mapping_error,
475 .dma_supported = sn_dma_supported, 474 .dma_supported = sn_dma_supported,
475 .get_required_mask = sn_dma_get_required_mask,
476}; 476};
477 477
478void sn_dma_init(void) 478void sn_dma_init(void)
diff --git a/arch/m68k/Kconfig b/arch/m68k/Kconfig
index 070553791e97..c7b2a8d60a41 100644
--- a/arch/m68k/Kconfig
+++ b/arch/m68k/Kconfig
@@ -26,7 +26,7 @@ config M68K
26 select MODULES_USE_ELF_RELA 26 select MODULES_USE_ELF_RELA
27 select OLD_SIGSUSPEND3 27 select OLD_SIGSUSPEND3
28 select OLD_SIGACTION 28 select OLD_SIGACTION
29 select DMA_NONCOHERENT_OPS if HAS_DMA 29 select DMA_DIRECT_OPS if HAS_DMA
30 select HAVE_MEMBLOCK 30 select HAVE_MEMBLOCK
31 select ARCH_DISCARD_MEMBLOCK 31 select ARCH_DISCARD_MEMBLOCK
32 select NO_BOOTMEM 32 select NO_BOOTMEM
diff --git a/arch/microblaze/Kconfig b/arch/microblaze/Kconfig
index ace5c5bf1836..164a4857737a 100644
--- a/arch/microblaze/Kconfig
+++ b/arch/microblaze/Kconfig
@@ -1,6 +1,7 @@
1config MICROBLAZE 1config MICROBLAZE
2 def_bool y 2 def_bool y
3 select ARCH_NO_SWAP 3 select ARCH_NO_SWAP
4 select ARCH_HAS_DMA_COHERENT_TO_PFN if MMU
4 select ARCH_HAS_GCOV_PROFILE_ALL 5 select ARCH_HAS_GCOV_PROFILE_ALL
5 select ARCH_HAS_SYNC_DMA_FOR_CPU 6 select ARCH_HAS_SYNC_DMA_FOR_CPU
6 select ARCH_HAS_SYNC_DMA_FOR_DEVICE 7 select ARCH_HAS_SYNC_DMA_FOR_DEVICE
@@ -11,8 +12,7 @@ config MICROBLAZE
11 select TIMER_OF 12 select TIMER_OF
12 select CLONE_BACKWARDS3 13 select CLONE_BACKWARDS3
13 select COMMON_CLK 14 select COMMON_CLK
14 select DMA_NONCOHERENT_OPS 15 select DMA_DIRECT_OPS
15 select DMA_NONCOHERENT_MMAP
16 select GENERIC_ATOMIC64 16 select GENERIC_ATOMIC64
17 select GENERIC_CLOCKEVENTS 17 select GENERIC_CLOCKEVENTS
18 select GENERIC_CPU_DEVICES 18 select GENERIC_CPU_DEVICES
diff --git a/arch/microblaze/include/asm/pgtable.h b/arch/microblaze/include/asm/pgtable.h
index 7b650ab14fa0..f64ebb9c9a41 100644
--- a/arch/microblaze/include/asm/pgtable.h
+++ b/arch/microblaze/include/asm/pgtable.h
@@ -553,8 +553,6 @@ void __init *early_get_page(void);
553 553
554extern unsigned long ioremap_bot, ioremap_base; 554extern unsigned long ioremap_bot, ioremap_base;
555 555
556unsigned long consistent_virt_to_pfn(void *vaddr);
557
558void setup_memory(void); 556void setup_memory(void);
559#endif /* __ASSEMBLY__ */ 557#endif /* __ASSEMBLY__ */
560 558
diff --git a/arch/microblaze/kernel/dma.c b/arch/microblaze/kernel/dma.c
index 71032cf64669..a89c2d4ed5ff 100644
--- a/arch/microblaze/kernel/dma.c
+++ b/arch/microblaze/kernel/dma.c
@@ -42,25 +42,3 @@ void arch_sync_dma_for_cpu(struct device *dev, phys_addr_t paddr,
42{ 42{
43 __dma_sync(dev, paddr, size, dir); 43 __dma_sync(dev, paddr, size, dir);
44} 44}
45
46int arch_dma_mmap(struct device *dev, struct vm_area_struct *vma,
47 void *cpu_addr, dma_addr_t handle, size_t size,
48 unsigned long attrs)
49{
50#ifdef CONFIG_MMU
51 unsigned long user_count = vma_pages(vma);
52 unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT;
53 unsigned long off = vma->vm_pgoff;
54 unsigned long pfn;
55
56 if (off >= count || user_count > (count - off))
57 return -ENXIO;
58
59 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
60 pfn = consistent_virt_to_pfn(cpu_addr);
61 return remap_pfn_range(vma, vma->vm_start, pfn + off,
62 vma->vm_end - vma->vm_start, vma->vm_page_prot);
63#else
64 return -ENXIO;
65#endif
66}
diff --git a/arch/microblaze/mm/consistent.c b/arch/microblaze/mm/consistent.c
index c9a278ac795a..d801cc5f5b95 100644
--- a/arch/microblaze/mm/consistent.c
+++ b/arch/microblaze/mm/consistent.c
@@ -165,7 +165,8 @@ static pte_t *consistent_virt_to_pte(void *vaddr)
165 return pte_offset_kernel(pmd_offset(pgd_offset_k(addr), addr), addr); 165 return pte_offset_kernel(pmd_offset(pgd_offset_k(addr), addr), addr);
166} 166}
167 167
168unsigned long consistent_virt_to_pfn(void *vaddr) 168long arch_dma_coherent_to_pfn(struct device *dev, void *vaddr,
169 dma_addr_t dma_addr)
169{ 170{
170 pte_t *ptep = consistent_virt_to_pte(vaddr); 171 pte_t *ptep = consistent_virt_to_pte(vaddr);
171 172
diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig
index 35511999156a..77c022e56e6e 100644
--- a/arch/mips/Kconfig
+++ b/arch/mips/Kconfig
@@ -1106,21 +1106,22 @@ config ARCH_SUPPORTS_UPROBES
1106 bool 1106 bool
1107 1107
1108config DMA_MAYBE_COHERENT 1108config DMA_MAYBE_COHERENT
1109 select ARCH_HAS_DMA_COHERENCE_H
1109 select DMA_NONCOHERENT 1110 select DMA_NONCOHERENT
1110 bool 1111 bool
1111 1112
1112config DMA_PERDEV_COHERENT 1113config DMA_PERDEV_COHERENT
1113 bool 1114 bool
1114 select DMA_MAYBE_COHERENT 1115 select DMA_NONCOHERENT
1115 1116
1116config DMA_NONCOHERENT 1117config DMA_NONCOHERENT
1117 bool 1118 bool
1119 select ARCH_HAS_DMA_MMAP_PGPROT
1118 select ARCH_HAS_SYNC_DMA_FOR_DEVICE 1120 select ARCH_HAS_SYNC_DMA_FOR_DEVICE
1119 select ARCH_HAS_SYNC_DMA_FOR_CPU 1121 select ARCH_HAS_SYNC_DMA_FOR_CPU
1120 select NEED_DMA_MAP_STATE 1122 select NEED_DMA_MAP_STATE
1121 select DMA_NONCOHERENT_MMAP 1123 select ARCH_HAS_DMA_COHERENT_TO_PFN
1122 select DMA_NONCOHERENT_CACHE_SYNC 1124 select DMA_NONCOHERENT_CACHE_SYNC
1123 select DMA_NONCOHERENT_OPS
1124 1125
1125config SYS_HAS_EARLY_PRINTK 1126config SYS_HAS_EARLY_PRINTK
1126 bool 1127 bool
diff --git a/arch/mips/include/asm/Kbuild b/arch/mips/include/asm/Kbuild
index 58351e48421e..9a81e72119da 100644
--- a/arch/mips/include/asm/Kbuild
+++ b/arch/mips/include/asm/Kbuild
@@ -1,6 +1,7 @@
1# MIPS headers 1# MIPS headers
2generic-(CONFIG_GENERIC_CSUM) += checksum.h 2generic-(CONFIG_GENERIC_CSUM) += checksum.h
3generic-y += current.h 3generic-y += current.h
4generic-y += device.h
4generic-y += dma-contiguous.h 5generic-y += dma-contiguous.h
5generic-y += emergency-restart.h 6generic-y += emergency-restart.h
6generic-y += export.h 7generic-y += export.h
diff --git a/arch/mips/include/asm/device.h b/arch/mips/include/asm/device.h
deleted file mode 100644
index 6aa796f1081a..000000000000
--- a/arch/mips/include/asm/device.h
+++ /dev/null
@@ -1,19 +0,0 @@
1/*
2 * Arch specific extensions to struct device
3 *
4 * This file is released under the GPLv2
5 */
6#ifndef _ASM_MIPS_DEVICE_H
7#define _ASM_MIPS_DEVICE_H
8
9struct dev_archdata {
10#ifdef CONFIG_DMA_PERDEV_COHERENT
11 /* Non-zero if DMA is coherent with CPU caches */
12 bool dma_coherent;
13#endif
14};
15
16struct pdev_archdata {
17};
18
19#endif /* _ASM_MIPS_DEVICE_H*/
diff --git a/arch/mips/include/asm/dma-coherence.h b/arch/mips/include/asm/dma-coherence.h
index 8eda48748ed5..5eaa1fcc878a 100644
--- a/arch/mips/include/asm/dma-coherence.h
+++ b/arch/mips/include/asm/dma-coherence.h
@@ -20,6 +20,12 @@ enum coherent_io_user_state {
20#elif defined(CONFIG_DMA_MAYBE_COHERENT) 20#elif defined(CONFIG_DMA_MAYBE_COHERENT)
21extern enum coherent_io_user_state coherentio; 21extern enum coherent_io_user_state coherentio;
22extern int hw_coherentio; 22extern int hw_coherentio;
23
24static inline bool dev_is_dma_coherent(struct device *dev)
25{
26 return coherentio == IO_COHERENCE_ENABLED ||
27 (coherentio == IO_COHERENCE_DEFAULT && hw_coherentio);
28}
23#else 29#else
24#ifdef CONFIG_DMA_NONCOHERENT 30#ifdef CONFIG_DMA_NONCOHERENT
25#define coherentio IO_COHERENCE_DISABLED 31#define coherentio IO_COHERENCE_DISABLED
diff --git a/arch/mips/include/asm/dma-mapping.h b/arch/mips/include/asm/dma-mapping.h
index e81c4e97ff1a..b4c477eb46ce 100644
--- a/arch/mips/include/asm/dma-mapping.h
+++ b/arch/mips/include/asm/dma-mapping.h
@@ -12,8 +12,6 @@ static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus)
12 return &jazz_dma_ops; 12 return &jazz_dma_ops;
13#elif defined(CONFIG_SWIOTLB) 13#elif defined(CONFIG_SWIOTLB)
14 return &swiotlb_dma_ops; 14 return &swiotlb_dma_ops;
15#elif defined(CONFIG_DMA_NONCOHERENT_OPS)
16 return &dma_noncoherent_ops;
17#else 15#else
18 return &dma_direct_ops; 16 return &dma_direct_ops;
19#endif 17#endif
@@ -25,7 +23,7 @@ static inline void arch_setup_dma_ops(struct device *dev, u64 dma_base,
25 bool coherent) 23 bool coherent)
26{ 24{
27#ifdef CONFIG_DMA_PERDEV_COHERENT 25#ifdef CONFIG_DMA_PERDEV_COHERENT
28 dev->archdata.dma_coherent = coherent; 26 dev->dma_coherent = coherent;
29#endif 27#endif
30} 28}
31 29
diff --git a/arch/mips/jazz/jazzdma.c b/arch/mips/jazz/jazzdma.c
index d31bc2f01208..0a0aaf39fd16 100644
--- a/arch/mips/jazz/jazzdma.c
+++ b/arch/mips/jazz/jazzdma.c
@@ -564,13 +564,13 @@ static void *jazz_dma_alloc(struct device *dev, size_t size,
564{ 564{
565 void *ret; 565 void *ret;
566 566
567 ret = dma_direct_alloc(dev, size, dma_handle, gfp, attrs); 567 ret = dma_direct_alloc_pages(dev, size, dma_handle, gfp, attrs);
568 if (!ret) 568 if (!ret)
569 return NULL; 569 return NULL;
570 570
571 *dma_handle = vdma_alloc(virt_to_phys(ret), size); 571 *dma_handle = vdma_alloc(virt_to_phys(ret), size);
572 if (*dma_handle == VDMA_ERROR) { 572 if (*dma_handle == VDMA_ERROR) {
573 dma_direct_free(dev, size, ret, *dma_handle, attrs); 573 dma_direct_free_pages(dev, size, ret, *dma_handle, attrs);
574 return NULL; 574 return NULL;
575 } 575 }
576 576
@@ -587,7 +587,7 @@ static void jazz_dma_free(struct device *dev, size_t size, void *vaddr,
587 vdma_free(dma_handle); 587 vdma_free(dma_handle);
588 if (!(attrs & DMA_ATTR_NON_CONSISTENT)) 588 if (!(attrs & DMA_ATTR_NON_CONSISTENT))
589 vaddr = (void *)CAC_ADDR((unsigned long)vaddr); 589 vaddr = (void *)CAC_ADDR((unsigned long)vaddr);
590 return dma_direct_free(dev, size, vaddr, dma_handle, attrs); 590 dma_direct_free_pages(dev, size, vaddr, dma_handle, attrs);
591} 591}
592 592
593static dma_addr_t jazz_dma_map_page(struct device *dev, struct page *page, 593static dma_addr_t jazz_dma_map_page(struct device *dev, struct page *page,
@@ -682,7 +682,6 @@ static int jazz_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
682const struct dma_map_ops jazz_dma_ops = { 682const struct dma_map_ops jazz_dma_ops = {
683 .alloc = jazz_dma_alloc, 683 .alloc = jazz_dma_alloc,
684 .free = jazz_dma_free, 684 .free = jazz_dma_free,
685 .mmap = arch_dma_mmap,
686 .map_page = jazz_dma_map_page, 685 .map_page = jazz_dma_map_page,
687 .unmap_page = jazz_dma_unmap_page, 686 .unmap_page = jazz_dma_unmap_page,
688 .map_sg = jazz_dma_map_sg, 687 .map_sg = jazz_dma_map_sg,
diff --git a/arch/mips/kernel/setup.c b/arch/mips/kernel/setup.c
index 8aaaa42f91ed..e64b9e8bb002 100644
--- a/arch/mips/kernel/setup.c
+++ b/arch/mips/kernel/setup.c
@@ -1075,7 +1075,7 @@ static int __init debugfs_mips(void)
1075arch_initcall(debugfs_mips); 1075arch_initcall(debugfs_mips);
1076#endif 1076#endif
1077 1077
1078#if defined(CONFIG_DMA_MAYBE_COHERENT) && !defined(CONFIG_DMA_PERDEV_COHERENT) 1078#ifdef CONFIG_DMA_MAYBE_COHERENT
1079/* User defined DMA coherency from command line. */ 1079/* User defined DMA coherency from command line. */
1080enum coherent_io_user_state coherentio = IO_COHERENCE_DEFAULT; 1080enum coherent_io_user_state coherentio = IO_COHERENCE_DEFAULT;
1081EXPORT_SYMBOL_GPL(coherentio); 1081EXPORT_SYMBOL_GPL(coherentio);
diff --git a/arch/mips/mm/c-r4k.c b/arch/mips/mm/c-r4k.c
index a9ef057c79fe..05bd77727fb9 100644
--- a/arch/mips/mm/c-r4k.c
+++ b/arch/mips/mm/c-r4k.c
@@ -1955,22 +1955,21 @@ void r4k_cache_init(void)
1955 __flush_icache_user_range = r4k_flush_icache_user_range; 1955 __flush_icache_user_range = r4k_flush_icache_user_range;
1956 __local_flush_icache_user_range = local_r4k_flush_icache_user_range; 1956 __local_flush_icache_user_range = local_r4k_flush_icache_user_range;
1957 1957
1958#if defined(CONFIG_DMA_NONCOHERENT) || defined(CONFIG_DMA_MAYBE_COHERENT) 1958#ifdef CONFIG_DMA_NONCOHERENT
1959# if defined(CONFIG_DMA_PERDEV_COHERENT) 1959#ifdef CONFIG_DMA_MAYBE_COHERENT
1960 if (0) { 1960 if (coherentio == IO_COHERENCE_ENABLED ||
1961# else 1961 (coherentio == IO_COHERENCE_DEFAULT && hw_coherentio)) {
1962 if ((coherentio == IO_COHERENCE_ENABLED) ||
1963 ((coherentio == IO_COHERENCE_DEFAULT) && hw_coherentio)) {
1964# endif
1965 _dma_cache_wback_inv = (void *)cache_noop; 1962 _dma_cache_wback_inv = (void *)cache_noop;
1966 _dma_cache_wback = (void *)cache_noop; 1963 _dma_cache_wback = (void *)cache_noop;
1967 _dma_cache_inv = (void *)cache_noop; 1964 _dma_cache_inv = (void *)cache_noop;
1968 } else { 1965 } else
1966#endif /* CONFIG_DMA_MAYBE_COHERENT */
1967 {
1969 _dma_cache_wback_inv = r4k_dma_cache_wback_inv; 1968 _dma_cache_wback_inv = r4k_dma_cache_wback_inv;
1970 _dma_cache_wback = r4k_dma_cache_wback_inv; 1969 _dma_cache_wback = r4k_dma_cache_wback_inv;
1971 _dma_cache_inv = r4k_dma_cache_inv; 1970 _dma_cache_inv = r4k_dma_cache_inv;
1972 } 1971 }
1973#endif 1972#endif /* CONFIG_DMA_NONCOHERENT */
1974 1973
1975 build_clear_page(); 1974 build_clear_page();
1976 build_copy_page(); 1975 build_copy_page();
diff --git a/arch/mips/mm/dma-noncoherent.c b/arch/mips/mm/dma-noncoherent.c
index 2aca1236af36..e6c9485cadcf 100644
--- a/arch/mips/mm/dma-noncoherent.c
+++ b/arch/mips/mm/dma-noncoherent.c
@@ -14,26 +14,6 @@
14#include <asm/dma-coherence.h> 14#include <asm/dma-coherence.h>
15#include <asm/io.h> 15#include <asm/io.h>
16 16
17#ifdef CONFIG_DMA_PERDEV_COHERENT
18static inline int dev_is_coherent(struct device *dev)
19{
20 return dev->archdata.dma_coherent;
21}
22#else
23static inline int dev_is_coherent(struct device *dev)
24{
25 switch (coherentio) {
26 default:
27 case IO_COHERENCE_DEFAULT:
28 return hw_coherentio;
29 case IO_COHERENCE_ENABLED:
30 return 1;
31 case IO_COHERENCE_DISABLED:
32 return 0;
33 }
34}
35#endif /* CONFIG_DMA_PERDEV_COHERENT */
36
37/* 17/*
38 * The affected CPUs below in 'cpu_needs_post_dma_flush()' can speculatively 18 * The affected CPUs below in 'cpu_needs_post_dma_flush()' can speculatively
39 * fill random cachelines with stale data at any time, requiring an extra 19 * fill random cachelines with stale data at any time, requiring an extra
@@ -49,9 +29,6 @@ static inline int dev_is_coherent(struct device *dev)
49 */ 29 */
50static inline bool cpu_needs_post_dma_flush(struct device *dev) 30static inline bool cpu_needs_post_dma_flush(struct device *dev)
51{ 31{
52 if (dev_is_coherent(dev))
53 return false;
54
55 switch (boot_cpu_type()) { 32 switch (boot_cpu_type()) {
56 case CPU_R10000: 33 case CPU_R10000:
57 case CPU_R12000: 34 case CPU_R12000:
@@ -72,11 +49,8 @@ void *arch_dma_alloc(struct device *dev, size_t size,
72{ 49{
73 void *ret; 50 void *ret;
74 51
75 ret = dma_direct_alloc(dev, size, dma_handle, gfp, attrs); 52 ret = dma_direct_alloc_pages(dev, size, dma_handle, gfp, attrs);
76 if (!ret) 53 if (!ret && !(attrs & DMA_ATTR_NON_CONSISTENT)) {
77 return NULL;
78
79 if (!dev_is_coherent(dev) && !(attrs & DMA_ATTR_NON_CONSISTENT)) {
80 dma_cache_wback_inv((unsigned long) ret, size); 54 dma_cache_wback_inv((unsigned long) ret, size);
81 ret = (void *)UNCAC_ADDR(ret); 55 ret = (void *)UNCAC_ADDR(ret);
82 } 56 }
@@ -87,43 +61,24 @@ void *arch_dma_alloc(struct device *dev, size_t size,
87void arch_dma_free(struct device *dev, size_t size, void *cpu_addr, 61void arch_dma_free(struct device *dev, size_t size, void *cpu_addr,
88 dma_addr_t dma_addr, unsigned long attrs) 62 dma_addr_t dma_addr, unsigned long attrs)
89{ 63{
90 if (!(attrs & DMA_ATTR_NON_CONSISTENT) && !dev_is_coherent(dev)) 64 if (!(attrs & DMA_ATTR_NON_CONSISTENT))
91 cpu_addr = (void *)CAC_ADDR((unsigned long)cpu_addr); 65 cpu_addr = (void *)CAC_ADDR((unsigned long)cpu_addr);
92 dma_direct_free(dev, size, cpu_addr, dma_addr, attrs); 66 dma_direct_free_pages(dev, size, cpu_addr, dma_addr, attrs);
93} 67}
94 68
95int arch_dma_mmap(struct device *dev, struct vm_area_struct *vma, 69long arch_dma_coherent_to_pfn(struct device *dev, void *cpu_addr,
96 void *cpu_addr, dma_addr_t dma_addr, size_t size, 70 dma_addr_t dma_addr)
97 unsigned long attrs)
98{ 71{
99 unsigned long user_count = vma_pages(vma); 72 unsigned long addr = CAC_ADDR((unsigned long)cpu_addr);
100 unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT; 73 return page_to_pfn(virt_to_page((void *)addr));
101 unsigned long addr = (unsigned long)cpu_addr; 74}
102 unsigned long off = vma->vm_pgoff;
103 unsigned long pfn;
104 int ret = -ENXIO;
105
106 if (!dev_is_coherent(dev))
107 addr = CAC_ADDR(addr);
108
109 pfn = page_to_pfn(virt_to_page((void *)addr));
110 75
76pgprot_t arch_dma_mmap_pgprot(struct device *dev, pgprot_t prot,
77 unsigned long attrs)
78{
111 if (attrs & DMA_ATTR_WRITE_COMBINE) 79 if (attrs & DMA_ATTR_WRITE_COMBINE)
112 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); 80 return pgprot_writecombine(prot);
113 else 81 return pgprot_noncached(prot);
114 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
115
116 if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret))
117 return ret;
118
119 if (off < count && user_count <= (count - off)) {
120 ret = remap_pfn_range(vma, vma->vm_start,
121 pfn + off,
122 user_count << PAGE_SHIFT,
123 vma->vm_page_prot);
124 }
125
126 return ret;
127} 82}
128 83
129static inline void dma_sync_virt(void *addr, size_t size, 84static inline void dma_sync_virt(void *addr, size_t size,
@@ -187,8 +142,7 @@ static inline void dma_sync_phys(phys_addr_t paddr, size_t size,
187void arch_sync_dma_for_device(struct device *dev, phys_addr_t paddr, 142void arch_sync_dma_for_device(struct device *dev, phys_addr_t paddr,
188 size_t size, enum dma_data_direction dir) 143 size_t size, enum dma_data_direction dir)
189{ 144{
190 if (!dev_is_coherent(dev)) 145 dma_sync_phys(paddr, size, dir);
191 dma_sync_phys(paddr, size, dir);
192} 146}
193 147
194void arch_sync_dma_for_cpu(struct device *dev, phys_addr_t paddr, 148void arch_sync_dma_for_cpu(struct device *dev, phys_addr_t paddr,
@@ -203,6 +157,5 @@ void arch_dma_cache_sync(struct device *dev, void *vaddr, size_t size,
203{ 157{
204 BUG_ON(direction == DMA_NONE); 158 BUG_ON(direction == DMA_NONE);
205 159
206 if (!dev_is_coherent(dev)) 160 dma_sync_virt(vaddr, size, direction);
207 dma_sync_virt(vaddr, size, direction);
208} 161}
diff --git a/arch/nds32/Kconfig b/arch/nds32/Kconfig
index 7068f341133d..56992330026a 100644
--- a/arch/nds32/Kconfig
+++ b/arch/nds32/Kconfig
@@ -11,7 +11,7 @@ config NDS32
11 select CLKSRC_MMIO 11 select CLKSRC_MMIO
12 select CLONE_BACKWARDS 12 select CLONE_BACKWARDS
13 select COMMON_CLK 13 select COMMON_CLK
14 select DMA_NONCOHERENT_OPS 14 select DMA_DIRECT_OPS
15 select GENERIC_ATOMIC64 15 select GENERIC_ATOMIC64
16 select GENERIC_CPU_DEVICES 16 select GENERIC_CPU_DEVICES
17 select GENERIC_CLOCKEVENTS 17 select GENERIC_CLOCKEVENTS
diff --git a/arch/nios2/Kconfig b/arch/nios2/Kconfig
index f4ad1138e6b9..03965692fbfe 100644
--- a/arch/nios2/Kconfig
+++ b/arch/nios2/Kconfig
@@ -4,7 +4,7 @@ config NIOS2
4 select ARCH_HAS_SYNC_DMA_FOR_CPU 4 select ARCH_HAS_SYNC_DMA_FOR_CPU
5 select ARCH_HAS_SYNC_DMA_FOR_DEVICE 5 select ARCH_HAS_SYNC_DMA_FOR_DEVICE
6 select ARCH_NO_SWAP 6 select ARCH_NO_SWAP
7 select DMA_NONCOHERENT_OPS 7 select DMA_DIRECT_OPS
8 select TIMER_OF 8 select TIMER_OF
9 select GENERIC_ATOMIC64 9 select GENERIC_ATOMIC64
10 select GENERIC_CLOCKEVENTS 10 select GENERIC_CLOCKEVENTS
diff --git a/arch/openrisc/Kconfig b/arch/openrisc/Kconfig
index e0081e734827..a655ae280637 100644
--- a/arch/openrisc/Kconfig
+++ b/arch/openrisc/Kconfig
@@ -7,7 +7,7 @@
7config OPENRISC 7config OPENRISC
8 def_bool y 8 def_bool y
9 select ARCH_HAS_SYNC_DMA_FOR_DEVICE 9 select ARCH_HAS_SYNC_DMA_FOR_DEVICE
10 select DMA_NONCOHERENT_OPS 10 select DMA_DIRECT_OPS
11 select OF 11 select OF
12 select OF_EARLY_FLATTREE 12 select OF_EARLY_FLATTREE
13 select IRQ_DOMAIN 13 select IRQ_DOMAIN
diff --git a/arch/parisc/Kconfig b/arch/parisc/Kconfig
index 8e6d83f79e72..f1cd12afd943 100644
--- a/arch/parisc/Kconfig
+++ b/arch/parisc/Kconfig
@@ -186,7 +186,7 @@ config PA11
186 depends on PA7000 || PA7100LC || PA7200 || PA7300LC 186 depends on PA7000 || PA7100LC || PA7200 || PA7300LC
187 select ARCH_HAS_SYNC_DMA_FOR_CPU 187 select ARCH_HAS_SYNC_DMA_FOR_CPU
188 select ARCH_HAS_SYNC_DMA_FOR_DEVICE 188 select ARCH_HAS_SYNC_DMA_FOR_DEVICE
189 select DMA_NONCOHERENT_OPS 189 select DMA_DIRECT_OPS
190 select DMA_NONCOHERENT_CACHE_SYNC 190 select DMA_NONCOHERENT_CACHE_SYNC
191 191
192config PREFETCH 192config PREFETCH
diff --git a/arch/parisc/kernel/setup.c b/arch/parisc/kernel/setup.c
index 4e87c35c22b7..755e89ec828a 100644
--- a/arch/parisc/kernel/setup.c
+++ b/arch/parisc/kernel/setup.c
@@ -102,7 +102,7 @@ void __init dma_ops_init(void)
102 case pcxl: /* falls through */ 102 case pcxl: /* falls through */
103 case pcxs: 103 case pcxs:
104 case pcxt: 104 case pcxt:
105 hppa_dma_ops = &dma_noncoherent_ops; 105 hppa_dma_ops = &dma_direct_ops;
106 break; 106 break;
107 default: 107 default:
108 break; 108 break;
diff --git a/arch/sh/Kconfig b/arch/sh/Kconfig
index 1fb7b6d72baf..475d786a65b0 100644
--- a/arch/sh/Kconfig
+++ b/arch/sh/Kconfig
@@ -7,6 +7,7 @@ config SUPERH
7 select ARCH_NO_COHERENT_DMA_MMAP if !MMU 7 select ARCH_NO_COHERENT_DMA_MMAP if !MMU
8 select HAVE_PATA_PLATFORM 8 select HAVE_PATA_PLATFORM
9 select CLKDEV_LOOKUP 9 select CLKDEV_LOOKUP
10 select DMA_DIRECT_OPS
10 select HAVE_IDE if HAS_IOPORT_MAP 11 select HAVE_IDE if HAS_IOPORT_MAP
11 select HAVE_MEMBLOCK 12 select HAVE_MEMBLOCK
12 select HAVE_MEMBLOCK_NODE_MAP 13 select HAVE_MEMBLOCK_NODE_MAP
@@ -158,13 +159,11 @@ config SWAP_IO_SPACE
158 bool 159 bool
159 160
160config DMA_COHERENT 161config DMA_COHERENT
161 select DMA_DIRECT_OPS
162 bool 162 bool
163 163
164config DMA_NONCOHERENT 164config DMA_NONCOHERENT
165 def_bool !DMA_COHERENT 165 def_bool !DMA_COHERENT
166 select ARCH_HAS_SYNC_DMA_FOR_DEVICE 166 select ARCH_HAS_SYNC_DMA_FOR_DEVICE
167 select DMA_NONCOHERENT_OPS
168 167
169config PGTABLE_LEVELS 168config PGTABLE_LEVELS
170 default 3 if X2TLB 169 default 3 if X2TLB
diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig
index e6f2a38d2e61..7e2aa59fcc29 100644
--- a/arch/sparc/Kconfig
+++ b/arch/sparc/Kconfig
@@ -51,7 +51,7 @@ config SPARC
51config SPARC32 51config SPARC32
52 def_bool !64BIT 52 def_bool !64BIT
53 select ARCH_HAS_SYNC_DMA_FOR_CPU 53 select ARCH_HAS_SYNC_DMA_FOR_CPU
54 select DMA_NONCOHERENT_OPS 54 select DMA_DIRECT_OPS
55 select GENERIC_ATOMIC64 55 select GENERIC_ATOMIC64
56 select CLZ_TAB 56 select CLZ_TAB
57 select HAVE_UID16 57 select HAVE_UID16
diff --git a/arch/sparc/include/asm/dma-mapping.h b/arch/sparc/include/asm/dma-mapping.h
index e17566376934..b0bb2fcaf1c9 100644
--- a/arch/sparc/include/asm/dma-mapping.h
+++ b/arch/sparc/include/asm/dma-mapping.h
@@ -14,11 +14,11 @@ static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus)
14{ 14{
15#ifdef CONFIG_SPARC_LEON 15#ifdef CONFIG_SPARC_LEON
16 if (sparc_cpu_model == sparc_leon) 16 if (sparc_cpu_model == sparc_leon)
17 return &dma_noncoherent_ops; 17 return &dma_direct_ops;
18#endif 18#endif
19#if defined(CONFIG_SPARC32) && defined(CONFIG_PCI) 19#if defined(CONFIG_SPARC32) && defined(CONFIG_PCI)
20 if (bus == &pci_bus_type) 20 if (bus == &pci_bus_type)
21 return &dma_noncoherent_ops; 21 return &dma_direct_ops;
22#endif 22#endif
23 return dma_ops; 23 return dma_ops;
24} 24}
diff --git a/arch/unicore32/Kconfig b/arch/unicore32/Kconfig
index 60eae744d8fd..3a3b40f79558 100644
--- a/arch/unicore32/Kconfig
+++ b/arch/unicore32/Kconfig
@@ -4,6 +4,7 @@ config UNICORE32
4 select ARCH_HAS_DEVMEM_IS_ALLOWED 4 select ARCH_HAS_DEVMEM_IS_ALLOWED
5 select ARCH_MIGHT_HAVE_PC_PARPORT 5 select ARCH_MIGHT_HAVE_PC_PARPORT
6 select ARCH_MIGHT_HAVE_PC_SERIO 6 select ARCH_MIGHT_HAVE_PC_SERIO
7 select DMA_DIRECT_OPS
7 select HAVE_MEMBLOCK 8 select HAVE_MEMBLOCK
8 select HAVE_GENERIC_DMA_COHERENT 9 select HAVE_GENERIC_DMA_COHERENT
9 select HAVE_KERNEL_GZIP 10 select HAVE_KERNEL_GZIP
@@ -20,7 +21,6 @@ config UNICORE32
20 select GENERIC_IOMAP 21 select GENERIC_IOMAP
21 select MODULES_USE_ELF_REL 22 select MODULES_USE_ELF_REL
22 select NEED_DMA_MAP_STATE 23 select NEED_DMA_MAP_STATE
23 select SWIOTLB
24 help 24 help
25 UniCore-32 is 32-bit Instruction Set Architecture, 25 UniCore-32 is 32-bit Instruction Set Architecture,
26 including a series of low-power-consumption RISC chip 26 including a series of low-power-consumption RISC chip
diff --git a/arch/unicore32/include/asm/Kbuild b/arch/unicore32/include/asm/Kbuild
index bfc7abe77905..1372553dc0a9 100644
--- a/arch/unicore32/include/asm/Kbuild
+++ b/arch/unicore32/include/asm/Kbuild
@@ -4,6 +4,7 @@ generic-y += compat.h
4generic-y += current.h 4generic-y += current.h
5generic-y += device.h 5generic-y += device.h
6generic-y += div64.h 6generic-y += div64.h
7generic-y += dma-mapping.h
7generic-y += emergency-restart.h 8generic-y += emergency-restart.h
8generic-y += exec.h 9generic-y += exec.h
9generic-y += extable.h 10generic-y += extable.h
diff --git a/arch/unicore32/include/asm/dma-mapping.h b/arch/unicore32/include/asm/dma-mapping.h
deleted file mode 100644
index 790bc2ef4af2..000000000000
--- a/arch/unicore32/include/asm/dma-mapping.h
+++ /dev/null
@@ -1,22 +0,0 @@
1/*
2 * linux/arch/unicore32/include/asm/dma-mapping.h
3 *
4 * Code specific to PKUnity SoC and UniCore ISA
5 *
6 * Copyright (C) 2001-2010 GUAN Xue-tao
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
11 */
12#ifndef __UNICORE_DMA_MAPPING_H__
13#define __UNICORE_DMA_MAPPING_H__
14
15#include <linux/swiotlb.h>
16
17static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus)
18{
19 return &swiotlb_dma_ops;
20}
21
22#endif
diff --git a/arch/unicore32/mm/init.c b/arch/unicore32/mm/init.c
index f4950fbfe574..5f72a8d1d953 100644
--- a/arch/unicore32/mm/init.c
+++ b/arch/unicore32/mm/init.c
@@ -234,9 +234,6 @@ void __init bootmem_init(void)
234 234
235 uc32_bootmem_init(min, max_low); 235 uc32_bootmem_init(min, max_low);
236 236
237#ifdef CONFIG_SWIOTLB
238 swiotlb_init(1);
239#endif
240 /* 237 /*
241 * Sparsemem tries to allocate bootmem in memory_present(), 238 * Sparsemem tries to allocate bootmem in memory_present(),
242 * so must be done after the fixed reservations 239 * so must be done after the fixed reservations
diff --git a/arch/x86/kernel/amd_gart_64.c b/arch/x86/kernel/amd_gart_64.c
index f299d8a479bb..3f9d1b4019bb 100644
--- a/arch/x86/kernel/amd_gart_64.c
+++ b/arch/x86/kernel/amd_gart_64.c
@@ -482,7 +482,7 @@ gart_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_addr,
482{ 482{
483 void *vaddr; 483 void *vaddr;
484 484
485 vaddr = dma_direct_alloc(dev, size, dma_addr, flag, attrs); 485 vaddr = dma_direct_alloc_pages(dev, size, dma_addr, flag, attrs);
486 if (!vaddr || 486 if (!vaddr ||
487 !force_iommu || dev->coherent_dma_mask <= DMA_BIT_MASK(24)) 487 !force_iommu || dev->coherent_dma_mask <= DMA_BIT_MASK(24))
488 return vaddr; 488 return vaddr;
@@ -494,7 +494,7 @@ gart_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_addr,
494 goto out_free; 494 goto out_free;
495 return vaddr; 495 return vaddr;
496out_free: 496out_free:
497 dma_direct_free(dev, size, vaddr, *dma_addr, attrs); 497 dma_direct_free_pages(dev, size, vaddr, *dma_addr, attrs);
498 return NULL; 498 return NULL;
499} 499}
500 500
@@ -504,7 +504,7 @@ gart_free_coherent(struct device *dev, size_t size, void *vaddr,
504 dma_addr_t dma_addr, unsigned long attrs) 504 dma_addr_t dma_addr, unsigned long attrs)
505{ 505{
506 gart_unmap_page(dev, dma_addr, size, DMA_BIDIRECTIONAL, 0); 506 gart_unmap_page(dev, dma_addr, size, DMA_BIDIRECTIONAL, 0);
507 dma_direct_free(dev, size, vaddr, dma_addr, attrs); 507 dma_direct_free_pages(dev, size, vaddr, dma_addr, attrs);
508} 508}
509 509
510static int gart_mapping_error(struct device *dev, dma_addr_t dma_addr) 510static int gart_mapping_error(struct device *dev, dma_addr_t dma_addr)
diff --git a/arch/xtensa/Kconfig b/arch/xtensa/Kconfig
index b9ad83a0ee5d..ea5d8d03e53b 100644
--- a/arch/xtensa/Kconfig
+++ b/arch/xtensa/Kconfig
@@ -13,7 +13,7 @@ config XTENSA
13 select BUILDTIME_EXTABLE_SORT 13 select BUILDTIME_EXTABLE_SORT
14 select CLONE_BACKWARDS 14 select CLONE_BACKWARDS
15 select COMMON_CLK 15 select COMMON_CLK
16 select DMA_NONCOHERENT_OPS 16 select DMA_DIRECT_OPS
17 select GENERIC_ATOMIC64 17 select GENERIC_ATOMIC64
18 select GENERIC_CLOCKEVENTS 18 select GENERIC_CLOCKEVENTS
19 select GENERIC_IRQ_SHOW 19 select GENERIC_IRQ_SHOW
diff --git a/drivers/acpi/arm64/iort.c b/drivers/acpi/arm64/iort.c
index 08f26db2da7e..2a361e22d38d 100644
--- a/drivers/acpi/arm64/iort.c
+++ b/drivers/acpi/arm64/iort.c
@@ -1428,7 +1428,7 @@ static int __init iort_add_platform_device(struct acpi_iort_node *node,
1428 return 0; 1428 return 0;
1429 1429
1430dma_deconfigure: 1430dma_deconfigure:
1431 acpi_dma_deconfigure(&pdev->dev); 1431 arch_teardown_dma_ops(&pdev->dev);
1432dev_put: 1432dev_put:
1433 platform_device_put(pdev); 1433 platform_device_put(pdev);
1434 1434
diff --git a/drivers/acpi/scan.c b/drivers/acpi/scan.c
index e1b6231cfa1c..56676a56b3e3 100644
--- a/drivers/acpi/scan.c
+++ b/drivers/acpi/scan.c
@@ -1469,16 +1469,6 @@ int acpi_dma_configure(struct device *dev, enum dev_dma_attr attr)
1469} 1469}
1470EXPORT_SYMBOL_GPL(acpi_dma_configure); 1470EXPORT_SYMBOL_GPL(acpi_dma_configure);
1471 1471
1472/**
1473 * acpi_dma_deconfigure - Tear-down DMA configuration for the device.
1474 * @dev: The pointer to the device
1475 */
1476void acpi_dma_deconfigure(struct device *dev)
1477{
1478 arch_teardown_dma_ops(dev);
1479}
1480EXPORT_SYMBOL_GPL(acpi_dma_deconfigure);
1481
1482static void acpi_init_coherency(struct acpi_device *adev) 1472static void acpi_init_coherency(struct acpi_device *adev)
1483{ 1473{
1484 unsigned long long cca = 0; 1474 unsigned long long cca = 0;
diff --git a/drivers/base/dd.c b/drivers/base/dd.c
index edfc9f0b1180..169412ee4ae8 100644
--- a/drivers/base/dd.c
+++ b/drivers/base/dd.c
@@ -480,9 +480,11 @@ re_probe:
480 if (ret) 480 if (ret)
481 goto pinctrl_bind_failed; 481 goto pinctrl_bind_failed;
482 482
483 ret = dma_configure(dev); 483 if (dev->bus->dma_configure) {
484 if (ret) 484 ret = dev->bus->dma_configure(dev);
485 goto dma_failed; 485 if (ret)
486 goto dma_failed;
487 }
486 488
487 if (driver_sysfs_add(dev)) { 489 if (driver_sysfs_add(dev)) {
488 printk(KERN_ERR "%s: driver_sysfs_add(%s) failed\n", 490 printk(KERN_ERR "%s: driver_sysfs_add(%s) failed\n",
@@ -537,7 +539,7 @@ re_probe:
537 goto done; 539 goto done;
538 540
539probe_failed: 541probe_failed:
540 dma_deconfigure(dev); 542 arch_teardown_dma_ops(dev);
541dma_failed: 543dma_failed:
542 if (dev->bus) 544 if (dev->bus)
543 blocking_notifier_call_chain(&dev->bus->p->bus_notifier, 545 blocking_notifier_call_chain(&dev->bus->p->bus_notifier,
@@ -966,7 +968,7 @@ static void __device_release_driver(struct device *dev, struct device *parent)
966 drv->remove(dev); 968 drv->remove(dev);
967 969
968 device_links_driver_cleanup(dev); 970 device_links_driver_cleanup(dev);
969 dma_deconfigure(dev); 971 arch_teardown_dma_ops(dev);
970 972
971 devres_release_all(dev); 973 devres_release_all(dev);
972 dev->driver = NULL; 974 dev->driver = NULL;
diff --git a/drivers/base/platform.c b/drivers/base/platform.c
index dff82a3c2caa..23cf4427f425 100644
--- a/drivers/base/platform.c
+++ b/drivers/base/platform.c
@@ -1180,7 +1180,7 @@ int __init platform_bus_init(void)
1180} 1180}
1181 1181
1182#ifndef ARCH_HAS_DMA_GET_REQUIRED_MASK 1182#ifndef ARCH_HAS_DMA_GET_REQUIRED_MASK
1183u64 dma_get_required_mask(struct device *dev) 1183static u64 dma_default_get_required_mask(struct device *dev)
1184{ 1184{
1185 u32 low_totalram = ((max_pfn - 1) << PAGE_SHIFT); 1185 u32 low_totalram = ((max_pfn - 1) << PAGE_SHIFT);
1186 u32 high_totalram = ((max_pfn - 1) >> (32 - PAGE_SHIFT)); 1186 u32 high_totalram = ((max_pfn - 1) >> (32 - PAGE_SHIFT));
@@ -1198,6 +1198,15 @@ u64 dma_get_required_mask(struct device *dev)
1198 } 1198 }
1199 return mask; 1199 return mask;
1200} 1200}
1201
1202u64 dma_get_required_mask(struct device *dev)
1203{
1204 const struct dma_map_ops *ops = get_dma_ops(dev);
1205
1206 if (ops->get_required_mask)
1207 return ops->get_required_mask(dev);
1208 return dma_default_get_required_mask(dev);
1209}
1201EXPORT_SYMBOL_GPL(dma_get_required_mask); 1210EXPORT_SYMBOL_GPL(dma_get_required_mask);
1202#endif 1211#endif
1203 1212
diff --git a/drivers/of/device.c b/drivers/of/device.c
index 5957cd4fa262..c7fa5a9697c9 100644
--- a/drivers/of/device.c
+++ b/drivers/of/device.c
@@ -170,18 +170,6 @@ int of_dma_configure(struct device *dev, struct device_node *np, bool force_dma)
170} 170}
171EXPORT_SYMBOL_GPL(of_dma_configure); 171EXPORT_SYMBOL_GPL(of_dma_configure);
172 172
173/**
174 * of_dma_deconfigure - Clean up DMA configuration
175 * @dev: Device for which to clean up DMA configuration
176 *
177 * Clean up all configuration performed by of_dma_configure_ops() and free all
178 * resources that have been allocated.
179 */
180void of_dma_deconfigure(struct device *dev)
181{
182 arch_teardown_dma_ops(dev);
183}
184
185int of_device_register(struct platform_device *pdev) 173int of_device_register(struct platform_device *pdev)
186{ 174{
187 device_initialize(&pdev->dev); 175 device_initialize(&pdev->dev);
diff --git a/drivers/pci/controller/vmd.c b/drivers/pci/controller/vmd.c
index fd2dbd7eed7b..f31ed62d518c 100644
--- a/drivers/pci/controller/vmd.c
+++ b/drivers/pci/controller/vmd.c
@@ -404,12 +404,10 @@ static int vmd_dma_supported(struct device *dev, u64 mask)
404 return vmd_dma_ops(dev)->dma_supported(to_vmd_dev(dev), mask); 404 return vmd_dma_ops(dev)->dma_supported(to_vmd_dev(dev), mask);
405} 405}
406 406
407#ifdef ARCH_HAS_DMA_GET_REQUIRED_MASK
408static u64 vmd_get_required_mask(struct device *dev) 407static u64 vmd_get_required_mask(struct device *dev)
409{ 408{
410 return vmd_dma_ops(dev)->get_required_mask(to_vmd_dev(dev)); 409 return vmd_dma_ops(dev)->get_required_mask(to_vmd_dev(dev));
411} 410}
412#endif
413 411
414static void vmd_teardown_dma_ops(struct vmd_dev *vmd) 412static void vmd_teardown_dma_ops(struct vmd_dev *vmd)
415{ 413{
@@ -450,9 +448,7 @@ static void vmd_setup_dma_ops(struct vmd_dev *vmd)
450 ASSIGN_VMD_DMA_OPS(source, dest, sync_sg_for_device); 448 ASSIGN_VMD_DMA_OPS(source, dest, sync_sg_for_device);
451 ASSIGN_VMD_DMA_OPS(source, dest, mapping_error); 449 ASSIGN_VMD_DMA_OPS(source, dest, mapping_error);
452 ASSIGN_VMD_DMA_OPS(source, dest, dma_supported); 450 ASSIGN_VMD_DMA_OPS(source, dest, dma_supported);
453#ifdef ARCH_HAS_DMA_GET_REQUIRED_MASK
454 ASSIGN_VMD_DMA_OPS(source, dest, get_required_mask); 451 ASSIGN_VMD_DMA_OPS(source, dest, get_required_mask);
455#endif
456 add_dma_domain(domain); 452 add_dma_domain(domain);
457} 453}
458#undef ASSIGN_VMD_DMA_OPS 454#undef ASSIGN_VMD_DMA_OPS
diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c
index a6f9ba85dc4b..28819a0e61d0 100644
--- a/drivers/xen/swiotlb-xen.c
+++ b/drivers/xen/swiotlb-xen.c
@@ -662,7 +662,7 @@ xen_swiotlb_dma_mmap(struct device *dev, struct vm_area_struct *vma,
662 return xen_get_dma_ops(dev)->mmap(dev, vma, cpu_addr, 662 return xen_get_dma_ops(dev)->mmap(dev, vma, cpu_addr,
663 dma_addr, size, attrs); 663 dma_addr, size, attrs);
664#endif 664#endif
665 return dma_common_mmap(dev, vma, cpu_addr, dma_addr, size); 665 return dma_common_mmap(dev, vma, cpu_addr, dma_addr, size, attrs);
666} 666}
667 667
668/* 668/*
@@ -689,7 +689,7 @@ xen_swiotlb_get_sgtable(struct device *dev, struct sg_table *sgt,
689 handle, size, attrs); 689 handle, size, attrs);
690 } 690 }
691#endif 691#endif
692 return dma_common_get_sgtable(dev, sgt, cpu_addr, handle, size); 692 return dma_common_get_sgtable(dev, sgt, cpu_addr, handle, size, attrs);
693} 693}
694 694
695static int xen_swiotlb_mapping_error(struct device *dev, dma_addr_t dma_addr) 695static int xen_swiotlb_mapping_error(struct device *dev, dma_addr_t dma_addr)
diff --git a/include/acpi/acpi_bus.h b/include/acpi/acpi_bus.h
index ba4dd54f2c82..53600f527a70 100644
--- a/include/acpi/acpi_bus.h
+++ b/include/acpi/acpi_bus.h
@@ -595,7 +595,6 @@ enum dev_dma_attr acpi_get_dma_attr(struct acpi_device *adev);
595int acpi_dma_get_range(struct device *dev, u64 *dma_addr, u64 *offset, 595int acpi_dma_get_range(struct device *dev, u64 *dma_addr, u64 *offset,
596 u64 *size); 596 u64 *size);
597int acpi_dma_configure(struct device *dev, enum dev_dma_attr attr); 597int acpi_dma_configure(struct device *dev, enum dev_dma_attr attr);
598void acpi_dma_deconfigure(struct device *dev);
599 598
600struct acpi_device *acpi_find_child_device(struct acpi_device *parent, 599struct acpi_device *acpi_find_child_device(struct acpi_device *parent,
601 u64 address, bool check_children); 600 u64 address, bool check_children);
diff --git a/include/asm-generic/dma-mapping.h b/include/asm-generic/dma-mapping.h
index ad2868263867..880a292d792f 100644
--- a/include/asm-generic/dma-mapping.h
+++ b/include/asm-generic/dma-mapping.h
@@ -4,16 +4,7 @@
4 4
5static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus) 5static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus)
6{ 6{
7 /*
8 * Use the non-coherent ops if available. If an architecture wants a
9 * more fine-grained selection of operations it will have to implement
10 * get_arch_dma_ops itself or use the per-device dma_ops.
11 */
12#ifdef CONFIG_DMA_NONCOHERENT_OPS
13 return &dma_noncoherent_ops;
14#else
15 return &dma_direct_ops; 7 return &dma_direct_ops;
16#endif
17} 8}
18 9
19#endif /* _ASM_GENERIC_DMA_MAPPING_H */ 10#endif /* _ASM_GENERIC_DMA_MAPPING_H */
diff --git a/include/linux/acpi.h b/include/linux/acpi.h
index de8d3d3fa651..af4628979d13 100644
--- a/include/linux/acpi.h
+++ b/include/linux/acpi.h
@@ -831,8 +831,6 @@ static inline int acpi_dma_configure(struct device *dev,
831 return 0; 831 return 0;
832} 832}
833 833
834static inline void acpi_dma_deconfigure(struct device *dev) { }
835
836#define ACPI_PTR(_ptr) (NULL) 834#define ACPI_PTR(_ptr) (NULL)
837 835
838static inline void acpi_device_set_enumerated(struct acpi_device *adev) 836static inline void acpi_device_set_enumerated(struct acpi_device *adev)
diff --git a/include/linux/device.h b/include/linux/device.h
index 8f882549edee..983506789402 100644
--- a/include/linux/device.h
+++ b/include/linux/device.h
@@ -927,6 +927,8 @@ struct dev_links_info {
927 * @offline: Set after successful invocation of bus type's .offline(). 927 * @offline: Set after successful invocation of bus type's .offline().
928 * @of_node_reused: Set if the device-tree node is shared with an ancestor 928 * @of_node_reused: Set if the device-tree node is shared with an ancestor
929 * device. 929 * device.
930 * @dma_coherent: this particular device is dma coherent, even if the
931 * architecture supports non-coherent devices.
930 * 932 *
931 * At the lowest level, every device in a Linux system is represented by an 933 * At the lowest level, every device in a Linux system is represented by an
932 * instance of struct device. The device structure contains the information 934 * instance of struct device. The device structure contains the information
@@ -1016,6 +1018,11 @@ struct device {
1016 bool offline_disabled:1; 1018 bool offline_disabled:1;
1017 bool offline:1; 1019 bool offline:1;
1018 bool of_node_reused:1; 1020 bool of_node_reused:1;
1021#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \
1022 defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
1023 defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL)
1024 bool dma_coherent:1;
1025#endif
1019}; 1026};
1020 1027
1021static inline struct device *kobj_to_dev(struct kobject *kobj) 1028static inline struct device *kobj_to_dev(struct kobject *kobj)
diff --git a/include/linux/dma-debug.h b/include/linux/dma-debug.h
index a785f2507159..30213adbb6b9 100644
--- a/include/linux/dma-debug.h
+++ b/include/linux/dma-debug.h
@@ -32,6 +32,9 @@ extern void dma_debug_add_bus(struct bus_type *bus);
32 32
33extern int dma_debug_resize_entries(u32 num_entries); 33extern int dma_debug_resize_entries(u32 num_entries);
34 34
35extern void debug_dma_map_single(struct device *dev, const void *addr,
36 unsigned long len);
37
35extern void debug_dma_map_page(struct device *dev, struct page *page, 38extern void debug_dma_map_page(struct device *dev, struct page *page,
36 size_t offset, size_t size, 39 size_t offset, size_t size,
37 int direction, dma_addr_t dma_addr, 40 int direction, dma_addr_t dma_addr,
@@ -103,6 +106,11 @@ static inline int dma_debug_resize_entries(u32 num_entries)
103 return 0; 106 return 0;
104} 107}
105 108
109static inline void debug_dma_map_single(struct device *dev, const void *addr,
110 unsigned long len)
111{
112}
113
106static inline void debug_dma_map_page(struct device *dev, struct page *page, 114static inline void debug_dma_map_page(struct device *dev, struct page *page,
107 size_t offset, size_t size, 115 size_t offset, size_t size,
108 int direction, dma_addr_t dma_addr, 116 int direction, dma_addr_t dma_addr,
diff --git a/include/linux/dma-direct.h b/include/linux/dma-direct.h
index 8d9f33febde5..fbca184ff5a0 100644
--- a/include/linux/dma-direct.h
+++ b/include/linux/dma-direct.h
@@ -27,7 +27,8 @@ static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size)
27 if (!dev->dma_mask) 27 if (!dev->dma_mask)
28 return false; 28 return false;
29 29
30 return addr + size - 1 <= *dev->dma_mask; 30 return addr + size - 1 <=
31 min_not_zero(*dev->dma_mask, dev->bus_dma_mask);
31} 32}
32#endif /* !CONFIG_ARCH_HAS_PHYS_TO_DMA */ 33#endif /* !CONFIG_ARCH_HAS_PHYS_TO_DMA */
33 34
@@ -55,10 +56,15 @@ static inline void dma_mark_clean(void *addr, size_t size)
55} 56}
56#endif /* CONFIG_ARCH_HAS_DMA_MARK_CLEAN */ 57#endif /* CONFIG_ARCH_HAS_DMA_MARK_CLEAN */
57 58
59u64 dma_direct_get_required_mask(struct device *dev);
58void *dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, 60void *dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
59 gfp_t gfp, unsigned long attrs); 61 gfp_t gfp, unsigned long attrs);
60void dma_direct_free(struct device *dev, size_t size, void *cpu_addr, 62void dma_direct_free(struct device *dev, size_t size, void *cpu_addr,
61 dma_addr_t dma_addr, unsigned long attrs); 63 dma_addr_t dma_addr, unsigned long attrs);
64void *dma_direct_alloc_pages(struct device *dev, size_t size,
65 dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs);
66void dma_direct_free_pages(struct device *dev, size_t size, void *cpu_addr,
67 dma_addr_t dma_addr, unsigned long attrs);
62dma_addr_t dma_direct_map_page(struct device *dev, struct page *page, 68dma_addr_t dma_direct_map_page(struct device *dev, struct page *page,
63 unsigned long offset, size_t size, enum dma_data_direction dir, 69 unsigned long offset, size_t size, enum dma_data_direction dir,
64 unsigned long attrs); 70 unsigned long attrs);
diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h
index 1db6a6b46d0d..15bd41447025 100644
--- a/include/linux/dma-mapping.h
+++ b/include/linux/dma-mapping.h
@@ -130,13 +130,10 @@ struct dma_map_ops {
130 enum dma_data_direction direction); 130 enum dma_data_direction direction);
131 int (*mapping_error)(struct device *dev, dma_addr_t dma_addr); 131 int (*mapping_error)(struct device *dev, dma_addr_t dma_addr);
132 int (*dma_supported)(struct device *dev, u64 mask); 132 int (*dma_supported)(struct device *dev, u64 mask);
133#ifdef ARCH_HAS_DMA_GET_REQUIRED_MASK
134 u64 (*get_required_mask)(struct device *dev); 133 u64 (*get_required_mask)(struct device *dev);
135#endif
136}; 134};
137 135
138extern const struct dma_map_ops dma_direct_ops; 136extern const struct dma_map_ops dma_direct_ops;
139extern const struct dma_map_ops dma_noncoherent_ops;
140extern const struct dma_map_ops dma_virt_ops; 137extern const struct dma_map_ops dma_virt_ops;
141 138
142#define DMA_BIT_MASK(n) (((n) == 64) ? ~0ULL : ((1ULL<<(n))-1)) 139#define DMA_BIT_MASK(n) (((n) == 64) ? ~0ULL : ((1ULL<<(n))-1))
@@ -232,6 +229,7 @@ static inline dma_addr_t dma_map_single_attrs(struct device *dev, void *ptr,
232 dma_addr_t addr; 229 dma_addr_t addr;
233 230
234 BUG_ON(!valid_dma_direction(dir)); 231 BUG_ON(!valid_dma_direction(dir));
232 debug_dma_map_single(dev, ptr, size);
235 addr = ops->map_page(dev, virt_to_page(ptr), 233 addr = ops->map_page(dev, virt_to_page(ptr),
236 offset_in_page(ptr), size, 234 offset_in_page(ptr), size,
237 dir, attrs); 235 dir, attrs);
@@ -445,7 +443,8 @@ dma_cache_sync(struct device *dev, void *vaddr, size_t size,
445} 443}
446 444
447extern int dma_common_mmap(struct device *dev, struct vm_area_struct *vma, 445extern int dma_common_mmap(struct device *dev, struct vm_area_struct *vma,
448 void *cpu_addr, dma_addr_t dma_addr, size_t size); 446 void *cpu_addr, dma_addr_t dma_addr, size_t size,
447 unsigned long attrs);
449 448
450void *dma_common_contiguous_remap(struct page *page, size_t size, 449void *dma_common_contiguous_remap(struct page *page, size_t size,
451 unsigned long vm_flags, 450 unsigned long vm_flags,
@@ -477,14 +476,14 @@ dma_mmap_attrs(struct device *dev, struct vm_area_struct *vma, void *cpu_addr,
477 BUG_ON(!ops); 476 BUG_ON(!ops);
478 if (ops->mmap) 477 if (ops->mmap)
479 return ops->mmap(dev, vma, cpu_addr, dma_addr, size, attrs); 478 return ops->mmap(dev, vma, cpu_addr, dma_addr, size, attrs);
480 return dma_common_mmap(dev, vma, cpu_addr, dma_addr, size); 479 return dma_common_mmap(dev, vma, cpu_addr, dma_addr, size, attrs);
481} 480}
482 481
483#define dma_mmap_coherent(d, v, c, h, s) dma_mmap_attrs(d, v, c, h, s, 0) 482#define dma_mmap_coherent(d, v, c, h, s) dma_mmap_attrs(d, v, c, h, s, 0)
484 483
485int 484int
486dma_common_get_sgtable(struct device *dev, struct sg_table *sgt, 485dma_common_get_sgtable(struct device *dev, struct sg_table *sgt, void *cpu_addr,
487 void *cpu_addr, dma_addr_t dma_addr, size_t size); 486 dma_addr_t dma_addr, size_t size, unsigned long attrs);
488 487
489static inline int 488static inline int
490dma_get_sgtable_attrs(struct device *dev, struct sg_table *sgt, void *cpu_addr, 489dma_get_sgtable_attrs(struct device *dev, struct sg_table *sgt, void *cpu_addr,
@@ -496,7 +495,8 @@ dma_get_sgtable_attrs(struct device *dev, struct sg_table *sgt, void *cpu_addr,
496 if (ops->get_sgtable) 495 if (ops->get_sgtable)
497 return ops->get_sgtable(dev, sgt, cpu_addr, dma_addr, size, 496 return ops->get_sgtable(dev, sgt, cpu_addr, dma_addr, size,
498 attrs); 497 attrs);
499 return dma_common_get_sgtable(dev, sgt, cpu_addr, dma_addr, size); 498 return dma_common_get_sgtable(dev, sgt, cpu_addr, dma_addr, size,
499 attrs);
500} 500}
501 501
502#define dma_get_sgtable(d, t, v, h, s) dma_get_sgtable_attrs(d, t, v, h, s, 0) 502#define dma_get_sgtable(d, t, v, h, s) dma_get_sgtable_attrs(d, t, v, h, s, 0)
@@ -558,9 +558,11 @@ static inline void dma_free_attrs(struct device *dev, size_t size,
558} 558}
559 559
560static inline void *dma_alloc_coherent(struct device *dev, size_t size, 560static inline void *dma_alloc_coherent(struct device *dev, size_t size,
561 dma_addr_t *dma_handle, gfp_t flag) 561 dma_addr_t *dma_handle, gfp_t gfp)
562{ 562{
563 return dma_alloc_attrs(dev, size, dma_handle, flag, 0); 563
564 return dma_alloc_attrs(dev, size, dma_handle, gfp,
565 (gfp & __GFP_NOWARN) ? DMA_ATTR_NO_WARN : 0);
564} 566}
565 567
566static inline void dma_free_coherent(struct device *dev, size_t size, 568static inline void dma_free_coherent(struct device *dev, size_t size,
@@ -753,18 +755,6 @@ dma_mark_declared_memory_occupied(struct device *dev,
753} 755}
754#endif /* CONFIG_HAVE_GENERIC_DMA_COHERENT */ 756#endif /* CONFIG_HAVE_GENERIC_DMA_COHERENT */
755 757
756#ifdef CONFIG_HAS_DMA
757int dma_configure(struct device *dev);
758void dma_deconfigure(struct device *dev);
759#else
760static inline int dma_configure(struct device *dev)
761{
762 return 0;
763}
764
765static inline void dma_deconfigure(struct device *dev) {}
766#endif
767
768/* 758/*
769 * Managed DMA API 759 * Managed DMA API
770 */ 760 */
@@ -806,8 +796,12 @@ static inline void dmam_release_declared_memory(struct device *dev)
806static inline void *dma_alloc_wc(struct device *dev, size_t size, 796static inline void *dma_alloc_wc(struct device *dev, size_t size,
807 dma_addr_t *dma_addr, gfp_t gfp) 797 dma_addr_t *dma_addr, gfp_t gfp)
808{ 798{
809 return dma_alloc_attrs(dev, size, dma_addr, gfp, 799 unsigned long attrs = DMA_ATTR_NO_WARN;
810 DMA_ATTR_WRITE_COMBINE); 800
801 if (gfp & __GFP_NOWARN)
802 attrs |= DMA_ATTR_NO_WARN;
803
804 return dma_alloc_attrs(dev, size, dma_addr, gfp, attrs);
811} 805}
812#ifndef dma_alloc_writecombine 806#ifndef dma_alloc_writecombine
813#define dma_alloc_writecombine dma_alloc_wc 807#define dma_alloc_writecombine dma_alloc_wc
diff --git a/include/linux/dma-noncoherent.h b/include/linux/dma-noncoherent.h
index a0aa00cc909d..9051b055beec 100644
--- a/include/linux/dma-noncoherent.h
+++ b/include/linux/dma-noncoherent.h
@@ -4,18 +4,35 @@
4 4
5#include <linux/dma-mapping.h> 5#include <linux/dma-mapping.h>
6 6
7#ifdef CONFIG_ARCH_HAS_DMA_COHERENCE_H
8#include <asm/dma-coherence.h>
9#elif defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \
10 defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
11 defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL)
12static inline bool dev_is_dma_coherent(struct device *dev)
13{
14 return dev->dma_coherent;
15}
16#else
17static inline bool dev_is_dma_coherent(struct device *dev)
18{
19 return true;
20}
21#endif /* CONFIG_ARCH_HAS_DMA_COHERENCE_H */
22
7void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, 23void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
8 gfp_t gfp, unsigned long attrs); 24 gfp_t gfp, unsigned long attrs);
9void arch_dma_free(struct device *dev, size_t size, void *cpu_addr, 25void arch_dma_free(struct device *dev, size_t size, void *cpu_addr,
10 dma_addr_t dma_addr, unsigned long attrs); 26 dma_addr_t dma_addr, unsigned long attrs);
27long arch_dma_coherent_to_pfn(struct device *dev, void *cpu_addr,
28 dma_addr_t dma_addr);
11 29
12#ifdef CONFIG_DMA_NONCOHERENT_MMAP 30#ifdef CONFIG_ARCH_HAS_DMA_MMAP_PGPROT
13int arch_dma_mmap(struct device *dev, struct vm_area_struct *vma, 31pgprot_t arch_dma_mmap_pgprot(struct device *dev, pgprot_t prot,
14 void *cpu_addr, dma_addr_t dma_addr, size_t size,
15 unsigned long attrs); 32 unsigned long attrs);
16#else 33#else
17#define arch_dma_mmap NULL 34# define arch_dma_mmap_pgprot(dev, prot, attrs) pgprot_noncached(prot)
18#endif /* CONFIG_DMA_NONCOHERENT_MMAP */ 35#endif
19 36
20#ifdef CONFIG_DMA_NONCOHERENT_CACHE_SYNC 37#ifdef CONFIG_DMA_NONCOHERENT_CACHE_SYNC
21void arch_dma_cache_sync(struct device *dev, void *vaddr, size_t size, 38void arch_dma_cache_sync(struct device *dev, void *vaddr, size_t size,
diff --git a/include/linux/of_device.h b/include/linux/of_device.h
index 165fd302b442..8d31e39dd564 100644
--- a/include/linux/of_device.h
+++ b/include/linux/of_device.h
@@ -58,7 +58,6 @@ static inline struct device_node *of_cpu_device_node_get(int cpu)
58int of_dma_configure(struct device *dev, 58int of_dma_configure(struct device *dev,
59 struct device_node *np, 59 struct device_node *np,
60 bool force_dma); 60 bool force_dma);
61void of_dma_deconfigure(struct device *dev);
62#else /* CONFIG_OF */ 61#else /* CONFIG_OF */
63 62
64static inline int of_driver_match_device(struct device *dev, 63static inline int of_driver_match_device(struct device *dev,
@@ -113,8 +112,6 @@ static inline int of_dma_configure(struct device *dev,
113{ 112{
114 return 0; 113 return 0;
115} 114}
116static inline void of_dma_deconfigure(struct device *dev)
117{}
118#endif /* CONFIG_OF */ 115#endif /* CONFIG_OF */
119 116
120#endif /* _LINUX_OF_DEVICE_H */ 117#endif /* _LINUX_OF_DEVICE_H */
diff --git a/kernel/dma/Kconfig b/kernel/dma/Kconfig
index 1b1d63b3634b..645c7a2ecde8 100644
--- a/kernel/dma/Kconfig
+++ b/kernel/dma/Kconfig
@@ -13,6 +13,9 @@ config NEED_DMA_MAP_STATE
13config ARCH_DMA_ADDR_T_64BIT 13config ARCH_DMA_ADDR_T_64BIT
14 def_bool 64BIT || PHYS_ADDR_T_64BIT 14 def_bool 64BIT || PHYS_ADDR_T_64BIT
15 15
16config ARCH_HAS_DMA_COHERENCE_H
17 bool
18
16config HAVE_GENERIC_DMA_COHERENT 19config HAVE_GENERIC_DMA_COHERENT
17 bool 20 bool
18 21
@@ -26,22 +29,19 @@ config ARCH_HAS_SYNC_DMA_FOR_CPU
26config ARCH_HAS_SYNC_DMA_FOR_CPU_ALL 29config ARCH_HAS_SYNC_DMA_FOR_CPU_ALL
27 bool 30 bool
28 31
29config DMA_DIRECT_OPS 32config ARCH_HAS_DMA_COHERENT_TO_PFN
30 bool 33 bool
31 depends on HAS_DMA
32 34
33config DMA_NONCOHERENT_OPS 35config ARCH_HAS_DMA_MMAP_PGPROT
34 bool 36 bool
35 depends on HAS_DMA
36 select DMA_DIRECT_OPS
37 37
38config DMA_NONCOHERENT_MMAP 38config DMA_DIRECT_OPS
39 bool 39 bool
40 depends on DMA_NONCOHERENT_OPS 40 depends on HAS_DMA
41 41
42config DMA_NONCOHERENT_CACHE_SYNC 42config DMA_NONCOHERENT_CACHE_SYNC
43 bool 43 bool
44 depends on DMA_NONCOHERENT_OPS 44 depends on DMA_DIRECT_OPS
45 45
46config DMA_VIRT_OPS 46config DMA_VIRT_OPS
47 bool 47 bool
diff --git a/kernel/dma/Makefile b/kernel/dma/Makefile
index 6de44e4eb454..7d581e4eea4a 100644
--- a/kernel/dma/Makefile
+++ b/kernel/dma/Makefile
@@ -4,7 +4,6 @@ obj-$(CONFIG_HAS_DMA) += mapping.o
4obj-$(CONFIG_DMA_CMA) += contiguous.o 4obj-$(CONFIG_DMA_CMA) += contiguous.o
5obj-$(CONFIG_HAVE_GENERIC_DMA_COHERENT) += coherent.o 5obj-$(CONFIG_HAVE_GENERIC_DMA_COHERENT) += coherent.o
6obj-$(CONFIG_DMA_DIRECT_OPS) += direct.o 6obj-$(CONFIG_DMA_DIRECT_OPS) += direct.o
7obj-$(CONFIG_DMA_NONCOHERENT_OPS) += noncoherent.o
8obj-$(CONFIG_DMA_VIRT_OPS) += virt.o 7obj-$(CONFIG_DMA_VIRT_OPS) += virt.o
9obj-$(CONFIG_DMA_API_DEBUG) += debug.o 8obj-$(CONFIG_DMA_API_DEBUG) += debug.o
10obj-$(CONFIG_SWIOTLB) += swiotlb.o 9obj-$(CONFIG_SWIOTLB) += swiotlb.o
diff --git a/kernel/dma/contiguous.c b/kernel/dma/contiguous.c
index 286d82329eb0..b2a87905846d 100644
--- a/kernel/dma/contiguous.c
+++ b/kernel/dma/contiguous.c
@@ -49,7 +49,11 @@ static phys_addr_t limit_cmdline;
49 49
50static int __init early_cma(char *p) 50static int __init early_cma(char *p)
51{ 51{
52 pr_debug("%s(%s)\n", __func__, p); 52 if (!p) {
53 pr_err("Config string not provided\n");
54 return -EINVAL;
55 }
56
53 size_cmdline = memparse(p, &p); 57 size_cmdline = memparse(p, &p);
54 if (*p != '@') 58 if (*p != '@')
55 return 0; 59 return 0;
diff --git a/kernel/dma/debug.c b/kernel/dma/debug.c
index c007d25bee09..231ca4628062 100644
--- a/kernel/dma/debug.c
+++ b/kernel/dma/debug.c
@@ -1312,6 +1312,22 @@ static void check_sg_segment(struct device *dev, struct scatterlist *sg)
1312#endif 1312#endif
1313} 1313}
1314 1314
1315void debug_dma_map_single(struct device *dev, const void *addr,
1316 unsigned long len)
1317{
1318 if (unlikely(dma_debug_disabled()))
1319 return;
1320
1321 if (!virt_addr_valid(addr))
1322 err_printk(dev, NULL, "DMA-API: device driver maps memory from invalid area [addr=%p] [len=%lu]\n",
1323 addr, len);
1324
1325 if (is_vmalloc_addr(addr))
1326 err_printk(dev, NULL, "DMA-API: device driver maps memory from vmalloc area [addr=%p] [len=%lu]\n",
1327 addr, len);
1328}
1329EXPORT_SYMBOL(debug_dma_map_single);
1330
1315void debug_dma_map_page(struct device *dev, struct page *page, size_t offset, 1331void debug_dma_map_page(struct device *dev, struct page *page, size_t offset,
1316 size_t size, int direction, dma_addr_t dma_addr, 1332 size_t size, int direction, dma_addr_t dma_addr,
1317 bool map_single) 1333 bool map_single)
diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c
index de87b0282e74..87a6bc2a96c0 100644
--- a/kernel/dma/direct.c
+++ b/kernel/dma/direct.c
@@ -1,13 +1,16 @@
1// SPDX-License-Identifier: GPL-2.0 1// SPDX-License-Identifier: GPL-2.0
2/* 2/*
3 * DMA operations that map physical memory directly without using an IOMMU or 3 * Copyright (C) 2018 Christoph Hellwig.
4 * flushing caches. 4 *
5 * DMA operations that map physical memory directly without using an IOMMU.
5 */ 6 */
7#include <linux/bootmem.h> /* for max_pfn */
6#include <linux/export.h> 8#include <linux/export.h>
7#include <linux/mm.h> 9#include <linux/mm.h>
8#include <linux/dma-direct.h> 10#include <linux/dma-direct.h>
9#include <linux/scatterlist.h> 11#include <linux/scatterlist.h>
10#include <linux/dma-contiguous.h> 12#include <linux/dma-contiguous.h>
13#include <linux/dma-noncoherent.h>
11#include <linux/pfn.h> 14#include <linux/pfn.h>
12#include <linux/set_memory.h> 15#include <linux/set_memory.h>
13 16
@@ -41,40 +44,83 @@ check_addr(struct device *dev, dma_addr_t dma_addr, size_t size,
41 return false; 44 return false;
42 } 45 }
43 46
44 if (*dev->dma_mask >= DMA_BIT_MASK(32)) { 47 if (*dev->dma_mask >= DMA_BIT_MASK(32) || dev->bus_dma_mask) {
45 dev_err(dev, 48 dev_err(dev,
46 "%s: overflow %pad+%zu of device mask %llx\n", 49 "%s: overflow %pad+%zu of device mask %llx bus mask %llx\n",
47 caller, &dma_addr, size, *dev->dma_mask); 50 caller, &dma_addr, size,
51 *dev->dma_mask, dev->bus_dma_mask);
48 } 52 }
49 return false; 53 return false;
50 } 54 }
51 return true; 55 return true;
52} 56}
53 57
58static inline dma_addr_t phys_to_dma_direct(struct device *dev,
59 phys_addr_t phys)
60{
61 if (force_dma_unencrypted())
62 return __phys_to_dma(dev, phys);
63 return phys_to_dma(dev, phys);
64}
65
66u64 dma_direct_get_required_mask(struct device *dev)
67{
68 u64 max_dma = phys_to_dma_direct(dev, (max_pfn - 1) << PAGE_SHIFT);
69
70 if (dev->bus_dma_mask && dev->bus_dma_mask < max_dma)
71 max_dma = dev->bus_dma_mask;
72
73 return (1ULL << (fls64(max_dma) - 1)) * 2 - 1;
74}
75
76static gfp_t __dma_direct_optimal_gfp_mask(struct device *dev, u64 dma_mask,
77 u64 *phys_mask)
78{
79 if (dev->bus_dma_mask && dev->bus_dma_mask < dma_mask)
80 dma_mask = dev->bus_dma_mask;
81
82 if (force_dma_unencrypted())
83 *phys_mask = __dma_to_phys(dev, dma_mask);
84 else
85 *phys_mask = dma_to_phys(dev, dma_mask);
86
87 /*
88 * Optimistically try the zone that the physical address mask falls
89 * into first. If that returns memory that isn't actually addressable
90 * we will fallback to the next lower zone and try again.
91 *
92 * Note that GFP_DMA32 and GFP_DMA are no ops without the corresponding
93 * zones.
94 */
95 if (*phys_mask <= DMA_BIT_MASK(ARCH_ZONE_DMA_BITS))
96 return GFP_DMA;
97 if (*phys_mask <= DMA_BIT_MASK(32))
98 return GFP_DMA32;
99 return 0;
100}
101
54static bool dma_coherent_ok(struct device *dev, phys_addr_t phys, size_t size) 102static bool dma_coherent_ok(struct device *dev, phys_addr_t phys, size_t size)
55{ 103{
56 dma_addr_t addr = force_dma_unencrypted() ? 104 return phys_to_dma_direct(dev, phys) + size - 1 <=
57 __phys_to_dma(dev, phys) : phys_to_dma(dev, phys); 105 min_not_zero(dev->coherent_dma_mask, dev->bus_dma_mask);
58 return addr + size - 1 <= dev->coherent_dma_mask;
59} 106}
60 107
61void *dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, 108void *dma_direct_alloc_pages(struct device *dev, size_t size,
62 gfp_t gfp, unsigned long attrs) 109 dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
63{ 110{
64 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; 111 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
65 int page_order = get_order(size); 112 int page_order = get_order(size);
66 struct page *page = NULL; 113 struct page *page = NULL;
114 u64 phys_mask;
67 void *ret; 115 void *ret;
68 116
117 if (attrs & DMA_ATTR_NO_WARN)
118 gfp |= __GFP_NOWARN;
119
69 /* we always manually zero the memory once we are done: */ 120 /* we always manually zero the memory once we are done: */
70 gfp &= ~__GFP_ZERO; 121 gfp &= ~__GFP_ZERO;
71 122 gfp |= __dma_direct_optimal_gfp_mask(dev, dev->coherent_dma_mask,
72 /* GFP_DMA32 and GFP_DMA are no ops without the corresponding zones: */ 123 &phys_mask);
73 if (dev->coherent_dma_mask <= DMA_BIT_MASK(ARCH_ZONE_DMA_BITS))
74 gfp |= GFP_DMA;
75 if (dev->coherent_dma_mask <= DMA_BIT_MASK(32) && !(gfp & GFP_DMA))
76 gfp |= GFP_DMA32;
77
78again: 124again:
79 /* CMA can be used only in the context which permits sleeping */ 125 /* CMA can be used only in the context which permits sleeping */
80 if (gfpflags_allow_blocking(gfp)) { 126 if (gfpflags_allow_blocking(gfp)) {
@@ -93,15 +139,14 @@ again:
93 page = NULL; 139 page = NULL;
94 140
95 if (IS_ENABLED(CONFIG_ZONE_DMA32) && 141 if (IS_ENABLED(CONFIG_ZONE_DMA32) &&
96 dev->coherent_dma_mask < DMA_BIT_MASK(64) && 142 phys_mask < DMA_BIT_MASK(64) &&
97 !(gfp & (GFP_DMA32 | GFP_DMA))) { 143 !(gfp & (GFP_DMA32 | GFP_DMA))) {
98 gfp |= GFP_DMA32; 144 gfp |= GFP_DMA32;
99 goto again; 145 goto again;
100 } 146 }
101 147
102 if (IS_ENABLED(CONFIG_ZONE_DMA) && 148 if (IS_ENABLED(CONFIG_ZONE_DMA) &&
103 dev->coherent_dma_mask < DMA_BIT_MASK(32) && 149 phys_mask < DMA_BIT_MASK(32) && !(gfp & GFP_DMA)) {
104 !(gfp & GFP_DMA)) {
105 gfp = (gfp & ~GFP_DMA32) | GFP_DMA; 150 gfp = (gfp & ~GFP_DMA32) | GFP_DMA;
106 goto again; 151 goto again;
107 } 152 }
@@ -124,7 +169,7 @@ again:
124 * NOTE: this function must never look at the dma_addr argument, because we want 169 * NOTE: this function must never look at the dma_addr argument, because we want
125 * to be able to use it as a helper for iommu implementations as well. 170 * to be able to use it as a helper for iommu implementations as well.
126 */ 171 */
127void dma_direct_free(struct device *dev, size_t size, void *cpu_addr, 172void dma_direct_free_pages(struct device *dev, size_t size, void *cpu_addr,
128 dma_addr_t dma_addr, unsigned long attrs) 173 dma_addr_t dma_addr, unsigned long attrs)
129{ 174{
130 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; 175 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
@@ -136,14 +181,96 @@ void dma_direct_free(struct device *dev, size_t size, void *cpu_addr,
136 free_pages((unsigned long)cpu_addr, page_order); 181 free_pages((unsigned long)cpu_addr, page_order);
137} 182}
138 183
184void *dma_direct_alloc(struct device *dev, size_t size,
185 dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
186{
187 if (!dev_is_dma_coherent(dev))
188 return arch_dma_alloc(dev, size, dma_handle, gfp, attrs);
189 return dma_direct_alloc_pages(dev, size, dma_handle, gfp, attrs);
190}
191
192void dma_direct_free(struct device *dev, size_t size,
193 void *cpu_addr, dma_addr_t dma_addr, unsigned long attrs)
194{
195 if (!dev_is_dma_coherent(dev))
196 arch_dma_free(dev, size, cpu_addr, dma_addr, attrs);
197 else
198 dma_direct_free_pages(dev, size, cpu_addr, dma_addr, attrs);
199}
200
201static void dma_direct_sync_single_for_device(struct device *dev,
202 dma_addr_t addr, size_t size, enum dma_data_direction dir)
203{
204 if (dev_is_dma_coherent(dev))
205 return;
206 arch_sync_dma_for_device(dev, dma_to_phys(dev, addr), size, dir);
207}
208
209static void dma_direct_sync_sg_for_device(struct device *dev,
210 struct scatterlist *sgl, int nents, enum dma_data_direction dir)
211{
212 struct scatterlist *sg;
213 int i;
214
215 if (dev_is_dma_coherent(dev))
216 return;
217
218 for_each_sg(sgl, sg, nents, i)
219 arch_sync_dma_for_device(dev, sg_phys(sg), sg->length, dir);
220}
221
222#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
223 defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL)
224static void dma_direct_sync_single_for_cpu(struct device *dev,
225 dma_addr_t addr, size_t size, enum dma_data_direction dir)
226{
227 if (dev_is_dma_coherent(dev))
228 return;
229 arch_sync_dma_for_cpu(dev, dma_to_phys(dev, addr), size, dir);
230 arch_sync_dma_for_cpu_all(dev);
231}
232
233static void dma_direct_sync_sg_for_cpu(struct device *dev,
234 struct scatterlist *sgl, int nents, enum dma_data_direction dir)
235{
236 struct scatterlist *sg;
237 int i;
238
239 if (dev_is_dma_coherent(dev))
240 return;
241
242 for_each_sg(sgl, sg, nents, i)
243 arch_sync_dma_for_cpu(dev, sg_phys(sg), sg->length, dir);
244 arch_sync_dma_for_cpu_all(dev);
245}
246
247static void dma_direct_unmap_page(struct device *dev, dma_addr_t addr,
248 size_t size, enum dma_data_direction dir, unsigned long attrs)
249{
250 if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
251 dma_direct_sync_single_for_cpu(dev, addr, size, dir);
252}
253
254static void dma_direct_unmap_sg(struct device *dev, struct scatterlist *sgl,
255 int nents, enum dma_data_direction dir, unsigned long attrs)
256{
257 if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
258 dma_direct_sync_sg_for_cpu(dev, sgl, nents, dir);
259}
260#endif
261
139dma_addr_t dma_direct_map_page(struct device *dev, struct page *page, 262dma_addr_t dma_direct_map_page(struct device *dev, struct page *page,
140 unsigned long offset, size_t size, enum dma_data_direction dir, 263 unsigned long offset, size_t size, enum dma_data_direction dir,
141 unsigned long attrs) 264 unsigned long attrs)
142{ 265{
143 dma_addr_t dma_addr = phys_to_dma(dev, page_to_phys(page)) + offset; 266 phys_addr_t phys = page_to_phys(page) + offset;
267 dma_addr_t dma_addr = phys_to_dma(dev, phys);
144 268
145 if (!check_addr(dev, dma_addr, size, __func__)) 269 if (!check_addr(dev, dma_addr, size, __func__))
146 return DIRECT_MAPPING_ERROR; 270 return DIRECT_MAPPING_ERROR;
271
272 if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
273 dma_direct_sync_single_for_device(dev, dma_addr, size, dir);
147 return dma_addr; 274 return dma_addr;
148} 275}
149 276
@@ -162,31 +289,29 @@ int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, int nents,
162 sg_dma_len(sg) = sg->length; 289 sg_dma_len(sg) = sg->length;
163 } 290 }
164 291
292 if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
293 dma_direct_sync_sg_for_device(dev, sgl, nents, dir);
165 return nents; 294 return nents;
166} 295}
167 296
297/*
298 * Because 32-bit DMA masks are so common we expect every architecture to be
299 * able to satisfy them - either by not supporting more physical memory, or by
300 * providing a ZONE_DMA32. If neither is the case, the architecture needs to
301 * use an IOMMU instead of the direct mapping.
302 */
168int dma_direct_supported(struct device *dev, u64 mask) 303int dma_direct_supported(struct device *dev, u64 mask)
169{ 304{
170#ifdef CONFIG_ZONE_DMA 305 u64 min_mask;
171 if (mask < phys_to_dma(dev, DMA_BIT_MASK(ARCH_ZONE_DMA_BITS))) 306
172 return 0; 307 if (IS_ENABLED(CONFIG_ZONE_DMA))
173#else 308 min_mask = DMA_BIT_MASK(ARCH_ZONE_DMA_BITS);
174 /* 309 else
175 * Because 32-bit DMA masks are so common we expect every architecture 310 min_mask = DMA_BIT_MASK(32);
176 * to be able to satisfy them - either by not supporting more physical 311
177 * memory, or by providing a ZONE_DMA32. If neither is the case, the 312 min_mask = min_t(u64, min_mask, (max_pfn - 1) << PAGE_SHIFT);
178 * architecture needs to use an IOMMU instead of the direct mapping. 313
179 */ 314 return mask >= phys_to_dma(dev, min_mask);
180 if (mask < phys_to_dma(dev, DMA_BIT_MASK(32)))
181 return 0;
182#endif
183 /*
184 * Upstream PCI/PCIe bridges or SoC interconnects may not carry
185 * as many DMA address bits as the device itself supports.
186 */
187 if (dev->bus_dma_mask && mask > dev->bus_dma_mask)
188 return 0;
189 return 1;
190} 315}
191 316
192int dma_direct_mapping_error(struct device *dev, dma_addr_t dma_addr) 317int dma_direct_mapping_error(struct device *dev, dma_addr_t dma_addr)
@@ -199,7 +324,20 @@ const struct dma_map_ops dma_direct_ops = {
199 .free = dma_direct_free, 324 .free = dma_direct_free,
200 .map_page = dma_direct_map_page, 325 .map_page = dma_direct_map_page,
201 .map_sg = dma_direct_map_sg, 326 .map_sg = dma_direct_map_sg,
327#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE)
328 .sync_single_for_device = dma_direct_sync_single_for_device,
329 .sync_sg_for_device = dma_direct_sync_sg_for_device,
330#endif
331#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
332 defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL)
333 .sync_single_for_cpu = dma_direct_sync_single_for_cpu,
334 .sync_sg_for_cpu = dma_direct_sync_sg_for_cpu,
335 .unmap_page = dma_direct_unmap_page,
336 .unmap_sg = dma_direct_unmap_sg,
337#endif
338 .get_required_mask = dma_direct_get_required_mask,
202 .dma_supported = dma_direct_supported, 339 .dma_supported = dma_direct_supported,
203 .mapping_error = dma_direct_mapping_error, 340 .mapping_error = dma_direct_mapping_error,
341 .cache_sync = arch_dma_cache_sync,
204}; 342};
205EXPORT_SYMBOL(dma_direct_ops); 343EXPORT_SYMBOL(dma_direct_ops);
diff --git a/kernel/dma/mapping.c b/kernel/dma/mapping.c
index d2a92ddaac4d..58dec7a92b7b 100644
--- a/kernel/dma/mapping.c
+++ b/kernel/dma/mapping.c
@@ -7,7 +7,7 @@
7 */ 7 */
8 8
9#include <linux/acpi.h> 9#include <linux/acpi.h>
10#include <linux/dma-mapping.h> 10#include <linux/dma-noncoherent.h>
11#include <linux/export.h> 11#include <linux/export.h>
12#include <linux/gfp.h> 12#include <linux/gfp.h>
13#include <linux/of_device.h> 13#include <linux/of_device.h>
@@ -202,17 +202,26 @@ EXPORT_SYMBOL(dmam_release_declared_memory);
202 * Create scatter-list for the already allocated DMA buffer. 202 * Create scatter-list for the already allocated DMA buffer.
203 */ 203 */
204int dma_common_get_sgtable(struct device *dev, struct sg_table *sgt, 204int dma_common_get_sgtable(struct device *dev, struct sg_table *sgt,
205 void *cpu_addr, dma_addr_t handle, size_t size) 205 void *cpu_addr, dma_addr_t dma_addr, size_t size,
206 unsigned long attrs)
206{ 207{
207 struct page *page = virt_to_page(cpu_addr); 208 struct page *page;
208 int ret; 209 int ret;
209 210
210 ret = sg_alloc_table(sgt, 1, GFP_KERNEL); 211 if (!dev_is_dma_coherent(dev)) {
211 if (unlikely(ret)) 212 if (!IS_ENABLED(CONFIG_ARCH_HAS_DMA_COHERENT_TO_PFN))
212 return ret; 213 return -ENXIO;
213 214
214 sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0); 215 page = pfn_to_page(arch_dma_coherent_to_pfn(dev, cpu_addr,
215 return 0; 216 dma_addr));
217 } else {
218 page = virt_to_page(cpu_addr);
219 }
220
221 ret = sg_alloc_table(sgt, 1, GFP_KERNEL);
222 if (!ret)
223 sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0);
224 return ret;
216} 225}
217EXPORT_SYMBOL(dma_common_get_sgtable); 226EXPORT_SYMBOL(dma_common_get_sgtable);
218 227
@@ -220,27 +229,37 @@ EXPORT_SYMBOL(dma_common_get_sgtable);
220 * Create userspace mapping for the DMA-coherent memory. 229 * Create userspace mapping for the DMA-coherent memory.
221 */ 230 */
222int dma_common_mmap(struct device *dev, struct vm_area_struct *vma, 231int dma_common_mmap(struct device *dev, struct vm_area_struct *vma,
223 void *cpu_addr, dma_addr_t dma_addr, size_t size) 232 void *cpu_addr, dma_addr_t dma_addr, size_t size,
233 unsigned long attrs)
224{ 234{
225 int ret = -ENXIO;
226#ifndef CONFIG_ARCH_NO_COHERENT_DMA_MMAP 235#ifndef CONFIG_ARCH_NO_COHERENT_DMA_MMAP
227 unsigned long user_count = vma_pages(vma); 236 unsigned long user_count = vma_pages(vma);
228 unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT; 237 unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT;
229 unsigned long off = vma->vm_pgoff; 238 unsigned long off = vma->vm_pgoff;
239 unsigned long pfn;
240 int ret = -ENXIO;
230 241
231 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 242 vma->vm_page_prot = arch_dma_mmap_pgprot(dev, vma->vm_page_prot, attrs);
232 243
233 if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret)) 244 if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret))
234 return ret; 245 return ret;
235 246
236 if (off < count && user_count <= (count - off)) 247 if (off >= count || user_count > count - off)
237 ret = remap_pfn_range(vma, vma->vm_start, 248 return -ENXIO;
238 page_to_pfn(virt_to_page(cpu_addr)) + off,
239 user_count << PAGE_SHIFT,
240 vma->vm_page_prot);
241#endif /* !CONFIG_ARCH_NO_COHERENT_DMA_MMAP */
242 249
243 return ret; 250 if (!dev_is_dma_coherent(dev)) {
251 if (!IS_ENABLED(CONFIG_ARCH_HAS_DMA_COHERENT_TO_PFN))
252 return -ENXIO;
253 pfn = arch_dma_coherent_to_pfn(dev, cpu_addr, dma_addr);
254 } else {
255 pfn = page_to_pfn(virt_to_page(cpu_addr));
256 }
257
258 return remap_pfn_range(vma, vma->vm_start, pfn + vma->vm_pgoff,
259 user_count << PAGE_SHIFT, vma->vm_page_prot);
260#else
261 return -ENXIO;
262#endif /* !CONFIG_ARCH_NO_COHERENT_DMA_MMAP */
244} 263}
245EXPORT_SYMBOL(dma_common_mmap); 264EXPORT_SYMBOL(dma_common_mmap);
246 265
@@ -327,19 +346,3 @@ void dma_common_free_remap(void *cpu_addr, size_t size, unsigned long vm_flags)
327 vunmap(cpu_addr); 346 vunmap(cpu_addr);
328} 347}
329#endif 348#endif
330
331/*
332 * enables DMA API use for a device
333 */
334int dma_configure(struct device *dev)
335{
336 if (dev->bus->dma_configure)
337 return dev->bus->dma_configure(dev);
338 return 0;
339}
340
341void dma_deconfigure(struct device *dev)
342{
343 of_dma_deconfigure(dev);
344 acpi_dma_deconfigure(dev);
345}
diff --git a/kernel/dma/noncoherent.c b/kernel/dma/noncoherent.c
deleted file mode 100644
index 031fe235d958..000000000000
--- a/kernel/dma/noncoherent.c
+++ /dev/null
@@ -1,106 +0,0 @@
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2018 Christoph Hellwig.
4 *
5 * DMA operations that map physical memory directly without providing cache
6 * coherence.
7 */
8#include <linux/export.h>
9#include <linux/mm.h>
10#include <linux/dma-direct.h>
11#include <linux/dma-noncoherent.h>
12#include <linux/scatterlist.h>
13
14static void dma_noncoherent_sync_single_for_device(struct device *dev,
15 dma_addr_t addr, size_t size, enum dma_data_direction dir)
16{
17 arch_sync_dma_for_device(dev, dma_to_phys(dev, addr), size, dir);
18}
19
20static void dma_noncoherent_sync_sg_for_device(struct device *dev,
21 struct scatterlist *sgl, int nents, enum dma_data_direction dir)
22{
23 struct scatterlist *sg;
24 int i;
25
26 for_each_sg(sgl, sg, nents, i)
27 arch_sync_dma_for_device(dev, sg_phys(sg), sg->length, dir);
28}
29
30static dma_addr_t dma_noncoherent_map_page(struct device *dev, struct page *page,
31 unsigned long offset, size_t size, enum dma_data_direction dir,
32 unsigned long attrs)
33{
34 dma_addr_t addr;
35
36 addr = dma_direct_map_page(dev, page, offset, size, dir, attrs);
37 if (!dma_mapping_error(dev, addr) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
38 arch_sync_dma_for_device(dev, page_to_phys(page) + offset,
39 size, dir);
40 return addr;
41}
42
43static int dma_noncoherent_map_sg(struct device *dev, struct scatterlist *sgl,
44 int nents, enum dma_data_direction dir, unsigned long attrs)
45{
46 nents = dma_direct_map_sg(dev, sgl, nents, dir, attrs);
47 if (nents > 0 && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
48 dma_noncoherent_sync_sg_for_device(dev, sgl, nents, dir);
49 return nents;
50}
51
52#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
53 defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL)
54static void dma_noncoherent_sync_single_for_cpu(struct device *dev,
55 dma_addr_t addr, size_t size, enum dma_data_direction dir)
56{
57 arch_sync_dma_for_cpu(dev, dma_to_phys(dev, addr), size, dir);
58 arch_sync_dma_for_cpu_all(dev);
59}
60
61static void dma_noncoherent_sync_sg_for_cpu(struct device *dev,
62 struct scatterlist *sgl, int nents, enum dma_data_direction dir)
63{
64 struct scatterlist *sg;
65 int i;
66
67 for_each_sg(sgl, sg, nents, i)
68 arch_sync_dma_for_cpu(dev, sg_phys(sg), sg->length, dir);
69 arch_sync_dma_for_cpu_all(dev);
70}
71
72static void dma_noncoherent_unmap_page(struct device *dev, dma_addr_t addr,
73 size_t size, enum dma_data_direction dir, unsigned long attrs)
74{
75 if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
76 dma_noncoherent_sync_single_for_cpu(dev, addr, size, dir);
77}
78
79static void dma_noncoherent_unmap_sg(struct device *dev, struct scatterlist *sgl,
80 int nents, enum dma_data_direction dir, unsigned long attrs)
81{
82 if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
83 dma_noncoherent_sync_sg_for_cpu(dev, sgl, nents, dir);
84}
85#endif
86
87const struct dma_map_ops dma_noncoherent_ops = {
88 .alloc = arch_dma_alloc,
89 .free = arch_dma_free,
90 .mmap = arch_dma_mmap,
91 .sync_single_for_device = dma_noncoherent_sync_single_for_device,
92 .sync_sg_for_device = dma_noncoherent_sync_sg_for_device,
93 .map_page = dma_noncoherent_map_page,
94 .map_sg = dma_noncoherent_map_sg,
95#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
96 defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL)
97 .sync_single_for_cpu = dma_noncoherent_sync_single_for_cpu,
98 .sync_sg_for_cpu = dma_noncoherent_sync_sg_for_cpu,
99 .unmap_page = dma_noncoherent_unmap_page,
100 .unmap_sg = dma_noncoherent_unmap_sg,
101#endif
102 .dma_supported = dma_direct_supported,
103 .mapping_error = dma_direct_mapping_error,
104 .cache_sync = arch_dma_cache_sync,
105};
106EXPORT_SYMBOL(dma_noncoherent_ops);
generated by cgit v1.2.2 (git 2.25.0) at 2026-05-27 13:00:36 -0400