aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2018-04-02 20:18:45 -0400
committerLinus Torvalds <torvalds@linux-foundation.org>2018-04-02 20:18:45 -0400
commit2fcd2b306aa80771e053275ed74b2dfe7e3d1434 (patch)
treeb19f4e3e9552eab00056c833650e692192fe8f5c
parentce6eba3dba366b607c0a363c7cdbd4ee8fcc6434 (diff)
parente89f5b37015309a8bdf0b21d08007580b92f92a4 (diff)
Merge branch 'x86-dma-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull x86 dma mapping updates from Ingo Molnar: "This tree, by Christoph Hellwig, switches over the x86 architecture to the generic dma-direct and swiotlb code, and also unifies more of the dma-direct code between architectures. The now unused x86-only primitives are removed" * 'x86-dma-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: dma-mapping: Don't clear GFP_ZERO in dma_alloc_attrs swiotlb: Make swiotlb_{alloc,free}_buffer depend on CONFIG_DMA_DIRECT_OPS dma/swiotlb: Remove swiotlb_{alloc,free}_coherent() dma/direct: Handle force decryption for DMA coherent buffers in common code dma/direct: Handle the memory encryption bit in common code dma/swiotlb: Remove swiotlb_set_mem_attributes() set_memory.h: Provide set_memory_{en,de}crypted() stubs x86/dma: Remove dma_alloc_coherent_gfp_flags() iommu/intel-iommu: Enable CONFIG_DMA_DIRECT_OPS=y and clean up intel_{alloc,free}_coherent() iommu/amd_iommu: Use CONFIG_DMA_DIRECT_OPS=y and dma_direct_{alloc,free}() x86/dma/amd_gart: Use dma_direct_{alloc,free}() x86/dma/amd_gart: Look at dev->coherent_dma_mask instead of GFP_DMA x86/dma: Use generic swiotlb_ops x86/dma: Use DMA-direct (CONFIG_DMA_DIRECT_OPS=y) x86/dma: Remove dma_alloc_coherent_mask()
Diffstat
-rw-r--r--arch/arm/include/asm/dma-direct.h4
-rw-r--r--arch/mips/cavium-octeon/dma-octeon.c10
-rw-r--r--arch/mips/include/asm/mach-cavium-octeon/dma-coherence.h4
-rw-r--r--arch/mips/include/asm/mach-loongson64/dma-coherence.h10
-rw-r--r--arch/mips/loongson64/common/dma-swiotlb.c4
-rw-r--r--arch/powerpc/include/asm/dma-direct.h4
-rw-r--r--arch/x86/Kconfig3
-rw-r--r--arch/x86/include/asm/device.h3
-rw-r--r--arch/x86/include/asm/dma-direct.h25
-rw-r--r--arch/x86/include/asm/dma-mapping.h33
-rw-r--r--arch/x86/include/asm/iommu.h3
-rw-r--r--arch/x86/include/asm/mem_encrypt.h2
-rw-r--r--arch/x86/include/asm/swiotlb.h8
-rw-r--r--arch/x86/kernel/Makefile2
-rw-r--r--arch/x86/kernel/amd_gart_64.c41
-rw-r--r--arch/x86/kernel/pci-calgary_64.c5
-rw-r--r--arch/x86/kernel/pci-dma.c70
-rw-r--r--arch/x86/kernel/pci-swiotlb.c48
-rw-r--r--arch/x86/mm/mem_encrypt.c90
-rw-r--r--arch/x86/pci/sta2x11-fixup.c52
-rw-r--r--drivers/iommu/Kconfig2
-rw-r--r--drivers/iommu/amd_iommu.c75
-rw-r--r--drivers/iommu/intel-iommu.c65
-rw-r--r--drivers/xen/swiotlb-xen.c16
-rw-r--r--include/linux/dma-direct.h21
-rw-r--r--include/linux/dma-mapping.h8
-rw-r--r--include/linux/set_memory.h12
-rw-r--r--include/linux/swiotlb.h8
-rw-r--r--lib/dma-direct.c35
-rw-r--r--lib/swiotlb.c77
30 files changed, 182 insertions, 558 deletions
diff --git a/arch/arm/include/asm/dma-direct.h b/arch/arm/include/asm/dma-direct.h
index 5b0a8a421894..b67e5fc1fe43 100644
--- a/arch/arm/include/asm/dma-direct.h
+++ b/arch/arm/include/asm/dma-direct.h
@@ -2,13 +2,13 @@
2#ifndef ASM_ARM_DMA_DIRECT_H 2#ifndef ASM_ARM_DMA_DIRECT_H
3#define ASM_ARM_DMA_DIRECT_H 1 3#define ASM_ARM_DMA_DIRECT_H 1
4 4
5static inline dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr) 5static inline dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t paddr)
6{ 6{
7 unsigned int offset = paddr & ~PAGE_MASK; 7 unsigned int offset = paddr & ~PAGE_MASK;
8 return pfn_to_dma(dev, __phys_to_pfn(paddr)) + offset; 8 return pfn_to_dma(dev, __phys_to_pfn(paddr)) + offset;
9} 9}
10 10
11static inline phys_addr_t dma_to_phys(struct device *dev, dma_addr_t dev_addr) 11static inline phys_addr_t __dma_to_phys(struct device *dev, dma_addr_t dev_addr)
12{ 12{
13 unsigned int offset = dev_addr & ~PAGE_MASK; 13 unsigned int offset = dev_addr & ~PAGE_MASK;
14 return __pfn_to_phys(dma_to_pfn(dev, dev_addr)) + offset; 14 return __pfn_to_phys(dma_to_pfn(dev, dev_addr)) + offset;
diff --git a/arch/mips/cavium-octeon/dma-octeon.c b/arch/mips/cavium-octeon/dma-octeon.c
index c7bb8a407041..7b335ab21697 100644
--- a/arch/mips/cavium-octeon/dma-octeon.c
+++ b/arch/mips/cavium-octeon/dma-octeon.c
@@ -10,7 +10,7 @@
10 * IP32 changes by Ilya. 10 * IP32 changes by Ilya.
11 * Copyright (C) 2010 Cavium Networks, Inc. 11 * Copyright (C) 2010 Cavium Networks, Inc.
12 */ 12 */
13#include <linux/dma-mapping.h> 13#include <linux/dma-direct.h>
14#include <linux/scatterlist.h> 14#include <linux/scatterlist.h>
15#include <linux/bootmem.h> 15#include <linux/bootmem.h>
16#include <linux/export.h> 16#include <linux/export.h>
@@ -182,7 +182,7 @@ struct octeon_dma_map_ops {
182 phys_addr_t (*dma_to_phys)(struct device *dev, dma_addr_t daddr); 182 phys_addr_t (*dma_to_phys)(struct device *dev, dma_addr_t daddr);
183}; 183};
184 184
185dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr) 185dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t paddr)
186{ 186{
187 struct octeon_dma_map_ops *ops = container_of(get_dma_ops(dev), 187 struct octeon_dma_map_ops *ops = container_of(get_dma_ops(dev),
188 struct octeon_dma_map_ops, 188 struct octeon_dma_map_ops,
@@ -190,9 +190,9 @@ dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr)
190 190
191 return ops->phys_to_dma(dev, paddr); 191 return ops->phys_to_dma(dev, paddr);
192} 192}
193EXPORT_SYMBOL(phys_to_dma); 193EXPORT_SYMBOL(__phys_to_dma);
194 194
195phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr) 195phys_addr_t __dma_to_phys(struct device *dev, dma_addr_t daddr)
196{ 196{
197 struct octeon_dma_map_ops *ops = container_of(get_dma_ops(dev), 197 struct octeon_dma_map_ops *ops = container_of(get_dma_ops(dev),
198 struct octeon_dma_map_ops, 198 struct octeon_dma_map_ops,
@@ -200,7 +200,7 @@ phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr)
200 200
201 return ops->dma_to_phys(dev, daddr); 201 return ops->dma_to_phys(dev, daddr);
202} 202}
203EXPORT_SYMBOL(dma_to_phys); 203EXPORT_SYMBOL(__dma_to_phys);
204 204
205static struct octeon_dma_map_ops octeon_linear_dma_map_ops = { 205static struct octeon_dma_map_ops octeon_linear_dma_map_ops = {
206 .dma_map_ops = { 206 .dma_map_ops = {
diff --git a/arch/mips/include/asm/mach-cavium-octeon/dma-coherence.h b/arch/mips/include/asm/mach-cavium-octeon/dma-coherence.h
index 138edf6b5b48..6eb1ee548b11 100644
--- a/arch/mips/include/asm/mach-cavium-octeon/dma-coherence.h
+++ b/arch/mips/include/asm/mach-cavium-octeon/dma-coherence.h
@@ -69,8 +69,8 @@ static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size)
69 return addr + size - 1 <= *dev->dma_mask; 69 return addr + size - 1 <= *dev->dma_mask;
70} 70}
71 71
72dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr); 72dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t paddr);
73phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr); 73phys_addr_t __dma_to_phys(struct device *dev, dma_addr_t daddr);
74 74
75struct dma_map_ops; 75struct dma_map_ops;
76extern const struct dma_map_ops *octeon_pci_dma_map_ops; 76extern const struct dma_map_ops *octeon_pci_dma_map_ops;
diff --git a/arch/mips/include/asm/mach-loongson64/dma-coherence.h b/arch/mips/include/asm/mach-loongson64/dma-coherence.h
index b1b575f5c6c1..64fc44dec0a8 100644
--- a/arch/mips/include/asm/mach-loongson64/dma-coherence.h
+++ b/arch/mips/include/asm/mach-loongson64/dma-coherence.h
@@ -25,13 +25,13 @@ static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size)
25 return addr + size - 1 <= *dev->dma_mask; 25 return addr + size - 1 <= *dev->dma_mask;
26} 26}
27 27
28extern dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr); 28extern dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t paddr);
29extern phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr); 29extern phys_addr_t __dma_to_phys(struct device *dev, dma_addr_t daddr);
30static inline dma_addr_t plat_map_dma_mem(struct device *dev, void *addr, 30static inline dma_addr_t plat_map_dma_mem(struct device *dev, void *addr,
31 size_t size) 31 size_t size)
32{ 32{
33#ifdef CONFIG_CPU_LOONGSON3 33#ifdef CONFIG_CPU_LOONGSON3
34 return phys_to_dma(dev, virt_to_phys(addr)); 34 return __phys_to_dma(dev, virt_to_phys(addr));
35#else 35#else
36 return virt_to_phys(addr) | 0x80000000; 36 return virt_to_phys(addr) | 0x80000000;
37#endif 37#endif
@@ -41,7 +41,7 @@ static inline dma_addr_t plat_map_dma_mem_page(struct device *dev,
41 struct page *page) 41 struct page *page)
42{ 42{
43#ifdef CONFIG_CPU_LOONGSON3 43#ifdef CONFIG_CPU_LOONGSON3
44 return phys_to_dma(dev, page_to_phys(page)); 44 return __phys_to_dma(dev, page_to_phys(page));
45#else 45#else
46 return page_to_phys(page) | 0x80000000; 46 return page_to_phys(page) | 0x80000000;
47#endif 47#endif
@@ -51,7 +51,7 @@ static inline unsigned long plat_dma_addr_to_phys(struct device *dev,
51 dma_addr_t dma_addr) 51 dma_addr_t dma_addr)
52{ 52{
53#if defined(CONFIG_CPU_LOONGSON3) && defined(CONFIG_64BIT) 53#if defined(CONFIG_CPU_LOONGSON3) && defined(CONFIG_64BIT)
54 return dma_to_phys(dev, dma_addr); 54 return __dma_to_phys(dev, dma_addr);
55#elif defined(CONFIG_CPU_LOONGSON2F) && defined(CONFIG_64BIT) 55#elif defined(CONFIG_CPU_LOONGSON2F) && defined(CONFIG_64BIT)
56 return (dma_addr > 0x8fffffff) ? dma_addr : (dma_addr & 0x0fffffff); 56 return (dma_addr > 0x8fffffff) ? dma_addr : (dma_addr & 0x0fffffff);
57#else 57#else
diff --git a/arch/mips/loongson64/common/dma-swiotlb.c b/arch/mips/loongson64/common/dma-swiotlb.c
index 7bbcf89475f3..6a739f8ae110 100644
--- a/arch/mips/loongson64/common/dma-swiotlb.c
+++ b/arch/mips/loongson64/common/dma-swiotlb.c
@@ -63,7 +63,7 @@ static int loongson_dma_supported(struct device *dev, u64 mask)
63 return swiotlb_dma_supported(dev, mask); 63 return swiotlb_dma_supported(dev, mask);
64} 64}
65 65
66dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr) 66dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t paddr)
67{ 67{
68 long nid; 68 long nid;
69#ifdef CONFIG_PHYS48_TO_HT40 69#ifdef CONFIG_PHYS48_TO_HT40
@@ -75,7 +75,7 @@ dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr)
75 return paddr; 75 return paddr;
76} 76}
77 77
78phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr) 78phys_addr_t __dma_to_phys(struct device *dev, dma_addr_t daddr)
79{ 79{
80 long nid; 80 long nid;
81#ifdef CONFIG_PHYS48_TO_HT40 81#ifdef CONFIG_PHYS48_TO_HT40
diff --git a/arch/powerpc/include/asm/dma-direct.h b/arch/powerpc/include/asm/dma-direct.h
index a5b59c765426..7702875aabb7 100644
--- a/arch/powerpc/include/asm/dma-direct.h
+++ b/arch/powerpc/include/asm/dma-direct.h
@@ -17,12 +17,12 @@ static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size)
17 return addr + size - 1 <= *dev->dma_mask; 17 return addr + size - 1 <= *dev->dma_mask;
18} 18}
19 19
20static inline dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr) 20static inline dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t paddr)
21{ 21{
22 return paddr + get_dma_offset(dev); 22 return paddr + get_dma_offset(dev);
23} 23}
24 24
25static inline phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr) 25static inline phys_addr_t __dma_to_phys(struct device *dev, dma_addr_t daddr)
26{ 26{
27 return daddr - get_dma_offset(dev); 27 return daddr - get_dma_offset(dev);
28} 28}
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index a0fb8bc346d5..27fede438959 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -54,7 +54,6 @@ config X86
54 select ARCH_HAS_FORTIFY_SOURCE 54 select ARCH_HAS_FORTIFY_SOURCE
55 select ARCH_HAS_GCOV_PROFILE_ALL 55 select ARCH_HAS_GCOV_PROFILE_ALL
56 select ARCH_HAS_KCOV if X86_64 56 select ARCH_HAS_KCOV if X86_64
57 select ARCH_HAS_PHYS_TO_DMA
58 select ARCH_HAS_MEMBARRIER_SYNC_CORE 57 select ARCH_HAS_MEMBARRIER_SYNC_CORE
59 select ARCH_HAS_PMEM_API if X86_64 58 select ARCH_HAS_PMEM_API if X86_64
60 select ARCH_HAS_REFCOUNT 59 select ARCH_HAS_REFCOUNT
@@ -83,6 +82,7 @@ config X86
83 select CLOCKSOURCE_VALIDATE_LAST_CYCLE 82 select CLOCKSOURCE_VALIDATE_LAST_CYCLE
84 select CLOCKSOURCE_WATCHDOG 83 select CLOCKSOURCE_WATCHDOG
85 select DCACHE_WORD_ACCESS 84 select DCACHE_WORD_ACCESS
85 select DMA_DIRECT_OPS
86 select EDAC_ATOMIC_SCRUB 86 select EDAC_ATOMIC_SCRUB
87 select EDAC_SUPPORT 87 select EDAC_SUPPORT
88 select GENERIC_CLOCKEVENTS 88 select GENERIC_CLOCKEVENTS
@@ -680,6 +680,7 @@ config X86_SUPPORTS_MEMORY_FAILURE
680config STA2X11 680config STA2X11
681 bool "STA2X11 Companion Chip Support" 681 bool "STA2X11 Companion Chip Support"
682 depends on X86_32_NON_STANDARD && PCI 682 depends on X86_32_NON_STANDARD && PCI
683 select ARCH_HAS_PHYS_TO_DMA
683 select X86_DEV_DMA_OPS 684 select X86_DEV_DMA_OPS
684 select X86_DMA_REMAP 685 select X86_DMA_REMAP
685 select SWIOTLB 686 select SWIOTLB
diff --git a/arch/x86/include/asm/device.h b/arch/x86/include/asm/device.h
index 5e12c63b47aa..a8f6c809d9b1 100644
--- a/arch/x86/include/asm/device.h
+++ b/arch/x86/include/asm/device.h
@@ -6,6 +6,9 @@ struct dev_archdata {
6#if defined(CONFIG_INTEL_IOMMU) || defined(CONFIG_AMD_IOMMU) 6#if defined(CONFIG_INTEL_IOMMU) || defined(CONFIG_AMD_IOMMU)
7 void *iommu; /* hook for IOMMU specific extension */ 7 void *iommu; /* hook for IOMMU specific extension */
8#endif 8#endif
9#ifdef CONFIG_STA2X11
10 bool is_sta2x11;
11#endif
9}; 12};
10 13
11#if defined(CONFIG_X86_DEV_DMA_OPS) && defined(CONFIG_PCI_DOMAINS) 14#if defined(CONFIG_X86_DEV_DMA_OPS) && defined(CONFIG_PCI_DOMAINS)
diff --git a/arch/x86/include/asm/dma-direct.h b/arch/x86/include/asm/dma-direct.h
index 1295bc622ebe..1a19251eaac9 100644
--- a/arch/x86/include/asm/dma-direct.h
+++ b/arch/x86/include/asm/dma-direct.h
@@ -2,29 +2,8 @@
2#ifndef ASM_X86_DMA_DIRECT_H 2#ifndef ASM_X86_DMA_DIRECT_H
3#define ASM_X86_DMA_DIRECT_H 1 3#define ASM_X86_DMA_DIRECT_H 1
4 4
5#include <linux/mem_encrypt.h>
6
7#ifdef CONFIG_X86_DMA_REMAP /* Platform code defines bridge-specific code */
8bool dma_capable(struct device *dev, dma_addr_t addr, size_t size); 5bool dma_capable(struct device *dev, dma_addr_t addr, size_t size);
9dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr); 6dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t paddr);
10phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr); 7phys_addr_t __dma_to_phys(struct device *dev, dma_addr_t daddr);
11#else
12static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size)
13{
14 if (!dev->dma_mask)
15 return 0;
16
17 return addr + size - 1 <= *dev->dma_mask;
18}
19
20static inline dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr)
21{
22 return __sme_set(paddr);
23}
24 8
25static inline phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr)
26{
27 return __sme_clr(daddr);
28}
29#endif /* CONFIG_X86_DMA_REMAP */
30#endif /* ASM_X86_DMA_DIRECT_H */ 9#endif /* ASM_X86_DMA_DIRECT_H */
diff --git a/arch/x86/include/asm/dma-mapping.h b/arch/x86/include/asm/dma-mapping.h
index 6277c83c0eb1..89ce4bfd241f 100644
--- a/arch/x86/include/asm/dma-mapping.h
+++ b/arch/x86/include/asm/dma-mapping.h
@@ -36,37 +36,4 @@ int arch_dma_supported(struct device *dev, u64 mask);
36bool arch_dma_alloc_attrs(struct device **dev, gfp_t *gfp); 36bool arch_dma_alloc_attrs(struct device **dev, gfp_t *gfp);
37#define arch_dma_alloc_attrs arch_dma_alloc_attrs 37#define arch_dma_alloc_attrs arch_dma_alloc_attrs
38 38
39extern void *dma_generic_alloc_coherent(struct device *dev, size_t size,
40 dma_addr_t *dma_addr, gfp_t flag,
41 unsigned long attrs);
42
43extern void dma_generic_free_coherent(struct device *dev, size_t size,
44 void *vaddr, dma_addr_t dma_addr,
45 unsigned long attrs);
46
47static inline unsigned long dma_alloc_coherent_mask(struct device *dev,
48 gfp_t gfp)
49{
50 unsigned long dma_mask = 0;
51
52 dma_mask = dev->coherent_dma_mask;
53 if (!dma_mask)
54 dma_mask = (gfp & GFP_DMA) ? DMA_BIT_MASK(24) : DMA_BIT_MASK(32);
55
56 return dma_mask;
57}
58
59static inline gfp_t dma_alloc_coherent_gfp_flags(struct device *dev, gfp_t gfp)
60{
61 unsigned long dma_mask = dma_alloc_coherent_mask(dev, gfp);
62
63 if (dma_mask <= DMA_BIT_MASK(24))
64 gfp |= GFP_DMA;
65#ifdef CONFIG_X86_64
66 if (dma_mask <= DMA_BIT_MASK(32) && !(gfp & GFP_DMA))
67 gfp |= GFP_DMA32;
68#endif
69 return gfp;
70}
71
72#endif 39#endif
diff --git a/arch/x86/include/asm/iommu.h b/arch/x86/include/asm/iommu.h
index 1e5d5d92eb40..baedab8ac538 100644
--- a/arch/x86/include/asm/iommu.h
+++ b/arch/x86/include/asm/iommu.h
@@ -2,13 +2,10 @@
2#ifndef _ASM_X86_IOMMU_H 2#ifndef _ASM_X86_IOMMU_H
3#define _ASM_X86_IOMMU_H 3#define _ASM_X86_IOMMU_H
4 4
5extern const struct dma_map_ops nommu_dma_ops;
6extern int force_iommu, no_iommu; 5extern int force_iommu, no_iommu;
7extern int iommu_detected; 6extern int iommu_detected;
8extern int iommu_pass_through; 7extern int iommu_pass_through;
9 8
10int x86_dma_supported(struct device *dev, u64 mask);
11
12/* 10 seconds */ 9/* 10 seconds */
13#define DMAR_OPERATION_TIMEOUT ((cycles_t) tsc_khz*10*1000) 10#define DMAR_OPERATION_TIMEOUT ((cycles_t) tsc_khz*10*1000)
14 11
diff --git a/arch/x86/include/asm/mem_encrypt.h b/arch/x86/include/asm/mem_encrypt.h
index 8fe61ad21047..c0643831706e 100644
--- a/arch/x86/include/asm/mem_encrypt.h
+++ b/arch/x86/include/asm/mem_encrypt.h
@@ -49,8 +49,6 @@ int __init early_set_memory_encrypted(unsigned long vaddr, unsigned long size);
49/* Architecture __weak replacement functions */ 49/* Architecture __weak replacement functions */
50void __init mem_encrypt_init(void); 50void __init mem_encrypt_init(void);
51 51
52void swiotlb_set_mem_attributes(void *vaddr, unsigned long size);
53
54bool sme_active(void); 52bool sme_active(void);
55bool sev_active(void); 53bool sev_active(void);
56 54
diff --git a/arch/x86/include/asm/swiotlb.h b/arch/x86/include/asm/swiotlb.h
index 1c6a6cb230ff..ff6c92eff035 100644
--- a/arch/x86/include/asm/swiotlb.h
+++ b/arch/x86/include/asm/swiotlb.h
@@ -27,12 +27,4 @@ static inline void pci_swiotlb_late_init(void)
27{ 27{
28} 28}
29#endif 29#endif
30
31extern void *x86_swiotlb_alloc_coherent(struct device *hwdev, size_t size,
32 dma_addr_t *dma_handle, gfp_t flags,
33 unsigned long attrs);
34extern void x86_swiotlb_free_coherent(struct device *dev, size_t size,
35 void *vaddr, dma_addr_t dma_addr,
36 unsigned long attrs);
37
38#endif /* _ASM_X86_SWIOTLB_H */ 30#endif /* _ASM_X86_SWIOTLB_H */
diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile
index 73ccf80c09a2..02d6f5cf4e70 100644
--- a/arch/x86/kernel/Makefile
+++ b/arch/x86/kernel/Makefile
@@ -57,7 +57,7 @@ obj-$(CONFIG_X86_ESPFIX64) += espfix_64.o
57obj-$(CONFIG_SYSFS) += ksysfs.o 57obj-$(CONFIG_SYSFS) += ksysfs.o
58obj-y += bootflag.o e820.o 58obj-y += bootflag.o e820.o
59obj-y += pci-dma.o quirks.o topology.o kdebugfs.o 59obj-y += pci-dma.o quirks.o topology.o kdebugfs.o
60obj-y += alternative.o i8253.o pci-nommu.o hw_breakpoint.o 60obj-y += alternative.o i8253.o hw_breakpoint.o
61obj-y += tsc.o tsc_msr.o io_delay.o rtc.o 61obj-y += tsc.o tsc_msr.o io_delay.o rtc.o
62obj-y += pci-iommu_table.o 62obj-y += pci-iommu_table.o
63obj-y += resource.o 63obj-y += resource.o
diff --git a/arch/x86/kernel/amd_gart_64.c b/arch/x86/kernel/amd_gart_64.c
index ecd486cb06ab..f299d8a479bb 100644
--- a/arch/x86/kernel/amd_gart_64.c
+++ b/arch/x86/kernel/amd_gart_64.c
@@ -480,30 +480,21 @@ static void *
480gart_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_addr, 480gart_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_addr,
481 gfp_t flag, unsigned long attrs) 481 gfp_t flag, unsigned long attrs)
482{ 482{
483 dma_addr_t paddr; 483 void *vaddr;
484 unsigned long align_mask; 484
485 struct page *page; 485 vaddr = dma_direct_alloc(dev, size, dma_addr, flag, attrs);
486 486 if (!vaddr ||
487 if (force_iommu && !(flag & GFP_DMA)) { 487 !force_iommu || dev->coherent_dma_mask <= DMA_BIT_MASK(24))
488 flag &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32); 488 return vaddr;
489 page = alloc_pages(flag | __GFP_ZERO, get_order(size));
490 if (!page)
491 return NULL;
492
493 align_mask = (1UL << get_order(size)) - 1;
494 paddr = dma_map_area(dev, page_to_phys(page), size,
495 DMA_BIDIRECTIONAL, align_mask);
496
497 flush_gart();
498 if (paddr != bad_dma_addr) {
499 *dma_addr = paddr;
500 return page_address(page);
501 }
502 __free_pages(page, get_order(size));
503 } else
504 return dma_generic_alloc_coherent(dev, size, dma_addr, flag,
505 attrs);
506 489
490 *dma_addr = dma_map_area(dev, virt_to_phys(vaddr), size,
491 DMA_BIDIRECTIONAL, (1UL << get_order(size)) - 1);
492 flush_gart();
493 if (unlikely(*dma_addr == bad_dma_addr))
494 goto out_free;
495 return vaddr;
496out_free:
497 dma_direct_free(dev, size, vaddr, *dma_addr, attrs);
507 return NULL; 498 return NULL;
508} 499}
509 500
@@ -513,7 +504,7 @@ gart_free_coherent(struct device *dev, size_t size, void *vaddr,
513 dma_addr_t dma_addr, unsigned long attrs) 504 dma_addr_t dma_addr, unsigned long attrs)
514{ 505{
515 gart_unmap_page(dev, dma_addr, size, DMA_BIDIRECTIONAL, 0); 506 gart_unmap_page(dev, dma_addr, size, DMA_BIDIRECTIONAL, 0);
516 dma_generic_free_coherent(dev, size, vaddr, dma_addr, attrs); 507 dma_direct_free(dev, size, vaddr, dma_addr, attrs);
517} 508}
518 509
519static int gart_mapping_error(struct device *dev, dma_addr_t dma_addr) 510static int gart_mapping_error(struct device *dev, dma_addr_t dma_addr)
@@ -705,7 +696,7 @@ static const struct dma_map_ops gart_dma_ops = {
705 .alloc = gart_alloc_coherent, 696 .alloc = gart_alloc_coherent,
706 .free = gart_free_coherent, 697 .free = gart_free_coherent,
707 .mapping_error = gart_mapping_error, 698 .mapping_error = gart_mapping_error,
708 .dma_supported = x86_dma_supported, 699 .dma_supported = dma_direct_supported,
709}; 700};
710 701
711static void gart_iommu_shutdown(void) 702static void gart_iommu_shutdown(void)
diff --git a/arch/x86/kernel/pci-calgary_64.c b/arch/x86/kernel/pci-calgary_64.c
index 35c461f21815..bbfc8b1e9104 100644
--- a/arch/x86/kernel/pci-calgary_64.c
+++ b/arch/x86/kernel/pci-calgary_64.c
@@ -33,6 +33,7 @@
33#include <linux/string.h> 33#include <linux/string.h>
34#include <linux/crash_dump.h> 34#include <linux/crash_dump.h>
35#include <linux/dma-mapping.h> 35#include <linux/dma-mapping.h>
36#include <linux/dma-direct.h>
36#include <linux/bitmap.h> 37#include <linux/bitmap.h>
37#include <linux/pci_ids.h> 38#include <linux/pci_ids.h>
38#include <linux/pci.h> 39#include <linux/pci.h>
@@ -445,8 +446,6 @@ static void* calgary_alloc_coherent(struct device *dev, size_t size,
445 npages = size >> PAGE_SHIFT; 446 npages = size >> PAGE_SHIFT;
446 order = get_order(size); 447 order = get_order(size);
447 448
448 flag &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32);
449
450 /* alloc enough pages (and possibly more) */ 449 /* alloc enough pages (and possibly more) */
451 ret = (void *)__get_free_pages(flag, order); 450 ret = (void *)__get_free_pages(flag, order);
452 if (!ret) 451 if (!ret)
@@ -493,7 +492,7 @@ static const struct dma_map_ops calgary_dma_ops = {
493 .map_page = calgary_map_page, 492 .map_page = calgary_map_page,
494 .unmap_page = calgary_unmap_page, 493 .unmap_page = calgary_unmap_page,
495 .mapping_error = calgary_mapping_error, 494 .mapping_error = calgary_mapping_error,
496 .dma_supported = x86_dma_supported, 495 .dma_supported = dma_direct_supported,
497}; 496};
498 497
499static inline void __iomem * busno_to_bbar(unsigned char num) 498static inline void __iomem * busno_to_bbar(unsigned char num)
diff --git a/arch/x86/kernel/pci-dma.c b/arch/x86/kernel/pci-dma.c
index df7ab02f959f..14437116ffea 100644
--- a/arch/x86/kernel/pci-dma.c
+++ b/arch/x86/kernel/pci-dma.c
@@ -18,7 +18,7 @@
18 18
19static int forbid_dac __read_mostly; 19static int forbid_dac __read_mostly;
20 20
21const struct dma_map_ops *dma_ops = &nommu_dma_ops; 21const struct dma_map_ops *dma_ops = &dma_direct_ops;
22EXPORT_SYMBOL(dma_ops); 22EXPORT_SYMBOL(dma_ops);
23 23
24static int iommu_sac_force __read_mostly; 24static int iommu_sac_force __read_mostly;
@@ -76,70 +76,12 @@ void __init pci_iommu_alloc(void)
76 } 76 }
77 } 77 }
78} 78}
79void *dma_generic_alloc_coherent(struct device *dev, size_t size,
80 dma_addr_t *dma_addr, gfp_t flag,
81 unsigned long attrs)
82{
83 unsigned long dma_mask;
84 struct page *page;
85 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
86 dma_addr_t addr;
87
88 dma_mask = dma_alloc_coherent_mask(dev, flag);
89
90again:
91 page = NULL;
92 /* CMA can be used only in the context which permits sleeping */
93 if (gfpflags_allow_blocking(flag)) {
94 page = dma_alloc_from_contiguous(dev, count, get_order(size),
95 flag);
96 if (page) {
97 addr = phys_to_dma(dev, page_to_phys(page));
98 if (addr + size > dma_mask) {
99 dma_release_from_contiguous(dev, page, count);
100 page = NULL;
101 }
102 }
103 }
104 /* fallback */
105 if (!page)
106 page = alloc_pages_node(dev_to_node(dev), flag, get_order(size));
107 if (!page)
108 return NULL;
109
110 addr = phys_to_dma(dev, page_to_phys(page));
111 if (addr + size > dma_mask) {
112 __free_pages(page, get_order(size));
113
114 if (dma_mask < DMA_BIT_MASK(32) && !(flag & GFP_DMA)) {
115 flag = (flag & ~GFP_DMA32) | GFP_DMA;
116 goto again;
117 }
118
119 return NULL;
120 }
121 memset(page_address(page), 0, size);
122 *dma_addr = addr;
123 return page_address(page);
124}
125
126void dma_generic_free_coherent(struct device *dev, size_t size, void *vaddr,
127 dma_addr_t dma_addr, unsigned long attrs)
128{
129 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
130 struct page *page = virt_to_page(vaddr);
131
132 if (!dma_release_from_contiguous(dev, page, count))
133 free_pages((unsigned long)vaddr, get_order(size));
134}
135 79
136bool arch_dma_alloc_attrs(struct device **dev, gfp_t *gfp) 80bool arch_dma_alloc_attrs(struct device **dev, gfp_t *gfp)
137{ 81{
138 if (!*dev) 82 if (!*dev)
139 *dev = &x86_dma_fallback_dev; 83 *dev = &x86_dma_fallback_dev;
140 84
141 *gfp = dma_alloc_coherent_gfp_flags(*dev, *gfp);
142
143 if (!is_device_dma_capable(*dev)) 85 if (!is_device_dma_capable(*dev))
144 return false; 86 return false;
145 return true; 87 return true;
@@ -245,16 +187,6 @@ int arch_dma_supported(struct device *dev, u64 mask)
245} 187}
246EXPORT_SYMBOL(arch_dma_supported); 188EXPORT_SYMBOL(arch_dma_supported);
247 189
248int x86_dma_supported(struct device *dev, u64 mask)
249{
250 /* Copied from i386. Doesn't make much sense, because it will
251 only work for pci_alloc_coherent.
252 The caller just has to use GFP_DMA in this case. */
253 if (mask < DMA_BIT_MASK(24))
254 return 0;
255 return 1;
256}
257
258static int __init pci_iommu_init(void) 190static int __init pci_iommu_init(void)
259{ 191{
260 struct iommu_table_entry *p; 192 struct iommu_table_entry *p;
diff --git a/arch/x86/kernel/pci-swiotlb.c b/arch/x86/kernel/pci-swiotlb.c
index 0ee0f8f34251..661583662430 100644
--- a/arch/x86/kernel/pci-swiotlb.c
+++ b/arch/x86/kernel/pci-swiotlb.c
@@ -17,52 +17,6 @@
17 17
18int swiotlb __read_mostly; 18int swiotlb __read_mostly;
19 19
20void *x86_swiotlb_alloc_coherent(struct device *hwdev, size_t size,
21 dma_addr_t *dma_handle, gfp_t flags,
22 unsigned long attrs)
23{
24 void *vaddr;
25
26 /*
27 * Don't print a warning when the first allocation attempt fails.
28 * swiotlb_alloc_coherent() will print a warning when the DMA
29 * memory allocation ultimately failed.
30 */
31 flags |= __GFP_NOWARN;
32
33 vaddr = dma_generic_alloc_coherent(hwdev, size, dma_handle, flags,
34 attrs);
35 if (vaddr)
36 return vaddr;
37
38 return swiotlb_alloc_coherent(hwdev, size, dma_handle, flags);
39}
40
41void x86_swiotlb_free_coherent(struct device *dev, size_t size,
42 void *vaddr, dma_addr_t dma_addr,
43 unsigned long attrs)
44{
45 if (is_swiotlb_buffer(dma_to_phys(dev, dma_addr)))
46 swiotlb_free_coherent(dev, size, vaddr, dma_addr);
47 else
48 dma_generic_free_coherent(dev, size, vaddr, dma_addr, attrs);
49}
50
51static const struct dma_map_ops x86_swiotlb_dma_ops = {
52 .mapping_error = swiotlb_dma_mapping_error,
53 .alloc = x86_swiotlb_alloc_coherent,
54 .free = x86_swiotlb_free_coherent,
55 .sync_single_for_cpu = swiotlb_sync_single_for_cpu,
56 .sync_single_for_device = swiotlb_sync_single_for_device,
57 .sync_sg_for_cpu = swiotlb_sync_sg_for_cpu,
58 .sync_sg_for_device = swiotlb_sync_sg_for_device,
59 .map_sg = swiotlb_map_sg_attrs,
60 .unmap_sg = swiotlb_unmap_sg_attrs,
61 .map_page = swiotlb_map_page,
62 .unmap_page = swiotlb_unmap_page,
63 .dma_supported = NULL,
64};
65
66/* 20/*
67 * pci_swiotlb_detect_override - set swiotlb to 1 if necessary 21 * pci_swiotlb_detect_override - set swiotlb to 1 if necessary
68 * 22 *
@@ -112,7 +66,7 @@ void __init pci_swiotlb_init(void)
112{ 66{
113 if (swiotlb) { 67 if (swiotlb) {
114 swiotlb_init(0); 68 swiotlb_init(0);
115 dma_ops = &x86_swiotlb_dma_ops; 69 dma_ops = &swiotlb_dma_ops;
116 } 70 }
117} 71}
118 72
diff --git a/arch/x86/mm/mem_encrypt.c b/arch/x86/mm/mem_encrypt.c
index 3a1b5fe4c2ca..b2de398d1fd3 100644
--- a/arch/x86/mm/mem_encrypt.c
+++ b/arch/x86/mm/mem_encrypt.c
@@ -195,67 +195,6 @@ void __init sme_early_init(void)
195 swiotlb_force = SWIOTLB_FORCE; 195 swiotlb_force = SWIOTLB_FORCE;
196} 196}
197 197
198static void *sev_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
199 gfp_t gfp, unsigned long attrs)
200{
201 unsigned long dma_mask;
202 unsigned int order;
203 struct page *page;
204 void *vaddr = NULL;
205
206 dma_mask = dma_alloc_coherent_mask(dev, gfp);
207 order = get_order(size);
208
209 /*
210 * Memory will be memset to zero after marking decrypted, so don't
211 * bother clearing it before.
212 */
213 gfp &= ~__GFP_ZERO;
214
215 page = alloc_pages_node(dev_to_node(dev), gfp, order);
216 if (page) {
217 dma_addr_t addr;
218
219 /*
220 * Since we will be clearing the encryption bit, check the
221 * mask with it already cleared.
222 */
223 addr = __sme_clr(phys_to_dma(dev, page_to_phys(page)));
224 if ((addr + size) > dma_mask) {
225 __free_pages(page, get_order(size));
226 } else {
227 vaddr = page_address(page);
228 *dma_handle = addr;
229 }
230 }
231
232 if (!vaddr)
233 vaddr = swiotlb_alloc_coherent(dev, size, dma_handle, gfp);
234
235 if (!vaddr)
236 return NULL;
237
238 /* Clear the SME encryption bit for DMA use if not swiotlb area */
239 if (!is_swiotlb_buffer(dma_to_phys(dev, *dma_handle))) {
240 set_memory_decrypted((unsigned long)vaddr, 1 << order);
241 memset(vaddr, 0, PAGE_SIZE << order);
242 *dma_handle = __sme_clr(*dma_handle);
243 }
244
245 return vaddr;
246}
247
248static void sev_free(struct device *dev, size_t size, void *vaddr,
249 dma_addr_t dma_handle, unsigned long attrs)
250{
251 /* Set the SME encryption bit for re-use if not swiotlb area */
252 if (!is_swiotlb_buffer(dma_to_phys(dev, dma_handle)))
253 set_memory_encrypted((unsigned long)vaddr,
254 1 << get_order(size));
255
256 swiotlb_free_coherent(dev, size, vaddr, dma_handle);
257}
258
259static void __init __set_clr_pte_enc(pte_t *kpte, int level, bool enc) 198static void __init __set_clr_pte_enc(pte_t *kpte, int level, bool enc)
260{ 199{
261 pgprot_t old_prot, new_prot; 200 pgprot_t old_prot, new_prot;
@@ -408,20 +347,6 @@ bool sev_active(void)
408} 347}
409EXPORT_SYMBOL(sev_active); 348EXPORT_SYMBOL(sev_active);
410 349
411static const struct dma_map_ops sev_dma_ops = {
412 .alloc = sev_alloc,
413 .free = sev_free,
414 .map_page = swiotlb_map_page,
415 .unmap_page = swiotlb_unmap_page,
416 .map_sg = swiotlb_map_sg_attrs,
417 .unmap_sg = swiotlb_unmap_sg_attrs,
418 .sync_single_for_cpu = swiotlb_sync_single_for_cpu,
419 .sync_single_for_device = swiotlb_sync_single_for_device,
420 .sync_sg_for_cpu = swiotlb_sync_sg_for_cpu,
421 .sync_sg_for_device = swiotlb_sync_sg_for_device,
422 .mapping_error = swiotlb_dma_mapping_error,
423};
424
425/* Architecture __weak replacement functions */ 350/* Architecture __weak replacement functions */
426void __init mem_encrypt_init(void) 351void __init mem_encrypt_init(void)
427{ 352{
@@ -432,12 +357,11 @@ void __init mem_encrypt_init(void)
432 swiotlb_update_mem_attributes(); 357 swiotlb_update_mem_attributes();
433 358
434 /* 359 /*
435 * With SEV, DMA operations cannot use encryption. New DMA ops 360 * With SEV, DMA operations cannot use encryption, we need to use
436 * are required in order to mark the DMA areas as decrypted or 361 * SWIOTLB to bounce buffer DMA operation.
437 * to use bounce buffers.
438 */ 362 */
439 if (sev_active()) 363 if (sev_active())
440 dma_ops = &sev_dma_ops; 364 dma_ops = &swiotlb_dma_ops;
441 365
442 /* 366 /*
443 * With SEV, we need to unroll the rep string I/O instructions. 367 * With SEV, we need to unroll the rep string I/O instructions.
@@ -450,11 +374,3 @@ void __init mem_encrypt_init(void)
450 : "Secure Memory Encryption (SME)"); 374 : "Secure Memory Encryption (SME)");
451} 375}
452 376
453void swiotlb_set_mem_attributes(void *vaddr, unsigned long size)
454{
455 WARN(PAGE_ALIGN(size) != size,
456 "size is not page-aligned (%#lx)\n", size);
457
458 /* Make the SWIOTLB buffer area decrypted */
459 set_memory_decrypted((unsigned long)vaddr, size >> PAGE_SHIFT);
460}
diff --git a/arch/x86/pci/sta2x11-fixup.c b/arch/x86/pci/sta2x11-fixup.c
index 75577c1490c4..7a5bafb76d77 100644
--- a/arch/x86/pci/sta2x11-fixup.c
+++ b/arch/x86/pci/sta2x11-fixup.c
@@ -159,43 +159,6 @@ static dma_addr_t a2p(dma_addr_t a, struct pci_dev *pdev)
159 return p; 159 return p;
160} 160}
161 161
162/**
163 * sta2x11_swiotlb_alloc_coherent - Allocate swiotlb bounce buffers
164 * returns virtual address. This is the only "special" function here.
165 * @dev: PCI device
166 * @size: Size of the buffer
167 * @dma_handle: DMA address
168 * @flags: memory flags
169 */
170static void *sta2x11_swiotlb_alloc_coherent(struct device *dev,
171 size_t size,
172 dma_addr_t *dma_handle,
173 gfp_t flags,
174 unsigned long attrs)
175{
176 void *vaddr;
177
178 vaddr = x86_swiotlb_alloc_coherent(dev, size, dma_handle, flags, attrs);
179 *dma_handle = p2a(*dma_handle, to_pci_dev(dev));
180 return vaddr;
181}
182
183/* We have our own dma_ops: the same as swiotlb but from alloc (above) */
184static const struct dma_map_ops sta2x11_dma_ops = {
185 .alloc = sta2x11_swiotlb_alloc_coherent,
186 .free = x86_swiotlb_free_coherent,
187 .map_page = swiotlb_map_page,
188 .unmap_page = swiotlb_unmap_page,
189 .map_sg = swiotlb_map_sg_attrs,
190 .unmap_sg = swiotlb_unmap_sg_attrs,
191 .sync_single_for_cpu = swiotlb_sync_single_for_cpu,
192 .sync_single_for_device = swiotlb_sync_single_for_device,
193 .sync_sg_for_cpu = swiotlb_sync_sg_for_cpu,
194 .sync_sg_for_device = swiotlb_sync_sg_for_device,
195 .mapping_error = swiotlb_dma_mapping_error,
196 .dma_supported = x86_dma_supported,
197};
198
199/* At setup time, we use our own ops if the device is a ConneXt one */ 162/* At setup time, we use our own ops if the device is a ConneXt one */
200static void sta2x11_setup_pdev(struct pci_dev *pdev) 163static void sta2x11_setup_pdev(struct pci_dev *pdev)
201{ 164{
@@ -205,7 +168,8 @@ static void sta2x11_setup_pdev(struct pci_dev *pdev)
205 return; 168 return;
206 pci_set_consistent_dma_mask(pdev, STA2X11_AMBA_SIZE - 1); 169 pci_set_consistent_dma_mask(pdev, STA2X11_AMBA_SIZE - 1);
207 pci_set_dma_mask(pdev, STA2X11_AMBA_SIZE - 1); 170 pci_set_dma_mask(pdev, STA2X11_AMBA_SIZE - 1);
208 pdev->dev.dma_ops = &sta2x11_dma_ops; 171 pdev->dev.dma_ops = &swiotlb_dma_ops;
172 pdev->dev.archdata.is_sta2x11 = true;
209 173
210 /* We must enable all devices as master, for audio DMA to work */ 174 /* We must enable all devices as master, for audio DMA to work */
211 pci_set_master(pdev); 175 pci_set_master(pdev);
@@ -225,7 +189,7 @@ bool dma_capable(struct device *dev, dma_addr_t addr, size_t size)
225{ 189{
226 struct sta2x11_mapping *map; 190 struct sta2x11_mapping *map;
227 191
228 if (dev->dma_ops != &sta2x11_dma_ops) { 192 if (!dev->archdata.is_sta2x11) {
229 if (!dev->dma_mask) 193 if (!dev->dma_mask)
230 return false; 194 return false;
231 return addr + size - 1 <= *dev->dma_mask; 195 return addr + size - 1 <= *dev->dma_mask;
@@ -243,13 +207,13 @@ bool dma_capable(struct device *dev, dma_addr_t addr, size_t size)
243} 207}
244 208
245/** 209/**
246 * phys_to_dma - Return the DMA AMBA address used for this STA2x11 device 210 * __phys_to_dma - Return the DMA AMBA address used for this STA2x11 device
247 * @dev: device for a PCI device 211 * @dev: device for a PCI device
248 * @paddr: Physical address 212 * @paddr: Physical address
249 */ 213 */
250dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr) 214dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t paddr)
251{ 215{
252 if (dev->dma_ops != &sta2x11_dma_ops) 216 if (!dev->archdata.is_sta2x11)
253 return paddr; 217 return paddr;
254 return p2a(paddr, to_pci_dev(dev)); 218 return p2a(paddr, to_pci_dev(dev));
255} 219}
@@ -259,9 +223,9 @@ dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr)
259 * @dev: device for a PCI device 223 * @dev: device for a PCI device
260 * @daddr: STA2x11 AMBA DMA address 224 * @daddr: STA2x11 AMBA DMA address
261 */ 225 */
262phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr) 226phys_addr_t __dma_to_phys(struct device *dev, dma_addr_t daddr)
263{ 227{
264 if (dev->dma_ops != &sta2x11_dma_ops) 228 if (!dev->archdata.is_sta2x11)
265 return daddr; 229 return daddr;
266 return a2p(daddr, to_pci_dev(dev)); 230 return a2p(daddr, to_pci_dev(dev));
267} 231}
diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig
index f3a21343e636..df171cb85822 100644
--- a/drivers/iommu/Kconfig
+++ b/drivers/iommu/Kconfig
@@ -107,6 +107,7 @@ config IOMMU_PGTABLES_L2
107# AMD IOMMU support 107# AMD IOMMU support
108config AMD_IOMMU 108config AMD_IOMMU
109 bool "AMD IOMMU support" 109 bool "AMD IOMMU support"
110 select DMA_DIRECT_OPS
110 select SWIOTLB 111 select SWIOTLB
111 select PCI_MSI 112 select PCI_MSI
112 select PCI_ATS 113 select PCI_ATS
@@ -142,6 +143,7 @@ config DMAR_TABLE
142config INTEL_IOMMU 143config INTEL_IOMMU
143 bool "Support for Intel IOMMU using DMA Remapping Devices" 144 bool "Support for Intel IOMMU using DMA Remapping Devices"
144 depends on PCI_MSI && ACPI && (X86 || IA64_GENERIC) 145 depends on PCI_MSI && ACPI && (X86 || IA64_GENERIC)
146 select DMA_DIRECT_OPS
145 select IOMMU_API 147 select IOMMU_API
146 select IOMMU_IOVA 148 select IOMMU_IOVA
147 select DMAR_TABLE 149 select DMAR_TABLE
diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c
index 74788fdeb773..83819d0cbf90 100644
--- a/drivers/iommu/amd_iommu.c
+++ b/drivers/iommu/amd_iommu.c
@@ -28,6 +28,7 @@
28#include <linux/debugfs.h> 28#include <linux/debugfs.h>
29#include <linux/scatterlist.h> 29#include <linux/scatterlist.h>
30#include <linux/dma-mapping.h> 30#include <linux/dma-mapping.h>
31#include <linux/dma-direct.h>
31#include <linux/iommu-helper.h> 32#include <linux/iommu-helper.h>
32#include <linux/iommu.h> 33#include <linux/iommu.h>
33#include <linux/delay.h> 34#include <linux/delay.h>
@@ -2193,7 +2194,7 @@ static int amd_iommu_add_device(struct device *dev)
2193 dev_name(dev)); 2194 dev_name(dev));
2194 2195
2195 iommu_ignore_device(dev); 2196 iommu_ignore_device(dev);
2196 dev->dma_ops = &nommu_dma_ops; 2197 dev->dma_ops = &dma_direct_ops;
2197 goto out; 2198 goto out;
2198 } 2199 }
2199 init_iommu_group(dev); 2200 init_iommu_group(dev);
@@ -2599,51 +2600,32 @@ static void *alloc_coherent(struct device *dev, size_t size,
2599 unsigned long attrs) 2600 unsigned long attrs)
2600{ 2601{
2601 u64 dma_mask = dev->coherent_dma_mask; 2602 u64 dma_mask = dev->coherent_dma_mask;
2602 struct protection_domain *domain; 2603 struct protection_domain *domain = get_domain(dev);
2603 struct dma_ops_domain *dma_dom; 2604 bool is_direct = false;
2604 struct page *page; 2605 void *virt_addr;
2605
2606 domain = get_domain(dev);
2607 if (PTR_ERR(domain) == -EINVAL) {
2608 page = alloc_pages(flag, get_order(size));
2609 *dma_addr = page_to_phys(page);
2610 return page_address(page);
2611 } else if (IS_ERR(domain))
2612 return NULL;
2613 2606
2614 dma_dom = to_dma_ops_domain(domain); 2607 if (IS_ERR(domain)) {
2615 size = PAGE_ALIGN(size); 2608 if (PTR_ERR(domain) != -EINVAL)
2616 dma_mask = dev->coherent_dma_mask;
2617 flag &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32);
2618 flag |= __GFP_ZERO;
2619
2620 page = alloc_pages(flag | __GFP_NOWARN, get_order(size));
2621 if (!page) {
2622 if (!gfpflags_allow_blocking(flag))
2623 return NULL;
2624
2625 page = dma_alloc_from_contiguous(dev, size >> PAGE_SHIFT,
2626 get_order(size), flag);
2627 if (!page)
2628 return NULL; 2609 return NULL;
2610 is_direct = true;
2629 } 2611 }
2630 2612
2613 virt_addr = dma_direct_alloc(dev, size, dma_addr, flag, attrs);
2614 if (!virt_addr || is_direct)
2615 return virt_addr;
2616
2631 if (!dma_mask) 2617 if (!dma_mask)
2632 dma_mask = *dev->dma_mask; 2618 dma_mask = *dev->dma_mask;
2633 2619
2634 *dma_addr = __map_single(dev, dma_dom, page_to_phys(page), 2620 *dma_addr = __map_single(dev, to_dma_ops_domain(domain),
2635 size, DMA_BIDIRECTIONAL, dma_mask); 2621 virt_to_phys(virt_addr), PAGE_ALIGN(size),
2636 2622 DMA_BIDIRECTIONAL, dma_mask);
2637 if (*dma_addr == AMD_IOMMU_MAPPING_ERROR) 2623 if (*dma_addr == AMD_IOMMU_MAPPING_ERROR)
2638 goto out_free; 2624 goto out_free;
2639 2625 return virt_addr;
2640 return page_address(page);
2641 2626
2642out_free: 2627out_free:
2643 2628 dma_direct_free(dev, size, virt_addr, *dma_addr, attrs);
2644 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
2645 __free_pages(page, get_order(size));
2646
2647 return NULL; 2629 return NULL;
2648} 2630}
2649 2631
@@ -2654,24 +2636,17 @@ static void free_coherent(struct device *dev, size_t size,
2654 void *virt_addr, dma_addr_t dma_addr, 2636 void *virt_addr, dma_addr_t dma_addr,
2655 unsigned long attrs) 2637 unsigned long attrs)
2656{ 2638{
2657 struct protection_domain *domain; 2639 struct protection_domain *domain = get_domain(dev);
2658 struct dma_ops_domain *dma_dom;
2659 struct page *page;
2660 2640
2661 page = virt_to_page(virt_addr);
2662 size = PAGE_ALIGN(size); 2641 size = PAGE_ALIGN(size);
2663 2642
2664 domain = get_domain(dev); 2643 if (!IS_ERR(domain)) {
2665 if (IS_ERR(domain)) 2644 struct dma_ops_domain *dma_dom = to_dma_ops_domain(domain);
2666 goto free_mem;
2667 2645
2668 dma_dom = to_dma_ops_domain(domain); 2646 __unmap_single(dma_dom, dma_addr, size, DMA_BIDIRECTIONAL);
2669 2647 }
2670 __unmap_single(dma_dom, dma_addr, size, DMA_BIDIRECTIONAL);
2671 2648
2672free_mem: 2649 dma_direct_free(dev, size, virt_addr, dma_addr, attrs);
2673 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
2674 __free_pages(page, get_order(size));
2675} 2650}
2676 2651
2677/* 2652/*
@@ -2680,7 +2655,7 @@ free_mem:
2680 */ 2655 */
2681static int amd_iommu_dma_supported(struct device *dev, u64 mask) 2656static int amd_iommu_dma_supported(struct device *dev, u64 mask)
2682{ 2657{
2683 if (!x86_dma_supported(dev, mask)) 2658 if (!dma_direct_supported(dev, mask))
2684 return 0; 2659 return 0;
2685 return check_device(dev); 2660 return check_device(dev);
2686} 2661}
@@ -2794,7 +2769,7 @@ int __init amd_iommu_init_dma_ops(void)
2794 * continue to be SWIOTLB. 2769 * continue to be SWIOTLB.
2795 */ 2770 */
2796 if (!swiotlb) 2771 if (!swiotlb)
2797 dma_ops = &nommu_dma_ops; 2772 dma_ops = &dma_direct_ops;
2798 2773
2799 if (amd_iommu_unmap_flush) 2774 if (amd_iommu_unmap_flush)
2800 pr_info("AMD-Vi: IO/TLB flush on unmap enabled\n"); 2775 pr_info("AMD-Vi: IO/TLB flush on unmap enabled\n");
diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c
index 582fd01cb7d1..24d1b1b42013 100644
--- a/drivers/iommu/intel-iommu.c
+++ b/drivers/iommu/intel-iommu.c
@@ -31,6 +31,7 @@
31#include <linux/pci.h> 31#include <linux/pci.h>
32#include <linux/dmar.h> 32#include <linux/dmar.h>
33#include <linux/dma-mapping.h> 33#include <linux/dma-mapping.h>
34#include <linux/dma-direct.h>
34#include <linux/mempool.h> 35#include <linux/mempool.h>
35#include <linux/memory.h> 36#include <linux/memory.h>
36#include <linux/cpu.h> 37#include <linux/cpu.h>
@@ -45,6 +46,7 @@
45#include <linux/pci-ats.h> 46#include <linux/pci-ats.h>
46#include <linux/memblock.h> 47#include <linux/memblock.h>
47#include <linux/dma-contiguous.h> 48#include <linux/dma-contiguous.h>
49#include <linux/dma-direct.h>
48#include <linux/crash_dump.h> 50#include <linux/crash_dump.h>
49#include <asm/irq_remapping.h> 51#include <asm/irq_remapping.h>
50#include <asm/cacheflush.h> 52#include <asm/cacheflush.h>
@@ -3707,61 +3709,30 @@ static void *intel_alloc_coherent(struct device *dev, size_t size,
3707 dma_addr_t *dma_handle, gfp_t flags, 3709 dma_addr_t *dma_handle, gfp_t flags,
3708 unsigned long attrs) 3710 unsigned long attrs)
3709{ 3711{
3710 struct page *page = NULL; 3712 void *vaddr;
3711 int order;
3712 3713
3713 size = PAGE_ALIGN(size); 3714 vaddr = dma_direct_alloc(dev, size, dma_handle, flags, attrs);
3714 order = get_order(size); 3715 if (iommu_no_mapping(dev) || !vaddr)
3716 return vaddr;
3715 3717
3716 if (!iommu_no_mapping(dev)) 3718 *dma_handle = __intel_map_single(dev, virt_to_phys(vaddr),
3717 flags &= ~(GFP_DMA | GFP_DMA32); 3719 PAGE_ALIGN(size), DMA_BIDIRECTIONAL,
3718 else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) { 3720 dev->coherent_dma_mask);
3719 if (dev->coherent_dma_mask < DMA_BIT_MASK(32)) 3721 if (!*dma_handle)
3720 flags |= GFP_DMA; 3722 goto out_free_pages;
3721 else 3723 return vaddr;
3722 flags |= GFP_DMA32;
3723 }
3724
3725 if (gfpflags_allow_blocking(flags)) {
3726 unsigned int count = size >> PAGE_SHIFT;
3727
3728 page = dma_alloc_from_contiguous(dev, count, order, flags);
3729 if (page && iommu_no_mapping(dev) &&
3730 page_to_phys(page) + size > dev->coherent_dma_mask) {
3731 dma_release_from_contiguous(dev, page, count);
3732 page = NULL;
3733 }
3734 }
3735
3736 if (!page)
3737 page = alloc_pages(flags, order);
3738 if (!page)
3739 return NULL;
3740 memset(page_address(page), 0, size);
3741
3742 *dma_handle = __intel_map_single(dev, page_to_phys(page), size,
3743 DMA_BIDIRECTIONAL,
3744 dev->coherent_dma_mask);
3745 if (*dma_handle)
3746 return page_address(page);
3747 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3748 __free_pages(page, order);
3749 3724
3725out_free_pages:
3726 dma_direct_free(dev, size, vaddr, *dma_handle, attrs);
3750 return NULL; 3727 return NULL;
3751} 3728}
3752 3729
3753static void intel_free_coherent(struct device *dev, size_t size, void *vaddr, 3730static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
3754 dma_addr_t dma_handle, unsigned long attrs) 3731 dma_addr_t dma_handle, unsigned long attrs)
3755{ 3732{
3756 int order; 3733 if (!iommu_no_mapping(dev))
3757 struct page *page = virt_to_page(vaddr); 3734 intel_unmap(dev, dma_handle, PAGE_ALIGN(size));
3758 3735 dma_direct_free(dev, size, vaddr, dma_handle, attrs);
3759 size = PAGE_ALIGN(size);
3760 order = get_order(size);
3761
3762 intel_unmap(dev, dma_handle, size);
3763 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3764 __free_pages(page, order);
3765} 3736}
3766 3737
3767static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist, 3738static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
@@ -3871,7 +3842,7 @@ const struct dma_map_ops intel_dma_ops = {
3871 .unmap_page = intel_unmap_page, 3842 .unmap_page = intel_unmap_page,
3872 .mapping_error = intel_mapping_error, 3843 .mapping_error = intel_mapping_error,
3873#ifdef CONFIG_X86 3844#ifdef CONFIG_X86
3874 .dma_supported = x86_dma_supported, 3845 .dma_supported = dma_direct_supported,
3875#endif 3846#endif
3876}; 3847};
3877 3848
diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c
index 5bb72d3f8337..e1c60899fdbc 100644
--- a/drivers/xen/swiotlb-xen.c
+++ b/drivers/xen/swiotlb-xen.c
@@ -53,20 +53,6 @@
53 * API. 53 * API.
54 */ 54 */
55 55
56#ifndef CONFIG_X86
57static unsigned long dma_alloc_coherent_mask(struct device *dev,
58 gfp_t gfp)
59{
60 unsigned long dma_mask = 0;
61
62 dma_mask = dev->coherent_dma_mask;
63 if (!dma_mask)
64 dma_mask = (gfp & GFP_DMA) ? DMA_BIT_MASK(24) : DMA_BIT_MASK(32);
65
66 return dma_mask;
67}
68#endif
69
70#define XEN_SWIOTLB_ERROR_CODE (~(dma_addr_t)0x0) 56#define XEN_SWIOTLB_ERROR_CODE (~(dma_addr_t)0x0)
71 57
72static char *xen_io_tlb_start, *xen_io_tlb_end; 58static char *xen_io_tlb_start, *xen_io_tlb_end;
@@ -328,7 +314,7 @@ xen_swiotlb_alloc_coherent(struct device *hwdev, size_t size,
328 return ret; 314 return ret;
329 315
330 if (hwdev && hwdev->coherent_dma_mask) 316 if (hwdev && hwdev->coherent_dma_mask)
331 dma_mask = dma_alloc_coherent_mask(hwdev, flags); 317 dma_mask = hwdev->coherent_dma_mask;
332 318
333 /* At this point dma_handle is the physical address, next we are 319 /* At this point dma_handle is the physical address, next we are
334 * going to set it to the machine address. 320 * going to set it to the machine address.
diff --git a/include/linux/dma-direct.h b/include/linux/dma-direct.h
index bcdb1a3e4b1f..53ad6a47f513 100644
--- a/include/linux/dma-direct.h
+++ b/include/linux/dma-direct.h
@@ -3,18 +3,19 @@
3#define _LINUX_DMA_DIRECT_H 1 3#define _LINUX_DMA_DIRECT_H 1
4 4
5#include <linux/dma-mapping.h> 5#include <linux/dma-mapping.h>
6#include <linux/mem_encrypt.h>
6 7
7#ifdef CONFIG_ARCH_HAS_PHYS_TO_DMA 8#ifdef CONFIG_ARCH_HAS_PHYS_TO_DMA
8#include <asm/dma-direct.h> 9#include <asm/dma-direct.h>
9#else 10#else
10static inline dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr) 11static inline dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t paddr)
11{ 12{
12 dma_addr_t dev_addr = (dma_addr_t)paddr; 13 dma_addr_t dev_addr = (dma_addr_t)paddr;
13 14
14 return dev_addr - ((dma_addr_t)dev->dma_pfn_offset << PAGE_SHIFT); 15 return dev_addr - ((dma_addr_t)dev->dma_pfn_offset << PAGE_SHIFT);
15} 16}
16 17
17static inline phys_addr_t dma_to_phys(struct device *dev, dma_addr_t dev_addr) 18static inline phys_addr_t __dma_to_phys(struct device *dev, dma_addr_t dev_addr)
18{ 19{
19 phys_addr_t paddr = (phys_addr_t)dev_addr; 20 phys_addr_t paddr = (phys_addr_t)dev_addr;
20 21
@@ -30,6 +31,22 @@ static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size)
30} 31}
31#endif /* !CONFIG_ARCH_HAS_PHYS_TO_DMA */ 32#endif /* !CONFIG_ARCH_HAS_PHYS_TO_DMA */
32 33
34/*
35 * If memory encryption is supported, phys_to_dma will set the memory encryption
36 * bit in the DMA address, and dma_to_phys will clear it. The raw __phys_to_dma
37 * and __dma_to_phys versions should only be used on non-encrypted memory for
38 * special occasions like DMA coherent buffers.
39 */
40static inline dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr)
41{
42 return __sme_set(__phys_to_dma(dev, paddr));
43}
44
45static inline phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr)
46{
47 return __sme_clr(__dma_to_phys(dev, daddr));
48}
49
33#ifdef CONFIG_ARCH_HAS_DMA_MARK_CLEAN 50#ifdef CONFIG_ARCH_HAS_DMA_MARK_CLEAN
34void dma_mark_clean(void *addr, size_t size); 51void dma_mark_clean(void *addr, size_t size);
35#else 52#else
diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h
index eb9eab4ecd6d..12fedcba9a9a 100644
--- a/include/linux/dma-mapping.h
+++ b/include/linux/dma-mapping.h
@@ -518,12 +518,8 @@ static inline void *dma_alloc_attrs(struct device *dev, size_t size,
518 if (dma_alloc_from_dev_coherent(dev, size, dma_handle, &cpu_addr)) 518 if (dma_alloc_from_dev_coherent(dev, size, dma_handle, &cpu_addr))
519 return cpu_addr; 519 return cpu_addr;
520 520
521 /* 521 /* let the implementation decide on the zone to allocate from: */
522 * Let the implementation decide on the zone to allocate from, and 522 flag &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM);
523 * decide on the way of zeroing the memory given that the memory
524 * returned should always be zeroed.
525 */
526 flag &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM | __GFP_ZERO);
527 523
528 if (!arch_dma_alloc_attrs(&dev, &flag)) 524 if (!arch_dma_alloc_attrs(&dev, &flag))
529 return NULL; 525 return NULL;
diff --git a/include/linux/set_memory.h b/include/linux/set_memory.h
index e5140648f638..da5178216da5 100644
--- a/include/linux/set_memory.h
+++ b/include/linux/set_memory.h
@@ -17,4 +17,16 @@ static inline int set_memory_x(unsigned long addr, int numpages) { return 0; }
17static inline int set_memory_nx(unsigned long addr, int numpages) { return 0; } 17static inline int set_memory_nx(unsigned long addr, int numpages) { return 0; }
18#endif 18#endif
19 19
20#ifndef CONFIG_ARCH_HAS_MEM_ENCRYPT
21static inline int set_memory_encrypted(unsigned long addr, int numpages)
22{
23 return 0;
24}
25
26static inline int set_memory_decrypted(unsigned long addr, int numpages)
27{
28 return 0;
29}
30#endif /* CONFIG_ARCH_HAS_MEM_ENCRYPT */
31
20#endif /* _LINUX_SET_MEMORY_H_ */ 32#endif /* _LINUX_SET_MEMORY_H_ */
diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h
index 5b1f2a00491c..965be92c33b5 100644
--- a/include/linux/swiotlb.h
+++ b/include/linux/swiotlb.h
@@ -72,14 +72,6 @@ void *swiotlb_alloc(struct device *hwdev, size_t size, dma_addr_t *dma_handle,
72void swiotlb_free(struct device *dev, size_t size, void *vaddr, 72void swiotlb_free(struct device *dev, size_t size, void *vaddr,
73 dma_addr_t dma_addr, unsigned long attrs); 73 dma_addr_t dma_addr, unsigned long attrs);
74 74
75extern void
76*swiotlb_alloc_coherent(struct device *hwdev, size_t size,
77 dma_addr_t *dma_handle, gfp_t flags);
78
79extern void
80swiotlb_free_coherent(struct device *hwdev, size_t size,
81 void *vaddr, dma_addr_t dma_handle);
82
83extern dma_addr_t swiotlb_map_page(struct device *dev, struct page *page, 75extern dma_addr_t swiotlb_map_page(struct device *dev, struct page *page,
84 unsigned long offset, size_t size, 76 unsigned long offset, size_t size,
85 enum dma_data_direction dir, 77 enum dma_data_direction dir,
diff --git a/lib/dma-direct.c b/lib/dma-direct.c
index c9e8e21cb334..c0bba30fef0a 100644
--- a/lib/dma-direct.c
+++ b/lib/dma-direct.c
@@ -9,6 +9,7 @@
9#include <linux/scatterlist.h> 9#include <linux/scatterlist.h>
10#include <linux/dma-contiguous.h> 10#include <linux/dma-contiguous.h>
11#include <linux/pfn.h> 11#include <linux/pfn.h>
12#include <linux/set_memory.h>
12 13
13#define DIRECT_MAPPING_ERROR 0 14#define DIRECT_MAPPING_ERROR 0
14 15
@@ -20,6 +21,14 @@
20#define ARCH_ZONE_DMA_BITS 24 21#define ARCH_ZONE_DMA_BITS 24
21#endif 22#endif
22 23
24/*
25 * For AMD SEV all DMA must be to unencrypted addresses.
26 */
27static inline bool force_dma_unencrypted(void)
28{
29 return sev_active();
30}
31
23static bool 32static bool
24check_addr(struct device *dev, dma_addr_t dma_addr, size_t size, 33check_addr(struct device *dev, dma_addr_t dma_addr, size_t size,
25 const char *caller) 34 const char *caller)
@@ -37,7 +46,9 @@ check_addr(struct device *dev, dma_addr_t dma_addr, size_t size,
37 46
38static bool dma_coherent_ok(struct device *dev, phys_addr_t phys, size_t size) 47static bool dma_coherent_ok(struct device *dev, phys_addr_t phys, size_t size)
39{ 48{
40 return phys_to_dma(dev, phys) + size - 1 <= dev->coherent_dma_mask; 49 dma_addr_t addr = force_dma_unencrypted() ?
50 __phys_to_dma(dev, phys) : phys_to_dma(dev, phys);
51 return addr + size - 1 <= dev->coherent_dma_mask;
41} 52}
42 53
43void *dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, 54void *dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
@@ -46,6 +57,10 @@ void *dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
46 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; 57 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
47 int page_order = get_order(size); 58 int page_order = get_order(size);
48 struct page *page = NULL; 59 struct page *page = NULL;
60 void *ret;
61
62 /* we always manually zero the memory once we are done: */
63 gfp &= ~__GFP_ZERO;
49 64
50 /* GFP_DMA32 and GFP_DMA are no ops without the corresponding zones: */ 65 /* GFP_DMA32 and GFP_DMA are no ops without the corresponding zones: */
51 if (dev->coherent_dma_mask <= DMA_BIT_MASK(ARCH_ZONE_DMA_BITS)) 66 if (dev->coherent_dma_mask <= DMA_BIT_MASK(ARCH_ZONE_DMA_BITS))
@@ -78,10 +93,15 @@ again:
78 93
79 if (!page) 94 if (!page)
80 return NULL; 95 return NULL;
81 96 ret = page_address(page);
82 *dma_handle = phys_to_dma(dev, page_to_phys(page)); 97 if (force_dma_unencrypted()) {
83 memset(page_address(page), 0, size); 98 set_memory_decrypted((unsigned long)ret, 1 << page_order);
84 return page_address(page); 99 *dma_handle = __phys_to_dma(dev, page_to_phys(page));
100 } else {
101 *dma_handle = phys_to_dma(dev, page_to_phys(page));
102 }
103 memset(ret, 0, size);
104 return ret;
85} 105}
86 106
87/* 107/*
@@ -92,9 +112,12 @@ void dma_direct_free(struct device *dev, size_t size, void *cpu_addr,
92 dma_addr_t dma_addr, unsigned long attrs) 112 dma_addr_t dma_addr, unsigned long attrs)
93{ 113{
94 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; 114 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
115 unsigned int page_order = get_order(size);
95 116
117 if (force_dma_unencrypted())
118 set_memory_encrypted((unsigned long)cpu_addr, 1 << page_order);
96 if (!dma_release_from_contiguous(dev, virt_to_page(cpu_addr), count)) 119 if (!dma_release_from_contiguous(dev, virt_to_page(cpu_addr), count))
97 free_pages((unsigned long)cpu_addr, get_order(size)); 120 free_pages((unsigned long)cpu_addr, page_order);
98} 121}
99 122
100static dma_addr_t dma_direct_map_page(struct device *dev, struct page *page, 123static dma_addr_t dma_direct_map_page(struct device *dev, struct page *page,
diff --git a/lib/swiotlb.c b/lib/swiotlb.c
index c43ec2271469..47aeb04c1997 100644
--- a/lib/swiotlb.c
+++ b/lib/swiotlb.c
@@ -31,6 +31,7 @@
31#include <linux/gfp.h> 31#include <linux/gfp.h>
32#include <linux/scatterlist.h> 32#include <linux/scatterlist.h>
33#include <linux/mem_encrypt.h> 33#include <linux/mem_encrypt.h>
34#include <linux/set_memory.h>
34 35
35#include <asm/io.h> 36#include <asm/io.h>
36#include <asm/dma.h> 37#include <asm/dma.h>
@@ -156,22 +157,6 @@ unsigned long swiotlb_size_or_default(void)
156 return size ? size : (IO_TLB_DEFAULT_SIZE); 157 return size ? size : (IO_TLB_DEFAULT_SIZE);
157} 158}
158 159
159void __weak swiotlb_set_mem_attributes(void *vaddr, unsigned long size) { }
160
161/* For swiotlb, clear memory encryption mask from dma addresses */
162static dma_addr_t swiotlb_phys_to_dma(struct device *hwdev,
163 phys_addr_t address)
164{
165 return __sme_clr(phys_to_dma(hwdev, address));
166}
167
168/* Note that this doesn't work with highmem page */
169static dma_addr_t swiotlb_virt_to_bus(struct device *hwdev,
170 volatile void *address)
171{
172 return phys_to_dma(hwdev, virt_to_phys(address));
173}
174
175static bool no_iotlb_memory; 160static bool no_iotlb_memory;
176 161
177void swiotlb_print_info(void) 162void swiotlb_print_info(void)
@@ -209,12 +194,12 @@ void __init swiotlb_update_mem_attributes(void)
209 194
210 vaddr = phys_to_virt(io_tlb_start); 195 vaddr = phys_to_virt(io_tlb_start);
211 bytes = PAGE_ALIGN(io_tlb_nslabs << IO_TLB_SHIFT); 196 bytes = PAGE_ALIGN(io_tlb_nslabs << IO_TLB_SHIFT);
212 swiotlb_set_mem_attributes(vaddr, bytes); 197 set_memory_decrypted((unsigned long)vaddr, bytes >> PAGE_SHIFT);
213 memset(vaddr, 0, bytes); 198 memset(vaddr, 0, bytes);
214 199
215 vaddr = phys_to_virt(io_tlb_overflow_buffer); 200 vaddr = phys_to_virt(io_tlb_overflow_buffer);
216 bytes = PAGE_ALIGN(io_tlb_overflow); 201 bytes = PAGE_ALIGN(io_tlb_overflow);
217 swiotlb_set_mem_attributes(vaddr, bytes); 202 set_memory_decrypted((unsigned long)vaddr, bytes >> PAGE_SHIFT);
218 memset(vaddr, 0, bytes); 203 memset(vaddr, 0, bytes);
219} 204}
220 205
@@ -355,7 +340,7 @@ swiotlb_late_init_with_tbl(char *tlb, unsigned long nslabs)
355 io_tlb_start = virt_to_phys(tlb); 340 io_tlb_start = virt_to_phys(tlb);
356 io_tlb_end = io_tlb_start + bytes; 341 io_tlb_end = io_tlb_start + bytes;
357 342
358 swiotlb_set_mem_attributes(tlb, bytes); 343 set_memory_decrypted((unsigned long)tlb, bytes >> PAGE_SHIFT);
359 memset(tlb, 0, bytes); 344 memset(tlb, 0, bytes);
360 345
361 /* 346 /*
@@ -366,7 +351,8 @@ swiotlb_late_init_with_tbl(char *tlb, unsigned long nslabs)
366 if (!v_overflow_buffer) 351 if (!v_overflow_buffer)
367 goto cleanup2; 352 goto cleanup2;
368 353
369 swiotlb_set_mem_attributes(v_overflow_buffer, io_tlb_overflow); 354 set_memory_decrypted((unsigned long)v_overflow_buffer,
355 io_tlb_overflow >> PAGE_SHIFT);
370 memset(v_overflow_buffer, 0, io_tlb_overflow); 356 memset(v_overflow_buffer, 0, io_tlb_overflow);
371 io_tlb_overflow_buffer = virt_to_phys(v_overflow_buffer); 357 io_tlb_overflow_buffer = virt_to_phys(v_overflow_buffer);
372 358
@@ -622,7 +608,7 @@ map_single(struct device *hwdev, phys_addr_t phys, size_t size,
622 return SWIOTLB_MAP_ERROR; 608 return SWIOTLB_MAP_ERROR;
623 } 609 }
624 610
625 start_dma_addr = swiotlb_phys_to_dma(hwdev, io_tlb_start); 611 start_dma_addr = __phys_to_dma(hwdev, io_tlb_start);
626 return swiotlb_tbl_map_single(hwdev, start_dma_addr, phys, size, 612 return swiotlb_tbl_map_single(hwdev, start_dma_addr, phys, size,
627 dir, attrs); 613 dir, attrs);
628} 614}
@@ -706,6 +692,7 @@ void swiotlb_tbl_sync_single(struct device *hwdev, phys_addr_t tlb_addr,
706 } 692 }
707} 693}
708 694
695#ifdef CONFIG_DMA_DIRECT_OPS
709static inline bool dma_coherent_ok(struct device *dev, dma_addr_t addr, 696static inline bool dma_coherent_ok(struct device *dev, dma_addr_t addr,
710 size_t size) 697 size_t size)
711{ 698{
@@ -726,12 +713,12 @@ swiotlb_alloc_buffer(struct device *dev, size_t size, dma_addr_t *dma_handle,
726 goto out_warn; 713 goto out_warn;
727 714
728 phys_addr = swiotlb_tbl_map_single(dev, 715 phys_addr = swiotlb_tbl_map_single(dev,
729 swiotlb_phys_to_dma(dev, io_tlb_start), 716 __phys_to_dma(dev, io_tlb_start),
730 0, size, DMA_FROM_DEVICE, 0); 717 0, size, DMA_FROM_DEVICE, 0);
731 if (phys_addr == SWIOTLB_MAP_ERROR) 718 if (phys_addr == SWIOTLB_MAP_ERROR)
732 goto out_warn; 719 goto out_warn;
733 720
734 *dma_handle = swiotlb_phys_to_dma(dev, phys_addr); 721 *dma_handle = __phys_to_dma(dev, phys_addr);
735 if (dma_coherent_ok(dev, *dma_handle, size)) 722 if (dma_coherent_ok(dev, *dma_handle, size))
736 goto out_unmap; 723 goto out_unmap;
737 724
@@ -759,28 +746,6 @@ out_warn:
759 return NULL; 746 return NULL;
760} 747}
761 748
762void *
763swiotlb_alloc_coherent(struct device *hwdev, size_t size,
764 dma_addr_t *dma_handle, gfp_t flags)
765{
766 int order = get_order(size);
767 unsigned long attrs = (flags & __GFP_NOWARN) ? DMA_ATTR_NO_WARN : 0;
768 void *ret;
769
770 ret = (void *)__get_free_pages(flags, order);
771 if (ret) {
772 *dma_handle = swiotlb_virt_to_bus(hwdev, ret);
773 if (dma_coherent_ok(hwdev, *dma_handle, size)) {
774 memset(ret, 0, size);
775 return ret;
776 }
777 free_pages((unsigned long)ret, order);
778 }
779
780 return swiotlb_alloc_buffer(hwdev, size, dma_handle, attrs);
781}
782EXPORT_SYMBOL(swiotlb_alloc_coherent);
783
784static bool swiotlb_free_buffer(struct device *dev, size_t size, 749static bool swiotlb_free_buffer(struct device *dev, size_t size,
785 dma_addr_t dma_addr) 750 dma_addr_t dma_addr)
786{ 751{
@@ -799,15 +764,7 @@ static bool swiotlb_free_buffer(struct device *dev, size_t size,
799 DMA_ATTR_SKIP_CPU_SYNC); 764 DMA_ATTR_SKIP_CPU_SYNC);
800 return true; 765 return true;
801} 766}
802 767#endif
803void
804swiotlb_free_coherent(struct device *hwdev, size_t size, void *vaddr,
805 dma_addr_t dev_addr)
806{
807 if (!swiotlb_free_buffer(hwdev, size, dev_addr))
808 free_pages((unsigned long)vaddr, get_order(size));
809}
810EXPORT_SYMBOL(swiotlb_free_coherent);
811 768
812static void 769static void
813swiotlb_full(struct device *dev, size_t size, enum dma_data_direction dir, 770swiotlb_full(struct device *dev, size_t size, enum dma_data_direction dir,
@@ -867,10 +824,10 @@ dma_addr_t swiotlb_map_page(struct device *dev, struct page *page,
867 map = map_single(dev, phys, size, dir, attrs); 824 map = map_single(dev, phys, size, dir, attrs);
868 if (map == SWIOTLB_MAP_ERROR) { 825 if (map == SWIOTLB_MAP_ERROR) {
869 swiotlb_full(dev, size, dir, 1); 826 swiotlb_full(dev, size, dir, 1);
870 return swiotlb_phys_to_dma(dev, io_tlb_overflow_buffer); 827 return __phys_to_dma(dev, io_tlb_overflow_buffer);
871 } 828 }
872 829
873 dev_addr = swiotlb_phys_to_dma(dev, map); 830 dev_addr = __phys_to_dma(dev, map);
874 831
875 /* Ensure that the address returned is DMA'ble */ 832 /* Ensure that the address returned is DMA'ble */
876 if (dma_capable(dev, dev_addr, size)) 833 if (dma_capable(dev, dev_addr, size))
@@ -879,7 +836,7 @@ dma_addr_t swiotlb_map_page(struct device *dev, struct page *page,
879 attrs |= DMA_ATTR_SKIP_CPU_SYNC; 836 attrs |= DMA_ATTR_SKIP_CPU_SYNC;
880 swiotlb_tbl_unmap_single(dev, map, size, dir, attrs); 837 swiotlb_tbl_unmap_single(dev, map, size, dir, attrs);
881 838
882 return swiotlb_phys_to_dma(dev, io_tlb_overflow_buffer); 839 return __phys_to_dma(dev, io_tlb_overflow_buffer);
883} 840}
884 841
885/* 842/*
@@ -1009,7 +966,7 @@ swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl, int nelems,
1009 sg_dma_len(sgl) = 0; 966 sg_dma_len(sgl) = 0;
1010 return 0; 967 return 0;
1011 } 968 }
1012 sg->dma_address = swiotlb_phys_to_dma(hwdev, map); 969 sg->dma_address = __phys_to_dma(hwdev, map);
1013 } else 970 } else
1014 sg->dma_address = dev_addr; 971 sg->dma_address = dev_addr;
1015 sg_dma_len(sg) = sg->length; 972 sg_dma_len(sg) = sg->length;
@@ -1073,7 +1030,7 @@ swiotlb_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg,
1073int 1030int
1074swiotlb_dma_mapping_error(struct device *hwdev, dma_addr_t dma_addr) 1031swiotlb_dma_mapping_error(struct device *hwdev, dma_addr_t dma_addr)
1075{ 1032{
1076 return (dma_addr == swiotlb_phys_to_dma(hwdev, io_tlb_overflow_buffer)); 1033 return (dma_addr == __phys_to_dma(hwdev, io_tlb_overflow_buffer));
1077} 1034}
1078 1035
1079/* 1036/*
@@ -1085,7 +1042,7 @@ swiotlb_dma_mapping_error(struct device *hwdev, dma_addr_t dma_addr)
1085int 1042int
1086swiotlb_dma_supported(struct device *hwdev, u64 mask) 1043swiotlb_dma_supported(struct device *hwdev, u64 mask)
1087{ 1044{
1088 return swiotlb_phys_to_dma(hwdev, io_tlb_end - 1) <= mask; 1045 return __phys_to_dma(hwdev, io_tlb_end - 1) <= mask;
1089} 1046}
1090 1047
1091#ifdef CONFIG_DMA_DIRECT_OPS 1048#ifdef CONFIG_DMA_DIRECT_OPS
generated by cgit v1.2.2 (git 2.25.0) at 2025-08-29 01:41:12 -0400