summaryrefslogtreecommitdiffstats
path: root/kernel/dma
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2018-12-28 17:12:21 -0500
committerLinus Torvalds <torvalds@linux-foundation.org>2018-12-28 17:12:21 -0500
commitaf7ddd8a627c62a835524b3f5b471edbbbcce025 (patch)
treeaf9777ddef6d394c7cc01fca599328b584ca2bc1 /kernel/dma
parentfe2b0cdabcd9e6aeca66a104bc03576946e5fee2 (diff)
parent8b1cce9f5832a8eda17d37a3c49fb7dd2d650f46 (diff)
Merge tag 'dma-mapping-4.21' of git://git.infradead.org/users/hch/dma-mapping
Pull DMA mapping updates from Christoph Hellwig: "A huge update this time, but a lot of that is just consolidating or removing code: - provide a common DMA_MAPPING_ERROR definition and avoid indirect calls for dma_map_* error checking - use direct calls for the DMA direct mapping case, avoiding huge retpoline overhead for high performance workloads - merge the swiotlb dma_map_ops into dma-direct - provide a generic remapping DMA consistent allocator for architectures that have devices that perform DMA that is not cache coherent. Based on the existing arm64 implementation and also used for csky now. - improve the dma-debug infrastructure, including dynamic allocation of entries (Robin Murphy) - default to providing chaining scatterlist everywhere, with opt-outs for the few architectures (alpha, parisc, most arm32 variants) that can't cope with it - misc sparc32 dma-related cleanups - remove the dma_mark_clean arch hook used by swiotlb on ia64 and replace it with the generic noncoherent infrastructure - fix the return type of dma_set_max_seg_size (Niklas Söderlund) - move the dummy dma ops for not DMA capable devices from arm64 to common code (Robin Murphy) - ensure dma_alloc_coherent returns zeroed memory to avoid kernel data leaks through userspace. We already did this for most common architectures, but this ensures we do it everywhere. dma_zalloc_coherent has been deprecated and can hopefully be removed after -rc1 with a coccinelle script" * tag 'dma-mapping-4.21' of git://git.infradead.org/users/hch/dma-mapping: (73 commits) dma-mapping: fix inverted logic in dma_supported dma-mapping: deprecate dma_zalloc_coherent dma-mapping: zero memory returned from dma_alloc_* sparc/iommu: fix ->map_sg return value sparc/io-unit: fix ->map_sg return value arm64: default to the direct mapping in get_arch_dma_ops PCI: Remove unused attr variable in pci_dma_configure ia64: only select ARCH_HAS_DMA_COHERENT_TO_PFN if swiotlb is enabled dma-mapping: bypass indirect calls for dma-direct vmd: use the proper dma_* APIs instead of direct methods calls dma-direct: merge swiotlb_dma_ops into the dma_direct code dma-direct: use dma_direct_map_page to implement dma_direct_map_sg dma-direct: improve addressability error reporting swiotlb: remove dma_mark_clean swiotlb: remove SWIOTLB_MAP_ERROR ACPI / scan: Refactor _CCA enforcement dma-mapping: factor out dummy DMA ops dma-mapping: always build the direct mapping code dma-mapping: move dma_cache_sync out of line dma-mapping: move various slow path functions out of line ...
Diffstat (limited to 'kernel/dma')
-rw-r--r--kernel/dma/Kconfig14
-rw-r--r--kernel/dma/Makefile5
-rw-r--r--kernel/dma/debug.c259
-rw-r--r--kernel/dma/direct.c222
-rw-r--r--kernel/dma/dummy.c39
-rw-r--r--kernel/dma/mapping.c223
-rw-r--r--kernel/dma/remap.c256
-rw-r--r--kernel/dma/swiotlb.c253
-rw-r--r--kernel/dma/virt.c2
9 files changed, 708 insertions, 565 deletions
diff --git a/kernel/dma/Kconfig b/kernel/dma/Kconfig
index 645c7a2ecde8..ca88b867e7fe 100644
--- a/kernel/dma/Kconfig
+++ b/kernel/dma/Kconfig
@@ -35,13 +35,8 @@ config ARCH_HAS_DMA_COHERENT_TO_PFN
35config ARCH_HAS_DMA_MMAP_PGPROT 35config ARCH_HAS_DMA_MMAP_PGPROT
36 bool 36 bool
37 37
38config DMA_DIRECT_OPS
39 bool
40 depends on HAS_DMA
41
42config DMA_NONCOHERENT_CACHE_SYNC 38config DMA_NONCOHERENT_CACHE_SYNC
43 bool 39 bool
44 depends on DMA_DIRECT_OPS
45 40
46config DMA_VIRT_OPS 41config DMA_VIRT_OPS
47 bool 42 bool
@@ -49,5 +44,12 @@ config DMA_VIRT_OPS
49 44
50config SWIOTLB 45config SWIOTLB
51 bool 46 bool
52 select DMA_DIRECT_OPS
53 select NEED_DMA_MAP_STATE 47 select NEED_DMA_MAP_STATE
48
49config DMA_REMAP
50 depends on MMU
51 bool
52
53config DMA_DIRECT_REMAP
54 bool
55 select DMA_REMAP
diff --git a/kernel/dma/Makefile b/kernel/dma/Makefile
index 7d581e4eea4a..72ff6e46aa86 100644
--- a/kernel/dma/Makefile
+++ b/kernel/dma/Makefile
@@ -1,10 +1,9 @@
1# SPDX-License-Identifier: GPL-2.0 1# SPDX-License-Identifier: GPL-2.0
2 2
3obj-$(CONFIG_HAS_DMA) += mapping.o 3obj-$(CONFIG_HAS_DMA) += mapping.o direct.o dummy.o
4obj-$(CONFIG_DMA_CMA) += contiguous.o 4obj-$(CONFIG_DMA_CMA) += contiguous.o
5obj-$(CONFIG_HAVE_GENERIC_DMA_COHERENT) += coherent.o 5obj-$(CONFIG_HAVE_GENERIC_DMA_COHERENT) += coherent.o
6obj-$(CONFIG_DMA_DIRECT_OPS) += direct.o
7obj-$(CONFIG_DMA_VIRT_OPS) += virt.o 6obj-$(CONFIG_DMA_VIRT_OPS) += virt.o
8obj-$(CONFIG_DMA_API_DEBUG) += debug.o 7obj-$(CONFIG_DMA_API_DEBUG) += debug.o
9obj-$(CONFIG_SWIOTLB) += swiotlb.o 8obj-$(CONFIG_SWIOTLB) += swiotlb.o
10 9obj-$(CONFIG_DMA_REMAP) += remap.o
diff --git a/kernel/dma/debug.c b/kernel/dma/debug.c
index 231ca4628062..164706da2a73 100644
--- a/kernel/dma/debug.c
+++ b/kernel/dma/debug.c
@@ -17,6 +17,8 @@
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18 */ 18 */
19 19
20#define pr_fmt(fmt) "DMA-API: " fmt
21
20#include <linux/sched/task_stack.h> 22#include <linux/sched/task_stack.h>
21#include <linux/scatterlist.h> 23#include <linux/scatterlist.h>
22#include <linux/dma-mapping.h> 24#include <linux/dma-mapping.h>
@@ -41,10 +43,9 @@
41#define HASH_FN_SHIFT 13 43#define HASH_FN_SHIFT 13
42#define HASH_FN_MASK (HASH_SIZE - 1) 44#define HASH_FN_MASK (HASH_SIZE - 1)
43 45
44/* allow architectures to override this if absolutely required */
45#ifndef PREALLOC_DMA_DEBUG_ENTRIES
46#define PREALLOC_DMA_DEBUG_ENTRIES (1 << 16) 46#define PREALLOC_DMA_DEBUG_ENTRIES (1 << 16)
47#endif 47/* If the pool runs out, add this many new entries at once */
48#define DMA_DEBUG_DYNAMIC_ENTRIES (PAGE_SIZE / sizeof(struct dma_debug_entry))
48 49
49enum { 50enum {
50 dma_debug_single, 51 dma_debug_single,
@@ -142,6 +143,7 @@ static struct dentry *show_all_errors_dent __read_mostly;
142static struct dentry *show_num_errors_dent __read_mostly; 143static struct dentry *show_num_errors_dent __read_mostly;
143static struct dentry *num_free_entries_dent __read_mostly; 144static struct dentry *num_free_entries_dent __read_mostly;
144static struct dentry *min_free_entries_dent __read_mostly; 145static struct dentry *min_free_entries_dent __read_mostly;
146static struct dentry *nr_total_entries_dent __read_mostly;
145static struct dentry *filter_dent __read_mostly; 147static struct dentry *filter_dent __read_mostly;
146 148
147/* per-driver filter related state */ 149/* per-driver filter related state */
@@ -234,7 +236,7 @@ static bool driver_filter(struct device *dev)
234 error_count += 1; \ 236 error_count += 1; \
235 if (driver_filter(dev) && \ 237 if (driver_filter(dev) && \
236 (show_all_errors || show_num_errors > 0)) { \ 238 (show_all_errors || show_num_errors > 0)) { \
237 WARN(1, "%s %s: " format, \ 239 WARN(1, pr_fmt("%s %s: ") format, \
238 dev ? dev_driver_string(dev) : "NULL", \ 240 dev ? dev_driver_string(dev) : "NULL", \
239 dev ? dev_name(dev) : "NULL", ## arg); \ 241 dev ? dev_name(dev) : "NULL", ## arg); \
240 dump_entry_trace(entry); \ 242 dump_entry_trace(entry); \
@@ -519,7 +521,7 @@ static void active_cacheline_inc_overlap(phys_addr_t cln)
519 * prematurely. 521 * prematurely.
520 */ 522 */
521 WARN_ONCE(overlap > ACTIVE_CACHELINE_MAX_OVERLAP, 523 WARN_ONCE(overlap > ACTIVE_CACHELINE_MAX_OVERLAP,
522 "DMA-API: exceeded %d overlapping mappings of cacheline %pa\n", 524 pr_fmt("exceeded %d overlapping mappings of cacheline %pa\n"),
523 ACTIVE_CACHELINE_MAX_OVERLAP, &cln); 525 ACTIVE_CACHELINE_MAX_OVERLAP, &cln);
524} 526}
525 527
@@ -614,7 +616,7 @@ void debug_dma_assert_idle(struct page *page)
614 616
615 cln = to_cacheline_number(entry); 617 cln = to_cacheline_number(entry);
616 err_printk(entry->dev, entry, 618 err_printk(entry->dev, entry,
617 "DMA-API: cpu touching an active dma mapped cacheline [cln=%pa]\n", 619 "cpu touching an active dma mapped cacheline [cln=%pa]\n",
618 &cln); 620 &cln);
619} 621}
620 622
@@ -634,7 +636,7 @@ static void add_dma_entry(struct dma_debug_entry *entry)
634 636
635 rc = active_cacheline_insert(entry); 637 rc = active_cacheline_insert(entry);
636 if (rc == -ENOMEM) { 638 if (rc == -ENOMEM) {
637 pr_err("DMA-API: cacheline tracking ENOMEM, dma-debug disabled\n"); 639 pr_err("cacheline tracking ENOMEM, dma-debug disabled\n");
638 global_disable = true; 640 global_disable = true;
639 } 641 }
640 642
@@ -643,6 +645,24 @@ static void add_dma_entry(struct dma_debug_entry *entry)
643 */ 645 */
644} 646}
645 647
648static int dma_debug_create_entries(gfp_t gfp)
649{
650 struct dma_debug_entry *entry;
651 int i;
652
653 entry = (void *)get_zeroed_page(gfp);
654 if (!entry)
655 return -ENOMEM;
656
657 for (i = 0; i < DMA_DEBUG_DYNAMIC_ENTRIES; i++)
658 list_add_tail(&entry[i].list, &free_entries);
659
660 num_free_entries += DMA_DEBUG_DYNAMIC_ENTRIES;
661 nr_total_entries += DMA_DEBUG_DYNAMIC_ENTRIES;
662
663 return 0;
664}
665
646static struct dma_debug_entry *__dma_entry_alloc(void) 666static struct dma_debug_entry *__dma_entry_alloc(void)
647{ 667{
648 struct dma_debug_entry *entry; 668 struct dma_debug_entry *entry;
@@ -658,6 +678,18 @@ static struct dma_debug_entry *__dma_entry_alloc(void)
658 return entry; 678 return entry;
659} 679}
660 680
681void __dma_entry_alloc_check_leak(void)
682{
683 u32 tmp = nr_total_entries % nr_prealloc_entries;
684
685 /* Shout each time we tick over some multiple of the initial pool */
686 if (tmp < DMA_DEBUG_DYNAMIC_ENTRIES) {
687 pr_info("dma_debug_entry pool grown to %u (%u00%%)\n",
688 nr_total_entries,
689 (nr_total_entries / nr_prealloc_entries));
690 }
691}
692
661/* struct dma_entry allocator 693/* struct dma_entry allocator
662 * 694 *
663 * The next two functions implement the allocator for 695 * The next two functions implement the allocator for
@@ -669,12 +701,14 @@ static struct dma_debug_entry *dma_entry_alloc(void)
669 unsigned long flags; 701 unsigned long flags;
670 702
671 spin_lock_irqsave(&free_entries_lock, flags); 703 spin_lock_irqsave(&free_entries_lock, flags);
672 704 if (num_free_entries == 0) {
673 if (list_empty(&free_entries)) { 705 if (dma_debug_create_entries(GFP_ATOMIC)) {
674 global_disable = true; 706 global_disable = true;
675 spin_unlock_irqrestore(&free_entries_lock, flags); 707 spin_unlock_irqrestore(&free_entries_lock, flags);
676 pr_err("DMA-API: debugging out of memory - disabling\n"); 708 pr_err("debugging out of memory - disabling\n");
677 return NULL; 709 return NULL;
710 }
711 __dma_entry_alloc_check_leak();
678 } 712 }
679 713
680 entry = __dma_entry_alloc(); 714 entry = __dma_entry_alloc();
@@ -707,52 +741,6 @@ static void dma_entry_free(struct dma_debug_entry *entry)
707 spin_unlock_irqrestore(&free_entries_lock, flags); 741 spin_unlock_irqrestore(&free_entries_lock, flags);
708} 742}
709 743
710int dma_debug_resize_entries(u32 num_entries)
711{
712 int i, delta, ret = 0;
713 unsigned long flags;
714 struct dma_debug_entry *entry;
715 LIST_HEAD(tmp);
716
717 spin_lock_irqsave(&free_entries_lock, flags);
718
719 if (nr_total_entries < num_entries) {
720 delta = num_entries - nr_total_entries;
721
722 spin_unlock_irqrestore(&free_entries_lock, flags);
723
724 for (i = 0; i < delta; i++) {
725 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
726 if (!entry)
727 break;
728
729 list_add_tail(&entry->list, &tmp);
730 }
731
732 spin_lock_irqsave(&free_entries_lock, flags);
733
734 list_splice(&tmp, &free_entries);
735 nr_total_entries += i;
736 num_free_entries += i;
737 } else {
738 delta = nr_total_entries - num_entries;
739
740 for (i = 0; i < delta && !list_empty(&free_entries); i++) {
741 entry = __dma_entry_alloc();
742 kfree(entry);
743 }
744
745 nr_total_entries -= i;
746 }
747
748 if (nr_total_entries != num_entries)
749 ret = 1;
750
751 spin_unlock_irqrestore(&free_entries_lock, flags);
752
753 return ret;
754}
755
756/* 744/*
757 * DMA-API debugging init code 745 * DMA-API debugging init code
758 * 746 *
@@ -761,36 +749,6 @@ int dma_debug_resize_entries(u32 num_entries)
761 * 2. Preallocate a given number of dma_debug_entry structs 749 * 2. Preallocate a given number of dma_debug_entry structs
762 */ 750 */
763 751
764static int prealloc_memory(u32 num_entries)
765{
766 struct dma_debug_entry *entry, *next_entry;
767 int i;
768
769 for (i = 0; i < num_entries; ++i) {
770 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
771 if (!entry)
772 goto out_err;
773
774 list_add_tail(&entry->list, &free_entries);
775 }
776
777 num_free_entries = num_entries;
778 min_free_entries = num_entries;
779
780 pr_info("DMA-API: preallocated %d debug entries\n", num_entries);
781
782 return 0;
783
784out_err:
785
786 list_for_each_entry_safe(entry, next_entry, &free_entries, list) {
787 list_del(&entry->list);
788 kfree(entry);
789 }
790
791 return -ENOMEM;
792}
793
794static ssize_t filter_read(struct file *file, char __user *user_buf, 752static ssize_t filter_read(struct file *file, char __user *user_buf,
795 size_t count, loff_t *ppos) 753 size_t count, loff_t *ppos)
796{ 754{
@@ -850,7 +808,7 @@ static ssize_t filter_write(struct file *file, const char __user *userbuf,
850 * switched off. 808 * switched off.
851 */ 809 */
852 if (current_driver_name[0]) 810 if (current_driver_name[0])
853 pr_info("DMA-API: switching off dma-debug driver filter\n"); 811 pr_info("switching off dma-debug driver filter\n");
854 current_driver_name[0] = 0; 812 current_driver_name[0] = 0;
855 current_driver = NULL; 813 current_driver = NULL;
856 goto out_unlock; 814 goto out_unlock;
@@ -868,7 +826,7 @@ static ssize_t filter_write(struct file *file, const char __user *userbuf,
868 current_driver_name[i] = 0; 826 current_driver_name[i] = 0;
869 current_driver = NULL; 827 current_driver = NULL;
870 828
871 pr_info("DMA-API: enable driver filter for driver [%s]\n", 829 pr_info("enable driver filter for driver [%s]\n",
872 current_driver_name); 830 current_driver_name);
873 831
874out_unlock: 832out_unlock:
@@ -887,7 +845,7 @@ static int dma_debug_fs_init(void)
887{ 845{
888 dma_debug_dent = debugfs_create_dir("dma-api", NULL); 846 dma_debug_dent = debugfs_create_dir("dma-api", NULL);
889 if (!dma_debug_dent) { 847 if (!dma_debug_dent) {
890 pr_err("DMA-API: can not create debugfs directory\n"); 848 pr_err("can not create debugfs directory\n");
891 return -ENOMEM; 849 return -ENOMEM;
892 } 850 }
893 851
@@ -926,6 +884,12 @@ static int dma_debug_fs_init(void)
926 if (!min_free_entries_dent) 884 if (!min_free_entries_dent)
927 goto out_err; 885 goto out_err;
928 886
887 nr_total_entries_dent = debugfs_create_u32("nr_total_entries", 0444,
888 dma_debug_dent,
889 &nr_total_entries);
890 if (!nr_total_entries_dent)
891 goto out_err;
892
929 filter_dent = debugfs_create_file("driver_filter", 0644, 893 filter_dent = debugfs_create_file("driver_filter", 0644,
930 dma_debug_dent, NULL, &filter_fops); 894 dma_debug_dent, NULL, &filter_fops);
931 if (!filter_dent) 895 if (!filter_dent)
@@ -973,7 +937,7 @@ static int dma_debug_device_change(struct notifier_block *nb, unsigned long acti
973 count = device_dma_allocations(dev, &entry); 937 count = device_dma_allocations(dev, &entry);
974 if (count == 0) 938 if (count == 0)
975 break; 939 break;
976 err_printk(dev, entry, "DMA-API: device driver has pending " 940 err_printk(dev, entry, "device driver has pending "
977 "DMA allocations while released from device " 941 "DMA allocations while released from device "
978 "[count=%d]\n" 942 "[count=%d]\n"
979 "One of leaked entries details: " 943 "One of leaked entries details: "
@@ -1009,7 +973,7 @@ void dma_debug_add_bus(struct bus_type *bus)
1009 973
1010static int dma_debug_init(void) 974static int dma_debug_init(void)
1011{ 975{
1012 int i; 976 int i, nr_pages;
1013 977
1014 /* Do not use dma_debug_initialized here, since we really want to be 978 /* Do not use dma_debug_initialized here, since we really want to be
1015 * called to set dma_debug_initialized 979 * called to set dma_debug_initialized
@@ -1023,24 +987,31 @@ static int dma_debug_init(void)
1023 } 987 }
1024 988
1025 if (dma_debug_fs_init() != 0) { 989 if (dma_debug_fs_init() != 0) {
1026 pr_err("DMA-API: error creating debugfs entries - disabling\n"); 990 pr_err("error creating debugfs entries - disabling\n");
1027 global_disable = true; 991 global_disable = true;
1028 992
1029 return 0; 993 return 0;
1030 } 994 }
1031 995
1032 if (prealloc_memory(nr_prealloc_entries) != 0) { 996 nr_pages = DIV_ROUND_UP(nr_prealloc_entries, DMA_DEBUG_DYNAMIC_ENTRIES);
1033 pr_err("DMA-API: debugging out of memory error - disabled\n"); 997 for (i = 0; i < nr_pages; ++i)
998 dma_debug_create_entries(GFP_KERNEL);
999 if (num_free_entries >= nr_prealloc_entries) {
1000 pr_info("preallocated %d debug entries\n", nr_total_entries);
1001 } else if (num_free_entries > 0) {
1002 pr_warn("%d debug entries requested but only %d allocated\n",
1003 nr_prealloc_entries, nr_total_entries);
1004 } else {
1005 pr_err("debugging out of memory error - disabled\n");
1034 global_disable = true; 1006 global_disable = true;
1035 1007
1036 return 0; 1008 return 0;
1037 } 1009 }
1038 1010 min_free_entries = num_free_entries;
1039 nr_total_entries = num_free_entries;
1040 1011
1041 dma_debug_initialized = true; 1012 dma_debug_initialized = true;
1042 1013
1043 pr_info("DMA-API: debugging enabled by kernel config\n"); 1014 pr_info("debugging enabled by kernel config\n");
1044 return 0; 1015 return 0;
1045} 1016}
1046core_initcall(dma_debug_init); 1017core_initcall(dma_debug_init);
@@ -1051,7 +1022,7 @@ static __init int dma_debug_cmdline(char *str)
1051 return -EINVAL; 1022 return -EINVAL;
1052 1023
1053 if (strncmp(str, "off", 3) == 0) { 1024 if (strncmp(str, "off", 3) == 0) {
1054 pr_info("DMA-API: debugging disabled on kernel command line\n"); 1025 pr_info("debugging disabled on kernel command line\n");
1055 global_disable = true; 1026 global_disable = true;
1056 } 1027 }
1057 1028
@@ -1085,11 +1056,11 @@ static void check_unmap(struct dma_debug_entry *ref)
1085 1056
1086 if (dma_mapping_error(ref->dev, ref->dev_addr)) { 1057 if (dma_mapping_error(ref->dev, ref->dev_addr)) {
1087 err_printk(ref->dev, NULL, 1058 err_printk(ref->dev, NULL,
1088 "DMA-API: device driver tries to free an " 1059 "device driver tries to free an "
1089 "invalid DMA memory address\n"); 1060 "invalid DMA memory address\n");
1090 } else { 1061 } else {
1091 err_printk(ref->dev, NULL, 1062 err_printk(ref->dev, NULL,
1092 "DMA-API: device driver tries to free DMA " 1063 "device driver tries to free DMA "
1093 "memory it has not allocated [device " 1064 "memory it has not allocated [device "
1094 "address=0x%016llx] [size=%llu bytes]\n", 1065 "address=0x%016llx] [size=%llu bytes]\n",
1095 ref->dev_addr, ref->size); 1066 ref->dev_addr, ref->size);
@@ -1098,7 +1069,7 @@ static void check_unmap(struct dma_debug_entry *ref)
1098 } 1069 }
1099 1070
1100 if (ref->size != entry->size) { 1071 if (ref->size != entry->size) {
1101 err_printk(ref->dev, entry, "DMA-API: device driver frees " 1072 err_printk(ref->dev, entry, "device driver frees "
1102 "DMA memory with different size " 1073 "DMA memory with different size "
1103 "[device address=0x%016llx] [map size=%llu bytes] " 1074 "[device address=0x%016llx] [map size=%llu bytes] "
1104 "[unmap size=%llu bytes]\n", 1075 "[unmap size=%llu bytes]\n",
@@ -1106,7 +1077,7 @@ static void check_unmap(struct dma_debug_entry *ref)
1106 } 1077 }
1107 1078
1108 if (ref->type != entry->type) { 1079 if (ref->type != entry->type) {
1109 err_printk(ref->dev, entry, "DMA-API: device driver frees " 1080 err_printk(ref->dev, entry, "device driver frees "
1110 "DMA memory with wrong function " 1081 "DMA memory with wrong function "
1111 "[device address=0x%016llx] [size=%llu bytes] " 1082 "[device address=0x%016llx] [size=%llu bytes] "
1112 "[mapped as %s] [unmapped as %s]\n", 1083 "[mapped as %s] [unmapped as %s]\n",
@@ -1114,7 +1085,7 @@ static void check_unmap(struct dma_debug_entry *ref)
1114 type2name[entry->type], type2name[ref->type]); 1085 type2name[entry->type], type2name[ref->type]);
1115 } else if ((entry->type == dma_debug_coherent) && 1086 } else if ((entry->type == dma_debug_coherent) &&
1116 (phys_addr(ref) != phys_addr(entry))) { 1087 (phys_addr(ref) != phys_addr(entry))) {
1117 err_printk(ref->dev, entry, "DMA-API: device driver frees " 1088 err_printk(ref->dev, entry, "device driver frees "
1118 "DMA memory with different CPU address " 1089 "DMA memory with different CPU address "
1119 "[device address=0x%016llx] [size=%llu bytes] " 1090 "[device address=0x%016llx] [size=%llu bytes] "
1120 "[cpu alloc address=0x%016llx] " 1091 "[cpu alloc address=0x%016llx] "
@@ -1126,7 +1097,7 @@ static void check_unmap(struct dma_debug_entry *ref)
1126 1097
1127 if (ref->sg_call_ents && ref->type == dma_debug_sg && 1098 if (ref->sg_call_ents && ref->type == dma_debug_sg &&
1128 ref->sg_call_ents != entry->sg_call_ents) { 1099 ref->sg_call_ents != entry->sg_call_ents) {
1129 err_printk(ref->dev, entry, "DMA-API: device driver frees " 1100 err_printk(ref->dev, entry, "device driver frees "
1130 "DMA sg list with different entry count " 1101 "DMA sg list with different entry count "
1131 "[map count=%d] [unmap count=%d]\n", 1102 "[map count=%d] [unmap count=%d]\n",
1132 entry->sg_call_ents, ref->sg_call_ents); 1103 entry->sg_call_ents, ref->sg_call_ents);
@@ -1137,7 +1108,7 @@ static void check_unmap(struct dma_debug_entry *ref)
1137 * DMA API don't handle this properly, so check for it here 1108 * DMA API don't handle this properly, so check for it here
1138 */ 1109 */
1139 if (ref->direction != entry->direction) { 1110 if (ref->direction != entry->direction) {
1140 err_printk(ref->dev, entry, "DMA-API: device driver frees " 1111 err_printk(ref->dev, entry, "device driver frees "
1141 "DMA memory with different direction " 1112 "DMA memory with different direction "
1142 "[device address=0x%016llx] [size=%llu bytes] " 1113 "[device address=0x%016llx] [size=%llu bytes] "
1143 "[mapped with %s] [unmapped with %s]\n", 1114 "[mapped with %s] [unmapped with %s]\n",
@@ -1153,7 +1124,7 @@ static void check_unmap(struct dma_debug_entry *ref)
1153 */ 1124 */
1154 if (entry->map_err_type == MAP_ERR_NOT_CHECKED) { 1125 if (entry->map_err_type == MAP_ERR_NOT_CHECKED) {
1155 err_printk(ref->dev, entry, 1126 err_printk(ref->dev, entry,
1156 "DMA-API: device driver failed to check map error" 1127 "device driver failed to check map error"
1157 "[device address=0x%016llx] [size=%llu bytes] " 1128 "[device address=0x%016llx] [size=%llu bytes] "
1158 "[mapped as %s]", 1129 "[mapped as %s]",
1159 ref->dev_addr, ref->size, 1130 ref->dev_addr, ref->size,
@@ -1178,7 +1149,7 @@ static void check_for_stack(struct device *dev,
1178 return; 1149 return;
1179 addr = page_address(page) + offset; 1150 addr = page_address(page) + offset;
1180 if (object_is_on_stack(addr)) 1151 if (object_is_on_stack(addr))
1181 err_printk(dev, NULL, "DMA-API: device driver maps memory from stack [addr=%p]\n", addr); 1152 err_printk(dev, NULL, "device driver maps memory from stack [addr=%p]\n", addr);
1182 } else { 1153 } else {
1183 /* Stack is vmalloced. */ 1154 /* Stack is vmalloced. */
1184 int i; 1155 int i;
@@ -1188,7 +1159,7 @@ static void check_for_stack(struct device *dev,
1188 continue; 1159 continue;
1189 1160
1190 addr = (u8 *)current->stack + i * PAGE_SIZE + offset; 1161 addr = (u8 *)current->stack + i * PAGE_SIZE + offset;
1191 err_printk(dev, NULL, "DMA-API: device driver maps memory from stack [probable addr=%p]\n", addr); 1162 err_printk(dev, NULL, "device driver maps memory from stack [probable addr=%p]\n", addr);
1192 break; 1163 break;
1193 } 1164 }
1194 } 1165 }
@@ -1208,7 +1179,7 @@ static void check_for_illegal_area(struct device *dev, void *addr, unsigned long
1208{ 1179{
1209 if (overlap(addr, len, _stext, _etext) || 1180 if (overlap(addr, len, _stext, _etext) ||
1210 overlap(addr, len, __start_rodata, __end_rodata)) 1181 overlap(addr, len, __start_rodata, __end_rodata))
1211 err_printk(dev, NULL, "DMA-API: device driver maps memory from kernel text or rodata [addr=%p] [len=%lu]\n", addr, len); 1182 err_printk(dev, NULL, "device driver maps memory from kernel text or rodata [addr=%p] [len=%lu]\n", addr, len);
1212} 1183}
1213 1184
1214static void check_sync(struct device *dev, 1185static void check_sync(struct device *dev,
@@ -1224,7 +1195,7 @@ static void check_sync(struct device *dev,
1224 entry = bucket_find_contain(&bucket, ref, &flags); 1195 entry = bucket_find_contain(&bucket, ref, &flags);
1225 1196
1226 if (!entry) { 1197 if (!entry) {
1227 err_printk(dev, NULL, "DMA-API: device driver tries " 1198 err_printk(dev, NULL, "device driver tries "
1228 "to sync DMA memory it has not allocated " 1199 "to sync DMA memory it has not allocated "
1229 "[device address=0x%016llx] [size=%llu bytes]\n", 1200 "[device address=0x%016llx] [size=%llu bytes]\n",
1230 (unsigned long long)ref->dev_addr, ref->size); 1201 (unsigned long long)ref->dev_addr, ref->size);
@@ -1232,7 +1203,7 @@ static void check_sync(struct device *dev,
1232 } 1203 }
1233 1204
1234 if (ref->size > entry->size) { 1205 if (ref->size > entry->size) {
1235 err_printk(dev, entry, "DMA-API: device driver syncs" 1206 err_printk(dev, entry, "device driver syncs"
1236 " DMA memory outside allocated range " 1207 " DMA memory outside allocated range "
1237 "[device address=0x%016llx] " 1208 "[device address=0x%016llx] "
1238 "[allocation size=%llu bytes] " 1209 "[allocation size=%llu bytes] "
@@ -1245,7 +1216,7 @@ static void check_sync(struct device *dev,
1245 goto out; 1216 goto out;
1246 1217
1247 if (ref->direction != entry->direction) { 1218 if (ref->direction != entry->direction) {
1248 err_printk(dev, entry, "DMA-API: device driver syncs " 1219 err_printk(dev, entry, "device driver syncs "
1249 "DMA memory with different direction " 1220 "DMA memory with different direction "
1250 "[device address=0x%016llx] [size=%llu bytes] " 1221 "[device address=0x%016llx] [size=%llu bytes] "
1251 "[mapped with %s] [synced with %s]\n", 1222 "[mapped with %s] [synced with %s]\n",
@@ -1256,7 +1227,7 @@ static void check_sync(struct device *dev,
1256 1227
1257 if (to_cpu && !(entry->direction == DMA_FROM_DEVICE) && 1228 if (to_cpu && !(entry->direction == DMA_FROM_DEVICE) &&
1258 !(ref->direction == DMA_TO_DEVICE)) 1229 !(ref->direction == DMA_TO_DEVICE))
1259 err_printk(dev, entry, "DMA-API: device driver syncs " 1230 err_printk(dev, entry, "device driver syncs "
1260 "device read-only DMA memory for cpu " 1231 "device read-only DMA memory for cpu "
1261 "[device address=0x%016llx] [size=%llu bytes] " 1232 "[device address=0x%016llx] [size=%llu bytes] "
1262 "[mapped with %s] [synced with %s]\n", 1233 "[mapped with %s] [synced with %s]\n",
@@ -1266,7 +1237,7 @@ static void check_sync(struct device *dev,
1266 1237
1267 if (!to_cpu && !(entry->direction == DMA_TO_DEVICE) && 1238 if (!to_cpu && !(entry->direction == DMA_TO_DEVICE) &&
1268 !(ref->direction == DMA_FROM_DEVICE)) 1239 !(ref->direction == DMA_FROM_DEVICE))
1269 err_printk(dev, entry, "DMA-API: device driver syncs " 1240 err_printk(dev, entry, "device driver syncs "
1270 "device write-only DMA memory to device " 1241 "device write-only DMA memory to device "
1271 "[device address=0x%016llx] [size=%llu bytes] " 1242 "[device address=0x%016llx] [size=%llu bytes] "
1272 "[mapped with %s] [synced with %s]\n", 1243 "[mapped with %s] [synced with %s]\n",
@@ -1276,7 +1247,7 @@ static void check_sync(struct device *dev,
1276 1247
1277 if (ref->sg_call_ents && ref->type == dma_debug_sg && 1248 if (ref->sg_call_ents && ref->type == dma_debug_sg &&
1278 ref->sg_call_ents != entry->sg_call_ents) { 1249 ref->sg_call_ents != entry->sg_call_ents) {
1279 err_printk(ref->dev, entry, "DMA-API: device driver syncs " 1250 err_printk(ref->dev, entry, "device driver syncs "
1280 "DMA sg list with different entry count " 1251 "DMA sg list with different entry count "
1281 "[map count=%d] [sync count=%d]\n", 1252 "[map count=%d] [sync count=%d]\n",
1282 entry->sg_call_ents, ref->sg_call_ents); 1253 entry->sg_call_ents, ref->sg_call_ents);
@@ -1297,7 +1268,7 @@ static void check_sg_segment(struct device *dev, struct scatterlist *sg)
1297 * whoever generated the list forgot to check them. 1268 * whoever generated the list forgot to check them.
1298 */ 1269 */
1299 if (sg->length > max_seg) 1270 if (sg->length > max_seg)
1300 err_printk(dev, NULL, "DMA-API: mapping sg segment longer than device claims to support [len=%u] [max=%u]\n", 1271 err_printk(dev, NULL, "mapping sg segment longer than device claims to support [len=%u] [max=%u]\n",
1301 sg->length, max_seg); 1272 sg->length, max_seg);
1302 /* 1273 /*
1303 * In some cases this could potentially be the DMA API 1274 * In some cases this could potentially be the DMA API
@@ -1307,7 +1278,7 @@ static void check_sg_segment(struct device *dev, struct scatterlist *sg)
1307 start = sg_dma_address(sg); 1278 start = sg_dma_address(sg);
1308 end = start + sg_dma_len(sg) - 1; 1279 end = start + sg_dma_len(sg) - 1;
1309 if ((start ^ end) & ~boundary) 1280 if ((start ^ end) & ~boundary)
1310 err_printk(dev, NULL, "DMA-API: mapping sg segment across boundary [start=0x%016llx] [end=0x%016llx] [boundary=0x%016llx]\n", 1281 err_printk(dev, NULL, "mapping sg segment across boundary [start=0x%016llx] [end=0x%016llx] [boundary=0x%016llx]\n",
1311 start, end, boundary); 1282 start, end, boundary);
1312#endif 1283#endif
1313} 1284}
@@ -1319,11 +1290,11 @@ void debug_dma_map_single(struct device *dev, const void *addr,
1319 return; 1290 return;
1320 1291
1321 if (!virt_addr_valid(addr)) 1292 if (!virt_addr_valid(addr))
1322 err_printk(dev, NULL, "DMA-API: device driver maps memory from invalid area [addr=%p] [len=%lu]\n", 1293 err_printk(dev, NULL, "device driver maps memory from invalid area [addr=%p] [len=%lu]\n",
1323 addr, len); 1294 addr, len);
1324 1295
1325 if (is_vmalloc_addr(addr)) 1296 if (is_vmalloc_addr(addr))
1326 err_printk(dev, NULL, "DMA-API: device driver maps memory from vmalloc area [addr=%p] [len=%lu]\n", 1297 err_printk(dev, NULL, "device driver maps memory from vmalloc area [addr=%p] [len=%lu]\n",
1327 addr, len); 1298 addr, len);
1328} 1299}
1329EXPORT_SYMBOL(debug_dma_map_single); 1300EXPORT_SYMBOL(debug_dma_map_single);
@@ -1662,48 +1633,6 @@ void debug_dma_sync_single_for_device(struct device *dev,
1662} 1633}
1663EXPORT_SYMBOL(debug_dma_sync_single_for_device); 1634EXPORT_SYMBOL(debug_dma_sync_single_for_device);
1664 1635
1665void debug_dma_sync_single_range_for_cpu(struct device *dev,
1666 dma_addr_t dma_handle,
1667 unsigned long offset, size_t size,
1668 int direction)
1669{
1670 struct dma_debug_entry ref;
1671
1672 if (unlikely(dma_debug_disabled()))
1673 return;
1674
1675 ref.type = dma_debug_single;
1676 ref.dev = dev;
1677 ref.dev_addr = dma_handle;
1678 ref.size = offset + size;
1679 ref.direction = direction;
1680 ref.sg_call_ents = 0;
1681
1682 check_sync(dev, &ref, true);
1683}
1684EXPORT_SYMBOL(debug_dma_sync_single_range_for_cpu);
1685
1686void debug_dma_sync_single_range_for_device(struct device *dev,
1687 dma_addr_t dma_handle,
1688 unsigned long offset,
1689 size_t size, int direction)
1690{
1691 struct dma_debug_entry ref;
1692
1693 if (unlikely(dma_debug_disabled()))
1694 return;
1695
1696 ref.type = dma_debug_single;
1697 ref.dev = dev;
1698 ref.dev_addr = dma_handle;
1699 ref.size = offset + size;
1700 ref.direction = direction;
1701 ref.sg_call_ents = 0;
1702
1703 check_sync(dev, &ref, false);
1704}
1705EXPORT_SYMBOL(debug_dma_sync_single_range_for_device);
1706
1707void debug_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, 1636void debug_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
1708 int nelems, int direction) 1637 int nelems, int direction)
1709{ 1638{
@@ -1780,7 +1709,7 @@ static int __init dma_debug_driver_setup(char *str)
1780 } 1709 }
1781 1710
1782 if (current_driver_name[0]) 1711 if (current_driver_name[0])
1783 pr_info("DMA-API: enable driver filter for driver [%s]\n", 1712 pr_info("enable driver filter for driver [%s]\n",
1784 current_driver_name); 1713 current_driver_name);
1785 1714
1786 1715
diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c
index 375c77e8d52f..355d16acee6d 100644
--- a/kernel/dma/direct.c
+++ b/kernel/dma/direct.c
@@ -13,6 +13,7 @@
13#include <linux/dma-noncoherent.h> 13#include <linux/dma-noncoherent.h>
14#include <linux/pfn.h> 14#include <linux/pfn.h>
15#include <linux/set_memory.h> 15#include <linux/set_memory.h>
16#include <linux/swiotlb.h>
16 17
17/* 18/*
18 * Most architectures use ZONE_DMA for the first 16 Megabytes, but 19 * Most architectures use ZONE_DMA for the first 16 Megabytes, but
@@ -30,27 +31,16 @@ static inline bool force_dma_unencrypted(void)
30 return sev_active(); 31 return sev_active();
31} 32}
32 33
33static bool 34static void report_addr(struct device *dev, dma_addr_t dma_addr, size_t size)
34check_addr(struct device *dev, dma_addr_t dma_addr, size_t size,
35 const char *caller)
36{ 35{
37 if (unlikely(dev && !dma_capable(dev, dma_addr, size))) { 36 if (!dev->dma_mask) {
38 if (!dev->dma_mask) { 37 dev_err_once(dev, "DMA map on device without dma_mask\n");
39 dev_err(dev, 38 } else if (*dev->dma_mask >= DMA_BIT_MASK(32) || dev->bus_dma_mask) {
40 "%s: call on device without dma_mask\n", 39 dev_err_once(dev,
41 caller); 40 "overflow %pad+%zu of DMA mask %llx bus mask %llx\n",
42 return false; 41 &dma_addr, size, *dev->dma_mask, dev->bus_dma_mask);
43 }
44
45 if (*dev->dma_mask >= DMA_BIT_MASK(32) || dev->bus_dma_mask) {
46 dev_err(dev,
47 "%s: overflow %pad+%zu of device mask %llx bus mask %llx\n",
48 caller, &dma_addr, size,
49 *dev->dma_mask, dev->bus_dma_mask);
50 }
51 return false;
52 } 42 }
53 return true; 43 WARN_ON_ONCE(1);
54} 44}
55 45
56static inline dma_addr_t phys_to_dma_direct(struct device *dev, 46static inline dma_addr_t phys_to_dma_direct(struct device *dev,
@@ -103,14 +93,13 @@ static bool dma_coherent_ok(struct device *dev, phys_addr_t phys, size_t size)
103 min_not_zero(dev->coherent_dma_mask, dev->bus_dma_mask); 93 min_not_zero(dev->coherent_dma_mask, dev->bus_dma_mask);
104} 94}
105 95
106void *dma_direct_alloc_pages(struct device *dev, size_t size, 96struct page *__dma_direct_alloc_pages(struct device *dev, size_t size,
107 dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) 97 dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
108{ 98{
109 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; 99 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
110 int page_order = get_order(size); 100 int page_order = get_order(size);
111 struct page *page = NULL; 101 struct page *page = NULL;
112 u64 phys_mask; 102 u64 phys_mask;
113 void *ret;
114 103
115 if (attrs & DMA_ATTR_NO_WARN) 104 if (attrs & DMA_ATTR_NO_WARN)
116 gfp |= __GFP_NOWARN; 105 gfp |= __GFP_NOWARN;
@@ -150,11 +139,34 @@ again:
150 } 139 }
151 } 140 }
152 141
142 return page;
143}
144
145void *dma_direct_alloc_pages(struct device *dev, size_t size,
146 dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
147{
148 struct page *page;
149 void *ret;
150
151 page = __dma_direct_alloc_pages(dev, size, dma_handle, gfp, attrs);
153 if (!page) 152 if (!page)
154 return NULL; 153 return NULL;
154
155 if (PageHighMem(page)) {
156 /*
157 * Depending on the cma= arguments and per-arch setup
158 * dma_alloc_from_contiguous could return highmem pages.
159 * Without remapping there is no way to return them here,
160 * so log an error and fail.
161 */
162 dev_info(dev, "Rejecting highmem page from CMA.\n");
163 __dma_direct_free_pages(dev, size, page);
164 return NULL;
165 }
166
155 ret = page_address(page); 167 ret = page_address(page);
156 if (force_dma_unencrypted()) { 168 if (force_dma_unencrypted()) {
157 set_memory_decrypted((unsigned long)ret, 1 << page_order); 169 set_memory_decrypted((unsigned long)ret, 1 << get_order(size));
158 *dma_handle = __phys_to_dma(dev, page_to_phys(page)); 170 *dma_handle = __phys_to_dma(dev, page_to_phys(page));
159 } else { 171 } else {
160 *dma_handle = phys_to_dma(dev, page_to_phys(page)); 172 *dma_handle = phys_to_dma(dev, page_to_phys(page));
@@ -163,20 +175,22 @@ again:
163 return ret; 175 return ret;
164} 176}
165 177
166/* 178void __dma_direct_free_pages(struct device *dev, size_t size, struct page *page)
167 * NOTE: this function must never look at the dma_addr argument, because we want 179{
168 * to be able to use it as a helper for iommu implementations as well. 180 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
169 */ 181
182 if (!dma_release_from_contiguous(dev, page, count))
183 __free_pages(page, get_order(size));
184}
185
170void dma_direct_free_pages(struct device *dev, size_t size, void *cpu_addr, 186void dma_direct_free_pages(struct device *dev, size_t size, void *cpu_addr,
171 dma_addr_t dma_addr, unsigned long attrs) 187 dma_addr_t dma_addr, unsigned long attrs)
172{ 188{
173 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
174 unsigned int page_order = get_order(size); 189 unsigned int page_order = get_order(size);
175 190
176 if (force_dma_unencrypted()) 191 if (force_dma_unencrypted())
177 set_memory_encrypted((unsigned long)cpu_addr, 1 << page_order); 192 set_memory_encrypted((unsigned long)cpu_addr, 1 << page_order);
178 if (!dma_release_from_contiguous(dev, virt_to_page(cpu_addr), count)) 193 __dma_direct_free_pages(dev, size, virt_to_page(cpu_addr));
179 free_pages((unsigned long)cpu_addr, page_order);
180} 194}
181 195
182void *dma_direct_alloc(struct device *dev, size_t size, 196void *dma_direct_alloc(struct device *dev, size_t size,
@@ -196,67 +210,111 @@ void dma_direct_free(struct device *dev, size_t size,
196 dma_direct_free_pages(dev, size, cpu_addr, dma_addr, attrs); 210 dma_direct_free_pages(dev, size, cpu_addr, dma_addr, attrs);
197} 211}
198 212
199static void dma_direct_sync_single_for_device(struct device *dev, 213#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \
214 defined(CONFIG_SWIOTLB)
215void dma_direct_sync_single_for_device(struct device *dev,
200 dma_addr_t addr, size_t size, enum dma_data_direction dir) 216 dma_addr_t addr, size_t size, enum dma_data_direction dir)
201{ 217{
202 if (dev_is_dma_coherent(dev)) 218 phys_addr_t paddr = dma_to_phys(dev, addr);
203 return; 219
204 arch_sync_dma_for_device(dev, dma_to_phys(dev, addr), size, dir); 220 if (unlikely(is_swiotlb_buffer(paddr)))
221 swiotlb_tbl_sync_single(dev, paddr, size, dir, SYNC_FOR_DEVICE);
222
223 if (!dev_is_dma_coherent(dev))
224 arch_sync_dma_for_device(dev, paddr, size, dir);
205} 225}
226EXPORT_SYMBOL(dma_direct_sync_single_for_device);
206 227
207static void dma_direct_sync_sg_for_device(struct device *dev, 228void dma_direct_sync_sg_for_device(struct device *dev,
208 struct scatterlist *sgl, int nents, enum dma_data_direction dir) 229 struct scatterlist *sgl, int nents, enum dma_data_direction dir)
209{ 230{
210 struct scatterlist *sg; 231 struct scatterlist *sg;
211 int i; 232 int i;
212 233
213 if (dev_is_dma_coherent(dev)) 234 for_each_sg(sgl, sg, nents, i) {
214 return; 235 if (unlikely(is_swiotlb_buffer(sg_phys(sg))))
236 swiotlb_tbl_sync_single(dev, sg_phys(sg), sg->length,
237 dir, SYNC_FOR_DEVICE);
215 238
216 for_each_sg(sgl, sg, nents, i) 239 if (!dev_is_dma_coherent(dev))
217 arch_sync_dma_for_device(dev, sg_phys(sg), sg->length, dir); 240 arch_sync_dma_for_device(dev, sg_phys(sg), sg->length,
241 dir);
242 }
218} 243}
244EXPORT_SYMBOL(dma_direct_sync_sg_for_device);
245#endif
219 246
220#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \ 247#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
221 defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL) 248 defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL) || \
222static void dma_direct_sync_single_for_cpu(struct device *dev, 249 defined(CONFIG_SWIOTLB)
250void dma_direct_sync_single_for_cpu(struct device *dev,
223 dma_addr_t addr, size_t size, enum dma_data_direction dir) 251 dma_addr_t addr, size_t size, enum dma_data_direction dir)
224{ 252{
225 if (dev_is_dma_coherent(dev)) 253 phys_addr_t paddr = dma_to_phys(dev, addr);
226 return; 254
227 arch_sync_dma_for_cpu(dev, dma_to_phys(dev, addr), size, dir); 255 if (!dev_is_dma_coherent(dev)) {
228 arch_sync_dma_for_cpu_all(dev); 256 arch_sync_dma_for_cpu(dev, paddr, size, dir);
257 arch_sync_dma_for_cpu_all(dev);
258 }
259
260 if (unlikely(is_swiotlb_buffer(paddr)))
261 swiotlb_tbl_sync_single(dev, paddr, size, dir, SYNC_FOR_CPU);
229} 262}
263EXPORT_SYMBOL(dma_direct_sync_single_for_cpu);
230 264
231static void dma_direct_sync_sg_for_cpu(struct device *dev, 265void dma_direct_sync_sg_for_cpu(struct device *dev,
232 struct scatterlist *sgl, int nents, enum dma_data_direction dir) 266 struct scatterlist *sgl, int nents, enum dma_data_direction dir)
233{ 267{
234 struct scatterlist *sg; 268 struct scatterlist *sg;
235 int i; 269 int i;
236 270
237 if (dev_is_dma_coherent(dev)) 271 for_each_sg(sgl, sg, nents, i) {
238 return; 272 if (!dev_is_dma_coherent(dev))
273 arch_sync_dma_for_cpu(dev, sg_phys(sg), sg->length, dir);
274
275 if (unlikely(is_swiotlb_buffer(sg_phys(sg))))
276 swiotlb_tbl_sync_single(dev, sg_phys(sg), sg->length, dir,
277 SYNC_FOR_CPU);
278 }
239 279
240 for_each_sg(sgl, sg, nents, i) 280 if (!dev_is_dma_coherent(dev))
241 arch_sync_dma_for_cpu(dev, sg_phys(sg), sg->length, dir); 281 arch_sync_dma_for_cpu_all(dev);
242 arch_sync_dma_for_cpu_all(dev);
243} 282}
283EXPORT_SYMBOL(dma_direct_sync_sg_for_cpu);
244 284
245static void dma_direct_unmap_page(struct device *dev, dma_addr_t addr, 285void dma_direct_unmap_page(struct device *dev, dma_addr_t addr,
246 size_t size, enum dma_data_direction dir, unsigned long attrs) 286 size_t size, enum dma_data_direction dir, unsigned long attrs)
247{ 287{
288 phys_addr_t phys = dma_to_phys(dev, addr);
289
248 if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) 290 if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
249 dma_direct_sync_single_for_cpu(dev, addr, size, dir); 291 dma_direct_sync_single_for_cpu(dev, addr, size, dir);
292
293 if (unlikely(is_swiotlb_buffer(phys)))
294 swiotlb_tbl_unmap_single(dev, phys, size, dir, attrs);
250} 295}
296EXPORT_SYMBOL(dma_direct_unmap_page);
251 297
252static void dma_direct_unmap_sg(struct device *dev, struct scatterlist *sgl, 298void dma_direct_unmap_sg(struct device *dev, struct scatterlist *sgl,
253 int nents, enum dma_data_direction dir, unsigned long attrs) 299 int nents, enum dma_data_direction dir, unsigned long attrs)
254{ 300{
255 if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) 301 struct scatterlist *sg;
256 dma_direct_sync_sg_for_cpu(dev, sgl, nents, dir); 302 int i;
303
304 for_each_sg(sgl, sg, nents, i)
305 dma_direct_unmap_page(dev, sg->dma_address, sg_dma_len(sg), dir,
306 attrs);
257} 307}
308EXPORT_SYMBOL(dma_direct_unmap_sg);
258#endif 309#endif
259 310
311static inline bool dma_direct_possible(struct device *dev, dma_addr_t dma_addr,
312 size_t size)
313{
314 return swiotlb_force != SWIOTLB_FORCE &&
315 (!dev || dma_capable(dev, dma_addr, size));
316}
317
260dma_addr_t dma_direct_map_page(struct device *dev, struct page *page, 318dma_addr_t dma_direct_map_page(struct device *dev, struct page *page,
261 unsigned long offset, size_t size, enum dma_data_direction dir, 319 unsigned long offset, size_t size, enum dma_data_direction dir,
262 unsigned long attrs) 320 unsigned long attrs)
@@ -264,13 +322,17 @@ dma_addr_t dma_direct_map_page(struct device *dev, struct page *page,
264 phys_addr_t phys = page_to_phys(page) + offset; 322 phys_addr_t phys = page_to_phys(page) + offset;
265 dma_addr_t dma_addr = phys_to_dma(dev, phys); 323 dma_addr_t dma_addr = phys_to_dma(dev, phys);
266 324
267 if (!check_addr(dev, dma_addr, size, __func__)) 325 if (unlikely(!dma_direct_possible(dev, dma_addr, size)) &&
268 return DIRECT_MAPPING_ERROR; 326 !swiotlb_map(dev, &phys, &dma_addr, size, dir, attrs)) {
327 report_addr(dev, dma_addr, size);
328 return DMA_MAPPING_ERROR;
329 }
269 330
270 if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) 331 if (!dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
271 dma_direct_sync_single_for_device(dev, dma_addr, size, dir); 332 arch_sync_dma_for_device(dev, phys, size, dir);
272 return dma_addr; 333 return dma_addr;
273} 334}
335EXPORT_SYMBOL(dma_direct_map_page);
274 336
275int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, int nents, 337int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, int nents,
276 enum dma_data_direction dir, unsigned long attrs) 338 enum dma_data_direction dir, unsigned long attrs)
@@ -279,18 +341,20 @@ int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, int nents,
279 struct scatterlist *sg; 341 struct scatterlist *sg;
280 342
281 for_each_sg(sgl, sg, nents, i) { 343 for_each_sg(sgl, sg, nents, i) {
282 BUG_ON(!sg_page(sg)); 344 sg->dma_address = dma_direct_map_page(dev, sg_page(sg),
283 345 sg->offset, sg->length, dir, attrs);
284 sg_dma_address(sg) = phys_to_dma(dev, sg_phys(sg)); 346 if (sg->dma_address == DMA_MAPPING_ERROR)
285 if (!check_addr(dev, sg_dma_address(sg), sg->length, __func__)) 347 goto out_unmap;
286 return 0;
287 sg_dma_len(sg) = sg->length; 348 sg_dma_len(sg) = sg->length;
288 } 349 }
289 350
290 if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
291 dma_direct_sync_sg_for_device(dev, sgl, nents, dir);
292 return nents; 351 return nents;
352
353out_unmap:
354 dma_direct_unmap_sg(dev, sgl, i, dir, attrs | DMA_ATTR_SKIP_CPU_SYNC);
355 return 0;
293} 356}
357EXPORT_SYMBOL(dma_direct_map_sg);
294 358
295/* 359/*
296 * Because 32-bit DMA masks are so common we expect every architecture to be 360 * Because 32-bit DMA masks are so common we expect every architecture to be
@@ -316,31 +380,3 @@ int dma_direct_supported(struct device *dev, u64 mask)
316 */ 380 */
317 return mask >= __phys_to_dma(dev, min_mask); 381 return mask >= __phys_to_dma(dev, min_mask);
318} 382}
319
320int dma_direct_mapping_error(struct device *dev, dma_addr_t dma_addr)
321{
322 return dma_addr == DIRECT_MAPPING_ERROR;
323}
324
325const struct dma_map_ops dma_direct_ops = {
326 .alloc = dma_direct_alloc,
327 .free = dma_direct_free,
328 .map_page = dma_direct_map_page,
329 .map_sg = dma_direct_map_sg,
330#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE)
331 .sync_single_for_device = dma_direct_sync_single_for_device,
332 .sync_sg_for_device = dma_direct_sync_sg_for_device,
333#endif
334#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
335 defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL)
336 .sync_single_for_cpu = dma_direct_sync_single_for_cpu,
337 .sync_sg_for_cpu = dma_direct_sync_sg_for_cpu,
338 .unmap_page = dma_direct_unmap_page,
339 .unmap_sg = dma_direct_unmap_sg,
340#endif
341 .get_required_mask = dma_direct_get_required_mask,
342 .dma_supported = dma_direct_supported,
343 .mapping_error = dma_direct_mapping_error,
344 .cache_sync = arch_dma_cache_sync,
345};
346EXPORT_SYMBOL(dma_direct_ops);
diff --git a/kernel/dma/dummy.c b/kernel/dma/dummy.c
new file mode 100644
index 000000000000..05607642c888
--- /dev/null
+++ b/kernel/dma/dummy.c
@@ -0,0 +1,39 @@
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Dummy DMA ops that always fail.
4 */
5#include <linux/dma-mapping.h>
6
7static int dma_dummy_mmap(struct device *dev, struct vm_area_struct *vma,
8 void *cpu_addr, dma_addr_t dma_addr, size_t size,
9 unsigned long attrs)
10{
11 return -ENXIO;
12}
13
14static dma_addr_t dma_dummy_map_page(struct device *dev, struct page *page,
15 unsigned long offset, size_t size, enum dma_data_direction dir,
16 unsigned long attrs)
17{
18 return DMA_MAPPING_ERROR;
19}
20
21static int dma_dummy_map_sg(struct device *dev, struct scatterlist *sgl,
22 int nelems, enum dma_data_direction dir,
23 unsigned long attrs)
24{
25 return 0;
26}
27
28static int dma_dummy_supported(struct device *hwdev, u64 mask)
29{
30 return 0;
31}
32
33const struct dma_map_ops dma_dummy_ops = {
34 .mmap = dma_dummy_mmap,
35 .map_page = dma_dummy_map_page,
36 .map_sg = dma_dummy_map_sg,
37 .dma_supported = dma_dummy_supported,
38};
39EXPORT_SYMBOL(dma_dummy_ops);
diff --git a/kernel/dma/mapping.c b/kernel/dma/mapping.c
index 58dec7a92b7b..d7c34d2d1ba5 100644
--- a/kernel/dma/mapping.c
+++ b/kernel/dma/mapping.c
@@ -5,8 +5,9 @@
5 * Copyright (c) 2006 SUSE Linux Products GmbH 5 * Copyright (c) 2006 SUSE Linux Products GmbH
6 * Copyright (c) 2006 Tejun Heo <teheo@suse.de> 6 * Copyright (c) 2006 Tejun Heo <teheo@suse.de>
7 */ 7 */
8 8#include <linux/memblock.h> /* for max_pfn */
9#include <linux/acpi.h> 9#include <linux/acpi.h>
10#include <linux/dma-direct.h>
10#include <linux/dma-noncoherent.h> 11#include <linux/dma-noncoherent.h>
11#include <linux/export.h> 12#include <linux/export.h>
12#include <linux/gfp.h> 13#include <linux/gfp.h>
@@ -223,7 +224,20 @@ int dma_common_get_sgtable(struct device *dev, struct sg_table *sgt,
223 sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0); 224 sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0);
224 return ret; 225 return ret;
225} 226}
226EXPORT_SYMBOL(dma_common_get_sgtable); 227
228int dma_get_sgtable_attrs(struct device *dev, struct sg_table *sgt,
229 void *cpu_addr, dma_addr_t dma_addr, size_t size,
230 unsigned long attrs)
231{
232 const struct dma_map_ops *ops = get_dma_ops(dev);
233
234 if (!dma_is_direct(ops) && ops->get_sgtable)
235 return ops->get_sgtable(dev, sgt, cpu_addr, dma_addr, size,
236 attrs);
237 return dma_common_get_sgtable(dev, sgt, cpu_addr, dma_addr, size,
238 attrs);
239}
240EXPORT_SYMBOL(dma_get_sgtable_attrs);
227 241
228/* 242/*
229 * Create userspace mapping for the DMA-coherent memory. 243 * Create userspace mapping for the DMA-coherent memory.
@@ -261,88 +275,179 @@ int dma_common_mmap(struct device *dev, struct vm_area_struct *vma,
261 return -ENXIO; 275 return -ENXIO;
262#endif /* !CONFIG_ARCH_NO_COHERENT_DMA_MMAP */ 276#endif /* !CONFIG_ARCH_NO_COHERENT_DMA_MMAP */
263} 277}
264EXPORT_SYMBOL(dma_common_mmap);
265 278
266#ifdef CONFIG_MMU 279/**
267static struct vm_struct *__dma_common_pages_remap(struct page **pages, 280 * dma_mmap_attrs - map a coherent DMA allocation into user space
268 size_t size, unsigned long vm_flags, pgprot_t prot, 281 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
269 const void *caller) 282 * @vma: vm_area_struct describing requested user mapping
283 * @cpu_addr: kernel CPU-view address returned from dma_alloc_attrs
284 * @dma_addr: device-view address returned from dma_alloc_attrs
285 * @size: size of memory originally requested in dma_alloc_attrs
286 * @attrs: attributes of mapping properties requested in dma_alloc_attrs
287 *
288 * Map a coherent DMA buffer previously allocated by dma_alloc_attrs into user
289 * space. The coherent DMA buffer must not be freed by the driver until the
290 * user space mapping has been released.
291 */
292int dma_mmap_attrs(struct device *dev, struct vm_area_struct *vma,
293 void *cpu_addr, dma_addr_t dma_addr, size_t size,
294 unsigned long attrs)
270{ 295{
271 struct vm_struct *area; 296 const struct dma_map_ops *ops = get_dma_ops(dev);
272 297
273 area = get_vm_area_caller(size, vm_flags, caller); 298 if (!dma_is_direct(ops) && ops->mmap)
274 if (!area) 299 return ops->mmap(dev, vma, cpu_addr, dma_addr, size, attrs);
275 return NULL; 300 return dma_common_mmap(dev, vma, cpu_addr, dma_addr, size, attrs);
301}
302EXPORT_SYMBOL(dma_mmap_attrs);
276 303
277 if (map_vm_area(area, prot, pages)) { 304#ifndef ARCH_HAS_DMA_GET_REQUIRED_MASK
278 vunmap(area->addr); 305static u64 dma_default_get_required_mask(struct device *dev)
279 return NULL; 306{
307 u32 low_totalram = ((max_pfn - 1) << PAGE_SHIFT);
308 u32 high_totalram = ((max_pfn - 1) >> (32 - PAGE_SHIFT));
309 u64 mask;
310
311 if (!high_totalram) {
312 /* convert to mask just covering totalram */
313 low_totalram = (1 << (fls(low_totalram) - 1));
314 low_totalram += low_totalram - 1;
315 mask = low_totalram;
316 } else {
317 high_totalram = (1 << (fls(high_totalram) - 1));
318 high_totalram += high_totalram - 1;
319 mask = (((u64)high_totalram) << 32) + 0xffffffff;
280 } 320 }
321 return mask;
322}
281 323
282 return area; 324u64 dma_get_required_mask(struct device *dev)
325{
326 const struct dma_map_ops *ops = get_dma_ops(dev);
327
328 if (dma_is_direct(ops))
329 return dma_direct_get_required_mask(dev);
330 if (ops->get_required_mask)
331 return ops->get_required_mask(dev);
332 return dma_default_get_required_mask(dev);
283} 333}
334EXPORT_SYMBOL_GPL(dma_get_required_mask);
335#endif
284 336
285/* 337#ifndef arch_dma_alloc_attrs
286 * remaps an array of PAGE_SIZE pages into another vm_area 338#define arch_dma_alloc_attrs(dev) (true)
287 * Cannot be used in non-sleeping contexts 339#endif
288 */ 340
289void *dma_common_pages_remap(struct page **pages, size_t size, 341void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle,
290 unsigned long vm_flags, pgprot_t prot, 342 gfp_t flag, unsigned long attrs)
291 const void *caller)
292{ 343{
293 struct vm_struct *area; 344 const struct dma_map_ops *ops = get_dma_ops(dev);
345 void *cpu_addr;
346
347 WARN_ON_ONCE(dev && !dev->coherent_dma_mask);
294 348
295 area = __dma_common_pages_remap(pages, size, vm_flags, prot, caller); 349 if (dma_alloc_from_dev_coherent(dev, size, dma_handle, &cpu_addr))
296 if (!area) 350 return cpu_addr;
351
352 /* let the implementation decide on the zone to allocate from: */
353 flag &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM);
354
355 if (!arch_dma_alloc_attrs(&dev))
297 return NULL; 356 return NULL;
298 357
299 area->pages = pages; 358 if (dma_is_direct(ops))
359 cpu_addr = dma_direct_alloc(dev, size, dma_handle, flag, attrs);
360 else if (ops->alloc)
361 cpu_addr = ops->alloc(dev, size, dma_handle, flag, attrs);
362 else
363 return NULL;
300 364
301 return area->addr; 365 debug_dma_alloc_coherent(dev, size, *dma_handle, cpu_addr);
366 return cpu_addr;
302} 367}
368EXPORT_SYMBOL(dma_alloc_attrs);
303 369
304/* 370void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr,
305 * remaps an allocated contiguous region into another vm_area. 371 dma_addr_t dma_handle, unsigned long attrs)
306 * Cannot be used in non-sleeping contexts
307 */
308
309void *dma_common_contiguous_remap(struct page *page, size_t size,
310 unsigned long vm_flags,
311 pgprot_t prot, const void *caller)
312{ 372{
313 int i; 373 const struct dma_map_ops *ops = get_dma_ops(dev);
314 struct page **pages;
315 struct vm_struct *area;
316 374
317 pages = kmalloc(sizeof(struct page *) << get_order(size), GFP_KERNEL); 375 if (dma_release_from_dev_coherent(dev, get_order(size), cpu_addr))
318 if (!pages) 376 return;
319 return NULL; 377 /*
378 * On non-coherent platforms which implement DMA-coherent buffers via
379 * non-cacheable remaps, ops->free() may call vunmap(). Thus getting
380 * this far in IRQ context is a) at risk of a BUG_ON() or trying to
381 * sleep on some machines, and b) an indication that the driver is
382 * probably misusing the coherent API anyway.
383 */
384 WARN_ON(irqs_disabled());
385
386 if (!cpu_addr)
387 return;
320 388
321 for (i = 0; i < (size >> PAGE_SHIFT); i++) 389 debug_dma_free_coherent(dev, size, cpu_addr, dma_handle);
322 pages[i] = nth_page(page, i); 390 if (dma_is_direct(ops))
391 dma_direct_free(dev, size, cpu_addr, dma_handle, attrs);
392 else if (ops->free)
393 ops->free(dev, size, cpu_addr, dma_handle, attrs);
394}
395EXPORT_SYMBOL(dma_free_attrs);
323 396
324 area = __dma_common_pages_remap(pages, size, vm_flags, prot, caller); 397static inline void dma_check_mask(struct device *dev, u64 mask)
398{
399 if (sme_active() && (mask < (((u64)sme_get_me_mask() << 1) - 1)))
400 dev_warn(dev, "SME is active, device will require DMA bounce buffers\n");
401}
325 402
326 kfree(pages); 403int dma_supported(struct device *dev, u64 mask)
404{
405 const struct dma_map_ops *ops = get_dma_ops(dev);
327 406
328 if (!area) 407 if (dma_is_direct(ops))
329 return NULL; 408 return dma_direct_supported(dev, mask);
330 return area->addr; 409 if (!ops->dma_supported)
410 return 1;
411 return ops->dma_supported(dev, mask);
331} 412}
413EXPORT_SYMBOL(dma_supported);
332 414
333/* 415#ifndef HAVE_ARCH_DMA_SET_MASK
334 * unmaps a range previously mapped by dma_common_*_remap 416int dma_set_mask(struct device *dev, u64 mask)
335 */
336void dma_common_free_remap(void *cpu_addr, size_t size, unsigned long vm_flags)
337{ 417{
338 struct vm_struct *area = find_vm_area(cpu_addr); 418 if (!dev->dma_mask || !dma_supported(dev, mask))
419 return -EIO;
339 420
340 if (!area || (area->flags & vm_flags) != vm_flags) { 421 dma_check_mask(dev, mask);
341 WARN(1, "trying to free invalid coherent area: %p\n", cpu_addr); 422 *dev->dma_mask = mask;
342 return; 423 return 0;
343 } 424}
425EXPORT_SYMBOL(dma_set_mask);
426#endif
427
428#ifndef CONFIG_ARCH_HAS_DMA_SET_COHERENT_MASK
429int dma_set_coherent_mask(struct device *dev, u64 mask)
430{
431 if (!dma_supported(dev, mask))
432 return -EIO;
344 433
345 unmap_kernel_range((unsigned long)cpu_addr, PAGE_ALIGN(size)); 434 dma_check_mask(dev, mask);
346 vunmap(cpu_addr); 435 dev->coherent_dma_mask = mask;
436 return 0;
347} 437}
438EXPORT_SYMBOL(dma_set_coherent_mask);
348#endif 439#endif
440
441void dma_cache_sync(struct device *dev, void *vaddr, size_t size,
442 enum dma_data_direction dir)
443{
444 const struct dma_map_ops *ops = get_dma_ops(dev);
445
446 BUG_ON(!valid_dma_direction(dir));
447
448 if (dma_is_direct(ops))
449 arch_dma_cache_sync(dev, vaddr, size, dir);
450 else if (ops->cache_sync)
451 ops->cache_sync(dev, vaddr, size, dir);
452}
453EXPORT_SYMBOL(dma_cache_sync);
diff --git a/kernel/dma/remap.c b/kernel/dma/remap.c
new file mode 100644
index 000000000000..18cc09fc27b9
--- /dev/null
+++ b/kernel/dma/remap.c
@@ -0,0 +1,256 @@
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2012 ARM Ltd.
4 * Copyright (c) 2014 The Linux Foundation
5 */
6#include <linux/dma-direct.h>
7#include <linux/dma-noncoherent.h>
8#include <linux/dma-contiguous.h>
9#include <linux/init.h>
10#include <linux/genalloc.h>
11#include <linux/slab.h>
12#include <linux/vmalloc.h>
13
14static struct vm_struct *__dma_common_pages_remap(struct page **pages,
15 size_t size, unsigned long vm_flags, pgprot_t prot,
16 const void *caller)
17{
18 struct vm_struct *area;
19
20 area = get_vm_area_caller(size, vm_flags, caller);
21 if (!area)
22 return NULL;
23
24 if (map_vm_area(area, prot, pages)) {
25 vunmap(area->addr);
26 return NULL;
27 }
28
29 return area;
30}
31
32/*
33 * Remaps an array of PAGE_SIZE pages into another vm_area.
34 * Cannot be used in non-sleeping contexts
35 */
36void *dma_common_pages_remap(struct page **pages, size_t size,
37 unsigned long vm_flags, pgprot_t prot,
38 const void *caller)
39{
40 struct vm_struct *area;
41
42 area = __dma_common_pages_remap(pages, size, vm_flags, prot, caller);
43 if (!area)
44 return NULL;
45
46 area->pages = pages;
47
48 return area->addr;
49}
50
51/*
52 * Remaps an allocated contiguous region into another vm_area.
53 * Cannot be used in non-sleeping contexts
54 */
55void *dma_common_contiguous_remap(struct page *page, size_t size,
56 unsigned long vm_flags,
57 pgprot_t prot, const void *caller)
58{
59 int i;
60 struct page **pages;
61 struct vm_struct *area;
62
63 pages = kmalloc(sizeof(struct page *) << get_order(size), GFP_KERNEL);
64 if (!pages)
65 return NULL;
66
67 for (i = 0; i < (size >> PAGE_SHIFT); i++)
68 pages[i] = nth_page(page, i);
69
70 area = __dma_common_pages_remap(pages, size, vm_flags, prot, caller);
71
72 kfree(pages);
73
74 if (!area)
75 return NULL;
76 return area->addr;
77}
78
79/*
80 * Unmaps a range previously mapped by dma_common_*_remap
81 */
82void dma_common_free_remap(void *cpu_addr, size_t size, unsigned long vm_flags)
83{
84 struct vm_struct *area = find_vm_area(cpu_addr);
85
86 if (!area || (area->flags & vm_flags) != vm_flags) {
87 WARN(1, "trying to free invalid coherent area: %p\n", cpu_addr);
88 return;
89 }
90
91 unmap_kernel_range((unsigned long)cpu_addr, PAGE_ALIGN(size));
92 vunmap(cpu_addr);
93}
94
95#ifdef CONFIG_DMA_DIRECT_REMAP
96static struct gen_pool *atomic_pool __ro_after_init;
97
98#define DEFAULT_DMA_COHERENT_POOL_SIZE SZ_256K
99static size_t atomic_pool_size __initdata = DEFAULT_DMA_COHERENT_POOL_SIZE;
100
101static int __init early_coherent_pool(char *p)
102{
103 atomic_pool_size = memparse(p, &p);
104 return 0;
105}
106early_param("coherent_pool", early_coherent_pool);
107
108int __init dma_atomic_pool_init(gfp_t gfp, pgprot_t prot)
109{
110 unsigned int pool_size_order = get_order(atomic_pool_size);
111 unsigned long nr_pages = atomic_pool_size >> PAGE_SHIFT;
112 struct page *page;
113 void *addr;
114 int ret;
115
116 if (dev_get_cma_area(NULL))
117 page = dma_alloc_from_contiguous(NULL, nr_pages,
118 pool_size_order, false);
119 else
120 page = alloc_pages(gfp, pool_size_order);
121 if (!page)
122 goto out;
123
124 arch_dma_prep_coherent(page, atomic_pool_size);
125
126 atomic_pool = gen_pool_create(PAGE_SHIFT, -1);
127 if (!atomic_pool)
128 goto free_page;
129
130 addr = dma_common_contiguous_remap(page, atomic_pool_size, VM_USERMAP,
131 prot, __builtin_return_address(0));
132 if (!addr)
133 goto destroy_genpool;
134
135 ret = gen_pool_add_virt(atomic_pool, (unsigned long)addr,
136 page_to_phys(page), atomic_pool_size, -1);
137 if (ret)
138 goto remove_mapping;
139 gen_pool_set_algo(atomic_pool, gen_pool_first_fit_order_align, NULL);
140
141 pr_info("DMA: preallocated %zu KiB pool for atomic allocations\n",
142 atomic_pool_size / 1024);
143 return 0;
144
145remove_mapping:
146 dma_common_free_remap(addr, atomic_pool_size, VM_USERMAP);
147destroy_genpool:
148 gen_pool_destroy(atomic_pool);
149 atomic_pool = NULL;
150free_page:
151 if (!dma_release_from_contiguous(NULL, page, nr_pages))
152 __free_pages(page, pool_size_order);
153out:
154 pr_err("DMA: failed to allocate %zu KiB pool for atomic coherent allocation\n",
155 atomic_pool_size / 1024);
156 return -ENOMEM;
157}
158
159bool dma_in_atomic_pool(void *start, size_t size)
160{
161 return addr_in_gen_pool(atomic_pool, (unsigned long)start, size);
162}
163
164void *dma_alloc_from_pool(size_t size, struct page **ret_page, gfp_t flags)
165{
166 unsigned long val;
167 void *ptr = NULL;
168
169 if (!atomic_pool) {
170 WARN(1, "coherent pool not initialised!\n");
171 return NULL;
172 }
173
174 val = gen_pool_alloc(atomic_pool, size);
175 if (val) {
176 phys_addr_t phys = gen_pool_virt_to_phys(atomic_pool, val);
177
178 *ret_page = pfn_to_page(__phys_to_pfn(phys));
179 ptr = (void *)val;
180 memset(ptr, 0, size);
181 }
182
183 return ptr;
184}
185
186bool dma_free_from_pool(void *start, size_t size)
187{
188 if (!dma_in_atomic_pool(start, size))
189 return false;
190 gen_pool_free(atomic_pool, (unsigned long)start, size);
191 return true;
192}
193
194void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
195 gfp_t flags, unsigned long attrs)
196{
197 struct page *page = NULL;
198 void *ret;
199
200 size = PAGE_ALIGN(size);
201
202 if (!gfpflags_allow_blocking(flags) &&
203 !(attrs & DMA_ATTR_NO_KERNEL_MAPPING)) {
204 ret = dma_alloc_from_pool(size, &page, flags);
205 if (!ret)
206 return NULL;
207 *dma_handle = phys_to_dma(dev, page_to_phys(page));
208 return ret;
209 }
210
211 page = __dma_direct_alloc_pages(dev, size, dma_handle, flags, attrs);
212 if (!page)
213 return NULL;
214
215 /* remove any dirty cache lines on the kernel alias */
216 arch_dma_prep_coherent(page, size);
217
218 if (attrs & DMA_ATTR_NO_KERNEL_MAPPING)
219 return page; /* opaque cookie */
220
221 /* create a coherent mapping */
222 ret = dma_common_contiguous_remap(page, size, VM_USERMAP,
223 arch_dma_mmap_pgprot(dev, PAGE_KERNEL, attrs),
224 __builtin_return_address(0));
225 if (!ret) {
226 __dma_direct_free_pages(dev, size, page);
227 return ret;
228 }
229
230 *dma_handle = phys_to_dma(dev, page_to_phys(page));
231 memset(ret, 0, size);
232
233 return ret;
234}
235
236void arch_dma_free(struct device *dev, size_t size, void *vaddr,
237 dma_addr_t dma_handle, unsigned long attrs)
238{
239 if (attrs & DMA_ATTR_NO_KERNEL_MAPPING) {
240 /* vaddr is a struct page cookie, not a kernel address */
241 __dma_direct_free_pages(dev, size, vaddr);
242 } else if (!dma_free_from_pool(vaddr, PAGE_ALIGN(size))) {
243 phys_addr_t phys = dma_to_phys(dev, dma_handle);
244 struct page *page = pfn_to_page(__phys_to_pfn(phys));
245
246 vunmap(vaddr);
247 __dma_direct_free_pages(dev, size, page);
248 }
249}
250
251long arch_dma_coherent_to_pfn(struct device *dev, void *cpu_addr,
252 dma_addr_t dma_addr)
253{
254 return __phys_to_pfn(dma_to_phys(dev, dma_addr));
255}
256#endif /* CONFIG_DMA_DIRECT_REMAP */
diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c
index 045930e32c0e..d6361776dc5c 100644
--- a/kernel/dma/swiotlb.c
+++ b/kernel/dma/swiotlb.c
@@ -21,7 +21,6 @@
21 21
22#include <linux/cache.h> 22#include <linux/cache.h>
23#include <linux/dma-direct.h> 23#include <linux/dma-direct.h>
24#include <linux/dma-noncoherent.h>
25#include <linux/mm.h> 24#include <linux/mm.h>
26#include <linux/export.h> 25#include <linux/export.h>
27#include <linux/spinlock.h> 26#include <linux/spinlock.h>
@@ -65,7 +64,7 @@ enum swiotlb_force swiotlb_force;
65 * swiotlb_tbl_sync_single_*, to see if the memory was in fact allocated by this 64 * swiotlb_tbl_sync_single_*, to see if the memory was in fact allocated by this
66 * API. 65 * API.
67 */ 66 */
68static phys_addr_t io_tlb_start, io_tlb_end; 67phys_addr_t io_tlb_start, io_tlb_end;
69 68
70/* 69/*
71 * The number of IO TLB blocks (in groups of 64) between io_tlb_start and 70 * The number of IO TLB blocks (in groups of 64) between io_tlb_start and
@@ -383,11 +382,6 @@ void __init swiotlb_exit(void)
383 max_segment = 0; 382 max_segment = 0;
384} 383}
385 384
386static int is_swiotlb_buffer(phys_addr_t paddr)
387{
388 return paddr >= io_tlb_start && paddr < io_tlb_end;
389}
390
391/* 385/*
392 * Bounce: copy the swiotlb buffer back to the original dma location 386 * Bounce: copy the swiotlb buffer back to the original dma location
393 */ 387 */
@@ -526,7 +520,7 @@ not_found:
526 spin_unlock_irqrestore(&io_tlb_lock, flags); 520 spin_unlock_irqrestore(&io_tlb_lock, flags);
527 if (!(attrs & DMA_ATTR_NO_WARN) && printk_ratelimit()) 521 if (!(attrs & DMA_ATTR_NO_WARN) && printk_ratelimit())
528 dev_warn(hwdev, "swiotlb buffer is full (sz: %zd bytes)\n", size); 522 dev_warn(hwdev, "swiotlb buffer is full (sz: %zd bytes)\n", size);
529 return SWIOTLB_MAP_ERROR; 523 return DMA_MAPPING_ERROR;
530found: 524found:
531 spin_unlock_irqrestore(&io_tlb_lock, flags); 525 spin_unlock_irqrestore(&io_tlb_lock, flags);
532 526
@@ -623,237 +617,36 @@ void swiotlb_tbl_sync_single(struct device *hwdev, phys_addr_t tlb_addr,
623 } 617 }
624} 618}
625 619
626static dma_addr_t swiotlb_bounce_page(struct device *dev, phys_addr_t *phys, 620/*
621 * Create a swiotlb mapping for the buffer at @phys, and in case of DMAing
622 * to the device copy the data into it as well.
623 */
624bool swiotlb_map(struct device *dev, phys_addr_t *phys, dma_addr_t *dma_addr,
627 size_t size, enum dma_data_direction dir, unsigned long attrs) 625 size_t size, enum dma_data_direction dir, unsigned long attrs)
628{ 626{
629 dma_addr_t dma_addr; 627 trace_swiotlb_bounced(dev, *dma_addr, size, swiotlb_force);
630 628
631 if (unlikely(swiotlb_force == SWIOTLB_NO_FORCE)) { 629 if (unlikely(swiotlb_force == SWIOTLB_NO_FORCE)) {
632 dev_warn_ratelimited(dev, 630 dev_warn_ratelimited(dev,
633 "Cannot do DMA to address %pa\n", phys); 631 "Cannot do DMA to address %pa\n", phys);
634 return DIRECT_MAPPING_ERROR; 632 return false;
635 } 633 }
636 634
637 /* Oh well, have to allocate and map a bounce buffer. */ 635 /* Oh well, have to allocate and map a bounce buffer. */
638 *phys = swiotlb_tbl_map_single(dev, __phys_to_dma(dev, io_tlb_start), 636 *phys = swiotlb_tbl_map_single(dev, __phys_to_dma(dev, io_tlb_start),
639 *phys, size, dir, attrs); 637 *phys, size, dir, attrs);
640 if (*phys == SWIOTLB_MAP_ERROR) 638 if (*phys == DMA_MAPPING_ERROR)
641 return DIRECT_MAPPING_ERROR; 639 return false;
642 640
643 /* Ensure that the address returned is DMA'ble */ 641 /* Ensure that the address returned is DMA'ble */
644 dma_addr = __phys_to_dma(dev, *phys); 642 *dma_addr = __phys_to_dma(dev, *phys);
645 if (unlikely(!dma_capable(dev, dma_addr, size))) { 643 if (unlikely(!dma_capable(dev, *dma_addr, size))) {
646 swiotlb_tbl_unmap_single(dev, *phys, size, dir, 644 swiotlb_tbl_unmap_single(dev, *phys, size, dir,
647 attrs | DMA_ATTR_SKIP_CPU_SYNC); 645 attrs | DMA_ATTR_SKIP_CPU_SYNC);
648 return DIRECT_MAPPING_ERROR; 646 return false;
649 }
650
651 return dma_addr;
652}
653
654/*
655 * Map a single buffer of the indicated size for DMA in streaming mode. The
656 * physical address to use is returned.
657 *
658 * Once the device is given the dma address, the device owns this memory until
659 * either swiotlb_unmap_page or swiotlb_dma_sync_single is performed.
660 */
661dma_addr_t swiotlb_map_page(struct device *dev, struct page *page,
662 unsigned long offset, size_t size,
663 enum dma_data_direction dir,
664 unsigned long attrs)
665{
666 phys_addr_t phys = page_to_phys(page) + offset;
667 dma_addr_t dev_addr = phys_to_dma(dev, phys);
668
669 BUG_ON(dir == DMA_NONE);
670 /*
671 * If the address happens to be in the device's DMA window,
672 * we can safely return the device addr and not worry about bounce
673 * buffering it.
674 */
675 if (!dma_capable(dev, dev_addr, size) ||
676 swiotlb_force == SWIOTLB_FORCE) {
677 trace_swiotlb_bounced(dev, dev_addr, size, swiotlb_force);
678 dev_addr = swiotlb_bounce_page(dev, &phys, size, dir, attrs);
679 }
680
681 if (!dev_is_dma_coherent(dev) &&
682 (attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0 &&
683 dev_addr != DIRECT_MAPPING_ERROR)
684 arch_sync_dma_for_device(dev, phys, size, dir);
685
686 return dev_addr;
687}
688
689/*
690 * Unmap a single streaming mode DMA translation. The dma_addr and size must
691 * match what was provided for in a previous swiotlb_map_page call. All
692 * other usages are undefined.
693 *
694 * After this call, reads by the cpu to the buffer are guaranteed to see
695 * whatever the device wrote there.
696 */
697void swiotlb_unmap_page(struct device *hwdev, dma_addr_t dev_addr,
698 size_t size, enum dma_data_direction dir,
699 unsigned long attrs)
700{
701 phys_addr_t paddr = dma_to_phys(hwdev, dev_addr);
702
703 BUG_ON(dir == DMA_NONE);
704
705 if (!dev_is_dma_coherent(hwdev) &&
706 (attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)
707 arch_sync_dma_for_cpu(hwdev, paddr, size, dir);
708
709 if (is_swiotlb_buffer(paddr)) {
710 swiotlb_tbl_unmap_single(hwdev, paddr, size, dir, attrs);
711 return;
712 } 647 }
713 648
714 if (dir != DMA_FROM_DEVICE) 649 return true;
715 return;
716
717 /*
718 * phys_to_virt doesn't work with hihgmem page but we could
719 * call dma_mark_clean() with hihgmem page here. However, we
720 * are fine since dma_mark_clean() is null on POWERPC. We can
721 * make dma_mark_clean() take a physical address if necessary.
722 */
723 dma_mark_clean(phys_to_virt(paddr), size);
724}
725
726/*
727 * Make physical memory consistent for a single streaming mode DMA translation
728 * after a transfer.
729 *
730 * If you perform a swiotlb_map_page() but wish to interrogate the buffer
731 * using the cpu, yet do not wish to teardown the dma mapping, you must
732 * call this function before doing so. At the next point you give the dma
733 * address back to the card, you must first perform a
734 * swiotlb_dma_sync_for_device, and then the device again owns the buffer
735 */
736static void
737swiotlb_sync_single(struct device *hwdev, dma_addr_t dev_addr,
738 size_t size, enum dma_data_direction dir,
739 enum dma_sync_target target)
740{
741 phys_addr_t paddr = dma_to_phys(hwdev, dev_addr);
742
743 BUG_ON(dir == DMA_NONE);
744
745 if (!dev_is_dma_coherent(hwdev) && target == SYNC_FOR_CPU)
746 arch_sync_dma_for_cpu(hwdev, paddr, size, dir);
747
748 if (is_swiotlb_buffer(paddr))
749 swiotlb_tbl_sync_single(hwdev, paddr, size, dir, target);
750
751 if (!dev_is_dma_coherent(hwdev) && target == SYNC_FOR_DEVICE)
752 arch_sync_dma_for_device(hwdev, paddr, size, dir);
753
754 if (!is_swiotlb_buffer(paddr) && dir == DMA_FROM_DEVICE)
755 dma_mark_clean(phys_to_virt(paddr), size);
756}
757
758void
759swiotlb_sync_single_for_cpu(struct device *hwdev, dma_addr_t dev_addr,
760 size_t size, enum dma_data_direction dir)
761{
762 swiotlb_sync_single(hwdev, dev_addr, size, dir, SYNC_FOR_CPU);
763}
764
765void
766swiotlb_sync_single_for_device(struct device *hwdev, dma_addr_t dev_addr,
767 size_t size, enum dma_data_direction dir)
768{
769 swiotlb_sync_single(hwdev, dev_addr, size, dir, SYNC_FOR_DEVICE);
770}
771
772/*
773 * Map a set of buffers described by scatterlist in streaming mode for DMA.
774 * This is the scatter-gather version of the above swiotlb_map_page
775 * interface. Here the scatter gather list elements are each tagged with the
776 * appropriate dma address and length. They are obtained via
777 * sg_dma_{address,length}(SG).
778 *
779 * Device ownership issues as mentioned above for swiotlb_map_page are the
780 * same here.
781 */
782int
783swiotlb_map_sg_attrs(struct device *dev, struct scatterlist *sgl, int nelems,
784 enum dma_data_direction dir, unsigned long attrs)
785{
786 struct scatterlist *sg;
787 int i;
788
789 for_each_sg(sgl, sg, nelems, i) {
790 sg->dma_address = swiotlb_map_page(dev, sg_page(sg), sg->offset,
791 sg->length, dir, attrs);
792 if (sg->dma_address == DIRECT_MAPPING_ERROR)
793 goto out_error;
794 sg_dma_len(sg) = sg->length;
795 }
796
797 return nelems;
798
799out_error:
800 swiotlb_unmap_sg_attrs(dev, sgl, i, dir,
801 attrs | DMA_ATTR_SKIP_CPU_SYNC);
802 sg_dma_len(sgl) = 0;
803 return 0;
804}
805
806/*
807 * Unmap a set of streaming mode DMA translations. Again, cpu read rules
808 * concerning calls here are the same as for swiotlb_unmap_page() above.
809 */
810void
811swiotlb_unmap_sg_attrs(struct device *hwdev, struct scatterlist *sgl,
812 int nelems, enum dma_data_direction dir,
813 unsigned long attrs)
814{
815 struct scatterlist *sg;
816 int i;
817
818 BUG_ON(dir == DMA_NONE);
819
820 for_each_sg(sgl, sg, nelems, i)
821 swiotlb_unmap_page(hwdev, sg->dma_address, sg_dma_len(sg), dir,
822 attrs);
823}
824
825/*
826 * Make physical memory consistent for a set of streaming mode DMA translations
827 * after a transfer.
828 *
829 * The same as swiotlb_sync_single_* but for a scatter-gather list, same rules
830 * and usage.
831 */
832static void
833swiotlb_sync_sg(struct device *hwdev, struct scatterlist *sgl,
834 int nelems, enum dma_data_direction dir,
835 enum dma_sync_target target)
836{
837 struct scatterlist *sg;
838 int i;
839
840 for_each_sg(sgl, sg, nelems, i)
841 swiotlb_sync_single(hwdev, sg->dma_address,
842 sg_dma_len(sg), dir, target);
843}
844
845void
846swiotlb_sync_sg_for_cpu(struct device *hwdev, struct scatterlist *sg,
847 int nelems, enum dma_data_direction dir)
848{
849 swiotlb_sync_sg(hwdev, sg, nelems, dir, SYNC_FOR_CPU);
850}
851
852void
853swiotlb_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg,
854 int nelems, enum dma_data_direction dir)
855{
856 swiotlb_sync_sg(hwdev, sg, nelems, dir, SYNC_FOR_DEVICE);
857} 650}
858 651
859/* 652/*
@@ -867,19 +660,3 @@ swiotlb_dma_supported(struct device *hwdev, u64 mask)
867{ 660{
868 return __phys_to_dma(hwdev, io_tlb_end - 1) <= mask; 661 return __phys_to_dma(hwdev, io_tlb_end - 1) <= mask;
869} 662}
870
871const struct dma_map_ops swiotlb_dma_ops = {
872 .mapping_error = dma_direct_mapping_error,
873 .alloc = dma_direct_alloc,
874 .free = dma_direct_free,
875 .sync_single_for_cpu = swiotlb_sync_single_for_cpu,
876 .sync_single_for_device = swiotlb_sync_single_for_device,
877 .sync_sg_for_cpu = swiotlb_sync_sg_for_cpu,
878 .sync_sg_for_device = swiotlb_sync_sg_for_device,
879 .map_sg = swiotlb_map_sg_attrs,
880 .unmap_sg = swiotlb_unmap_sg_attrs,
881 .map_page = swiotlb_map_page,
882 .unmap_page = swiotlb_unmap_page,
883 .dma_supported = dma_direct_supported,
884};
885EXPORT_SYMBOL(swiotlb_dma_ops);
diff --git a/kernel/dma/virt.c b/kernel/dma/virt.c
index 631ddec4b60a..ebe128833af7 100644
--- a/kernel/dma/virt.c
+++ b/kernel/dma/virt.c
@@ -13,7 +13,7 @@ static void *dma_virt_alloc(struct device *dev, size_t size,
13{ 13{
14 void *ret; 14 void *ret;
15 15
16 ret = (void *)__get_free_pages(gfp, get_order(size)); 16 ret = (void *)__get_free_pages(gfp | __GFP_ZERO, get_order(size));
17 if (ret) 17 if (ret)
18 *dma_handle = (uintptr_t)ret; 18 *dma_handle = (uintptr_t)ret;
19 return ret; 19 return ret;
generated by cgit v1.2.2 (git 2.25.0) at 2026-02-24 23:02:05 -0500