aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2018-10-26 13:50:10 -0400
committerLinus Torvalds <torvalds@linux-foundation.org>2018-10-26 13:50:10 -0400
commitd1f2b1710d92a80d60351503bbf41cdac95ed7a8 (patch)
treecc8247f2917c38b476b8777c29e3c9a154445d9e
parent18d0eae30e6a4f8644d589243d7ac1d70d29203d (diff)
parent2f2fbfb71ecc221352d84ae6430b42031ae5b654 (diff)
Merge tag 'iommu-updates-v4.20' of git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu
Pull IOMMU updates from Joerg Roedel: - Debugfs support for the Intel VT-d driver. When enabled, it now also exposes some of its internal data structures to user-space for debugging purposes. - ARM-SMMU driver now uses the generic deferred flushing and fast-path iova allocation code. This is expected to be a major performance improvement, as this allocation path scales a lot better. - Support for r8a7744 in the Renesas iommu driver - Couple of minor fixes and improvements all over the place * tag 'iommu-updates-v4.20' of git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu: (39 commits) iommu/arm-smmu-v3: Remove unnecessary wrapper function iommu/arm-smmu-v3: Add SPDX header iommu/amd: Add default branch in amd_iommu_capable() dt-bindings: iommu: ipmmu-vmsa: Add r8a7744 support iommu/amd: Move iommu_init_pci() to .init section iommu/arm-smmu: Support non-strict mode iommu/io-pgtable-arm-v7s: Add support for non-strict mode iommu/arm-smmu-v3: Add support for non-strict mode iommu/io-pgtable-arm: Add support for non-strict mode iommu: Add "iommu.strict" command line option iommu/dma: Add support for non-strict mode iommu/arm-smmu: Ensure that page-table updates are visible before TLBI iommu/arm-smmu-v3: Implement flush_iotlb_all hook iommu/arm-smmu-v3: Avoid back-to-back CMD_SYNC operations iommu/arm-smmu-v3: Fix unexpected CMD_SYNC timeout iommu/io-pgtable-arm: Fix race handling in split_blk_unmap() iommu/arm-smmu-v3: Fix a couple of minor comment typos iommu: Fix a typo iommu: Remove .domain_{get,set}_windows iommu: Tidy up window attributes ...
Diffstat
-rw-r--r--Documentation/admin-guide/kernel-parameters.txt12
-rw-r--r--Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.txt1
-rw-r--r--Documentation/devicetree/bindings/misc/fsl,qoriq-mc.txt39
-rw-r--r--arch/arm64/boot/dts/freescale/fsl-ls208xa.dtsi7
-rw-r--r--arch/arm64/mm/dma-mapping.c10
-rw-r--r--arch/x86/include/asm/irq_remapping.h2
-rw-r--r--drivers/bus/fsl-mc/fsl-mc-bus.c16
-rw-r--r--drivers/iommu/Kconfig13
-rw-r--r--drivers/iommu/Makefile1
-rw-r--r--drivers/iommu/amd_iommu.c2
-rw-r--r--drivers/iommu/amd_iommu_init.c2
-rw-r--r--drivers/iommu/arm-smmu-v3.c140
-rw-r--r--drivers/iommu/arm-smmu.c106
-rw-r--r--drivers/iommu/dma-iommu.c55
-rw-r--r--drivers/iommu/fsl_pamu_domain.c119
-rw-r--r--drivers/iommu/intel-iommu-debugfs.c314
-rw-r--r--drivers/iommu/intel-iommu.c32
-rw-r--r--drivers/iommu/intel_irq_remapping.c2
-rw-r--r--drivers/iommu/io-pgtable-arm-v7s.c11
-rw-r--r--drivers/iommu/io-pgtable-arm.c23
-rw-r--r--drivers/iommu/io-pgtable.h5
-rw-r--r--drivers/iommu/iommu.c58
-rw-r--r--drivers/iommu/iova.c22
-rw-r--r--drivers/iommu/ipmmu-vmsa.c5
-rw-r--r--drivers/iommu/of_iommu.c25
-rw-r--r--drivers/of/base.c102
-rw-r--r--drivers/of/irq.c5
-rw-r--r--drivers/pci/of.c101
-rw-r--r--include/linux/fsl/mc.h8
-rw-r--r--include/linux/intel-iommu.h72
-rw-r--r--include/linux/iommu.h10
-rw-r--r--include/linux/iova.h1
-rw-r--r--include/linux/of.h11
-rw-r--r--include/linux/of_pci.h10
34 files changed, 986 insertions, 356 deletions
diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index 8022d902e770..5b5f1ba76bba 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -1759,6 +1759,18 @@
1759 nobypass [PPC/POWERNV] 1759 nobypass [PPC/POWERNV]
1760 Disable IOMMU bypass, using IOMMU for PCI devices. 1760 Disable IOMMU bypass, using IOMMU for PCI devices.
1761 1761
1762 iommu.strict= [ARM64] Configure TLB invalidation behaviour
1763 Format: { "0" | "1" }
1764 0 - Lazy mode.
1765 Request that DMA unmap operations use deferred
1766 invalidation of hardware TLBs, for increased
1767 throughput at the cost of reduced device isolation.
1768 Will fall back to strict mode if not supported by
1769 the relevant IOMMU driver.
1770 1 - Strict mode (default).
1771 DMA unmap operations invalidate IOMMU hardware TLBs
1772 synchronously.
1773
1762 iommu.passthrough= 1774 iommu.passthrough=
1763 [ARM64] Configure DMA to bypass the IOMMU by default. 1775 [ARM64] Configure DMA to bypass the IOMMU by default.
1764 Format: { "0" | "1" } 1776 Format: { "0" | "1" }
diff --git a/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.txt b/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.txt
index c6e2d855fe13..377ee639d103 100644
--- a/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.txt
+++ b/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.txt
@@ -12,6 +12,7 @@ Required Properties:
12 12
13 - "renesas,ipmmu-r8a73a4" for the R8A73A4 (R-Mobile APE6) IPMMU. 13 - "renesas,ipmmu-r8a73a4" for the R8A73A4 (R-Mobile APE6) IPMMU.
14 - "renesas,ipmmu-r8a7743" for the R8A7743 (RZ/G1M) IPMMU. 14 - "renesas,ipmmu-r8a7743" for the R8A7743 (RZ/G1M) IPMMU.
15 - "renesas,ipmmu-r8a7744" for the R8A7744 (RZ/G1N) IPMMU.
15 - "renesas,ipmmu-r8a7745" for the R8A7745 (RZ/G1E) IPMMU. 16 - "renesas,ipmmu-r8a7745" for the R8A7745 (RZ/G1E) IPMMU.
16 - "renesas,ipmmu-r8a7790" for the R8A7790 (R-Car H2) IPMMU. 17 - "renesas,ipmmu-r8a7790" for the R8A7790 (R-Car H2) IPMMU.
17 - "renesas,ipmmu-r8a7791" for the R8A7791 (R-Car M2-W) IPMMU. 18 - "renesas,ipmmu-r8a7791" for the R8A7791 (R-Car M2-W) IPMMU.
diff --git a/Documentation/devicetree/bindings/misc/fsl,qoriq-mc.txt b/Documentation/devicetree/bindings/misc/fsl,qoriq-mc.txt
index 6611a7c2053a..01fdc33a41d0 100644
--- a/Documentation/devicetree/bindings/misc/fsl,qoriq-mc.txt
+++ b/Documentation/devicetree/bindings/misc/fsl,qoriq-mc.txt
@@ -9,6 +9,25 @@ blocks that can be used to create functional hardware objects/devices
9such as network interfaces, crypto accelerator instances, L2 switches, 9such as network interfaces, crypto accelerator instances, L2 switches,
10etc. 10etc.
11 11
12For an overview of the DPAA2 architecture and fsl-mc bus see:
13Documentation/networking/dpaa2/overview.rst
14
15As described in the above overview, all DPAA2 objects in a DPRC share the
16same hardware "isolation context" and a 10-bit value called an ICID
17(isolation context id) is expressed by the hardware to identify
18the requester.
19
20The generic 'iommus' property is insufficient to describe the relationship
21between ICIDs and IOMMUs, so an iommu-map property is used to define
22the set of possible ICIDs under a root DPRC and how they map to
23an IOMMU.
24
25For generic IOMMU bindings, see
26Documentation/devicetree/bindings/iommu/iommu.txt.
27
28For arm-smmu binding, see:
29Documentation/devicetree/bindings/iommu/arm,smmu.txt.
30
12Required properties: 31Required properties:
13 32
14 - compatible 33 - compatible
@@ -88,14 +107,34 @@ Sub-nodes:
88 Value type: <phandle> 107 Value type: <phandle>
89 Definition: Specifies the phandle to the PHY device node associated 108 Definition: Specifies the phandle to the PHY device node associated
90 with the this dpmac. 109 with the this dpmac.
110Optional properties:
111
112- iommu-map: Maps an ICID to an IOMMU and associated iommu-specifier
113 data.
114
115 The property is an arbitrary number of tuples of
116 (icid-base,iommu,iommu-base,length).
117
118 Any ICID i in the interval [icid-base, icid-base + length) is
119 associated with the listed IOMMU, with the iommu-specifier
120 (i - icid-base + iommu-base).
91 121
92Example: 122Example:
93 123
124 smmu: iommu@5000000 {
125 compatible = "arm,mmu-500";
126 #iommu-cells = <1>;
127 stream-match-mask = <0x7C00>;
128 ...
129 };
130
94 fsl_mc: fsl-mc@80c000000 { 131 fsl_mc: fsl-mc@80c000000 {
95 compatible = "fsl,qoriq-mc"; 132 compatible = "fsl,qoriq-mc";
96 reg = <0x00000008 0x0c000000 0 0x40>, /* MC portal base */ 133 reg = <0x00000008 0x0c000000 0 0x40>, /* MC portal base */
97 <0x00000000 0x08340000 0 0x40000>; /* MC control reg */ 134 <0x00000000 0x08340000 0 0x40000>; /* MC control reg */
98 msi-parent = <&its>; 135 msi-parent = <&its>;
136 /* define map for ICIDs 23-64 */
137 iommu-map = <23 &smmu 23 41>;
99 #address-cells = <3>; 138 #address-cells = <3>;
100 #size-cells = <1>; 139 #size-cells = <1>;
101 140
diff --git a/arch/arm64/boot/dts/freescale/fsl-ls208xa.dtsi b/arch/arm64/boot/dts/freescale/fsl-ls208xa.dtsi
index 8cb78dd99672..90c0faf8579f 100644
--- a/arch/arm64/boot/dts/freescale/fsl-ls208xa.dtsi
+++ b/arch/arm64/boot/dts/freescale/fsl-ls208xa.dtsi
@@ -148,6 +148,7 @@
148 #address-cells = <2>; 148 #address-cells = <2>;
149 #size-cells = <2>; 149 #size-cells = <2>;
150 ranges; 150 ranges;
151 dma-ranges = <0x0 0x0 0x0 0x0 0x10000 0x00000000>;
151 152
152 clockgen: clocking@1300000 { 153 clockgen: clocking@1300000 {
153 compatible = "fsl,ls2080a-clockgen"; 154 compatible = "fsl,ls2080a-clockgen";
@@ -321,6 +322,8 @@
321 reg = <0x00000008 0x0c000000 0 0x40>, /* MC portal base */ 322 reg = <0x00000008 0x0c000000 0 0x40>, /* MC portal base */
322 <0x00000000 0x08340000 0 0x40000>; /* MC control reg */ 323 <0x00000000 0x08340000 0 0x40000>; /* MC control reg */
323 msi-parent = <&its>; 324 msi-parent = <&its>;
325 iommu-map = <0 &smmu 0 0>; /* This is fixed-up by u-boot */
326 dma-coherent;
324 #address-cells = <3>; 327 #address-cells = <3>;
325 #size-cells = <1>; 328 #size-cells = <1>;
326 329
@@ -424,6 +427,9 @@
424 compatible = "arm,mmu-500"; 427 compatible = "arm,mmu-500";
425 reg = <0 0x5000000 0 0x800000>; 428 reg = <0 0x5000000 0 0x800000>;
426 #global-interrupts = <12>; 429 #global-interrupts = <12>;
430 #iommu-cells = <1>;
431 stream-match-mask = <0x7C00>;
432 dma-coherent;
427 interrupts = <0 13 4>, /* global secure fault */ 433 interrupts = <0 13 4>, /* global secure fault */
428 <0 14 4>, /* combined secure interrupt */ 434 <0 14 4>, /* combined secure interrupt */
429 <0 15 4>, /* global non-secure fault */ 435 <0 15 4>, /* global non-secure fault */
@@ -466,7 +472,6 @@
466 <0 204 4>, <0 205 4>, 472 <0 204 4>, <0 205 4>,
467 <0 206 4>, <0 207 4>, 473 <0 206 4>, <0 207 4>,
468 <0 208 4>, <0 209 4>; 474 <0 208 4>, <0 209 4>;
469 mmu-masters = <&fsl_mc 0x300 0>;
470 }; 475 };
471 476
472 dspi: dspi@2100000 { 477 dspi: dspi@2100000 {
diff --git a/arch/arm64/mm/dma-mapping.c b/arch/arm64/mm/dma-mapping.c
index 072c51fb07d7..cf017c5bb5e7 100644
--- a/arch/arm64/mm/dma-mapping.c
+++ b/arch/arm64/mm/dma-mapping.c
@@ -712,7 +712,7 @@ static void __iommu_sync_single_for_cpu(struct device *dev,
712 if (is_device_dma_coherent(dev)) 712 if (is_device_dma_coherent(dev))
713 return; 713 return;
714 714
715 phys = iommu_iova_to_phys(iommu_get_domain_for_dev(dev), dev_addr); 715 phys = iommu_iova_to_phys(iommu_get_dma_domain(dev), dev_addr);
716 __dma_unmap_area(phys_to_virt(phys), size, dir); 716 __dma_unmap_area(phys_to_virt(phys), size, dir);
717} 717}
718 718
@@ -725,7 +725,7 @@ static void __iommu_sync_single_for_device(struct device *dev,
725 if (is_device_dma_coherent(dev)) 725 if (is_device_dma_coherent(dev))
726 return; 726 return;
727 727
728 phys = iommu_iova_to_phys(iommu_get_domain_for_dev(dev), dev_addr); 728 phys = iommu_iova_to_phys(iommu_get_dma_domain(dev), dev_addr);
729 __dma_map_area(phys_to_virt(phys), size, dir); 729 __dma_map_area(phys_to_virt(phys), size, dir);
730} 730}
731 731
@@ -738,9 +738,9 @@ static dma_addr_t __iommu_map_page(struct device *dev, struct page *page,
738 int prot = dma_info_to_prot(dir, coherent, attrs); 738 int prot = dma_info_to_prot(dir, coherent, attrs);
739 dma_addr_t dev_addr = iommu_dma_map_page(dev, page, offset, size, prot); 739 dma_addr_t dev_addr = iommu_dma_map_page(dev, page, offset, size, prot);
740 740
741 if (!iommu_dma_mapping_error(dev, dev_addr) && 741 if (!coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&
742 (attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) 742 !iommu_dma_mapping_error(dev, dev_addr))
743 __iommu_sync_single_for_device(dev, dev_addr, size, dir); 743 __dma_map_area(page_address(page) + offset, size, dir);
744 744
745 return dev_addr; 745 return dev_addr;
746} 746}
diff --git a/arch/x86/include/asm/irq_remapping.h b/arch/x86/include/asm/irq_remapping.h
index 5f26962eff42..67ed72f31cc2 100644
--- a/arch/x86/include/asm/irq_remapping.h
+++ b/arch/x86/include/asm/irq_remapping.h
@@ -45,6 +45,8 @@ struct vcpu_data {
45 45
46#ifdef CONFIG_IRQ_REMAP 46#ifdef CONFIG_IRQ_REMAP
47 47
48extern raw_spinlock_t irq_2_ir_lock;
49
48extern bool irq_remapping_cap(enum irq_remap_cap cap); 50extern bool irq_remapping_cap(enum irq_remap_cap cap);
49extern void set_irq_remapping_broken(void); 51extern void set_irq_remapping_broken(void);
50extern int irq_remapping_prepare(void); 52extern int irq_remapping_prepare(void);
diff --git a/drivers/bus/fsl-mc/fsl-mc-bus.c b/drivers/bus/fsl-mc/fsl-mc-bus.c
index 4552b06fe601..439ba5c23693 100644
--- a/drivers/bus/fsl-mc/fsl-mc-bus.c
+++ b/drivers/bus/fsl-mc/fsl-mc-bus.c
@@ -127,6 +127,16 @@ static int fsl_mc_bus_uevent(struct device *dev, struct kobj_uevent_env *env)
127 return 0; 127 return 0;
128} 128}
129 129
130static int fsl_mc_dma_configure(struct device *dev)
131{
132 struct device *dma_dev = dev;
133
134 while (dev_is_fsl_mc(dma_dev))
135 dma_dev = dma_dev->parent;
136
137 return of_dma_configure(dev, dma_dev->of_node, 0);
138}
139
130static ssize_t modalias_show(struct device *dev, struct device_attribute *attr, 140static ssize_t modalias_show(struct device *dev, struct device_attribute *attr,
131 char *buf) 141 char *buf)
132{ 142{
@@ -148,6 +158,7 @@ struct bus_type fsl_mc_bus_type = {
148 .name = "fsl-mc", 158 .name = "fsl-mc",
149 .match = fsl_mc_bus_match, 159 .match = fsl_mc_bus_match,
150 .uevent = fsl_mc_bus_uevent, 160 .uevent = fsl_mc_bus_uevent,
161 .dma_configure = fsl_mc_dma_configure,
151 .dev_groups = fsl_mc_dev_groups, 162 .dev_groups = fsl_mc_dev_groups,
152}; 163};
153EXPORT_SYMBOL_GPL(fsl_mc_bus_type); 164EXPORT_SYMBOL_GPL(fsl_mc_bus_type);
@@ -621,6 +632,7 @@ int fsl_mc_device_add(struct fsl_mc_obj_desc *obj_desc,
621 mc_dev->icid = parent_mc_dev->icid; 632 mc_dev->icid = parent_mc_dev->icid;
622 mc_dev->dma_mask = FSL_MC_DEFAULT_DMA_MASK; 633 mc_dev->dma_mask = FSL_MC_DEFAULT_DMA_MASK;
623 mc_dev->dev.dma_mask = &mc_dev->dma_mask; 634 mc_dev->dev.dma_mask = &mc_dev->dma_mask;
635 mc_dev->dev.coherent_dma_mask = mc_dev->dma_mask;
624 dev_set_msi_domain(&mc_dev->dev, 636 dev_set_msi_domain(&mc_dev->dev,
625 dev_get_msi_domain(&parent_mc_dev->dev)); 637 dev_get_msi_domain(&parent_mc_dev->dev));
626 } 638 }
@@ -638,10 +650,6 @@ int fsl_mc_device_add(struct fsl_mc_obj_desc *obj_desc,
638 goto error_cleanup_dev; 650 goto error_cleanup_dev;
639 } 651 }
640 652
641 /* Objects are coherent, unless 'no shareability' flag set. */
642 if (!(obj_desc->flags & FSL_MC_OBJ_FLAG_NO_MEM_SHAREABILITY))
643 arch_setup_dma_ops(&mc_dev->dev, 0, 0, NULL, true);
644
645 /* 653 /*
646 * The device-specific probe callback will get invoked by device_add() 654 * The device-specific probe callback will get invoked by device_add()
647 */ 655 */
diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig
index 83e6d993fca5..d9a25715650e 100644
--- a/drivers/iommu/Kconfig
+++ b/drivers/iommu/Kconfig
@@ -186,6 +186,19 @@ config INTEL_IOMMU
186 and include PCI device scope covered by these DMA 186 and include PCI device scope covered by these DMA
187 remapping devices. 187 remapping devices.
188 188
189config INTEL_IOMMU_DEBUGFS
190 bool "Export Intel IOMMU internals in Debugfs"
191 depends on INTEL_IOMMU && IOMMU_DEBUGFS
192 help
193 !!!WARNING!!!
194
195 DO NOT ENABLE THIS OPTION UNLESS YOU REALLY KNOW WHAT YOU ARE DOING!!!
196
197 Expose Intel IOMMU internals in Debugfs.
198
199 This option is -NOT- intended for production environments, and should
200 only be enabled for debugging Intel IOMMU.
201
189config INTEL_IOMMU_SVM 202config INTEL_IOMMU_SVM
190 bool "Support for Shared Virtual Memory with Intel IOMMU" 203 bool "Support for Shared Virtual Memory with Intel IOMMU"
191 depends on INTEL_IOMMU && X86 204 depends on INTEL_IOMMU && X86
diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile
index ab5eba6edf82..a158a68c8ea8 100644
--- a/drivers/iommu/Makefile
+++ b/drivers/iommu/Makefile
@@ -17,6 +17,7 @@ obj-$(CONFIG_ARM_SMMU) += arm-smmu.o
17obj-$(CONFIG_ARM_SMMU_V3) += arm-smmu-v3.o 17obj-$(CONFIG_ARM_SMMU_V3) += arm-smmu-v3.o
18obj-$(CONFIG_DMAR_TABLE) += dmar.o 18obj-$(CONFIG_DMAR_TABLE) += dmar.o
19obj-$(CONFIG_INTEL_IOMMU) += intel-iommu.o intel-pasid.o 19obj-$(CONFIG_INTEL_IOMMU) += intel-iommu.o intel-pasid.o
20obj-$(CONFIG_INTEL_IOMMU_DEBUGFS) += intel-iommu-debugfs.o
20obj-$(CONFIG_INTEL_IOMMU_SVM) += intel-svm.o 21obj-$(CONFIG_INTEL_IOMMU_SVM) += intel-svm.o
21obj-$(CONFIG_IPMMU_VMSA) += ipmmu-vmsa.o 22obj-$(CONFIG_IPMMU_VMSA) += ipmmu-vmsa.o
22obj-$(CONFIG_IRQ_REMAP) += intel_irq_remapping.o irq_remapping.o 23obj-$(CONFIG_IRQ_REMAP) += intel_irq_remapping.o irq_remapping.o
diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c
index bee0dfb7b93b..1167ff0416cf 100644
--- a/drivers/iommu/amd_iommu.c
+++ b/drivers/iommu/amd_iommu.c
@@ -3083,6 +3083,8 @@ static bool amd_iommu_capable(enum iommu_cap cap)
3083 return (irq_remapping_enabled == 1); 3083 return (irq_remapping_enabled == 1);
3084 case IOMMU_CAP_NOEXEC: 3084 case IOMMU_CAP_NOEXEC:
3085 return false; 3085 return false;
3086 default:
3087 break;
3086 } 3088 }
3087 3089
3088 return false; 3090 return false;
diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c
index 3931c7de7c69..bb2cd29e1658 100644
--- a/drivers/iommu/amd_iommu_init.c
+++ b/drivers/iommu/amd_iommu_init.c
@@ -1719,7 +1719,7 @@ static const struct attribute_group *amd_iommu_groups[] = {
1719 NULL, 1719 NULL,
1720}; 1720};
1721 1721
1722static int iommu_init_pci(struct amd_iommu *iommu) 1722static int __init iommu_init_pci(struct amd_iommu *iommu)
1723{ 1723{
1724 int cap_ptr = iommu->cap_ptr; 1724 int cap_ptr = iommu->cap_ptr;
1725 u32 range, misc, low, high; 1725 u32 range, misc, low, high;
diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c
index 5059d09f3202..6947ccf26512 100644
--- a/drivers/iommu/arm-smmu-v3.c
+++ b/drivers/iommu/arm-smmu-v3.c
@@ -1,18 +1,7 @@
1// SPDX-License-Identifier: GPL-2.0
1/* 2/*
2 * IOMMU API for ARM architected SMMUv3 implementations. 3 * IOMMU API for ARM architected SMMUv3 implementations.
3 * 4 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * along with this program. If not, see <http://www.gnu.org/licenses/>.
15 *
16 * Copyright (C) 2015 ARM Limited 5 * Copyright (C) 2015 ARM Limited
17 * 6 *
18 * Author: Will Deacon <will.deacon@arm.com> 7 * Author: Will Deacon <will.deacon@arm.com>
@@ -567,7 +556,8 @@ struct arm_smmu_device {
567 556
568 int gerr_irq; 557 int gerr_irq;
569 int combined_irq; 558 int combined_irq;
570 atomic_t sync_nr; 559 u32 sync_nr;
560 u8 prev_cmd_opcode;
571 561
572 unsigned long ias; /* IPA */ 562 unsigned long ias; /* IPA */
573 unsigned long oas; /* PA */ 563 unsigned long oas; /* PA */
@@ -611,6 +601,7 @@ struct arm_smmu_domain {
611 struct mutex init_mutex; /* Protects smmu pointer */ 601 struct mutex init_mutex; /* Protects smmu pointer */
612 602
613 struct io_pgtable_ops *pgtbl_ops; 603 struct io_pgtable_ops *pgtbl_ops;
604 bool non_strict;
614 605
615 enum arm_smmu_domain_stage stage; 606 enum arm_smmu_domain_stage stage;
616 union { 607 union {
@@ -708,7 +699,7 @@ static void queue_inc_prod(struct arm_smmu_queue *q)
708} 699}
709 700
710/* 701/*
711 * Wait for the SMMU to consume items. If drain is true, wait until the queue 702 * Wait for the SMMU to consume items. If sync is true, wait until the queue
712 * is empty. Otherwise, wait until there is at least one free slot. 703 * is empty. Otherwise, wait until there is at least one free slot.
713 */ 704 */
714static int queue_poll_cons(struct arm_smmu_queue *q, bool sync, bool wfe) 705static int queue_poll_cons(struct arm_smmu_queue *q, bool sync, bool wfe)
@@ -901,6 +892,8 @@ static void arm_smmu_cmdq_insert_cmd(struct arm_smmu_device *smmu, u64 *cmd)
901 struct arm_smmu_queue *q = &smmu->cmdq.q; 892 struct arm_smmu_queue *q = &smmu->cmdq.q;
902 bool wfe = !!(smmu->features & ARM_SMMU_FEAT_SEV); 893 bool wfe = !!(smmu->features & ARM_SMMU_FEAT_SEV);
903 894
895 smmu->prev_cmd_opcode = FIELD_GET(CMDQ_0_OP, cmd[0]);
896
904 while (queue_insert_raw(q, cmd) == -ENOSPC) { 897 while (queue_insert_raw(q, cmd) == -ENOSPC) {
905 if (queue_poll_cons(q, false, wfe)) 898 if (queue_poll_cons(q, false, wfe))
906 dev_err_ratelimited(smmu->dev, "CMDQ timeout\n"); 899 dev_err_ratelimited(smmu->dev, "CMDQ timeout\n");
@@ -948,15 +941,21 @@ static int __arm_smmu_cmdq_issue_sync_msi(struct arm_smmu_device *smmu)
948 struct arm_smmu_cmdq_ent ent = { 941 struct arm_smmu_cmdq_ent ent = {
949 .opcode = CMDQ_OP_CMD_SYNC, 942 .opcode = CMDQ_OP_CMD_SYNC,
950 .sync = { 943 .sync = {
951 .msidata = atomic_inc_return_relaxed(&smmu->sync_nr),
952 .msiaddr = virt_to_phys(&smmu->sync_count), 944 .msiaddr = virt_to_phys(&smmu->sync_count),
953 }, 945 },
954 }; 946 };
955 947
956 arm_smmu_cmdq_build_cmd(cmd, &ent);
957
958 spin_lock_irqsave(&smmu->cmdq.lock, flags); 948 spin_lock_irqsave(&smmu->cmdq.lock, flags);
959 arm_smmu_cmdq_insert_cmd(smmu, cmd); 949
950 /* Piggy-back on the previous command if it's a SYNC */
951 if (smmu->prev_cmd_opcode == CMDQ_OP_CMD_SYNC) {
952 ent.sync.msidata = smmu->sync_nr;
953 } else {
954 ent.sync.msidata = ++smmu->sync_nr;
955 arm_smmu_cmdq_build_cmd(cmd, &ent);
956 arm_smmu_cmdq_insert_cmd(smmu, cmd);
957 }
958
960 spin_unlock_irqrestore(&smmu->cmdq.lock, flags); 959 spin_unlock_irqrestore(&smmu->cmdq.lock, flags);
961 960
962 return __arm_smmu_sync_poll_msi(smmu, ent.sync.msidata); 961 return __arm_smmu_sync_poll_msi(smmu, ent.sync.msidata);
@@ -1372,15 +1371,11 @@ static irqreturn_t arm_smmu_combined_irq_handler(int irq, void *dev)
1372} 1371}
1373 1372
1374/* IO_PGTABLE API */ 1373/* IO_PGTABLE API */
1375static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu)
1376{
1377 arm_smmu_cmdq_issue_sync(smmu);
1378}
1379
1380static void arm_smmu_tlb_sync(void *cookie) 1374static void arm_smmu_tlb_sync(void *cookie)
1381{ 1375{
1382 struct arm_smmu_domain *smmu_domain = cookie; 1376 struct arm_smmu_domain *smmu_domain = cookie;
1383 __arm_smmu_tlb_sync(smmu_domain->smmu); 1377
1378 arm_smmu_cmdq_issue_sync(smmu_domain->smmu);
1384} 1379}
1385 1380
1386static void arm_smmu_tlb_inv_context(void *cookie) 1381static void arm_smmu_tlb_inv_context(void *cookie)
@@ -1398,8 +1393,14 @@ static void arm_smmu_tlb_inv_context(void *cookie)
1398 cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid; 1393 cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid;
1399 } 1394 }
1400 1395
1396 /*
1397 * NOTE: when io-pgtable is in non-strict mode, we may get here with
1398 * PTEs previously cleared by unmaps on the current CPU not yet visible
1399 * to the SMMU. We are relying on the DSB implicit in queue_inc_prod()
1400 * to guarantee those are observed before the TLBI. Do be careful, 007.
1401 */
1401 arm_smmu_cmdq_issue_cmd(smmu, &cmd); 1402 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1402 __arm_smmu_tlb_sync(smmu); 1403 arm_smmu_cmdq_issue_sync(smmu);
1403} 1404}
1404 1405
1405static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size, 1406static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
@@ -1624,6 +1625,9 @@ static int arm_smmu_domain_finalise(struct iommu_domain *domain)
1624 if (smmu->features & ARM_SMMU_FEAT_COHERENCY) 1625 if (smmu->features & ARM_SMMU_FEAT_COHERENCY)
1625 pgtbl_cfg.quirks = IO_PGTABLE_QUIRK_NO_DMA; 1626 pgtbl_cfg.quirks = IO_PGTABLE_QUIRK_NO_DMA;
1626 1627
1628 if (smmu_domain->non_strict)
1629 pgtbl_cfg.quirks |= IO_PGTABLE_QUIRK_NON_STRICT;
1630
1627 pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain); 1631 pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
1628 if (!pgtbl_ops) 1632 if (!pgtbl_ops)
1629 return -ENOMEM; 1633 return -ENOMEM;
@@ -1772,12 +1776,20 @@ arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size)
1772 return ops->unmap(ops, iova, size); 1776 return ops->unmap(ops, iova, size);
1773} 1777}
1774 1778
1779static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
1780{
1781 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1782
1783 if (smmu_domain->smmu)
1784 arm_smmu_tlb_inv_context(smmu_domain);
1785}
1786
1775static void arm_smmu_iotlb_sync(struct iommu_domain *domain) 1787static void arm_smmu_iotlb_sync(struct iommu_domain *domain)
1776{ 1788{
1777 struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu; 1789 struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1778 1790
1779 if (smmu) 1791 if (smmu)
1780 __arm_smmu_tlb_sync(smmu); 1792 arm_smmu_cmdq_issue_sync(smmu);
1781} 1793}
1782 1794
1783static phys_addr_t 1795static phys_addr_t
@@ -1917,15 +1929,27 @@ static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
1917{ 1929{
1918 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 1930 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1919 1931
1920 if (domain->type != IOMMU_DOMAIN_UNMANAGED) 1932 switch (domain->type) {
1921 return -EINVAL; 1933 case IOMMU_DOMAIN_UNMANAGED:
1922 1934 switch (attr) {
1923 switch (attr) { 1935 case DOMAIN_ATTR_NESTING:
1924 case DOMAIN_ATTR_NESTING: 1936 *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
1925 *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED); 1937 return 0;
1926 return 0; 1938 default:
1939 return -ENODEV;
1940 }
1941 break;
1942 case IOMMU_DOMAIN_DMA:
1943 switch (attr) {
1944 case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE:
1945 *(int *)data = smmu_domain->non_strict;
1946 return 0;
1947 default:
1948 return -ENODEV;
1949 }
1950 break;
1927 default: 1951 default:
1928 return -ENODEV; 1952 return -EINVAL;
1929 } 1953 }
1930} 1954}
1931 1955
@@ -1935,26 +1959,37 @@ static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
1935 int ret = 0; 1959 int ret = 0;
1936 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 1960 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1937 1961
1938 if (domain->type != IOMMU_DOMAIN_UNMANAGED)
1939 return -EINVAL;
1940
1941 mutex_lock(&smmu_domain->init_mutex); 1962 mutex_lock(&smmu_domain->init_mutex);
1942 1963
1943 switch (attr) { 1964 switch (domain->type) {
1944 case DOMAIN_ATTR_NESTING: 1965 case IOMMU_DOMAIN_UNMANAGED:
1945 if (smmu_domain->smmu) { 1966 switch (attr) {
1946 ret = -EPERM; 1967 case DOMAIN_ATTR_NESTING:
1947 goto out_unlock; 1968 if (smmu_domain->smmu) {
1969 ret = -EPERM;
1970 goto out_unlock;
1971 }
1972
1973 if (*(int *)data)
1974 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1975 else
1976 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1977 break;
1978 default:
1979 ret = -ENODEV;
1980 }
1981 break;
1982 case IOMMU_DOMAIN_DMA:
1983 switch(attr) {
1984 case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE:
1985 smmu_domain->non_strict = *(int *)data;
1986 break;
1987 default:
1988 ret = -ENODEV;
1948 } 1989 }
1949
1950 if (*(int *)data)
1951 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1952 else
1953 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1954
1955 break; 1990 break;
1956 default: 1991 default:
1957 ret = -ENODEV; 1992 ret = -EINVAL;
1958 } 1993 }
1959 1994
1960out_unlock: 1995out_unlock:
@@ -1999,7 +2034,7 @@ static struct iommu_ops arm_smmu_ops = {
1999 .attach_dev = arm_smmu_attach_dev, 2034 .attach_dev = arm_smmu_attach_dev,
2000 .map = arm_smmu_map, 2035 .map = arm_smmu_map,
2001 .unmap = arm_smmu_unmap, 2036 .unmap = arm_smmu_unmap,
2002 .flush_iotlb_all = arm_smmu_iotlb_sync, 2037 .flush_iotlb_all = arm_smmu_flush_iotlb_all,
2003 .iotlb_sync = arm_smmu_iotlb_sync, 2038 .iotlb_sync = arm_smmu_iotlb_sync,
2004 .iova_to_phys = arm_smmu_iova_to_phys, 2039 .iova_to_phys = arm_smmu_iova_to_phys,
2005 .add_device = arm_smmu_add_device, 2040 .add_device = arm_smmu_add_device,
@@ -2180,7 +2215,6 @@ static int arm_smmu_init_structures(struct arm_smmu_device *smmu)
2180{ 2215{
2181 int ret; 2216 int ret;
2182 2217
2183 atomic_set(&smmu->sync_nr, 0);
2184 ret = arm_smmu_init_queues(smmu); 2218 ret = arm_smmu_init_queues(smmu);
2185 if (ret) 2219 if (ret)
2186 return ret; 2220 return ret;
@@ -2353,8 +2387,8 @@ static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu)
2353 irq = smmu->combined_irq; 2387 irq = smmu->combined_irq;
2354 if (irq) { 2388 if (irq) {
2355 /* 2389 /*
2356 * Cavium ThunderX2 implementation doesn't not support unique 2390 * Cavium ThunderX2 implementation doesn't support unique irq
2357 * irq lines. Use single irq line for all the SMMUv3 interrupts. 2391 * lines. Use a single irq line for all the SMMUv3 interrupts.
2358 */ 2392 */
2359 ret = devm_request_threaded_irq(smmu->dev, irq, 2393 ret = devm_request_threaded_irq(smmu->dev, irq,
2360 arm_smmu_combined_irq_handler, 2394 arm_smmu_combined_irq_handler,
diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c
index fd1b80ef9490..5a28ae892504 100644
--- a/drivers/iommu/arm-smmu.c
+++ b/drivers/iommu/arm-smmu.c
@@ -52,6 +52,7 @@
52#include <linux/spinlock.h> 52#include <linux/spinlock.h>
53 53
54#include <linux/amba/bus.h> 54#include <linux/amba/bus.h>
55#include <linux/fsl/mc.h>
55 56
56#include "io-pgtable.h" 57#include "io-pgtable.h"
57#include "arm-smmu-regs.h" 58#include "arm-smmu-regs.h"
@@ -246,6 +247,7 @@ struct arm_smmu_domain {
246 const struct iommu_gather_ops *tlb_ops; 247 const struct iommu_gather_ops *tlb_ops;
247 struct arm_smmu_cfg cfg; 248 struct arm_smmu_cfg cfg;
248 enum arm_smmu_domain_stage stage; 249 enum arm_smmu_domain_stage stage;
250 bool non_strict;
249 struct mutex init_mutex; /* Protects smmu pointer */ 251 struct mutex init_mutex; /* Protects smmu pointer */
250 spinlock_t cb_lock; /* Serialises ATS1* ops and TLB syncs */ 252 spinlock_t cb_lock; /* Serialises ATS1* ops and TLB syncs */
251 struct iommu_domain domain; 253 struct iommu_domain domain;
@@ -447,7 +449,11 @@ static void arm_smmu_tlb_inv_context_s1(void *cookie)
447 struct arm_smmu_cfg *cfg = &smmu_domain->cfg; 449 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
448 void __iomem *base = ARM_SMMU_CB(smmu_domain->smmu, cfg->cbndx); 450 void __iomem *base = ARM_SMMU_CB(smmu_domain->smmu, cfg->cbndx);
449 451
450 writel_relaxed(cfg->asid, base + ARM_SMMU_CB_S1_TLBIASID); 452 /*
453 * NOTE: this is not a relaxed write; it needs to guarantee that PTEs
454 * cleared by the current CPU are visible to the SMMU before the TLBI.
455 */
456 writel(cfg->asid, base + ARM_SMMU_CB_S1_TLBIASID);
451 arm_smmu_tlb_sync_context(cookie); 457 arm_smmu_tlb_sync_context(cookie);
452} 458}
453 459
@@ -457,7 +463,8 @@ static void arm_smmu_tlb_inv_context_s2(void *cookie)
457 struct arm_smmu_device *smmu = smmu_domain->smmu; 463 struct arm_smmu_device *smmu = smmu_domain->smmu;
458 void __iomem *base = ARM_SMMU_GR0(smmu); 464 void __iomem *base = ARM_SMMU_GR0(smmu);
459 465
460 writel_relaxed(smmu_domain->cfg.vmid, base + ARM_SMMU_GR0_TLBIVMID); 466 /* NOTE: see above */
467 writel(smmu_domain->cfg.vmid, base + ARM_SMMU_GR0_TLBIVMID);
461 arm_smmu_tlb_sync_global(smmu); 468 arm_smmu_tlb_sync_global(smmu);
462} 469}
463 470
@@ -469,6 +476,9 @@ static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
469 bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS; 476 bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
470 void __iomem *reg = ARM_SMMU_CB(smmu_domain->smmu, cfg->cbndx); 477 void __iomem *reg = ARM_SMMU_CB(smmu_domain->smmu, cfg->cbndx);
471 478
479 if (smmu_domain->smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
480 wmb();
481
472 if (stage1) { 482 if (stage1) {
473 reg += leaf ? ARM_SMMU_CB_S1_TLBIVAL : ARM_SMMU_CB_S1_TLBIVA; 483 reg += leaf ? ARM_SMMU_CB_S1_TLBIVAL : ARM_SMMU_CB_S1_TLBIVA;
474 484
@@ -510,6 +520,9 @@ static void arm_smmu_tlb_inv_vmid_nosync(unsigned long iova, size_t size,
510 struct arm_smmu_domain *smmu_domain = cookie; 520 struct arm_smmu_domain *smmu_domain = cookie;
511 void __iomem *base = ARM_SMMU_GR0(smmu_domain->smmu); 521 void __iomem *base = ARM_SMMU_GR0(smmu_domain->smmu);
512 522
523 if (smmu_domain->smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
524 wmb();
525
513 writel_relaxed(smmu_domain->cfg.vmid, base + ARM_SMMU_GR0_TLBIVMID); 526 writel_relaxed(smmu_domain->cfg.vmid, base + ARM_SMMU_GR0_TLBIVMID);
514} 527}
515 528
@@ -863,6 +876,9 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain,
863 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK) 876 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
864 pgtbl_cfg.quirks = IO_PGTABLE_QUIRK_NO_DMA; 877 pgtbl_cfg.quirks = IO_PGTABLE_QUIRK_NO_DMA;
865 878
879 if (smmu_domain->non_strict)
880 pgtbl_cfg.quirks |= IO_PGTABLE_QUIRK_NON_STRICT;
881
866 smmu_domain->smmu = smmu; 882 smmu_domain->smmu = smmu;
867 pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain); 883 pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
868 if (!pgtbl_ops) { 884 if (!pgtbl_ops) {
@@ -1252,6 +1268,14 @@ static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
1252 return ops->unmap(ops, iova, size); 1268 return ops->unmap(ops, iova, size);
1253} 1269}
1254 1270
1271static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
1272{
1273 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1274
1275 if (smmu_domain->tlb_ops)
1276 smmu_domain->tlb_ops->tlb_flush_all(smmu_domain);
1277}
1278
1255static void arm_smmu_iotlb_sync(struct iommu_domain *domain) 1279static void arm_smmu_iotlb_sync(struct iommu_domain *domain)
1256{ 1280{
1257 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 1281 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
@@ -1459,6 +1483,8 @@ static struct iommu_group *arm_smmu_device_group(struct device *dev)
1459 1483
1460 if (dev_is_pci(dev)) 1484 if (dev_is_pci(dev))
1461 group = pci_device_group(dev); 1485 group = pci_device_group(dev);
1486 else if (dev_is_fsl_mc(dev))
1487 group = fsl_mc_device_group(dev);
1462 else 1488 else
1463 group = generic_device_group(dev); 1489 group = generic_device_group(dev);
1464 1490
@@ -1470,15 +1496,27 @@ static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
1470{ 1496{
1471 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 1497 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1472 1498
1473 if (domain->type != IOMMU_DOMAIN_UNMANAGED) 1499 switch(domain->type) {
1474 return -EINVAL; 1500 case IOMMU_DOMAIN_UNMANAGED:
1475 1501 switch (attr) {
1476 switch (attr) { 1502 case DOMAIN_ATTR_NESTING:
1477 case DOMAIN_ATTR_NESTING: 1503 *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
1478 *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED); 1504 return 0;
1479 return 0; 1505 default:
1506 return -ENODEV;
1507 }
1508 break;
1509 case IOMMU_DOMAIN_DMA:
1510 switch (attr) {
1511 case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE:
1512 *(int *)data = smmu_domain->non_strict;
1513 return 0;
1514 default:
1515 return -ENODEV;
1516 }
1517 break;
1480 default: 1518 default:
1481 return -ENODEV; 1519 return -EINVAL;
1482 } 1520 }
1483} 1521}
1484 1522
@@ -1488,28 +1526,38 @@ static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
1488 int ret = 0; 1526 int ret = 0;
1489 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 1527 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1490 1528
1491 if (domain->type != IOMMU_DOMAIN_UNMANAGED)
1492 return -EINVAL;
1493
1494 mutex_lock(&smmu_domain->init_mutex); 1529 mutex_lock(&smmu_domain->init_mutex);
1495 1530
1496 switch (attr) { 1531 switch(domain->type) {
1497 case DOMAIN_ATTR_NESTING: 1532 case IOMMU_DOMAIN_UNMANAGED:
1498 if (smmu_domain->smmu) { 1533 switch (attr) {
1499 ret = -EPERM; 1534 case DOMAIN_ATTR_NESTING:
1500 goto out_unlock; 1535 if (smmu_domain->smmu) {
1536 ret = -EPERM;
1537 goto out_unlock;
1538 }
1539
1540 if (*(int *)data)
1541 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1542 else
1543 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1544 break;
1545 default:
1546 ret = -ENODEV;
1547 }
1548 break;
1549 case IOMMU_DOMAIN_DMA:
1550 switch (attr) {
1551 case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE:
1552 smmu_domain->non_strict = *(int *)data;
1553 break;
1554 default:
1555 ret = -ENODEV;
1501 } 1556 }
1502
1503 if (*(int *)data)
1504 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1505 else
1506 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1507
1508 break; 1557 break;
1509 default: 1558 default:
1510 ret = -ENODEV; 1559 ret = -EINVAL;
1511 } 1560 }
1512
1513out_unlock: 1561out_unlock:
1514 mutex_unlock(&smmu_domain->init_mutex); 1562 mutex_unlock(&smmu_domain->init_mutex);
1515 return ret; 1563 return ret;
@@ -1562,7 +1610,7 @@ static struct iommu_ops arm_smmu_ops = {
1562 .attach_dev = arm_smmu_attach_dev, 1610 .attach_dev = arm_smmu_attach_dev,
1563 .map = arm_smmu_map, 1611 .map = arm_smmu_map,
1564 .unmap = arm_smmu_unmap, 1612 .unmap = arm_smmu_unmap,
1565 .flush_iotlb_all = arm_smmu_iotlb_sync, 1613 .flush_iotlb_all = arm_smmu_flush_iotlb_all,
1566 .iotlb_sync = arm_smmu_iotlb_sync, 1614 .iotlb_sync = arm_smmu_iotlb_sync,
1567 .iova_to_phys = arm_smmu_iova_to_phys, 1615 .iova_to_phys = arm_smmu_iova_to_phys,
1568 .add_device = arm_smmu_add_device, 1616 .add_device = arm_smmu_add_device,
@@ -2036,6 +2084,10 @@ static void arm_smmu_bus_init(void)
2036 bus_set_iommu(&pci_bus_type, &arm_smmu_ops); 2084 bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
2037 } 2085 }
2038#endif 2086#endif
2087#ifdef CONFIG_FSL_MC_BUS
2088 if (!iommu_present(&fsl_mc_bus_type))
2089 bus_set_iommu(&fsl_mc_bus_type, &arm_smmu_ops);
2090#endif
2039} 2091}
2040 2092
2041static int arm_smmu_device_probe(struct platform_device *pdev) 2093static int arm_smmu_device_probe(struct platform_device *pdev)
diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c
index 511ff9a1d6d9..d1b04753b204 100644
--- a/drivers/iommu/dma-iommu.c
+++ b/drivers/iommu/dma-iommu.c
@@ -55,6 +55,9 @@ struct iommu_dma_cookie {
55 }; 55 };
56 struct list_head msi_page_list; 56 struct list_head msi_page_list;
57 spinlock_t msi_lock; 57 spinlock_t msi_lock;
58
59 /* Domain for flush queue callback; NULL if flush queue not in use */
60 struct iommu_domain *fq_domain;
58}; 61};
59 62
60static inline size_t cookie_msi_granule(struct iommu_dma_cookie *cookie) 63static inline size_t cookie_msi_granule(struct iommu_dma_cookie *cookie)
@@ -257,6 +260,20 @@ static int iova_reserve_iommu_regions(struct device *dev,
257 return ret; 260 return ret;
258} 261}
259 262
263static void iommu_dma_flush_iotlb_all(struct iova_domain *iovad)
264{
265 struct iommu_dma_cookie *cookie;
266 struct iommu_domain *domain;
267
268 cookie = container_of(iovad, struct iommu_dma_cookie, iovad);
269 domain = cookie->fq_domain;
270 /*
271 * The IOMMU driver supporting DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE
272 * implies that ops->flush_iotlb_all must be non-NULL.
273 */
274 domain->ops->flush_iotlb_all(domain);
275}
276
260/** 277/**
261 * iommu_dma_init_domain - Initialise a DMA mapping domain 278 * iommu_dma_init_domain - Initialise a DMA mapping domain
262 * @domain: IOMMU domain previously prepared by iommu_get_dma_cookie() 279 * @domain: IOMMU domain previously prepared by iommu_get_dma_cookie()
@@ -275,6 +292,7 @@ int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base,
275 struct iommu_dma_cookie *cookie = domain->iova_cookie; 292 struct iommu_dma_cookie *cookie = domain->iova_cookie;
276 struct iova_domain *iovad = &cookie->iovad; 293 struct iova_domain *iovad = &cookie->iovad;
277 unsigned long order, base_pfn, end_pfn; 294 unsigned long order, base_pfn, end_pfn;
295 int attr;
278 296
279 if (!cookie || cookie->type != IOMMU_DMA_IOVA_COOKIE) 297 if (!cookie || cookie->type != IOMMU_DMA_IOVA_COOKIE)
280 return -EINVAL; 298 return -EINVAL;
@@ -308,6 +326,13 @@ int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base,
308 } 326 }
309 327
310 init_iova_domain(iovad, 1UL << order, base_pfn); 328 init_iova_domain(iovad, 1UL << order, base_pfn);
329
330 if (!cookie->fq_domain && !iommu_domain_get_attr(domain,
331 DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE, &attr) && attr) {
332 cookie->fq_domain = domain;
333 init_iova_flush_queue(iovad, iommu_dma_flush_iotlb_all, NULL);
334 }
335
311 if (!dev) 336 if (!dev)
312 return 0; 337 return 0;
313 338
@@ -393,6 +418,9 @@ static void iommu_dma_free_iova(struct iommu_dma_cookie *cookie,
393 /* The MSI case is only ever cleaning up its most recent allocation */ 418 /* The MSI case is only ever cleaning up its most recent allocation */
394 if (cookie->type == IOMMU_DMA_MSI_COOKIE) 419 if (cookie->type == IOMMU_DMA_MSI_COOKIE)
395 cookie->msi_iova -= size; 420 cookie->msi_iova -= size;
421 else if (cookie->fq_domain) /* non-strict mode */
422 queue_iova(iovad, iova_pfn(iovad, iova),
423 size >> iova_shift(iovad), 0);
396 else 424 else
397 free_iova_fast(iovad, iova_pfn(iovad, iova), 425 free_iova_fast(iovad, iova_pfn(iovad, iova),
398 size >> iova_shift(iovad)); 426 size >> iova_shift(iovad));
@@ -408,7 +436,9 @@ static void __iommu_dma_unmap(struct iommu_domain *domain, dma_addr_t dma_addr,
408 dma_addr -= iova_off; 436 dma_addr -= iova_off;
409 size = iova_align(iovad, size + iova_off); 437 size = iova_align(iovad, size + iova_off);
410 438
411 WARN_ON(iommu_unmap(domain, dma_addr, size) != size); 439 WARN_ON(iommu_unmap_fast(domain, dma_addr, size) != size);
440 if (!cookie->fq_domain)
441 iommu_tlb_sync(domain);
412 iommu_dma_free_iova(cookie, dma_addr, size); 442 iommu_dma_free_iova(cookie, dma_addr, size);
413} 443}
414 444
@@ -491,7 +521,7 @@ static struct page **__iommu_dma_alloc_pages(unsigned int count,
491void iommu_dma_free(struct device *dev, struct page **pages, size_t size, 521void iommu_dma_free(struct device *dev, struct page **pages, size_t size,
492 dma_addr_t *handle) 522 dma_addr_t *handle)
493{ 523{
494 __iommu_dma_unmap(iommu_get_domain_for_dev(dev), *handle, size); 524 __iommu_dma_unmap(iommu_get_dma_domain(dev), *handle, size);
495 __iommu_dma_free_pages(pages, PAGE_ALIGN(size) >> PAGE_SHIFT); 525 __iommu_dma_free_pages(pages, PAGE_ALIGN(size) >> PAGE_SHIFT);
496 *handle = IOMMU_MAPPING_ERROR; 526 *handle = IOMMU_MAPPING_ERROR;
497} 527}
@@ -518,7 +548,7 @@ struct page **iommu_dma_alloc(struct device *dev, size_t size, gfp_t gfp,
518 unsigned long attrs, int prot, dma_addr_t *handle, 548 unsigned long attrs, int prot, dma_addr_t *handle,
519 void (*flush_page)(struct device *, const void *, phys_addr_t)) 549 void (*flush_page)(struct device *, const void *, phys_addr_t))
520{ 550{
521 struct iommu_domain *domain = iommu_get_domain_for_dev(dev); 551 struct iommu_domain *domain = iommu_get_dma_domain(dev);
522 struct iommu_dma_cookie *cookie = domain->iova_cookie; 552 struct iommu_dma_cookie *cookie = domain->iova_cookie;
523 struct iova_domain *iovad = &cookie->iovad; 553 struct iova_domain *iovad = &cookie->iovad;
524 struct page **pages; 554 struct page **pages;
@@ -606,9 +636,8 @@ int iommu_dma_mmap(struct page **pages, size_t size, struct vm_area_struct *vma)
606} 636}
607 637
608static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys, 638static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys,
609 size_t size, int prot) 639 size_t size, int prot, struct iommu_domain *domain)
610{ 640{
611 struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
612 struct iommu_dma_cookie *cookie = domain->iova_cookie; 641 struct iommu_dma_cookie *cookie = domain->iova_cookie;
613 size_t iova_off = 0; 642 size_t iova_off = 0;
614 dma_addr_t iova; 643 dma_addr_t iova;
@@ -632,13 +661,14 @@ static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys,
632dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page, 661dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page,
633 unsigned long offset, size_t size, int prot) 662 unsigned long offset, size_t size, int prot)
634{ 663{
635 return __iommu_dma_map(dev, page_to_phys(page) + offset, size, prot); 664 return __iommu_dma_map(dev, page_to_phys(page) + offset, size, prot,
665 iommu_get_dma_domain(dev));
636} 666}
637 667
638void iommu_dma_unmap_page(struct device *dev, dma_addr_t handle, size_t size, 668void iommu_dma_unmap_page(struct device *dev, dma_addr_t handle, size_t size,
639 enum dma_data_direction dir, unsigned long attrs) 669 enum dma_data_direction dir, unsigned long attrs)
640{ 670{
641 __iommu_dma_unmap(iommu_get_domain_for_dev(dev), handle, size); 671 __iommu_dma_unmap(iommu_get_dma_domain(dev), handle, size);
642} 672}
643 673
644/* 674/*
@@ -726,7 +756,7 @@ static void __invalidate_sg(struct scatterlist *sg, int nents)
726int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, 756int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg,
727 int nents, int prot) 757 int nents, int prot)
728{ 758{
729 struct iommu_domain *domain = iommu_get_domain_for_dev(dev); 759 struct iommu_domain *domain = iommu_get_dma_domain(dev);
730 struct iommu_dma_cookie *cookie = domain->iova_cookie; 760 struct iommu_dma_cookie *cookie = domain->iova_cookie;
731 struct iova_domain *iovad = &cookie->iovad; 761 struct iova_domain *iovad = &cookie->iovad;
732 struct scatterlist *s, *prev = NULL; 762 struct scatterlist *s, *prev = NULL;
@@ -811,20 +841,21 @@ void iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
811 sg = tmp; 841 sg = tmp;
812 } 842 }
813 end = sg_dma_address(sg) + sg_dma_len(sg); 843 end = sg_dma_address(sg) + sg_dma_len(sg);
814 __iommu_dma_unmap(iommu_get_domain_for_dev(dev), start, end - start); 844 __iommu_dma_unmap(iommu_get_dma_domain(dev), start, end - start);
815} 845}
816 846
817dma_addr_t iommu_dma_map_resource(struct device *dev, phys_addr_t phys, 847dma_addr_t iommu_dma_map_resource(struct device *dev, phys_addr_t phys,
818 size_t size, enum dma_data_direction dir, unsigned long attrs) 848 size_t size, enum dma_data_direction dir, unsigned long attrs)
819{ 849{
820 return __iommu_dma_map(dev, phys, size, 850 return __iommu_dma_map(dev, phys, size,
821 dma_info_to_prot(dir, false, attrs) | IOMMU_MMIO); 851 dma_info_to_prot(dir, false, attrs) | IOMMU_MMIO,
852 iommu_get_dma_domain(dev));
822} 853}
823 854
824void iommu_dma_unmap_resource(struct device *dev, dma_addr_t handle, 855void iommu_dma_unmap_resource(struct device *dev, dma_addr_t handle,
825 size_t size, enum dma_data_direction dir, unsigned long attrs) 856 size_t size, enum dma_data_direction dir, unsigned long attrs)
826{ 857{
827 __iommu_dma_unmap(iommu_get_domain_for_dev(dev), handle, size); 858 __iommu_dma_unmap(iommu_get_dma_domain(dev), handle, size);
828} 859}
829 860
830int iommu_dma_mapping_error(struct device *dev, dma_addr_t dma_addr) 861int iommu_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
@@ -850,7 +881,7 @@ static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev,
850 if (!msi_page) 881 if (!msi_page)
851 return NULL; 882 return NULL;
852 883
853 iova = __iommu_dma_map(dev, msi_addr, size, prot); 884 iova = __iommu_dma_map(dev, msi_addr, size, prot, domain);
854 if (iommu_dma_mapping_error(dev, iova)) 885 if (iommu_dma_mapping_error(dev, iova))
855 goto out_free_page; 886 goto out_free_page;
856 887
diff --git a/drivers/iommu/fsl_pamu_domain.c b/drivers/iommu/fsl_pamu_domain.c
index f089136e9c3f..9b528cfcc6c3 100644
--- a/drivers/iommu/fsl_pamu_domain.c
+++ b/drivers/iommu/fsl_pamu_domain.c
@@ -814,6 +814,55 @@ static int configure_domain_dma_state(struct fsl_dma_domain *dma_domain, bool en
814 return 0; 814 return 0;
815} 815}
816 816
817static int fsl_pamu_set_windows(struct iommu_domain *domain, u32 w_count)
818{
819 struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
820 unsigned long flags;
821 int ret;
822
823 spin_lock_irqsave(&dma_domain->domain_lock, flags);
824 /* Ensure domain is inactive i.e. DMA should be disabled for the domain */
825 if (dma_domain->enabled) {
826 pr_debug("Can't set geometry attributes as domain is active\n");
827 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
828 return -EBUSY;
829 }
830
831 /* Ensure that the geometry has been set for the domain */
832 if (!dma_domain->geom_size) {
833 pr_debug("Please configure geometry before setting the number of windows\n");
834 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
835 return -EINVAL;
836 }
837
838 /*
839 * Ensure we have valid window count i.e. it should be less than
840 * maximum permissible limit and should be a power of two.
841 */
842 if (w_count > pamu_get_max_subwin_cnt() || !is_power_of_2(w_count)) {
843 pr_debug("Invalid window count\n");
844 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
845 return -EINVAL;
846 }
847
848 ret = pamu_set_domain_geometry(dma_domain, &domain->geometry,
849 w_count > 1 ? w_count : 0);
850 if (!ret) {
851 kfree(dma_domain->win_arr);
852 dma_domain->win_arr = kcalloc(w_count,
853 sizeof(*dma_domain->win_arr),
854 GFP_ATOMIC);
855 if (!dma_domain->win_arr) {
856 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
857 return -ENOMEM;
858 }
859 dma_domain->win_cnt = w_count;
860 }
861 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
862
863 return ret;
864}
865
817static int fsl_pamu_set_domain_attr(struct iommu_domain *domain, 866static int fsl_pamu_set_domain_attr(struct iommu_domain *domain,
818 enum iommu_attr attr_type, void *data) 867 enum iommu_attr attr_type, void *data)
819{ 868{
@@ -830,6 +879,9 @@ static int fsl_pamu_set_domain_attr(struct iommu_domain *domain,
830 case DOMAIN_ATTR_FSL_PAMU_ENABLE: 879 case DOMAIN_ATTR_FSL_PAMU_ENABLE:
831 ret = configure_domain_dma_state(dma_domain, *(int *)data); 880 ret = configure_domain_dma_state(dma_domain, *(int *)data);
832 break; 881 break;
882 case DOMAIN_ATTR_WINDOWS:
883 ret = fsl_pamu_set_windows(domain, *(u32 *)data);
884 break;
833 default: 885 default:
834 pr_debug("Unsupported attribute type\n"); 886 pr_debug("Unsupported attribute type\n");
835 ret = -EINVAL; 887 ret = -EINVAL;
@@ -856,6 +908,9 @@ static int fsl_pamu_get_domain_attr(struct iommu_domain *domain,
856 case DOMAIN_ATTR_FSL_PAMUV1: 908 case DOMAIN_ATTR_FSL_PAMUV1:
857 *(int *)data = DOMAIN_ATTR_FSL_PAMUV1; 909 *(int *)data = DOMAIN_ATTR_FSL_PAMUV1;
858 break; 910 break;
911 case DOMAIN_ATTR_WINDOWS:
912 *(u32 *)data = dma_domain->win_cnt;
913 break;
859 default: 914 default:
860 pr_debug("Unsupported attribute type\n"); 915 pr_debug("Unsupported attribute type\n");
861 ret = -EINVAL; 916 ret = -EINVAL;
@@ -916,13 +971,13 @@ static struct iommu_group *get_shared_pci_device_group(struct pci_dev *pdev)
916static struct iommu_group *get_pci_device_group(struct pci_dev *pdev) 971static struct iommu_group *get_pci_device_group(struct pci_dev *pdev)
917{ 972{
918 struct pci_controller *pci_ctl; 973 struct pci_controller *pci_ctl;
919 bool pci_endpt_partioning; 974 bool pci_endpt_partitioning;
920 struct iommu_group *group = NULL; 975 struct iommu_group *group = NULL;
921 976
922 pci_ctl = pci_bus_to_host(pdev->bus); 977 pci_ctl = pci_bus_to_host(pdev->bus);
923 pci_endpt_partioning = check_pci_ctl_endpt_part(pci_ctl); 978 pci_endpt_partitioning = check_pci_ctl_endpt_part(pci_ctl);
924 /* We can partition PCIe devices so assign device group to the device */ 979 /* We can partition PCIe devices so assign device group to the device */
925 if (pci_endpt_partioning) { 980 if (pci_endpt_partitioning) {
926 group = pci_device_group(&pdev->dev); 981 group = pci_device_group(&pdev->dev);
927 982
928 /* 983 /*
@@ -994,62 +1049,6 @@ static void fsl_pamu_remove_device(struct device *dev)
994 iommu_group_remove_device(dev); 1049 iommu_group_remove_device(dev);
995} 1050}
996 1051
997static int fsl_pamu_set_windows(struct iommu_domain *domain, u32 w_count)
998{
999 struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
1000 unsigned long flags;
1001 int ret;
1002
1003 spin_lock_irqsave(&dma_domain->domain_lock, flags);
1004 /* Ensure domain is inactive i.e. DMA should be disabled for the domain */
1005 if (dma_domain->enabled) {
1006 pr_debug("Can't set geometry attributes as domain is active\n");
1007 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
1008 return -EBUSY;
1009 }
1010
1011 /* Ensure that the geometry has been set for the domain */
1012 if (!dma_domain->geom_size) {
1013 pr_debug("Please configure geometry before setting the number of windows\n");
1014 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
1015 return -EINVAL;
1016 }
1017
1018 /*
1019 * Ensure we have valid window count i.e. it should be less than
1020 * maximum permissible limit and should be a power of two.
1021 */
1022 if (w_count > pamu_get_max_subwin_cnt() || !is_power_of_2(w_count)) {
1023 pr_debug("Invalid window count\n");
1024 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
1025 return -EINVAL;
1026 }
1027
1028 ret = pamu_set_domain_geometry(dma_domain, &domain->geometry,
1029 w_count > 1 ? w_count : 0);
1030 if (!ret) {
1031 kfree(dma_domain->win_arr);
1032 dma_domain->win_arr = kcalloc(w_count,
1033 sizeof(*dma_domain->win_arr),
1034 GFP_ATOMIC);
1035 if (!dma_domain->win_arr) {
1036 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
1037 return -ENOMEM;
1038 }
1039 dma_domain->win_cnt = w_count;
1040 }
1041 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
1042
1043 return ret;
1044}
1045
1046static u32 fsl_pamu_get_windows(struct iommu_domain *domain)
1047{
1048 struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
1049
1050 return dma_domain->win_cnt;
1051}
1052
1053static const struct iommu_ops fsl_pamu_ops = { 1052static const struct iommu_ops fsl_pamu_ops = {
1054 .capable = fsl_pamu_capable, 1053 .capable = fsl_pamu_capable,
1055 .domain_alloc = fsl_pamu_domain_alloc, 1054 .domain_alloc = fsl_pamu_domain_alloc,
@@ -1058,8 +1057,6 @@ static const struct iommu_ops fsl_pamu_ops = {
1058 .detach_dev = fsl_pamu_detach_device, 1057 .detach_dev = fsl_pamu_detach_device,
1059 .domain_window_enable = fsl_pamu_window_enable, 1058 .domain_window_enable = fsl_pamu_window_enable,
1060 .domain_window_disable = fsl_pamu_window_disable, 1059 .domain_window_disable = fsl_pamu_window_disable,
1061 .domain_get_windows = fsl_pamu_get_windows,
1062 .domain_set_windows = fsl_pamu_set_windows,
1063 .iova_to_phys = fsl_pamu_iova_to_phys, 1060 .iova_to_phys = fsl_pamu_iova_to_phys,
1064 .domain_set_attr = fsl_pamu_set_domain_attr, 1061 .domain_set_attr = fsl_pamu_set_domain_attr,
1065 .domain_get_attr = fsl_pamu_get_domain_attr, 1062 .domain_get_attr = fsl_pamu_get_domain_attr,
diff --git a/drivers/iommu/intel-iommu-debugfs.c b/drivers/iommu/intel-iommu-debugfs.c
new file mode 100644
index 000000000000..7fabf9b1c2dc
--- /dev/null
+++ b/drivers/iommu/intel-iommu-debugfs.c
@@ -0,0 +1,314 @@
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright © 2018 Intel Corporation.
4 *
5 * Authors: Gayatri Kammela <gayatri.kammela@intel.com>
6 * Sohil Mehta <sohil.mehta@intel.com>
7 * Jacob Pan <jacob.jun.pan@linux.intel.com>
8 */
9
10#include <linux/debugfs.h>
11#include <linux/dmar.h>
12#include <linux/intel-iommu.h>
13#include <linux/pci.h>
14
15#include <asm/irq_remapping.h>
16
17struct iommu_regset {
18 int offset;
19 const char *regs;
20};
21
22#define IOMMU_REGSET_ENTRY(_reg_) \
23 { DMAR_##_reg_##_REG, __stringify(_reg_) }
24static const struct iommu_regset iommu_regs[] = {
25 IOMMU_REGSET_ENTRY(VER),
26 IOMMU_REGSET_ENTRY(CAP),
27 IOMMU_REGSET_ENTRY(ECAP),
28 IOMMU_REGSET_ENTRY(GCMD),
29 IOMMU_REGSET_ENTRY(GSTS),
30 IOMMU_REGSET_ENTRY(RTADDR),
31 IOMMU_REGSET_ENTRY(CCMD),
32 IOMMU_REGSET_ENTRY(FSTS),
33 IOMMU_REGSET_ENTRY(FECTL),
34 IOMMU_REGSET_ENTRY(FEDATA),
35 IOMMU_REGSET_ENTRY(FEADDR),
36 IOMMU_REGSET_ENTRY(FEUADDR),
37 IOMMU_REGSET_ENTRY(AFLOG),
38 IOMMU_REGSET_ENTRY(PMEN),
39 IOMMU_REGSET_ENTRY(PLMBASE),
40 IOMMU_REGSET_ENTRY(PLMLIMIT),
41 IOMMU_REGSET_ENTRY(PHMBASE),
42 IOMMU_REGSET_ENTRY(PHMLIMIT),
43 IOMMU_REGSET_ENTRY(IQH),
44 IOMMU_REGSET_ENTRY(IQT),
45 IOMMU_REGSET_ENTRY(IQA),
46 IOMMU_REGSET_ENTRY(ICS),
47 IOMMU_REGSET_ENTRY(IRTA),
48 IOMMU_REGSET_ENTRY(PQH),
49 IOMMU_REGSET_ENTRY(PQT),
50 IOMMU_REGSET_ENTRY(PQA),
51 IOMMU_REGSET_ENTRY(PRS),
52 IOMMU_REGSET_ENTRY(PECTL),
53 IOMMU_REGSET_ENTRY(PEDATA),
54 IOMMU_REGSET_ENTRY(PEADDR),
55 IOMMU_REGSET_ENTRY(PEUADDR),
56 IOMMU_REGSET_ENTRY(MTRRCAP),
57 IOMMU_REGSET_ENTRY(MTRRDEF),
58 IOMMU_REGSET_ENTRY(MTRR_FIX64K_00000),
59 IOMMU_REGSET_ENTRY(MTRR_FIX16K_80000),
60 IOMMU_REGSET_ENTRY(MTRR_FIX16K_A0000),
61 IOMMU_REGSET_ENTRY(MTRR_FIX4K_C0000),
62 IOMMU_REGSET_ENTRY(MTRR_FIX4K_C8000),
63 IOMMU_REGSET_ENTRY(MTRR_FIX4K_D0000),
64 IOMMU_REGSET_ENTRY(MTRR_FIX4K_D8000),
65 IOMMU_REGSET_ENTRY(MTRR_FIX4K_E0000),
66 IOMMU_REGSET_ENTRY(MTRR_FIX4K_E8000),
67 IOMMU_REGSET_ENTRY(MTRR_FIX4K_F0000),
68 IOMMU_REGSET_ENTRY(MTRR_FIX4K_F8000),
69 IOMMU_REGSET_ENTRY(MTRR_PHYSBASE0),
70 IOMMU_REGSET_ENTRY(MTRR_PHYSMASK0),
71 IOMMU_REGSET_ENTRY(MTRR_PHYSBASE1),
72 IOMMU_REGSET_ENTRY(MTRR_PHYSMASK1),
73 IOMMU_REGSET_ENTRY(MTRR_PHYSBASE2),
74 IOMMU_REGSET_ENTRY(MTRR_PHYSMASK2),
75 IOMMU_REGSET_ENTRY(MTRR_PHYSBASE3),
76 IOMMU_REGSET_ENTRY(MTRR_PHYSMASK3),
77 IOMMU_REGSET_ENTRY(MTRR_PHYSBASE4),
78 IOMMU_REGSET_ENTRY(MTRR_PHYSMASK4),
79 IOMMU_REGSET_ENTRY(MTRR_PHYSBASE5),
80 IOMMU_REGSET_ENTRY(MTRR_PHYSMASK5),
81 IOMMU_REGSET_ENTRY(MTRR_PHYSBASE6),
82 IOMMU_REGSET_ENTRY(MTRR_PHYSMASK6),
83 IOMMU_REGSET_ENTRY(MTRR_PHYSBASE7),
84 IOMMU_REGSET_ENTRY(MTRR_PHYSMASK7),
85 IOMMU_REGSET_ENTRY(MTRR_PHYSBASE8),
86 IOMMU_REGSET_ENTRY(MTRR_PHYSMASK8),
87 IOMMU_REGSET_ENTRY(MTRR_PHYSBASE9),
88 IOMMU_REGSET_ENTRY(MTRR_PHYSMASK9),
89 IOMMU_REGSET_ENTRY(VCCAP),
90 IOMMU_REGSET_ENTRY(VCMD),
91 IOMMU_REGSET_ENTRY(VCRSP),
92};
93
94static int iommu_regset_show(struct seq_file *m, void *unused)
95{
96 struct dmar_drhd_unit *drhd;
97 struct intel_iommu *iommu;
98 unsigned long flag;
99 int i, ret = 0;
100 u64 value;
101
102 rcu_read_lock();
103 for_each_active_iommu(iommu, drhd) {
104 if (!drhd->reg_base_addr) {
105 seq_puts(m, "IOMMU: Invalid base address\n");
106 ret = -EINVAL;
107 goto out;
108 }
109
110 seq_printf(m, "IOMMU: %s Register Base Address: %llx\n",
111 iommu->name, drhd->reg_base_addr);
112 seq_puts(m, "Name\t\t\tOffset\t\tContents\n");
113 /*
114 * Publish the contents of the 64-bit hardware registers
115 * by adding the offset to the pointer (virtual address).
116 */
117 raw_spin_lock_irqsave(&iommu->register_lock, flag);
118 for (i = 0 ; i < ARRAY_SIZE(iommu_regs); i++) {
119 value = dmar_readq(iommu->reg + iommu_regs[i].offset);
120 seq_printf(m, "%-16s\t0x%02x\t\t0x%016llx\n",
121 iommu_regs[i].regs, iommu_regs[i].offset,
122 value);
123 }
124 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
125 seq_putc(m, '\n');
126 }
127out:
128 rcu_read_unlock();
129
130 return ret;
131}
132DEFINE_SHOW_ATTRIBUTE(iommu_regset);
133
134static void ctx_tbl_entry_show(struct seq_file *m, struct intel_iommu *iommu,
135 int bus)
136{
137 struct context_entry *context;
138 int devfn;
139
140 seq_printf(m, " Context Table Entries for Bus: %d\n", bus);
141 seq_puts(m, " Entry\tB:D.F\tHigh\tLow\n");
142
143 for (devfn = 0; devfn < 256; devfn++) {
144 context = iommu_context_addr(iommu, bus, devfn, 0);
145 if (!context)
146 return;
147
148 if (!context_present(context))
149 continue;
150
151 seq_printf(m, " %-5d\t%02x:%02x.%x\t%-6llx\t%llx\n", devfn,
152 bus, PCI_SLOT(devfn), PCI_FUNC(devfn),
153 context[0].hi, context[0].lo);
154 }
155}
156
157static void root_tbl_entry_show(struct seq_file *m, struct intel_iommu *iommu)
158{
159 unsigned long flags;
160 int bus;
161
162 spin_lock_irqsave(&iommu->lock, flags);
163 seq_printf(m, "IOMMU %s: Root Table Address:%llx\n", iommu->name,
164 (u64)virt_to_phys(iommu->root_entry));
165 seq_puts(m, "Root Table Entries:\n");
166
167 for (bus = 0; bus < 256; bus++) {
168 if (!(iommu->root_entry[bus].lo & 1))
169 continue;
170
171 seq_printf(m, " Bus: %d H: %llx L: %llx\n", bus,
172 iommu->root_entry[bus].hi,
173 iommu->root_entry[bus].lo);
174
175 ctx_tbl_entry_show(m, iommu, bus);
176 seq_putc(m, '\n');
177 }
178 spin_unlock_irqrestore(&iommu->lock, flags);
179}
180
181static int dmar_translation_struct_show(struct seq_file *m, void *unused)
182{
183 struct dmar_drhd_unit *drhd;
184 struct intel_iommu *iommu;
185
186 rcu_read_lock();
187 for_each_active_iommu(iommu, drhd) {
188 root_tbl_entry_show(m, iommu);
189 seq_putc(m, '\n');
190 }
191 rcu_read_unlock();
192
193 return 0;
194}
195DEFINE_SHOW_ATTRIBUTE(dmar_translation_struct);
196
197#ifdef CONFIG_IRQ_REMAP
198static void ir_tbl_remap_entry_show(struct seq_file *m,
199 struct intel_iommu *iommu)
200{
201 struct irte *ri_entry;
202 unsigned long flags;
203 int idx;
204
205 seq_puts(m, " Entry SrcID DstID Vct IRTE_high\t\tIRTE_low\n");
206
207 raw_spin_lock_irqsave(&irq_2_ir_lock, flags);
208 for (idx = 0; idx < INTR_REMAP_TABLE_ENTRIES; idx++) {
209 ri_entry = &iommu->ir_table->base[idx];
210 if (!ri_entry->present || ri_entry->p_pst)
211 continue;
212
213 seq_printf(m, " %-5d %02x:%02x.%01x %08x %02x %016llx\t%016llx\n",
214 idx, PCI_BUS_NUM(ri_entry->sid),
215 PCI_SLOT(ri_entry->sid), PCI_FUNC(ri_entry->sid),
216 ri_entry->dest_id, ri_entry->vector,
217 ri_entry->high, ri_entry->low);
218 }
219 raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags);
220}
221
222static void ir_tbl_posted_entry_show(struct seq_file *m,
223 struct intel_iommu *iommu)
224{
225 struct irte *pi_entry;
226 unsigned long flags;
227 int idx;
228
229 seq_puts(m, " Entry SrcID PDA_high PDA_low Vct IRTE_high\t\tIRTE_low\n");
230
231 raw_spin_lock_irqsave(&irq_2_ir_lock, flags);
232 for (idx = 0; idx < INTR_REMAP_TABLE_ENTRIES; idx++) {
233 pi_entry = &iommu->ir_table->base[idx];
234 if (!pi_entry->present || !pi_entry->p_pst)
235 continue;
236
237 seq_printf(m, " %-5d %02x:%02x.%01x %08x %08x %02x %016llx\t%016llx\n",
238 idx, PCI_BUS_NUM(pi_entry->sid),
239 PCI_SLOT(pi_entry->sid), PCI_FUNC(pi_entry->sid),
240 pi_entry->pda_h, pi_entry->pda_l << 6,
241 pi_entry->vector, pi_entry->high,
242 pi_entry->low);
243 }
244 raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags);
245}
246
247/*
248 * For active IOMMUs go through the Interrupt remapping
249 * table and print valid entries in a table format for
250 * Remapped and Posted Interrupts.
251 */
252static int ir_translation_struct_show(struct seq_file *m, void *unused)
253{
254 struct dmar_drhd_unit *drhd;
255 struct intel_iommu *iommu;
256 u64 irta;
257
258 rcu_read_lock();
259 for_each_active_iommu(iommu, drhd) {
260 if (!ecap_ir_support(iommu->ecap))
261 continue;
262
263 seq_printf(m, "Remapped Interrupt supported on IOMMU: %s\n",
264 iommu->name);
265
266 if (iommu->ir_table) {
267 irta = virt_to_phys(iommu->ir_table->base);
268 seq_printf(m, " IR table address:%llx\n", irta);
269 ir_tbl_remap_entry_show(m, iommu);
270 } else {
271 seq_puts(m, "Interrupt Remapping is not enabled\n");
272 }
273 seq_putc(m, '\n');
274 }
275
276 seq_puts(m, "****\n\n");
277
278 for_each_active_iommu(iommu, drhd) {
279 if (!cap_pi_support(iommu->cap))
280 continue;
281
282 seq_printf(m, "Posted Interrupt supported on IOMMU: %s\n",
283 iommu->name);
284
285 if (iommu->ir_table) {
286 irta = virt_to_phys(iommu->ir_table->base);
287 seq_printf(m, " IR table address:%llx\n", irta);
288 ir_tbl_posted_entry_show(m, iommu);
289 } else {
290 seq_puts(m, "Interrupt Remapping is not enabled\n");
291 }
292 seq_putc(m, '\n');
293 }
294 rcu_read_unlock();
295
296 return 0;
297}
298DEFINE_SHOW_ATTRIBUTE(ir_translation_struct);
299#endif
300
301void __init intel_iommu_debugfs_init(void)
302{
303 struct dentry *intel_iommu_debug = debugfs_create_dir("intel",
304 iommu_debugfs_dir);
305
306 debugfs_create_file("iommu_regset", 0444, intel_iommu_debug, NULL,
307 &iommu_regset_fops);
308 debugfs_create_file("dmar_translation_struct", 0444, intel_iommu_debug,
309 NULL, &dmar_translation_struct_fops);
310#ifdef CONFIG_IRQ_REMAP
311 debugfs_create_file("ir_translation_struct", 0444, intel_iommu_debug,
312 NULL, &ir_translation_struct_fops);
313#endif
314}
diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c
index 76f0a5d16ed3..f3ccf025108b 100644
--- a/drivers/iommu/intel-iommu.c
+++ b/drivers/iommu/intel-iommu.c
@@ -185,16 +185,6 @@ static int rwbf_quirk;
185static int force_on = 0; 185static int force_on = 0;
186int intel_iommu_tboot_noforce; 186int intel_iommu_tboot_noforce;
187 187
188/*
189 * 0: Present
190 * 1-11: Reserved
191 * 12-63: Context Ptr (12 - (haw-1))
192 * 64-127: Reserved
193 */
194struct root_entry {
195 u64 lo;
196 u64 hi;
197};
198#define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry)) 188#define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
199 189
200/* 190/*
@@ -220,21 +210,6 @@ static phys_addr_t root_entry_uctp(struct root_entry *re)
220 210
221 return re->hi & VTD_PAGE_MASK; 211 return re->hi & VTD_PAGE_MASK;
222} 212}
223/*
224 * low 64 bits:
225 * 0: present
226 * 1: fault processing disable
227 * 2-3: translation type
228 * 12-63: address space root
229 * high 64 bits:
230 * 0-2: address width
231 * 3-6: aval
232 * 8-23: domain id
233 */
234struct context_entry {
235 u64 lo;
236 u64 hi;
237};
238 213
239static inline void context_clear_pasid_enable(struct context_entry *context) 214static inline void context_clear_pasid_enable(struct context_entry *context)
240{ 215{
@@ -261,7 +236,7 @@ static inline bool __context_present(struct context_entry *context)
261 return (context->lo & 1); 236 return (context->lo & 1);
262} 237}
263 238
264static inline bool context_present(struct context_entry *context) 239bool context_present(struct context_entry *context)
265{ 240{
266 return context_pasid_enabled(context) ? 241 return context_pasid_enabled(context) ?
267 __context_present(context) : 242 __context_present(context) :
@@ -788,8 +763,8 @@ static void domain_update_iommu_cap(struct dmar_domain *domain)
788 domain->iommu_superpage = domain_update_iommu_superpage(NULL); 763 domain->iommu_superpage = domain_update_iommu_superpage(NULL);
789} 764}
790 765
791static inline struct context_entry *iommu_context_addr(struct intel_iommu *iommu, 766struct context_entry *iommu_context_addr(struct intel_iommu *iommu, u8 bus,
792 u8 bus, u8 devfn, int alloc) 767 u8 devfn, int alloc)
793{ 768{
794 struct root_entry *root = &iommu->root_entry[bus]; 769 struct root_entry *root = &iommu->root_entry[bus];
795 struct context_entry *context; 770 struct context_entry *context;
@@ -4860,6 +4835,7 @@ int __init intel_iommu_init(void)
4860 cpuhp_setup_state(CPUHP_IOMMU_INTEL_DEAD, "iommu/intel:dead", NULL, 4835 cpuhp_setup_state(CPUHP_IOMMU_INTEL_DEAD, "iommu/intel:dead", NULL,
4861 intel_iommu_cpu_dead); 4836 intel_iommu_cpu_dead);
4862 intel_iommu_enabled = 1; 4837 intel_iommu_enabled = 1;
4838 intel_iommu_debugfs_init();
4863 4839
4864 return 0; 4840 return 0;
4865 4841
diff --git a/drivers/iommu/intel_irq_remapping.c b/drivers/iommu/intel_irq_remapping.c
index 967450bd421a..c2d6c11431de 100644
--- a/drivers/iommu/intel_irq_remapping.c
+++ b/drivers/iommu/intel_irq_remapping.c
@@ -76,7 +76,7 @@ static struct hpet_scope ir_hpet[MAX_HPET_TBS];
76 * in single-threaded environment with interrupt disabled, so no need to tabke 76 * in single-threaded environment with interrupt disabled, so no need to tabke
77 * the dmar_global_lock. 77 * the dmar_global_lock.
78 */ 78 */
79static DEFINE_RAW_SPINLOCK(irq_2_ir_lock); 79DEFINE_RAW_SPINLOCK(irq_2_ir_lock);
80static const struct irq_domain_ops intel_ir_domain_ops; 80static const struct irq_domain_ops intel_ir_domain_ops;
81 81
82static void iommu_disable_irq_remapping(struct intel_iommu *iommu); 82static void iommu_disable_irq_remapping(struct intel_iommu *iommu);
diff --git a/drivers/iommu/io-pgtable-arm-v7s.c b/drivers/iommu/io-pgtable-arm-v7s.c
index b5948ba6b3b3..445c3bde0480 100644
--- a/drivers/iommu/io-pgtable-arm-v7s.c
+++ b/drivers/iommu/io-pgtable-arm-v7s.c
@@ -587,6 +587,7 @@ static size_t arm_v7s_split_blk_unmap(struct arm_v7s_io_pgtable *data,
587 } 587 }
588 588
589 io_pgtable_tlb_add_flush(&data->iop, iova, size, size, true); 589 io_pgtable_tlb_add_flush(&data->iop, iova, size, size, true);
590 io_pgtable_tlb_sync(&data->iop);
590 return size; 591 return size;
591} 592}
592 593
@@ -642,6 +643,13 @@ static size_t __arm_v7s_unmap(struct arm_v7s_io_pgtable *data,
642 io_pgtable_tlb_sync(iop); 643 io_pgtable_tlb_sync(iop);
643 ptep = iopte_deref(pte[i], lvl); 644 ptep = iopte_deref(pte[i], lvl);
644 __arm_v7s_free_table(ptep, lvl + 1, data); 645 __arm_v7s_free_table(ptep, lvl + 1, data);
646 } else if (iop->cfg.quirks & IO_PGTABLE_QUIRK_NON_STRICT) {
647 /*
648 * Order the PTE update against queueing the IOVA, to
649 * guarantee that a flush callback from a different CPU
650 * has observed it before the TLBIALL can be issued.
651 */
652 smp_wmb();
645 } else { 653 } else {
646 io_pgtable_tlb_add_flush(iop, iova, blk_size, 654 io_pgtable_tlb_add_flush(iop, iova, blk_size,
647 blk_size, true); 655 blk_size, true);
@@ -712,7 +720,8 @@ static struct io_pgtable *arm_v7s_alloc_pgtable(struct io_pgtable_cfg *cfg,
712 IO_PGTABLE_QUIRK_NO_PERMS | 720 IO_PGTABLE_QUIRK_NO_PERMS |
713 IO_PGTABLE_QUIRK_TLBI_ON_MAP | 721 IO_PGTABLE_QUIRK_TLBI_ON_MAP |
714 IO_PGTABLE_QUIRK_ARM_MTK_4GB | 722 IO_PGTABLE_QUIRK_ARM_MTK_4GB |
715 IO_PGTABLE_QUIRK_NO_DMA)) 723 IO_PGTABLE_QUIRK_NO_DMA |
724 IO_PGTABLE_QUIRK_NON_STRICT))
716 return NULL; 725 return NULL;
717 726
718 /* If ARM_MTK_4GB is enabled, the NO_PERMS is also expected. */ 727 /* If ARM_MTK_4GB is enabled, the NO_PERMS is also expected. */
diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c
index 88641b4560bc..237cacd4a62b 100644
--- a/drivers/iommu/io-pgtable-arm.c
+++ b/drivers/iommu/io-pgtable-arm.c
@@ -574,13 +574,13 @@ static size_t arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data,
574 return 0; 574 return 0;
575 575
576 tablep = iopte_deref(pte, data); 576 tablep = iopte_deref(pte, data);
577 } else if (unmap_idx >= 0) {
578 io_pgtable_tlb_add_flush(&data->iop, iova, size, size, true);
579 io_pgtable_tlb_sync(&data->iop);
580 return size;
577 } 581 }
578 582
579 if (unmap_idx < 0) 583 return __arm_lpae_unmap(data, iova, size, lvl, tablep);
580 return __arm_lpae_unmap(data, iova, size, lvl, tablep);
581
582 io_pgtable_tlb_add_flush(&data->iop, iova, size, size, true);
583 return size;
584} 584}
585 585
586static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, 586static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
@@ -610,6 +610,13 @@ static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
610 io_pgtable_tlb_sync(iop); 610 io_pgtable_tlb_sync(iop);
611 ptep = iopte_deref(pte, data); 611 ptep = iopte_deref(pte, data);
612 __arm_lpae_free_pgtable(data, lvl + 1, ptep); 612 __arm_lpae_free_pgtable(data, lvl + 1, ptep);
613 } else if (iop->cfg.quirks & IO_PGTABLE_QUIRK_NON_STRICT) {
614 /*
615 * Order the PTE update against queueing the IOVA, to
616 * guarantee that a flush callback from a different CPU
617 * has observed it before the TLBIALL can be issued.
618 */
619 smp_wmb();
613 } else { 620 } else {
614 io_pgtable_tlb_add_flush(iop, iova, size, size, true); 621 io_pgtable_tlb_add_flush(iop, iova, size, size, true);
615 } 622 }
@@ -772,7 +779,8 @@ arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie)
772 u64 reg; 779 u64 reg;
773 struct arm_lpae_io_pgtable *data; 780 struct arm_lpae_io_pgtable *data;
774 781
775 if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_NS | IO_PGTABLE_QUIRK_NO_DMA)) 782 if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_NS | IO_PGTABLE_QUIRK_NO_DMA |
783 IO_PGTABLE_QUIRK_NON_STRICT))
776 return NULL; 784 return NULL;
777 785
778 data = arm_lpae_alloc_pgtable(cfg); 786 data = arm_lpae_alloc_pgtable(cfg);
@@ -864,7 +872,8 @@ arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie)
864 struct arm_lpae_io_pgtable *data; 872 struct arm_lpae_io_pgtable *data;
865 873
866 /* The NS quirk doesn't apply at stage 2 */ 874 /* The NS quirk doesn't apply at stage 2 */
867 if (cfg->quirks & ~IO_PGTABLE_QUIRK_NO_DMA) 875 if (cfg->quirks & ~(IO_PGTABLE_QUIRK_NO_DMA |
876 IO_PGTABLE_QUIRK_NON_STRICT))
868 return NULL; 877 return NULL;
869 878
870 data = arm_lpae_alloc_pgtable(cfg); 879 data = arm_lpae_alloc_pgtable(cfg);
diff --git a/drivers/iommu/io-pgtable.h b/drivers/iommu/io-pgtable.h
index 2df79093cad9..47d5ae559329 100644
--- a/drivers/iommu/io-pgtable.h
+++ b/drivers/iommu/io-pgtable.h
@@ -71,12 +71,17 @@ struct io_pgtable_cfg {
71 * be accessed by a fully cache-coherent IOMMU or CPU (e.g. for a 71 * be accessed by a fully cache-coherent IOMMU or CPU (e.g. for a
72 * software-emulated IOMMU), such that pagetable updates need not 72 * software-emulated IOMMU), such that pagetable updates need not
73 * be treated as explicit DMA data. 73 * be treated as explicit DMA data.
74 *
75 * IO_PGTABLE_QUIRK_NON_STRICT: Skip issuing synchronous leaf TLBIs
76 * on unmap, for DMA domains using the flush queue mechanism for
77 * delayed invalidation.
74 */ 78 */
75 #define IO_PGTABLE_QUIRK_ARM_NS BIT(0) 79 #define IO_PGTABLE_QUIRK_ARM_NS BIT(0)
76 #define IO_PGTABLE_QUIRK_NO_PERMS BIT(1) 80 #define IO_PGTABLE_QUIRK_NO_PERMS BIT(1)
77 #define IO_PGTABLE_QUIRK_TLBI_ON_MAP BIT(2) 81 #define IO_PGTABLE_QUIRK_TLBI_ON_MAP BIT(2)
78 #define IO_PGTABLE_QUIRK_ARM_MTK_4GB BIT(3) 82 #define IO_PGTABLE_QUIRK_ARM_MTK_4GB BIT(3)
79 #define IO_PGTABLE_QUIRK_NO_DMA BIT(4) 83 #define IO_PGTABLE_QUIRK_NO_DMA BIT(4)
84 #define IO_PGTABLE_QUIRK_NON_STRICT BIT(5)
80 unsigned long quirks; 85 unsigned long quirks;
81 unsigned long pgsize_bitmap; 86 unsigned long pgsize_bitmap;
82 unsigned int ias; 87 unsigned int ias;
diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
index 8c15c5980299..edbdf5d6962c 100644
--- a/drivers/iommu/iommu.c
+++ b/drivers/iommu/iommu.c
@@ -32,6 +32,7 @@
32#include <linux/pci.h> 32#include <linux/pci.h>
33#include <linux/bitops.h> 33#include <linux/bitops.h>
34#include <linux/property.h> 34#include <linux/property.h>
35#include <linux/fsl/mc.h>
35#include <trace/events/iommu.h> 36#include <trace/events/iommu.h>
36 37
37static struct kset *iommu_group_kset; 38static struct kset *iommu_group_kset;
@@ -41,6 +42,7 @@ static unsigned int iommu_def_domain_type = IOMMU_DOMAIN_IDENTITY;
41#else 42#else
42static unsigned int iommu_def_domain_type = IOMMU_DOMAIN_DMA; 43static unsigned int iommu_def_domain_type = IOMMU_DOMAIN_DMA;
43#endif 44#endif
45static bool iommu_dma_strict __read_mostly = true;
44 46
45struct iommu_callback_data { 47struct iommu_callback_data {
46 const struct iommu_ops *ops; 48 const struct iommu_ops *ops;
@@ -131,6 +133,12 @@ static int __init iommu_set_def_domain_type(char *str)
131} 133}
132early_param("iommu.passthrough", iommu_set_def_domain_type); 134early_param("iommu.passthrough", iommu_set_def_domain_type);
133 135
136static int __init iommu_dma_setup(char *str)
137{
138 return kstrtobool(str, &iommu_dma_strict);
139}
140early_param("iommu.strict", iommu_dma_setup);
141
134static ssize_t iommu_group_attr_show(struct kobject *kobj, 142static ssize_t iommu_group_attr_show(struct kobject *kobj,
135 struct attribute *__attr, char *buf) 143 struct attribute *__attr, char *buf)
136{ 144{
@@ -1024,6 +1032,18 @@ struct iommu_group *pci_device_group(struct device *dev)
1024 return iommu_group_alloc(); 1032 return iommu_group_alloc();
1025} 1033}
1026 1034
1035/* Get the IOMMU group for device on fsl-mc bus */
1036struct iommu_group *fsl_mc_device_group(struct device *dev)
1037{
1038 struct device *cont_dev = fsl_mc_cont_dev(dev);
1039 struct iommu_group *group;
1040
1041 group = iommu_group_get(cont_dev);
1042 if (!group)
1043 group = iommu_group_alloc();
1044 return group;
1045}
1046
1027/** 1047/**
1028 * iommu_group_get_for_dev - Find or create the IOMMU group for a device 1048 * iommu_group_get_for_dev - Find or create the IOMMU group for a device
1029 * @dev: target device 1049 * @dev: target device
@@ -1072,6 +1092,13 @@ struct iommu_group *iommu_group_get_for_dev(struct device *dev)
1072 group->default_domain = dom; 1092 group->default_domain = dom;
1073 if (!group->domain) 1093 if (!group->domain)
1074 group->domain = dom; 1094 group->domain = dom;
1095
1096 if (dom && !iommu_dma_strict) {
1097 int attr = 1;
1098 iommu_domain_set_attr(dom,
1099 DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE,
1100 &attr);
1101 }
1075 } 1102 }
1076 1103
1077 ret = iommu_group_add_device(group, dev); 1104 ret = iommu_group_add_device(group, dev);
@@ -1416,7 +1443,16 @@ struct iommu_domain *iommu_get_domain_for_dev(struct device *dev)
1416EXPORT_SYMBOL_GPL(iommu_get_domain_for_dev); 1443EXPORT_SYMBOL_GPL(iommu_get_domain_for_dev);
1417 1444
1418/* 1445/*
1419 * IOMMU groups are really the natrual working unit of the IOMMU, but 1446 * For IOMMU_DOMAIN_DMA implementations which already provide their own
1447 * guarantees that the group and its default domain are valid and correct.
1448 */
1449struct iommu_domain *iommu_get_dma_domain(struct device *dev)
1450{
1451 return dev->iommu_group->default_domain;
1452}
1453
1454/*
1455 * IOMMU groups are really the natural working unit of the IOMMU, but
1420 * the IOMMU API works on domains and devices. Bridge that gap by 1456 * the IOMMU API works on domains and devices. Bridge that gap by
1421 * iterating over the devices in a group. Ideally we'd have a single 1457 * iterating over the devices in a group. Ideally we'd have a single
1422 * device which represents the requestor ID of the group, but we also 1458 * device which represents the requestor ID of the group, but we also
@@ -1796,7 +1832,6 @@ int iommu_domain_get_attr(struct iommu_domain *domain,
1796 struct iommu_domain_geometry *geometry; 1832 struct iommu_domain_geometry *geometry;
1797 bool *paging; 1833 bool *paging;
1798 int ret = 0; 1834 int ret = 0;
1799 u32 *count;
1800 1835
1801 switch (attr) { 1836 switch (attr) {
1802 case DOMAIN_ATTR_GEOMETRY: 1837 case DOMAIN_ATTR_GEOMETRY:
@@ -1808,15 +1843,6 @@ int iommu_domain_get_attr(struct iommu_domain *domain,
1808 paging = data; 1843 paging = data;
1809 *paging = (domain->pgsize_bitmap != 0UL); 1844 *paging = (domain->pgsize_bitmap != 0UL);
1810 break; 1845 break;
1811 case DOMAIN_ATTR_WINDOWS:
1812 count = data;
1813
1814 if (domain->ops->domain_get_windows != NULL)
1815 *count = domain->ops->domain_get_windows(domain);
1816 else
1817 ret = -ENODEV;
1818
1819 break;
1820 default: 1846 default:
1821 if (!domain->ops->domain_get_attr) 1847 if (!domain->ops->domain_get_attr)
1822 return -EINVAL; 1848 return -EINVAL;
@@ -1832,18 +1858,8 @@ int iommu_domain_set_attr(struct iommu_domain *domain,
1832 enum iommu_attr attr, void *data) 1858 enum iommu_attr attr, void *data)
1833{ 1859{
1834 int ret = 0; 1860 int ret = 0;
1835 u32 *count;
1836 1861
1837 switch (attr) { 1862 switch (attr) {
1838 case DOMAIN_ATTR_WINDOWS:
1839 count = data;
1840
1841 if (domain->ops->domain_set_windows != NULL)
1842 ret = domain->ops->domain_set_windows(domain, *count);
1843 else
1844 ret = -ENODEV;
1845
1846 break;
1847 default: 1863 default:
1848 if (domain->ops->domain_set_attr == NULL) 1864 if (domain->ops->domain_set_attr == NULL)
1849 return -EINVAL; 1865 return -EINVAL;
diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c
index 83fe2621effe..f8d3ba247523 100644
--- a/drivers/iommu/iova.c
+++ b/drivers/iommu/iova.c
@@ -56,6 +56,7 @@ init_iova_domain(struct iova_domain *iovad, unsigned long granule,
56 iovad->granule = granule; 56 iovad->granule = granule;
57 iovad->start_pfn = start_pfn; 57 iovad->start_pfn = start_pfn;
58 iovad->dma_32bit_pfn = 1UL << (32 - iova_shift(iovad)); 58 iovad->dma_32bit_pfn = 1UL << (32 - iova_shift(iovad));
59 iovad->max32_alloc_size = iovad->dma_32bit_pfn;
59 iovad->flush_cb = NULL; 60 iovad->flush_cb = NULL;
60 iovad->fq = NULL; 61 iovad->fq = NULL;
61 iovad->anchor.pfn_lo = iovad->anchor.pfn_hi = IOVA_ANCHOR; 62 iovad->anchor.pfn_lo = iovad->anchor.pfn_hi = IOVA_ANCHOR;
@@ -139,8 +140,10 @@ __cached_rbnode_delete_update(struct iova_domain *iovad, struct iova *free)
139 140
140 cached_iova = rb_entry(iovad->cached32_node, struct iova, node); 141 cached_iova = rb_entry(iovad->cached32_node, struct iova, node);
141 if (free->pfn_hi < iovad->dma_32bit_pfn && 142 if (free->pfn_hi < iovad->dma_32bit_pfn &&
142 free->pfn_lo >= cached_iova->pfn_lo) 143 free->pfn_lo >= cached_iova->pfn_lo) {
143 iovad->cached32_node = rb_next(&free->node); 144 iovad->cached32_node = rb_next(&free->node);
145 iovad->max32_alloc_size = iovad->dma_32bit_pfn;
146 }
144 147
145 cached_iova = rb_entry(iovad->cached_node, struct iova, node); 148 cached_iova = rb_entry(iovad->cached_node, struct iova, node);
146 if (free->pfn_lo >= cached_iova->pfn_lo) 149 if (free->pfn_lo >= cached_iova->pfn_lo)
@@ -190,6 +193,10 @@ static int __alloc_and_insert_iova_range(struct iova_domain *iovad,
190 193
191 /* Walk the tree backwards */ 194 /* Walk the tree backwards */
192 spin_lock_irqsave(&iovad->iova_rbtree_lock, flags); 195 spin_lock_irqsave(&iovad->iova_rbtree_lock, flags);
196 if (limit_pfn <= iovad->dma_32bit_pfn &&
197 size >= iovad->max32_alloc_size)
198 goto iova32_full;
199
193 curr = __get_cached_rbnode(iovad, limit_pfn); 200 curr = __get_cached_rbnode(iovad, limit_pfn);
194 curr_iova = rb_entry(curr, struct iova, node); 201 curr_iova = rb_entry(curr, struct iova, node);
195 do { 202 do {
@@ -200,10 +207,8 @@ static int __alloc_and_insert_iova_range(struct iova_domain *iovad,
200 curr_iova = rb_entry(curr, struct iova, node); 207 curr_iova = rb_entry(curr, struct iova, node);
201 } while (curr && new_pfn <= curr_iova->pfn_hi); 208 } while (curr && new_pfn <= curr_iova->pfn_hi);
202 209
203 if (limit_pfn < size || new_pfn < iovad->start_pfn) { 210 if (limit_pfn < size || new_pfn < iovad->start_pfn)
204 spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags); 211 goto iova32_full;
205 return -ENOMEM;
206 }
207 212
208 /* pfn_lo will point to size aligned address if size_aligned is set */ 213 /* pfn_lo will point to size aligned address if size_aligned is set */
209 new->pfn_lo = new_pfn; 214 new->pfn_lo = new_pfn;
@@ -214,9 +219,12 @@ static int __alloc_and_insert_iova_range(struct iova_domain *iovad,
214 __cached_rbnode_insert_update(iovad, new); 219 __cached_rbnode_insert_update(iovad, new);
215 220
216 spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags); 221 spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);
217
218
219 return 0; 222 return 0;
223
224iova32_full:
225 iovad->max32_alloc_size = size;
226 spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);
227 return -ENOMEM;
220} 228}
221 229
222static struct kmem_cache *iova_cache; 230static struct kmem_cache *iova_cache;
diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c
index 22b94f8a9a04..b98a03189580 100644
--- a/drivers/iommu/ipmmu-vmsa.c
+++ b/drivers/iommu/ipmmu-vmsa.c
@@ -1,11 +1,8 @@
1// SPDX-License-Identifier: GPL-2.0
1/* 2/*
2 * IPMMU VMSA 3 * IPMMU VMSA
3 * 4 *
4 * Copyright (C) 2014 Renesas Electronics Corporation 5 * Copyright (C) 2014 Renesas Electronics Corporation
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; version 2 of the License.
9 */ 6 */
10 7
11#include <linux/bitmap.h> 8#include <linux/bitmap.h>
diff --git a/drivers/iommu/of_iommu.c b/drivers/iommu/of_iommu.c
index f7787e757244..c5dd63072529 100644
--- a/drivers/iommu/of_iommu.c
+++ b/drivers/iommu/of_iommu.c
@@ -24,6 +24,7 @@
24#include <linux/of_iommu.h> 24#include <linux/of_iommu.h>
25#include <linux/of_pci.h> 25#include <linux/of_pci.h>
26#include <linux/slab.h> 26#include <linux/slab.h>
27#include <linux/fsl/mc.h>
27 28
28#define NO_IOMMU 1 29#define NO_IOMMU 1
29 30
@@ -132,9 +133,8 @@ static int of_pci_iommu_init(struct pci_dev *pdev, u16 alias, void *data)
132 struct of_phandle_args iommu_spec = { .args_count = 1 }; 133 struct of_phandle_args iommu_spec = { .args_count = 1 };
133 int err; 134 int err;
134 135
135 err = of_pci_map_rid(info->np, alias, "iommu-map", 136 err = of_map_rid(info->np, alias, "iommu-map", "iommu-map-mask",
136 "iommu-map-mask", &iommu_spec.np, 137 &iommu_spec.np, iommu_spec.args);
137 iommu_spec.args);
138 if (err) 138 if (err)
139 return err == -ENODEV ? NO_IOMMU : err; 139 return err == -ENODEV ? NO_IOMMU : err;
140 140
@@ -143,6 +143,23 @@ static int of_pci_iommu_init(struct pci_dev *pdev, u16 alias, void *data)
143 return err; 143 return err;
144} 144}
145 145
146static int of_fsl_mc_iommu_init(struct fsl_mc_device *mc_dev,
147 struct device_node *master_np)
148{
149 struct of_phandle_args iommu_spec = { .args_count = 1 };
150 int err;
151
152 err = of_map_rid(master_np, mc_dev->icid, "iommu-map",
153 "iommu-map-mask", &iommu_spec.np,
154 iommu_spec.args);
155 if (err)
156 return err == -ENODEV ? NO_IOMMU : err;
157
158 err = of_iommu_xlate(&mc_dev->dev, &iommu_spec);
159 of_node_put(iommu_spec.np);
160 return err;
161}
162
146const struct iommu_ops *of_iommu_configure(struct device *dev, 163const struct iommu_ops *of_iommu_configure(struct device *dev,
147 struct device_node *master_np) 164 struct device_node *master_np)
148{ 165{
@@ -174,6 +191,8 @@ const struct iommu_ops *of_iommu_configure(struct device *dev,
174 191
175 err = pci_for_each_dma_alias(to_pci_dev(dev), 192 err = pci_for_each_dma_alias(to_pci_dev(dev),
176 of_pci_iommu_init, &info); 193 of_pci_iommu_init, &info);
194 } else if (dev_is_fsl_mc(dev)) {
195 err = of_fsl_mc_iommu_init(to_fsl_mc_device(dev), master_np);
177 } else { 196 } else {
178 struct of_phandle_args iommu_spec; 197 struct of_phandle_args iommu_spec;
179 int idx = 0; 198 int idx = 0;
diff --git a/drivers/of/base.c b/drivers/of/base.c
index 74eaedd5b860..90bf7d9fa17b 100644
--- a/drivers/of/base.c
+++ b/drivers/of/base.c
@@ -2045,3 +2045,105 @@ int of_find_last_cache_level(unsigned int cpu)
2045 2045
2046 return cache_level; 2046 return cache_level;
2047} 2047}
2048
2049/**
2050 * of_map_rid - Translate a requester ID through a downstream mapping.
2051 * @np: root complex device node.
2052 * @rid: device requester ID to map.
2053 * @map_name: property name of the map to use.
2054 * @map_mask_name: optional property name of the mask to use.
2055 * @target: optional pointer to a target device node.
2056 * @id_out: optional pointer to receive the translated ID.
2057 *
2058 * Given a device requester ID, look up the appropriate implementation-defined
2059 * platform ID and/or the target device which receives transactions on that
2060 * ID, as per the "iommu-map" and "msi-map" bindings. Either of @target or
2061 * @id_out may be NULL if only the other is required. If @target points to
2062 * a non-NULL device node pointer, only entries targeting that node will be
2063 * matched; if it points to a NULL value, it will receive the device node of
2064 * the first matching target phandle, with a reference held.
2065 *
2066 * Return: 0 on success or a standard error code on failure.
2067 */
2068int of_map_rid(struct device_node *np, u32 rid,
2069 const char *map_name, const char *map_mask_name,
2070 struct device_node **target, u32 *id_out)
2071{
2072 u32 map_mask, masked_rid;
2073 int map_len;
2074 const __be32 *map = NULL;
2075
2076 if (!np || !map_name || (!target && !id_out))
2077 return -EINVAL;
2078
2079 map = of_get_property(np, map_name, &map_len);
2080 if (!map) {
2081 if (target)
2082 return -ENODEV;
2083 /* Otherwise, no map implies no translation */
2084 *id_out = rid;
2085 return 0;
2086 }
2087
2088 if (!map_len || map_len % (4 * sizeof(*map))) {
2089 pr_err("%pOF: Error: Bad %s length: %d\n", np,
2090 map_name, map_len);
2091 return -EINVAL;
2092 }
2093
2094 /* The default is to select all bits. */
2095 map_mask = 0xffffffff;
2096
2097 /*
2098 * Can be overridden by "{iommu,msi}-map-mask" property.
2099 * If of_property_read_u32() fails, the default is used.
2100 */
2101 if (map_mask_name)
2102 of_property_read_u32(np, map_mask_name, &map_mask);
2103
2104 masked_rid = map_mask & rid;
2105 for ( ; map_len > 0; map_len -= 4 * sizeof(*map), map += 4) {
2106 struct device_node *phandle_node;
2107 u32 rid_base = be32_to_cpup(map + 0);
2108 u32 phandle = be32_to_cpup(map + 1);
2109 u32 out_base = be32_to_cpup(map + 2);
2110 u32 rid_len = be32_to_cpup(map + 3);
2111
2112 if (rid_base & ~map_mask) {
2113 pr_err("%pOF: Invalid %s translation - %s-mask (0x%x) ignores rid-base (0x%x)\n",
2114 np, map_name, map_name,
2115 map_mask, rid_base);
2116 return -EFAULT;
2117 }
2118
2119 if (masked_rid < rid_base || masked_rid >= rid_base + rid_len)
2120 continue;
2121
2122 phandle_node = of_find_node_by_phandle(phandle);
2123 if (!phandle_node)
2124 return -ENODEV;
2125
2126 if (target) {
2127 if (*target)
2128 of_node_put(phandle_node);
2129 else
2130 *target = phandle_node;
2131
2132 if (*target != phandle_node)
2133 continue;
2134 }
2135
2136 if (id_out)
2137 *id_out = masked_rid - rid_base + out_base;
2138
2139 pr_debug("%pOF: %s, using mask %08x, rid-base: %08x, out-base: %08x, length: %08x, rid: %08x -> %08x\n",
2140 np, map_name, map_mask, rid_base, out_base,
2141 rid_len, rid, masked_rid - rid_base + out_base);
2142 return 0;
2143 }
2144
2145 pr_err("%pOF: Invalid %s translation - no match for rid 0x%x on %pOF\n",
2146 np, map_name, rid, target && *target ? *target : NULL);
2147 return -EFAULT;
2148}
2149EXPORT_SYMBOL_GPL(of_map_rid);
diff --git a/drivers/of/irq.c b/drivers/of/irq.c
index 02ad93a304a4..e1f6f392a4c0 100644
--- a/drivers/of/irq.c
+++ b/drivers/of/irq.c
@@ -22,7 +22,6 @@
22#include <linux/module.h> 22#include <linux/module.h>
23#include <linux/of.h> 23#include <linux/of.h>
24#include <linux/of_irq.h> 24#include <linux/of_irq.h>
25#include <linux/of_pci.h>
26#include <linux/string.h> 25#include <linux/string.h>
27#include <linux/slab.h> 26#include <linux/slab.h>
28 27
@@ -588,8 +587,8 @@ static u32 __of_msi_map_rid(struct device *dev, struct device_node **np,
588 * "msi-map" property. 587 * "msi-map" property.
589 */ 588 */
590 for (parent_dev = dev; parent_dev; parent_dev = parent_dev->parent) 589 for (parent_dev = dev; parent_dev; parent_dev = parent_dev->parent)
591 if (!of_pci_map_rid(parent_dev->of_node, rid_in, "msi-map", 590 if (!of_map_rid(parent_dev->of_node, rid_in, "msi-map",
592 "msi-map-mask", np, &rid_out)) 591 "msi-map-mask", np, &rid_out))
593 break; 592 break;
594 return rid_out; 593 return rid_out;
595} 594}
diff --git a/drivers/pci/of.c b/drivers/pci/of.c
index 1836b8ddf292..4c4217d0c3f1 100644
--- a/drivers/pci/of.c
+++ b/drivers/pci/of.c
@@ -355,107 +355,6 @@ failed:
355EXPORT_SYMBOL_GPL(devm_of_pci_get_host_bridge_resources); 355EXPORT_SYMBOL_GPL(devm_of_pci_get_host_bridge_resources);
356#endif /* CONFIG_OF_ADDRESS */ 356#endif /* CONFIG_OF_ADDRESS */
357 357
358/**
359 * of_pci_map_rid - Translate a requester ID through a downstream mapping.
360 * @np: root complex device node.
361 * @rid: PCI requester ID to map.
362 * @map_name: property name of the map to use.
363 * @map_mask_name: optional property name of the mask to use.
364 * @target: optional pointer to a target device node.
365 * @id_out: optional pointer to receive the translated ID.
366 *
367 * Given a PCI requester ID, look up the appropriate implementation-defined
368 * platform ID and/or the target device which receives transactions on that
369 * ID, as per the "iommu-map" and "msi-map" bindings. Either of @target or
370 * @id_out may be NULL if only the other is required. If @target points to
371 * a non-NULL device node pointer, only entries targeting that node will be
372 * matched; if it points to a NULL value, it will receive the device node of
373 * the first matching target phandle, with a reference held.
374 *
375 * Return: 0 on success or a standard error code on failure.
376 */
377int of_pci_map_rid(struct device_node *np, u32 rid,
378 const char *map_name, const char *map_mask_name,
379 struct device_node **target, u32 *id_out)
380{
381 u32 map_mask, masked_rid;
382 int map_len;
383 const __be32 *map = NULL;
384
385 if (!np || !map_name || (!target && !id_out))
386 return -EINVAL;
387
388 map = of_get_property(np, map_name, &map_len);
389 if (!map) {
390 if (target)
391 return -ENODEV;
392 /* Otherwise, no map implies no translation */
393 *id_out = rid;
394 return 0;
395 }
396
397 if (!map_len || map_len % (4 * sizeof(*map))) {
398 pr_err("%pOF: Error: Bad %s length: %d\n", np,
399 map_name, map_len);
400 return -EINVAL;
401 }
402
403 /* The default is to select all bits. */
404 map_mask = 0xffffffff;
405
406 /*
407 * Can be overridden by "{iommu,msi}-map-mask" property.
408 * If of_property_read_u32() fails, the default is used.
409 */
410 if (map_mask_name)
411 of_property_read_u32(np, map_mask_name, &map_mask);
412
413 masked_rid = map_mask & rid;
414 for ( ; map_len > 0; map_len -= 4 * sizeof(*map), map += 4) {
415 struct device_node *phandle_node;
416 u32 rid_base = be32_to_cpup(map + 0);
417 u32 phandle = be32_to_cpup(map + 1);
418 u32 out_base = be32_to_cpup(map + 2);
419 u32 rid_len = be32_to_cpup(map + 3);
420
421 if (rid_base & ~map_mask) {
422 pr_err("%pOF: Invalid %s translation - %s-mask (0x%x) ignores rid-base (0x%x)\n",
423 np, map_name, map_name,
424 map_mask, rid_base);
425 return -EFAULT;
426 }
427
428 if (masked_rid < rid_base || masked_rid >= rid_base + rid_len)
429 continue;
430
431 phandle_node = of_find_node_by_phandle(phandle);
432 if (!phandle_node)
433 return -ENODEV;
434
435 if (target) {
436 if (*target)
437 of_node_put(phandle_node);
438 else
439 *target = phandle_node;
440
441 if (*target != phandle_node)
442 continue;
443 }
444
445 if (id_out)
446 *id_out = masked_rid - rid_base + out_base;
447
448 pr_debug("%pOF: %s, using mask %08x, rid-base: %08x, out-base: %08x, length: %08x, rid: %08x -> %08x\n",
449 np, map_name, map_mask, rid_base, out_base,
450 rid_len, rid, masked_rid - rid_base + out_base);
451 return 0;
452 }
453
454 pr_err("%pOF: Invalid %s translation - no match for rid 0x%x on %pOF\n",
455 np, map_name, rid, target && *target ? *target : NULL);
456 return -EFAULT;
457}
458
459#if IS_ENABLED(CONFIG_OF_IRQ) 358#if IS_ENABLED(CONFIG_OF_IRQ)
460/** 359/**
461 * of_irq_parse_pci - Resolve the interrupt for a PCI device 360 * of_irq_parse_pci - Resolve the interrupt for a PCI device
diff --git a/include/linux/fsl/mc.h b/include/linux/fsl/mc.h
index 5160f06ffbac..9d3f668df7df 100644
--- a/include/linux/fsl/mc.h
+++ b/include/linux/fsl/mc.h
@@ -351,6 +351,14 @@ int mc_send_command(struct fsl_mc_io *mc_io, struct fsl_mc_command *cmd);
351#define dev_is_fsl_mc(_dev) (0) 351#define dev_is_fsl_mc(_dev) (0)
352#endif 352#endif
353 353
354/* Macro to check if a device is a container device */
355#define fsl_mc_is_cont_dev(_dev) (to_fsl_mc_device(_dev)->flags & \
356 FSL_MC_IS_DPRC)
357
358/* Macro to get the container device of a MC device */
359#define fsl_mc_cont_dev(_dev) (fsl_mc_is_cont_dev(_dev) ? \
360 (_dev) : (_dev)->parent)
361
354/* 362/*
355 * module_fsl_mc_driver() - Helper macro for drivers that don't do 363 * module_fsl_mc_driver() - Helper macro for drivers that don't do
356 * anything special in module init/exit. This eliminates a lot of 364 * anything special in module init/exit. This eliminates a lot of
diff --git a/include/linux/intel-iommu.h b/include/linux/intel-iommu.h
index 28004d74ae04..b0ae25837361 100644
--- a/include/linux/intel-iommu.h
+++ b/include/linux/intel-iommu.h
@@ -72,6 +72,42 @@
72#define DMAR_PEDATA_REG 0xe4 /* Page request event interrupt data register */ 72#define DMAR_PEDATA_REG 0xe4 /* Page request event interrupt data register */
73#define DMAR_PEADDR_REG 0xe8 /* Page request event interrupt addr register */ 73#define DMAR_PEADDR_REG 0xe8 /* Page request event interrupt addr register */
74#define DMAR_PEUADDR_REG 0xec /* Page request event Upper address register */ 74#define DMAR_PEUADDR_REG 0xec /* Page request event Upper address register */
75#define DMAR_MTRRCAP_REG 0x100 /* MTRR capability register */
76#define DMAR_MTRRDEF_REG 0x108 /* MTRR default type register */
77#define DMAR_MTRR_FIX64K_00000_REG 0x120 /* MTRR Fixed range registers */
78#define DMAR_MTRR_FIX16K_80000_REG 0x128
79#define DMAR_MTRR_FIX16K_A0000_REG 0x130
80#define DMAR_MTRR_FIX4K_C0000_REG 0x138
81#define DMAR_MTRR_FIX4K_C8000_REG 0x140
82#define DMAR_MTRR_FIX4K_D0000_REG 0x148
83#define DMAR_MTRR_FIX4K_D8000_REG 0x150
84#define DMAR_MTRR_FIX4K_E0000_REG 0x158
85#define DMAR_MTRR_FIX4K_E8000_REG 0x160
86#define DMAR_MTRR_FIX4K_F0000_REG 0x168
87#define DMAR_MTRR_FIX4K_F8000_REG 0x170
88#define DMAR_MTRR_PHYSBASE0_REG 0x180 /* MTRR Variable range registers */
89#define DMAR_MTRR_PHYSMASK0_REG 0x188
90#define DMAR_MTRR_PHYSBASE1_REG 0x190
91#define DMAR_MTRR_PHYSMASK1_REG 0x198
92#define DMAR_MTRR_PHYSBASE2_REG 0x1a0
93#define DMAR_MTRR_PHYSMASK2_REG 0x1a8
94#define DMAR_MTRR_PHYSBASE3_REG 0x1b0
95#define DMAR_MTRR_PHYSMASK3_REG 0x1b8
96#define DMAR_MTRR_PHYSBASE4_REG 0x1c0
97#define DMAR_MTRR_PHYSMASK4_REG 0x1c8
98#define DMAR_MTRR_PHYSBASE5_REG 0x1d0
99#define DMAR_MTRR_PHYSMASK5_REG 0x1d8
100#define DMAR_MTRR_PHYSBASE6_REG 0x1e0
101#define DMAR_MTRR_PHYSMASK6_REG 0x1e8
102#define DMAR_MTRR_PHYSBASE7_REG 0x1f0
103#define DMAR_MTRR_PHYSMASK7_REG 0x1f8
104#define DMAR_MTRR_PHYSBASE8_REG 0x200
105#define DMAR_MTRR_PHYSMASK8_REG 0x208
106#define DMAR_MTRR_PHYSBASE9_REG 0x210
107#define DMAR_MTRR_PHYSMASK9_REG 0x218
108#define DMAR_VCCAP_REG 0xe00 /* Virtual command capability register */
109#define DMAR_VCMD_REG 0xe10 /* Virtual command register */
110#define DMAR_VCRSP_REG 0xe20 /* Virtual command response register */
75 111
76#define OFFSET_STRIDE (9) 112#define OFFSET_STRIDE (9)
77 113
@@ -389,6 +425,33 @@ struct pasid_entry;
389struct pasid_state_entry; 425struct pasid_state_entry;
390struct page_req_dsc; 426struct page_req_dsc;
391 427
428/*
429 * 0: Present
430 * 1-11: Reserved
431 * 12-63: Context Ptr (12 - (haw-1))
432 * 64-127: Reserved
433 */
434struct root_entry {
435 u64 lo;
436 u64 hi;
437};
438
439/*
440 * low 64 bits:
441 * 0: present
442 * 1: fault processing disable
443 * 2-3: translation type
444 * 12-63: address space root
445 * high 64 bits:
446 * 0-2: address width
447 * 3-6: aval
448 * 8-23: domain id
449 */
450struct context_entry {
451 u64 lo;
452 u64 hi;
453};
454
392struct dmar_domain { 455struct dmar_domain {
393 int nid; /* node id */ 456 int nid; /* node id */
394 457
@@ -558,6 +621,15 @@ extern int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct intel_svm_
558extern struct intel_iommu *intel_svm_device_to_iommu(struct device *dev); 621extern struct intel_iommu *intel_svm_device_to_iommu(struct device *dev);
559#endif 622#endif
560 623
624#ifdef CONFIG_INTEL_IOMMU_DEBUGFS
625void intel_iommu_debugfs_init(void);
626#else
627static inline void intel_iommu_debugfs_init(void) {}
628#endif /* CONFIG_INTEL_IOMMU_DEBUGFS */
629
561extern const struct attribute_group *intel_iommu_groups[]; 630extern const struct attribute_group *intel_iommu_groups[];
631bool context_present(struct context_entry *context);
632struct context_entry *iommu_context_addr(struct intel_iommu *iommu, u8 bus,
633 u8 devfn, int alloc);
562 634
563#endif 635#endif
diff --git a/include/linux/iommu.h b/include/linux/iommu.h
index 87994c265bf5..a1d28f42cb77 100644
--- a/include/linux/iommu.h
+++ b/include/linux/iommu.h
@@ -124,6 +124,7 @@ enum iommu_attr {
124 DOMAIN_ATTR_FSL_PAMU_ENABLE, 124 DOMAIN_ATTR_FSL_PAMU_ENABLE,
125 DOMAIN_ATTR_FSL_PAMUV1, 125 DOMAIN_ATTR_FSL_PAMUV1,
126 DOMAIN_ATTR_NESTING, /* two stages of translation */ 126 DOMAIN_ATTR_NESTING, /* two stages of translation */
127 DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE,
127 DOMAIN_ATTR_MAX, 128 DOMAIN_ATTR_MAX,
128}; 129};
129 130
@@ -181,8 +182,6 @@ struct iommu_resv_region {
181 * @apply_resv_region: Temporary helper call-back for iova reserved ranges 182 * @apply_resv_region: Temporary helper call-back for iova reserved ranges
182 * @domain_window_enable: Configure and enable a particular window for a domain 183 * @domain_window_enable: Configure and enable a particular window for a domain
183 * @domain_window_disable: Disable a particular window for a domain 184 * @domain_window_disable: Disable a particular window for a domain
184 * @domain_set_windows: Set the number of windows for a domain
185 * @domain_get_windows: Return the number of windows for a domain
186 * @of_xlate: add OF master IDs to iommu grouping 185 * @of_xlate: add OF master IDs to iommu grouping
187 * @pgsize_bitmap: bitmap of all possible supported page sizes 186 * @pgsize_bitmap: bitmap of all possible supported page sizes
188 */ 187 */
@@ -223,10 +222,6 @@ struct iommu_ops {
223 int (*domain_window_enable)(struct iommu_domain *domain, u32 wnd_nr, 222 int (*domain_window_enable)(struct iommu_domain *domain, u32 wnd_nr,
224 phys_addr_t paddr, u64 size, int prot); 223 phys_addr_t paddr, u64 size, int prot);
225 void (*domain_window_disable)(struct iommu_domain *domain, u32 wnd_nr); 224 void (*domain_window_disable)(struct iommu_domain *domain, u32 wnd_nr);
226 /* Set the number of windows per domain */
227 int (*domain_set_windows)(struct iommu_domain *domain, u32 w_count);
228 /* Get the number of windows per domain */
229 u32 (*domain_get_windows)(struct iommu_domain *domain);
230 225
231 int (*of_xlate)(struct device *dev, struct of_phandle_args *args); 226 int (*of_xlate)(struct device *dev, struct of_phandle_args *args);
232 bool (*is_attach_deferred)(struct iommu_domain *domain, struct device *dev); 227 bool (*is_attach_deferred)(struct iommu_domain *domain, struct device *dev);
@@ -293,6 +288,7 @@ extern int iommu_attach_device(struct iommu_domain *domain,
293extern void iommu_detach_device(struct iommu_domain *domain, 288extern void iommu_detach_device(struct iommu_domain *domain,
294 struct device *dev); 289 struct device *dev);
295extern struct iommu_domain *iommu_get_domain_for_dev(struct device *dev); 290extern struct iommu_domain *iommu_get_domain_for_dev(struct device *dev);
291extern struct iommu_domain *iommu_get_dma_domain(struct device *dev);
296extern int iommu_map(struct iommu_domain *domain, unsigned long iova, 292extern int iommu_map(struct iommu_domain *domain, unsigned long iova,
297 phys_addr_t paddr, size_t size, int prot); 293 phys_addr_t paddr, size_t size, int prot);
298extern size_t iommu_unmap(struct iommu_domain *domain, unsigned long iova, 294extern size_t iommu_unmap(struct iommu_domain *domain, unsigned long iova,
@@ -377,6 +373,8 @@ static inline void iommu_tlb_sync(struct iommu_domain *domain)
377extern struct iommu_group *pci_device_group(struct device *dev); 373extern struct iommu_group *pci_device_group(struct device *dev);
378/* Generic device grouping function */ 374/* Generic device grouping function */
379extern struct iommu_group *generic_device_group(struct device *dev); 375extern struct iommu_group *generic_device_group(struct device *dev);
376/* FSL-MC device grouping function */
377struct iommu_group *fsl_mc_device_group(struct device *dev);
380 378
381/** 379/**
382 * struct iommu_fwspec - per-device IOMMU instance data 380 * struct iommu_fwspec - per-device IOMMU instance data
diff --git a/include/linux/iova.h b/include/linux/iova.h
index 928442dda565..0b93bf96693e 100644
--- a/include/linux/iova.h
+++ b/include/linux/iova.h
@@ -75,6 +75,7 @@ struct iova_domain {
75 unsigned long granule; /* pfn granularity for this domain */ 75 unsigned long granule; /* pfn granularity for this domain */
76 unsigned long start_pfn; /* Lower limit for this domain */ 76 unsigned long start_pfn; /* Lower limit for this domain */
77 unsigned long dma_32bit_pfn; 77 unsigned long dma_32bit_pfn;
78 unsigned long max32_alloc_size; /* Size of last failed allocation */
78 struct iova anchor; /* rbtree lookup anchor */ 79 struct iova anchor; /* rbtree lookup anchor */
79 struct iova_rcache rcaches[IOVA_RANGE_CACHE_MAX_SIZE]; /* IOVA range caches */ 80 struct iova_rcache rcaches[IOVA_RANGE_CACHE_MAX_SIZE]; /* IOVA range caches */
80 81
diff --git a/include/linux/of.h b/include/linux/of.h
index 99b0ebf49632..bf577ca3c8e7 100644
--- a/include/linux/of.h
+++ b/include/linux/of.h
@@ -550,6 +550,10 @@ bool of_console_check(struct device_node *dn, char *name, int index);
550 550
551extern int of_cpu_node_to_id(struct device_node *np); 551extern int of_cpu_node_to_id(struct device_node *np);
552 552
553int of_map_rid(struct device_node *np, u32 rid,
554 const char *map_name, const char *map_mask_name,
555 struct device_node **target, u32 *id_out);
556
553#else /* CONFIG_OF */ 557#else /* CONFIG_OF */
554 558
555static inline void of_core_init(void) 559static inline void of_core_init(void)
@@ -952,6 +956,13 @@ static inline int of_cpu_node_to_id(struct device_node *np)
952 return -ENODEV; 956 return -ENODEV;
953} 957}
954 958
959static inline int of_map_rid(struct device_node *np, u32 rid,
960 const char *map_name, const char *map_mask_name,
961 struct device_node **target, u32 *id_out)
962{
963 return -EINVAL;
964}
965
955#define of_match_ptr(_ptr) NULL 966#define of_match_ptr(_ptr) NULL
956#define of_match_node(_matches, _node) NULL 967#define of_match_node(_matches, _node) NULL
957#endif /* CONFIG_OF */ 968#endif /* CONFIG_OF */
diff --git a/include/linux/of_pci.h b/include/linux/of_pci.h
index e83d87fc5673..21a89c4880fa 100644
--- a/include/linux/of_pci.h
+++ b/include/linux/of_pci.h
@@ -14,9 +14,6 @@ struct device_node *of_pci_find_child_device(struct device_node *parent,
14 unsigned int devfn); 14 unsigned int devfn);
15int of_pci_get_devfn(struct device_node *np); 15int of_pci_get_devfn(struct device_node *np);
16void of_pci_check_probe_only(void); 16void of_pci_check_probe_only(void);
17int of_pci_map_rid(struct device_node *np, u32 rid,
18 const char *map_name, const char *map_mask_name,
19 struct device_node **target, u32 *id_out);
20#else 17#else
21static inline struct device_node *of_pci_find_child_device(struct device_node *parent, 18static inline struct device_node *of_pci_find_child_device(struct device_node *parent,
22 unsigned int devfn) 19 unsigned int devfn)
@@ -29,13 +26,6 @@ static inline int of_pci_get_devfn(struct device_node *np)
29 return -EINVAL; 26 return -EINVAL;
30} 27}
31 28
32static inline int of_pci_map_rid(struct device_node *np, u32 rid,
33 const char *map_name, const char *map_mask_name,
34 struct device_node **target, u32 *id_out)
35{
36 return -EINVAL;
37}
38
39static inline void of_pci_check_probe_only(void) { } 29static inline void of_pci_check_probe_only(void) { }
40#endif 30#endif
41 31
generated by cgit v1.2.2 (git 2.25.0) at 2026-07-22 01:13:36 -0400