aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2018-08-24 16:10:38 -0400
committerLinus Torvalds <torvalds@linux-foundation.org>2018-08-24 16:10:38 -0400
commit18b8bfdfbae5821a7df691bc1e542bbab6c31e9c (patch)
tree4c7c1d942c71b19fc6430e4acbe5b2f5967ab93d
parentd972604f6f87478212f012af5560c4fd4bb2b01d (diff)
parent6488a7f35eeab463f6c9c2ea34d30ca856f0bc8e (diff)
Merge tag 'iommu-updates-v4.19' of git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu
Pull IOMMU updates from Joerg Roedel: - PASID table handling updates for the Intel VT-d driver. It implements a global PASID space now so that applications usings multiple devices will just have one PASID. - A new config option to make iommu passthroug mode the default. - New sysfs attribute for iommu groups to export the type of the default domain. - A debugfs interface (for debug only) usable by IOMMU drivers to export internals to user-space. - R-Car Gen3 SoCs support for the ipmmu-vmsa driver - The ARM-SMMU now aborts transactions from unknown devices and devices not attached to any domain. - Various cleanups and smaller fixes all over the place. * tag 'iommu-updates-v4.19' of git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu: (42 commits) iommu/omap: Fix cache flushes on L2 table entries iommu: Remove the ->map_sg indirection iommu/arm-smmu-v3: Abort all transactions if SMMU is enabled in kdump kernel iommu/arm-smmu-v3: Prevent any devices access to memory without registration iommu/ipmmu-vmsa: Don't register as BUS IOMMU if machine doesn't have IPMMU-VMSA iommu/ipmmu-vmsa: Clarify supported platforms iommu/ipmmu-vmsa: Fix allocation in atomic context iommu: Add config option to set passthrough as default iommu: Add sysfs attribyte for domain type iommu/arm-smmu-v3: sync the OVACKFLG to PRIQ consumer register iommu/arm-smmu: Error out only if not enough context interrupts iommu/io-pgtable-arm-v7s: Abort allocation when table address overflows the PTE iommu/io-pgtable-arm: Fix pgtable allocation in selftest iommu/vt-d: Remove the obsolete per iommu pasid tables iommu/vt-d: Apply per pci device pasid table in SVA iommu/vt-d: Allocate and free pasid table iommu/vt-d: Per PCI device pasid table interfaces iommu/vt-d: Add for_each_device_domain() helper iommu/vt-d: Move device_domain_info to header iommu/vt-d: Apply global PASID in SVA ...
Diffstat
-rw-r--r--Documentation/admin-guide/kernel-parameters.txt3
-rw-r--r--Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.txt3
-rw-r--r--arch/x86/include/asm/irq_remapping.h5
-rw-r--r--arch/x86/kernel/pci-dma.c8
-rw-r--r--drivers/iommu/Kconfig37
-rw-r--r--drivers/iommu/Makefile4
-rw-r--r--drivers/iommu/amd_iommu.c38
-rw-r--r--drivers/iommu/amd_iommu_debugfs.c33
-rw-r--r--drivers/iommu/amd_iommu_init.c57
-rw-r--r--drivers/iommu/amd_iommu_proto.h6
-rw-r--r--drivers/iommu/amd_iommu_types.h22
-rw-r--r--drivers/iommu/arm-smmu-v3.c26
-rw-r--r--drivers/iommu/arm-smmu.c17
-rw-r--r--drivers/iommu/dmar.c6
-rw-r--r--drivers/iommu/exynos-iommu.c1
-rw-r--r--drivers/iommu/intel-iommu.c160
-rw-r--r--drivers/iommu/intel-pasid.c239
-rw-r--r--drivers/iommu/intel-pasid.h39
-rw-r--r--drivers/iommu/intel-svm.c79
-rw-r--r--drivers/iommu/io-pgtable-arm-v7s.c7
-rw-r--r--drivers/iommu/io-pgtable-arm.c3
-rw-r--r--drivers/iommu/iommu-debugfs.c66
-rw-r--r--drivers/iommu/iommu.c44
-rw-r--r--drivers/iommu/ipmmu-vmsa.c59
-rw-r--r--drivers/iommu/msm_iommu.c17
-rw-r--r--drivers/iommu/mtk_iommu.c1
-rw-r--r--drivers/iommu/mtk_iommu_v1.c1
-rw-r--r--drivers/iommu/omap-iommu.c5
-rw-r--r--drivers/iommu/qcom_iommu.c1
-rw-r--r--drivers/iommu/rockchip-iommu.c1
-rw-r--r--drivers/iommu/tegra-gart.c1
-rw-r--r--drivers/iommu/tegra-smmu.c1
-rw-r--r--include/linux/dmar.h5
-rw-r--r--include/linux/intel-iommu.h80
-rw-r--r--include/linux/iommu.h23
-rw-r--r--include/linux/platform_data/sh_ipmmu.h18
36 files changed, 839 insertions, 277 deletions
diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index 970d837bd57f..9871e649ffef 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -1744,7 +1744,8 @@
1744 merge 1744 merge
1745 nomerge 1745 nomerge
1746 soft 1746 soft
1747 pt [x86, IA-64] 1747 pt [x86]
1748 nopt [x86]
1748 nobypass [PPC/POWERNV] 1749 nobypass [PPC/POWERNV]
1749 Disable IOMMU bypass, using IOMMU for PCI devices. 1750 Disable IOMMU bypass, using IOMMU for PCI devices.
1750 1751
diff --git a/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.txt b/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.txt
index ffadb7c6f1f3..c6e2d855fe13 100644
--- a/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.txt
+++ b/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.txt
@@ -19,7 +19,10 @@ Required Properties:
19 - "renesas,ipmmu-r8a7794" for the R8A7794 (R-Car E2) IPMMU. 19 - "renesas,ipmmu-r8a7794" for the R8A7794 (R-Car E2) IPMMU.
20 - "renesas,ipmmu-r8a7795" for the R8A7795 (R-Car H3) IPMMU. 20 - "renesas,ipmmu-r8a7795" for the R8A7795 (R-Car H3) IPMMU.
21 - "renesas,ipmmu-r8a7796" for the R8A7796 (R-Car M3-W) IPMMU. 21 - "renesas,ipmmu-r8a7796" for the R8A7796 (R-Car M3-W) IPMMU.
22 - "renesas,ipmmu-r8a77965" for the R8A77965 (R-Car M3-N) IPMMU.
22 - "renesas,ipmmu-r8a77970" for the R8A77970 (R-Car V3M) IPMMU. 23 - "renesas,ipmmu-r8a77970" for the R8A77970 (R-Car V3M) IPMMU.
24 - "renesas,ipmmu-r8a77980" for the R8A77980 (R-Car V3H) IPMMU.
25 - "renesas,ipmmu-r8a77990" for the R8A77990 (R-Car E3) IPMMU.
23 - "renesas,ipmmu-r8a77995" for the R8A77995 (R-Car D3) IPMMU. 26 - "renesas,ipmmu-r8a77995" for the R8A77995 (R-Car D3) IPMMU.
24 - "renesas,ipmmu-vmsa" for generic R-Car Gen2 or RZ/G1 VMSA-compatible 27 - "renesas,ipmmu-vmsa" for generic R-Car Gen2 or RZ/G1 VMSA-compatible
25 IPMMU. 28 IPMMU.
diff --git a/arch/x86/include/asm/irq_remapping.h b/arch/x86/include/asm/irq_remapping.h
index 023b4a9fc846..5f26962eff42 100644
--- a/arch/x86/include/asm/irq_remapping.h
+++ b/arch/x86/include/asm/irq_remapping.h
@@ -33,6 +33,11 @@ enum irq_remap_cap {
33 IRQ_POSTING_CAP = 0, 33 IRQ_POSTING_CAP = 0,
34}; 34};
35 35
36enum {
37 IRQ_REMAP_XAPIC_MODE,
38 IRQ_REMAP_X2APIC_MODE,
39};
40
36struct vcpu_data { 41struct vcpu_data {
37 u64 pi_desc_addr; /* Physical address of PI Descriptor */ 42 u64 pi_desc_addr; /* Physical address of PI Descriptor */
38 u32 vector; /* Guest vector of the interrupt */ 43 u32 vector; /* Guest vector of the interrupt */
diff --git a/arch/x86/kernel/pci-dma.c b/arch/x86/kernel/pci-dma.c
index acfd04121da3..7ba73fe0d917 100644
--- a/arch/x86/kernel/pci-dma.c
+++ b/arch/x86/kernel/pci-dma.c
@@ -40,8 +40,14 @@ int iommu_detected __read_mostly = 0;
40 * devices and allow every device to access to whole physical memory. This is 40 * devices and allow every device to access to whole physical memory. This is
41 * useful if a user wants to use an IOMMU only for KVM device assignment to 41 * useful if a user wants to use an IOMMU only for KVM device assignment to
42 * guests and not for driver dma translation. 42 * guests and not for driver dma translation.
43 * It is also possible to disable by default in kernel config, and enable with
44 * iommu=nopt at boot time.
43 */ 45 */
46#ifdef CONFIG_IOMMU_DEFAULT_PASSTHROUGH
47int iommu_pass_through __read_mostly = 1;
48#else
44int iommu_pass_through __read_mostly; 49int iommu_pass_through __read_mostly;
50#endif
45 51
46extern struct iommu_table_entry __iommu_table[], __iommu_table_end[]; 52extern struct iommu_table_entry __iommu_table[], __iommu_table_end[];
47 53
@@ -135,6 +141,8 @@ static __init int iommu_setup(char *p)
135#endif 141#endif
136 if (!strncmp(p, "pt", 2)) 142 if (!strncmp(p, "pt", 2))
137 iommu_pass_through = 1; 143 iommu_pass_through = 1;
144 if (!strncmp(p, "nopt", 4))
145 iommu_pass_through = 0;
138 146
139 gart_parse_options(p); 147 gart_parse_options(p);
140 148
diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig
index 689ffe538370..c60395b7470f 100644
--- a/drivers/iommu/Kconfig
+++ b/drivers/iommu/Kconfig
@@ -60,6 +60,27 @@ config IOMMU_IO_PGTABLE_ARMV7S_SELFTEST
60 60
61endmenu 61endmenu
62 62
63config IOMMU_DEBUGFS
64 bool "Export IOMMU internals in DebugFS"
65 depends on DEBUG_FS
66 help
67 Allows exposure of IOMMU device internals. This option enables
68 the use of debugfs by IOMMU drivers as required. Devices can,
69 at initialization time, cause the IOMMU code to create a top-level
70 debug/iommu directory, and then populate a subdirectory with
71 entries as required.
72
73config IOMMU_DEFAULT_PASSTHROUGH
74 bool "IOMMU passthrough by default"
75 depends on IOMMU_API
76 help
77 Enable passthrough by default, removing the need to pass in
78 iommu.passthrough=on or iommu=pt through command line. If this
79 is enabled, you can still disable with iommu.passthrough=off
80 or iommu=nopt depending on the architecture.
81
82 If unsure, say N here.
83
63config IOMMU_IOVA 84config IOMMU_IOVA
64 tristate 85 tristate
65 86
@@ -135,6 +156,18 @@ config AMD_IOMMU_V2
135 hardware. Select this option if you want to use devices that support 156 hardware. Select this option if you want to use devices that support
136 the PCI PRI and PASID interface. 157 the PCI PRI and PASID interface.
137 158
159config AMD_IOMMU_DEBUGFS
160 bool "Enable AMD IOMMU internals in DebugFS"
161 depends on AMD_IOMMU && IOMMU_DEBUGFS
162 ---help---
163 !!!WARNING!!! !!!WARNING!!! !!!WARNING!!! !!!WARNING!!!
164
165 DO NOT ENABLE THIS OPTION UNLESS YOU REALLY, -REALLY- KNOW WHAT YOU ARE DOING!!!
166 Exposes AMD IOMMU device internals in DebugFS.
167
168 This option is -NOT- intended for production environments, and should
169 not generally be enabled.
170
138# Intel IOMMU support 171# Intel IOMMU support
139config DMAR_TABLE 172config DMAR_TABLE
140 bool 173 bool
@@ -284,8 +317,8 @@ config IPMMU_VMSA
284 select IOMMU_IO_PGTABLE_LPAE 317 select IOMMU_IO_PGTABLE_LPAE
285 select ARM_DMA_USE_IOMMU 318 select ARM_DMA_USE_IOMMU
286 help 319 help
287 Support for the Renesas VMSA-compatible IPMMU Renesas found in the 320 Support for the Renesas VMSA-compatible IPMMU found in the R-Mobile
288 R-Mobile APE6 and R-Car H2/M2 SoCs. 321 APE6, R-Car Gen2, and R-Car Gen3 SoCs.
289 322
290 If unsure, say N. 323 If unsure, say N.
291 324
diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile
index 1fb695854809..ab5eba6edf82 100644
--- a/drivers/iommu/Makefile
+++ b/drivers/iommu/Makefile
@@ -2,6 +2,7 @@
2obj-$(CONFIG_IOMMU_API) += iommu.o 2obj-$(CONFIG_IOMMU_API) += iommu.o
3obj-$(CONFIG_IOMMU_API) += iommu-traces.o 3obj-$(CONFIG_IOMMU_API) += iommu-traces.o
4obj-$(CONFIG_IOMMU_API) += iommu-sysfs.o 4obj-$(CONFIG_IOMMU_API) += iommu-sysfs.o
5obj-$(CONFIG_IOMMU_DEBUGFS) += iommu-debugfs.o
5obj-$(CONFIG_IOMMU_DMA) += dma-iommu.o 6obj-$(CONFIG_IOMMU_DMA) += dma-iommu.o
6obj-$(CONFIG_IOMMU_IO_PGTABLE) += io-pgtable.o 7obj-$(CONFIG_IOMMU_IO_PGTABLE) += io-pgtable.o
7obj-$(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) += io-pgtable-arm-v7s.o 8obj-$(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) += io-pgtable-arm-v7s.o
@@ -10,11 +11,12 @@ obj-$(CONFIG_IOMMU_IOVA) += iova.o
10obj-$(CONFIG_OF_IOMMU) += of_iommu.o 11obj-$(CONFIG_OF_IOMMU) += of_iommu.o
11obj-$(CONFIG_MSM_IOMMU) += msm_iommu.o 12obj-$(CONFIG_MSM_IOMMU) += msm_iommu.o
12obj-$(CONFIG_AMD_IOMMU) += amd_iommu.o amd_iommu_init.o 13obj-$(CONFIG_AMD_IOMMU) += amd_iommu.o amd_iommu_init.o
14obj-$(CONFIG_AMD_IOMMU_DEBUGFS) += amd_iommu_debugfs.o
13obj-$(CONFIG_AMD_IOMMU_V2) += amd_iommu_v2.o 15obj-$(CONFIG_AMD_IOMMU_V2) += amd_iommu_v2.o
14obj-$(CONFIG_ARM_SMMU) += arm-smmu.o 16obj-$(CONFIG_ARM_SMMU) += arm-smmu.o
15obj-$(CONFIG_ARM_SMMU_V3) += arm-smmu-v3.o 17obj-$(CONFIG_ARM_SMMU_V3) += arm-smmu-v3.o
16obj-$(CONFIG_DMAR_TABLE) += dmar.o 18obj-$(CONFIG_DMAR_TABLE) += dmar.o
17obj-$(CONFIG_INTEL_IOMMU) += intel-iommu.o 19obj-$(CONFIG_INTEL_IOMMU) += intel-iommu.o intel-pasid.o
18obj-$(CONFIG_INTEL_IOMMU_SVM) += intel-svm.o 20obj-$(CONFIG_INTEL_IOMMU_SVM) += intel-svm.o
19obj-$(CONFIG_IPMMU_VMSA) += ipmmu-vmsa.o 21obj-$(CONFIG_IPMMU_VMSA) += ipmmu-vmsa.o
20obj-$(CONFIG_IRQ_REMAP) += intel_irq_remapping.o irq_remapping.o 22obj-$(CONFIG_IRQ_REMAP) += intel_irq_remapping.o irq_remapping.o
diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c
index 60b2eab29cd8..4e04fff23977 100644
--- a/drivers/iommu/amd_iommu.c
+++ b/drivers/iommu/amd_iommu.c
@@ -1404,6 +1404,8 @@ static u64 *fetch_pte(struct protection_domain *domain,
1404 int level; 1404 int level;
1405 u64 *pte; 1405 u64 *pte;
1406 1406
1407 *page_size = 0;
1408
1407 if (address > PM_LEVEL_SIZE(domain->mode)) 1409 if (address > PM_LEVEL_SIZE(domain->mode))
1408 return NULL; 1410 return NULL;
1409 1411
@@ -1944,12 +1946,6 @@ static int __attach_device(struct iommu_dev_data *dev_data,
1944{ 1946{
1945 int ret; 1947 int ret;
1946 1948
1947 /*
1948 * Must be called with IRQs disabled. Warn here to detect early
1949 * when its not.
1950 */
1951 WARN_ON(!irqs_disabled());
1952
1953 /* lock domain */ 1949 /* lock domain */
1954 spin_lock(&domain->lock); 1950 spin_lock(&domain->lock);
1955 1951
@@ -2115,12 +2111,6 @@ static void __detach_device(struct iommu_dev_data *dev_data)
2115{ 2111{
2116 struct protection_domain *domain; 2112 struct protection_domain *domain;
2117 2113
2118 /*
2119 * Must be called with IRQs disabled. Warn here to detect early
2120 * when its not.
2121 */
2122 WARN_ON(!irqs_disabled());
2123
2124 domain = dev_data->domain; 2114 domain = dev_data->domain;
2125 2115
2126 spin_lock(&domain->lock); 2116 spin_lock(&domain->lock);
@@ -2405,9 +2395,9 @@ static void __unmap_single(struct dma_ops_domain *dma_dom,
2405 } 2395 }
2406 2396
2407 if (amd_iommu_unmap_flush) { 2397 if (amd_iommu_unmap_flush) {
2408 dma_ops_free_iova(dma_dom, dma_addr, pages);
2409 domain_flush_tlb(&dma_dom->domain); 2398 domain_flush_tlb(&dma_dom->domain);
2410 domain_flush_complete(&dma_dom->domain); 2399 domain_flush_complete(&dma_dom->domain);
2400 dma_ops_free_iova(dma_dom, dma_addr, pages);
2411 } else { 2401 } else {
2412 pages = __roundup_pow_of_two(pages); 2402 pages = __roundup_pow_of_two(pages);
2413 queue_iova(&dma_dom->iovad, dma_addr >> PAGE_SHIFT, pages, 0); 2403 queue_iova(&dma_dom->iovad, dma_addr >> PAGE_SHIFT, pages, 0);
@@ -3192,7 +3182,6 @@ const struct iommu_ops amd_iommu_ops = {
3192 .detach_dev = amd_iommu_detach_device, 3182 .detach_dev = amd_iommu_detach_device,
3193 .map = amd_iommu_map, 3183 .map = amd_iommu_map,
3194 .unmap = amd_iommu_unmap, 3184 .unmap = amd_iommu_unmap,
3195 .map_sg = default_iommu_map_sg,
3196 .iova_to_phys = amd_iommu_iova_to_phys, 3185 .iova_to_phys = amd_iommu_iova_to_phys,
3197 .add_device = amd_iommu_add_device, 3186 .add_device = amd_iommu_add_device,
3198 .remove_device = amd_iommu_remove_device, 3187 .remove_device = amd_iommu_remove_device,
@@ -3874,7 +3863,8 @@ static void irte_ga_prepare(void *entry,
3874 irte->lo.fields_remap.int_type = delivery_mode; 3863 irte->lo.fields_remap.int_type = delivery_mode;
3875 irte->lo.fields_remap.dm = dest_mode; 3864 irte->lo.fields_remap.dm = dest_mode;
3876 irte->hi.fields.vector = vector; 3865 irte->hi.fields.vector = vector;
3877 irte->lo.fields_remap.destination = dest_apicid; 3866 irte->lo.fields_remap.destination = APICID_TO_IRTE_DEST_LO(dest_apicid);
3867 irte->hi.fields.destination = APICID_TO_IRTE_DEST_HI(dest_apicid);
3878 irte->lo.fields_remap.valid = 1; 3868 irte->lo.fields_remap.valid = 1;
3879} 3869}
3880 3870
@@ -3927,7 +3917,10 @@ static void irte_ga_set_affinity(void *entry, u16 devid, u16 index,
3927 3917
3928 if (!irte->lo.fields_remap.guest_mode) { 3918 if (!irte->lo.fields_remap.guest_mode) {
3929 irte->hi.fields.vector = vector; 3919 irte->hi.fields.vector = vector;
3930 irte->lo.fields_remap.destination = dest_apicid; 3920 irte->lo.fields_remap.destination =
3921 APICID_TO_IRTE_DEST_LO(dest_apicid);
3922 irte->hi.fields.destination =
3923 APICID_TO_IRTE_DEST_HI(dest_apicid);
3931 modify_irte_ga(devid, index, irte, NULL); 3924 modify_irte_ga(devid, index, irte, NULL);
3932 } 3925 }
3933} 3926}
@@ -4344,7 +4337,10 @@ static int amd_ir_set_vcpu_affinity(struct irq_data *data, void *vcpu_info)
4344 irte->lo.val = 0; 4337 irte->lo.val = 0;
4345 irte->hi.fields.vector = cfg->vector; 4338 irte->hi.fields.vector = cfg->vector;
4346 irte->lo.fields_remap.guest_mode = 0; 4339 irte->lo.fields_remap.guest_mode = 0;
4347 irte->lo.fields_remap.destination = cfg->dest_apicid; 4340 irte->lo.fields_remap.destination =
4341 APICID_TO_IRTE_DEST_LO(cfg->dest_apicid);
4342 irte->hi.fields.destination =
4343 APICID_TO_IRTE_DEST_HI(cfg->dest_apicid);
4348 irte->lo.fields_remap.int_type = apic->irq_delivery_mode; 4344 irte->lo.fields_remap.int_type = apic->irq_delivery_mode;
4349 irte->lo.fields_remap.dm = apic->irq_dest_mode; 4345 irte->lo.fields_remap.dm = apic->irq_dest_mode;
4350 4346
@@ -4461,8 +4457,12 @@ int amd_iommu_update_ga(int cpu, bool is_run, void *data)
4461 raw_spin_lock_irqsave(&table->lock, flags); 4457 raw_spin_lock_irqsave(&table->lock, flags);
4462 4458
4463 if (ref->lo.fields_vapic.guest_mode) { 4459 if (ref->lo.fields_vapic.guest_mode) {
4464 if (cpu >= 0) 4460 if (cpu >= 0) {
4465 ref->lo.fields_vapic.destination = cpu; 4461 ref->lo.fields_vapic.destination =
4462 APICID_TO_IRTE_DEST_LO(cpu);
4463 ref->hi.fields.destination =
4464 APICID_TO_IRTE_DEST_HI(cpu);
4465 }
4466 ref->lo.fields_vapic.is_run = is_run; 4466 ref->lo.fields_vapic.is_run = is_run;
4467 barrier(); 4467 barrier();
4468 } 4468 }
diff --git a/drivers/iommu/amd_iommu_debugfs.c b/drivers/iommu/amd_iommu_debugfs.c
new file mode 100644
index 000000000000..c6a5c737ef09
--- /dev/null
+++ b/drivers/iommu/amd_iommu_debugfs.c
@@ -0,0 +1,33 @@
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * AMD IOMMU driver
4 *
5 * Copyright (C) 2018 Advanced Micro Devices, Inc.
6 *
7 * Author: Gary R Hook <gary.hook@amd.com>
8 */
9
10#include <linux/debugfs.h>
11#include <linux/iommu.h>
12#include <linux/pci.h>
13#include "amd_iommu_proto.h"
14#include "amd_iommu_types.h"
15
16static struct dentry *amd_iommu_debugfs;
17static DEFINE_MUTEX(amd_iommu_debugfs_lock);
18
19#define MAX_NAME_LEN 20
20
21void amd_iommu_debugfs_setup(struct amd_iommu *iommu)
22{
23 char name[MAX_NAME_LEN + 1];
24
25 mutex_lock(&amd_iommu_debugfs_lock);
26 if (!amd_iommu_debugfs)
27 amd_iommu_debugfs = debugfs_create_dir("amd",
28 iommu_debugfs_dir);
29 mutex_unlock(&amd_iommu_debugfs_lock);
30
31 snprintf(name, MAX_NAME_LEN, "iommu%02d", iommu->index);
32 iommu->debugfs = debugfs_create_dir(name, amd_iommu_debugfs);
33}
diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c
index 904c575d1677..84b3e4445d46 100644
--- a/drivers/iommu/amd_iommu_init.c
+++ b/drivers/iommu/amd_iommu_init.c
@@ -153,6 +153,7 @@ bool amd_iommu_dump;
153bool amd_iommu_irq_remap __read_mostly; 153bool amd_iommu_irq_remap __read_mostly;
154 154
155int amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_VAPIC; 155int amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_VAPIC;
156static int amd_iommu_xt_mode = IRQ_REMAP_X2APIC_MODE;
156 157
157static bool amd_iommu_detected; 158static bool amd_iommu_detected;
158static bool __initdata amd_iommu_disabled; 159static bool __initdata amd_iommu_disabled;
@@ -280,9 +281,9 @@ static void clear_translation_pre_enabled(struct amd_iommu *iommu)
280 281
281static void init_translation_status(struct amd_iommu *iommu) 282static void init_translation_status(struct amd_iommu *iommu)
282{ 283{
283 u32 ctrl; 284 u64 ctrl;
284 285
285 ctrl = readl(iommu->mmio_base + MMIO_CONTROL_OFFSET); 286 ctrl = readq(iommu->mmio_base + MMIO_CONTROL_OFFSET);
286 if (ctrl & (1<<CONTROL_IOMMU_EN)) 287 if (ctrl & (1<<CONTROL_IOMMU_EN))
287 iommu->flags |= AMD_IOMMU_FLAG_TRANS_PRE_ENABLED; 288 iommu->flags |= AMD_IOMMU_FLAG_TRANS_PRE_ENABLED;
288} 289}
@@ -386,30 +387,30 @@ static void iommu_set_device_table(struct amd_iommu *iommu)
386/* Generic functions to enable/disable certain features of the IOMMU. */ 387/* Generic functions to enable/disable certain features of the IOMMU. */
387static void iommu_feature_enable(struct amd_iommu *iommu, u8 bit) 388static void iommu_feature_enable(struct amd_iommu *iommu, u8 bit)
388{ 389{
389 u32 ctrl; 390 u64 ctrl;
390 391
391 ctrl = readl(iommu->mmio_base + MMIO_CONTROL_OFFSET); 392 ctrl = readq(iommu->mmio_base + MMIO_CONTROL_OFFSET);
392 ctrl |= (1 << bit); 393 ctrl |= (1ULL << bit);
393 writel(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET); 394 writeq(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET);
394} 395}
395 396
396static void iommu_feature_disable(struct amd_iommu *iommu, u8 bit) 397static void iommu_feature_disable(struct amd_iommu *iommu, u8 bit)
397{ 398{
398 u32 ctrl; 399 u64 ctrl;
399 400
400 ctrl = readl(iommu->mmio_base + MMIO_CONTROL_OFFSET); 401 ctrl = readq(iommu->mmio_base + MMIO_CONTROL_OFFSET);
401 ctrl &= ~(1 << bit); 402 ctrl &= ~(1ULL << bit);
402 writel(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET); 403 writeq(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET);
403} 404}
404 405
405static void iommu_set_inv_tlb_timeout(struct amd_iommu *iommu, int timeout) 406static void iommu_set_inv_tlb_timeout(struct amd_iommu *iommu, int timeout)
406{ 407{
407 u32 ctrl; 408 u64 ctrl;
408 409
409 ctrl = readl(iommu->mmio_base + MMIO_CONTROL_OFFSET); 410 ctrl = readq(iommu->mmio_base + MMIO_CONTROL_OFFSET);
410 ctrl &= ~CTRL_INV_TO_MASK; 411 ctrl &= ~CTRL_INV_TO_MASK;
411 ctrl |= (timeout << CONTROL_INV_TIMEOUT) & CTRL_INV_TO_MASK; 412 ctrl |= (timeout << CONTROL_INV_TIMEOUT) & CTRL_INV_TO_MASK;
412 writel(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET); 413 writeq(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET);
413} 414}
414 415
415/* Function to enable the hardware */ 416/* Function to enable the hardware */
@@ -827,6 +828,19 @@ static int iommu_init_ga(struct amd_iommu *iommu)
827 return ret; 828 return ret;
828} 829}
829 830
831static void iommu_enable_xt(struct amd_iommu *iommu)
832{
833#ifdef CONFIG_IRQ_REMAP
834 /*
835 * XT mode (32-bit APIC destination ID) requires
836 * GA mode (128-bit IRTE support) as a prerequisite.
837 */
838 if (AMD_IOMMU_GUEST_IR_GA(amd_iommu_guest_ir) &&
839 amd_iommu_xt_mode == IRQ_REMAP_X2APIC_MODE)
840 iommu_feature_enable(iommu, CONTROL_XT_EN);
841#endif /* CONFIG_IRQ_REMAP */
842}
843
830static void iommu_enable_gt(struct amd_iommu *iommu) 844static void iommu_enable_gt(struct amd_iommu *iommu)
831{ 845{
832 if (!iommu_feature(iommu, FEATURE_GT)) 846 if (!iommu_feature(iommu, FEATURE_GT))
@@ -1507,6 +1521,8 @@ static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h)
1507 iommu->mmio_phys_end = MMIO_CNTR_CONF_OFFSET; 1521 iommu->mmio_phys_end = MMIO_CNTR_CONF_OFFSET;
1508 if (((h->efr_attr & (0x1 << IOMMU_FEAT_GASUP_SHIFT)) == 0)) 1522 if (((h->efr_attr & (0x1 << IOMMU_FEAT_GASUP_SHIFT)) == 0))
1509 amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY; 1523 amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY;
1524 if (((h->efr_attr & (0x1 << IOMMU_FEAT_XTSUP_SHIFT)) == 0))
1525 amd_iommu_xt_mode = IRQ_REMAP_XAPIC_MODE;
1510 break; 1526 break;
1511 case 0x11: 1527 case 0x11:
1512 case 0x40: 1528 case 0x40:
@@ -1516,6 +1532,8 @@ static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h)
1516 iommu->mmio_phys_end = MMIO_CNTR_CONF_OFFSET; 1532 iommu->mmio_phys_end = MMIO_CNTR_CONF_OFFSET;
1517 if (((h->efr_reg & (0x1 << IOMMU_EFR_GASUP_SHIFT)) == 0)) 1533 if (((h->efr_reg & (0x1 << IOMMU_EFR_GASUP_SHIFT)) == 0))
1518 amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY; 1534 amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY;
1535 if (((h->efr_reg & (0x1 << IOMMU_EFR_XTSUP_SHIFT)) == 0))
1536 amd_iommu_xt_mode = IRQ_REMAP_XAPIC_MODE;
1519 break; 1537 break;
1520 default: 1538 default:
1521 return -EINVAL; 1539 return -EINVAL;
@@ -1832,6 +1850,8 @@ static void print_iommu_info(void)
1832 pr_info("AMD-Vi: Interrupt remapping enabled\n"); 1850 pr_info("AMD-Vi: Interrupt remapping enabled\n");
1833 if (AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)) 1851 if (AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir))
1834 pr_info("AMD-Vi: virtual APIC enabled\n"); 1852 pr_info("AMD-Vi: virtual APIC enabled\n");
1853 if (amd_iommu_xt_mode == IRQ_REMAP_X2APIC_MODE)
1854 pr_info("AMD-Vi: X2APIC enabled\n");
1835 } 1855 }
1836} 1856}
1837 1857
@@ -2168,6 +2188,7 @@ static void early_enable_iommu(struct amd_iommu *iommu)
2168 iommu_enable_event_buffer(iommu); 2188 iommu_enable_event_buffer(iommu);
2169 iommu_set_exclusion_range(iommu); 2189 iommu_set_exclusion_range(iommu);
2170 iommu_enable_ga(iommu); 2190 iommu_enable_ga(iommu);
2191 iommu_enable_xt(iommu);
2171 iommu_enable(iommu); 2192 iommu_enable(iommu);
2172 iommu_flush_all_caches(iommu); 2193 iommu_flush_all_caches(iommu);
2173} 2194}
@@ -2212,6 +2233,7 @@ static void early_enable_iommus(void)
2212 iommu_enable_command_buffer(iommu); 2233 iommu_enable_command_buffer(iommu);
2213 iommu_enable_event_buffer(iommu); 2234 iommu_enable_event_buffer(iommu);
2214 iommu_enable_ga(iommu); 2235 iommu_enable_ga(iommu);
2236 iommu_enable_xt(iommu);
2215 iommu_set_device_table(iommu); 2237 iommu_set_device_table(iommu);
2216 iommu_flush_all_caches(iommu); 2238 iommu_flush_all_caches(iommu);
2217 } 2239 }
@@ -2691,8 +2713,7 @@ int __init amd_iommu_enable(void)
2691 return ret; 2713 return ret;
2692 2714
2693 irq_remapping_enabled = 1; 2715 irq_remapping_enabled = 1;
2694 2716 return amd_iommu_xt_mode;
2695 return 0;
2696} 2717}
2697 2718
2698void amd_iommu_disable(void) 2719void amd_iommu_disable(void)
@@ -2721,6 +2742,7 @@ int __init amd_iommu_enable_faulting(void)
2721 */ 2742 */
2722static int __init amd_iommu_init(void) 2743static int __init amd_iommu_init(void)
2723{ 2744{
2745 struct amd_iommu *iommu;
2724 int ret; 2746 int ret;
2725 2747
2726 ret = iommu_go_to_state(IOMMU_INITIALIZED); 2748 ret = iommu_go_to_state(IOMMU_INITIALIZED);
@@ -2730,14 +2752,15 @@ static int __init amd_iommu_init(void)
2730 disable_iommus(); 2752 disable_iommus();
2731 free_iommu_resources(); 2753 free_iommu_resources();
2732 } else { 2754 } else {
2733 struct amd_iommu *iommu;
2734
2735 uninit_device_table_dma(); 2755 uninit_device_table_dma();
2736 for_each_iommu(iommu) 2756 for_each_iommu(iommu)
2737 iommu_flush_all_caches(iommu); 2757 iommu_flush_all_caches(iommu);
2738 } 2758 }
2739 } 2759 }
2740 2760
2761 for_each_iommu(iommu)
2762 amd_iommu_debugfs_setup(iommu);
2763
2741 return ret; 2764 return ret;
2742} 2765}
2743 2766
diff --git a/drivers/iommu/amd_iommu_proto.h b/drivers/iommu/amd_iommu_proto.h
index 640c286a0ab9..a8cd0296fb16 100644
--- a/drivers/iommu/amd_iommu_proto.h
+++ b/drivers/iommu/amd_iommu_proto.h
@@ -33,6 +33,12 @@ extern void amd_iommu_uninit_devices(void);
33extern void amd_iommu_init_notifier(void); 33extern void amd_iommu_init_notifier(void);
34extern int amd_iommu_init_api(void); 34extern int amd_iommu_init_api(void);
35 35
36#ifdef CONFIG_AMD_IOMMU_DEBUGFS
37void amd_iommu_debugfs_setup(struct amd_iommu *iommu);
38#else
39static inline void amd_iommu_debugfs_setup(struct amd_iommu *iommu) {}
40#endif
41
36/* Needed for interrupt remapping */ 42/* Needed for interrupt remapping */
37extern int amd_iommu_prepare(void); 43extern int amd_iommu_prepare(void);
38extern int amd_iommu_enable(void); 44extern int amd_iommu_enable(void);
diff --git a/drivers/iommu/amd_iommu_types.h b/drivers/iommu/amd_iommu_types.h
index 986cbe0cc189..e2b342e65a7b 100644
--- a/drivers/iommu/amd_iommu_types.h
+++ b/drivers/iommu/amd_iommu_types.h
@@ -161,6 +161,7 @@
161#define CONTROL_GAM_EN 0x19ULL 161#define CONTROL_GAM_EN 0x19ULL
162#define CONTROL_GALOG_EN 0x1CULL 162#define CONTROL_GALOG_EN 0x1CULL
163#define CONTROL_GAINT_EN 0x1DULL 163#define CONTROL_GAINT_EN 0x1DULL
164#define CONTROL_XT_EN 0x32ULL
164 165
165#define CTRL_INV_TO_MASK (7 << CONTROL_INV_TIMEOUT) 166#define CTRL_INV_TO_MASK (7 << CONTROL_INV_TIMEOUT)
166#define CTRL_INV_TO_NONE 0 167#define CTRL_INV_TO_NONE 0
@@ -378,9 +379,11 @@
378#define IOMMU_CAP_EFR 27 379#define IOMMU_CAP_EFR 27
379 380
380/* IOMMU Feature Reporting Field (for IVHD type 10h */ 381/* IOMMU Feature Reporting Field (for IVHD type 10h */
382#define IOMMU_FEAT_XTSUP_SHIFT 0
381#define IOMMU_FEAT_GASUP_SHIFT 6 383#define IOMMU_FEAT_GASUP_SHIFT 6
382 384
383/* IOMMU Extended Feature Register (EFR) */ 385/* IOMMU Extended Feature Register (EFR) */
386#define IOMMU_EFR_XTSUP_SHIFT 2
384#define IOMMU_EFR_GASUP_SHIFT 7 387#define IOMMU_EFR_GASUP_SHIFT 7
385 388
386#define MAX_DOMAIN_ID 65536 389#define MAX_DOMAIN_ID 65536
@@ -437,7 +440,6 @@ extern struct kmem_cache *amd_iommu_irq_cache;
437#define APERTURE_RANGE_INDEX(a) ((a) >> APERTURE_RANGE_SHIFT) 440#define APERTURE_RANGE_INDEX(a) ((a) >> APERTURE_RANGE_SHIFT)
438#define APERTURE_PAGE_INDEX(a) (((a) >> 21) & 0x3fULL) 441#define APERTURE_PAGE_INDEX(a) (((a) >> 21) & 0x3fULL)
439 442
440
441/* 443/*
442 * This struct is used to pass information about 444 * This struct is used to pass information about
443 * incoming PPR faults around. 445 * incoming PPR faults around.
@@ -594,6 +596,11 @@ struct amd_iommu {
594 596
595 u32 flags; 597 u32 flags;
596 volatile u64 __aligned(8) cmd_sem; 598 volatile u64 __aligned(8) cmd_sem;
599
600#ifdef CONFIG_AMD_IOMMU_DEBUGFS
601 /* DebugFS Info */
602 struct dentry *debugfs;
603#endif
597}; 604};
598 605
599static inline struct amd_iommu *dev_to_amd_iommu(struct device *dev) 606static inline struct amd_iommu *dev_to_amd_iommu(struct device *dev)
@@ -810,6 +817,9 @@ union irte {
810 } fields; 817 } fields;
811}; 818};
812 819
820#define APICID_TO_IRTE_DEST_LO(x) (x & 0xffffff)
821#define APICID_TO_IRTE_DEST_HI(x) ((x >> 24) & 0xff)
822
813union irte_ga_lo { 823union irte_ga_lo {
814 u64 val; 824 u64 val;
815 825
@@ -823,8 +833,8 @@ union irte_ga_lo {
823 dm : 1, 833 dm : 1,
824 /* ------ */ 834 /* ------ */
825 guest_mode : 1, 835 guest_mode : 1,
826 destination : 8, 836 destination : 24,
827 rsvd : 48; 837 ga_tag : 32;
828 } fields_remap; 838 } fields_remap;
829 839
830 /* For guest vAPIC */ 840 /* For guest vAPIC */
@@ -837,8 +847,7 @@ union irte_ga_lo {
837 is_run : 1, 847 is_run : 1,
838 /* ------ */ 848 /* ------ */
839 guest_mode : 1, 849 guest_mode : 1,
840 destination : 8, 850 destination : 24,
841 rsvd2 : 16,
842 ga_tag : 32; 851 ga_tag : 32;
843 } fields_vapic; 852 } fields_vapic;
844}; 853};
@@ -849,7 +858,8 @@ union irte_ga_hi {
849 u64 vector : 8, 858 u64 vector : 8,
850 rsvd_1 : 4, 859 rsvd_1 : 4,
851 ga_root_ptr : 40, 860 ga_root_ptr : 40,
852 rsvd_2 : 12; 861 rsvd_2 : 4,
862 destination : 8;
853 } fields; 863 } fields;
854}; 864};
855 865
diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c
index 22bdabd3d8e0..5059d09f3202 100644
--- a/drivers/iommu/arm-smmu-v3.c
+++ b/drivers/iommu/arm-smmu-v3.c
@@ -24,6 +24,7 @@
24#include <linux/acpi_iort.h> 24#include <linux/acpi_iort.h>
25#include <linux/bitfield.h> 25#include <linux/bitfield.h>
26#include <linux/bitops.h> 26#include <linux/bitops.h>
27#include <linux/crash_dump.h>
27#include <linux/delay.h> 28#include <linux/delay.h>
28#include <linux/dma-iommu.h> 29#include <linux/dma-iommu.h>
29#include <linux/err.h> 30#include <linux/err.h>
@@ -366,7 +367,7 @@
366#define MSI_IOVA_BASE 0x8000000 367#define MSI_IOVA_BASE 0x8000000
367#define MSI_IOVA_LENGTH 0x100000 368#define MSI_IOVA_LENGTH 0x100000
368 369
369static bool disable_bypass; 370static bool disable_bypass = 1;
370module_param_named(disable_bypass, disable_bypass, bool, S_IRUGO); 371module_param_named(disable_bypass, disable_bypass, bool, S_IRUGO);
371MODULE_PARM_DESC(disable_bypass, 372MODULE_PARM_DESC(disable_bypass,
372 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU."); 373 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
@@ -1301,6 +1302,7 @@ static irqreturn_t arm_smmu_priq_thread(int irq, void *dev)
1301 1302
1302 /* Sync our overflow flag, as we believe we're up to speed */ 1303 /* Sync our overflow flag, as we believe we're up to speed */
1303 q->cons = Q_OVF(q, q->prod) | Q_WRP(q, q->cons) | Q_IDX(q, q->cons); 1304 q->cons = Q_OVF(q, q->prod) | Q_WRP(q, q->cons) | Q_IDX(q, q->cons);
1305 writel(q->cons, q->cons_reg);
1304 return IRQ_HANDLED; 1306 return IRQ_HANDLED;
1305} 1307}
1306 1308
@@ -1997,7 +1999,6 @@ static struct iommu_ops arm_smmu_ops = {
1997 .attach_dev = arm_smmu_attach_dev, 1999 .attach_dev = arm_smmu_attach_dev,
1998 .map = arm_smmu_map, 2000 .map = arm_smmu_map,
1999 .unmap = arm_smmu_unmap, 2001 .unmap = arm_smmu_unmap,
2000 .map_sg = default_iommu_map_sg,
2001 .flush_iotlb_all = arm_smmu_iotlb_sync, 2002 .flush_iotlb_all = arm_smmu_iotlb_sync,
2002 .iotlb_sync = arm_smmu_iotlb_sync, 2003 .iotlb_sync = arm_smmu_iotlb_sync,
2003 .iova_to_phys = arm_smmu_iova_to_phys, 2004 .iova_to_phys = arm_smmu_iova_to_phys,
@@ -2211,8 +2212,12 @@ static int arm_smmu_update_gbpa(struct arm_smmu_device *smmu, u32 set, u32 clr)
2211 reg &= ~clr; 2212 reg &= ~clr;
2212 reg |= set; 2213 reg |= set;
2213 writel_relaxed(reg | GBPA_UPDATE, gbpa); 2214 writel_relaxed(reg | GBPA_UPDATE, gbpa);
2214 return readl_relaxed_poll_timeout(gbpa, reg, !(reg & GBPA_UPDATE), 2215 ret = readl_relaxed_poll_timeout(gbpa, reg, !(reg & GBPA_UPDATE),
2215 1, ARM_SMMU_POLL_TIMEOUT_US); 2216 1, ARM_SMMU_POLL_TIMEOUT_US);
2217
2218 if (ret)
2219 dev_err(smmu->dev, "GBPA not responding to update\n");
2220 return ret;
2216} 2221}
2217 2222
2218static void arm_smmu_free_msis(void *data) 2223static void arm_smmu_free_msis(void *data)
@@ -2392,8 +2397,15 @@ static int arm_smmu_device_reset(struct arm_smmu_device *smmu, bool bypass)
2392 2397
2393 /* Clear CR0 and sync (disables SMMU and queue processing) */ 2398 /* Clear CR0 and sync (disables SMMU and queue processing) */
2394 reg = readl_relaxed(smmu->base + ARM_SMMU_CR0); 2399 reg = readl_relaxed(smmu->base + ARM_SMMU_CR0);
2395 if (reg & CR0_SMMUEN) 2400 if (reg & CR0_SMMUEN) {
2401 if (is_kdump_kernel()) {
2402 arm_smmu_update_gbpa(smmu, GBPA_ABORT, 0);
2403 arm_smmu_device_disable(smmu);
2404 return -EBUSY;
2405 }
2406
2396 dev_warn(smmu->dev, "SMMU currently enabled! Resetting...\n"); 2407 dev_warn(smmu->dev, "SMMU currently enabled! Resetting...\n");
2408 }
2397 2409
2398 ret = arm_smmu_device_disable(smmu); 2410 ret = arm_smmu_device_disable(smmu);
2399 if (ret) 2411 if (ret)
@@ -2491,10 +2503,8 @@ static int arm_smmu_device_reset(struct arm_smmu_device *smmu, bool bypass)
2491 enables |= CR0_SMMUEN; 2503 enables |= CR0_SMMUEN;
2492 } else { 2504 } else {
2493 ret = arm_smmu_update_gbpa(smmu, 0, GBPA_ABORT); 2505 ret = arm_smmu_update_gbpa(smmu, 0, GBPA_ABORT);
2494 if (ret) { 2506 if (ret)
2495 dev_err(smmu->dev, "GBPA not responding to update\n");
2496 return ret; 2507 return ret;
2497 }
2498 } 2508 }
2499 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0, 2509 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
2500 ARM_SMMU_CR0ACK); 2510 ARM_SMMU_CR0ACK);
diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c
index c73cfce1ccc0..fd1b80ef9490 100644
--- a/drivers/iommu/arm-smmu.c
+++ b/drivers/iommu/arm-smmu.c
@@ -1562,7 +1562,6 @@ static struct iommu_ops arm_smmu_ops = {
1562 .attach_dev = arm_smmu_attach_dev, 1562 .attach_dev = arm_smmu_attach_dev,
1563 .map = arm_smmu_map, 1563 .map = arm_smmu_map,
1564 .unmap = arm_smmu_unmap, 1564 .unmap = arm_smmu_unmap,
1565 .map_sg = default_iommu_map_sg,
1566 .flush_iotlb_all = arm_smmu_iotlb_sync, 1565 .flush_iotlb_all = arm_smmu_iotlb_sync,
1567 .iotlb_sync = arm_smmu_iotlb_sync, 1566 .iotlb_sync = arm_smmu_iotlb_sync,
1568 .iova_to_phys = arm_smmu_iova_to_phys, 1567 .iova_to_phys = arm_smmu_iova_to_phys,
@@ -2103,12 +2102,16 @@ static int arm_smmu_device_probe(struct platform_device *pdev)
2103 if (err) 2102 if (err)
2104 return err; 2103 return err;
2105 2104
2106 if (smmu->version == ARM_SMMU_V2 && 2105 if (smmu->version == ARM_SMMU_V2) {
2107 smmu->num_context_banks != smmu->num_context_irqs) { 2106 if (smmu->num_context_banks > smmu->num_context_irqs) {
2108 dev_err(dev, 2107 dev_err(dev,
2109 "found only %d context interrupt(s) but %d required\n", 2108 "found only %d context irq(s) but %d required\n",
2110 smmu->num_context_irqs, smmu->num_context_banks); 2109 smmu->num_context_irqs, smmu->num_context_banks);
2111 return -ENODEV; 2110 return -ENODEV;
2111 }
2112
2113 /* Ignore superfluous interrupts */
2114 smmu->num_context_irqs = smmu->num_context_banks;
2112 } 2115 }
2113 2116
2114 for (i = 0; i < smmu->num_global_irqs; ++i) { 2117 for (i = 0; i < smmu->num_global_irqs; ++i) {
diff --git a/drivers/iommu/dmar.c b/drivers/iommu/dmar.c
index 75456b5aa825..d9c748b6f9e4 100644
--- a/drivers/iommu/dmar.c
+++ b/drivers/iommu/dmar.c
@@ -1339,8 +1339,8 @@ void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
1339 qi_submit_sync(&desc, iommu); 1339 qi_submit_sync(&desc, iommu);
1340} 1340}
1341 1341
1342void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 qdep, 1342void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid,
1343 u64 addr, unsigned mask) 1343 u16 qdep, u64 addr, unsigned mask)
1344{ 1344{
1345 struct qi_desc desc; 1345 struct qi_desc desc;
1346 1346
@@ -1355,7 +1355,7 @@ void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 qdep,
1355 qdep = 0; 1355 qdep = 0;
1356 1356
1357 desc.low = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) | 1357 desc.low = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
1358 QI_DIOTLB_TYPE; 1358 QI_DIOTLB_TYPE | QI_DEV_IOTLB_PFSID(pfsid);
1359 1359
1360 qi_submit_sync(&desc, iommu); 1360 qi_submit_sync(&desc, iommu);
1361} 1361}
diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c
index b128cb4372d3..1bd0cd7168df 100644
--- a/drivers/iommu/exynos-iommu.c
+++ b/drivers/iommu/exynos-iommu.c
@@ -1332,7 +1332,6 @@ static const struct iommu_ops exynos_iommu_ops = {
1332 .detach_dev = exynos_iommu_detach_device, 1332 .detach_dev = exynos_iommu_detach_device,
1333 .map = exynos_iommu_map, 1333 .map = exynos_iommu_map,
1334 .unmap = exynos_iommu_unmap, 1334 .unmap = exynos_iommu_unmap,
1335 .map_sg = default_iommu_map_sg,
1336 .iova_to_phys = exynos_iommu_iova_to_phys, 1335 .iova_to_phys = exynos_iommu_iova_to_phys,
1337 .device_group = generic_device_group, 1336 .device_group = generic_device_group,
1338 .add_device = exynos_iommu_add_device, 1337 .add_device = exynos_iommu_add_device,
diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c
index 6a237d18fabf..5f3f10cf9d9d 100644
--- a/drivers/iommu/intel-iommu.c
+++ b/drivers/iommu/intel-iommu.c
@@ -52,6 +52,7 @@
52#include <asm/iommu.h> 52#include <asm/iommu.h>
53 53
54#include "irq_remapping.h" 54#include "irq_remapping.h"
55#include "intel-pasid.h"
55 56
56#define ROOT_SIZE VTD_PAGE_SIZE 57#define ROOT_SIZE VTD_PAGE_SIZE
57#define CONTEXT_SIZE VTD_PAGE_SIZE 58#define CONTEXT_SIZE VTD_PAGE_SIZE
@@ -379,60 +380,6 @@ static int hw_pass_through = 1;
379 for (idx = 0; idx < g_num_of_iommus; idx++) \ 380 for (idx = 0; idx < g_num_of_iommus; idx++) \
380 if (domain->iommu_refcnt[idx]) 381 if (domain->iommu_refcnt[idx])
381 382
382struct dmar_domain {
383 int nid; /* node id */
384
385 unsigned iommu_refcnt[DMAR_UNITS_SUPPORTED];
386 /* Refcount of devices per iommu */
387
388
389 u16 iommu_did[DMAR_UNITS_SUPPORTED];
390 /* Domain ids per IOMMU. Use u16 since
391 * domain ids are 16 bit wide according
392 * to VT-d spec, section 9.3 */
393
394 bool has_iotlb_device;
395 struct list_head devices; /* all devices' list */
396 struct iova_domain iovad; /* iova's that belong to this domain */
397
398 struct dma_pte *pgd; /* virtual address */
399 int gaw; /* max guest address width */
400
401 /* adjusted guest address width, 0 is level 2 30-bit */
402 int agaw;
403
404 int flags; /* flags to find out type of domain */
405
406 int iommu_coherency;/* indicate coherency of iommu access */
407 int iommu_snooping; /* indicate snooping control feature*/
408 int iommu_count; /* reference count of iommu */
409 int iommu_superpage;/* Level of superpages supported:
410 0 == 4KiB (no superpages), 1 == 2MiB,
411 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
412 u64 max_addr; /* maximum mapped address */
413
414 struct iommu_domain domain; /* generic domain data structure for
415 iommu core */
416};
417
418/* PCI domain-device relationship */
419struct device_domain_info {
420 struct list_head link; /* link to domain siblings */
421 struct list_head global; /* link to global list */
422 u8 bus; /* PCI bus number */
423 u8 devfn; /* PCI devfn number */
424 u8 pasid_supported:3;
425 u8 pasid_enabled:1;
426 u8 pri_supported:1;
427 u8 pri_enabled:1;
428 u8 ats_supported:1;
429 u8 ats_enabled:1;
430 u8 ats_qdep;
431 struct device *dev; /* it's NULL for PCIe-to-PCI bridge */
432 struct intel_iommu *iommu; /* IOMMU used by this device */
433 struct dmar_domain *domain; /* pointer to domain */
434};
435
436struct dmar_rmrr_unit { 383struct dmar_rmrr_unit {
437 struct list_head list; /* list of rmrr units */ 384 struct list_head list; /* list of rmrr units */
438 struct acpi_dmar_header *hdr; /* ACPI header */ 385 struct acpi_dmar_header *hdr; /* ACPI header */
@@ -523,6 +470,27 @@ EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
523static DEFINE_SPINLOCK(device_domain_lock); 470static DEFINE_SPINLOCK(device_domain_lock);
524static LIST_HEAD(device_domain_list); 471static LIST_HEAD(device_domain_list);
525 472
473/*
474 * Iterate over elements in device_domain_list and call the specified
475 * callback @fn against each element. This helper should only be used
476 * in the context where the device_domain_lock has already been holden.
477 */
478int for_each_device_domain(int (*fn)(struct device_domain_info *info,
479 void *data), void *data)
480{
481 int ret = 0;
482 struct device_domain_info *info;
483
484 assert_spin_locked(&device_domain_lock);
485 list_for_each_entry(info, &device_domain_list, global) {
486 ret = fn(info, data);
487 if (ret)
488 return ret;
489 }
490
491 return 0;
492}
493
526const struct iommu_ops intel_iommu_ops; 494const struct iommu_ops intel_iommu_ops;
527 495
528static bool translation_pre_enabled(struct intel_iommu *iommu) 496static bool translation_pre_enabled(struct intel_iommu *iommu)
@@ -629,7 +597,7 @@ static void set_iommu_domain(struct intel_iommu *iommu, u16 did,
629 domains[did & 0xff] = domain; 597 domains[did & 0xff] = domain;
630} 598}
631 599
632static inline void *alloc_pgtable_page(int node) 600void *alloc_pgtable_page(int node)
633{ 601{
634 struct page *page; 602 struct page *page;
635 void *vaddr = NULL; 603 void *vaddr = NULL;
@@ -640,7 +608,7 @@ static inline void *alloc_pgtable_page(int node)
640 return vaddr; 608 return vaddr;
641} 609}
642 610
643static inline void free_pgtable_page(void *vaddr) 611void free_pgtable_page(void *vaddr)
644{ 612{
645 free_page((unsigned long)vaddr); 613 free_page((unsigned long)vaddr);
646} 614}
@@ -723,7 +691,7 @@ int iommu_calculate_agaw(struct intel_iommu *iommu)
723} 691}
724 692
725/* This functionin only returns single iommu in a domain */ 693/* This functionin only returns single iommu in a domain */
726static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain) 694struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
727{ 695{
728 int iommu_id; 696 int iommu_id;
729 697
@@ -1501,6 +1469,20 @@ static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1501 return; 1469 return;
1502 1470
1503 pdev = to_pci_dev(info->dev); 1471 pdev = to_pci_dev(info->dev);
1472 /* For IOMMU that supports device IOTLB throttling (DIT), we assign
1473 * PFSID to the invalidation desc of a VF such that IOMMU HW can gauge
1474 * queue depth at PF level. If DIT is not set, PFSID will be treated as
1475 * reserved, which should be set to 0.
1476 */
1477 if (!ecap_dit(info->iommu->ecap))
1478 info->pfsid = 0;
1479 else {
1480 struct pci_dev *pf_pdev;
1481
1482 /* pdev will be returned if device is not a vf */
1483 pf_pdev = pci_physfn(pdev);
1484 info->pfsid = PCI_DEVID(pf_pdev->bus->number, pf_pdev->devfn);
1485 }
1504 1486
1505#ifdef CONFIG_INTEL_IOMMU_SVM 1487#ifdef CONFIG_INTEL_IOMMU_SVM
1506 /* The PCIe spec, in its wisdom, declares that the behaviour of 1488 /* The PCIe spec, in its wisdom, declares that the behaviour of
@@ -1566,7 +1548,8 @@ static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1566 1548
1567 sid = info->bus << 8 | info->devfn; 1549 sid = info->bus << 8 | info->devfn;
1568 qdep = info->ats_qdep; 1550 qdep = info->ats_qdep;
1569 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask); 1551 qi_flush_dev_iotlb(info->iommu, sid, info->pfsid,
1552 qdep, addr, mask);
1570 } 1553 }
1571 spin_unlock_irqrestore(&device_domain_lock, flags); 1554 spin_unlock_irqrestore(&device_domain_lock, flags);
1572} 1555}
@@ -1800,7 +1783,7 @@ static void free_dmar_iommu(struct intel_iommu *iommu)
1800 if (pasid_enabled(iommu)) { 1783 if (pasid_enabled(iommu)) {
1801 if (ecap_prs(iommu->ecap)) 1784 if (ecap_prs(iommu->ecap))
1802 intel_svm_finish_prq(iommu); 1785 intel_svm_finish_prq(iommu);
1803 intel_svm_free_pasid_tables(iommu); 1786 intel_svm_exit(iommu);
1804 } 1787 }
1805#endif 1788#endif
1806} 1789}
@@ -2495,6 +2478,7 @@ static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu,
2495 info->dev = dev; 2478 info->dev = dev;
2496 info->domain = domain; 2479 info->domain = domain;
2497 info->iommu = iommu; 2480 info->iommu = iommu;
2481 info->pasid_table = NULL;
2498 2482
2499 if (dev && dev_is_pci(dev)) { 2483 if (dev && dev_is_pci(dev)) {
2500 struct pci_dev *pdev = to_pci_dev(info->dev); 2484 struct pci_dev *pdev = to_pci_dev(info->dev);
@@ -2552,6 +2536,15 @@ static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu,
2552 list_add(&info->global, &device_domain_list); 2536 list_add(&info->global, &device_domain_list);
2553 if (dev) 2537 if (dev)
2554 dev->archdata.iommu = info; 2538 dev->archdata.iommu = info;
2539
2540 if (dev && dev_is_pci(dev) && info->pasid_supported) {
2541 ret = intel_pasid_alloc_table(dev);
2542 if (ret) {
2543 __dmar_remove_one_dev_info(info);
2544 spin_unlock_irqrestore(&device_domain_lock, flags);
2545 return NULL;
2546 }
2547 }
2555 spin_unlock_irqrestore(&device_domain_lock, flags); 2548 spin_unlock_irqrestore(&device_domain_lock, flags);
2556 2549
2557 if (dev && domain_context_mapping(domain, dev)) { 2550 if (dev && domain_context_mapping(domain, dev)) {
@@ -3304,6 +3297,18 @@ static int __init init_dmars(void)
3304 } 3297 }
3305 3298
3306 for_each_active_iommu(iommu, drhd) { 3299 for_each_active_iommu(iommu, drhd) {
3300 /*
3301 * Find the max pasid size of all IOMMU's in the system.
3302 * We need to ensure the system pasid table is no bigger
3303 * than the smallest supported.
3304 */
3305 if (pasid_enabled(iommu)) {
3306 u32 temp = 2 << ecap_pss(iommu->ecap);
3307
3308 intel_pasid_max_id = min_t(u32, temp,
3309 intel_pasid_max_id);
3310 }
3311
3307 g_iommus[iommu->seq_id] = iommu; 3312 g_iommus[iommu->seq_id] = iommu;
3308 3313
3309 intel_iommu_init_qi(iommu); 3314 intel_iommu_init_qi(iommu);
@@ -3359,7 +3364,7 @@ static int __init init_dmars(void)
3359 hw_pass_through = 0; 3364 hw_pass_through = 0;
3360#ifdef CONFIG_INTEL_IOMMU_SVM 3365#ifdef CONFIG_INTEL_IOMMU_SVM
3361 if (pasid_enabled(iommu)) 3366 if (pasid_enabled(iommu))
3362 intel_svm_alloc_pasid_tables(iommu); 3367 intel_svm_init(iommu);
3363#endif 3368#endif
3364 } 3369 }
3365 3370
@@ -3526,7 +3531,7 @@ static unsigned long intel_alloc_iova(struct device *dev,
3526 return iova_pfn; 3531 return iova_pfn;
3527} 3532}
3528 3533
3529static struct dmar_domain *get_valid_domain_for_dev(struct device *dev) 3534struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
3530{ 3535{
3531 struct dmar_domain *domain, *tmp; 3536 struct dmar_domain *domain, *tmp;
3532 struct dmar_rmrr_unit *rmrr; 3537 struct dmar_rmrr_unit *rmrr;
@@ -4354,7 +4359,7 @@ static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
4354 4359
4355#ifdef CONFIG_INTEL_IOMMU_SVM 4360#ifdef CONFIG_INTEL_IOMMU_SVM
4356 if (pasid_enabled(iommu)) 4361 if (pasid_enabled(iommu))
4357 intel_svm_alloc_pasid_tables(iommu); 4362 intel_svm_init(iommu);
4358#endif 4363#endif
4359 4364
4360 if (dmaru->ignored) { 4365 if (dmaru->ignored) {
@@ -4906,6 +4911,7 @@ static void __dmar_remove_one_dev_info(struct device_domain_info *info)
4906 if (info->dev) { 4911 if (info->dev) {
4907 iommu_disable_dev_iotlb(info); 4912 iommu_disable_dev_iotlb(info);
4908 domain_context_clear(iommu, info->dev); 4913 domain_context_clear(iommu, info->dev);
4914 intel_pasid_free_table(info->dev);
4909 } 4915 }
4910 4916
4911 unlink_domain_info(info); 4917 unlink_domain_info(info);
@@ -5231,22 +5237,16 @@ static void intel_iommu_put_resv_regions(struct device *dev,
5231 5237
5232#ifdef CONFIG_INTEL_IOMMU_SVM 5238#ifdef CONFIG_INTEL_IOMMU_SVM
5233#define MAX_NR_PASID_BITS (20) 5239#define MAX_NR_PASID_BITS (20)
5234static inline unsigned long intel_iommu_get_pts(struct intel_iommu *iommu) 5240static inline unsigned long intel_iommu_get_pts(struct device *dev)
5235{ 5241{
5236 /* 5242 int pts, max_pasid;
5237 * Convert ecap_pss to extend context entry pts encoding, also 5243
5238 * respect the soft pasid_max value set by the iommu. 5244 max_pasid = intel_pasid_get_dev_max_id(dev);
5239 * - number of PASID bits = ecap_pss + 1 5245 pts = find_first_bit((unsigned long *)&max_pasid, MAX_NR_PASID_BITS);
5240 * - number of PASID table entries = 2^(pts + 5) 5246 if (pts < 5)
5241 * Therefore, pts = ecap_pss - 4
5242 * e.g. KBL ecap_pss = 0x13, PASID has 20 bits, pts = 15
5243 */
5244 if (ecap_pss(iommu->ecap) < 5)
5245 return 0; 5247 return 0;
5246 5248
5247 /* pasid_max is encoded as actual number of entries not the bits */ 5249 return pts - 5;
5248 return find_first_bit((unsigned long *)&iommu->pasid_max,
5249 MAX_NR_PASID_BITS) - 5;
5250} 5250}
5251 5251
5252int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct intel_svm_dev *sdev) 5252int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct intel_svm_dev *sdev)
@@ -5282,8 +5282,8 @@ int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct intel_svm_dev *sd
5282 if (!(ctx_lo & CONTEXT_PASIDE)) { 5282 if (!(ctx_lo & CONTEXT_PASIDE)) {
5283 if (iommu->pasid_state_table) 5283 if (iommu->pasid_state_table)
5284 context[1].hi = (u64)virt_to_phys(iommu->pasid_state_table); 5284 context[1].hi = (u64)virt_to_phys(iommu->pasid_state_table);
5285 context[1].lo = (u64)virt_to_phys(iommu->pasid_table) | 5285 context[1].lo = (u64)virt_to_phys(info->pasid_table->table) |
5286 intel_iommu_get_pts(iommu); 5286 intel_iommu_get_pts(sdev->dev);
5287 5287
5288 wmb(); 5288 wmb();
5289 /* CONTEXT_TT_MULTI_LEVEL and CONTEXT_TT_DEV_IOTLB are both 5289 /* CONTEXT_TT_MULTI_LEVEL and CONTEXT_TT_DEV_IOTLB are both
@@ -5350,11 +5350,6 @@ struct intel_iommu *intel_svm_device_to_iommu(struct device *dev)
5350 return NULL; 5350 return NULL;
5351 } 5351 }
5352 5352
5353 if (!iommu->pasid_table) {
5354 dev_err(dev, "PASID not enabled on IOMMU; cannot enable SVM\n");
5355 return NULL;
5356 }
5357
5358 return iommu; 5353 return iommu;
5359} 5354}
5360#endif /* CONFIG_INTEL_IOMMU_SVM */ 5355#endif /* CONFIG_INTEL_IOMMU_SVM */
@@ -5367,7 +5362,6 @@ const struct iommu_ops intel_iommu_ops = {
5367 .detach_dev = intel_iommu_detach_device, 5362 .detach_dev = intel_iommu_detach_device,
5368 .map = intel_iommu_map, 5363 .map = intel_iommu_map,
5369 .unmap = intel_iommu_unmap, 5364 .unmap = intel_iommu_unmap,
5370 .map_sg = default_iommu_map_sg,
5371 .iova_to_phys = intel_iommu_iova_to_phys, 5365 .iova_to_phys = intel_iommu_iova_to_phys,
5372 .add_device = intel_iommu_add_device, 5366 .add_device = intel_iommu_add_device,
5373 .remove_device = intel_iommu_remove_device, 5367 .remove_device = intel_iommu_remove_device,
diff --git a/drivers/iommu/intel-pasid.c b/drivers/iommu/intel-pasid.c
new file mode 100644
index 000000000000..fe95c9bd4d33
--- /dev/null
+++ b/drivers/iommu/intel-pasid.c
@@ -0,0 +1,239 @@
1// SPDX-License-Identifier: GPL-2.0
2/**
3 * intel-pasid.c - PASID idr, table and entry manipulation
4 *
5 * Copyright (C) 2018 Intel Corporation
6 *
7 * Author: Lu Baolu <baolu.lu@linux.intel.com>
8 */
9
10#define pr_fmt(fmt) "DMAR: " fmt
11
12#include <linux/dmar.h>
13#include <linux/intel-iommu.h>
14#include <linux/iommu.h>
15#include <linux/memory.h>
16#include <linux/pci.h>
17#include <linux/pci-ats.h>
18#include <linux/spinlock.h>
19
20#include "intel-pasid.h"
21
22/*
23 * Intel IOMMU system wide PASID name space:
24 */
25static DEFINE_SPINLOCK(pasid_lock);
26u32 intel_pasid_max_id = PASID_MAX;
27static DEFINE_IDR(pasid_idr);
28
29int intel_pasid_alloc_id(void *ptr, int start, int end, gfp_t gfp)
30{
31 int ret, min, max;
32
33 min = max_t(int, start, PASID_MIN);
34 max = min_t(int, end, intel_pasid_max_id);
35
36 WARN_ON(in_interrupt());
37 idr_preload(gfp);
38 spin_lock(&pasid_lock);
39 ret = idr_alloc(&pasid_idr, ptr, min, max, GFP_ATOMIC);
40 spin_unlock(&pasid_lock);
41 idr_preload_end();
42
43 return ret;
44}
45
46void intel_pasid_free_id(int pasid)
47{
48 spin_lock(&pasid_lock);
49 idr_remove(&pasid_idr, pasid);
50 spin_unlock(&pasid_lock);
51}
52
53void *intel_pasid_lookup_id(int pasid)
54{
55 void *p;
56
57 spin_lock(&pasid_lock);
58 p = idr_find(&pasid_idr, pasid);
59 spin_unlock(&pasid_lock);
60
61 return p;
62}
63
64/*
65 * Per device pasid table management:
66 */
67static inline void
68device_attach_pasid_table(struct device_domain_info *info,
69 struct pasid_table *pasid_table)
70{
71 info->pasid_table = pasid_table;
72 list_add(&info->table, &pasid_table->dev);
73}
74
75static inline void
76device_detach_pasid_table(struct device_domain_info *info,
77 struct pasid_table *pasid_table)
78{
79 info->pasid_table = NULL;
80 list_del(&info->table);
81}
82
83struct pasid_table_opaque {
84 struct pasid_table **pasid_table;
85 int segment;
86 int bus;
87 int devfn;
88};
89
90static int search_pasid_table(struct device_domain_info *info, void *opaque)
91{
92 struct pasid_table_opaque *data = opaque;
93
94 if (info->iommu->segment == data->segment &&
95 info->bus == data->bus &&
96 info->devfn == data->devfn &&
97 info->pasid_table) {
98 *data->pasid_table = info->pasid_table;
99 return 1;
100 }
101
102 return 0;
103}
104
105static int get_alias_pasid_table(struct pci_dev *pdev, u16 alias, void *opaque)
106{
107 struct pasid_table_opaque *data = opaque;
108
109 data->segment = pci_domain_nr(pdev->bus);
110 data->bus = PCI_BUS_NUM(alias);
111 data->devfn = alias & 0xff;
112
113 return for_each_device_domain(&search_pasid_table, data);
114}
115
116/*
117 * Allocate a pasid table for @dev. It should be called in a
118 * single-thread context.
119 */
120int intel_pasid_alloc_table(struct device *dev)
121{
122 struct device_domain_info *info;
123 struct pasid_table *pasid_table;
124 struct pasid_table_opaque data;
125 struct page *pages;
126 size_t size, count;
127 int ret, order;
128
129 info = dev->archdata.iommu;
130 if (WARN_ON(!info || !dev_is_pci(dev) ||
131 !info->pasid_supported || info->pasid_table))
132 return -EINVAL;
133
134 /* DMA alias device already has a pasid table, use it: */
135 data.pasid_table = &pasid_table;
136 ret = pci_for_each_dma_alias(to_pci_dev(dev),
137 &get_alias_pasid_table, &data);
138 if (ret)
139 goto attach_out;
140
141 pasid_table = kzalloc(sizeof(*pasid_table), GFP_ATOMIC);
142 if (!pasid_table)
143 return -ENOMEM;
144 INIT_LIST_HEAD(&pasid_table->dev);
145
146 size = sizeof(struct pasid_entry);
147 count = min_t(int, pci_max_pasids(to_pci_dev(dev)), intel_pasid_max_id);
148 order = get_order(size * count);
149 pages = alloc_pages_node(info->iommu->node,
150 GFP_ATOMIC | __GFP_ZERO,
151 order);
152 if (!pages)
153 return -ENOMEM;
154
155 pasid_table->table = page_address(pages);
156 pasid_table->order = order;
157 pasid_table->max_pasid = count;
158
159attach_out:
160 device_attach_pasid_table(info, pasid_table);
161
162 return 0;
163}
164
165void intel_pasid_free_table(struct device *dev)
166{
167 struct device_domain_info *info;
168 struct pasid_table *pasid_table;
169
170 info = dev->archdata.iommu;
171 if (!info || !dev_is_pci(dev) ||
172 !info->pasid_supported || !info->pasid_table)
173 return;
174
175 pasid_table = info->pasid_table;
176 device_detach_pasid_table(info, pasid_table);
177
178 if (!list_empty(&pasid_table->dev))
179 return;
180
181 free_pages((unsigned long)pasid_table->table, pasid_table->order);
182 kfree(pasid_table);
183}
184
185struct pasid_table *intel_pasid_get_table(struct device *dev)
186{
187 struct device_domain_info *info;
188
189 info = dev->archdata.iommu;
190 if (!info)
191 return NULL;
192
193 return info->pasid_table;
194}
195
196int intel_pasid_get_dev_max_id(struct device *dev)
197{
198 struct device_domain_info *info;
199
200 info = dev->archdata.iommu;
201 if (!info || !info->pasid_table)
202 return 0;
203
204 return info->pasid_table->max_pasid;
205}
206
207struct pasid_entry *intel_pasid_get_entry(struct device *dev, int pasid)
208{
209 struct pasid_table *pasid_table;
210 struct pasid_entry *entries;
211
212 pasid_table = intel_pasid_get_table(dev);
213 if (WARN_ON(!pasid_table || pasid < 0 ||
214 pasid >= intel_pasid_get_dev_max_id(dev)))
215 return NULL;
216
217 entries = pasid_table->table;
218
219 return &entries[pasid];
220}
221
222/*
223 * Interfaces for PASID table entry manipulation:
224 */
225static inline void pasid_clear_entry(struct pasid_entry *pe)
226{
227 WRITE_ONCE(pe->val, 0);
228}
229
230void intel_pasid_clear_entry(struct device *dev, int pasid)
231{
232 struct pasid_entry *pe;
233
234 pe = intel_pasid_get_entry(dev, pasid);
235 if (WARN_ON(!pe))
236 return;
237
238 pasid_clear_entry(pe);
239}
diff --git a/drivers/iommu/intel-pasid.h b/drivers/iommu/intel-pasid.h
new file mode 100644
index 000000000000..1c05ed6fc5a5
--- /dev/null
+++ b/drivers/iommu/intel-pasid.h
@@ -0,0 +1,39 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * intel-pasid.h - PASID idr, table and entry header
4 *
5 * Copyright (C) 2018 Intel Corporation
6 *
7 * Author: Lu Baolu <baolu.lu@linux.intel.com>
8 */
9
10#ifndef __INTEL_PASID_H
11#define __INTEL_PASID_H
12
13#define PASID_MIN 0x1
14#define PASID_MAX 0x100000
15
16struct pasid_entry {
17 u64 val;
18};
19
20/* The representative of a PASID table */
21struct pasid_table {
22 void *table; /* pasid table pointer */
23 int order; /* page order of pasid table */
24 int max_pasid; /* max pasid */
25 struct list_head dev; /* device list */
26};
27
28extern u32 intel_pasid_max_id;
29int intel_pasid_alloc_id(void *ptr, int start, int end, gfp_t gfp);
30void intel_pasid_free_id(int pasid);
31void *intel_pasid_lookup_id(int pasid);
32int intel_pasid_alloc_table(struct device *dev);
33void intel_pasid_free_table(struct device *dev);
34struct pasid_table *intel_pasid_get_table(struct device *dev);
35int intel_pasid_get_dev_max_id(struct device *dev);
36struct pasid_entry *intel_pasid_get_entry(struct device *dev, int pasid);
37void intel_pasid_clear_entry(struct device *dev, int pasid);
38
39#endif /* __INTEL_PASID_H */
diff --git a/drivers/iommu/intel-svm.c b/drivers/iommu/intel-svm.c
index 7d65aab36a96..4a03e5090952 100644
--- a/drivers/iommu/intel-svm.c
+++ b/drivers/iommu/intel-svm.c
@@ -27,21 +27,19 @@
27#include <linux/mm_types.h> 27#include <linux/mm_types.h>
28#include <asm/page.h> 28#include <asm/page.h>
29 29
30#include "intel-pasid.h"
31
30#define PASID_ENTRY_P BIT_ULL(0) 32#define PASID_ENTRY_P BIT_ULL(0)
31#define PASID_ENTRY_FLPM_5LP BIT_ULL(9) 33#define PASID_ENTRY_FLPM_5LP BIT_ULL(9)
32#define PASID_ENTRY_SRE BIT_ULL(11) 34#define PASID_ENTRY_SRE BIT_ULL(11)
33 35
34static irqreturn_t prq_event_thread(int irq, void *d); 36static irqreturn_t prq_event_thread(int irq, void *d);
35 37
36struct pasid_entry {
37 u64 val;
38};
39
40struct pasid_state_entry { 38struct pasid_state_entry {
41 u64 val; 39 u64 val;
42}; 40};
43 41
44int intel_svm_alloc_pasid_tables(struct intel_iommu *iommu) 42int intel_svm_init(struct intel_iommu *iommu)
45{ 43{
46 struct page *pages; 44 struct page *pages;
47 int order; 45 int order;
@@ -66,15 +64,6 @@ int intel_svm_alloc_pasid_tables(struct intel_iommu *iommu)
66 iommu->pasid_max = 0x20000; 64 iommu->pasid_max = 0x20000;
67 65
68 order = get_order(sizeof(struct pasid_entry) * iommu->pasid_max); 66 order = get_order(sizeof(struct pasid_entry) * iommu->pasid_max);
69 pages = alloc_pages(GFP_KERNEL | __GFP_ZERO, order);
70 if (!pages) {
71 pr_warn("IOMMU: %s: Failed to allocate PASID table\n",
72 iommu->name);
73 return -ENOMEM;
74 }
75 iommu->pasid_table = page_address(pages);
76 pr_info("%s: Allocated order %d PASID table.\n", iommu->name, order);
77
78 if (ecap_dis(iommu->ecap)) { 67 if (ecap_dis(iommu->ecap)) {
79 /* Just making it explicit... */ 68 /* Just making it explicit... */
80 BUILD_BUG_ON(sizeof(struct pasid_entry) != sizeof(struct pasid_state_entry)); 69 BUILD_BUG_ON(sizeof(struct pasid_entry) != sizeof(struct pasid_state_entry));
@@ -86,24 +75,18 @@ int intel_svm_alloc_pasid_tables(struct intel_iommu *iommu)
86 iommu->name); 75 iommu->name);
87 } 76 }
88 77
89 idr_init(&iommu->pasid_idr);
90
91 return 0; 78 return 0;
92} 79}
93 80
94int intel_svm_free_pasid_tables(struct intel_iommu *iommu) 81int intel_svm_exit(struct intel_iommu *iommu)
95{ 82{
96 int order = get_order(sizeof(struct pasid_entry) * iommu->pasid_max); 83 int order = get_order(sizeof(struct pasid_entry) * iommu->pasid_max);
97 84
98 if (iommu->pasid_table) {
99 free_pages((unsigned long)iommu->pasid_table, order);
100 iommu->pasid_table = NULL;
101 }
102 if (iommu->pasid_state_table) { 85 if (iommu->pasid_state_table) {
103 free_pages((unsigned long)iommu->pasid_state_table, order); 86 free_pages((unsigned long)iommu->pasid_state_table, order);
104 iommu->pasid_state_table = NULL; 87 iommu->pasid_state_table = NULL;
105 } 88 }
106 idr_destroy(&iommu->pasid_idr); 89
107 return 0; 90 return 0;
108} 91}
109 92
@@ -279,11 +262,9 @@ static void intel_mm_release(struct mmu_notifier *mn, struct mm_struct *mm)
279 * page) so that we end up taking a fault that the hardware really 262 * page) so that we end up taking a fault that the hardware really
280 * *has* to handle gracefully without affecting other processes. 263 * *has* to handle gracefully without affecting other processes.
281 */ 264 */
282 svm->iommu->pasid_table[svm->pasid].val = 0;
283 wmb();
284
285 rcu_read_lock(); 265 rcu_read_lock();
286 list_for_each_entry_rcu(sdev, &svm->devs, list) { 266 list_for_each_entry_rcu(sdev, &svm->devs, list) {
267 intel_pasid_clear_entry(sdev->dev, svm->pasid);
287 intel_flush_pasid_dev(svm, sdev, svm->pasid); 268 intel_flush_pasid_dev(svm, sdev, svm->pasid);
288 intel_flush_svm_range_dev(svm, sdev, 0, -1, 0, !svm->mm); 269 intel_flush_svm_range_dev(svm, sdev, 0, -1, 0, !svm->mm);
289 } 270 }
@@ -299,10 +280,12 @@ static const struct mmu_notifier_ops intel_mmuops = {
299}; 280};
300 281
301static DEFINE_MUTEX(pasid_mutex); 282static DEFINE_MUTEX(pasid_mutex);
283static LIST_HEAD(global_svm_list);
302 284
303int intel_svm_bind_mm(struct device *dev, int *pasid, int flags, struct svm_dev_ops *ops) 285int intel_svm_bind_mm(struct device *dev, int *pasid, int flags, struct svm_dev_ops *ops)
304{ 286{
305 struct intel_iommu *iommu = intel_svm_device_to_iommu(dev); 287 struct intel_iommu *iommu = intel_svm_device_to_iommu(dev);
288 struct pasid_entry *entry;
306 struct intel_svm_dev *sdev; 289 struct intel_svm_dev *sdev;
307 struct intel_svm *svm = NULL; 290 struct intel_svm *svm = NULL;
308 struct mm_struct *mm = NULL; 291 struct mm_struct *mm = NULL;
@@ -310,7 +293,7 @@ int intel_svm_bind_mm(struct device *dev, int *pasid, int flags, struct svm_dev_
310 int pasid_max; 293 int pasid_max;
311 int ret; 294 int ret;
312 295
313 if (WARN_ON(!iommu || !iommu->pasid_table)) 296 if (!iommu)
314 return -EINVAL; 297 return -EINVAL;
315 298
316 if (dev_is_pci(dev)) { 299 if (dev_is_pci(dev)) {
@@ -330,13 +313,13 @@ int intel_svm_bind_mm(struct device *dev, int *pasid, int flags, struct svm_dev_
330 313
331 mutex_lock(&pasid_mutex); 314 mutex_lock(&pasid_mutex);
332 if (pasid && !(flags & SVM_FLAG_PRIVATE_PASID)) { 315 if (pasid && !(flags & SVM_FLAG_PRIVATE_PASID)) {
333 int i; 316 struct intel_svm *t;
334 317
335 idr_for_each_entry(&iommu->pasid_idr, svm, i) { 318 list_for_each_entry(t, &global_svm_list, list) {
336 if (svm->mm != mm || 319 if (t->mm != mm || (t->flags & SVM_FLAG_PRIVATE_PASID))
337 (svm->flags & SVM_FLAG_PRIVATE_PASID))
338 continue; 320 continue;
339 321
322 svm = t;
340 if (svm->pasid >= pasid_max) { 323 if (svm->pasid >= pasid_max) {
341 dev_warn(dev, 324 dev_warn(dev,
342 "Limited PASID width. Cannot use existing PASID %d\n", 325 "Limited PASID width. Cannot use existing PASID %d\n",
@@ -388,13 +371,13 @@ int intel_svm_bind_mm(struct device *dev, int *pasid, int flags, struct svm_dev_
388 } 371 }
389 svm->iommu = iommu; 372 svm->iommu = iommu;
390 373
391 if (pasid_max > iommu->pasid_max) 374 if (pasid_max > intel_pasid_max_id)
392 pasid_max = iommu->pasid_max; 375 pasid_max = intel_pasid_max_id;
393 376
394 /* Do not use PASID 0 in caching mode (virtualised IOMMU) */ 377 /* Do not use PASID 0 in caching mode (virtualised IOMMU) */
395 ret = idr_alloc(&iommu->pasid_idr, svm, 378 ret = intel_pasid_alloc_id(svm,
396 !!cap_caching_mode(iommu->cap), 379 !!cap_caching_mode(iommu->cap),
397 pasid_max - 1, GFP_KERNEL); 380 pasid_max - 1, GFP_KERNEL);
398 if (ret < 0) { 381 if (ret < 0) {
399 kfree(svm); 382 kfree(svm);
400 kfree(sdev); 383 kfree(sdev);
@@ -405,11 +388,12 @@ int intel_svm_bind_mm(struct device *dev, int *pasid, int flags, struct svm_dev_
405 svm->mm = mm; 388 svm->mm = mm;
406 svm->flags = flags; 389 svm->flags = flags;
407 INIT_LIST_HEAD_RCU(&svm->devs); 390 INIT_LIST_HEAD_RCU(&svm->devs);
391 INIT_LIST_HEAD(&svm->list);
408 ret = -ENOMEM; 392 ret = -ENOMEM;
409 if (mm) { 393 if (mm) {
410 ret = mmu_notifier_register(&svm->notifier, mm); 394 ret = mmu_notifier_register(&svm->notifier, mm);
411 if (ret) { 395 if (ret) {
412 idr_remove(&svm->iommu->pasid_idr, svm->pasid); 396 intel_pasid_free_id(svm->pasid);
413 kfree(svm); 397 kfree(svm);
414 kfree(sdev); 398 kfree(sdev);
415 goto out; 399 goto out;
@@ -421,7 +405,8 @@ int intel_svm_bind_mm(struct device *dev, int *pasid, int flags, struct svm_dev_
421 if (cpu_feature_enabled(X86_FEATURE_LA57)) 405 if (cpu_feature_enabled(X86_FEATURE_LA57))
422 pasid_entry_val |= PASID_ENTRY_FLPM_5LP; 406 pasid_entry_val |= PASID_ENTRY_FLPM_5LP;
423 407
424 iommu->pasid_table[svm->pasid].val = pasid_entry_val; 408 entry = intel_pasid_get_entry(dev, svm->pasid);
409 entry->val = pasid_entry_val;
425 410
426 wmb(); 411 wmb();
427 412
@@ -431,6 +416,8 @@ int intel_svm_bind_mm(struct device *dev, int *pasid, int flags, struct svm_dev_
431 */ 416 */
432 if (cap_caching_mode(iommu->cap)) 417 if (cap_caching_mode(iommu->cap))
433 intel_flush_pasid_dev(svm, sdev, svm->pasid); 418 intel_flush_pasid_dev(svm, sdev, svm->pasid);
419
420 list_add_tail(&svm->list, &global_svm_list);
434 } 421 }
435 list_add_rcu(&sdev->list, &svm->devs); 422 list_add_rcu(&sdev->list, &svm->devs);
436 423
@@ -454,10 +441,10 @@ int intel_svm_unbind_mm(struct device *dev, int pasid)
454 441
455 mutex_lock(&pasid_mutex); 442 mutex_lock(&pasid_mutex);
456 iommu = intel_svm_device_to_iommu(dev); 443 iommu = intel_svm_device_to_iommu(dev);
457 if (!iommu || !iommu->pasid_table) 444 if (!iommu)
458 goto out; 445 goto out;
459 446
460 svm = idr_find(&iommu->pasid_idr, pasid); 447 svm = intel_pasid_lookup_id(pasid);
461 if (!svm) 448 if (!svm)
462 goto out; 449 goto out;
463 450
@@ -477,15 +464,15 @@ int intel_svm_unbind_mm(struct device *dev, int pasid)
477 intel_flush_pasid_dev(svm, sdev, svm->pasid); 464 intel_flush_pasid_dev(svm, sdev, svm->pasid);
478 intel_flush_svm_range_dev(svm, sdev, 0, -1, 0, !svm->mm); 465 intel_flush_svm_range_dev(svm, sdev, 0, -1, 0, !svm->mm);
479 kfree_rcu(sdev, rcu); 466 kfree_rcu(sdev, rcu);
467 intel_pasid_clear_entry(dev, svm->pasid);
480 468
481 if (list_empty(&svm->devs)) { 469 if (list_empty(&svm->devs)) {
482 svm->iommu->pasid_table[svm->pasid].val = 0; 470 intel_pasid_free_id(svm->pasid);
483 wmb();
484
485 idr_remove(&svm->iommu->pasid_idr, svm->pasid);
486 if (svm->mm) 471 if (svm->mm)
487 mmu_notifier_unregister(&svm->notifier, svm->mm); 472 mmu_notifier_unregister(&svm->notifier, svm->mm);
488 473
474 list_del(&svm->list);
475
489 /* We mandate that no page faults may be outstanding 476 /* We mandate that no page faults may be outstanding
490 * for the PASID when intel_svm_unbind_mm() is called. 477 * for the PASID when intel_svm_unbind_mm() is called.
491 * If that is not obeyed, subtle errors will happen. 478 * If that is not obeyed, subtle errors will happen.
@@ -512,10 +499,10 @@ int intel_svm_is_pasid_valid(struct device *dev, int pasid)
512 499
513 mutex_lock(&pasid_mutex); 500 mutex_lock(&pasid_mutex);
514 iommu = intel_svm_device_to_iommu(dev); 501 iommu = intel_svm_device_to_iommu(dev);
515 if (!iommu || !iommu->pasid_table) 502 if (!iommu)
516 goto out; 503 goto out;
517 504
518 svm = idr_find(&iommu->pasid_idr, pasid); 505 svm = intel_pasid_lookup_id(pasid);
519 if (!svm) 506 if (!svm)
520 goto out; 507 goto out;
521 508
@@ -614,7 +601,7 @@ static irqreturn_t prq_event_thread(int irq, void *d)
614 601
615 if (!svm || svm->pasid != req->pasid) { 602 if (!svm || svm->pasid != req->pasid) {
616 rcu_read_lock(); 603 rcu_read_lock();
617 svm = idr_find(&iommu->pasid_idr, req->pasid); 604 svm = intel_pasid_lookup_id(req->pasid);
618 /* It *can't* go away, because the driver is not permitted 605 /* It *can't* go away, because the driver is not permitted
619 * to unbind the mm while any page faults are outstanding. 606 * to unbind the mm while any page faults are outstanding.
620 * So we only need RCU to protect the internal idr code. */ 607 * So we only need RCU to protect the internal idr code. */
diff --git a/drivers/iommu/io-pgtable-arm-v7s.c b/drivers/iommu/io-pgtable-arm-v7s.c
index 50e3a9fcf43e..b5948ba6b3b3 100644
--- a/drivers/iommu/io-pgtable-arm-v7s.c
+++ b/drivers/iommu/io-pgtable-arm-v7s.c
@@ -192,6 +192,7 @@ static void *__arm_v7s_alloc_table(int lvl, gfp_t gfp,
192{ 192{
193 struct io_pgtable_cfg *cfg = &data->iop.cfg; 193 struct io_pgtable_cfg *cfg = &data->iop.cfg;
194 struct device *dev = cfg->iommu_dev; 194 struct device *dev = cfg->iommu_dev;
195 phys_addr_t phys;
195 dma_addr_t dma; 196 dma_addr_t dma;
196 size_t size = ARM_V7S_TABLE_SIZE(lvl); 197 size_t size = ARM_V7S_TABLE_SIZE(lvl);
197 void *table = NULL; 198 void *table = NULL;
@@ -200,6 +201,10 @@ static void *__arm_v7s_alloc_table(int lvl, gfp_t gfp,
200 table = (void *)__get_dma_pages(__GFP_ZERO, get_order(size)); 201 table = (void *)__get_dma_pages(__GFP_ZERO, get_order(size));
201 else if (lvl == 2) 202 else if (lvl == 2)
202 table = kmem_cache_zalloc(data->l2_tables, gfp | GFP_DMA); 203 table = kmem_cache_zalloc(data->l2_tables, gfp | GFP_DMA);
204 phys = virt_to_phys(table);
205 if (phys != (arm_v7s_iopte)phys)
206 /* Doesn't fit in PTE */
207 goto out_free;
203 if (table && !(cfg->quirks & IO_PGTABLE_QUIRK_NO_DMA)) { 208 if (table && !(cfg->quirks & IO_PGTABLE_QUIRK_NO_DMA)) {
204 dma = dma_map_single(dev, table, size, DMA_TO_DEVICE); 209 dma = dma_map_single(dev, table, size, DMA_TO_DEVICE);
205 if (dma_mapping_error(dev, dma)) 210 if (dma_mapping_error(dev, dma))
@@ -209,7 +214,7 @@ static void *__arm_v7s_alloc_table(int lvl, gfp_t gfp,
209 * address directly, so if the DMA layer suggests otherwise by 214 * address directly, so if the DMA layer suggests otherwise by
210 * translating or truncating them, that bodes very badly... 215 * translating or truncating them, that bodes very badly...
211 */ 216 */
212 if (dma != virt_to_phys(table)) 217 if (dma != phys)
213 goto out_unmap; 218 goto out_unmap;
214 } 219 }
215 kmemleak_ignore(table); 220 kmemleak_ignore(table);
diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c
index 010a254305dd..88641b4560bc 100644
--- a/drivers/iommu/io-pgtable-arm.c
+++ b/drivers/iommu/io-pgtable-arm.c
@@ -237,7 +237,8 @@ static void *__arm_lpae_alloc_pages(size_t size, gfp_t gfp,
237 void *pages; 237 void *pages;
238 238
239 VM_BUG_ON((gfp & __GFP_HIGHMEM)); 239 VM_BUG_ON((gfp & __GFP_HIGHMEM));
240 p = alloc_pages_node(dev_to_node(dev), gfp | __GFP_ZERO, order); 240 p = alloc_pages_node(dev ? dev_to_node(dev) : NUMA_NO_NODE,
241 gfp | __GFP_ZERO, order);
241 if (!p) 242 if (!p)
242 return NULL; 243 return NULL;
243 244
diff --git a/drivers/iommu/iommu-debugfs.c b/drivers/iommu/iommu-debugfs.c
new file mode 100644
index 000000000000..3b1bf88fd1b0
--- /dev/null
+++ b/drivers/iommu/iommu-debugfs.c
@@ -0,0 +1,66 @@
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * IOMMU debugfs core infrastructure
4 *
5 * Copyright (C) 2018 Advanced Micro Devices, Inc.
6 *
7 * Author: Gary R Hook <gary.hook@amd.com>
8 */
9
10#include <linux/pci.h>
11#include <linux/iommu.h>
12#include <linux/debugfs.h>
13
14struct dentry *iommu_debugfs_dir;
15
16/**
17 * iommu_debugfs_setup - create the top-level iommu directory in debugfs
18 *
19 * Provide base enablement for using debugfs to expose internal data of an
20 * IOMMU driver. When called, this function creates the
21 * /sys/kernel/debug/iommu directory.
22 *
23 * Emit a strong warning at boot time to indicate that this feature is
24 * enabled.
25 *
26 * This function is called from iommu_init; drivers may then call
27 * iommu_debugfs_new_driver_dir() to instantiate a vendor-specific
28 * directory to be used to expose internal data.
29 */
30void iommu_debugfs_setup(void)
31{
32 if (!iommu_debugfs_dir) {
33 iommu_debugfs_dir = debugfs_create_dir("iommu", NULL);
34 pr_warn("\n");
35 pr_warn("*************************************************************\n");
36 pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
37 pr_warn("** **\n");
38 pr_warn("** IOMMU DebugFS SUPPORT HAS BEEN ENABLED IN THIS KERNEL **\n");
39 pr_warn("** **\n");
40 pr_warn("** This means that this kernel is built to expose internal **\n");
41 pr_warn("** IOMMU data structures, which may compromise security on **\n");
42 pr_warn("** your system. **\n");
43 pr_warn("** **\n");
44 pr_warn("** If you see this message and you are not debugging the **\n");
45 pr_warn("** kernel, report this immediately to your vendor! **\n");
46 pr_warn("** **\n");
47 pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
48 pr_warn("*************************************************************\n");
49 }
50}
51
52/**
53 * iommu_debugfs_new_driver_dir - create a vendor directory under debugfs/iommu
54 * @vendor: name of the vendor-specific subdirectory to create
55 *
56 * This function is called by an IOMMU driver to create the top-level debugfs
57 * directory for that driver.
58 *
59 * Return: upon success, a pointer to the dentry for the new directory.
60 * NULL in case of failure.
61 */
62struct dentry *iommu_debugfs_new_driver_dir(const char *vendor)
63{
64 return debugfs_create_dir(vendor, iommu_debugfs_dir);
65}
66EXPORT_SYMBOL_GPL(iommu_debugfs_new_driver_dir);
diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
index 63b37563db7e..8c15c5980299 100644
--- a/drivers/iommu/iommu.c
+++ b/drivers/iommu/iommu.c
@@ -36,7 +36,11 @@
36 36
37static struct kset *iommu_group_kset; 37static struct kset *iommu_group_kset;
38static DEFINE_IDA(iommu_group_ida); 38static DEFINE_IDA(iommu_group_ida);
39#ifdef CONFIG_IOMMU_DEFAULT_PASSTHROUGH
40static unsigned int iommu_def_domain_type = IOMMU_DOMAIN_IDENTITY;
41#else
39static unsigned int iommu_def_domain_type = IOMMU_DOMAIN_DMA; 42static unsigned int iommu_def_domain_type = IOMMU_DOMAIN_DMA;
43#endif
40 44
41struct iommu_callback_data { 45struct iommu_callback_data {
42 const struct iommu_ops *ops; 46 const struct iommu_ops *ops;
@@ -294,11 +298,39 @@ static ssize_t iommu_group_show_resv_regions(struct iommu_group *group,
294 return (str - buf); 298 return (str - buf);
295} 299}
296 300
301static ssize_t iommu_group_show_type(struct iommu_group *group,
302 char *buf)
303{
304 char *type = "unknown\n";
305
306 if (group->default_domain) {
307 switch (group->default_domain->type) {
308 case IOMMU_DOMAIN_BLOCKED:
309 type = "blocked\n";
310 break;
311 case IOMMU_DOMAIN_IDENTITY:
312 type = "identity\n";
313 break;
314 case IOMMU_DOMAIN_UNMANAGED:
315 type = "unmanaged\n";
316 break;
317 case IOMMU_DOMAIN_DMA:
318 type = "DMA";
319 break;
320 }
321 }
322 strcpy(buf, type);
323
324 return strlen(type);
325}
326
297static IOMMU_GROUP_ATTR(name, S_IRUGO, iommu_group_show_name, NULL); 327static IOMMU_GROUP_ATTR(name, S_IRUGO, iommu_group_show_name, NULL);
298 328
299static IOMMU_GROUP_ATTR(reserved_regions, 0444, 329static IOMMU_GROUP_ATTR(reserved_regions, 0444,
300 iommu_group_show_resv_regions, NULL); 330 iommu_group_show_resv_regions, NULL);
301 331
332static IOMMU_GROUP_ATTR(type, 0444, iommu_group_show_type, NULL);
333
302static void iommu_group_release(struct kobject *kobj) 334static void iommu_group_release(struct kobject *kobj)
303{ 335{
304 struct iommu_group *group = to_iommu_group(kobj); 336 struct iommu_group *group = to_iommu_group(kobj);
@@ -380,6 +412,10 @@ struct iommu_group *iommu_group_alloc(void)
380 if (ret) 412 if (ret)
381 return ERR_PTR(ret); 413 return ERR_PTR(ret);
382 414
415 ret = iommu_group_create_file(group, &iommu_group_attr_type);
416 if (ret)
417 return ERR_PTR(ret);
418
383 pr_debug("Allocated group %d\n", group->id); 419 pr_debug("Allocated group %d\n", group->id);
384 420
385 return group; 421 return group;
@@ -1637,8 +1673,8 @@ size_t iommu_unmap_fast(struct iommu_domain *domain,
1637} 1673}
1638EXPORT_SYMBOL_GPL(iommu_unmap_fast); 1674EXPORT_SYMBOL_GPL(iommu_unmap_fast);
1639 1675
1640size_t default_iommu_map_sg(struct iommu_domain *domain, unsigned long iova, 1676size_t iommu_map_sg(struct iommu_domain *domain, unsigned long iova,
1641 struct scatterlist *sg, unsigned int nents, int prot) 1677 struct scatterlist *sg, unsigned int nents, int prot)
1642{ 1678{
1643 struct scatterlist *s; 1679 struct scatterlist *s;
1644 size_t mapped = 0; 1680 size_t mapped = 0;
@@ -1678,7 +1714,7 @@ out_err:
1678 return 0; 1714 return 0;
1679 1715
1680} 1716}
1681EXPORT_SYMBOL_GPL(default_iommu_map_sg); 1717EXPORT_SYMBOL_GPL(iommu_map_sg);
1682 1718
1683int iommu_domain_window_enable(struct iommu_domain *domain, u32 wnd_nr, 1719int iommu_domain_window_enable(struct iommu_domain *domain, u32 wnd_nr,
1684 phys_addr_t paddr, u64 size, int prot) 1720 phys_addr_t paddr, u64 size, int prot)
@@ -1748,6 +1784,8 @@ static int __init iommu_init(void)
1748 NULL, kernel_kobj); 1784 NULL, kernel_kobj);
1749 BUG_ON(!iommu_group_kset); 1785 BUG_ON(!iommu_group_kset);
1750 1786
1787 iommu_debugfs_setup();
1788
1751 return 0; 1789 return 0;
1752} 1790}
1753core_initcall(iommu_init); 1791core_initcall(iommu_init);
diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c
index f026aa16d5f1..22b94f8a9a04 100644
--- a/drivers/iommu/ipmmu-vmsa.c
+++ b/drivers/iommu/ipmmu-vmsa.c
@@ -47,6 +47,7 @@ struct ipmmu_features {
47 unsigned int number_of_contexts; 47 unsigned int number_of_contexts;
48 bool setup_imbuscr; 48 bool setup_imbuscr;
49 bool twobit_imttbcr_sl0; 49 bool twobit_imttbcr_sl0;
50 bool reserved_context;
50}; 51};
51 52
52struct ipmmu_vmsa_device { 53struct ipmmu_vmsa_device {
@@ -73,7 +74,7 @@ struct ipmmu_vmsa_domain {
73 struct io_pgtable_ops *iop; 74 struct io_pgtable_ops *iop;
74 75
75 unsigned int context_id; 76 unsigned int context_id;
76 spinlock_t lock; /* Protects mappings */ 77 struct mutex mutex; /* Protects mappings */
77}; 78};
78 79
79static struct ipmmu_vmsa_domain *to_vmsa_domain(struct iommu_domain *dom) 80static struct ipmmu_vmsa_domain *to_vmsa_domain(struct iommu_domain *dom)
@@ -194,7 +195,9 @@ static struct ipmmu_vmsa_device *to_ipmmu(struct device *dev)
194#define IMPMBA(n) (0x0280 + ((n) * 4)) 195#define IMPMBA(n) (0x0280 + ((n) * 4))
195#define IMPMBD(n) (0x02c0 + ((n) * 4)) 196#define IMPMBD(n) (0x02c0 + ((n) * 4))
196 197
197#define IMUCTR(n) (0x0300 + ((n) * 16)) 198#define IMUCTR(n) ((n) < 32 ? IMUCTR0(n) : IMUCTR32(n))
199#define IMUCTR0(n) (0x0300 + ((n) * 16))
200#define IMUCTR32(n) (0x0600 + (((n) - 32) * 16))
198#define IMUCTR_FIXADDEN (1 << 31) 201#define IMUCTR_FIXADDEN (1 << 31)
199#define IMUCTR_FIXADD_MASK (0xff << 16) 202#define IMUCTR_FIXADD_MASK (0xff << 16)
200#define IMUCTR_FIXADD_SHIFT 16 203#define IMUCTR_FIXADD_SHIFT 16
@@ -204,7 +207,9 @@ static struct ipmmu_vmsa_device *to_ipmmu(struct device *dev)
204#define IMUCTR_FLUSH (1 << 1) 207#define IMUCTR_FLUSH (1 << 1)
205#define IMUCTR_MMUEN (1 << 0) 208#define IMUCTR_MMUEN (1 << 0)
206 209
207#define IMUASID(n) (0x0308 + ((n) * 16)) 210#define IMUASID(n) ((n) < 32 ? IMUASID0(n) : IMUASID32(n))
211#define IMUASID0(n) (0x0308 + ((n) * 16))
212#define IMUASID32(n) (0x0608 + (((n) - 32) * 16))
208#define IMUASID_ASID8_MASK (0xff << 8) 213#define IMUASID_ASID8_MASK (0xff << 8)
209#define IMUASID_ASID8_SHIFT 8 214#define IMUASID_ASID8_SHIFT 8
210#define IMUASID_ASID0_MASK (0xff << 0) 215#define IMUASID_ASID0_MASK (0xff << 0)
@@ -595,7 +600,7 @@ static struct iommu_domain *__ipmmu_domain_alloc(unsigned type)
595 if (!domain) 600 if (!domain)
596 return NULL; 601 return NULL;
597 602
598 spin_lock_init(&domain->lock); 603 mutex_init(&domain->mutex);
599 604
600 return &domain->io_domain; 605 return &domain->io_domain;
601} 606}
@@ -641,7 +646,6 @@ static int ipmmu_attach_device(struct iommu_domain *io_domain,
641 struct iommu_fwspec *fwspec = dev->iommu_fwspec; 646 struct iommu_fwspec *fwspec = dev->iommu_fwspec;
642 struct ipmmu_vmsa_device *mmu = to_ipmmu(dev); 647 struct ipmmu_vmsa_device *mmu = to_ipmmu(dev);
643 struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain); 648 struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);
644 unsigned long flags;
645 unsigned int i; 649 unsigned int i;
646 int ret = 0; 650 int ret = 0;
647 651
@@ -650,7 +654,7 @@ static int ipmmu_attach_device(struct iommu_domain *io_domain,
650 return -ENXIO; 654 return -ENXIO;
651 } 655 }
652 656
653 spin_lock_irqsave(&domain->lock, flags); 657 mutex_lock(&domain->mutex);
654 658
655 if (!domain->mmu) { 659 if (!domain->mmu) {
656 /* The domain hasn't been used yet, initialize it. */ 660 /* The domain hasn't been used yet, initialize it. */
@@ -674,7 +678,7 @@ static int ipmmu_attach_device(struct iommu_domain *io_domain,
674 } else 678 } else
675 dev_info(dev, "Reusing IPMMU context %u\n", domain->context_id); 679 dev_info(dev, "Reusing IPMMU context %u\n", domain->context_id);
676 680
677 spin_unlock_irqrestore(&domain->lock, flags); 681 mutex_unlock(&domain->mutex);
678 682
679 if (ret < 0) 683 if (ret < 0)
680 return ret; 684 return ret;
@@ -756,8 +760,12 @@ static bool ipmmu_slave_whitelist(struct device *dev)
756 return false; 760 return false;
757} 761}
758 762
759static const struct soc_device_attribute soc_r8a7795[] = { 763static const struct soc_device_attribute soc_rcar_gen3[] = {
760 { .soc_id = "r8a7795", }, 764 { .soc_id = "r8a7795", },
765 { .soc_id = "r8a7796", },
766 { .soc_id = "r8a77965", },
767 { .soc_id = "r8a77970", },
768 { .soc_id = "r8a77995", },
761 { /* sentinel */ } 769 { /* sentinel */ }
762}; 770};
763 771
@@ -765,7 +773,7 @@ static int ipmmu_of_xlate(struct device *dev,
765 struct of_phandle_args *spec) 773 struct of_phandle_args *spec)
766{ 774{
767 /* For R-Car Gen3 use a white list to opt-in slave devices */ 775 /* For R-Car Gen3 use a white list to opt-in slave devices */
768 if (soc_device_match(soc_r8a7795) && !ipmmu_slave_whitelist(dev)) 776 if (soc_device_match(soc_rcar_gen3) && !ipmmu_slave_whitelist(dev))
769 return -ENODEV; 777 return -ENODEV;
770 778
771 iommu_fwspec_add_ids(dev, spec->args, 1); 779 iommu_fwspec_add_ids(dev, spec->args, 1);
@@ -889,7 +897,6 @@ static const struct iommu_ops ipmmu_ops = {
889 .unmap = ipmmu_unmap, 897 .unmap = ipmmu_unmap,
890 .flush_iotlb_all = ipmmu_iotlb_sync, 898 .flush_iotlb_all = ipmmu_iotlb_sync,
891 .iotlb_sync = ipmmu_iotlb_sync, 899 .iotlb_sync = ipmmu_iotlb_sync,
892 .map_sg = default_iommu_map_sg,
893 .iova_to_phys = ipmmu_iova_to_phys, 900 .iova_to_phys = ipmmu_iova_to_phys,
894 .add_device = ipmmu_add_device, 901 .add_device = ipmmu_add_device,
895 .remove_device = ipmmu_remove_device, 902 .remove_device = ipmmu_remove_device,
@@ -917,14 +924,16 @@ static const struct ipmmu_features ipmmu_features_default = {
917 .number_of_contexts = 1, /* software only tested with one context */ 924 .number_of_contexts = 1, /* software only tested with one context */
918 .setup_imbuscr = true, 925 .setup_imbuscr = true,
919 .twobit_imttbcr_sl0 = false, 926 .twobit_imttbcr_sl0 = false,
927 .reserved_context = false,
920}; 928};
921 929
922static const struct ipmmu_features ipmmu_features_r8a7795 = { 930static const struct ipmmu_features ipmmu_features_rcar_gen3 = {
923 .use_ns_alias_offset = false, 931 .use_ns_alias_offset = false,
924 .has_cache_leaf_nodes = true, 932 .has_cache_leaf_nodes = true,
925 .number_of_contexts = 8, 933 .number_of_contexts = 8,
926 .setup_imbuscr = false, 934 .setup_imbuscr = false,
927 .twobit_imttbcr_sl0 = true, 935 .twobit_imttbcr_sl0 = true,
936 .reserved_context = true,
928}; 937};
929 938
930static const struct of_device_id ipmmu_of_ids[] = { 939static const struct of_device_id ipmmu_of_ids[] = {
@@ -933,7 +942,19 @@ static const struct of_device_id ipmmu_of_ids[] = {
933 .data = &ipmmu_features_default, 942 .data = &ipmmu_features_default,
934 }, { 943 }, {
935 .compatible = "renesas,ipmmu-r8a7795", 944 .compatible = "renesas,ipmmu-r8a7795",
936 .data = &ipmmu_features_r8a7795, 945 .data = &ipmmu_features_rcar_gen3,
946 }, {
947 .compatible = "renesas,ipmmu-r8a7796",
948 .data = &ipmmu_features_rcar_gen3,
949 }, {
950 .compatible = "renesas,ipmmu-r8a77965",
951 .data = &ipmmu_features_rcar_gen3,
952 }, {
953 .compatible = "renesas,ipmmu-r8a77970",
954 .data = &ipmmu_features_rcar_gen3,
955 }, {
956 .compatible = "renesas,ipmmu-r8a77995",
957 .data = &ipmmu_features_rcar_gen3,
937 }, { 958 }, {
938 /* Terminator */ 959 /* Terminator */
939 }, 960 },
@@ -955,7 +976,7 @@ static int ipmmu_probe(struct platform_device *pdev)
955 } 976 }
956 977
957 mmu->dev = &pdev->dev; 978 mmu->dev = &pdev->dev;
958 mmu->num_utlbs = 32; 979 mmu->num_utlbs = 48;
959 spin_lock_init(&mmu->lock); 980 spin_lock_init(&mmu->lock);
960 bitmap_zero(mmu->ctx, IPMMU_CTX_MAX); 981 bitmap_zero(mmu->ctx, IPMMU_CTX_MAX);
961 mmu->features = of_device_get_match_data(&pdev->dev); 982 mmu->features = of_device_get_match_data(&pdev->dev);
@@ -1018,6 +1039,11 @@ static int ipmmu_probe(struct platform_device *pdev)
1018 } 1039 }
1019 1040
1020 ipmmu_device_reset(mmu); 1041 ipmmu_device_reset(mmu);
1042
1043 if (mmu->features->reserved_context) {
1044 dev_info(&pdev->dev, "IPMMU context 0 is reserved\n");
1045 set_bit(0, mmu->ctx);
1046 }
1021 } 1047 }
1022 1048
1023 /* 1049 /*
@@ -1081,12 +1107,19 @@ static struct platform_driver ipmmu_driver = {
1081 1107
1082static int __init ipmmu_init(void) 1108static int __init ipmmu_init(void)
1083{ 1109{
1110 struct device_node *np;
1084 static bool setup_done; 1111 static bool setup_done;
1085 int ret; 1112 int ret;
1086 1113
1087 if (setup_done) 1114 if (setup_done)
1088 return 0; 1115 return 0;
1089 1116
1117 np = of_find_matching_node(NULL, ipmmu_of_ids);
1118 if (!np)
1119 return 0;
1120
1121 of_node_put(np);
1122
1090 ret = platform_driver_register(&ipmmu_driver); 1123 ret = platform_driver_register(&ipmmu_driver);
1091 if (ret < 0) 1124 if (ret < 0)
1092 return ret; 1125 return ret;
diff --git a/drivers/iommu/msm_iommu.c b/drivers/iommu/msm_iommu.c
index 27377742600d..fc5f0b53adaf 100644
--- a/drivers/iommu/msm_iommu.c
+++ b/drivers/iommu/msm_iommu.c
@@ -395,20 +395,15 @@ static int msm_iommu_add_device(struct device *dev)
395 struct msm_iommu_dev *iommu; 395 struct msm_iommu_dev *iommu;
396 struct iommu_group *group; 396 struct iommu_group *group;
397 unsigned long flags; 397 unsigned long flags;
398 int ret = 0;
399 398
400 spin_lock_irqsave(&msm_iommu_lock, flags); 399 spin_lock_irqsave(&msm_iommu_lock, flags);
401
402 iommu = find_iommu_for_dev(dev); 400 iommu = find_iommu_for_dev(dev);
401 spin_unlock_irqrestore(&msm_iommu_lock, flags);
402
403 if (iommu) 403 if (iommu)
404 iommu_device_link(&iommu->iommu, dev); 404 iommu_device_link(&iommu->iommu, dev);
405 else 405 else
406 ret = -ENODEV; 406 return -ENODEV;
407
408 spin_unlock_irqrestore(&msm_iommu_lock, flags);
409
410 if (ret)
411 return ret;
412 407
413 group = iommu_group_get_for_dev(dev); 408 group = iommu_group_get_for_dev(dev);
414 if (IS_ERR(group)) 409 if (IS_ERR(group))
@@ -425,13 +420,12 @@ static void msm_iommu_remove_device(struct device *dev)
425 unsigned long flags; 420 unsigned long flags;
426 421
427 spin_lock_irqsave(&msm_iommu_lock, flags); 422 spin_lock_irqsave(&msm_iommu_lock, flags);
428
429 iommu = find_iommu_for_dev(dev); 423 iommu = find_iommu_for_dev(dev);
424 spin_unlock_irqrestore(&msm_iommu_lock, flags);
425
430 if (iommu) 426 if (iommu)
431 iommu_device_unlink(&iommu->iommu, dev); 427 iommu_device_unlink(&iommu->iommu, dev);
432 428
433 spin_unlock_irqrestore(&msm_iommu_lock, flags);
434
435 iommu_group_remove_device(dev); 429 iommu_group_remove_device(dev);
436} 430}
437 431
@@ -708,7 +702,6 @@ static struct iommu_ops msm_iommu_ops = {
708 .detach_dev = msm_iommu_detach_dev, 702 .detach_dev = msm_iommu_detach_dev,
709 .map = msm_iommu_map, 703 .map = msm_iommu_map,
710 .unmap = msm_iommu_unmap, 704 .unmap = msm_iommu_unmap,
711 .map_sg = default_iommu_map_sg,
712 .iova_to_phys = msm_iommu_iova_to_phys, 705 .iova_to_phys = msm_iommu_iova_to_phys,
713 .add_device = msm_iommu_add_device, 706 .add_device = msm_iommu_add_device,
714 .remove_device = msm_iommu_remove_device, 707 .remove_device = msm_iommu_remove_device,
diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c
index f2832a10fcea..f9f69f7111a9 100644
--- a/drivers/iommu/mtk_iommu.c
+++ b/drivers/iommu/mtk_iommu.c
@@ -495,7 +495,6 @@ static struct iommu_ops mtk_iommu_ops = {
495 .detach_dev = mtk_iommu_detach_device, 495 .detach_dev = mtk_iommu_detach_device,
496 .map = mtk_iommu_map, 496 .map = mtk_iommu_map,
497 .unmap = mtk_iommu_unmap, 497 .unmap = mtk_iommu_unmap,
498 .map_sg = default_iommu_map_sg,
499 .flush_iotlb_all = mtk_iommu_iotlb_sync, 498 .flush_iotlb_all = mtk_iommu_iotlb_sync,
500 .iotlb_sync = mtk_iommu_iotlb_sync, 499 .iotlb_sync = mtk_iommu_iotlb_sync,
501 .iova_to_phys = mtk_iommu_iova_to_phys, 500 .iova_to_phys = mtk_iommu_iova_to_phys,
diff --git a/drivers/iommu/mtk_iommu_v1.c b/drivers/iommu/mtk_iommu_v1.c
index a7c2a973784f..676c029494e4 100644
--- a/drivers/iommu/mtk_iommu_v1.c
+++ b/drivers/iommu/mtk_iommu_v1.c
@@ -531,7 +531,6 @@ static struct iommu_ops mtk_iommu_ops = {
531 .detach_dev = mtk_iommu_detach_device, 531 .detach_dev = mtk_iommu_detach_device,
532 .map = mtk_iommu_map, 532 .map = mtk_iommu_map,
533 .unmap = mtk_iommu_unmap, 533 .unmap = mtk_iommu_unmap,
534 .map_sg = default_iommu_map_sg,
535 .iova_to_phys = mtk_iommu_iova_to_phys, 534 .iova_to_phys = mtk_iommu_iova_to_phys,
536 .add_device = mtk_iommu_add_device, 535 .add_device = mtk_iommu_add_device,
537 .remove_device = mtk_iommu_remove_device, 536 .remove_device = mtk_iommu_remove_device,
diff --git a/drivers/iommu/omap-iommu.c b/drivers/iommu/omap-iommu.c
index af4a8e7fcd27..d2fb347aa4ff 100644
--- a/drivers/iommu/omap-iommu.c
+++ b/drivers/iommu/omap-iommu.c
@@ -550,7 +550,7 @@ static u32 *iopte_alloc(struct omap_iommu *obj, u32 *iopgd,
550 550
551pte_ready: 551pte_ready:
552 iopte = iopte_offset(iopgd, da); 552 iopte = iopte_offset(iopgd, da);
553 *pt_dma = virt_to_phys(iopte); 553 *pt_dma = iopgd_page_paddr(iopgd);
554 dev_vdbg(obj->dev, 554 dev_vdbg(obj->dev,
555 "%s: da:%08x pgd:%p *pgd:%08x pte:%p *pte:%08x\n", 555 "%s: da:%08x pgd:%p *pgd:%08x pte:%p *pte:%08x\n",
556 __func__, da, iopgd, *iopgd, iopte, *iopte); 556 __func__, da, iopgd, *iopgd, iopte, *iopte);
@@ -738,7 +738,7 @@ static size_t iopgtable_clear_entry_core(struct omap_iommu *obj, u32 da)
738 } 738 }
739 bytes *= nent; 739 bytes *= nent;
740 memset(iopte, 0, nent * sizeof(*iopte)); 740 memset(iopte, 0, nent * sizeof(*iopte));
741 pt_dma = virt_to_phys(iopte); 741 pt_dma = iopgd_page_paddr(iopgd);
742 flush_iopte_range(obj->dev, pt_dma, pt_offset, nent); 742 flush_iopte_range(obj->dev, pt_dma, pt_offset, nent);
743 743
744 /* 744 /*
@@ -1548,7 +1548,6 @@ static const struct iommu_ops omap_iommu_ops = {
1548 .detach_dev = omap_iommu_detach_dev, 1548 .detach_dev = omap_iommu_detach_dev,
1549 .map = omap_iommu_map, 1549 .map = omap_iommu_map,
1550 .unmap = omap_iommu_unmap, 1550 .unmap = omap_iommu_unmap,
1551 .map_sg = default_iommu_map_sg,
1552 .iova_to_phys = omap_iommu_iova_to_phys, 1551 .iova_to_phys = omap_iommu_iova_to_phys,
1553 .add_device = omap_iommu_add_device, 1552 .add_device = omap_iommu_add_device,
1554 .remove_device = omap_iommu_remove_device, 1553 .remove_device = omap_iommu_remove_device,
diff --git a/drivers/iommu/qcom_iommu.c b/drivers/iommu/qcom_iommu.c
index b48aee82d14b..ee70e9921cf1 100644
--- a/drivers/iommu/qcom_iommu.c
+++ b/drivers/iommu/qcom_iommu.c
@@ -590,7 +590,6 @@ static const struct iommu_ops qcom_iommu_ops = {
590 .detach_dev = qcom_iommu_detach_dev, 590 .detach_dev = qcom_iommu_detach_dev,
591 .map = qcom_iommu_map, 591 .map = qcom_iommu_map,
592 .unmap = qcom_iommu_unmap, 592 .unmap = qcom_iommu_unmap,
593 .map_sg = default_iommu_map_sg,
594 .flush_iotlb_all = qcom_iommu_iotlb_sync, 593 .flush_iotlb_all = qcom_iommu_iotlb_sync,
595 .iotlb_sync = qcom_iommu_iotlb_sync, 594 .iotlb_sync = qcom_iommu_iotlb_sync,
596 .iova_to_phys = qcom_iommu_iova_to_phys, 595 .iova_to_phys = qcom_iommu_iova_to_phys,
diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c
index de8d3bf91b23..d393160e3e38 100644
--- a/drivers/iommu/rockchip-iommu.c
+++ b/drivers/iommu/rockchip-iommu.c
@@ -1110,7 +1110,6 @@ static const struct iommu_ops rk_iommu_ops = {
1110 .detach_dev = rk_iommu_detach_device, 1110 .detach_dev = rk_iommu_detach_device,
1111 .map = rk_iommu_map, 1111 .map = rk_iommu_map,
1112 .unmap = rk_iommu_unmap, 1112 .unmap = rk_iommu_unmap,
1113 .map_sg = default_iommu_map_sg,
1114 .add_device = rk_iommu_add_device, 1113 .add_device = rk_iommu_add_device,
1115 .remove_device = rk_iommu_remove_device, 1114 .remove_device = rk_iommu_remove_device,
1116 .iova_to_phys = rk_iommu_iova_to_phys, 1115 .iova_to_phys = rk_iommu_iova_to_phys,
diff --git a/drivers/iommu/tegra-gart.c b/drivers/iommu/tegra-gart.c
index a004f6da35f2..7b1361d57a17 100644
--- a/drivers/iommu/tegra-gart.c
+++ b/drivers/iommu/tegra-gart.c
@@ -377,7 +377,6 @@ static const struct iommu_ops gart_iommu_ops = {
377 .remove_device = gart_iommu_remove_device, 377 .remove_device = gart_iommu_remove_device,
378 .device_group = generic_device_group, 378 .device_group = generic_device_group,
379 .map = gart_iommu_map, 379 .map = gart_iommu_map,
380 .map_sg = default_iommu_map_sg,
381 .unmap = gart_iommu_unmap, 380 .unmap = gart_iommu_unmap,
382 .iova_to_phys = gart_iommu_iova_to_phys, 381 .iova_to_phys = gart_iommu_iova_to_phys,
383 .pgsize_bitmap = GART_IOMMU_PGSIZES, 382 .pgsize_bitmap = GART_IOMMU_PGSIZES,
diff --git a/drivers/iommu/tegra-smmu.c b/drivers/iommu/tegra-smmu.c
index 44d40bc771b5..0d03341317c4 100644
--- a/drivers/iommu/tegra-smmu.c
+++ b/drivers/iommu/tegra-smmu.c
@@ -876,7 +876,6 @@ static const struct iommu_ops tegra_smmu_ops = {
876 .device_group = tegra_smmu_device_group, 876 .device_group = tegra_smmu_device_group,
877 .map = tegra_smmu_map, 877 .map = tegra_smmu_map,
878 .unmap = tegra_smmu_unmap, 878 .unmap = tegra_smmu_unmap,
879 .map_sg = default_iommu_map_sg,
880 .iova_to_phys = tegra_smmu_iova_to_phys, 879 .iova_to_phys = tegra_smmu_iova_to_phys,
881 .of_xlate = tegra_smmu_of_xlate, 880 .of_xlate = tegra_smmu_of_xlate,
882 .pgsize_bitmap = SZ_4K, 881 .pgsize_bitmap = SZ_4K,
diff --git a/include/linux/dmar.h b/include/linux/dmar.h
index e2433bc50210..843a41ba7e28 100644
--- a/include/linux/dmar.h
+++ b/include/linux/dmar.h
@@ -265,11 +265,6 @@ static inline void dmar_copy_shared_irte(struct irte *dst, struct irte *src)
265#define PDA_LOW_BIT 26 265#define PDA_LOW_BIT 26
266#define PDA_HIGH_BIT 32 266#define PDA_HIGH_BIT 32
267 267
268enum {
269 IRQ_REMAP_XAPIC_MODE,
270 IRQ_REMAP_X2APIC_MODE,
271};
272
273/* Can't use the common MSI interrupt functions 268/* Can't use the common MSI interrupt functions
274 * since DMAR is not a pci device 269 * since DMAR is not a pci device
275 */ 270 */
diff --git a/include/linux/intel-iommu.h b/include/linux/intel-iommu.h
index ef169d67df92..28004d74ae04 100644
--- a/include/linux/intel-iommu.h
+++ b/include/linux/intel-iommu.h
@@ -31,6 +31,7 @@
31#include <linux/list.h> 31#include <linux/list.h>
32#include <linux/iommu.h> 32#include <linux/iommu.h>
33#include <linux/io-64-nonatomic-lo-hi.h> 33#include <linux/io-64-nonatomic-lo-hi.h>
34#include <linux/dmar.h>
34 35
35#include <asm/cacheflush.h> 36#include <asm/cacheflush.h>
36#include <asm/iommu.h> 37#include <asm/iommu.h>
@@ -114,6 +115,7 @@
114 * Extended Capability Register 115 * Extended Capability Register
115 */ 116 */
116 117
118#define ecap_dit(e) ((e >> 41) & 0x1)
117#define ecap_pasid(e) ((e >> 40) & 0x1) 119#define ecap_pasid(e) ((e >> 40) & 0x1)
118#define ecap_pss(e) ((e >> 35) & 0x1f) 120#define ecap_pss(e) ((e >> 35) & 0x1f)
119#define ecap_eafs(e) ((e >> 34) & 0x1) 121#define ecap_eafs(e) ((e >> 34) & 0x1)
@@ -284,6 +286,7 @@ enum {
284#define QI_DEV_IOTLB_SID(sid) ((u64)((sid) & 0xffff) << 32) 286#define QI_DEV_IOTLB_SID(sid) ((u64)((sid) & 0xffff) << 32)
285#define QI_DEV_IOTLB_QDEP(qdep) (((qdep) & 0x1f) << 16) 287#define QI_DEV_IOTLB_QDEP(qdep) (((qdep) & 0x1f) << 16)
286#define QI_DEV_IOTLB_ADDR(addr) ((u64)(addr) & VTD_PAGE_MASK) 288#define QI_DEV_IOTLB_ADDR(addr) ((u64)(addr) & VTD_PAGE_MASK)
289#define QI_DEV_IOTLB_PFSID(pfsid) (((u64)(pfsid & 0xf) << 12) | ((u64)(pfsid & 0xfff) << 52))
287#define QI_DEV_IOTLB_SIZE 1 290#define QI_DEV_IOTLB_SIZE 1
288#define QI_DEV_IOTLB_MAX_INVS 32 291#define QI_DEV_IOTLB_MAX_INVS 32
289 292
@@ -308,6 +311,7 @@ enum {
308#define QI_DEV_EIOTLB_PASID(p) (((u64)p) << 32) 311#define QI_DEV_EIOTLB_PASID(p) (((u64)p) << 32)
309#define QI_DEV_EIOTLB_SID(sid) ((u64)((sid) & 0xffff) << 16) 312#define QI_DEV_EIOTLB_SID(sid) ((u64)((sid) & 0xffff) << 16)
310#define QI_DEV_EIOTLB_QDEP(qd) ((u64)((qd) & 0x1f) << 4) 313#define QI_DEV_EIOTLB_QDEP(qd) ((u64)((qd) & 0x1f) << 4)
314#define QI_DEV_EIOTLB_PFSID(pfsid) (((u64)(pfsid & 0xf) << 12) | ((u64)(pfsid & 0xfff) << 52))
311#define QI_DEV_EIOTLB_MAX_INVS 32 315#define QI_DEV_EIOTLB_MAX_INVS 32
312 316
313#define QI_PGRP_IDX(idx) (((u64)(idx)) << 55) 317#define QI_PGRP_IDX(idx) (((u64)(idx)) << 55)
@@ -385,6 +389,42 @@ struct pasid_entry;
385struct pasid_state_entry; 389struct pasid_state_entry;
386struct page_req_dsc; 390struct page_req_dsc;
387 391
392struct dmar_domain {
393 int nid; /* node id */
394
395 unsigned iommu_refcnt[DMAR_UNITS_SUPPORTED];
396 /* Refcount of devices per iommu */
397
398
399 u16 iommu_did[DMAR_UNITS_SUPPORTED];
400 /* Domain ids per IOMMU. Use u16 since
401 * domain ids are 16 bit wide according
402 * to VT-d spec, section 9.3 */
403
404 bool has_iotlb_device;
405 struct list_head devices; /* all devices' list */
406 struct iova_domain iovad; /* iova's that belong to this domain */
407
408 struct dma_pte *pgd; /* virtual address */
409 int gaw; /* max guest address width */
410
411 /* adjusted guest address width, 0 is level 2 30-bit */
412 int agaw;
413
414 int flags; /* flags to find out type of domain */
415
416 int iommu_coherency;/* indicate coherency of iommu access */
417 int iommu_snooping; /* indicate snooping control feature*/
418 int iommu_count; /* reference count of iommu */
419 int iommu_superpage;/* Level of superpages supported:
420 0 == 4KiB (no superpages), 1 == 2MiB,
421 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
422 u64 max_addr; /* maximum mapped address */
423
424 struct iommu_domain domain; /* generic domain data structure for
425 iommu core */
426};
427
388struct intel_iommu { 428struct intel_iommu {
389 void __iomem *reg; /* Pointer to hardware regs, virtual addr */ 429 void __iomem *reg; /* Pointer to hardware regs, virtual addr */
390 u64 reg_phys; /* physical address of hw register set */ 430 u64 reg_phys; /* physical address of hw register set */
@@ -414,11 +454,9 @@ struct intel_iommu {
414 * devices away to userspace processes (e.g. for DPDK) and don't 454 * devices away to userspace processes (e.g. for DPDK) and don't
415 * want to trust that userspace will use *only* the PASID it was 455 * want to trust that userspace will use *only* the PASID it was
416 * told to. But while it's all driver-arbitrated, we're fine. */ 456 * told to. But while it's all driver-arbitrated, we're fine. */
417 struct pasid_entry *pasid_table;
418 struct pasid_state_entry *pasid_state_table; 457 struct pasid_state_entry *pasid_state_table;
419 struct page_req_dsc *prq; 458 struct page_req_dsc *prq;
420 unsigned char prq_name[16]; /* Name for PRQ interrupt */ 459 unsigned char prq_name[16]; /* Name for PRQ interrupt */
421 struct idr pasid_idr;
422 u32 pasid_max; 460 u32 pasid_max;
423#endif 461#endif
424 struct q_inval *qi; /* Queued invalidation info */ 462 struct q_inval *qi; /* Queued invalidation info */
@@ -434,6 +472,27 @@ struct intel_iommu {
434 u32 flags; /* Software defined flags */ 472 u32 flags; /* Software defined flags */
435}; 473};
436 474
475/* PCI domain-device relationship */
476struct device_domain_info {
477 struct list_head link; /* link to domain siblings */
478 struct list_head global; /* link to global list */
479 struct list_head table; /* link to pasid table */
480 u8 bus; /* PCI bus number */
481 u8 devfn; /* PCI devfn number */
482 u16 pfsid; /* SRIOV physical function source ID */
483 u8 pasid_supported:3;
484 u8 pasid_enabled:1;
485 u8 pri_supported:1;
486 u8 pri_enabled:1;
487 u8 ats_supported:1;
488 u8 ats_enabled:1;
489 u8 ats_qdep;
490 struct device *dev; /* it's NULL for PCIe-to-PCI bridge */
491 struct intel_iommu *iommu; /* IOMMU used by this device */
492 struct dmar_domain *domain; /* pointer to domain */
493 struct pasid_table *pasid_table; /* pasid table */
494};
495
437static inline void __iommu_flush_cache( 496static inline void __iommu_flush_cache(
438 struct intel_iommu *iommu, void *addr, int size) 497 struct intel_iommu *iommu, void *addr, int size)
439{ 498{
@@ -453,16 +512,22 @@ extern void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid,
453 u8 fm, u64 type); 512 u8 fm, u64 type);
454extern void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr, 513extern void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
455 unsigned int size_order, u64 type); 514 unsigned int size_order, u64 type);
456extern void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 qdep, 515extern void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid,
457 u64 addr, unsigned mask); 516 u16 qdep, u64 addr, unsigned mask);
458
459extern int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu); 517extern int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu);
460 518
461extern int dmar_ir_support(void); 519extern int dmar_ir_support(void);
462 520
521struct dmar_domain *get_valid_domain_for_dev(struct device *dev);
522void *alloc_pgtable_page(int node);
523void free_pgtable_page(void *vaddr);
524struct intel_iommu *domain_get_iommu(struct dmar_domain *domain);
525int for_each_device_domain(int (*fn)(struct device_domain_info *info,
526 void *data), void *data);
527
463#ifdef CONFIG_INTEL_IOMMU_SVM 528#ifdef CONFIG_INTEL_IOMMU_SVM
464extern int intel_svm_alloc_pasid_tables(struct intel_iommu *iommu); 529int intel_svm_init(struct intel_iommu *iommu);
465extern int intel_svm_free_pasid_tables(struct intel_iommu *iommu); 530int intel_svm_exit(struct intel_iommu *iommu);
466extern int intel_svm_enable_prq(struct intel_iommu *iommu); 531extern int intel_svm_enable_prq(struct intel_iommu *iommu);
467extern int intel_svm_finish_prq(struct intel_iommu *iommu); 532extern int intel_svm_finish_prq(struct intel_iommu *iommu);
468 533
@@ -486,6 +551,7 @@ struct intel_svm {
486 int flags; 551 int flags;
487 int pasid; 552 int pasid;
488 struct list_head devs; 553 struct list_head devs;
554 struct list_head list;
489}; 555};
490 556
491extern int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct intel_svm_dev *sdev); 557extern int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct intel_svm_dev *sdev);
diff --git a/include/linux/iommu.h b/include/linux/iommu.h
index 19938ee6eb31..87994c265bf5 100644
--- a/include/linux/iommu.h
+++ b/include/linux/iommu.h
@@ -166,8 +166,6 @@ struct iommu_resv_region {
166 * @detach_dev: detach device from an iommu domain 166 * @detach_dev: detach device from an iommu domain
167 * @map: map a physically contiguous memory region to an iommu domain 167 * @map: map a physically contiguous memory region to an iommu domain
168 * @unmap: unmap a physically contiguous memory region from an iommu domain 168 * @unmap: unmap a physically contiguous memory region from an iommu domain
169 * @map_sg: map a scatter-gather list of physically contiguous memory chunks
170 * to an iommu domain
171 * @flush_tlb_all: Synchronously flush all hardware TLBs for this domain 169 * @flush_tlb_all: Synchronously flush all hardware TLBs for this domain
172 * @tlb_range_add: Add a given iova range to the flush queue for this domain 170 * @tlb_range_add: Add a given iova range to the flush queue for this domain
173 * @tlb_sync: Flush all queued ranges from the hardware TLBs and empty flush 171 * @tlb_sync: Flush all queued ranges from the hardware TLBs and empty flush
@@ -201,8 +199,6 @@ struct iommu_ops {
201 phys_addr_t paddr, size_t size, int prot); 199 phys_addr_t paddr, size_t size, int prot);
202 size_t (*unmap)(struct iommu_domain *domain, unsigned long iova, 200 size_t (*unmap)(struct iommu_domain *domain, unsigned long iova,
203 size_t size); 201 size_t size);
204 size_t (*map_sg)(struct iommu_domain *domain, unsigned long iova,
205 struct scatterlist *sg, unsigned int nents, int prot);
206 void (*flush_iotlb_all)(struct iommu_domain *domain); 202 void (*flush_iotlb_all)(struct iommu_domain *domain);
207 void (*iotlb_range_add)(struct iommu_domain *domain, 203 void (*iotlb_range_add)(struct iommu_domain *domain,
208 unsigned long iova, size_t size); 204 unsigned long iova, size_t size);
@@ -303,9 +299,8 @@ extern size_t iommu_unmap(struct iommu_domain *domain, unsigned long iova,
303 size_t size); 299 size_t size);
304extern size_t iommu_unmap_fast(struct iommu_domain *domain, 300extern size_t iommu_unmap_fast(struct iommu_domain *domain,
305 unsigned long iova, size_t size); 301 unsigned long iova, size_t size);
306extern size_t default_iommu_map_sg(struct iommu_domain *domain, unsigned long iova, 302extern size_t iommu_map_sg(struct iommu_domain *domain, unsigned long iova,
307 struct scatterlist *sg,unsigned int nents, 303 struct scatterlist *sg,unsigned int nents, int prot);
308 int prot);
309extern phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova); 304extern phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova);
310extern void iommu_set_fault_handler(struct iommu_domain *domain, 305extern void iommu_set_fault_handler(struct iommu_domain *domain,
311 iommu_fault_handler_t handler, void *token); 306 iommu_fault_handler_t handler, void *token);
@@ -378,13 +373,6 @@ static inline void iommu_tlb_sync(struct iommu_domain *domain)
378 domain->ops->iotlb_sync(domain); 373 domain->ops->iotlb_sync(domain);
379} 374}
380 375
381static inline size_t iommu_map_sg(struct iommu_domain *domain,
382 unsigned long iova, struct scatterlist *sg,
383 unsigned int nents, int prot)
384{
385 return domain->ops->map_sg(domain, iova, sg, nents, prot);
386}
387
388/* PCI device grouping function */ 376/* PCI device grouping function */
389extern struct iommu_group *pci_device_group(struct device *dev); 377extern struct iommu_group *pci_device_group(struct device *dev);
390/* Generic device grouping function */ 378/* Generic device grouping function */
@@ -698,4 +686,11 @@ const struct iommu_ops *iommu_ops_from_fwnode(struct fwnode_handle *fwnode)
698 686
699#endif /* CONFIG_IOMMU_API */ 687#endif /* CONFIG_IOMMU_API */
700 688
689#ifdef CONFIG_IOMMU_DEBUGFS
690extern struct dentry *iommu_debugfs_dir;
691void iommu_debugfs_setup(void);
692#else
693static inline void iommu_debugfs_setup(void) {}
694#endif
695
701#endif /* __LINUX_IOMMU_H */ 696#endif /* __LINUX_IOMMU_H */
diff --git a/include/linux/platform_data/sh_ipmmu.h b/include/linux/platform_data/sh_ipmmu.h
deleted file mode 100644
index 39f7405cdac5..000000000000
--- a/include/linux/platform_data/sh_ipmmu.h
+++ /dev/null
@@ -1,18 +0,0 @@
1/* sh_ipmmu.h
2 *
3 * Copyright (C) 2012 Hideki EIRAKU
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; version 2 of the License.
8 */
9
10#ifndef __SH_IPMMU_H__
11#define __SH_IPMMU_H__
12
13struct shmobile_ipmmu_platform_data {
14 const char * const *dev_names;
15 unsigned int num_dev_names;
16};
17
18#endif /* __SH_IPMMU_H__ */
generated by cgit v1.2.2 (git 2.25.0) at 2025-05-23 17:14:57 -0400