summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2019-05-13 09:23:18 -0400
committerLinus Torvalds <torvalds@linux-foundation.org>2019-05-13 09:23:18 -0400
commita13f0655503a4a89df67fdc7cac6a7810795d4b3 (patch)
tree116a389d382f7ae1e1c4b83dd987cd3ade59defc
parent55472bae5331f33582d9f0e8919fed8bebcda0da (diff)
parentb5531563e8a0b8fcc5344a38d1fad9217e08e09b (diff)
Merge tag 'iommu-updates-v5.2' of ssh://gitolite.kernel.org/pub/scm/linux/kernel/git/joro/iommu
Pull IOMMU updates from Joerg Roedel: - ATS support for ARM-SMMU-v3. - AUX domain support in the IOMMU-API and the Intel VT-d driver. This adds support for multiple DMA address spaces per (PCI-)device. The use-case is to multiplex devices between host and KVM guests in a more flexible way than supported by SR-IOV. - the rest are smaller cleanups and fixes, two of which needed to be reverted after testing in linux-next. * tag 'iommu-updates-v5.2' of ssh://gitolite.kernel.org/pub/scm/linux/kernel/git/joro/iommu: (45 commits) Revert "iommu/amd: Flush not present cache in iommu_map_page" Revert "iommu/amd: Remove the leftover of bypass support" iommu/vt-d: Fix leak in intel_pasid_alloc_table on error path iommu/vt-d: Make kernel parameter igfx_off work with vIOMMU iommu/vt-d: Set intel_iommu_gfx_mapped correctly iommu/amd: Flush not present cache in iommu_map_page iommu/vt-d: Cleanup: no spaces at the start of a line iommu/vt-d: Don't request page request irq under dmar_global_lock iommu/vt-d: Use struct_size() helper iommu/mediatek: Fix leaked of_node references iommu/amd: Remove amd_iommu_pd_list iommu/arm-smmu: Log CBFRSYNRA register on context fault iommu/arm-smmu-v3: Don't disable SMMU in kdump kernel iommu/arm-smmu-v3: Disable tagged pointers iommu/arm-smmu-v3: Add support for PCI ATS iommu/arm-smmu-v3: Link domains and devices iommu/arm-smmu-v3: Add a master->domain pointer iommu/arm-smmu-v3: Store SteamIDs in master iommu/arm-smmu-v3: Rename arm_smmu_master_data to arm_smmu_master ACPI/IORT: Check ATS capability in root complex nodes ...
Diffstat
-rw-r--r--drivers/acpi/arm64/iort.c11
-rw-r--r--drivers/iommu/Kconfig25
-rw-r--r--drivers/iommu/amd_iommu.c52
-rw-r--r--drivers/iommu/amd_iommu_init.c8
-rw-r--r--drivers/iommu/amd_iommu_types.h6
-rw-r--r--drivers/iommu/arm-smmu-regs.h2
-rw-r--r--drivers/iommu/arm-smmu-v3.c355
-rw-r--r--drivers/iommu/arm-smmu.c11
-rw-r--r--drivers/iommu/dmar.c2
-rw-r--r--drivers/iommu/intel-iommu.c584
-rw-r--r--drivers/iommu/intel-pasid.c4
-rw-r--r--drivers/iommu/intel-svm.c19
-rw-r--r--drivers/iommu/intel_irq_remapping.c7
-rw-r--r--drivers/iommu/iommu.c211
-rw-r--r--drivers/iommu/mtk_iommu.c8
-rw-r--r--drivers/iommu/tegra-smmu.c41
-rw-r--r--drivers/vfio/mdev/mdev_core.c18
-rw-r--r--drivers/vfio/mdev/mdev_private.h1
-rw-r--r--drivers/vfio/vfio_iommu_type1.c139
-rw-r--r--include/linux/intel-iommu.h13
-rw-r--r--include/linux/iommu.h144
-rw-r--r--include/linux/iova.h16
-rw-r--r--include/linux/mdev.h14
-rw-r--r--include/linux/pci.h31
24 files changed, 1376 insertions, 346 deletions
diff --git a/drivers/acpi/arm64/iort.c b/drivers/acpi/arm64/iort.c
index adbf7cbedf80..9058cb084b91 100644
--- a/drivers/acpi/arm64/iort.c
+++ b/drivers/acpi/arm64/iort.c
@@ -1031,6 +1031,14 @@ void iort_dma_setup(struct device *dev, u64 *dma_addr, u64 *dma_size)
1031 dev_dbg(dev, "dma_pfn_offset(%#08llx)\n", offset); 1031 dev_dbg(dev, "dma_pfn_offset(%#08llx)\n", offset);
1032} 1032}
1033 1033
1034static bool iort_pci_rc_supports_ats(struct acpi_iort_node *node)
1035{
1036 struct acpi_iort_root_complex *pci_rc;
1037
1038 pci_rc = (struct acpi_iort_root_complex *)node->node_data;
1039 return pci_rc->ats_attribute & ACPI_IORT_ATS_SUPPORTED;
1040}
1041
1034/** 1042/**
1035 * iort_iommu_configure - Set-up IOMMU configuration for a device. 1043 * iort_iommu_configure - Set-up IOMMU configuration for a device.
1036 * 1044 *
@@ -1066,6 +1074,9 @@ const struct iommu_ops *iort_iommu_configure(struct device *dev)
1066 info.node = node; 1074 info.node = node;
1067 err = pci_for_each_dma_alias(to_pci_dev(dev), 1075 err = pci_for_each_dma_alias(to_pci_dev(dev),
1068 iort_pci_iommu_init, &info); 1076 iort_pci_iommu_init, &info);
1077
1078 if (!err && iort_pci_rc_supports_ats(node))
1079 dev->iommu_fwspec->flags |= IOMMU_FWSPEC_PCI_RC_ATS;
1069 } else { 1080 } else {
1070 int i = 0; 1081 int i = 0;
1071 1082
diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig
index 6f07f3b21816..15b831113ded 100644
--- a/drivers/iommu/Kconfig
+++ b/drivers/iommu/Kconfig
@@ -359,6 +359,31 @@ config ARM_SMMU
359 Say Y here if your SoC includes an IOMMU device implementing 359 Say Y here if your SoC includes an IOMMU device implementing
360 the ARM SMMU architecture. 360 the ARM SMMU architecture.
361 361
362config ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT
363 bool "Default to disabling bypass on ARM SMMU v1 and v2"
364 depends on ARM_SMMU
365 default y
366 help
367 Say Y here to (by default) disable bypass streams such that
368 incoming transactions from devices that are not attached to
369 an iommu domain will report an abort back to the device and
370 will not be allowed to pass through the SMMU.
371
372 Any old kernels that existed before this KConfig was
373 introduced would default to _allowing_ bypass (AKA the
374 equivalent of NO for this config). However the default for
375 this option is YES because the old behavior is insecure.
376
377 There are few reasons to allow unmatched stream bypass, and
378 even fewer good ones. If saying YES here breaks your board
379 you should work on fixing your board. This KConfig option
380 is expected to be removed in the future and we'll simply
381 hardcode the bypass disable in the code.
382
383 NOTE: the kernel command line parameter
384 'arm-smmu.disable_bypass' will continue to override this
385 config.
386
362config ARM_SMMU_V3 387config ARM_SMMU_V3
363 bool "ARM Ltd. System MMU Version 3 (SMMUv3) Support" 388 bool "ARM Ltd. System MMU Version 3 (SMMUv3) Support"
364 depends on ARM64 389 depends on ARM64
diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c
index f7cdd2ab7f11..fde16c5b0a70 100644
--- a/drivers/iommu/amd_iommu.c
+++ b/drivers/iommu/amd_iommu.c
@@ -1723,31 +1723,6 @@ static void dma_ops_free_iova(struct dma_ops_domain *dma_dom,
1723 * 1723 *
1724 ****************************************************************************/ 1724 ****************************************************************************/
1725 1725
1726/*
1727 * This function adds a protection domain to the global protection domain list
1728 */
1729static void add_domain_to_list(struct protection_domain *domain)
1730{
1731 unsigned long flags;
1732
1733 spin_lock_irqsave(&amd_iommu_pd_lock, flags);
1734 list_add(&domain->list, &amd_iommu_pd_list);
1735 spin_unlock_irqrestore(&amd_iommu_pd_lock, flags);
1736}
1737
1738/*
1739 * This function removes a protection domain to the global
1740 * protection domain list
1741 */
1742static void del_domain_from_list(struct protection_domain *domain)
1743{
1744 unsigned long flags;
1745
1746 spin_lock_irqsave(&amd_iommu_pd_lock, flags);
1747 list_del(&domain->list);
1748 spin_unlock_irqrestore(&amd_iommu_pd_lock, flags);
1749}
1750
1751static u16 domain_id_alloc(void) 1726static u16 domain_id_alloc(void)
1752{ 1727{
1753 int id; 1728 int id;
@@ -1838,8 +1813,6 @@ static void dma_ops_domain_free(struct dma_ops_domain *dom)
1838 if (!dom) 1813 if (!dom)
1839 return; 1814 return;
1840 1815
1841 del_domain_from_list(&dom->domain);
1842
1843 put_iova_domain(&dom->iovad); 1816 put_iova_domain(&dom->iovad);
1844 1817
1845 free_pagetable(&dom->domain); 1818 free_pagetable(&dom->domain);
@@ -1880,8 +1853,6 @@ static struct dma_ops_domain *dma_ops_domain_alloc(void)
1880 /* Initialize reserved ranges */ 1853 /* Initialize reserved ranges */
1881 copy_reserved_iova(&reserved_iova_ranges, &dma_dom->iovad); 1854 copy_reserved_iova(&reserved_iova_ranges, &dma_dom->iovad);
1882 1855
1883 add_domain_to_list(&dma_dom->domain);
1884
1885 return dma_dom; 1856 return dma_dom;
1886 1857
1887free_dma_dom: 1858free_dma_dom:
@@ -2122,23 +2093,6 @@ out_err:
2122 return ret; 2093 return ret;
2123} 2094}
2124 2095
2125/* FIXME: Move this to PCI code */
2126#define PCI_PRI_TLP_OFF (1 << 15)
2127
2128static bool pci_pri_tlp_required(struct pci_dev *pdev)
2129{
2130 u16 status;
2131 int pos;
2132
2133 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
2134 if (!pos)
2135 return false;
2136
2137 pci_read_config_word(pdev, pos + PCI_PRI_STATUS, &status);
2138
2139 return (status & PCI_PRI_TLP_OFF) ? true : false;
2140}
2141
2142/* 2096/*
2143 * If a device is not yet associated with a domain, this function makes the 2097 * If a device is not yet associated with a domain, this function makes the
2144 * device visible in the domain 2098 * device visible in the domain
@@ -2167,7 +2121,7 @@ static int attach_device(struct device *dev,
2167 2121
2168 dev_data->ats.enabled = true; 2122 dev_data->ats.enabled = true;
2169 dev_data->ats.qdep = pci_ats_queue_depth(pdev); 2123 dev_data->ats.qdep = pci_ats_queue_depth(pdev);
2170 dev_data->pri_tlp = pci_pri_tlp_required(pdev); 2124 dev_data->pri_tlp = pci_prg_resp_pasid_required(pdev);
2171 } 2125 }
2172 } else if (amd_iommu_iotlb_sup && 2126 } else if (amd_iommu_iotlb_sup &&
2173 pci_enable_ats(pdev, PAGE_SHIFT) == 0) { 2127 pci_enable_ats(pdev, PAGE_SHIFT) == 0) {
@@ -2897,8 +2851,6 @@ static void protection_domain_free(struct protection_domain *domain)
2897 if (!domain) 2851 if (!domain)
2898 return; 2852 return;
2899 2853
2900 del_domain_from_list(domain);
2901
2902 if (domain->id) 2854 if (domain->id)
2903 domain_id_free(domain->id); 2855 domain_id_free(domain->id);
2904 2856
@@ -2928,8 +2880,6 @@ static struct protection_domain *protection_domain_alloc(void)
2928 if (protection_domain_init(domain)) 2880 if (protection_domain_init(domain))
2929 goto out_err; 2881 goto out_err;
2930 2882
2931 add_domain_to_list(domain);
2932
2933 return domain; 2883 return domain;
2934 2884
2935out_err: 2885out_err:
diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c
index ff40ba758cf3..f977df90d2a4 100644
--- a/drivers/iommu/amd_iommu_init.c
+++ b/drivers/iommu/amd_iommu_init.c
@@ -189,12 +189,6 @@ static bool amd_iommu_pc_present __read_mostly;
189bool amd_iommu_force_isolation __read_mostly; 189bool amd_iommu_force_isolation __read_mostly;
190 190
191/* 191/*
192 * List of protection domains - used during resume
193 */
194LIST_HEAD(amd_iommu_pd_list);
195spinlock_t amd_iommu_pd_lock;
196
197/*
198 * Pointer to the device table which is shared by all AMD IOMMUs 192 * Pointer to the device table which is shared by all AMD IOMMUs
199 * it is indexed by the PCI device id or the HT unit id and contains 193 * it is indexed by the PCI device id or the HT unit id and contains
200 * information about the domain the device belongs to as well as the 194 * information about the domain the device belongs to as well as the
@@ -2526,8 +2520,6 @@ static int __init early_amd_iommu_init(void)
2526 */ 2520 */
2527 __set_bit(0, amd_iommu_pd_alloc_bitmap); 2521 __set_bit(0, amd_iommu_pd_alloc_bitmap);
2528 2522
2529 spin_lock_init(&amd_iommu_pd_lock);
2530
2531 /* 2523 /*
2532 * now the data structures are allocated and basically initialized 2524 * now the data structures are allocated and basically initialized
2533 * start the real acpi table scan 2525 * start the real acpi table scan
diff --git a/drivers/iommu/amd_iommu_types.h b/drivers/iommu/amd_iommu_types.h
index 87965e4d9647..85c488b8daea 100644
--- a/drivers/iommu/amd_iommu_types.h
+++ b/drivers/iommu/amd_iommu_types.h
@@ -675,12 +675,6 @@ extern struct list_head amd_iommu_list;
675extern struct amd_iommu *amd_iommus[MAX_IOMMUS]; 675extern struct amd_iommu *amd_iommus[MAX_IOMMUS];
676 676
677/* 677/*
678 * Declarations for the global list of all protection domains
679 */
680extern spinlock_t amd_iommu_pd_lock;
681extern struct list_head amd_iommu_pd_list;
682
683/*
684 * Structure defining one entry in the device table 678 * Structure defining one entry in the device table
685 */ 679 */
686struct dev_table_entry { 680struct dev_table_entry {
diff --git a/drivers/iommu/arm-smmu-regs.h b/drivers/iommu/arm-smmu-regs.h
index a1226e4ab5f8..e9132a926761 100644
--- a/drivers/iommu/arm-smmu-regs.h
+++ b/drivers/iommu/arm-smmu-regs.h
@@ -147,6 +147,8 @@ enum arm_smmu_s2cr_privcfg {
147#define CBAR_IRPTNDX_SHIFT 24 147#define CBAR_IRPTNDX_SHIFT 24
148#define CBAR_IRPTNDX_MASK 0xff 148#define CBAR_IRPTNDX_MASK 0xff
149 149
150#define ARM_SMMU_GR1_CBFRSYNRA(n) (0x400 + ((n) << 2))
151
150#define ARM_SMMU_GR1_CBA2R(n) (0x800 + ((n) << 2)) 152#define ARM_SMMU_GR1_CBA2R(n) (0x800 + ((n) << 2))
151#define CBA2R_RW64_32BIT (0 << 0) 153#define CBA2R_RW64_32BIT (0 << 0)
152#define CBA2R_RW64_64BIT (1 << 0) 154#define CBA2R_RW64_64BIT (1 << 0)
diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c
index d3880010c6cf..4d5a694f02c2 100644
--- a/drivers/iommu/arm-smmu-v3.c
+++ b/drivers/iommu/arm-smmu-v3.c
@@ -29,6 +29,7 @@
29#include <linux/of_iommu.h> 29#include <linux/of_iommu.h>
30#include <linux/of_platform.h> 30#include <linux/of_platform.h>
31#include <linux/pci.h> 31#include <linux/pci.h>
32#include <linux/pci-ats.h>
32#include <linux/platform_device.h> 33#include <linux/platform_device.h>
33 34
34#include <linux/amba/bus.h> 35#include <linux/amba/bus.h>
@@ -86,6 +87,7 @@
86#define IDR5_VAX_52_BIT 1 87#define IDR5_VAX_52_BIT 1
87 88
88#define ARM_SMMU_CR0 0x20 89#define ARM_SMMU_CR0 0x20
90#define CR0_ATSCHK (1 << 4)
89#define CR0_CMDQEN (1 << 3) 91#define CR0_CMDQEN (1 << 3)
90#define CR0_EVTQEN (1 << 2) 92#define CR0_EVTQEN (1 << 2)
91#define CR0_PRIQEN (1 << 1) 93#define CR0_PRIQEN (1 << 1)
@@ -294,6 +296,7 @@
294#define CMDQ_ERR_CERROR_NONE_IDX 0 296#define CMDQ_ERR_CERROR_NONE_IDX 0
295#define CMDQ_ERR_CERROR_ILL_IDX 1 297#define CMDQ_ERR_CERROR_ILL_IDX 1
296#define CMDQ_ERR_CERROR_ABT_IDX 2 298#define CMDQ_ERR_CERROR_ABT_IDX 2
299#define CMDQ_ERR_CERROR_ATC_INV_IDX 3
297 300
298#define CMDQ_0_OP GENMASK_ULL(7, 0) 301#define CMDQ_0_OP GENMASK_ULL(7, 0)
299#define CMDQ_0_SSV (1UL << 11) 302#define CMDQ_0_SSV (1UL << 11)
@@ -312,6 +315,12 @@
312#define CMDQ_TLBI_1_VA_MASK GENMASK_ULL(63, 12) 315#define CMDQ_TLBI_1_VA_MASK GENMASK_ULL(63, 12)
313#define CMDQ_TLBI_1_IPA_MASK GENMASK_ULL(51, 12) 316#define CMDQ_TLBI_1_IPA_MASK GENMASK_ULL(51, 12)
314 317
318#define CMDQ_ATC_0_SSID GENMASK_ULL(31, 12)
319#define CMDQ_ATC_0_SID GENMASK_ULL(63, 32)
320#define CMDQ_ATC_0_GLOBAL (1UL << 9)
321#define CMDQ_ATC_1_SIZE GENMASK_ULL(5, 0)
322#define CMDQ_ATC_1_ADDR_MASK GENMASK_ULL(63, 12)
323
315#define CMDQ_PRI_0_SSID GENMASK_ULL(31, 12) 324#define CMDQ_PRI_0_SSID GENMASK_ULL(31, 12)
316#define CMDQ_PRI_0_SID GENMASK_ULL(63, 32) 325#define CMDQ_PRI_0_SID GENMASK_ULL(63, 32)
317#define CMDQ_PRI_1_GRPID GENMASK_ULL(8, 0) 326#define CMDQ_PRI_1_GRPID GENMASK_ULL(8, 0)
@@ -433,6 +442,16 @@ struct arm_smmu_cmdq_ent {
433 u64 addr; 442 u64 addr;
434 } tlbi; 443 } tlbi;
435 444
445 #define CMDQ_OP_ATC_INV 0x40
446 #define ATC_INV_SIZE_ALL 52
447 struct {
448 u32 sid;
449 u32 ssid;
450 u64 addr;
451 u8 size;
452 bool global;
453 } atc;
454
436 #define CMDQ_OP_PRI_RESP 0x41 455 #define CMDQ_OP_PRI_RESP 0x41
437 struct { 456 struct {
438 u32 sid; 457 u32 sid;
@@ -505,19 +524,6 @@ struct arm_smmu_s2_cfg {
505 u64 vtcr; 524 u64 vtcr;
506}; 525};
507 526
508struct arm_smmu_strtab_ent {
509 /*
510 * An STE is "assigned" if the master emitting the corresponding SID
511 * is attached to a domain. The behaviour of an unassigned STE is
512 * determined by the disable_bypass parameter, whereas an assigned
513 * STE behaves according to s1_cfg/s2_cfg, which themselves are
514 * configured according to the domain type.
515 */
516 bool assigned;
517 struct arm_smmu_s1_cfg *s1_cfg;
518 struct arm_smmu_s2_cfg *s2_cfg;
519};
520
521struct arm_smmu_strtab_cfg { 527struct arm_smmu_strtab_cfg {
522 __le64 *strtab; 528 __le64 *strtab;
523 dma_addr_t strtab_dma; 529 dma_addr_t strtab_dma;
@@ -591,9 +597,14 @@ struct arm_smmu_device {
591}; 597};
592 598
593/* SMMU private data for each master */ 599/* SMMU private data for each master */
594struct arm_smmu_master_data { 600struct arm_smmu_master {
595 struct arm_smmu_device *smmu; 601 struct arm_smmu_device *smmu;
596 struct arm_smmu_strtab_ent ste; 602 struct device *dev;
603 struct arm_smmu_domain *domain;
604 struct list_head domain_head;
605 u32 *sids;
606 unsigned int num_sids;
607 bool ats_enabled :1;
597}; 608};
598 609
599/* SMMU private data for an IOMMU domain */ 610/* SMMU private data for an IOMMU domain */
@@ -618,6 +629,9 @@ struct arm_smmu_domain {
618 }; 629 };
619 630
620 struct iommu_domain domain; 631 struct iommu_domain domain;
632
633 struct list_head devices;
634 spinlock_t devices_lock;
621}; 635};
622 636
623struct arm_smmu_option_prop { 637struct arm_smmu_option_prop {
@@ -820,6 +834,14 @@ static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct arm_smmu_cmdq_ent *ent)
820 case CMDQ_OP_TLBI_S12_VMALL: 834 case CMDQ_OP_TLBI_S12_VMALL:
821 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid); 835 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
822 break; 836 break;
837 case CMDQ_OP_ATC_INV:
838 cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid);
839 cmd[0] |= FIELD_PREP(CMDQ_ATC_0_GLOBAL, ent->atc.global);
840 cmd[0] |= FIELD_PREP(CMDQ_ATC_0_SSID, ent->atc.ssid);
841 cmd[0] |= FIELD_PREP(CMDQ_ATC_0_SID, ent->atc.sid);
842 cmd[1] |= FIELD_PREP(CMDQ_ATC_1_SIZE, ent->atc.size);
843 cmd[1] |= ent->atc.addr & CMDQ_ATC_1_ADDR_MASK;
844 break;
823 case CMDQ_OP_PRI_RESP: 845 case CMDQ_OP_PRI_RESP:
824 cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid); 846 cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid);
825 cmd[0] |= FIELD_PREP(CMDQ_PRI_0_SSID, ent->pri.ssid); 847 cmd[0] |= FIELD_PREP(CMDQ_PRI_0_SSID, ent->pri.ssid);
@@ -864,6 +886,7 @@ static void arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu)
864 [CMDQ_ERR_CERROR_NONE_IDX] = "No error", 886 [CMDQ_ERR_CERROR_NONE_IDX] = "No error",
865 [CMDQ_ERR_CERROR_ILL_IDX] = "Illegal command", 887 [CMDQ_ERR_CERROR_ILL_IDX] = "Illegal command",
866 [CMDQ_ERR_CERROR_ABT_IDX] = "Abort on command fetch", 888 [CMDQ_ERR_CERROR_ABT_IDX] = "Abort on command fetch",
889 [CMDQ_ERR_CERROR_ATC_INV_IDX] = "ATC invalidate timeout",
867 }; 890 };
868 891
869 int i; 892 int i;
@@ -883,6 +906,14 @@ static void arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu)
883 dev_err(smmu->dev, "retrying command fetch\n"); 906 dev_err(smmu->dev, "retrying command fetch\n");
884 case CMDQ_ERR_CERROR_NONE_IDX: 907 case CMDQ_ERR_CERROR_NONE_IDX:
885 return; 908 return;
909 case CMDQ_ERR_CERROR_ATC_INV_IDX:
910 /*
911 * ATC Invalidation Completion timeout. CONS is still pointing
912 * at the CMD_SYNC. Attempt to complete other pending commands
913 * by repeating the CMD_SYNC, though we might well end up back
914 * here since the ATC invalidation may still be pending.
915 */
916 return;
886 case CMDQ_ERR_CERROR_ILL_IDX: 917 case CMDQ_ERR_CERROR_ILL_IDX:
887 /* Fallthrough */ 918 /* Fallthrough */
888 default: 919 default:
@@ -999,7 +1030,7 @@ static int __arm_smmu_cmdq_issue_sync(struct arm_smmu_device *smmu)
999 return ret; 1030 return ret;
1000} 1031}
1001 1032
1002static void arm_smmu_cmdq_issue_sync(struct arm_smmu_device *smmu) 1033static int arm_smmu_cmdq_issue_sync(struct arm_smmu_device *smmu)
1003{ 1034{
1004 int ret; 1035 int ret;
1005 bool msi = (smmu->features & ARM_SMMU_FEAT_MSI) && 1036 bool msi = (smmu->features & ARM_SMMU_FEAT_MSI) &&
@@ -1009,6 +1040,7 @@ static void arm_smmu_cmdq_issue_sync(struct arm_smmu_device *smmu)
1009 : __arm_smmu_cmdq_issue_sync(smmu); 1040 : __arm_smmu_cmdq_issue_sync(smmu);
1010 if (ret) 1041 if (ret)
1011 dev_err_ratelimited(smmu->dev, "CMD_SYNC timeout\n"); 1042 dev_err_ratelimited(smmu->dev, "CMD_SYNC timeout\n");
1043 return ret;
1012} 1044}
1013 1045
1014/* Context descriptor manipulation functions */ 1046/* Context descriptor manipulation functions */
@@ -1025,7 +1057,6 @@ static u64 arm_smmu_cpu_tcr_to_cd(u64 tcr)
1025 val |= ARM_SMMU_TCR2CD(tcr, EPD0); 1057 val |= ARM_SMMU_TCR2CD(tcr, EPD0);
1026 val |= ARM_SMMU_TCR2CD(tcr, EPD1); 1058 val |= ARM_SMMU_TCR2CD(tcr, EPD1);
1027 val |= ARM_SMMU_TCR2CD(tcr, IPS); 1059 val |= ARM_SMMU_TCR2CD(tcr, IPS);
1028 val |= ARM_SMMU_TCR2CD(tcr, TBI0);
1029 1060
1030 return val; 1061 return val;
1031} 1062}
@@ -1085,8 +1116,8 @@ static void arm_smmu_sync_ste_for_sid(struct arm_smmu_device *smmu, u32 sid)
1085 arm_smmu_cmdq_issue_sync(smmu); 1116 arm_smmu_cmdq_issue_sync(smmu);
1086} 1117}
1087 1118
1088static void arm_smmu_write_strtab_ent(struct arm_smmu_device *smmu, u32 sid, 1119static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid,
1089 __le64 *dst, struct arm_smmu_strtab_ent *ste) 1120 __le64 *dst)
1090{ 1121{
1091 /* 1122 /*
1092 * This is hideously complicated, but we only really care about 1123 * This is hideously complicated, but we only really care about
@@ -1106,6 +1137,10 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_device *smmu, u32 sid,
1106 */ 1137 */
1107 u64 val = le64_to_cpu(dst[0]); 1138 u64 val = le64_to_cpu(dst[0]);
1108 bool ste_live = false; 1139 bool ste_live = false;
1140 struct arm_smmu_device *smmu = NULL;
1141 struct arm_smmu_s1_cfg *s1_cfg = NULL;
1142 struct arm_smmu_s2_cfg *s2_cfg = NULL;
1143 struct arm_smmu_domain *smmu_domain = NULL;
1109 struct arm_smmu_cmdq_ent prefetch_cmd = { 1144 struct arm_smmu_cmdq_ent prefetch_cmd = {
1110 .opcode = CMDQ_OP_PREFETCH_CFG, 1145 .opcode = CMDQ_OP_PREFETCH_CFG,
1111 .prefetch = { 1146 .prefetch = {
@@ -1113,6 +1148,25 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_device *smmu, u32 sid,
1113 }, 1148 },
1114 }; 1149 };
1115 1150
1151 if (master) {
1152 smmu_domain = master->domain;
1153 smmu = master->smmu;
1154 }
1155
1156 if (smmu_domain) {
1157 switch (smmu_domain->stage) {
1158 case ARM_SMMU_DOMAIN_S1:
1159 s1_cfg = &smmu_domain->s1_cfg;
1160 break;
1161 case ARM_SMMU_DOMAIN_S2:
1162 case ARM_SMMU_DOMAIN_NESTED:
1163 s2_cfg = &smmu_domain->s2_cfg;
1164 break;
1165 default:
1166 break;
1167 }
1168 }
1169
1116 if (val & STRTAB_STE_0_V) { 1170 if (val & STRTAB_STE_0_V) {
1117 switch (FIELD_GET(STRTAB_STE_0_CFG, val)) { 1171 switch (FIELD_GET(STRTAB_STE_0_CFG, val)) {
1118 case STRTAB_STE_0_CFG_BYPASS: 1172 case STRTAB_STE_0_CFG_BYPASS:
@@ -1133,8 +1187,8 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_device *smmu, u32 sid,
1133 val = STRTAB_STE_0_V; 1187 val = STRTAB_STE_0_V;
1134 1188
1135 /* Bypass/fault */ 1189 /* Bypass/fault */
1136 if (!ste->assigned || !(ste->s1_cfg || ste->s2_cfg)) { 1190 if (!smmu_domain || !(s1_cfg || s2_cfg)) {
1137 if (!ste->assigned && disable_bypass) 1191 if (!smmu_domain && disable_bypass)
1138 val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_ABORT); 1192 val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_ABORT);
1139 else 1193 else
1140 val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_BYPASS); 1194 val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_BYPASS);
@@ -1152,41 +1206,42 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_device *smmu, u32 sid,
1152 return; 1206 return;
1153 } 1207 }
1154 1208
1155 if (ste->s1_cfg) { 1209 if (s1_cfg) {
1156 BUG_ON(ste_live); 1210 BUG_ON(ste_live);
1157 dst[1] = cpu_to_le64( 1211 dst[1] = cpu_to_le64(
1158 FIELD_PREP(STRTAB_STE_1_S1CIR, STRTAB_STE_1_S1C_CACHE_WBRA) | 1212 FIELD_PREP(STRTAB_STE_1_S1CIR, STRTAB_STE_1_S1C_CACHE_WBRA) |
1159 FIELD_PREP(STRTAB_STE_1_S1COR, STRTAB_STE_1_S1C_CACHE_WBRA) | 1213 FIELD_PREP(STRTAB_STE_1_S1COR, STRTAB_STE_1_S1C_CACHE_WBRA) |
1160 FIELD_PREP(STRTAB_STE_1_S1CSH, ARM_SMMU_SH_ISH) | 1214 FIELD_PREP(STRTAB_STE_1_S1CSH, ARM_SMMU_SH_ISH) |
1161#ifdef CONFIG_PCI_ATS
1162 FIELD_PREP(STRTAB_STE_1_EATS, STRTAB_STE_1_EATS_TRANS) |
1163#endif
1164 FIELD_PREP(STRTAB_STE_1_STRW, STRTAB_STE_1_STRW_NSEL1)); 1215 FIELD_PREP(STRTAB_STE_1_STRW, STRTAB_STE_1_STRW_NSEL1));
1165 1216
1166 if (smmu->features & ARM_SMMU_FEAT_STALLS && 1217 if (smmu->features & ARM_SMMU_FEAT_STALLS &&
1167 !(smmu->features & ARM_SMMU_FEAT_STALL_FORCE)) 1218 !(smmu->features & ARM_SMMU_FEAT_STALL_FORCE))
1168 dst[1] |= cpu_to_le64(STRTAB_STE_1_S1STALLD); 1219 dst[1] |= cpu_to_le64(STRTAB_STE_1_S1STALLD);
1169 1220
1170 val |= (ste->s1_cfg->cdptr_dma & STRTAB_STE_0_S1CTXPTR_MASK) | 1221 val |= (s1_cfg->cdptr_dma & STRTAB_STE_0_S1CTXPTR_MASK) |
1171 FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S1_TRANS); 1222 FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S1_TRANS);
1172 } 1223 }
1173 1224
1174 if (ste->s2_cfg) { 1225 if (s2_cfg) {
1175 BUG_ON(ste_live); 1226 BUG_ON(ste_live);
1176 dst[2] = cpu_to_le64( 1227 dst[2] = cpu_to_le64(
1177 FIELD_PREP(STRTAB_STE_2_S2VMID, ste->s2_cfg->vmid) | 1228 FIELD_PREP(STRTAB_STE_2_S2VMID, s2_cfg->vmid) |
1178 FIELD_PREP(STRTAB_STE_2_VTCR, ste->s2_cfg->vtcr) | 1229 FIELD_PREP(STRTAB_STE_2_VTCR, s2_cfg->vtcr) |
1179#ifdef __BIG_ENDIAN 1230#ifdef __BIG_ENDIAN
1180 STRTAB_STE_2_S2ENDI | 1231 STRTAB_STE_2_S2ENDI |
1181#endif 1232#endif
1182 STRTAB_STE_2_S2PTW | STRTAB_STE_2_S2AA64 | 1233 STRTAB_STE_2_S2PTW | STRTAB_STE_2_S2AA64 |
1183 STRTAB_STE_2_S2R); 1234 STRTAB_STE_2_S2R);
1184 1235
1185 dst[3] = cpu_to_le64(ste->s2_cfg->vttbr & STRTAB_STE_3_S2TTB_MASK); 1236 dst[3] = cpu_to_le64(s2_cfg->vttbr & STRTAB_STE_3_S2TTB_MASK);
1186 1237
1187 val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S2_TRANS); 1238 val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S2_TRANS);
1188 } 1239 }
1189 1240
1241 if (master->ats_enabled)
1242 dst[1] |= cpu_to_le64(FIELD_PREP(STRTAB_STE_1_EATS,
1243 STRTAB_STE_1_EATS_TRANS));
1244
1190 arm_smmu_sync_ste_for_sid(smmu, sid); 1245 arm_smmu_sync_ste_for_sid(smmu, sid);
1191 dst[0] = cpu_to_le64(val); 1246 dst[0] = cpu_to_le64(val);
1192 arm_smmu_sync_ste_for_sid(smmu, sid); 1247 arm_smmu_sync_ste_for_sid(smmu, sid);
@@ -1199,10 +1254,9 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_device *smmu, u32 sid,
1199static void arm_smmu_init_bypass_stes(u64 *strtab, unsigned int nent) 1254static void arm_smmu_init_bypass_stes(u64 *strtab, unsigned int nent)
1200{ 1255{
1201 unsigned int i; 1256 unsigned int i;
1202 struct arm_smmu_strtab_ent ste = { .assigned = false };
1203 1257
1204 for (i = 0; i < nent; ++i) { 1258 for (i = 0; i < nent; ++i) {
1205 arm_smmu_write_strtab_ent(NULL, -1, strtab, &ste); 1259 arm_smmu_write_strtab_ent(NULL, -1, strtab);
1206 strtab += STRTAB_STE_DWORDS; 1260 strtab += STRTAB_STE_DWORDS;
1207 } 1261 }
1208} 1262}
@@ -1390,6 +1444,96 @@ static irqreturn_t arm_smmu_combined_irq_handler(int irq, void *dev)
1390 return IRQ_WAKE_THREAD; 1444 return IRQ_WAKE_THREAD;
1391} 1445}
1392 1446
1447static void
1448arm_smmu_atc_inv_to_cmd(int ssid, unsigned long iova, size_t size,
1449 struct arm_smmu_cmdq_ent *cmd)
1450{
1451 size_t log2_span;
1452 size_t span_mask;
1453 /* ATC invalidates are always on 4096-bytes pages */
1454 size_t inval_grain_shift = 12;
1455 unsigned long page_start, page_end;
1456
1457 *cmd = (struct arm_smmu_cmdq_ent) {
1458 .opcode = CMDQ_OP_ATC_INV,
1459 .substream_valid = !!ssid,
1460 .atc.ssid = ssid,
1461 };
1462
1463 if (!size) {
1464 cmd->atc.size = ATC_INV_SIZE_ALL;
1465 return;
1466 }
1467
1468 page_start = iova >> inval_grain_shift;
1469 page_end = (iova + size - 1) >> inval_grain_shift;
1470
1471 /*
1472 * In an ATS Invalidate Request, the address must be aligned on the
1473 * range size, which must be a power of two number of page sizes. We
1474 * thus have to choose between grossly over-invalidating the region, or
1475 * splitting the invalidation into multiple commands. For simplicity
1476 * we'll go with the first solution, but should refine it in the future
1477 * if multiple commands are shown to be more efficient.
1478 *
1479 * Find the smallest power of two that covers the range. The most
1480 * significant differing bit between the start and end addresses,
1481 * fls(start ^ end), indicates the required span. For example:
1482 *
1483 * We want to invalidate pages [8; 11]. This is already the ideal range:
1484 * x = 0b1000 ^ 0b1011 = 0b11
1485 * span = 1 << fls(x) = 4
1486 *
1487 * To invalidate pages [7; 10], we need to invalidate [0; 15]:
1488 * x = 0b0111 ^ 0b1010 = 0b1101
1489 * span = 1 << fls(x) = 16
1490 */
1491 log2_span = fls_long(page_start ^ page_end);
1492 span_mask = (1ULL << log2_span) - 1;
1493
1494 page_start &= ~span_mask;
1495
1496 cmd->atc.addr = page_start << inval_grain_shift;
1497 cmd->atc.size = log2_span;
1498}
1499
1500static int arm_smmu_atc_inv_master(struct arm_smmu_master *master,
1501 struct arm_smmu_cmdq_ent *cmd)
1502{
1503 int i;
1504
1505 if (!master->ats_enabled)
1506 return 0;
1507
1508 for (i = 0; i < master->num_sids; i++) {
1509 cmd->atc.sid = master->sids[i];
1510 arm_smmu_cmdq_issue_cmd(master->smmu, cmd);
1511 }
1512
1513 return arm_smmu_cmdq_issue_sync(master->smmu);
1514}
1515
1516static int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain,
1517 int ssid, unsigned long iova, size_t size)
1518{
1519 int ret = 0;
1520 unsigned long flags;
1521 struct arm_smmu_cmdq_ent cmd;
1522 struct arm_smmu_master *master;
1523
1524 if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_ATS))
1525 return 0;
1526
1527 arm_smmu_atc_inv_to_cmd(ssid, iova, size, &cmd);
1528
1529 spin_lock_irqsave(&smmu_domain->devices_lock, flags);
1530 list_for_each_entry(master, &smmu_domain->devices, domain_head)
1531 ret |= arm_smmu_atc_inv_master(master, &cmd);
1532 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
1533
1534 return ret ? -ETIMEDOUT : 0;
1535}
1536
1393/* IO_PGTABLE API */ 1537/* IO_PGTABLE API */
1394static void arm_smmu_tlb_sync(void *cookie) 1538static void arm_smmu_tlb_sync(void *cookie)
1395{ 1539{
@@ -1493,6 +1637,9 @@ static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
1493 } 1637 }
1494 1638
1495 mutex_init(&smmu_domain->init_mutex); 1639 mutex_init(&smmu_domain->init_mutex);
1640 INIT_LIST_HEAD(&smmu_domain->devices);
1641 spin_lock_init(&smmu_domain->devices_lock);
1642
1496 return &smmu_domain->domain; 1643 return &smmu_domain->domain;
1497} 1644}
1498 1645
@@ -1688,55 +1835,97 @@ static __le64 *arm_smmu_get_step_for_sid(struct arm_smmu_device *smmu, u32 sid)
1688 return step; 1835 return step;
1689} 1836}
1690 1837
1691static void arm_smmu_install_ste_for_dev(struct iommu_fwspec *fwspec) 1838static void arm_smmu_install_ste_for_dev(struct arm_smmu_master *master)
1692{ 1839{
1693 int i, j; 1840 int i, j;
1694 struct arm_smmu_master_data *master = fwspec->iommu_priv;
1695 struct arm_smmu_device *smmu = master->smmu; 1841 struct arm_smmu_device *smmu = master->smmu;
1696 1842
1697 for (i = 0; i < fwspec->num_ids; ++i) { 1843 for (i = 0; i < master->num_sids; ++i) {
1698 u32 sid = fwspec->ids[i]; 1844 u32 sid = master->sids[i];
1699 __le64 *step = arm_smmu_get_step_for_sid(smmu, sid); 1845 __le64 *step = arm_smmu_get_step_for_sid(smmu, sid);
1700 1846
1701 /* Bridged PCI devices may end up with duplicated IDs */ 1847 /* Bridged PCI devices may end up with duplicated IDs */
1702 for (j = 0; j < i; j++) 1848 for (j = 0; j < i; j++)
1703 if (fwspec->ids[j] == sid) 1849 if (master->sids[j] == sid)
1704 break; 1850 break;
1705 if (j < i) 1851 if (j < i)
1706 continue; 1852 continue;
1707 1853
1708 arm_smmu_write_strtab_ent(smmu, sid, step, &master->ste); 1854 arm_smmu_write_strtab_ent(master, sid, step);
1709 } 1855 }
1710} 1856}
1711 1857
1712static void arm_smmu_detach_dev(struct device *dev) 1858static int arm_smmu_enable_ats(struct arm_smmu_master *master)
1713{ 1859{
1714 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 1860 int ret;
1715 struct arm_smmu_master_data *master = fwspec->iommu_priv; 1861 size_t stu;
1862 struct pci_dev *pdev;
1863 struct arm_smmu_device *smmu = master->smmu;
1864 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(master->dev);
1716 1865
1717 master->ste.assigned = false; 1866 if (!(smmu->features & ARM_SMMU_FEAT_ATS) || !dev_is_pci(master->dev) ||
1718 arm_smmu_install_ste_for_dev(fwspec); 1867 !(fwspec->flags & IOMMU_FWSPEC_PCI_RC_ATS) || pci_ats_disabled())
1868 return -ENXIO;
1869
1870 pdev = to_pci_dev(master->dev);
1871 if (pdev->untrusted)
1872 return -EPERM;
1873
1874 /* Smallest Translation Unit: log2 of the smallest supported granule */
1875 stu = __ffs(smmu->pgsize_bitmap);
1876
1877 ret = pci_enable_ats(pdev, stu);
1878 if (ret)
1879 return ret;
1880
1881 master->ats_enabled = true;
1882 return 0;
1883}
1884
1885static void arm_smmu_disable_ats(struct arm_smmu_master *master)
1886{
1887 if (!master->ats_enabled || !dev_is_pci(master->dev))
1888 return;
1889
1890 pci_disable_ats(to_pci_dev(master->dev));
1891 master->ats_enabled = false;
1892}
1893
1894static void arm_smmu_detach_dev(struct arm_smmu_master *master)
1895{
1896 unsigned long flags;
1897 struct arm_smmu_domain *smmu_domain = master->domain;
1898
1899 if (!smmu_domain)
1900 return;
1901
1902 spin_lock_irqsave(&smmu_domain->devices_lock, flags);
1903 list_del(&master->domain_head);
1904 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
1905
1906 master->domain = NULL;
1907 arm_smmu_install_ste_for_dev(master);
1908
1909 /* Disabling ATS invalidates all ATC entries */
1910 arm_smmu_disable_ats(master);
1719} 1911}
1720 1912
1721static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) 1913static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1722{ 1914{
1723 int ret = 0; 1915 int ret = 0;
1916 unsigned long flags;
1724 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 1917 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1725 struct arm_smmu_device *smmu; 1918 struct arm_smmu_device *smmu;
1726 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 1919 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1727 struct arm_smmu_master_data *master; 1920 struct arm_smmu_master *master;
1728 struct arm_smmu_strtab_ent *ste;
1729 1921
1730 if (!fwspec) 1922 if (!fwspec)
1731 return -ENOENT; 1923 return -ENOENT;
1732 1924
1733 master = fwspec->iommu_priv; 1925 master = fwspec->iommu_priv;
1734 smmu = master->smmu; 1926 smmu = master->smmu;
1735 ste = &master->ste;
1736 1927
1737 /* Already attached to a different domain? */ 1928 arm_smmu_detach_dev(master);
1738 if (ste->assigned)
1739 arm_smmu_detach_dev(dev);
1740 1929
1741 mutex_lock(&smmu_domain->init_mutex); 1930 mutex_lock(&smmu_domain->init_mutex);
1742 1931
@@ -1756,21 +1945,19 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1756 goto out_unlock; 1945 goto out_unlock;
1757 } 1946 }
1758 1947
1759 ste->assigned = true; 1948 master->domain = smmu_domain;
1760 1949
1761 if (smmu_domain->stage == ARM_SMMU_DOMAIN_BYPASS) { 1950 spin_lock_irqsave(&smmu_domain->devices_lock, flags);
1762 ste->s1_cfg = NULL; 1951 list_add(&master->domain_head, &smmu_domain->devices);
1763 ste->s2_cfg = NULL; 1952 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
1764 } else if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1765 ste->s1_cfg = &smmu_domain->s1_cfg;
1766 ste->s2_cfg = NULL;
1767 arm_smmu_write_ctx_desc(smmu, ste->s1_cfg);
1768 } else {
1769 ste->s1_cfg = NULL;
1770 ste->s2_cfg = &smmu_domain->s2_cfg;
1771 }
1772 1953
1773 arm_smmu_install_ste_for_dev(fwspec); 1954 if (smmu_domain->stage != ARM_SMMU_DOMAIN_BYPASS)
1955 arm_smmu_enable_ats(master);
1956
1957 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1958 arm_smmu_write_ctx_desc(smmu, &smmu_domain->s1_cfg);
1959
1960 arm_smmu_install_ste_for_dev(master);
1774out_unlock: 1961out_unlock:
1775 mutex_unlock(&smmu_domain->init_mutex); 1962 mutex_unlock(&smmu_domain->init_mutex);
1776 return ret; 1963 return ret;
@@ -1790,12 +1977,18 @@ static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
1790static size_t 1977static size_t
1791arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size) 1978arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size)
1792{ 1979{
1793 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops; 1980 int ret;
1981 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1982 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
1794 1983
1795 if (!ops) 1984 if (!ops)
1796 return 0; 1985 return 0;
1797 1986
1798 return ops->unmap(ops, iova, size); 1987 ret = ops->unmap(ops, iova, size);
1988 if (ret && arm_smmu_atc_inv_domain(smmu_domain, 0, iova, size))
1989 return 0;
1990
1991 return ret;
1799} 1992}
1800 1993
1801static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain) 1994static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
@@ -1860,7 +2053,7 @@ static int arm_smmu_add_device(struct device *dev)
1860{ 2053{
1861 int i, ret; 2054 int i, ret;
1862 struct arm_smmu_device *smmu; 2055 struct arm_smmu_device *smmu;
1863 struct arm_smmu_master_data *master; 2056 struct arm_smmu_master *master;
1864 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 2057 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1865 struct iommu_group *group; 2058 struct iommu_group *group;
1866 2059
@@ -1882,13 +2075,16 @@ static int arm_smmu_add_device(struct device *dev)
1882 if (!master) 2075 if (!master)
1883 return -ENOMEM; 2076 return -ENOMEM;
1884 2077
2078 master->dev = dev;
1885 master->smmu = smmu; 2079 master->smmu = smmu;
2080 master->sids = fwspec->ids;
2081 master->num_sids = fwspec->num_ids;
1886 fwspec->iommu_priv = master; 2082 fwspec->iommu_priv = master;
1887 } 2083 }
1888 2084
1889 /* Check the SIDs are in range of the SMMU and our stream table */ 2085 /* Check the SIDs are in range of the SMMU and our stream table */
1890 for (i = 0; i < fwspec->num_ids; i++) { 2086 for (i = 0; i < master->num_sids; i++) {
1891 u32 sid = fwspec->ids[i]; 2087 u32 sid = master->sids[i];
1892 2088
1893 if (!arm_smmu_sid_in_range(smmu, sid)) 2089 if (!arm_smmu_sid_in_range(smmu, sid))
1894 return -ERANGE; 2090 return -ERANGE;
@@ -1913,7 +2109,7 @@ static int arm_smmu_add_device(struct device *dev)
1913static void arm_smmu_remove_device(struct device *dev) 2109static void arm_smmu_remove_device(struct device *dev)
1914{ 2110{
1915 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 2111 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1916 struct arm_smmu_master_data *master; 2112 struct arm_smmu_master *master;
1917 struct arm_smmu_device *smmu; 2113 struct arm_smmu_device *smmu;
1918 2114
1919 if (!fwspec || fwspec->ops != &arm_smmu_ops) 2115 if (!fwspec || fwspec->ops != &arm_smmu_ops)
@@ -1921,8 +2117,7 @@ static void arm_smmu_remove_device(struct device *dev)
1921 2117
1922 master = fwspec->iommu_priv; 2118 master = fwspec->iommu_priv;
1923 smmu = master->smmu; 2119 smmu = master->smmu;
1924 if (master && master->ste.assigned) 2120 arm_smmu_detach_dev(master);
1925 arm_smmu_detach_dev(dev);
1926 iommu_group_remove_device(dev); 2121 iommu_group_remove_device(dev);
1927 iommu_device_unlink(&smmu->iommu, dev); 2122 iommu_device_unlink(&smmu->iommu, dev);
1928 kfree(master); 2123 kfree(master);
@@ -2454,13 +2649,9 @@ static int arm_smmu_device_reset(struct arm_smmu_device *smmu, bool bypass)
2454 /* Clear CR0 and sync (disables SMMU and queue processing) */ 2649 /* Clear CR0 and sync (disables SMMU and queue processing) */
2455 reg = readl_relaxed(smmu->base + ARM_SMMU_CR0); 2650 reg = readl_relaxed(smmu->base + ARM_SMMU_CR0);
2456 if (reg & CR0_SMMUEN) { 2651 if (reg & CR0_SMMUEN) {
2457 if (is_kdump_kernel()) {
2458 arm_smmu_update_gbpa(smmu, GBPA_ABORT, 0);
2459 arm_smmu_device_disable(smmu);
2460 return -EBUSY;
2461 }
2462
2463 dev_warn(smmu->dev, "SMMU currently enabled! Resetting...\n"); 2652 dev_warn(smmu->dev, "SMMU currently enabled! Resetting...\n");
2653 WARN_ON(is_kdump_kernel() && !disable_bypass);
2654 arm_smmu_update_gbpa(smmu, GBPA_ABORT, 0);
2464 } 2655 }
2465 2656
2466 ret = arm_smmu_device_disable(smmu); 2657 ret = arm_smmu_device_disable(smmu);
@@ -2547,12 +2738,24 @@ static int arm_smmu_device_reset(struct arm_smmu_device *smmu, bool bypass)
2547 } 2738 }
2548 } 2739 }
2549 2740
2741 if (smmu->features & ARM_SMMU_FEAT_ATS) {
2742 enables |= CR0_ATSCHK;
2743 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
2744 ARM_SMMU_CR0ACK);
2745 if (ret) {
2746 dev_err(smmu->dev, "failed to enable ATS check\n");
2747 return ret;
2748 }
2749 }
2750
2550 ret = arm_smmu_setup_irqs(smmu); 2751 ret = arm_smmu_setup_irqs(smmu);
2551 if (ret) { 2752 if (ret) {
2552 dev_err(smmu->dev, "failed to setup irqs\n"); 2753 dev_err(smmu->dev, "failed to setup irqs\n");
2553 return ret; 2754 return ret;
2554 } 2755 }
2555 2756
2757 if (is_kdump_kernel())
2758 enables &= ~(CR0_EVTQEN | CR0_PRIQEN);
2556 2759
2557 /* Enable the SMMU interface, or ensure bypass */ 2760 /* Enable the SMMU interface, or ensure bypass */
2558 if (!bypass || disable_bypass) { 2761 if (!bypass || disable_bypass) {
diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c
index 045d93884164..5e54cc0a28b3 100644
--- a/drivers/iommu/arm-smmu.c
+++ b/drivers/iommu/arm-smmu.c
@@ -110,7 +110,8 @@ static int force_stage;
110module_param(force_stage, int, S_IRUGO); 110module_param(force_stage, int, S_IRUGO);
111MODULE_PARM_DESC(force_stage, 111MODULE_PARM_DESC(force_stage,
112 "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation."); 112 "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
113static bool disable_bypass; 113static bool disable_bypass =
114 IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT);
114module_param(disable_bypass, bool, S_IRUGO); 115module_param(disable_bypass, bool, S_IRUGO);
115MODULE_PARM_DESC(disable_bypass, 116MODULE_PARM_DESC(disable_bypass,
116 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU."); 117 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
@@ -569,12 +570,13 @@ static const struct iommu_gather_ops arm_smmu_s2_tlb_ops_v1 = {
569 570
570static irqreturn_t arm_smmu_context_fault(int irq, void *dev) 571static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
571{ 572{
572 u32 fsr, fsynr; 573 u32 fsr, fsynr, cbfrsynra;
573 unsigned long iova; 574 unsigned long iova;
574 struct iommu_domain *domain = dev; 575 struct iommu_domain *domain = dev;
575 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 576 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
576 struct arm_smmu_cfg *cfg = &smmu_domain->cfg; 577 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
577 struct arm_smmu_device *smmu = smmu_domain->smmu; 578 struct arm_smmu_device *smmu = smmu_domain->smmu;
579 void __iomem *gr1_base = ARM_SMMU_GR1(smmu);
578 void __iomem *cb_base; 580 void __iomem *cb_base;
579 581
580 cb_base = ARM_SMMU_CB(smmu, cfg->cbndx); 582 cb_base = ARM_SMMU_CB(smmu, cfg->cbndx);
@@ -585,10 +587,11 @@ static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
585 587
586 fsynr = readl_relaxed(cb_base + ARM_SMMU_CB_FSYNR0); 588 fsynr = readl_relaxed(cb_base + ARM_SMMU_CB_FSYNR0);
587 iova = readq_relaxed(cb_base + ARM_SMMU_CB_FAR); 589 iova = readq_relaxed(cb_base + ARM_SMMU_CB_FAR);
590 cbfrsynra = readl_relaxed(gr1_base + ARM_SMMU_GR1_CBFRSYNRA(cfg->cbndx));
588 591
589 dev_err_ratelimited(smmu->dev, 592 dev_err_ratelimited(smmu->dev,
590 "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cb=%d\n", 593 "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cbfrsynra=0x%x, cb=%d\n",
591 fsr, iova, fsynr, cfg->cbndx); 594 fsr, iova, fsynr, cbfrsynra, cfg->cbndx);
592 595
593 writel(fsr, cb_base + ARM_SMMU_CB_FSR); 596 writel(fsr, cb_base + ARM_SMMU_CB_FSR);
594 return IRQ_HANDLED; 597 return IRQ_HANDLED;
diff --git a/drivers/iommu/dmar.c b/drivers/iommu/dmar.c
index 9c49300e9fb7..6d969a172fbb 100644
--- a/drivers/iommu/dmar.c
+++ b/drivers/iommu/dmar.c
@@ -145,7 +145,7 @@ dmar_alloc_pci_notify_info(struct pci_dev *dev, unsigned long event)
145 for (tmp = dev; tmp; tmp = tmp->bus->self) 145 for (tmp = dev; tmp; tmp = tmp->bus->self)
146 level++; 146 level++;
147 147
148 size = sizeof(*info) + level * sizeof(info->path[0]); 148 size = struct_size(info, path, level);
149 if (size <= sizeof(dmar_pci_notify_info_buf)) { 149 if (size <= sizeof(dmar_pci_notify_info_buf)) {
150 info = (struct dmar_pci_notify_info *)dmar_pci_notify_info_buf; 150 info = (struct dmar_pci_notify_info *)dmar_pci_notify_info_buf;
151 } else { 151 } else {
diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c
index 28cb713d728c..a320bda2c305 100644
--- a/drivers/iommu/intel-iommu.c
+++ b/drivers/iommu/intel-iommu.c
@@ -2341,32 +2341,33 @@ static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2341} 2341}
2342 2342
2343static int domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn, 2343static int domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2344 struct scatterlist *sg, unsigned long phys_pfn, 2344 struct scatterlist *sg, unsigned long phys_pfn,
2345 unsigned long nr_pages, int prot) 2345 unsigned long nr_pages, int prot)
2346{ 2346{
2347 int ret; 2347 int ret;
2348 struct intel_iommu *iommu; 2348 struct intel_iommu *iommu;
2349 2349
2350 /* Do the real mapping first */ 2350 /* Do the real mapping first */
2351 ret = __domain_mapping(domain, iov_pfn, sg, phys_pfn, nr_pages, prot); 2351 ret = __domain_mapping(domain, iov_pfn, sg, phys_pfn, nr_pages, prot);
2352 if (ret) 2352 if (ret)
2353 return ret; 2353 return ret;
2354 2354
2355 /* Notify about the new mapping */ 2355 /* Notify about the new mapping */
2356 if (domain_type_is_vm(domain)) { 2356 if (domain_type_is_vm(domain)) {
2357 /* VM typed domains can have more than one IOMMUs */ 2357 /* VM typed domains can have more than one IOMMUs */
2358 int iommu_id; 2358 int iommu_id;
2359 for_each_domain_iommu(iommu_id, domain) { 2359
2360 iommu = g_iommus[iommu_id]; 2360 for_each_domain_iommu(iommu_id, domain) {
2361 __mapping_notify_one(iommu, domain, iov_pfn, nr_pages); 2361 iommu = g_iommus[iommu_id];
2362 } 2362 __mapping_notify_one(iommu, domain, iov_pfn, nr_pages);
2363 } else { 2363 }
2364 /* General domains only have one IOMMU */ 2364 } else {
2365 iommu = domain_get_iommu(domain); 2365 /* General domains only have one IOMMU */
2366 __mapping_notify_one(iommu, domain, iov_pfn, nr_pages); 2366 iommu = domain_get_iommu(domain);
2367 } 2367 __mapping_notify_one(iommu, domain, iov_pfn, nr_pages);
2368 }
2368 2369
2369 return 0; 2370 return 0;
2370} 2371}
2371 2372
2372static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn, 2373static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
@@ -2485,6 +2486,8 @@ static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu,
2485 info->domain = domain; 2486 info->domain = domain;
2486 info->iommu = iommu; 2487 info->iommu = iommu;
2487 info->pasid_table = NULL; 2488 info->pasid_table = NULL;
2489 info->auxd_enabled = 0;
2490 INIT_LIST_HEAD(&info->auxiliary_domains);
2488 2491
2489 if (dev && dev_is_pci(dev)) { 2492 if (dev && dev_is_pci(dev)) {
2490 struct pci_dev *pdev = to_pci_dev(info->dev); 2493 struct pci_dev *pdev = to_pci_dev(info->dev);
@@ -3412,9 +3415,12 @@ static int __init init_dmars(void)
3412 iommu_identity_mapping |= IDENTMAP_ALL; 3415 iommu_identity_mapping |= IDENTMAP_ALL;
3413 3416
3414#ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA 3417#ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
3415 iommu_identity_mapping |= IDENTMAP_GFX; 3418 dmar_map_gfx = 0;
3416#endif 3419#endif
3417 3420
3421 if (!dmar_map_gfx)
3422 iommu_identity_mapping |= IDENTMAP_GFX;
3423
3418 check_tylersburg_isoch(); 3424 check_tylersburg_isoch();
3419 3425
3420 if (iommu_identity_mapping) { 3426 if (iommu_identity_mapping) {
@@ -3496,7 +3502,13 @@ domains_done:
3496 3502
3497#ifdef CONFIG_INTEL_IOMMU_SVM 3503#ifdef CONFIG_INTEL_IOMMU_SVM
3498 if (pasid_supported(iommu) && ecap_prs(iommu->ecap)) { 3504 if (pasid_supported(iommu) && ecap_prs(iommu->ecap)) {
3505 /*
3506 * Call dmar_alloc_hwirq() with dmar_global_lock held,
3507 * could cause possible lock race condition.
3508 */
3509 up_write(&dmar_global_lock);
3499 ret = intel_svm_enable_prq(iommu); 3510 ret = intel_svm_enable_prq(iommu);
3511 down_write(&dmar_global_lock);
3500 if (ret) 3512 if (ret)
3501 goto free_iommu; 3513 goto free_iommu;
3502 } 3514 }
@@ -3606,45 +3618,40 @@ out:
3606} 3618}
3607 3619
3608/* Check if the dev needs to go through non-identity map and unmap process.*/ 3620/* Check if the dev needs to go through non-identity map and unmap process.*/
3609static int iommu_no_mapping(struct device *dev) 3621static bool iommu_need_mapping(struct device *dev)
3610{ 3622{
3611 int found; 3623 int found;
3612 3624
3613 if (iommu_dummy(dev)) 3625 if (iommu_dummy(dev))
3614 return 1; 3626 return false;
3615 3627
3616 if (!iommu_identity_mapping) 3628 if (!iommu_identity_mapping)
3617 return 0; 3629 return true;
3618 3630
3619 found = identity_mapping(dev); 3631 found = identity_mapping(dev);
3620 if (found) { 3632 if (found) {
3621 if (iommu_should_identity_map(dev, 0)) 3633 if (iommu_should_identity_map(dev, 0))
3622 return 1; 3634 return false;
3623 else { 3635
3624 /* 3636 /*
3625 * 32 bit DMA is removed from si_domain and fall back 3637 * 32 bit DMA is removed from si_domain and fall back to
3626 * to non-identity mapping. 3638 * non-identity mapping.
3627 */ 3639 */
3628 dmar_remove_one_dev_info(dev); 3640 dmar_remove_one_dev_info(dev);
3629 dev_info(dev, "32bit DMA uses non-identity mapping\n"); 3641 dev_info(dev, "32bit DMA uses non-identity mapping\n");
3630 return 0;
3631 }
3632 } else { 3642 } else {
3633 /* 3643 /*
3634 * In case of a detached 64 bit DMA device from vm, the device 3644 * In case of a detached 64 bit DMA device from vm, the device
3635 * is put into si_domain for identity mapping. 3645 * is put into si_domain for identity mapping.
3636 */ 3646 */
3637 if (iommu_should_identity_map(dev, 0)) { 3647 if (iommu_should_identity_map(dev, 0) &&
3638 int ret; 3648 !domain_add_dev_info(si_domain, dev)) {
3639 ret = domain_add_dev_info(si_domain, dev); 3649 dev_info(dev, "64bit DMA uses identity mapping\n");
3640 if (!ret) { 3650 return false;
3641 dev_info(dev, "64bit DMA uses identity mapping\n");
3642 return 1;
3643 }
3644 } 3651 }
3645 } 3652 }
3646 3653
3647 return 0; 3654 return true;
3648} 3655}
3649 3656
3650static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr, 3657static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
@@ -3660,9 +3667,6 @@ static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
3660 3667
3661 BUG_ON(dir == DMA_NONE); 3668 BUG_ON(dir == DMA_NONE);
3662 3669
3663 if (iommu_no_mapping(dev))
3664 return paddr;
3665
3666 domain = get_valid_domain_for_dev(dev); 3670 domain = get_valid_domain_for_dev(dev);
3667 if (!domain) 3671 if (!domain)
3668 return DMA_MAPPING_ERROR; 3672 return DMA_MAPPING_ERROR;
@@ -3711,15 +3715,20 @@ static dma_addr_t intel_map_page(struct device *dev, struct page *page,
3711 enum dma_data_direction dir, 3715 enum dma_data_direction dir,
3712 unsigned long attrs) 3716 unsigned long attrs)
3713{ 3717{
3714 return __intel_map_single(dev, page_to_phys(page) + offset, size, 3718 if (iommu_need_mapping(dev))
3715 dir, *dev->dma_mask); 3719 return __intel_map_single(dev, page_to_phys(page) + offset,
3720 size, dir, *dev->dma_mask);
3721 return dma_direct_map_page(dev, page, offset, size, dir, attrs);
3716} 3722}
3717 3723
3718static dma_addr_t intel_map_resource(struct device *dev, phys_addr_t phys_addr, 3724static dma_addr_t intel_map_resource(struct device *dev, phys_addr_t phys_addr,
3719 size_t size, enum dma_data_direction dir, 3725 size_t size, enum dma_data_direction dir,
3720 unsigned long attrs) 3726 unsigned long attrs)
3721{ 3727{
3722 return __intel_map_single(dev, phys_addr, size, dir, *dev->dma_mask); 3728 if (iommu_need_mapping(dev))
3729 return __intel_map_single(dev, phys_addr, size, dir,
3730 *dev->dma_mask);
3731 return dma_direct_map_resource(dev, phys_addr, size, dir, attrs);
3723} 3732}
3724 3733
3725static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size) 3734static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size)
@@ -3730,9 +3739,7 @@ static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size)
3730 unsigned long iova_pfn; 3739 unsigned long iova_pfn;
3731 struct intel_iommu *iommu; 3740 struct intel_iommu *iommu;
3732 struct page *freelist; 3741 struct page *freelist;
3733 3742 struct pci_dev *pdev = NULL;
3734 if (iommu_no_mapping(dev))
3735 return;
3736 3743
3737 domain = find_domain(dev); 3744 domain = find_domain(dev);
3738 BUG_ON(!domain); 3745 BUG_ON(!domain);
@@ -3745,11 +3752,14 @@ static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size)
3745 start_pfn = mm_to_dma_pfn(iova_pfn); 3752 start_pfn = mm_to_dma_pfn(iova_pfn);
3746 last_pfn = start_pfn + nrpages - 1; 3753 last_pfn = start_pfn + nrpages - 1;
3747 3754
3755 if (dev_is_pci(dev))
3756 pdev = to_pci_dev(dev);
3757
3748 dev_dbg(dev, "Device unmapping: pfn %lx-%lx\n", start_pfn, last_pfn); 3758 dev_dbg(dev, "Device unmapping: pfn %lx-%lx\n", start_pfn, last_pfn);
3749 3759
3750 freelist = domain_unmap(domain, start_pfn, last_pfn); 3760 freelist = domain_unmap(domain, start_pfn, last_pfn);
3751 3761
3752 if (intel_iommu_strict) { 3762 if (intel_iommu_strict || (pdev && pdev->untrusted)) {
3753 iommu_flush_iotlb_psi(iommu, domain, start_pfn, 3763 iommu_flush_iotlb_psi(iommu, domain, start_pfn,
3754 nrpages, !freelist, 0); 3764 nrpages, !freelist, 0);
3755 /* free iova */ 3765 /* free iova */
@@ -3769,7 +3779,17 @@ static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3769 size_t size, enum dma_data_direction dir, 3779 size_t size, enum dma_data_direction dir,
3770 unsigned long attrs) 3780 unsigned long attrs)
3771{ 3781{
3772 intel_unmap(dev, dev_addr, size); 3782 if (iommu_need_mapping(dev))
3783 intel_unmap(dev, dev_addr, size);
3784 else
3785 dma_direct_unmap_page(dev, dev_addr, size, dir, attrs);
3786}
3787
3788static void intel_unmap_resource(struct device *dev, dma_addr_t dev_addr,
3789 size_t size, enum dma_data_direction dir, unsigned long attrs)
3790{
3791 if (iommu_need_mapping(dev))
3792 intel_unmap(dev, dev_addr, size);
3773} 3793}
3774 3794
3775static void *intel_alloc_coherent(struct device *dev, size_t size, 3795static void *intel_alloc_coherent(struct device *dev, size_t size,
@@ -3779,28 +3799,17 @@ static void *intel_alloc_coherent(struct device *dev, size_t size,
3779 struct page *page = NULL; 3799 struct page *page = NULL;
3780 int order; 3800 int order;
3781 3801
3802 if (!iommu_need_mapping(dev))
3803 return dma_direct_alloc(dev, size, dma_handle, flags, attrs);
3804
3782 size = PAGE_ALIGN(size); 3805 size = PAGE_ALIGN(size);
3783 order = get_order(size); 3806 order = get_order(size);
3784 3807
3785 if (!iommu_no_mapping(dev))
3786 flags &= ~(GFP_DMA | GFP_DMA32);
3787 else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) {
3788 if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
3789 flags |= GFP_DMA;
3790 else
3791 flags |= GFP_DMA32;
3792 }
3793
3794 if (gfpflags_allow_blocking(flags)) { 3808 if (gfpflags_allow_blocking(flags)) {
3795 unsigned int count = size >> PAGE_SHIFT; 3809 unsigned int count = size >> PAGE_SHIFT;
3796 3810
3797 page = dma_alloc_from_contiguous(dev, count, order, 3811 page = dma_alloc_from_contiguous(dev, count, order,
3798 flags & __GFP_NOWARN); 3812 flags & __GFP_NOWARN);
3799 if (page && iommu_no_mapping(dev) &&
3800 page_to_phys(page) + size > dev->coherent_dma_mask) {
3801 dma_release_from_contiguous(dev, page, count);
3802 page = NULL;
3803 }
3804 } 3813 }
3805 3814
3806 if (!page) 3815 if (!page)
@@ -3826,6 +3835,9 @@ static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
3826 int order; 3835 int order;
3827 struct page *page = virt_to_page(vaddr); 3836 struct page *page = virt_to_page(vaddr);
3828 3837
3838 if (!iommu_need_mapping(dev))
3839 return dma_direct_free(dev, size, vaddr, dma_handle, attrs);
3840
3829 size = PAGE_ALIGN(size); 3841 size = PAGE_ALIGN(size);
3830 order = get_order(size); 3842 order = get_order(size);
3831 3843
@@ -3843,6 +3855,9 @@ static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
3843 struct scatterlist *sg; 3855 struct scatterlist *sg;
3844 int i; 3856 int i;
3845 3857
3858 if (!iommu_need_mapping(dev))
3859 return dma_direct_unmap_sg(dev, sglist, nelems, dir, attrs);
3860
3846 for_each_sg(sglist, sg, nelems, i) { 3861 for_each_sg(sglist, sg, nelems, i) {
3847 nrpages += aligned_nrpages(sg_dma_address(sg), sg_dma_len(sg)); 3862 nrpages += aligned_nrpages(sg_dma_address(sg), sg_dma_len(sg));
3848 } 3863 }
@@ -3850,20 +3865,6 @@ static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
3850 intel_unmap(dev, startaddr, nrpages << VTD_PAGE_SHIFT); 3865 intel_unmap(dev, startaddr, nrpages << VTD_PAGE_SHIFT);
3851} 3866}
3852 3867
3853static int intel_nontranslate_map_sg(struct device *hddev,
3854 struct scatterlist *sglist, int nelems, int dir)
3855{
3856 int i;
3857 struct scatterlist *sg;
3858
3859 for_each_sg(sglist, sg, nelems, i) {
3860 BUG_ON(!sg_page(sg));
3861 sg->dma_address = sg_phys(sg);
3862 sg->dma_length = sg->length;
3863 }
3864 return nelems;
3865}
3866
3867static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems, 3868static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
3868 enum dma_data_direction dir, unsigned long attrs) 3869 enum dma_data_direction dir, unsigned long attrs)
3869{ 3870{
@@ -3878,8 +3879,8 @@ static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nele
3878 struct intel_iommu *iommu; 3879 struct intel_iommu *iommu;
3879 3880
3880 BUG_ON(dir == DMA_NONE); 3881 BUG_ON(dir == DMA_NONE);
3881 if (iommu_no_mapping(dev)) 3882 if (!iommu_need_mapping(dev))
3882 return intel_nontranslate_map_sg(dev, sglist, nelems, dir); 3883 return dma_direct_map_sg(dev, sglist, nelems, dir, attrs);
3883 3884
3884 domain = get_valid_domain_for_dev(dev); 3885 domain = get_valid_domain_for_dev(dev);
3885 if (!domain) 3886 if (!domain)
@@ -3929,7 +3930,7 @@ static const struct dma_map_ops intel_dma_ops = {
3929 .map_page = intel_map_page, 3930 .map_page = intel_map_page,
3930 .unmap_page = intel_unmap_page, 3931 .unmap_page = intel_unmap_page,
3931 .map_resource = intel_map_resource, 3932 .map_resource = intel_map_resource,
3932 .unmap_resource = intel_unmap_page, 3933 .unmap_resource = intel_unmap_resource,
3933 .dma_supported = dma_direct_supported, 3934 .dma_supported = dma_direct_supported,
3934}; 3935};
3935 3936
@@ -4055,9 +4056,7 @@ static void __init init_no_remapping_devices(void)
4055 4056
4056 /* This IOMMU has *only* gfx devices. Either bypass it or 4057 /* This IOMMU has *only* gfx devices. Either bypass it or
4057 set the gfx_mapped flag, as appropriate */ 4058 set the gfx_mapped flag, as appropriate */
4058 if (dmar_map_gfx) { 4059 if (!dmar_map_gfx) {
4059 intel_iommu_gfx_mapped = 1;
4060 } else {
4061 drhd->ignored = 1; 4060 drhd->ignored = 1;
4062 for_each_active_dev_scope(drhd->devices, 4061 for_each_active_dev_scope(drhd->devices,
4063 drhd->devices_cnt, i, dev) 4062 drhd->devices_cnt, i, dev)
@@ -4086,7 +4085,7 @@ static int init_iommu_hw(void)
4086 iommu_disable_protect_mem_regions(iommu); 4085 iommu_disable_protect_mem_regions(iommu);
4087 continue; 4086 continue;
4088 } 4087 }
4089 4088
4090 iommu_flush_write_buffer(iommu); 4089 iommu_flush_write_buffer(iommu);
4091 4090
4092 iommu_set_root_entry(iommu); 4091 iommu_set_root_entry(iommu);
@@ -4896,6 +4895,9 @@ int __init intel_iommu_init(void)
4896 goto out_free_reserved_range; 4895 goto out_free_reserved_range;
4897 } 4896 }
4898 4897
4898 if (dmar_map_gfx)
4899 intel_iommu_gfx_mapped = 1;
4900
4899 init_no_remapping_devices(); 4901 init_no_remapping_devices();
4900 4902
4901 ret = init_dmars(); 4903 ret = init_dmars();
@@ -5065,35 +5067,139 @@ static void intel_iommu_domain_free(struct iommu_domain *domain)
5065 domain_exit(to_dmar_domain(domain)); 5067 domain_exit(to_dmar_domain(domain));
5066} 5068}
5067 5069
5068static int intel_iommu_attach_device(struct iommu_domain *domain, 5070/*
5069 struct device *dev) 5071 * Check whether a @domain could be attached to the @dev through the
5072 * aux-domain attach/detach APIs.
5073 */
5074static inline bool
5075is_aux_domain(struct device *dev, struct iommu_domain *domain)
5070{ 5076{
5071 struct dmar_domain *dmar_domain = to_dmar_domain(domain); 5077 struct device_domain_info *info = dev->archdata.iommu;
5072 struct intel_iommu *iommu;
5073 int addr_width;
5074 u8 bus, devfn;
5075 5078
5076 if (device_is_rmrr_locked(dev)) { 5079 return info && info->auxd_enabled &&
5077 dev_warn(dev, "Device is ineligible for IOMMU domain attach due to platform RMRR requirement. Contact your platform vendor.\n"); 5080 domain->type == IOMMU_DOMAIN_UNMANAGED;
5078 return -EPERM; 5081}
5079 }
5080 5082
5081 /* normally dev is not mapped */ 5083static void auxiliary_link_device(struct dmar_domain *domain,
5082 if (unlikely(domain_context_mapped(dev))) { 5084 struct device *dev)
5083 struct dmar_domain *old_domain; 5085{
5086 struct device_domain_info *info = dev->archdata.iommu;
5084 5087
5085 old_domain = find_domain(dev); 5088 assert_spin_locked(&device_domain_lock);
5086 if (old_domain) { 5089 if (WARN_ON(!info))
5087 rcu_read_lock(); 5090 return;
5088 dmar_remove_one_dev_info(dev);
5089 rcu_read_unlock();
5090 5091
5091 if (!domain_type_is_vm_or_si(old_domain) && 5092 domain->auxd_refcnt++;
5092 list_empty(&old_domain->devices)) 5093 list_add(&domain->auxd, &info->auxiliary_domains);
5093 domain_exit(old_domain); 5094}
5095
5096static void auxiliary_unlink_device(struct dmar_domain *domain,
5097 struct device *dev)
5098{
5099 struct device_domain_info *info = dev->archdata.iommu;
5100
5101 assert_spin_locked(&device_domain_lock);
5102 if (WARN_ON(!info))
5103 return;
5104
5105 list_del(&domain->auxd);
5106 domain->auxd_refcnt--;
5107
5108 if (!domain->auxd_refcnt && domain->default_pasid > 0)
5109 intel_pasid_free_id(domain->default_pasid);
5110}
5111
5112static int aux_domain_add_dev(struct dmar_domain *domain,
5113 struct device *dev)
5114{
5115 int ret;
5116 u8 bus, devfn;
5117 unsigned long flags;
5118 struct intel_iommu *iommu;
5119
5120 iommu = device_to_iommu(dev, &bus, &devfn);
5121 if (!iommu)
5122 return -ENODEV;
5123
5124 if (domain->default_pasid <= 0) {
5125 int pasid;
5126
5127 pasid = intel_pasid_alloc_id(domain, PASID_MIN,
5128 pci_max_pasids(to_pci_dev(dev)),
5129 GFP_KERNEL);
5130 if (pasid <= 0) {
5131 pr_err("Can't allocate default pasid\n");
5132 return -ENODEV;
5094 } 5133 }
5134 domain->default_pasid = pasid;
5095 } 5135 }
5096 5136
5137 spin_lock_irqsave(&device_domain_lock, flags);
5138 /*
5139 * iommu->lock must be held to attach domain to iommu and setup the
5140 * pasid entry for second level translation.
5141 */
5142 spin_lock(&iommu->lock);
5143 ret = domain_attach_iommu(domain, iommu);
5144 if (ret)
5145 goto attach_failed;
5146
5147 /* Setup the PASID entry for mediated devices: */
5148 ret = intel_pasid_setup_second_level(iommu, domain, dev,
5149 domain->default_pasid);
5150 if (ret)
5151 goto table_failed;
5152 spin_unlock(&iommu->lock);
5153
5154 auxiliary_link_device(domain, dev);
5155
5156 spin_unlock_irqrestore(&device_domain_lock, flags);
5157
5158 return 0;
5159
5160table_failed:
5161 domain_detach_iommu(domain, iommu);
5162attach_failed:
5163 spin_unlock(&iommu->lock);
5164 spin_unlock_irqrestore(&device_domain_lock, flags);
5165 if (!domain->auxd_refcnt && domain->default_pasid > 0)
5166 intel_pasid_free_id(domain->default_pasid);
5167
5168 return ret;
5169}
5170
5171static void aux_domain_remove_dev(struct dmar_domain *domain,
5172 struct device *dev)
5173{
5174 struct device_domain_info *info;
5175 struct intel_iommu *iommu;
5176 unsigned long flags;
5177
5178 if (!is_aux_domain(dev, &domain->domain))
5179 return;
5180
5181 spin_lock_irqsave(&device_domain_lock, flags);
5182 info = dev->archdata.iommu;
5183 iommu = info->iommu;
5184
5185 auxiliary_unlink_device(domain, dev);
5186
5187 spin_lock(&iommu->lock);
5188 intel_pasid_tear_down_entry(iommu, dev, domain->default_pasid);
5189 domain_detach_iommu(domain, iommu);
5190 spin_unlock(&iommu->lock);
5191
5192 spin_unlock_irqrestore(&device_domain_lock, flags);
5193}
5194
5195static int prepare_domain_attach_device(struct iommu_domain *domain,
5196 struct device *dev)
5197{
5198 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5199 struct intel_iommu *iommu;
5200 int addr_width;
5201 u8 bus, devfn;
5202
5097 iommu = device_to_iommu(dev, &bus, &devfn); 5203 iommu = device_to_iommu(dev, &bus, &devfn);
5098 if (!iommu) 5204 if (!iommu)
5099 return -ENODEV; 5205 return -ENODEV;
@@ -5126,7 +5232,58 @@ static int intel_iommu_attach_device(struct iommu_domain *domain,
5126 dmar_domain->agaw--; 5232 dmar_domain->agaw--;
5127 } 5233 }
5128 5234
5129 return domain_add_dev_info(dmar_domain, dev); 5235 return 0;
5236}
5237
5238static int intel_iommu_attach_device(struct iommu_domain *domain,
5239 struct device *dev)
5240{
5241 int ret;
5242
5243 if (device_is_rmrr_locked(dev)) {
5244 dev_warn(dev, "Device is ineligible for IOMMU domain attach due to platform RMRR requirement. Contact your platform vendor.\n");
5245 return -EPERM;
5246 }
5247
5248 if (is_aux_domain(dev, domain))
5249 return -EPERM;
5250
5251 /* normally dev is not mapped */
5252 if (unlikely(domain_context_mapped(dev))) {
5253 struct dmar_domain *old_domain;
5254
5255 old_domain = find_domain(dev);
5256 if (old_domain) {
5257 rcu_read_lock();
5258 dmar_remove_one_dev_info(dev);
5259 rcu_read_unlock();
5260
5261 if (!domain_type_is_vm_or_si(old_domain) &&
5262 list_empty(&old_domain->devices))
5263 domain_exit(old_domain);
5264 }
5265 }
5266
5267 ret = prepare_domain_attach_device(domain, dev);
5268 if (ret)
5269 return ret;
5270
5271 return domain_add_dev_info(to_dmar_domain(domain), dev);
5272}
5273
5274static int intel_iommu_aux_attach_device(struct iommu_domain *domain,
5275 struct device *dev)
5276{
5277 int ret;
5278
5279 if (!is_aux_domain(dev, domain))
5280 return -EPERM;
5281
5282 ret = prepare_domain_attach_device(domain, dev);
5283 if (ret)
5284 return ret;
5285
5286 return aux_domain_add_dev(to_dmar_domain(domain), dev);
5130} 5287}
5131 5288
5132static void intel_iommu_detach_device(struct iommu_domain *domain, 5289static void intel_iommu_detach_device(struct iommu_domain *domain,
@@ -5135,6 +5292,12 @@ static void intel_iommu_detach_device(struct iommu_domain *domain,
5135 dmar_remove_one_dev_info(dev); 5292 dmar_remove_one_dev_info(dev);
5136} 5293}
5137 5294
5295static void intel_iommu_aux_detach_device(struct iommu_domain *domain,
5296 struct device *dev)
5297{
5298 aux_domain_remove_dev(to_dmar_domain(domain), dev);
5299}
5300
5138static int intel_iommu_map(struct iommu_domain *domain, 5301static int intel_iommu_map(struct iommu_domain *domain,
5139 unsigned long iova, phys_addr_t hpa, 5302 unsigned long iova, phys_addr_t hpa,
5140 size_t size, int iommu_prot) 5303 size_t size, int iommu_prot)
@@ -5223,6 +5386,42 @@ static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
5223 return phys; 5386 return phys;
5224} 5387}
5225 5388
5389static inline bool scalable_mode_support(void)
5390{
5391 struct dmar_drhd_unit *drhd;
5392 struct intel_iommu *iommu;
5393 bool ret = true;
5394
5395 rcu_read_lock();
5396 for_each_active_iommu(iommu, drhd) {
5397 if (!sm_supported(iommu)) {
5398 ret = false;
5399 break;
5400 }
5401 }
5402 rcu_read_unlock();
5403
5404 return ret;
5405}
5406
5407static inline bool iommu_pasid_support(void)
5408{
5409 struct dmar_drhd_unit *drhd;
5410 struct intel_iommu *iommu;
5411 bool ret = true;
5412
5413 rcu_read_lock();
5414 for_each_active_iommu(iommu, drhd) {
5415 if (!pasid_supported(iommu)) {
5416 ret = false;
5417 break;
5418 }
5419 }
5420 rcu_read_unlock();
5421
5422 return ret;
5423}
5424
5226static bool intel_iommu_capable(enum iommu_cap cap) 5425static bool intel_iommu_capable(enum iommu_cap cap)
5227{ 5426{
5228 if (cap == IOMMU_CAP_CACHE_COHERENCY) 5427 if (cap == IOMMU_CAP_CACHE_COHERENCY)
@@ -5307,8 +5506,7 @@ static void intel_iommu_put_resv_regions(struct device *dev,
5307 } 5506 }
5308} 5507}
5309 5508
5310#ifdef CONFIG_INTEL_IOMMU_SVM 5509int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct device *dev)
5311int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct intel_svm_dev *sdev)
5312{ 5510{
5313 struct device_domain_info *info; 5511 struct device_domain_info *info;
5314 struct context_entry *context; 5512 struct context_entry *context;
@@ -5317,7 +5515,7 @@ int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct intel_svm_dev *sd
5317 u64 ctx_lo; 5515 u64 ctx_lo;
5318 int ret; 5516 int ret;
5319 5517
5320 domain = get_valid_domain_for_dev(sdev->dev); 5518 domain = get_valid_domain_for_dev(dev);
5321 if (!domain) 5519 if (!domain)
5322 return -EINVAL; 5520 return -EINVAL;
5323 5521
@@ -5325,7 +5523,7 @@ int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct intel_svm_dev *sd
5325 spin_lock(&iommu->lock); 5523 spin_lock(&iommu->lock);
5326 5524
5327 ret = -EINVAL; 5525 ret = -EINVAL;
5328 info = sdev->dev->archdata.iommu; 5526 info = dev->archdata.iommu;
5329 if (!info || !info->pasid_supported) 5527 if (!info || !info->pasid_supported)
5330 goto out; 5528 goto out;
5331 5529
@@ -5335,14 +5533,13 @@ int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct intel_svm_dev *sd
5335 5533
5336 ctx_lo = context[0].lo; 5534 ctx_lo = context[0].lo;
5337 5535
5338 sdev->did = FLPT_DEFAULT_DID;
5339 sdev->sid = PCI_DEVID(info->bus, info->devfn);
5340
5341 if (!(ctx_lo & CONTEXT_PASIDE)) { 5536 if (!(ctx_lo & CONTEXT_PASIDE)) {
5342 ctx_lo |= CONTEXT_PASIDE; 5537 ctx_lo |= CONTEXT_PASIDE;
5343 context[0].lo = ctx_lo; 5538 context[0].lo = ctx_lo;
5344 wmb(); 5539 wmb();
5345 iommu->flush.flush_context(iommu, sdev->did, sdev->sid, 5540 iommu->flush.flush_context(iommu,
5541 domain->iommu_did[iommu->seq_id],
5542 PCI_DEVID(info->bus, info->devfn),
5346 DMA_CCMD_MASK_NOBIT, 5543 DMA_CCMD_MASK_NOBIT,
5347 DMA_CCMD_DEVICE_INVL); 5544 DMA_CCMD_DEVICE_INVL);
5348 } 5545 }
@@ -5351,12 +5548,6 @@ int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct intel_svm_dev *sd
5351 if (!info->pasid_enabled) 5548 if (!info->pasid_enabled)
5352 iommu_enable_dev_iotlb(info); 5549 iommu_enable_dev_iotlb(info);
5353 5550
5354 if (info->ats_enabled) {
5355 sdev->dev_iotlb = 1;
5356 sdev->qdep = info->ats_qdep;
5357 if (sdev->qdep >= QI_DEV_EIOTLB_MAX_INVS)
5358 sdev->qdep = 0;
5359 }
5360 ret = 0; 5551 ret = 0;
5361 5552
5362 out: 5553 out:
@@ -5366,6 +5557,7 @@ int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct intel_svm_dev *sd
5366 return ret; 5557 return ret;
5367} 5558}
5368 5559
5560#ifdef CONFIG_INTEL_IOMMU_SVM
5369struct intel_iommu *intel_svm_device_to_iommu(struct device *dev) 5561struct intel_iommu *intel_svm_device_to_iommu(struct device *dev)
5370{ 5562{
5371 struct intel_iommu *iommu; 5563 struct intel_iommu *iommu;
@@ -5387,12 +5579,142 @@ struct intel_iommu *intel_svm_device_to_iommu(struct device *dev)
5387} 5579}
5388#endif /* CONFIG_INTEL_IOMMU_SVM */ 5580#endif /* CONFIG_INTEL_IOMMU_SVM */
5389 5581
5582static int intel_iommu_enable_auxd(struct device *dev)
5583{
5584 struct device_domain_info *info;
5585 struct intel_iommu *iommu;
5586 unsigned long flags;
5587 u8 bus, devfn;
5588 int ret;
5589
5590 iommu = device_to_iommu(dev, &bus, &devfn);
5591 if (!iommu || dmar_disabled)
5592 return -EINVAL;
5593
5594 if (!sm_supported(iommu) || !pasid_supported(iommu))
5595 return -EINVAL;
5596
5597 ret = intel_iommu_enable_pasid(iommu, dev);
5598 if (ret)
5599 return -ENODEV;
5600
5601 spin_lock_irqsave(&device_domain_lock, flags);
5602 info = dev->archdata.iommu;
5603 info->auxd_enabled = 1;
5604 spin_unlock_irqrestore(&device_domain_lock, flags);
5605
5606 return 0;
5607}
5608
5609static int intel_iommu_disable_auxd(struct device *dev)
5610{
5611 struct device_domain_info *info;
5612 unsigned long flags;
5613
5614 spin_lock_irqsave(&device_domain_lock, flags);
5615 info = dev->archdata.iommu;
5616 if (!WARN_ON(!info))
5617 info->auxd_enabled = 0;
5618 spin_unlock_irqrestore(&device_domain_lock, flags);
5619
5620 return 0;
5621}
5622
5623/*
5624 * A PCI express designated vendor specific extended capability is defined
5625 * in the section 3.7 of Intel scalable I/O virtualization technical spec
5626 * for system software and tools to detect endpoint devices supporting the
5627 * Intel scalable IO virtualization without host driver dependency.
5628 *
5629 * Returns the address of the matching extended capability structure within
5630 * the device's PCI configuration space or 0 if the device does not support
5631 * it.
5632 */
5633static int siov_find_pci_dvsec(struct pci_dev *pdev)
5634{
5635 int pos;
5636 u16 vendor, id;
5637
5638 pos = pci_find_next_ext_capability(pdev, 0, 0x23);
5639 while (pos) {
5640 pci_read_config_word(pdev, pos + 4, &vendor);
5641 pci_read_config_word(pdev, pos + 8, &id);
5642 if (vendor == PCI_VENDOR_ID_INTEL && id == 5)
5643 return pos;
5644
5645 pos = pci_find_next_ext_capability(pdev, pos, 0x23);
5646 }
5647
5648 return 0;
5649}
5650
5651static bool
5652intel_iommu_dev_has_feat(struct device *dev, enum iommu_dev_features feat)
5653{
5654 if (feat == IOMMU_DEV_FEAT_AUX) {
5655 int ret;
5656
5657 if (!dev_is_pci(dev) || dmar_disabled ||
5658 !scalable_mode_support() || !iommu_pasid_support())
5659 return false;
5660
5661 ret = pci_pasid_features(to_pci_dev(dev));
5662 if (ret < 0)
5663 return false;
5664
5665 return !!siov_find_pci_dvsec(to_pci_dev(dev));
5666 }
5667
5668 return false;
5669}
5670
5671static int
5672intel_iommu_dev_enable_feat(struct device *dev, enum iommu_dev_features feat)
5673{
5674 if (feat == IOMMU_DEV_FEAT_AUX)
5675 return intel_iommu_enable_auxd(dev);
5676
5677 return -ENODEV;
5678}
5679
5680static int
5681intel_iommu_dev_disable_feat(struct device *dev, enum iommu_dev_features feat)
5682{
5683 if (feat == IOMMU_DEV_FEAT_AUX)
5684 return intel_iommu_disable_auxd(dev);
5685
5686 return -ENODEV;
5687}
5688
5689static bool
5690intel_iommu_dev_feat_enabled(struct device *dev, enum iommu_dev_features feat)
5691{
5692 struct device_domain_info *info = dev->archdata.iommu;
5693
5694 if (feat == IOMMU_DEV_FEAT_AUX)
5695 return scalable_mode_support() && info && info->auxd_enabled;
5696
5697 return false;
5698}
5699
5700static int
5701intel_iommu_aux_get_pasid(struct iommu_domain *domain, struct device *dev)
5702{
5703 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5704
5705 return dmar_domain->default_pasid > 0 ?
5706 dmar_domain->default_pasid : -EINVAL;
5707}
5708
5390const struct iommu_ops intel_iommu_ops = { 5709const struct iommu_ops intel_iommu_ops = {
5391 .capable = intel_iommu_capable, 5710 .capable = intel_iommu_capable,
5392 .domain_alloc = intel_iommu_domain_alloc, 5711 .domain_alloc = intel_iommu_domain_alloc,
5393 .domain_free = intel_iommu_domain_free, 5712 .domain_free = intel_iommu_domain_free,
5394 .attach_dev = intel_iommu_attach_device, 5713 .attach_dev = intel_iommu_attach_device,
5395 .detach_dev = intel_iommu_detach_device, 5714 .detach_dev = intel_iommu_detach_device,
5715 .aux_attach_dev = intel_iommu_aux_attach_device,
5716 .aux_detach_dev = intel_iommu_aux_detach_device,
5717 .aux_get_pasid = intel_iommu_aux_get_pasid,
5396 .map = intel_iommu_map, 5718 .map = intel_iommu_map,
5397 .unmap = intel_iommu_unmap, 5719 .unmap = intel_iommu_unmap,
5398 .iova_to_phys = intel_iommu_iova_to_phys, 5720 .iova_to_phys = intel_iommu_iova_to_phys,
@@ -5401,6 +5723,10 @@ const struct iommu_ops intel_iommu_ops = {
5401 .get_resv_regions = intel_iommu_get_resv_regions, 5723 .get_resv_regions = intel_iommu_get_resv_regions,
5402 .put_resv_regions = intel_iommu_put_resv_regions, 5724 .put_resv_regions = intel_iommu_put_resv_regions,
5403 .device_group = pci_device_group, 5725 .device_group = pci_device_group,
5726 .dev_has_feat = intel_iommu_dev_has_feat,
5727 .dev_feat_enabled = intel_iommu_dev_feat_enabled,
5728 .dev_enable_feat = intel_iommu_dev_enable_feat,
5729 .dev_disable_feat = intel_iommu_dev_disable_feat,
5404 .pgsize_bitmap = INTEL_IOMMU_PGSIZES, 5730 .pgsize_bitmap = INTEL_IOMMU_PGSIZES,
5405}; 5731};
5406 5732
diff --git a/drivers/iommu/intel-pasid.c b/drivers/iommu/intel-pasid.c
index 03b12d2ee213..2fefeafda437 100644
--- a/drivers/iommu/intel-pasid.c
+++ b/drivers/iommu/intel-pasid.c
@@ -154,8 +154,10 @@ int intel_pasid_alloc_table(struct device *dev)
154 order = size ? get_order(size) : 0; 154 order = size ? get_order(size) : 0;
155 pages = alloc_pages_node(info->iommu->node, 155 pages = alloc_pages_node(info->iommu->node,
156 GFP_KERNEL | __GFP_ZERO, order); 156 GFP_KERNEL | __GFP_ZERO, order);
157 if (!pages) 157 if (!pages) {
158 kfree(pasid_table);
158 return -ENOMEM; 159 return -ENOMEM;
160 }
159 161
160 pasid_table->table = page_address(pages); 162 pasid_table->table = page_address(pages);
161 pasid_table->order = order; 163 pasid_table->order = order;
diff --git a/drivers/iommu/intel-svm.c b/drivers/iommu/intel-svm.c
index 3a4b09ae8561..8f87304f915c 100644
--- a/drivers/iommu/intel-svm.c
+++ b/drivers/iommu/intel-svm.c
@@ -228,6 +228,7 @@ static LIST_HEAD(global_svm_list);
228int intel_svm_bind_mm(struct device *dev, int *pasid, int flags, struct svm_dev_ops *ops) 228int intel_svm_bind_mm(struct device *dev, int *pasid, int flags, struct svm_dev_ops *ops)
229{ 229{
230 struct intel_iommu *iommu = intel_svm_device_to_iommu(dev); 230 struct intel_iommu *iommu = intel_svm_device_to_iommu(dev);
231 struct device_domain_info *info;
231 struct intel_svm_dev *sdev; 232 struct intel_svm_dev *sdev;
232 struct intel_svm *svm = NULL; 233 struct intel_svm *svm = NULL;
233 struct mm_struct *mm = NULL; 234 struct mm_struct *mm = NULL;
@@ -291,13 +292,29 @@ int intel_svm_bind_mm(struct device *dev, int *pasid, int flags, struct svm_dev_
291 } 292 }
292 sdev->dev = dev; 293 sdev->dev = dev;
293 294
294 ret = intel_iommu_enable_pasid(iommu, sdev); 295 ret = intel_iommu_enable_pasid(iommu, dev);
295 if (ret || !pasid) { 296 if (ret || !pasid) {
296 /* If they don't actually want to assign a PASID, this is 297 /* If they don't actually want to assign a PASID, this is
297 * just an enabling check/preparation. */ 298 * just an enabling check/preparation. */
298 kfree(sdev); 299 kfree(sdev);
299 goto out; 300 goto out;
300 } 301 }
302
303 info = dev->archdata.iommu;
304 if (!info || !info->pasid_supported) {
305 kfree(sdev);
306 goto out;
307 }
308
309 sdev->did = FLPT_DEFAULT_DID;
310 sdev->sid = PCI_DEVID(info->bus, info->devfn);
311 if (info->ats_enabled) {
312 sdev->dev_iotlb = 1;
313 sdev->qdep = info->ats_qdep;
314 if (sdev->qdep >= QI_DEV_EIOTLB_MAX_INVS)
315 sdev->qdep = 0;
316 }
317
301 /* Finish the setup now we know we're keeping it */ 318 /* Finish the setup now we know we're keeping it */
302 sdev->users = 1; 319 sdev->users = 1;
303 sdev->ops = ops; 320 sdev->ops = ops;
diff --git a/drivers/iommu/intel_irq_remapping.c b/drivers/iommu/intel_irq_remapping.c
index 2d74641b7f7b..634d8f059019 100644
--- a/drivers/iommu/intel_irq_remapping.c
+++ b/drivers/iommu/intel_irq_remapping.c
@@ -548,8 +548,7 @@ static int intel_setup_irq_remapping(struct intel_iommu *iommu)
548 goto out_free_table; 548 goto out_free_table;
549 } 549 }
550 550
551 bitmap = kcalloc(BITS_TO_LONGS(INTR_REMAP_TABLE_ENTRIES), 551 bitmap = bitmap_zalloc(INTR_REMAP_TABLE_ENTRIES, GFP_ATOMIC);
552 sizeof(long), GFP_ATOMIC);
553 if (bitmap == NULL) { 552 if (bitmap == NULL) {
554 pr_err("IR%d: failed to allocate bitmap\n", iommu->seq_id); 553 pr_err("IR%d: failed to allocate bitmap\n", iommu->seq_id);
555 goto out_free_pages; 554 goto out_free_pages;
@@ -616,7 +615,7 @@ static int intel_setup_irq_remapping(struct intel_iommu *iommu)
616 return 0; 615 return 0;
617 616
618out_free_bitmap: 617out_free_bitmap:
619 kfree(bitmap); 618 bitmap_free(bitmap);
620out_free_pages: 619out_free_pages:
621 __free_pages(pages, INTR_REMAP_PAGE_ORDER); 620 __free_pages(pages, INTR_REMAP_PAGE_ORDER);
622out_free_table: 621out_free_table:
@@ -640,7 +639,7 @@ static void intel_teardown_irq_remapping(struct intel_iommu *iommu)
640 } 639 }
641 free_pages((unsigned long)iommu->ir_table->base, 640 free_pages((unsigned long)iommu->ir_table->base,
642 INTR_REMAP_PAGE_ORDER); 641 INTR_REMAP_PAGE_ORDER);
643 kfree(iommu->ir_table->bitmap); 642 bitmap_free(iommu->ir_table->bitmap);
644 kfree(iommu->ir_table); 643 kfree(iommu->ir_table);
645 iommu->ir_table = NULL; 644 iommu->ir_table = NULL;
646 } 645 }
diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
index 109de67d5d72..67ee6623f9b2 100644
--- a/drivers/iommu/iommu.c
+++ b/drivers/iommu/iommu.c
@@ -45,10 +45,6 @@ static unsigned int iommu_def_domain_type = IOMMU_DOMAIN_DMA;
45#endif 45#endif
46static bool iommu_dma_strict __read_mostly = true; 46static bool iommu_dma_strict __read_mostly = true;
47 47
48struct iommu_callback_data {
49 const struct iommu_ops *ops;
50};
51
52struct iommu_group { 48struct iommu_group {
53 struct kobject kobj; 49 struct kobject kobj;
54 struct kobject *devices_kobj; 50 struct kobject *devices_kobj;
@@ -1217,9 +1213,6 @@ static int iommu_bus_init(struct bus_type *bus, const struct iommu_ops *ops)
1217{ 1213{
1218 int err; 1214 int err;
1219 struct notifier_block *nb; 1215 struct notifier_block *nb;
1220 struct iommu_callback_data cb = {
1221 .ops = ops,
1222 };
1223 1216
1224 nb = kzalloc(sizeof(struct notifier_block), GFP_KERNEL); 1217 nb = kzalloc(sizeof(struct notifier_block), GFP_KERNEL);
1225 if (!nb) 1218 if (!nb)
@@ -1231,7 +1224,7 @@ static int iommu_bus_init(struct bus_type *bus, const struct iommu_ops *ops)
1231 if (err) 1224 if (err)
1232 goto out_free; 1225 goto out_free;
1233 1226
1234 err = bus_for_each_dev(bus, NULL, &cb, add_iommu_group); 1227 err = bus_for_each_dev(bus, NULL, NULL, add_iommu_group);
1235 if (err) 1228 if (err)
1236 goto out_err; 1229 goto out_err;
1237 1230
@@ -1240,7 +1233,7 @@ static int iommu_bus_init(struct bus_type *bus, const struct iommu_ops *ops)
1240 1233
1241out_err: 1234out_err:
1242 /* Clean up */ 1235 /* Clean up */
1243 bus_for_each_dev(bus, NULL, &cb, remove_iommu_group); 1236 bus_for_each_dev(bus, NULL, NULL, remove_iommu_group);
1244 bus_unregister_notifier(bus, nb); 1237 bus_unregister_notifier(bus, nb);
1245 1238
1246out_free: 1239out_free:
@@ -2039,3 +2032,203 @@ int iommu_fwspec_add_ids(struct device *dev, u32 *ids, int num_ids)
2039 return 0; 2032 return 0;
2040} 2033}
2041EXPORT_SYMBOL_GPL(iommu_fwspec_add_ids); 2034EXPORT_SYMBOL_GPL(iommu_fwspec_add_ids);
2035
2036/*
2037 * Per device IOMMU features.
2038 */
2039bool iommu_dev_has_feature(struct device *dev, enum iommu_dev_features feat)
2040{
2041 const struct iommu_ops *ops = dev->bus->iommu_ops;
2042
2043 if (ops && ops->dev_has_feat)
2044 return ops->dev_has_feat(dev, feat);
2045
2046 return false;
2047}
2048EXPORT_SYMBOL_GPL(iommu_dev_has_feature);
2049
2050int iommu_dev_enable_feature(struct device *dev, enum iommu_dev_features feat)
2051{
2052 const struct iommu_ops *ops = dev->bus->iommu_ops;
2053
2054 if (ops && ops->dev_enable_feat)
2055 return ops->dev_enable_feat(dev, feat);
2056
2057 return -ENODEV;
2058}
2059EXPORT_SYMBOL_GPL(iommu_dev_enable_feature);
2060
2061/*
2062 * The device drivers should do the necessary cleanups before calling this.
2063 * For example, before disabling the aux-domain feature, the device driver
2064 * should detach all aux-domains. Otherwise, this will return -EBUSY.
2065 */
2066int iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features feat)
2067{
2068 const struct iommu_ops *ops = dev->bus->iommu_ops;
2069
2070 if (ops && ops->dev_disable_feat)
2071 return ops->dev_disable_feat(dev, feat);
2072
2073 return -EBUSY;
2074}
2075EXPORT_SYMBOL_GPL(iommu_dev_disable_feature);
2076
2077bool iommu_dev_feature_enabled(struct device *dev, enum iommu_dev_features feat)
2078{
2079 const struct iommu_ops *ops = dev->bus->iommu_ops;
2080
2081 if (ops && ops->dev_feat_enabled)
2082 return ops->dev_feat_enabled(dev, feat);
2083
2084 return false;
2085}
2086EXPORT_SYMBOL_GPL(iommu_dev_feature_enabled);
2087
2088/*
2089 * Aux-domain specific attach/detach.
2090 *
2091 * Only works if iommu_dev_feature_enabled(dev, IOMMU_DEV_FEAT_AUX) returns
2092 * true. Also, as long as domains are attached to a device through this
2093 * interface, any tries to call iommu_attach_device() should fail
2094 * (iommu_detach_device() can't fail, so we fail when trying to re-attach).
2095 * This should make us safe against a device being attached to a guest as a
2096 * whole while there are still pasid users on it (aux and sva).
2097 */
2098int iommu_aux_attach_device(struct iommu_domain *domain, struct device *dev)
2099{
2100 int ret = -ENODEV;
2101
2102 if (domain->ops->aux_attach_dev)
2103 ret = domain->ops->aux_attach_dev(domain, dev);
2104
2105 if (!ret)
2106 trace_attach_device_to_domain(dev);
2107
2108 return ret;
2109}
2110EXPORT_SYMBOL_GPL(iommu_aux_attach_device);
2111
2112void iommu_aux_detach_device(struct iommu_domain *domain, struct device *dev)
2113{
2114 if (domain->ops->aux_detach_dev) {
2115 domain->ops->aux_detach_dev(domain, dev);
2116 trace_detach_device_from_domain(dev);
2117 }
2118}
2119EXPORT_SYMBOL_GPL(iommu_aux_detach_device);
2120
2121int iommu_aux_get_pasid(struct iommu_domain *domain, struct device *dev)
2122{
2123 int ret = -ENODEV;
2124
2125 if (domain->ops->aux_get_pasid)
2126 ret = domain->ops->aux_get_pasid(domain, dev);
2127
2128 return ret;
2129}
2130EXPORT_SYMBOL_GPL(iommu_aux_get_pasid);
2131
2132/**
2133 * iommu_sva_bind_device() - Bind a process address space to a device
2134 * @dev: the device
2135 * @mm: the mm to bind, caller must hold a reference to it
2136 *
2137 * Create a bond between device and address space, allowing the device to access
2138 * the mm using the returned PASID. If a bond already exists between @device and
2139 * @mm, it is returned and an additional reference is taken. Caller must call
2140 * iommu_sva_unbind_device() to release each reference.
2141 *
2142 * iommu_dev_enable_feature(dev, IOMMU_DEV_FEAT_SVA) must be called first, to
2143 * initialize the required SVA features.
2144 *
2145 * On error, returns an ERR_PTR value.
2146 */
2147struct iommu_sva *
2148iommu_sva_bind_device(struct device *dev, struct mm_struct *mm, void *drvdata)
2149{
2150 struct iommu_group *group;
2151 struct iommu_sva *handle = ERR_PTR(-EINVAL);
2152 const struct iommu_ops *ops = dev->bus->iommu_ops;
2153
2154 if (!ops || !ops->sva_bind)
2155 return ERR_PTR(-ENODEV);
2156
2157 group = iommu_group_get(dev);
2158 if (!group)
2159 return ERR_PTR(-ENODEV);
2160
2161 /* Ensure device count and domain don't change while we're binding */
2162 mutex_lock(&group->mutex);
2163
2164 /*
2165 * To keep things simple, SVA currently doesn't support IOMMU groups
2166 * with more than one device. Existing SVA-capable systems are not
2167 * affected by the problems that required IOMMU groups (lack of ACS
2168 * isolation, device ID aliasing and other hardware issues).
2169 */
2170 if (iommu_group_device_count(group) != 1)
2171 goto out_unlock;
2172
2173 handle = ops->sva_bind(dev, mm, drvdata);
2174
2175out_unlock:
2176 mutex_unlock(&group->mutex);
2177 iommu_group_put(group);
2178
2179 return handle;
2180}
2181EXPORT_SYMBOL_GPL(iommu_sva_bind_device);
2182
2183/**
2184 * iommu_sva_unbind_device() - Remove a bond created with iommu_sva_bind_device
2185 * @handle: the handle returned by iommu_sva_bind_device()
2186 *
2187 * Put reference to a bond between device and address space. The device should
2188 * not be issuing any more transaction for this PASID. All outstanding page
2189 * requests for this PASID must have been flushed to the IOMMU.
2190 *
2191 * Returns 0 on success, or an error value
2192 */
2193void iommu_sva_unbind_device(struct iommu_sva *handle)
2194{
2195 struct iommu_group *group;
2196 struct device *dev = handle->dev;
2197 const struct iommu_ops *ops = dev->bus->iommu_ops;
2198
2199 if (!ops || !ops->sva_unbind)
2200 return;
2201
2202 group = iommu_group_get(dev);
2203 if (!group)
2204 return;
2205
2206 mutex_lock(&group->mutex);
2207 ops->sva_unbind(handle);
2208 mutex_unlock(&group->mutex);
2209
2210 iommu_group_put(group);
2211}
2212EXPORT_SYMBOL_GPL(iommu_sva_unbind_device);
2213
2214int iommu_sva_set_ops(struct iommu_sva *handle,
2215 const struct iommu_sva_ops *sva_ops)
2216{
2217 if (handle->ops && handle->ops != sva_ops)
2218 return -EEXIST;
2219
2220 handle->ops = sva_ops;
2221 return 0;
2222}
2223EXPORT_SYMBOL_GPL(iommu_sva_set_ops);
2224
2225int iommu_sva_get_pasid(struct iommu_sva *handle)
2226{
2227 const struct iommu_ops *ops = handle->dev->bus->iommu_ops;
2228
2229 if (!ops || !ops->sva_get_pasid)
2230 return IOMMU_PASID_INVALID;
2231
2232 return ops->sva_get_pasid(handle);
2233}
2234EXPORT_SYMBOL_GPL(iommu_sva_get_pasid);
diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c
index de3e02277b70..b66d11b0286e 100644
--- a/drivers/iommu/mtk_iommu.c
+++ b/drivers/iommu/mtk_iommu.c
@@ -632,16 +632,20 @@ static int mtk_iommu_probe(struct platform_device *pdev)
632 if (!larbnode) 632 if (!larbnode)
633 return -EINVAL; 633 return -EINVAL;
634 634
635 if (!of_device_is_available(larbnode)) 635 if (!of_device_is_available(larbnode)) {
636 of_node_put(larbnode);
636 continue; 637 continue;
638 }
637 639
638 ret = of_property_read_u32(larbnode, "mediatek,larb-id", &id); 640 ret = of_property_read_u32(larbnode, "mediatek,larb-id", &id);
639 if (ret)/* The id is consecutive if there is no this property */ 641 if (ret)/* The id is consecutive if there is no this property */
640 id = i; 642 id = i;
641 643
642 plarbdev = of_find_device_by_node(larbnode); 644 plarbdev = of_find_device_by_node(larbnode);
643 if (!plarbdev) 645 if (!plarbdev) {
646 of_node_put(larbnode);
644 return -EPROBE_DEFER; 647 return -EPROBE_DEFER;
648 }
645 data->smi_imu.larb_imu[id].dev = &plarbdev->dev; 649 data->smi_imu.larb_imu[id].dev = &plarbdev->dev;
646 650
647 component_match_add_release(dev, &match, release_of, 651 component_match_add_release(dev, &match, release_of,
diff --git a/drivers/iommu/tegra-smmu.c b/drivers/iommu/tegra-smmu.c
index 5182c7d6171e..463ee08f7d3a 100644
--- a/drivers/iommu/tegra-smmu.c
+++ b/drivers/iommu/tegra-smmu.c
@@ -102,7 +102,6 @@ static inline u32 smmu_readl(struct tegra_smmu *smmu, unsigned long offset)
102#define SMMU_TLB_FLUSH_VA_MATCH_ALL (0 << 0) 102#define SMMU_TLB_FLUSH_VA_MATCH_ALL (0 << 0)
103#define SMMU_TLB_FLUSH_VA_MATCH_SECTION (2 << 0) 103#define SMMU_TLB_FLUSH_VA_MATCH_SECTION (2 << 0)
104#define SMMU_TLB_FLUSH_VA_MATCH_GROUP (3 << 0) 104#define SMMU_TLB_FLUSH_VA_MATCH_GROUP (3 << 0)
105#define SMMU_TLB_FLUSH_ASID(x) (((x) & 0x7f) << 24)
106#define SMMU_TLB_FLUSH_VA_SECTION(addr) ((((addr) & 0xffc00000) >> 12) | \ 105#define SMMU_TLB_FLUSH_VA_SECTION(addr) ((((addr) & 0xffc00000) >> 12) | \
107 SMMU_TLB_FLUSH_VA_MATCH_SECTION) 106 SMMU_TLB_FLUSH_VA_MATCH_SECTION)
108#define SMMU_TLB_FLUSH_VA_GROUP(addr) ((((addr) & 0xffffc000) >> 12) | \ 107#define SMMU_TLB_FLUSH_VA_GROUP(addr) ((((addr) & 0xffffc000) >> 12) | \
@@ -146,8 +145,6 @@ static inline u32 smmu_readl(struct tegra_smmu *smmu, unsigned long offset)
146 145
147#define SMMU_PDE_ATTR (SMMU_PDE_READABLE | SMMU_PDE_WRITABLE | \ 146#define SMMU_PDE_ATTR (SMMU_PDE_READABLE | SMMU_PDE_WRITABLE | \
148 SMMU_PDE_NONSECURE) 147 SMMU_PDE_NONSECURE)
149#define SMMU_PTE_ATTR (SMMU_PTE_READABLE | SMMU_PTE_WRITABLE | \
150 SMMU_PTE_NONSECURE)
151 148
152static unsigned int iova_pd_index(unsigned long iova) 149static unsigned int iova_pd_index(unsigned long iova)
153{ 150{
@@ -205,8 +202,12 @@ static inline void smmu_flush_tlb_asid(struct tegra_smmu *smmu,
205{ 202{
206 u32 value; 203 u32 value;
207 204
208 value = SMMU_TLB_FLUSH_ASID_MATCH | SMMU_TLB_FLUSH_ASID(asid) | 205 if (smmu->soc->num_asids == 4)
209 SMMU_TLB_FLUSH_VA_MATCH_ALL; 206 value = (asid & 0x3) << 29;
207 else
208 value = (asid & 0x7f) << 24;
209
210 value |= SMMU_TLB_FLUSH_ASID_MATCH | SMMU_TLB_FLUSH_VA_MATCH_ALL;
210 smmu_writel(smmu, value, SMMU_TLB_FLUSH); 211 smmu_writel(smmu, value, SMMU_TLB_FLUSH);
211} 212}
212 213
@@ -216,8 +217,12 @@ static inline void smmu_flush_tlb_section(struct tegra_smmu *smmu,
216{ 217{
217 u32 value; 218 u32 value;
218 219
219 value = SMMU_TLB_FLUSH_ASID_MATCH | SMMU_TLB_FLUSH_ASID(asid) | 220 if (smmu->soc->num_asids == 4)
220 SMMU_TLB_FLUSH_VA_SECTION(iova); 221 value = (asid & 0x3) << 29;
222 else
223 value = (asid & 0x7f) << 24;
224
225 value |= SMMU_TLB_FLUSH_ASID_MATCH | SMMU_TLB_FLUSH_VA_SECTION(iova);
221 smmu_writel(smmu, value, SMMU_TLB_FLUSH); 226 smmu_writel(smmu, value, SMMU_TLB_FLUSH);
222} 227}
223 228
@@ -227,8 +232,12 @@ static inline void smmu_flush_tlb_group(struct tegra_smmu *smmu,
227{ 232{
228 u32 value; 233 u32 value;
229 234
230 value = SMMU_TLB_FLUSH_ASID_MATCH | SMMU_TLB_FLUSH_ASID(asid) | 235 if (smmu->soc->num_asids == 4)
231 SMMU_TLB_FLUSH_VA_GROUP(iova); 236 value = (asid & 0x3) << 29;
237 else
238 value = (asid & 0x7f) << 24;
239
240 value |= SMMU_TLB_FLUSH_ASID_MATCH | SMMU_TLB_FLUSH_VA_GROUP(iova);
232 smmu_writel(smmu, value, SMMU_TLB_FLUSH); 241 smmu_writel(smmu, value, SMMU_TLB_FLUSH);
233} 242}
234 243
@@ -316,6 +325,9 @@ static void tegra_smmu_domain_free(struct iommu_domain *domain)
316 325
317 /* TODO: free page directory and page tables */ 326 /* TODO: free page directory and page tables */
318 327
328 WARN_ON_ONCE(as->use_count);
329 kfree(as->count);
330 kfree(as->pts);
319 kfree(as); 331 kfree(as);
320} 332}
321 333
@@ -645,6 +657,7 @@ static int tegra_smmu_map(struct iommu_domain *domain, unsigned long iova,
645{ 657{
646 struct tegra_smmu_as *as = to_smmu_as(domain); 658 struct tegra_smmu_as *as = to_smmu_as(domain);
647 dma_addr_t pte_dma; 659 dma_addr_t pte_dma;
660 u32 pte_attrs;
648 u32 *pte; 661 u32 *pte;
649 662
650 pte = as_get_pte(as, iova, &pte_dma); 663 pte = as_get_pte(as, iova, &pte_dma);
@@ -655,8 +668,16 @@ static int tegra_smmu_map(struct iommu_domain *domain, unsigned long iova,
655 if (*pte == 0) 668 if (*pte == 0)
656 tegra_smmu_pte_get_use(as, iova); 669 tegra_smmu_pte_get_use(as, iova);
657 670
671 pte_attrs = SMMU_PTE_NONSECURE;
672
673 if (prot & IOMMU_READ)
674 pte_attrs |= SMMU_PTE_READABLE;
675
676 if (prot & IOMMU_WRITE)
677 pte_attrs |= SMMU_PTE_WRITABLE;
678
658 tegra_smmu_set_pte(as, iova, pte, pte_dma, 679 tegra_smmu_set_pte(as, iova, pte, pte_dma,
659 __phys_to_pfn(paddr) | SMMU_PTE_ATTR); 680 __phys_to_pfn(paddr) | pte_attrs);
660 681
661 return 0; 682 return 0;
662} 683}
diff --git a/drivers/vfio/mdev/mdev_core.c b/drivers/vfio/mdev/mdev_core.c
index 1040a4a2dcbc..3cc1a05fde1c 100644
--- a/drivers/vfio/mdev/mdev_core.c
+++ b/drivers/vfio/mdev/mdev_core.c
@@ -388,6 +388,24 @@ int mdev_device_remove(struct device *dev, bool force_remove)
388 return 0; 388 return 0;
389} 389}
390 390
391int mdev_set_iommu_device(struct device *dev, struct device *iommu_device)
392{
393 struct mdev_device *mdev = to_mdev_device(dev);
394
395 mdev->iommu_device = iommu_device;
396
397 return 0;
398}
399EXPORT_SYMBOL(mdev_set_iommu_device);
400
401struct device *mdev_get_iommu_device(struct device *dev)
402{
403 struct mdev_device *mdev = to_mdev_device(dev);
404
405 return mdev->iommu_device;
406}
407EXPORT_SYMBOL(mdev_get_iommu_device);
408
391static int __init mdev_init(void) 409static int __init mdev_init(void)
392{ 410{
393 return mdev_bus_register(); 411 return mdev_bus_register();
diff --git a/drivers/vfio/mdev/mdev_private.h b/drivers/vfio/mdev/mdev_private.h
index ddcf9c72bd8a..36cbbdb754de 100644
--- a/drivers/vfio/mdev/mdev_private.h
+++ b/drivers/vfio/mdev/mdev_private.h
@@ -32,6 +32,7 @@ struct mdev_device {
32 void *driver_data; 32 void *driver_data;
33 struct list_head next; 33 struct list_head next;
34 struct kobject *type_kobj; 34 struct kobject *type_kobj;
35 struct device *iommu_device;
35 bool active; 36 bool active;
36}; 37};
37 38
diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c
index d0f731c9920a..3be1db3501cc 100644
--- a/drivers/vfio/vfio_iommu_type1.c
+++ b/drivers/vfio/vfio_iommu_type1.c
@@ -97,6 +97,7 @@ struct vfio_dma {
97struct vfio_group { 97struct vfio_group {
98 struct iommu_group *iommu_group; 98 struct iommu_group *iommu_group;
99 struct list_head next; 99 struct list_head next;
100 bool mdev_group; /* An mdev group */
100}; 101};
101 102
102/* 103/*
@@ -564,7 +565,7 @@ static int vfio_iommu_type1_pin_pages(void *iommu_data,
564 mutex_lock(&iommu->lock); 565 mutex_lock(&iommu->lock);
565 566
566 /* Fail if notifier list is empty */ 567 /* Fail if notifier list is empty */
567 if ((!iommu->external_domain) || (!iommu->notifier.head)) { 568 if (!iommu->notifier.head) {
568 ret = -EINVAL; 569 ret = -EINVAL;
569 goto pin_done; 570 goto pin_done;
570 } 571 }
@@ -646,11 +647,6 @@ static int vfio_iommu_type1_unpin_pages(void *iommu_data,
646 647
647 mutex_lock(&iommu->lock); 648 mutex_lock(&iommu->lock);
648 649
649 if (!iommu->external_domain) {
650 mutex_unlock(&iommu->lock);
651 return -EINVAL;
652 }
653
654 do_accounting = !IS_IOMMU_CAP_DOMAIN_IN_CONTAINER(iommu); 650 do_accounting = !IS_IOMMU_CAP_DOMAIN_IN_CONTAINER(iommu);
655 for (i = 0; i < npage; i++) { 651 for (i = 0; i < npage; i++) {
656 struct vfio_dma *dma; 652 struct vfio_dma *dma;
@@ -1311,13 +1307,109 @@ static bool vfio_iommu_has_sw_msi(struct iommu_group *group, phys_addr_t *base)
1311 return ret; 1307 return ret;
1312} 1308}
1313 1309
1310static struct device *vfio_mdev_get_iommu_device(struct device *dev)
1311{
1312 struct device *(*fn)(struct device *dev);
1313 struct device *iommu_device;
1314
1315 fn = symbol_get(mdev_get_iommu_device);
1316 if (fn) {
1317 iommu_device = fn(dev);
1318 symbol_put(mdev_get_iommu_device);
1319
1320 return iommu_device;
1321 }
1322
1323 return NULL;
1324}
1325
1326static int vfio_mdev_attach_domain(struct device *dev, void *data)
1327{
1328 struct iommu_domain *domain = data;
1329 struct device *iommu_device;
1330
1331 iommu_device = vfio_mdev_get_iommu_device(dev);
1332 if (iommu_device) {
1333 if (iommu_dev_feature_enabled(iommu_device, IOMMU_DEV_FEAT_AUX))
1334 return iommu_aux_attach_device(domain, iommu_device);
1335 else
1336 return iommu_attach_device(domain, iommu_device);
1337 }
1338
1339 return -EINVAL;
1340}
1341
1342static int vfio_mdev_detach_domain(struct device *dev, void *data)
1343{
1344 struct iommu_domain *domain = data;
1345 struct device *iommu_device;
1346
1347 iommu_device = vfio_mdev_get_iommu_device(dev);
1348 if (iommu_device) {
1349 if (iommu_dev_feature_enabled(iommu_device, IOMMU_DEV_FEAT_AUX))
1350 iommu_aux_detach_device(domain, iommu_device);
1351 else
1352 iommu_detach_device(domain, iommu_device);
1353 }
1354
1355 return 0;
1356}
1357
1358static int vfio_iommu_attach_group(struct vfio_domain *domain,
1359 struct vfio_group *group)
1360{
1361 if (group->mdev_group)
1362 return iommu_group_for_each_dev(group->iommu_group,
1363 domain->domain,
1364 vfio_mdev_attach_domain);
1365 else
1366 return iommu_attach_group(domain->domain, group->iommu_group);
1367}
1368
1369static void vfio_iommu_detach_group(struct vfio_domain *domain,
1370 struct vfio_group *group)
1371{
1372 if (group->mdev_group)
1373 iommu_group_for_each_dev(group->iommu_group, domain->domain,
1374 vfio_mdev_detach_domain);
1375 else
1376 iommu_detach_group(domain->domain, group->iommu_group);
1377}
1378
1379static bool vfio_bus_is_mdev(struct bus_type *bus)
1380{
1381 struct bus_type *mdev_bus;
1382 bool ret = false;
1383
1384 mdev_bus = symbol_get(mdev_bus_type);
1385 if (mdev_bus) {
1386 ret = (bus == mdev_bus);
1387 symbol_put(mdev_bus_type);
1388 }
1389
1390 return ret;
1391}
1392
1393static int vfio_mdev_iommu_device(struct device *dev, void *data)
1394{
1395 struct device **old = data, *new;
1396
1397 new = vfio_mdev_get_iommu_device(dev);
1398 if (!new || (*old && *old != new))
1399 return -EINVAL;
1400
1401 *old = new;
1402
1403 return 0;
1404}
1405
1314static int vfio_iommu_type1_attach_group(void *iommu_data, 1406static int vfio_iommu_type1_attach_group(void *iommu_data,
1315 struct iommu_group *iommu_group) 1407 struct iommu_group *iommu_group)
1316{ 1408{
1317 struct vfio_iommu *iommu = iommu_data; 1409 struct vfio_iommu *iommu = iommu_data;
1318 struct vfio_group *group; 1410 struct vfio_group *group;
1319 struct vfio_domain *domain, *d; 1411 struct vfio_domain *domain, *d;
1320 struct bus_type *bus = NULL, *mdev_bus; 1412 struct bus_type *bus = NULL;
1321 int ret; 1413 int ret;
1322 bool resv_msi, msi_remap; 1414 bool resv_msi, msi_remap;
1323 phys_addr_t resv_msi_base; 1415 phys_addr_t resv_msi_base;
@@ -1352,23 +1444,30 @@ static int vfio_iommu_type1_attach_group(void *iommu_data,
1352 if (ret) 1444 if (ret)
1353 goto out_free; 1445 goto out_free;
1354 1446
1355 mdev_bus = symbol_get(mdev_bus_type); 1447 if (vfio_bus_is_mdev(bus)) {
1448 struct device *iommu_device = NULL;
1356 1449
1357 if (mdev_bus) { 1450 group->mdev_group = true;
1358 if ((bus == mdev_bus) && !iommu_present(bus)) { 1451
1359 symbol_put(mdev_bus_type); 1452 /* Determine the isolation type */
1453 ret = iommu_group_for_each_dev(iommu_group, &iommu_device,
1454 vfio_mdev_iommu_device);
1455 if (ret || !iommu_device) {
1360 if (!iommu->external_domain) { 1456 if (!iommu->external_domain) {
1361 INIT_LIST_HEAD(&domain->group_list); 1457 INIT_LIST_HEAD(&domain->group_list);
1362 iommu->external_domain = domain; 1458 iommu->external_domain = domain;
1363 } else 1459 } else {
1364 kfree(domain); 1460 kfree(domain);
1461 }
1365 1462
1366 list_add(&group->next, 1463 list_add(&group->next,
1367 &iommu->external_domain->group_list); 1464 &iommu->external_domain->group_list);
1368 mutex_unlock(&iommu->lock); 1465 mutex_unlock(&iommu->lock);
1466
1369 return 0; 1467 return 0;
1370 } 1468 }
1371 symbol_put(mdev_bus_type); 1469
1470 bus = iommu_device->bus;
1372 } 1471 }
1373 1472
1374 domain->domain = iommu_domain_alloc(bus); 1473 domain->domain = iommu_domain_alloc(bus);
@@ -1386,7 +1485,7 @@ static int vfio_iommu_type1_attach_group(void *iommu_data,
1386 goto out_domain; 1485 goto out_domain;
1387 } 1486 }
1388 1487
1389 ret = iommu_attach_group(domain->domain, iommu_group); 1488 ret = vfio_iommu_attach_group(domain, group);
1390 if (ret) 1489 if (ret)
1391 goto out_domain; 1490 goto out_domain;
1392 1491
@@ -1418,8 +1517,8 @@ static int vfio_iommu_type1_attach_group(void *iommu_data,
1418 list_for_each_entry(d, &iommu->domain_list, next) { 1517 list_for_each_entry(d, &iommu->domain_list, next) {
1419 if (d->domain->ops == domain->domain->ops && 1518 if (d->domain->ops == domain->domain->ops &&
1420 d->prot == domain->prot) { 1519 d->prot == domain->prot) {
1421 iommu_detach_group(domain->domain, iommu_group); 1520 vfio_iommu_detach_group(domain, group);
1422 if (!iommu_attach_group(d->domain, iommu_group)) { 1521 if (!vfio_iommu_attach_group(d, group)) {
1423 list_add(&group->next, &d->group_list); 1522 list_add(&group->next, &d->group_list);
1424 iommu_domain_free(domain->domain); 1523 iommu_domain_free(domain->domain);
1425 kfree(domain); 1524 kfree(domain);
@@ -1427,7 +1526,7 @@ static int vfio_iommu_type1_attach_group(void *iommu_data,
1427 return 0; 1526 return 0;
1428 } 1527 }
1429 1528
1430 ret = iommu_attach_group(domain->domain, iommu_group); 1529 ret = vfio_iommu_attach_group(domain, group);
1431 if (ret) 1530 if (ret)
1432 goto out_domain; 1531 goto out_domain;
1433 } 1532 }
@@ -1453,7 +1552,7 @@ static int vfio_iommu_type1_attach_group(void *iommu_data,
1453 return 0; 1552 return 0;
1454 1553
1455out_detach: 1554out_detach:
1456 iommu_detach_group(domain->domain, iommu_group); 1555 vfio_iommu_detach_group(domain, group);
1457out_domain: 1556out_domain:
1458 iommu_domain_free(domain->domain); 1557 iommu_domain_free(domain->domain);
1459out_free: 1558out_free:
@@ -1544,7 +1643,7 @@ static void vfio_iommu_type1_detach_group(void *iommu_data,
1544 if (!group) 1643 if (!group)
1545 continue; 1644 continue;
1546 1645
1547 iommu_detach_group(domain->domain, iommu_group); 1646 vfio_iommu_detach_group(domain, group);
1548 list_del(&group->next); 1647 list_del(&group->next);
1549 kfree(group); 1648 kfree(group);
1550 /* 1649 /*
@@ -1610,7 +1709,7 @@ static void vfio_release_domain(struct vfio_domain *domain, bool external)
1610 list_for_each_entry_safe(group, group_tmp, 1709 list_for_each_entry_safe(group, group_tmp,
1611 &domain->group_list, next) { 1710 &domain->group_list, next) {
1612 if (!external) 1711 if (!external)
1613 iommu_detach_group(domain->domain, group->iommu_group); 1712 vfio_iommu_detach_group(domain, group);
1614 list_del(&group->next); 1713 list_del(&group->next);
1615 kfree(group); 1714 kfree(group);
1616 } 1715 }
diff --git a/include/linux/intel-iommu.h b/include/linux/intel-iommu.h
index fa364de9db18..6925a18a5ca3 100644
--- a/include/linux/intel-iommu.h
+++ b/include/linux/intel-iommu.h
@@ -489,9 +489,11 @@ struct dmar_domain {
489 /* Domain ids per IOMMU. Use u16 since 489 /* Domain ids per IOMMU. Use u16 since
490 * domain ids are 16 bit wide according 490 * domain ids are 16 bit wide according
491 * to VT-d spec, section 9.3 */ 491 * to VT-d spec, section 9.3 */
492 unsigned int auxd_refcnt; /* Refcount of auxiliary attaching */
492 493
493 bool has_iotlb_device; 494 bool has_iotlb_device;
494 struct list_head devices; /* all devices' list */ 495 struct list_head devices; /* all devices' list */
496 struct list_head auxd; /* link to device's auxiliary list */
495 struct iova_domain iovad; /* iova's that belong to this domain */ 497 struct iova_domain iovad; /* iova's that belong to this domain */
496 498
497 struct dma_pte *pgd; /* virtual address */ 499 struct dma_pte *pgd; /* virtual address */
@@ -510,6 +512,11 @@ struct dmar_domain {
510 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */ 512 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
511 u64 max_addr; /* maximum mapped address */ 513 u64 max_addr; /* maximum mapped address */
512 514
515 int default_pasid; /*
516 * The default pasid used for non-SVM
517 * traffic on mediated devices.
518 */
519
513 struct iommu_domain domain; /* generic domain data structure for 520 struct iommu_domain domain; /* generic domain data structure for
514 iommu core */ 521 iommu core */
515}; 522};
@@ -559,6 +566,9 @@ struct device_domain_info {
559 struct list_head link; /* link to domain siblings */ 566 struct list_head link; /* link to domain siblings */
560 struct list_head global; /* link to global list */ 567 struct list_head global; /* link to global list */
561 struct list_head table; /* link to pasid table */ 568 struct list_head table; /* link to pasid table */
569 struct list_head auxiliary_domains; /* auxiliary domains
570 * attached to this device
571 */
562 u8 bus; /* PCI bus number */ 572 u8 bus; /* PCI bus number */
563 u8 devfn; /* PCI devfn number */ 573 u8 devfn; /* PCI devfn number */
564 u16 pfsid; /* SRIOV physical function source ID */ 574 u16 pfsid; /* SRIOV physical function source ID */
@@ -568,6 +578,7 @@ struct device_domain_info {
568 u8 pri_enabled:1; 578 u8 pri_enabled:1;
569 u8 ats_supported:1; 579 u8 ats_supported:1;
570 u8 ats_enabled:1; 580 u8 ats_enabled:1;
581 u8 auxd_enabled:1; /* Multiple domains per device */
571 u8 ats_qdep; 582 u8 ats_qdep;
572 struct device *dev; /* it's NULL for PCIe-to-PCI bridge */ 583 struct device *dev; /* it's NULL for PCIe-to-PCI bridge */
573 struct intel_iommu *iommu; /* IOMMU used by this device */ 584 struct intel_iommu *iommu; /* IOMMU used by this device */
@@ -650,6 +661,7 @@ struct intel_iommu *domain_get_iommu(struct dmar_domain *domain);
650int for_each_device_domain(int (*fn)(struct device_domain_info *info, 661int for_each_device_domain(int (*fn)(struct device_domain_info *info,
651 void *data), void *data); 662 void *data), void *data);
652void iommu_flush_write_buffer(struct intel_iommu *iommu); 663void iommu_flush_write_buffer(struct intel_iommu *iommu);
664int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct device *dev);
653 665
654#ifdef CONFIG_INTEL_IOMMU_SVM 666#ifdef CONFIG_INTEL_IOMMU_SVM
655int intel_svm_init(struct intel_iommu *iommu); 667int intel_svm_init(struct intel_iommu *iommu);
@@ -679,7 +691,6 @@ struct intel_svm {
679 struct list_head list; 691 struct list_head list;
680}; 692};
681 693
682extern int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct intel_svm_dev *sdev);
683extern struct intel_iommu *intel_svm_device_to_iommu(struct device *dev); 694extern struct intel_iommu *intel_svm_device_to_iommu(struct device *dev);
684#endif 695#endif
685 696
diff --git a/include/linux/iommu.h b/include/linux/iommu.h
index ffbbc7e39cee..a815cf6f6f47 100644
--- a/include/linux/iommu.h
+++ b/include/linux/iommu.h
@@ -48,6 +48,7 @@ struct bus_type;
48struct device; 48struct device;
49struct iommu_domain; 49struct iommu_domain;
50struct notifier_block; 50struct notifier_block;
51struct iommu_sva;
51 52
52/* iommu fault flags */ 53/* iommu fault flags */
53#define IOMMU_FAULT_READ 0x0 54#define IOMMU_FAULT_READ 0x0
@@ -55,6 +56,8 @@ struct notifier_block;
55 56
56typedef int (*iommu_fault_handler_t)(struct iommu_domain *, 57typedef int (*iommu_fault_handler_t)(struct iommu_domain *,
57 struct device *, unsigned long, int, void *); 58 struct device *, unsigned long, int, void *);
59typedef int (*iommu_mm_exit_handler_t)(struct device *dev, struct iommu_sva *,
60 void *);
58 61
59struct iommu_domain_geometry { 62struct iommu_domain_geometry {
60 dma_addr_t aperture_start; /* First address that can be mapped */ 63 dma_addr_t aperture_start; /* First address that can be mapped */
@@ -156,6 +159,33 @@ struct iommu_resv_region {
156 enum iommu_resv_type type; 159 enum iommu_resv_type type;
157}; 160};
158 161
162/* Per device IOMMU features */
163enum iommu_dev_features {
164 IOMMU_DEV_FEAT_AUX, /* Aux-domain feature */
165 IOMMU_DEV_FEAT_SVA, /* Shared Virtual Addresses */
166};
167
168#define IOMMU_PASID_INVALID (-1U)
169
170/**
171 * struct iommu_sva_ops - device driver callbacks for an SVA context
172 *
173 * @mm_exit: called when the mm is about to be torn down by exit_mmap. After
174 * @mm_exit returns, the device must not issue any more transaction
175 * with the PASID given as argument.
176 *
177 * The @mm_exit handler is allowed to sleep. Be careful about the
178 * locks taken in @mm_exit, because they might lead to deadlocks if
179 * they are also held when dropping references to the mm. Consider the
180 * following call chain:
181 * mutex_lock(A); mmput(mm) -> exit_mm() -> @mm_exit() -> mutex_lock(A)
182 * Using mmput_async() prevents this scenario.
183 *
184 */
185struct iommu_sva_ops {
186 iommu_mm_exit_handler_t mm_exit;
187};
188
159#ifdef CONFIG_IOMMU_API 189#ifdef CONFIG_IOMMU_API
160 190
161/** 191/**
@@ -186,6 +216,14 @@ struct iommu_resv_region {
186 * @of_xlate: add OF master IDs to iommu grouping 216 * @of_xlate: add OF master IDs to iommu grouping
187 * @is_attach_deferred: Check if domain attach should be deferred from iommu 217 * @is_attach_deferred: Check if domain attach should be deferred from iommu
188 * driver init to device driver init (default no) 218 * driver init to device driver init (default no)
219 * @dev_has/enable/disable_feat: per device entries to check/enable/disable
220 * iommu specific features.
221 * @dev_feat_enabled: check enabled feature
222 * @aux_attach/detach_dev: aux-domain specific attach/detach entries.
223 * @aux_get_pasid: get the pasid given an aux-domain
224 * @sva_bind: Bind process address space to device
225 * @sva_unbind: Unbind process address space from device
226 * @sva_get_pasid: Get PASID associated to a SVA handle
189 * @pgsize_bitmap: bitmap of all possible supported page sizes 227 * @pgsize_bitmap: bitmap of all possible supported page sizes
190 */ 228 */
191struct iommu_ops { 229struct iommu_ops {
@@ -230,6 +268,22 @@ struct iommu_ops {
230 int (*of_xlate)(struct device *dev, struct of_phandle_args *args); 268 int (*of_xlate)(struct device *dev, struct of_phandle_args *args);
231 bool (*is_attach_deferred)(struct iommu_domain *domain, struct device *dev); 269 bool (*is_attach_deferred)(struct iommu_domain *domain, struct device *dev);
232 270
271 /* Per device IOMMU features */
272 bool (*dev_has_feat)(struct device *dev, enum iommu_dev_features f);
273 bool (*dev_feat_enabled)(struct device *dev, enum iommu_dev_features f);
274 int (*dev_enable_feat)(struct device *dev, enum iommu_dev_features f);
275 int (*dev_disable_feat)(struct device *dev, enum iommu_dev_features f);
276
277 /* Aux-domain specific attach/detach entries */
278 int (*aux_attach_dev)(struct iommu_domain *domain, struct device *dev);
279 void (*aux_detach_dev)(struct iommu_domain *domain, struct device *dev);
280 int (*aux_get_pasid)(struct iommu_domain *domain, struct device *dev);
281
282 struct iommu_sva *(*sva_bind)(struct device *dev, struct mm_struct *mm,
283 void *drvdata);
284 void (*sva_unbind)(struct iommu_sva *handle);
285 int (*sva_get_pasid)(struct iommu_sva *handle);
286
233 unsigned long pgsize_bitmap; 287 unsigned long pgsize_bitmap;
234}; 288};
235 289
@@ -392,10 +446,22 @@ struct iommu_fwspec {
392 const struct iommu_ops *ops; 446 const struct iommu_ops *ops;
393 struct fwnode_handle *iommu_fwnode; 447 struct fwnode_handle *iommu_fwnode;
394 void *iommu_priv; 448 void *iommu_priv;
449 u32 flags;
395 unsigned int num_ids; 450 unsigned int num_ids;
396 u32 ids[1]; 451 u32 ids[1];
397}; 452};
398 453
454/* ATS is supported */
455#define IOMMU_FWSPEC_PCI_RC_ATS (1 << 0)
456
457/**
458 * struct iommu_sva - handle to a device-mm bond
459 */
460struct iommu_sva {
461 struct device *dev;
462 const struct iommu_sva_ops *ops;
463};
464
399int iommu_fwspec_init(struct device *dev, struct fwnode_handle *iommu_fwnode, 465int iommu_fwspec_init(struct device *dev, struct fwnode_handle *iommu_fwnode,
400 const struct iommu_ops *ops); 466 const struct iommu_ops *ops);
401void iommu_fwspec_free(struct device *dev); 467void iommu_fwspec_free(struct device *dev);
@@ -416,6 +482,22 @@ static inline void dev_iommu_fwspec_set(struct device *dev,
416int iommu_probe_device(struct device *dev); 482int iommu_probe_device(struct device *dev);
417void iommu_release_device(struct device *dev); 483void iommu_release_device(struct device *dev);
418 484
485bool iommu_dev_has_feature(struct device *dev, enum iommu_dev_features f);
486int iommu_dev_enable_feature(struct device *dev, enum iommu_dev_features f);
487int iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features f);
488bool iommu_dev_feature_enabled(struct device *dev, enum iommu_dev_features f);
489int iommu_aux_attach_device(struct iommu_domain *domain, struct device *dev);
490void iommu_aux_detach_device(struct iommu_domain *domain, struct device *dev);
491int iommu_aux_get_pasid(struct iommu_domain *domain, struct device *dev);
492
493struct iommu_sva *iommu_sva_bind_device(struct device *dev,
494 struct mm_struct *mm,
495 void *drvdata);
496void iommu_sva_unbind_device(struct iommu_sva *handle);
497int iommu_sva_set_ops(struct iommu_sva *handle,
498 const struct iommu_sva_ops *ops);
499int iommu_sva_get_pasid(struct iommu_sva *handle);
500
419#else /* CONFIG_IOMMU_API */ 501#else /* CONFIG_IOMMU_API */
420 502
421struct iommu_ops {}; 503struct iommu_ops {};
@@ -700,6 +782,68 @@ const struct iommu_ops *iommu_ops_from_fwnode(struct fwnode_handle *fwnode)
700 return NULL; 782 return NULL;
701} 783}
702 784
785static inline bool
786iommu_dev_has_feature(struct device *dev, enum iommu_dev_features feat)
787{
788 return false;
789}
790
791static inline bool
792iommu_dev_feature_enabled(struct device *dev, enum iommu_dev_features feat)
793{
794 return false;
795}
796
797static inline int
798iommu_dev_enable_feature(struct device *dev, enum iommu_dev_features feat)
799{
800 return -ENODEV;
801}
802
803static inline int
804iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features feat)
805{
806 return -ENODEV;
807}
808
809static inline int
810iommu_aux_attach_device(struct iommu_domain *domain, struct device *dev)
811{
812 return -ENODEV;
813}
814
815static inline void
816iommu_aux_detach_device(struct iommu_domain *domain, struct device *dev)
817{
818}
819
820static inline int
821iommu_aux_get_pasid(struct iommu_domain *domain, struct device *dev)
822{
823 return -ENODEV;
824}
825
826static inline struct iommu_sva *
827iommu_sva_bind_device(struct device *dev, struct mm_struct *mm, void *drvdata)
828{
829 return NULL;
830}
831
832static inline void iommu_sva_unbind_device(struct iommu_sva *handle)
833{
834}
835
836static inline int iommu_sva_set_ops(struct iommu_sva *handle,
837 const struct iommu_sva_ops *ops)
838{
839 return -EINVAL;
840}
841
842static inline int iommu_sva_get_pasid(struct iommu_sva *handle)
843{
844 return IOMMU_PASID_INVALID;
845}
846
703#endif /* CONFIG_IOMMU_API */ 847#endif /* CONFIG_IOMMU_API */
704 848
705#ifdef CONFIG_IOMMU_DEBUGFS 849#ifdef CONFIG_IOMMU_DEBUGFS
diff --git a/include/linux/iova.h b/include/linux/iova.h
index 0b93bf96693e..28a5128405f8 100644
--- a/include/linux/iova.h
+++ b/include/linux/iova.h
@@ -76,6 +76,14 @@ struct iova_domain {
76 unsigned long start_pfn; /* Lower limit for this domain */ 76 unsigned long start_pfn; /* Lower limit for this domain */
77 unsigned long dma_32bit_pfn; 77 unsigned long dma_32bit_pfn;
78 unsigned long max32_alloc_size; /* Size of last failed allocation */ 78 unsigned long max32_alloc_size; /* Size of last failed allocation */
79 struct iova_fq __percpu *fq; /* Flush Queue */
80
81 atomic64_t fq_flush_start_cnt; /* Number of TLB flushes that
82 have been started */
83
84 atomic64_t fq_flush_finish_cnt; /* Number of TLB flushes that
85 have been finished */
86
79 struct iova anchor; /* rbtree lookup anchor */ 87 struct iova anchor; /* rbtree lookup anchor */
80 struct iova_rcache rcaches[IOVA_RANGE_CACHE_MAX_SIZE]; /* IOVA range caches */ 88 struct iova_rcache rcaches[IOVA_RANGE_CACHE_MAX_SIZE]; /* IOVA range caches */
81 89
@@ -85,14 +93,6 @@ struct iova_domain {
85 iova_entry_dtor entry_dtor; /* IOMMU driver specific destructor for 93 iova_entry_dtor entry_dtor; /* IOMMU driver specific destructor for
86 iova entry */ 94 iova entry */
87 95
88 struct iova_fq __percpu *fq; /* Flush Queue */
89
90 atomic64_t fq_flush_start_cnt; /* Number of TLB flushes that
91 have been started */
92
93 atomic64_t fq_flush_finish_cnt; /* Number of TLB flushes that
94 have been finished */
95
96 struct timer_list fq_timer; /* Timer to regularily empty the 96 struct timer_list fq_timer; /* Timer to regularily empty the
97 flush-queues */ 97 flush-queues */
98 atomic_t fq_timer_on; /* 1 when timer is active, 0 98 atomic_t fq_timer_on; /* 1 when timer is active, 0
diff --git a/include/linux/mdev.h b/include/linux/mdev.h
index 4924d8038814..89a52fd5756e 100644
--- a/include/linux/mdev.h
+++ b/include/linux/mdev.h
@@ -15,6 +15,20 @@
15 15
16struct mdev_device; 16struct mdev_device;
17 17
18/*
19 * Called by the parent device driver to set the device which represents
20 * this mdev in iommu protection scope. By default, the iommu device is
21 * NULL, that indicates using vendor defined isolation.
22 *
23 * @dev: the mediated device that iommu will isolate.
24 * @iommu_device: a pci device which represents the iommu for @dev.
25 *
26 * Return 0 for success, otherwise negative error value.
27 */
28int mdev_set_iommu_device(struct device *dev, struct device *iommu_device);
29
30struct device *mdev_get_iommu_device(struct device *dev);
31
18/** 32/**
19 * struct mdev_parent_ops - Structure to be registered for each parent device to 33 * struct mdev_parent_ops - Structure to be registered for each parent device to
20 * register the device to mdev module. 34 * register the device to mdev module.
diff --git a/include/linux/pci.h b/include/linux/pci.h
index 27854731afc4..1250806dc94a 100644
--- a/include/linux/pci.h
+++ b/include/linux/pci.h
@@ -1521,21 +1521,6 @@ static inline void pcie_ecrc_get_policy(char *str) { }
1521 1521
1522bool pci_ats_disabled(void); 1522bool pci_ats_disabled(void);
1523 1523
1524#ifdef CONFIG_PCI_ATS
1525/* Address Translation Service */
1526void pci_ats_init(struct pci_dev *dev);
1527int pci_enable_ats(struct pci_dev *dev, int ps);
1528void pci_disable_ats(struct pci_dev *dev);
1529int pci_ats_queue_depth(struct pci_dev *dev);
1530int pci_ats_page_aligned(struct pci_dev *dev);
1531#else
1532static inline void pci_ats_init(struct pci_dev *d) { }
1533static inline int pci_enable_ats(struct pci_dev *d, int ps) { return -ENODEV; }
1534static inline void pci_disable_ats(struct pci_dev *d) { }
1535static inline int pci_ats_queue_depth(struct pci_dev *d) { return -ENODEV; }
1536static inline int pci_ats_page_aligned(struct pci_dev *dev) { return 0; }
1537#endif
1538
1539#ifdef CONFIG_PCIE_PTM 1524#ifdef CONFIG_PCIE_PTM
1540int pci_enable_ptm(struct pci_dev *dev, u8 *granularity); 1525int pci_enable_ptm(struct pci_dev *dev, u8 *granularity);
1541#else 1526#else
@@ -1728,8 +1713,24 @@ static inline int pci_irqd_intx_xlate(struct irq_domain *d,
1728static inline const struct pci_device_id *pci_match_id(const struct pci_device_id *ids, 1713static inline const struct pci_device_id *pci_match_id(const struct pci_device_id *ids,
1729 struct pci_dev *dev) 1714 struct pci_dev *dev)
1730{ return NULL; } 1715{ return NULL; }
1716static inline bool pci_ats_disabled(void) { return true; }
1731#endif /* CONFIG_PCI */ 1717#endif /* CONFIG_PCI */
1732 1718
1719#ifdef CONFIG_PCI_ATS
1720/* Address Translation Service */
1721void pci_ats_init(struct pci_dev *dev);
1722int pci_enable_ats(struct pci_dev *dev, int ps);
1723void pci_disable_ats(struct pci_dev *dev);
1724int pci_ats_queue_depth(struct pci_dev *dev);
1725int pci_ats_page_aligned(struct pci_dev *dev);
1726#else
1727static inline void pci_ats_init(struct pci_dev *d) { }
1728static inline int pci_enable_ats(struct pci_dev *d, int ps) { return -ENODEV; }
1729static inline void pci_disable_ats(struct pci_dev *d) { }
1730static inline int pci_ats_queue_depth(struct pci_dev *d) { return -ENODEV; }
1731static inline int pci_ats_page_aligned(struct pci_dev *dev) { return 0; }
1732#endif
1733
1733/* Include architecture-dependent settings and functions */ 1734/* Include architecture-dependent settings and functions */
1734 1735
1735#include <asm/pci.h> 1736#include <asm/pci.h>
generated by cgit v1.2.2 (git 2.25.0) at 2025-09-11 13:30:30 -0400