aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2015-09-08 20:22:35 -0400
committerLinus Torvalds <torvalds@linux-foundation.org>2015-09-08 20:22:35 -0400
commit9a9952bbd76a13fc2c95c28f09ba1801a3664929 (patch)
tree8ec6955ec8706dd28cc43161821b4350ebd80424
parente81b594cdae73f341ea13bc9fb2b57a5b739c1a3 (diff)
parent4ad79562577a3936b08365260f86eeb83156479f (diff)
Merge tag 'iommu-updates-v4.3' of git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu
Pull iommu updates for from Joerg Roedel: "This time the IOMMU updates are mostly cleanups or fixes. No big new features or drivers this time. In particular the changes include: - Bigger cleanup of the Domain<->IOMMU data structures and the code that manages them in the Intel VT-d driver. This makes the code easier to understand and maintain, and also easier to keep the data structures in sync. It is also a preparation step to make use of default domains from the IOMMU core in the Intel VT-d driver. - Fixes for a couple of DMA-API misuses in ARM IOMMU drivers, namely in the ARM and Tegra SMMU drivers. - Fix for a potential buffer overflow in the OMAP iommu driver's debug code - A couple of smaller fixes and cleanups in various drivers - One small new feature: Report domain-id usage in the Intel VT-d driver to easier detect bugs where these are leaked" * tag 'iommu-updates-v4.3' of git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu: (83 commits) iommu/vt-d: Really use upper context table when necessary x86/vt-d: Fix documentation of DRHD iommu/fsl: Really fix init section(s) content iommu/io-pgtable-arm: Unmap and free table when overwriting with block iommu/io-pgtable-arm: Move init-fn declarations to io-pgtable.h iommu/msm: Use BUG_ON instead of if () BUG() iommu/vt-d: Access iomem correctly iommu/vt-d: Make two functions static iommu/vt-d: Use BUG_ON instead of if () BUG() iommu/vt-d: Return false instead of 0 in irq_remapping_cap() iommu/amd: Use BUG_ON instead of if () BUG() iommu/amd: Make a symbol static iommu/amd: Simplify allocation in irq_remapping_alloc() iommu/tegra-smmu: Parameterize number of TLB lines iommu/tegra-smmu: Factor out tegra_smmu_set_pde() iommu/tegra-smmu: Extract tegra_smmu_pte_get_use() iommu/tegra-smmu: Use __GFP_ZERO to allocate zeroed pages iommu/tegra-smmu: Remove PageReserved manipulation iommu/tegra-smmu: Convert to use DMA API iommu/tegra-smmu: smmu_flush_ptc() wants device addresses ...
Diffstat
-rw-r--r--Documentation/devicetree/bindings/iommu/arm,smmu.txt6
-rw-r--r--Documentation/devicetree/bindings/iommu/ti,omap-iommu.txt6
-rw-r--r--drivers/iommu/Kconfig3
-rw-r--r--drivers/iommu/amd_iommu.c21
-rw-r--r--drivers/iommu/amd_iommu_init.c2
-rw-r--r--drivers/iommu/amd_iommu_v2.c4
-rw-r--r--drivers/iommu/arm-smmu-v3.c66
-rw-r--r--drivers/iommu/arm-smmu.c45
-rw-r--r--drivers/iommu/dmar.c2
-rw-r--r--drivers/iommu/fsl_pamu.c26
-rw-r--r--drivers/iommu/intel-iommu.c716
-rw-r--r--drivers/iommu/intel_irq_remapping.c6
-rw-r--r--drivers/iommu/io-pgtable-arm.c143
-rw-r--r--drivers/iommu/io-pgtable.c5
-rw-r--r--drivers/iommu/io-pgtable.h14
-rw-r--r--drivers/iommu/ipmmu-vmsa.c19
-rw-r--r--drivers/iommu/irq_remapping.c2
-rw-r--r--drivers/iommu/msm_iommu.c4
-rw-r--r--drivers/iommu/of_iommu.c8
-rw-r--r--drivers/iommu/omap-iommu-debug.c131
-rw-r--r--drivers/iommu/omap-iommu.c198
-rw-r--r--drivers/iommu/omap-iommu.h79
-rw-r--r--drivers/iommu/omap-iopgtable.h27
-rw-r--r--drivers/iommu/tegra-smmu.c306
-rw-r--r--drivers/memory/tegra/tegra114.c18
-rw-r--r--drivers/memory/tegra/tegra124.c31
-rw-r--r--drivers/memory/tegra/tegra30.c18
-rw-r--r--include/linux/intel-iommu.h2
-rw-r--r--include/soc/tegra/mc.h8
29 files changed, 942 insertions, 974 deletions
diff --git a/Documentation/devicetree/bindings/iommu/arm,smmu.txt b/Documentation/devicetree/bindings/iommu/arm,smmu.txt
index 06760503a819..718074501fcb 100644
--- a/Documentation/devicetree/bindings/iommu/arm,smmu.txt
+++ b/Documentation/devicetree/bindings/iommu/arm,smmu.txt
@@ -43,6 +43,12 @@ conditions.
43 43
44** System MMU optional properties: 44** System MMU optional properties:
45 45
46- dma-coherent : Present if page table walks made by the SMMU are
47 cache coherent with the CPU.
48
49 NOTE: this only applies to the SMMU itself, not
50 masters connected upstream of the SMMU.
51
46- calxeda,smmu-secure-config-access : Enable proper handling of buggy 52- calxeda,smmu-secure-config-access : Enable proper handling of buggy
47 implementations that always use secure access to 53 implementations that always use secure access to
48 SMMU configuration registers. In this case non-secure 54 SMMU configuration registers. In this case non-secure
diff --git a/Documentation/devicetree/bindings/iommu/ti,omap-iommu.txt b/Documentation/devicetree/bindings/iommu/ti,omap-iommu.txt
index 42531dc387aa..869699925fd5 100644
--- a/Documentation/devicetree/bindings/iommu/ti,omap-iommu.txt
+++ b/Documentation/devicetree/bindings/iommu/ti,omap-iommu.txt
@@ -8,6 +8,11 @@ Required properties:
8- ti,hwmods : Name of the hwmod associated with the IOMMU instance 8- ti,hwmods : Name of the hwmod associated with the IOMMU instance
9- reg : Address space for the configuration registers 9- reg : Address space for the configuration registers
10- interrupts : Interrupt specifier for the IOMMU instance 10- interrupts : Interrupt specifier for the IOMMU instance
11- #iommu-cells : Should be 0. OMAP IOMMUs are all "single-master" devices,
12 and needs no additional data in the pargs specifier. Please
13 also refer to the generic bindings document for more info
14 on this property,
15 Documentation/devicetree/bindings/iommu/iommu.txt
11 16
12Optional properties: 17Optional properties:
13- ti,#tlb-entries : Number of entries in the translation look-aside buffer. 18- ti,#tlb-entries : Number of entries in the translation look-aside buffer.
@@ -18,6 +23,7 @@ Optional properties:
18Example: 23Example:
19 /* OMAP3 ISP MMU */ 24 /* OMAP3 ISP MMU */
20 mmu_isp: mmu@480bd400 { 25 mmu_isp: mmu@480bd400 {
26 #iommu-cells = <0>;
21 compatible = "ti,omap2-iommu"; 27 compatible = "ti,omap2-iommu";
22 reg = <0x480bd400 0x80>; 28 reg = <0x480bd400 0x80>;
23 interrupts = <24>; 29 interrupts = <24>;
diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig
index f491aec95160..4664c2a96c67 100644
--- a/drivers/iommu/Kconfig
+++ b/drivers/iommu/Kconfig
@@ -23,7 +23,8 @@ config IOMMU_IO_PGTABLE
23config IOMMU_IO_PGTABLE_LPAE 23config IOMMU_IO_PGTABLE_LPAE
24 bool "ARMv7/v8 Long Descriptor Format" 24 bool "ARMv7/v8 Long Descriptor Format"
25 select IOMMU_IO_PGTABLE 25 select IOMMU_IO_PGTABLE
26 depends on ARM || ARM64 || COMPILE_TEST 26 # SWIOTLB guarantees a dma_to_phys() implementation
27 depends on ARM || ARM64 || (COMPILE_TEST && SWIOTLB)
27 help 28 help
28 Enable support for the ARM long descriptor pagetable format. 29 Enable support for the ARM long descriptor pagetable format.
29 This allocator supports 4K/2M/1G, 16K/32M and 64K/512M page 30 This allocator supports 4K/2M/1G, 16K/32M and 64K/512M page
diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c
index 658ee39e6569..f82060e778a2 100644
--- a/drivers/iommu/amd_iommu.c
+++ b/drivers/iommu/amd_iommu.c
@@ -1835,8 +1835,8 @@ static void free_gcr3_table(struct protection_domain *domain)
1835 free_gcr3_tbl_level2(domain->gcr3_tbl); 1835 free_gcr3_tbl_level2(domain->gcr3_tbl);
1836 else if (domain->glx == 1) 1836 else if (domain->glx == 1)
1837 free_gcr3_tbl_level1(domain->gcr3_tbl); 1837 free_gcr3_tbl_level1(domain->gcr3_tbl);
1838 else if (domain->glx != 0) 1838 else
1839 BUG(); 1839 BUG_ON(domain->glx != 0);
1840 1840
1841 free_page((unsigned long)domain->gcr3_tbl); 1841 free_page((unsigned long)domain->gcr3_tbl);
1842} 1842}
@@ -3947,11 +3947,6 @@ static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq,
3947 if (ret < 0) 3947 if (ret < 0)
3948 return ret; 3948 return ret;
3949 3949
3950 ret = -ENOMEM;
3951 data = kzalloc(sizeof(*data), GFP_KERNEL);
3952 if (!data)
3953 goto out_free_parent;
3954
3955 if (info->type == X86_IRQ_ALLOC_TYPE_IOAPIC) { 3950 if (info->type == X86_IRQ_ALLOC_TYPE_IOAPIC) {
3956 if (get_irq_table(devid, true)) 3951 if (get_irq_table(devid, true))
3957 index = info->ioapic_pin; 3952 index = info->ioapic_pin;
@@ -3962,7 +3957,6 @@ static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq,
3962 } 3957 }
3963 if (index < 0) { 3958 if (index < 0) {
3964 pr_warn("Failed to allocate IRTE\n"); 3959 pr_warn("Failed to allocate IRTE\n");
3965 kfree(data);
3966 goto out_free_parent; 3960 goto out_free_parent;
3967 } 3961 }
3968 3962
@@ -3974,17 +3968,18 @@ static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq,
3974 goto out_free_data; 3968 goto out_free_data;
3975 } 3969 }
3976 3970
3977 if (i > 0) { 3971 ret = -ENOMEM;
3978 data = kzalloc(sizeof(*data), GFP_KERNEL); 3972 data = kzalloc(sizeof(*data), GFP_KERNEL);
3979 if (!data) 3973 if (!data)
3980 goto out_free_data; 3974 goto out_free_data;
3981 } 3975
3982 irq_data->hwirq = (devid << 16) + i; 3976 irq_data->hwirq = (devid << 16) + i;
3983 irq_data->chip_data = data; 3977 irq_data->chip_data = data;
3984 irq_data->chip = &amd_ir_chip; 3978 irq_data->chip = &amd_ir_chip;
3985 irq_remapping_prepare_irte(data, cfg, info, devid, index, i); 3979 irq_remapping_prepare_irte(data, cfg, info, devid, index, i);
3986 irq_set_status_flags(virq + i, IRQ_MOVE_PCNTXT); 3980 irq_set_status_flags(virq + i, IRQ_MOVE_PCNTXT);
3987 } 3981 }
3982
3988 return 0; 3983 return 0;
3989 3984
3990out_free_data: 3985out_free_data:
diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c
index a24495eb4e26..5ef347a13cb5 100644
--- a/drivers/iommu/amd_iommu_init.c
+++ b/drivers/iommu/amd_iommu_init.c
@@ -154,7 +154,7 @@ bool amd_iommu_iotlb_sup __read_mostly = true;
154u32 amd_iommu_max_pasid __read_mostly = ~0; 154u32 amd_iommu_max_pasid __read_mostly = ~0;
155 155
156bool amd_iommu_v2_present __read_mostly; 156bool amd_iommu_v2_present __read_mostly;
157bool amd_iommu_pc_present __read_mostly; 157static bool amd_iommu_pc_present __read_mostly;
158 158
159bool amd_iommu_force_isolation __read_mostly; 159bool amd_iommu_force_isolation __read_mostly;
160 160
diff --git a/drivers/iommu/amd_iommu_v2.c b/drivers/iommu/amd_iommu_v2.c
index f7b875bb70d4..1131664b918b 100644
--- a/drivers/iommu/amd_iommu_v2.c
+++ b/drivers/iommu/amd_iommu_v2.c
@@ -356,8 +356,8 @@ static void free_pasid_states(struct device_state *dev_state)
356 free_pasid_states_level2(dev_state->states); 356 free_pasid_states_level2(dev_state->states);
357 else if (dev_state->pasid_levels == 1) 357 else if (dev_state->pasid_levels == 1)
358 free_pasid_states_level1(dev_state->states); 358 free_pasid_states_level1(dev_state->states);
359 else if (dev_state->pasid_levels != 0) 359 else
360 BUG(); 360 BUG_ON(dev_state->pasid_levels != 0);
361 361
362 free_page((unsigned long)dev_state->states); 362 free_page((unsigned long)dev_state->states);
363} 363}
diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c
index da902baaa794..dafaf59dc3b8 100644
--- a/drivers/iommu/arm-smmu-v3.c
+++ b/drivers/iommu/arm-smmu-v3.c
@@ -118,6 +118,7 @@
118 118
119#define ARM_SMMU_IRQ_CTRL 0x50 119#define ARM_SMMU_IRQ_CTRL 0x50
120#define IRQ_CTRL_EVTQ_IRQEN (1 << 2) 120#define IRQ_CTRL_EVTQ_IRQEN (1 << 2)
121#define IRQ_CTRL_PRIQ_IRQEN (1 << 1)
121#define IRQ_CTRL_GERROR_IRQEN (1 << 0) 122#define IRQ_CTRL_GERROR_IRQEN (1 << 0)
122 123
123#define ARM_SMMU_IRQ_CTRLACK 0x54 124#define ARM_SMMU_IRQ_CTRLACK 0x54
@@ -173,14 +174,14 @@
173#define ARM_SMMU_PRIQ_IRQ_CFG2 0xdc 174#define ARM_SMMU_PRIQ_IRQ_CFG2 0xdc
174 175
175/* Common MSI config fields */ 176/* Common MSI config fields */
176#define MSI_CFG0_SH_SHIFT 60
177#define MSI_CFG0_SH_NSH (0UL << MSI_CFG0_SH_SHIFT)
178#define MSI_CFG0_SH_OSH (2UL << MSI_CFG0_SH_SHIFT)
179#define MSI_CFG0_SH_ISH (3UL << MSI_CFG0_SH_SHIFT)
180#define MSI_CFG0_MEMATTR_SHIFT 56
181#define MSI_CFG0_MEMATTR_DEVICE_nGnRE (0x1 << MSI_CFG0_MEMATTR_SHIFT)
182#define MSI_CFG0_ADDR_SHIFT 2 177#define MSI_CFG0_ADDR_SHIFT 2
183#define MSI_CFG0_ADDR_MASK 0x3fffffffffffUL 178#define MSI_CFG0_ADDR_MASK 0x3fffffffffffUL
179#define MSI_CFG2_SH_SHIFT 4
180#define MSI_CFG2_SH_NSH (0UL << MSI_CFG2_SH_SHIFT)
181#define MSI_CFG2_SH_OSH (2UL << MSI_CFG2_SH_SHIFT)
182#define MSI_CFG2_SH_ISH (3UL << MSI_CFG2_SH_SHIFT)
183#define MSI_CFG2_MEMATTR_SHIFT 0
184#define MSI_CFG2_MEMATTR_DEVICE_nGnRE (0x1 << MSI_CFG2_MEMATTR_SHIFT)
184 185
185#define Q_IDX(q, p) ((p) & ((1 << (q)->max_n_shift) - 1)) 186#define Q_IDX(q, p) ((p) & ((1 << (q)->max_n_shift) - 1))
186#define Q_WRP(q, p) ((p) & (1 << (q)->max_n_shift)) 187#define Q_WRP(q, p) ((p) & (1 << (q)->max_n_shift))
@@ -1330,33 +1331,10 @@ static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
1330 arm_smmu_cmdq_issue_cmd(smmu, &cmd); 1331 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1331} 1332}
1332 1333
1333static void arm_smmu_flush_pgtable(void *addr, size_t size, void *cookie)
1334{
1335 struct arm_smmu_domain *smmu_domain = cookie;
1336 struct arm_smmu_device *smmu = smmu_domain->smmu;
1337 unsigned long offset = (unsigned long)addr & ~PAGE_MASK;
1338
1339 if (smmu->features & ARM_SMMU_FEAT_COHERENCY) {
1340 dsb(ishst);
1341 } else {
1342 dma_addr_t dma_addr;
1343 struct device *dev = smmu->dev;
1344
1345 dma_addr = dma_map_page(dev, virt_to_page(addr), offset, size,
1346 DMA_TO_DEVICE);
1347
1348 if (dma_mapping_error(dev, dma_addr))
1349 dev_err(dev, "failed to flush pgtable at %p\n", addr);
1350 else
1351 dma_unmap_page(dev, dma_addr, size, DMA_TO_DEVICE);
1352 }
1353}
1354
1355static struct iommu_gather_ops arm_smmu_gather_ops = { 1334static struct iommu_gather_ops arm_smmu_gather_ops = {
1356 .tlb_flush_all = arm_smmu_tlb_inv_context, 1335 .tlb_flush_all = arm_smmu_tlb_inv_context,
1357 .tlb_add_flush = arm_smmu_tlb_inv_range_nosync, 1336 .tlb_add_flush = arm_smmu_tlb_inv_range_nosync,
1358 .tlb_sync = arm_smmu_tlb_sync, 1337 .tlb_sync = arm_smmu_tlb_sync,
1359 .flush_pgtable = arm_smmu_flush_pgtable,
1360}; 1338};
1361 1339
1362/* IOMMU API */ 1340/* IOMMU API */
@@ -1531,6 +1509,7 @@ static int arm_smmu_domain_finalise(struct iommu_domain *domain)
1531 .ias = ias, 1509 .ias = ias,
1532 .oas = oas, 1510 .oas = oas,
1533 .tlb = &arm_smmu_gather_ops, 1511 .tlb = &arm_smmu_gather_ops,
1512 .iommu_dev = smmu->dev,
1534 }; 1513 };
1535 1514
1536 pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain); 1515 pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
@@ -2053,9 +2032,17 @@ static int arm_smmu_init_strtab_2lvl(struct arm_smmu_device *smmu)
2053 int ret; 2032 int ret;
2054 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; 2033 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2055 2034
2056 /* Calculate the L1 size, capped to the SIDSIZE */ 2035 /*
2057 size = STRTAB_L1_SZ_SHIFT - (ilog2(STRTAB_L1_DESC_DWORDS) + 3); 2036 * If we can resolve everything with a single L2 table, then we
2058 size = min(size, smmu->sid_bits - STRTAB_SPLIT); 2037 * just need a single L1 descriptor. Otherwise, calculate the L1
2038 * size, capped to the SIDSIZE.
2039 */
2040 if (smmu->sid_bits < STRTAB_SPLIT) {
2041 size = 0;
2042 } else {
2043 size = STRTAB_L1_SZ_SHIFT - (ilog2(STRTAB_L1_DESC_DWORDS) + 3);
2044 size = min(size, smmu->sid_bits - STRTAB_SPLIT);
2045 }
2059 cfg->num_l1_ents = 1 << size; 2046 cfg->num_l1_ents = 1 << size;
2060 2047
2061 size += STRTAB_SPLIT; 2048 size += STRTAB_SPLIT;
@@ -2198,6 +2185,7 @@ static int arm_smmu_write_reg_sync(struct arm_smmu_device *smmu, u32 val,
2198static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu) 2185static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu)
2199{ 2186{
2200 int ret, irq; 2187 int ret, irq;
2188 u32 irqen_flags = IRQ_CTRL_EVTQ_IRQEN | IRQ_CTRL_GERROR_IRQEN;
2201 2189
2202 /* Disable IRQs first */ 2190 /* Disable IRQs first */
2203 ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_IRQ_CTRL, 2191 ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_IRQ_CTRL,
@@ -2252,13 +2240,13 @@ static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu)
2252 if (IS_ERR_VALUE(ret)) 2240 if (IS_ERR_VALUE(ret))
2253 dev_warn(smmu->dev, 2241 dev_warn(smmu->dev,
2254 "failed to enable priq irq\n"); 2242 "failed to enable priq irq\n");
2243 else
2244 irqen_flags |= IRQ_CTRL_PRIQ_IRQEN;
2255 } 2245 }
2256 } 2246 }
2257 2247
2258 /* Enable interrupt generation on the SMMU */ 2248 /* Enable interrupt generation on the SMMU */
2259 ret = arm_smmu_write_reg_sync(smmu, 2249 ret = arm_smmu_write_reg_sync(smmu, irqen_flags,
2260 IRQ_CTRL_EVTQ_IRQEN |
2261 IRQ_CTRL_GERROR_IRQEN,
2262 ARM_SMMU_IRQ_CTRL, ARM_SMMU_IRQ_CTRLACK); 2250 ARM_SMMU_IRQ_CTRL, ARM_SMMU_IRQ_CTRLACK);
2263 if (ret) 2251 if (ret)
2264 dev_warn(smmu->dev, "failed to enable irqs\n"); 2252 dev_warn(smmu->dev, "failed to enable irqs\n");
@@ -2540,12 +2528,12 @@ static int arm_smmu_device_probe(struct arm_smmu_device *smmu)
2540 case IDR5_OAS_44_BIT: 2528 case IDR5_OAS_44_BIT:
2541 smmu->oas = 44; 2529 smmu->oas = 44;
2542 break; 2530 break;
2531 default:
2532 dev_info(smmu->dev,
2533 "unknown output address size. Truncating to 48-bit\n");
2534 /* Fallthrough */
2543 case IDR5_OAS_48_BIT: 2535 case IDR5_OAS_48_BIT:
2544 smmu->oas = 48; 2536 smmu->oas = 48;
2545 break;
2546 default:
2547 dev_err(smmu->dev, "unknown output address size!\n");
2548 return -ENXIO;
2549 } 2537 }
2550 2538
2551 /* Set the DMA mask for our table walker */ 2539 /* Set the DMA mask for our table walker */
diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c
index 4cd0c29cb585..48a39dfa9777 100644
--- a/drivers/iommu/arm-smmu.c
+++ b/drivers/iommu/arm-smmu.c
@@ -37,6 +37,7 @@
37#include <linux/iopoll.h> 37#include <linux/iopoll.h>
38#include <linux/module.h> 38#include <linux/module.h>
39#include <linux/of.h> 39#include <linux/of.h>
40#include <linux/of_address.h>
40#include <linux/pci.h> 41#include <linux/pci.h>
41#include <linux/platform_device.h> 42#include <linux/platform_device.h>
42#include <linux/slab.h> 43#include <linux/slab.h>
@@ -607,34 +608,10 @@ static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
607 } 608 }
608} 609}
609 610
610static void arm_smmu_flush_pgtable(void *addr, size_t size, void *cookie)
611{
612 struct arm_smmu_domain *smmu_domain = cookie;
613 struct arm_smmu_device *smmu = smmu_domain->smmu;
614 unsigned long offset = (unsigned long)addr & ~PAGE_MASK;
615
616
617 /* Ensure new page tables are visible to the hardware walker */
618 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK) {
619 dsb(ishst);
620 } else {
621 /*
622 * If the SMMU can't walk tables in the CPU caches, treat them
623 * like non-coherent DMA since we need to flush the new entries
624 * all the way out to memory. There's no possibility of
625 * recursion here as the SMMU table walker will not be wired
626 * through another SMMU.
627 */
628 dma_map_page(smmu->dev, virt_to_page(addr), offset, size,
629 DMA_TO_DEVICE);
630 }
631}
632
633static struct iommu_gather_ops arm_smmu_gather_ops = { 611static struct iommu_gather_ops arm_smmu_gather_ops = {
634 .tlb_flush_all = arm_smmu_tlb_inv_context, 612 .tlb_flush_all = arm_smmu_tlb_inv_context,
635 .tlb_add_flush = arm_smmu_tlb_inv_range_nosync, 613 .tlb_add_flush = arm_smmu_tlb_inv_range_nosync,
636 .tlb_sync = arm_smmu_tlb_sync, 614 .tlb_sync = arm_smmu_tlb_sync,
637 .flush_pgtable = arm_smmu_flush_pgtable,
638}; 615};
639 616
640static irqreturn_t arm_smmu_context_fault(int irq, void *dev) 617static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
@@ -898,6 +875,7 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain,
898 .ias = ias, 875 .ias = ias,
899 .oas = oas, 876 .oas = oas,
900 .tlb = &arm_smmu_gather_ops, 877 .tlb = &arm_smmu_gather_ops,
878 .iommu_dev = smmu->dev,
901 }; 879 };
902 880
903 smmu_domain->smmu = smmu; 881 smmu_domain->smmu = smmu;
@@ -1532,6 +1510,7 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1532 unsigned long size; 1510 unsigned long size;
1533 void __iomem *gr0_base = ARM_SMMU_GR0(smmu); 1511 void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1534 u32 id; 1512 u32 id;
1513 bool cttw_dt, cttw_reg;
1535 1514
1536 dev_notice(smmu->dev, "probing hardware configuration...\n"); 1515 dev_notice(smmu->dev, "probing hardware configuration...\n");
1537 dev_notice(smmu->dev, "SMMUv%d with:\n", smmu->version); 1516 dev_notice(smmu->dev, "SMMUv%d with:\n", smmu->version);
@@ -1571,10 +1550,22 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1571 dev_notice(smmu->dev, "\taddress translation ops\n"); 1550 dev_notice(smmu->dev, "\taddress translation ops\n");
1572 } 1551 }
1573 1552
1574 if (id & ID0_CTTW) { 1553 /*
1554 * In order for DMA API calls to work properly, we must defer to what
1555 * the DT says about coherency, regardless of what the hardware claims.
1556 * Fortunately, this also opens up a workaround for systems where the
1557 * ID register value has ended up configured incorrectly.
1558 */
1559 cttw_dt = of_dma_is_coherent(smmu->dev->of_node);
1560 cttw_reg = !!(id & ID0_CTTW);
1561 if (cttw_dt)
1575 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK; 1562 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
1576 dev_notice(smmu->dev, "\tcoherent table walk\n"); 1563 if (cttw_dt || cttw_reg)
1577 } 1564 dev_notice(smmu->dev, "\t%scoherent table walk\n",
1565 cttw_dt ? "" : "non-");
1566 if (cttw_dt != cttw_reg)
1567 dev_notice(smmu->dev,
1568 "\t(IDR0.CTTW overridden by dma-coherent property)\n");
1578 1569
1579 if (id & ID0_SMS) { 1570 if (id & ID0_SMS) {
1580 u32 smr, sid, mask; 1571 u32 smr, sid, mask;
diff --git a/drivers/iommu/dmar.c b/drivers/iommu/dmar.c
index c9db04d4ef39..8757f8dfc4e5 100644
--- a/drivers/iommu/dmar.c
+++ b/drivers/iommu/dmar.c
@@ -1068,7 +1068,7 @@ static int alloc_iommu(struct dmar_drhd_unit *drhd)
1068 if (intel_iommu_enabled) 1068 if (intel_iommu_enabled)
1069 iommu->iommu_dev = iommu_device_create(NULL, iommu, 1069 iommu->iommu_dev = iommu_device_create(NULL, iommu,
1070 intel_iommu_groups, 1070 intel_iommu_groups,
1071 iommu->name); 1071 "%s", iommu->name);
1072 1072
1073 return 0; 1073 return 0;
1074 1074
diff --git a/drivers/iommu/fsl_pamu.c b/drivers/iommu/fsl_pamu.c
index abeedc9a78c2..2570f2a25dc4 100644
--- a/drivers/iommu/fsl_pamu.c
+++ b/drivers/iommu/fsl_pamu.c
@@ -41,7 +41,6 @@ struct pamu_isr_data {
41 41
42static struct paace *ppaact; 42static struct paace *ppaact;
43static struct paace *spaact; 43static struct paace *spaact;
44static struct ome *omt __initdata;
45 44
46/* 45/*
47 * Table for matching compatible strings, for device tree 46 * Table for matching compatible strings, for device tree
@@ -50,7 +49,7 @@ static struct ome *omt __initdata;
50 * SOCs. For the older SOCs "fsl,qoriq-device-config-1.0" 49 * SOCs. For the older SOCs "fsl,qoriq-device-config-1.0"
51 * string would be used. 50 * string would be used.
52 */ 51 */
53static const struct of_device_id guts_device_ids[] __initconst = { 52static const struct of_device_id guts_device_ids[] = {
54 { .compatible = "fsl,qoriq-device-config-1.0", }, 53 { .compatible = "fsl,qoriq-device-config-1.0", },
55 { .compatible = "fsl,qoriq-device-config-2.0", }, 54 { .compatible = "fsl,qoriq-device-config-2.0", },
56 {} 55 {}
@@ -599,7 +598,7 @@ found_cpu_node:
599 * Memory accesses to QMAN and BMAN private memory need not be coherent, so 598 * Memory accesses to QMAN and BMAN private memory need not be coherent, so
600 * clear the PAACE entry coherency attribute for them. 599 * clear the PAACE entry coherency attribute for them.
601 */ 600 */
602static void __init setup_qbman_paace(struct paace *ppaace, int paace_type) 601static void setup_qbman_paace(struct paace *ppaace, int paace_type)
603{ 602{
604 switch (paace_type) { 603 switch (paace_type) {
605 case QMAN_PAACE: 604 case QMAN_PAACE:
@@ -629,7 +628,7 @@ static void __init setup_qbman_paace(struct paace *ppaace, int paace_type)
629 * this table to translate device transaction to appropriate corenet 628 * this table to translate device transaction to appropriate corenet
630 * transaction. 629 * transaction.
631 */ 630 */
632static void __init setup_omt(struct ome *omt) 631static void setup_omt(struct ome *omt)
633{ 632{
634 struct ome *ome; 633 struct ome *ome;
635 634
@@ -666,7 +665,7 @@ static void __init setup_omt(struct ome *omt)
666 * Get the maximum number of PAACT table entries 665 * Get the maximum number of PAACT table entries
667 * and subwindows supported by PAMU 666 * and subwindows supported by PAMU
668 */ 667 */
669static void __init get_pamu_cap_values(unsigned long pamu_reg_base) 668static void get_pamu_cap_values(unsigned long pamu_reg_base)
670{ 669{
671 u32 pc_val; 670 u32 pc_val;
672 671
@@ -676,9 +675,9 @@ static void __init get_pamu_cap_values(unsigned long pamu_reg_base)
676} 675}
677 676
678/* Setup PAMU registers pointing to PAACT, SPAACT and OMT */ 677/* Setup PAMU registers pointing to PAACT, SPAACT and OMT */
679static int __init setup_one_pamu(unsigned long pamu_reg_base, unsigned long pamu_reg_size, 678static int setup_one_pamu(unsigned long pamu_reg_base, unsigned long pamu_reg_size,
680 phys_addr_t ppaact_phys, phys_addr_t spaact_phys, 679 phys_addr_t ppaact_phys, phys_addr_t spaact_phys,
681 phys_addr_t omt_phys) 680 phys_addr_t omt_phys)
682{ 681{
683 u32 *pc; 682 u32 *pc;
684 struct pamu_mmap_regs *pamu_regs; 683 struct pamu_mmap_regs *pamu_regs;
@@ -720,7 +719,7 @@ static int __init setup_one_pamu(unsigned long pamu_reg_base, unsigned long pamu
720} 719}
721 720
722/* Enable all device LIODNS */ 721/* Enable all device LIODNS */
723static void __init setup_liodns(void) 722static void setup_liodns(void)
724{ 723{
725 int i, len; 724 int i, len;
726 struct paace *ppaace; 725 struct paace *ppaace;
@@ -846,7 +845,7 @@ struct ccsr_law {
846/* 845/*
847 * Create a coherence subdomain for a given memory block. 846 * Create a coherence subdomain for a given memory block.
848 */ 847 */
849static int __init create_csd(phys_addr_t phys, size_t size, u32 csd_port_id) 848static int create_csd(phys_addr_t phys, size_t size, u32 csd_port_id)
850{ 849{
851 struct device_node *np; 850 struct device_node *np;
852 const __be32 *iprop; 851 const __be32 *iprop;
@@ -988,7 +987,7 @@ error:
988static const struct { 987static const struct {
989 u32 svr; 988 u32 svr;
990 u32 port_id; 989 u32 port_id;
991} port_id_map[] __initconst = { 990} port_id_map[] = {
992 {(SVR_P2040 << 8) | 0x10, 0xFF000000}, /* P2040 1.0 */ 991 {(SVR_P2040 << 8) | 0x10, 0xFF000000}, /* P2040 1.0 */
993 {(SVR_P2040 << 8) | 0x11, 0xFF000000}, /* P2040 1.1 */ 992 {(SVR_P2040 << 8) | 0x11, 0xFF000000}, /* P2040 1.1 */
994 {(SVR_P2041 << 8) | 0x10, 0xFF000000}, /* P2041 1.0 */ 993 {(SVR_P2041 << 8) | 0x10, 0xFF000000}, /* P2041 1.0 */
@@ -1006,7 +1005,7 @@ static const struct {
1006 1005
1007#define SVR_SECURITY 0x80000 /* The Security (E) bit */ 1006#define SVR_SECURITY 0x80000 /* The Security (E) bit */
1008 1007
1009static int __init fsl_pamu_probe(struct platform_device *pdev) 1008static int fsl_pamu_probe(struct platform_device *pdev)
1010{ 1009{
1011 struct device *dev = &pdev->dev; 1010 struct device *dev = &pdev->dev;
1012 void __iomem *pamu_regs = NULL; 1011 void __iomem *pamu_regs = NULL;
@@ -1022,6 +1021,7 @@ static int __init fsl_pamu_probe(struct platform_device *pdev)
1022 int irq; 1021 int irq;
1023 phys_addr_t ppaact_phys; 1022 phys_addr_t ppaact_phys;
1024 phys_addr_t spaact_phys; 1023 phys_addr_t spaact_phys;
1024 struct ome *omt;
1025 phys_addr_t omt_phys; 1025 phys_addr_t omt_phys;
1026 size_t mem_size = 0; 1026 size_t mem_size = 0;
1027 unsigned int order = 0; 1027 unsigned int order = 0;
@@ -1200,7 +1200,7 @@ error:
1200 return ret; 1200 return ret;
1201} 1201}
1202 1202
1203static struct platform_driver fsl_of_pamu_driver __initdata = { 1203static struct platform_driver fsl_of_pamu_driver = {
1204 .driver = { 1204 .driver = {
1205 .name = "fsl-of-pamu", 1205 .name = "fsl-of-pamu",
1206 }, 1206 },
diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c
index c82ebee6c7e5..2d7349a3ee14 100644
--- a/drivers/iommu/intel-iommu.c
+++ b/drivers/iommu/intel-iommu.c
@@ -364,7 +364,8 @@ static inline int first_pte_in_page(struct dma_pte *pte)
364static struct dmar_domain *si_domain; 364static struct dmar_domain *si_domain;
365static int hw_pass_through = 1; 365static int hw_pass_through = 1;
366 366
367/* domain represents a virtual machine, more than one devices 367/*
368 * Domain represents a virtual machine, more than one devices
368 * across iommus may be owned in one domain, e.g. kvm guest. 369 * across iommus may be owned in one domain, e.g. kvm guest.
369 */ 370 */
370#define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 0) 371#define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 0)
@@ -372,11 +373,21 @@ static int hw_pass_through = 1;
372/* si_domain contains mulitple devices */ 373/* si_domain contains mulitple devices */
373#define DOMAIN_FLAG_STATIC_IDENTITY (1 << 1) 374#define DOMAIN_FLAG_STATIC_IDENTITY (1 << 1)
374 375
376#define for_each_domain_iommu(idx, domain) \
377 for (idx = 0; idx < g_num_of_iommus; idx++) \
378 if (domain->iommu_refcnt[idx])
379
375struct dmar_domain { 380struct dmar_domain {
376 int id; /* domain id */
377 int nid; /* node id */ 381 int nid; /* node id */
378 DECLARE_BITMAP(iommu_bmp, DMAR_UNITS_SUPPORTED); 382
379 /* bitmap of iommus this domain uses*/ 383 unsigned iommu_refcnt[DMAR_UNITS_SUPPORTED];
384 /* Refcount of devices per iommu */
385
386
387 u16 iommu_did[DMAR_UNITS_SUPPORTED];
388 /* Domain ids per IOMMU. Use u16 since
389 * domain ids are 16 bit wide according
390 * to VT-d spec, section 9.3 */
380 391
381 struct list_head devices; /* all devices' list */ 392 struct list_head devices; /* all devices' list */
382 struct iova_domain iovad; /* iova's that belong to this domain */ 393 struct iova_domain iovad; /* iova's that belong to this domain */
@@ -395,7 +406,6 @@ struct dmar_domain {
395 int iommu_superpage;/* Level of superpages supported: 406 int iommu_superpage;/* Level of superpages supported:
396 0 == 4KiB (no superpages), 1 == 2MiB, 407 0 == 4KiB (no superpages), 1 == 2MiB,
397 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */ 408 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
398 spinlock_t iommu_lock; /* protect iommu set in domain */
399 u64 max_addr; /* maximum mapped address */ 409 u64 max_addr; /* maximum mapped address */
400 410
401 struct iommu_domain domain; /* generic domain data structure for 411 struct iommu_domain domain; /* generic domain data structure for
@@ -465,10 +475,11 @@ static long list_size;
465 475
466static void domain_exit(struct dmar_domain *domain); 476static void domain_exit(struct dmar_domain *domain);
467static void domain_remove_dev_info(struct dmar_domain *domain); 477static void domain_remove_dev_info(struct dmar_domain *domain);
468static void domain_remove_one_dev_info(struct dmar_domain *domain, 478static void dmar_remove_one_dev_info(struct dmar_domain *domain,
469 struct device *dev); 479 struct device *dev);
470static void iommu_detach_dependent_devices(struct intel_iommu *iommu, 480static void __dmar_remove_one_dev_info(struct device_domain_info *info);
471 struct device *dev); 481static void domain_context_clear(struct intel_iommu *iommu,
482 struct device *dev);
472static int domain_detach_iommu(struct dmar_domain *domain, 483static int domain_detach_iommu(struct dmar_domain *domain,
473 struct intel_iommu *iommu); 484 struct intel_iommu *iommu);
474 485
@@ -568,6 +579,36 @@ __setup("intel_iommu=", intel_iommu_setup);
568static struct kmem_cache *iommu_domain_cache; 579static struct kmem_cache *iommu_domain_cache;
569static struct kmem_cache *iommu_devinfo_cache; 580static struct kmem_cache *iommu_devinfo_cache;
570 581
582static struct dmar_domain* get_iommu_domain(struct intel_iommu *iommu, u16 did)
583{
584 struct dmar_domain **domains;
585 int idx = did >> 8;
586
587 domains = iommu->domains[idx];
588 if (!domains)
589 return NULL;
590
591 return domains[did & 0xff];
592}
593
594static void set_iommu_domain(struct intel_iommu *iommu, u16 did,
595 struct dmar_domain *domain)
596{
597 struct dmar_domain **domains;
598 int idx = did >> 8;
599
600 if (!iommu->domains[idx]) {
601 size_t size = 256 * sizeof(struct dmar_domain *);
602 iommu->domains[idx] = kzalloc(size, GFP_ATOMIC);
603 }
604
605 domains = iommu->domains[idx];
606 if (WARN_ON(!domains))
607 return;
608 else
609 domains[did & 0xff] = domain;
610}
611
571static inline void *alloc_pgtable_page(int node) 612static inline void *alloc_pgtable_page(int node)
572{ 613{
573 struct page *page; 614 struct page *page;
@@ -609,6 +650,11 @@ static inline int domain_type_is_vm(struct dmar_domain *domain)
609 return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE; 650 return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE;
610} 651}
611 652
653static inline int domain_type_is_si(struct dmar_domain *domain)
654{
655 return domain->flags & DOMAIN_FLAG_STATIC_IDENTITY;
656}
657
612static inline int domain_type_is_vm_or_si(struct dmar_domain *domain) 658static inline int domain_type_is_vm_or_si(struct dmar_domain *domain)
613{ 659{
614 return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE | 660 return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE |
@@ -663,7 +709,9 @@ static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
663 709
664 /* si_domain and vm domain should not get here. */ 710 /* si_domain and vm domain should not get here. */
665 BUG_ON(domain_type_is_vm_or_si(domain)); 711 BUG_ON(domain_type_is_vm_or_si(domain));
666 iommu_id = find_first_bit(domain->iommu_bmp, g_num_of_iommus); 712 for_each_domain_iommu(iommu_id, domain)
713 break;
714
667 if (iommu_id < 0 || iommu_id >= g_num_of_iommus) 715 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
668 return NULL; 716 return NULL;
669 717
@@ -679,7 +727,7 @@ static void domain_update_iommu_coherency(struct dmar_domain *domain)
679 727
680 domain->iommu_coherency = 1; 728 domain->iommu_coherency = 1;
681 729
682 for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus) { 730 for_each_domain_iommu(i, domain) {
683 found = true; 731 found = true;
684 if (!ecap_coherent(g_iommus[i]->ecap)) { 732 if (!ecap_coherent(g_iommus[i]->ecap)) {
685 domain->iommu_coherency = 0; 733 domain->iommu_coherency = 0;
@@ -759,6 +807,7 @@ static inline struct context_entry *iommu_context_addr(struct intel_iommu *iommu
759 struct context_entry *context; 807 struct context_entry *context;
760 u64 *entry; 808 u64 *entry;
761 809
810 entry = &root->lo;
762 if (ecs_enabled(iommu)) { 811 if (ecs_enabled(iommu)) {
763 if (devfn >= 0x80) { 812 if (devfn >= 0x80) {
764 devfn -= 0x80; 813 devfn -= 0x80;
@@ -766,7 +815,6 @@ static inline struct context_entry *iommu_context_addr(struct intel_iommu *iommu
766 } 815 }
767 devfn *= 2; 816 devfn *= 2;
768 } 817 }
769 entry = &root->lo;
770 if (*entry & 1) 818 if (*entry & 1)
771 context = phys_to_virt(*entry & VTD_PAGE_MASK); 819 context = phys_to_virt(*entry & VTD_PAGE_MASK);
772 else { 820 else {
@@ -1166,9 +1214,9 @@ next:
1166/* We can't just free the pages because the IOMMU may still be walking 1214/* We can't just free the pages because the IOMMU may still be walking
1167 the page tables, and may have cached the intermediate levels. The 1215 the page tables, and may have cached the intermediate levels. The
1168 pages can only be freed after the IOTLB flush has been done. */ 1216 pages can only be freed after the IOTLB flush has been done. */
1169struct page *domain_unmap(struct dmar_domain *domain, 1217static struct page *domain_unmap(struct dmar_domain *domain,
1170 unsigned long start_pfn, 1218 unsigned long start_pfn,
1171 unsigned long last_pfn) 1219 unsigned long last_pfn)
1172{ 1220{
1173 struct page *freelist = NULL; 1221 struct page *freelist = NULL;
1174 1222
@@ -1192,7 +1240,7 @@ struct page *domain_unmap(struct dmar_domain *domain,
1192 return freelist; 1240 return freelist;
1193} 1241}
1194 1242
1195void dma_free_pagelist(struct page *freelist) 1243static void dma_free_pagelist(struct page *freelist)
1196{ 1244{
1197 struct page *pg; 1245 struct page *pg;
1198 1246
@@ -1360,24 +1408,23 @@ iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1360 u8 bus, u8 devfn) 1408 u8 bus, u8 devfn)
1361{ 1409{
1362 bool found = false; 1410 bool found = false;
1363 unsigned long flags;
1364 struct device_domain_info *info; 1411 struct device_domain_info *info;
1365 struct pci_dev *pdev; 1412 struct pci_dev *pdev;
1366 1413
1414 assert_spin_locked(&device_domain_lock);
1415
1367 if (!ecap_dev_iotlb_support(iommu->ecap)) 1416 if (!ecap_dev_iotlb_support(iommu->ecap))
1368 return NULL; 1417 return NULL;
1369 1418
1370 if (!iommu->qi) 1419 if (!iommu->qi)
1371 return NULL; 1420 return NULL;
1372 1421
1373 spin_lock_irqsave(&device_domain_lock, flags);
1374 list_for_each_entry(info, &domain->devices, link) 1422 list_for_each_entry(info, &domain->devices, link)
1375 if (info->iommu == iommu && info->bus == bus && 1423 if (info->iommu == iommu && info->bus == bus &&
1376 info->devfn == devfn) { 1424 info->devfn == devfn) {
1377 found = true; 1425 found = true;
1378 break; 1426 break;
1379 } 1427 }
1380 spin_unlock_irqrestore(&device_domain_lock, flags);
1381 1428
1382 if (!found || !info->dev || !dev_is_pci(info->dev)) 1429 if (!found || !info->dev || !dev_is_pci(info->dev))
1383 return NULL; 1430 return NULL;
@@ -1436,11 +1483,14 @@ static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1436 spin_unlock_irqrestore(&device_domain_lock, flags); 1483 spin_unlock_irqrestore(&device_domain_lock, flags);
1437} 1484}
1438 1485
1439static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did, 1486static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
1440 unsigned long pfn, unsigned int pages, int ih, int map) 1487 struct dmar_domain *domain,
1488 unsigned long pfn, unsigned int pages,
1489 int ih, int map)
1441{ 1490{
1442 unsigned int mask = ilog2(__roundup_pow_of_two(pages)); 1491 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
1443 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT; 1492 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1493 u16 did = domain->iommu_did[iommu->seq_id];
1444 1494
1445 BUG_ON(pages == 0); 1495 BUG_ON(pages == 0);
1446 1496
@@ -1464,7 +1514,8 @@ static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
1464 * flush. However, device IOTLB doesn't need to be flushed in this case. 1514 * flush. However, device IOTLB doesn't need to be flushed in this case.
1465 */ 1515 */
1466 if (!cap_caching_mode(iommu->cap) || !map) 1516 if (!cap_caching_mode(iommu->cap) || !map)
1467 iommu_flush_dev_iotlb(iommu->domains[did], addr, mask); 1517 iommu_flush_dev_iotlb(get_iommu_domain(iommu, did),
1518 addr, mask);
1468} 1519}
1469 1520
1470static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu) 1521static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
@@ -1519,65 +1570,80 @@ static void iommu_disable_translation(struct intel_iommu *iommu)
1519 1570
1520static int iommu_init_domains(struct intel_iommu *iommu) 1571static int iommu_init_domains(struct intel_iommu *iommu)
1521{ 1572{
1522 unsigned long ndomains; 1573 u32 ndomains, nlongs;
1523 unsigned long nlongs; 1574 size_t size;
1524 1575
1525 ndomains = cap_ndoms(iommu->cap); 1576 ndomains = cap_ndoms(iommu->cap);
1526 pr_debug("%s: Number of Domains supported <%ld>\n", 1577 pr_debug("%s: Number of Domains supported <%d>\n",
1527 iommu->name, ndomains); 1578 iommu->name, ndomains);
1528 nlongs = BITS_TO_LONGS(ndomains); 1579 nlongs = BITS_TO_LONGS(ndomains);
1529 1580
1530 spin_lock_init(&iommu->lock); 1581 spin_lock_init(&iommu->lock);
1531 1582
1532 /* TBD: there might be 64K domains,
1533 * consider other allocation for future chip
1534 */
1535 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL); 1583 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1536 if (!iommu->domain_ids) { 1584 if (!iommu->domain_ids) {
1537 pr_err("%s: Allocating domain id array failed\n", 1585 pr_err("%s: Allocating domain id array failed\n",
1538 iommu->name); 1586 iommu->name);
1539 return -ENOMEM; 1587 return -ENOMEM;
1540 } 1588 }
1541 iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *), 1589
1542 GFP_KERNEL); 1590 size = ((ndomains >> 8) + 1) * sizeof(struct dmar_domain **);
1543 if (!iommu->domains) { 1591 iommu->domains = kzalloc(size, GFP_KERNEL);
1592
1593 if (iommu->domains) {
1594 size = 256 * sizeof(struct dmar_domain *);
1595 iommu->domains[0] = kzalloc(size, GFP_KERNEL);
1596 }
1597
1598 if (!iommu->domains || !iommu->domains[0]) {
1544 pr_err("%s: Allocating domain array failed\n", 1599 pr_err("%s: Allocating domain array failed\n",
1545 iommu->name); 1600 iommu->name);
1546 kfree(iommu->domain_ids); 1601 kfree(iommu->domain_ids);
1602 kfree(iommu->domains);
1547 iommu->domain_ids = NULL; 1603 iommu->domain_ids = NULL;
1604 iommu->domains = NULL;
1548 return -ENOMEM; 1605 return -ENOMEM;
1549 } 1606 }
1550 1607
1608
1609
1551 /* 1610 /*
1552 * if Caching mode is set, then invalid translations are tagged 1611 * If Caching mode is set, then invalid translations are tagged
1553 * with domainid 0. Hence we need to pre-allocate it. 1612 * with domain-id 0, hence we need to pre-allocate it. We also
1613 * use domain-id 0 as a marker for non-allocated domain-id, so
1614 * make sure it is not used for a real domain.
1554 */ 1615 */
1555 if (cap_caching_mode(iommu->cap)) 1616 set_bit(0, iommu->domain_ids);
1556 set_bit(0, iommu->domain_ids); 1617
1557 return 0; 1618 return 0;
1558} 1619}
1559 1620
1560static void disable_dmar_iommu(struct intel_iommu *iommu) 1621static void disable_dmar_iommu(struct intel_iommu *iommu)
1561{ 1622{
1562 struct dmar_domain *domain; 1623 struct device_domain_info *info, *tmp;
1563 int i; 1624 unsigned long flags;
1564 1625
1565 if ((iommu->domains) && (iommu->domain_ids)) { 1626 if (!iommu->domains || !iommu->domain_ids)
1566 for_each_set_bit(i, iommu->domain_ids, cap_ndoms(iommu->cap)) { 1627 return;
1567 /*
1568 * Domain id 0 is reserved for invalid translation
1569 * if hardware supports caching mode.
1570 */
1571 if (cap_caching_mode(iommu->cap) && i == 0)
1572 continue;
1573 1628
1574 domain = iommu->domains[i]; 1629 spin_lock_irqsave(&device_domain_lock, flags);
1575 clear_bit(i, iommu->domain_ids); 1630 list_for_each_entry_safe(info, tmp, &device_domain_list, global) {
1576 if (domain_detach_iommu(domain, iommu) == 0 && 1631 struct dmar_domain *domain;
1577 !domain_type_is_vm(domain)) 1632
1578 domain_exit(domain); 1633 if (info->iommu != iommu)
1579 } 1634 continue;
1635
1636 if (!info->dev || !info->domain)
1637 continue;
1638
1639 domain = info->domain;
1640
1641 dmar_remove_one_dev_info(domain, info->dev);
1642
1643 if (!domain_type_is_vm_or_si(domain))
1644 domain_exit(domain);
1580 } 1645 }
1646 spin_unlock_irqrestore(&device_domain_lock, flags);
1581 1647
1582 if (iommu->gcmd & DMA_GCMD_TE) 1648 if (iommu->gcmd & DMA_GCMD_TE)
1583 iommu_disable_translation(iommu); 1649 iommu_disable_translation(iommu);
@@ -1586,6 +1652,11 @@ static void disable_dmar_iommu(struct intel_iommu *iommu)
1586static void free_dmar_iommu(struct intel_iommu *iommu) 1652static void free_dmar_iommu(struct intel_iommu *iommu)
1587{ 1653{
1588 if ((iommu->domains) && (iommu->domain_ids)) { 1654 if ((iommu->domains) && (iommu->domain_ids)) {
1655 int elems = (cap_ndoms(iommu->cap) >> 8) + 1;
1656 int i;
1657
1658 for (i = 0; i < elems; i++)
1659 kfree(iommu->domains[i]);
1589 kfree(iommu->domains); 1660 kfree(iommu->domains);
1590 kfree(iommu->domain_ids); 1661 kfree(iommu->domain_ids);
1591 iommu->domains = NULL; 1662 iommu->domains = NULL;
@@ -1600,8 +1671,6 @@ static void free_dmar_iommu(struct intel_iommu *iommu)
1600 1671
1601static struct dmar_domain *alloc_domain(int flags) 1672static struct dmar_domain *alloc_domain(int flags)
1602{ 1673{
1603 /* domain id for virtual machine, it won't be set in context */
1604 static atomic_t vm_domid = ATOMIC_INIT(0);
1605 struct dmar_domain *domain; 1674 struct dmar_domain *domain;
1606 1675
1607 domain = alloc_domain_mem(); 1676 domain = alloc_domain_mem();
@@ -1611,111 +1680,64 @@ static struct dmar_domain *alloc_domain(int flags)
1611 memset(domain, 0, sizeof(*domain)); 1680 memset(domain, 0, sizeof(*domain));
1612 domain->nid = -1; 1681 domain->nid = -1;
1613 domain->flags = flags; 1682 domain->flags = flags;
1614 spin_lock_init(&domain->iommu_lock);
1615 INIT_LIST_HEAD(&domain->devices); 1683 INIT_LIST_HEAD(&domain->devices);
1616 if (flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
1617 domain->id = atomic_inc_return(&vm_domid);
1618 1684
1619 return domain; 1685 return domain;
1620} 1686}
1621 1687
1622static int __iommu_attach_domain(struct dmar_domain *domain, 1688/* Must be called with iommu->lock */
1623 struct intel_iommu *iommu) 1689static int domain_attach_iommu(struct dmar_domain *domain,
1624{
1625 int num;
1626 unsigned long ndomains;
1627
1628 ndomains = cap_ndoms(iommu->cap);
1629 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1630 if (num < ndomains) {
1631 set_bit(num, iommu->domain_ids);
1632 iommu->domains[num] = domain;
1633 } else {
1634 num = -ENOSPC;
1635 }
1636
1637 return num;
1638}
1639
1640static int iommu_attach_domain(struct dmar_domain *domain,
1641 struct intel_iommu *iommu) 1690 struct intel_iommu *iommu)
1642{ 1691{
1643 int num;
1644 unsigned long flags;
1645
1646 spin_lock_irqsave(&iommu->lock, flags);
1647 num = __iommu_attach_domain(domain, iommu);
1648 spin_unlock_irqrestore(&iommu->lock, flags);
1649 if (num < 0)
1650 pr_err("%s: No free domain ids\n", iommu->name);
1651
1652 return num;
1653}
1654
1655static int iommu_attach_vm_domain(struct dmar_domain *domain,
1656 struct intel_iommu *iommu)
1657{
1658 int num;
1659 unsigned long ndomains; 1692 unsigned long ndomains;
1693 int num;
1660 1694
1661 ndomains = cap_ndoms(iommu->cap); 1695 assert_spin_locked(&device_domain_lock);
1662 for_each_set_bit(num, iommu->domain_ids, ndomains) 1696 assert_spin_locked(&iommu->lock);
1663 if (iommu->domains[num] == domain)
1664 return num;
1665
1666 return __iommu_attach_domain(domain, iommu);
1667}
1668
1669static void iommu_detach_domain(struct dmar_domain *domain,
1670 struct intel_iommu *iommu)
1671{
1672 unsigned long flags;
1673 int num, ndomains;
1674 1697
1675 spin_lock_irqsave(&iommu->lock, flags); 1698 domain->iommu_refcnt[iommu->seq_id] += 1;
1676 if (domain_type_is_vm_or_si(domain)) { 1699 domain->iommu_count += 1;
1700 if (domain->iommu_refcnt[iommu->seq_id] == 1) {
1677 ndomains = cap_ndoms(iommu->cap); 1701 ndomains = cap_ndoms(iommu->cap);
1678 for_each_set_bit(num, iommu->domain_ids, ndomains) { 1702 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1679 if (iommu->domains[num] == domain) { 1703
1680 clear_bit(num, iommu->domain_ids); 1704 if (num >= ndomains) {
1681 iommu->domains[num] = NULL; 1705 pr_err("%s: No free domain ids\n", iommu->name);
1682 break; 1706 domain->iommu_refcnt[iommu->seq_id] -= 1;
1683 } 1707 domain->iommu_count -= 1;
1708 return -ENOSPC;
1684 } 1709 }
1685 } else {
1686 clear_bit(domain->id, iommu->domain_ids);
1687 iommu->domains[domain->id] = NULL;
1688 }
1689 spin_unlock_irqrestore(&iommu->lock, flags);
1690}
1691 1710
1692static void domain_attach_iommu(struct dmar_domain *domain, 1711 set_bit(num, iommu->domain_ids);
1693 struct intel_iommu *iommu) 1712 set_iommu_domain(iommu, num, domain);
1694{ 1713
1695 unsigned long flags; 1714 domain->iommu_did[iommu->seq_id] = num;
1715 domain->nid = iommu->node;
1696 1716
1697 spin_lock_irqsave(&domain->iommu_lock, flags);
1698 if (!test_and_set_bit(iommu->seq_id, domain->iommu_bmp)) {
1699 domain->iommu_count++;
1700 if (domain->iommu_count == 1)
1701 domain->nid = iommu->node;
1702 domain_update_iommu_cap(domain); 1717 domain_update_iommu_cap(domain);
1703 } 1718 }
1704 spin_unlock_irqrestore(&domain->iommu_lock, flags); 1719
1720 return 0;
1705} 1721}
1706 1722
1707static int domain_detach_iommu(struct dmar_domain *domain, 1723static int domain_detach_iommu(struct dmar_domain *domain,
1708 struct intel_iommu *iommu) 1724 struct intel_iommu *iommu)
1709{ 1725{
1710 unsigned long flags; 1726 int num, count = INT_MAX;
1711 int count = INT_MAX; 1727
1728 assert_spin_locked(&device_domain_lock);
1729 assert_spin_locked(&iommu->lock);
1730
1731 domain->iommu_refcnt[iommu->seq_id] -= 1;
1732 count = --domain->iommu_count;
1733 if (domain->iommu_refcnt[iommu->seq_id] == 0) {
1734 num = domain->iommu_did[iommu->seq_id];
1735 clear_bit(num, iommu->domain_ids);
1736 set_iommu_domain(iommu, num, NULL);
1712 1737
1713 spin_lock_irqsave(&domain->iommu_lock, flags);
1714 if (test_and_clear_bit(iommu->seq_id, domain->iommu_bmp)) {
1715 count = --domain->iommu_count;
1716 domain_update_iommu_cap(domain); 1738 domain_update_iommu_cap(domain);
1739 domain->iommu_did[iommu->seq_id] = 0;
1717 } 1740 }
1718 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1719 1741
1720 return count; 1742 return count;
1721} 1743}
@@ -1782,9 +1804,9 @@ static inline int guestwidth_to_adjustwidth(int gaw)
1782 return agaw; 1804 return agaw;
1783} 1805}
1784 1806
1785static int domain_init(struct dmar_domain *domain, int guest_width) 1807static int domain_init(struct dmar_domain *domain, struct intel_iommu *iommu,
1808 int guest_width)
1786{ 1809{
1787 struct intel_iommu *iommu;
1788 int adjust_width, agaw; 1810 int adjust_width, agaw;
1789 unsigned long sagaw; 1811 unsigned long sagaw;
1790 1812
@@ -1793,7 +1815,6 @@ static int domain_init(struct dmar_domain *domain, int guest_width)
1793 domain_reserve_special_ranges(domain); 1815 domain_reserve_special_ranges(domain);
1794 1816
1795 /* calculate AGAW */ 1817 /* calculate AGAW */
1796 iommu = domain_get_iommu(domain);
1797 if (guest_width > cap_mgaw(iommu->cap)) 1818 if (guest_width > cap_mgaw(iommu->cap))
1798 guest_width = cap_mgaw(iommu->cap); 1819 guest_width = cap_mgaw(iommu->cap);
1799 domain->gaw = guest_width; 1820 domain->gaw = guest_width;
@@ -1836,8 +1857,6 @@ static int domain_init(struct dmar_domain *domain, int guest_width)
1836 1857
1837static void domain_exit(struct dmar_domain *domain) 1858static void domain_exit(struct dmar_domain *domain)
1838{ 1859{
1839 struct dmar_drhd_unit *drhd;
1840 struct intel_iommu *iommu;
1841 struct page *freelist = NULL; 1860 struct page *freelist = NULL;
1842 1861
1843 /* Domain 0 is reserved, so dont process it */ 1862 /* Domain 0 is reserved, so dont process it */
@@ -1848,22 +1867,16 @@ static void domain_exit(struct dmar_domain *domain)
1848 if (!intel_iommu_strict) 1867 if (!intel_iommu_strict)
1849 flush_unmaps_timeout(0); 1868 flush_unmaps_timeout(0);
1850 1869
1851 /* remove associated devices */ 1870 /* Remove associated devices and clear attached or cached domains */
1871 rcu_read_lock();
1852 domain_remove_dev_info(domain); 1872 domain_remove_dev_info(domain);
1873 rcu_read_unlock();
1853 1874
1854 /* destroy iovas */ 1875 /* destroy iovas */
1855 put_iova_domain(&domain->iovad); 1876 put_iova_domain(&domain->iovad);
1856 1877
1857 freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw)); 1878 freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1858 1879
1859 /* clear attached or cached domains */
1860 rcu_read_lock();
1861 for_each_active_iommu(iommu, drhd)
1862 if (domain_type_is_vm(domain) ||
1863 test_bit(iommu->seq_id, domain->iommu_bmp))
1864 iommu_detach_domain(domain, iommu);
1865 rcu_read_unlock();
1866
1867 dma_free_pagelist(freelist); 1880 dma_free_pagelist(freelist);
1868 1881
1869 free_domain_mem(domain); 1882 free_domain_mem(domain);
@@ -1871,79 +1884,68 @@ static void domain_exit(struct dmar_domain *domain)
1871 1884
1872static int domain_context_mapping_one(struct dmar_domain *domain, 1885static int domain_context_mapping_one(struct dmar_domain *domain,
1873 struct intel_iommu *iommu, 1886 struct intel_iommu *iommu,
1874 u8 bus, u8 devfn, int translation) 1887 u8 bus, u8 devfn)
1875{ 1888{
1889 u16 did = domain->iommu_did[iommu->seq_id];
1890 int translation = CONTEXT_TT_MULTI_LEVEL;
1891 struct device_domain_info *info = NULL;
1876 struct context_entry *context; 1892 struct context_entry *context;
1877 unsigned long flags; 1893 unsigned long flags;
1878 struct dma_pte *pgd; 1894 struct dma_pte *pgd;
1879 int id; 1895 int ret, agaw;
1880 int agaw; 1896
1881 struct device_domain_info *info = NULL; 1897 WARN_ON(did == 0);
1898
1899 if (hw_pass_through && domain_type_is_si(domain))
1900 translation = CONTEXT_TT_PASS_THROUGH;
1882 1901
1883 pr_debug("Set context mapping for %02x:%02x.%d\n", 1902 pr_debug("Set context mapping for %02x:%02x.%d\n",
1884 bus, PCI_SLOT(devfn), PCI_FUNC(devfn)); 1903 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1885 1904
1886 BUG_ON(!domain->pgd); 1905 BUG_ON(!domain->pgd);
1887 BUG_ON(translation != CONTEXT_TT_PASS_THROUGH &&
1888 translation != CONTEXT_TT_MULTI_LEVEL);
1889 1906
1890 spin_lock_irqsave(&iommu->lock, flags); 1907 spin_lock_irqsave(&device_domain_lock, flags);
1908 spin_lock(&iommu->lock);
1909
1910 ret = -ENOMEM;
1891 context = iommu_context_addr(iommu, bus, devfn, 1); 1911 context = iommu_context_addr(iommu, bus, devfn, 1);
1892 spin_unlock_irqrestore(&iommu->lock, flags);
1893 if (!context) 1912 if (!context)
1894 return -ENOMEM; 1913 goto out_unlock;
1895 spin_lock_irqsave(&iommu->lock, flags);
1896 if (context_present(context)) {
1897 spin_unlock_irqrestore(&iommu->lock, flags);
1898 return 0;
1899 }
1900 1914
1901 context_clear_entry(context); 1915 ret = 0;
1916 if (context_present(context))
1917 goto out_unlock;
1902 1918
1903 id = domain->id;
1904 pgd = domain->pgd; 1919 pgd = domain->pgd;
1905 1920
1906 if (domain_type_is_vm_or_si(domain)) { 1921 context_clear_entry(context);
1907 if (domain_type_is_vm(domain)) { 1922 context_set_domain_id(context, did);
1908 id = iommu_attach_vm_domain(domain, iommu);
1909 if (id < 0) {
1910 spin_unlock_irqrestore(&iommu->lock, flags);
1911 pr_err("%s: No free domain ids\n", iommu->name);
1912 return -EFAULT;
1913 }
1914 }
1915 1923
1916 /* Skip top levels of page tables for 1924 /*
1917 * iommu which has less agaw than default. 1925 * Skip top levels of page tables for iommu which has less agaw
1918 * Unnecessary for PT mode. 1926 * than default. Unnecessary for PT mode.
1919 */ 1927 */
1920 if (translation != CONTEXT_TT_PASS_THROUGH) { 1928 if (translation != CONTEXT_TT_PASS_THROUGH) {
1921 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) { 1929 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1922 pgd = phys_to_virt(dma_pte_addr(pgd)); 1930 ret = -ENOMEM;
1923 if (!dma_pte_present(pgd)) { 1931 pgd = phys_to_virt(dma_pte_addr(pgd));
1924 spin_unlock_irqrestore(&iommu->lock, flags); 1932 if (!dma_pte_present(pgd))
1925 return -ENOMEM; 1933 goto out_unlock;
1926 }
1927 }
1928 } 1934 }
1929 }
1930
1931 context_set_domain_id(context, id);
1932 1935
1933 if (translation != CONTEXT_TT_PASS_THROUGH) {
1934 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn); 1936 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
1935 translation = info ? CONTEXT_TT_DEV_IOTLB : 1937 translation = info ? CONTEXT_TT_DEV_IOTLB :
1936 CONTEXT_TT_MULTI_LEVEL; 1938 CONTEXT_TT_MULTI_LEVEL;
1937 } 1939
1938 /*
1939 * In pass through mode, AW must be programmed to indicate the largest
1940 * AGAW value supported by hardware. And ASR is ignored by hardware.
1941 */
1942 if (unlikely(translation == CONTEXT_TT_PASS_THROUGH))
1943 context_set_address_width(context, iommu->msagaw);
1944 else {
1945 context_set_address_root(context, virt_to_phys(pgd)); 1940 context_set_address_root(context, virt_to_phys(pgd));
1946 context_set_address_width(context, iommu->agaw); 1941 context_set_address_width(context, iommu->agaw);
1942 } else {
1943 /*
1944 * In pass through mode, AW must be programmed to
1945 * indicate the largest AGAW value supported by
1946 * hardware. And ASR is ignored by hardware.
1947 */
1948 context_set_address_width(context, iommu->msagaw);
1947 } 1949 }
1948 1950
1949 context_set_translation_type(context, translation); 1951 context_set_translation_type(context, translation);
@@ -1962,14 +1964,17 @@ static int domain_context_mapping_one(struct dmar_domain *domain,
1962 (((u16)bus) << 8) | devfn, 1964 (((u16)bus) << 8) | devfn,
1963 DMA_CCMD_MASK_NOBIT, 1965 DMA_CCMD_MASK_NOBIT,
1964 DMA_CCMD_DEVICE_INVL); 1966 DMA_CCMD_DEVICE_INVL);
1965 iommu->flush.flush_iotlb(iommu, id, 0, 0, DMA_TLB_DSI_FLUSH); 1967 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
1966 } else { 1968 } else {
1967 iommu_flush_write_buffer(iommu); 1969 iommu_flush_write_buffer(iommu);
1968 } 1970 }
1969 iommu_enable_dev_iotlb(info); 1971 iommu_enable_dev_iotlb(info);
1970 spin_unlock_irqrestore(&iommu->lock, flags);
1971 1972
1972 domain_attach_iommu(domain, iommu); 1973 ret = 0;
1974
1975out_unlock:
1976 spin_unlock(&iommu->lock);
1977 spin_unlock_irqrestore(&device_domain_lock, flags);
1973 1978
1974 return 0; 1979 return 0;
1975} 1980}
@@ -1977,7 +1982,6 @@ static int domain_context_mapping_one(struct dmar_domain *domain,
1977struct domain_context_mapping_data { 1982struct domain_context_mapping_data {
1978 struct dmar_domain *domain; 1983 struct dmar_domain *domain;
1979 struct intel_iommu *iommu; 1984 struct intel_iommu *iommu;
1980 int translation;
1981}; 1985};
1982 1986
1983static int domain_context_mapping_cb(struct pci_dev *pdev, 1987static int domain_context_mapping_cb(struct pci_dev *pdev,
@@ -1986,13 +1990,11 @@ static int domain_context_mapping_cb(struct pci_dev *pdev,
1986 struct domain_context_mapping_data *data = opaque; 1990 struct domain_context_mapping_data *data = opaque;
1987 1991
1988 return domain_context_mapping_one(data->domain, data->iommu, 1992 return domain_context_mapping_one(data->domain, data->iommu,
1989 PCI_BUS_NUM(alias), alias & 0xff, 1993 PCI_BUS_NUM(alias), alias & 0xff);
1990 data->translation);
1991} 1994}
1992 1995
1993static int 1996static int
1994domain_context_mapping(struct dmar_domain *domain, struct device *dev, 1997domain_context_mapping(struct dmar_domain *domain, struct device *dev)
1995 int translation)
1996{ 1998{
1997 struct intel_iommu *iommu; 1999 struct intel_iommu *iommu;
1998 u8 bus, devfn; 2000 u8 bus, devfn;
@@ -2003,12 +2005,10 @@ domain_context_mapping(struct dmar_domain *domain, struct device *dev,
2003 return -ENODEV; 2005 return -ENODEV;
2004 2006
2005 if (!dev_is_pci(dev)) 2007 if (!dev_is_pci(dev))
2006 return domain_context_mapping_one(domain, iommu, bus, devfn, 2008 return domain_context_mapping_one(domain, iommu, bus, devfn);
2007 translation);
2008 2009
2009 data.domain = domain; 2010 data.domain = domain;
2010 data.iommu = iommu; 2011 data.iommu = iommu;
2011 data.translation = translation;
2012 2012
2013 return pci_for_each_dma_alias(to_pci_dev(dev), 2013 return pci_for_each_dma_alias(to_pci_dev(dev),
2014 &domain_context_mapping_cb, &data); 2014 &domain_context_mapping_cb, &data);
@@ -2194,7 +2194,7 @@ static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long i
2194 return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot); 2194 return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
2195} 2195}
2196 2196
2197static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn) 2197static void domain_context_clear_one(struct intel_iommu *iommu, u8 bus, u8 devfn)
2198{ 2198{
2199 if (!iommu) 2199 if (!iommu)
2200 return; 2200 return;
@@ -2220,21 +2220,8 @@ static void domain_remove_dev_info(struct dmar_domain *domain)
2220 unsigned long flags; 2220 unsigned long flags;
2221 2221
2222 spin_lock_irqsave(&device_domain_lock, flags); 2222 spin_lock_irqsave(&device_domain_lock, flags);
2223 list_for_each_entry_safe(info, tmp, &domain->devices, link) { 2223 list_for_each_entry_safe(info, tmp, &domain->devices, link)
2224 unlink_domain_info(info); 2224 __dmar_remove_one_dev_info(info);
2225 spin_unlock_irqrestore(&device_domain_lock, flags);
2226
2227 iommu_disable_dev_iotlb(info);
2228 iommu_detach_dev(info->iommu, info->bus, info->devfn);
2229
2230 if (domain_type_is_vm(domain)) {
2231 iommu_detach_dependent_devices(info->iommu, info->dev);
2232 domain_detach_iommu(domain, info->iommu);
2233 }
2234
2235 free_devinfo_mem(info);
2236 spin_lock_irqsave(&device_domain_lock, flags);
2237 }
2238 spin_unlock_irqrestore(&device_domain_lock, flags); 2225 spin_unlock_irqrestore(&device_domain_lock, flags);
2239} 2226}
2240 2227
@@ -2266,14 +2253,15 @@ dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2266 return NULL; 2253 return NULL;
2267} 2254}
2268 2255
2269static struct dmar_domain *dmar_insert_dev_info(struct intel_iommu *iommu, 2256static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu,
2270 int bus, int devfn, 2257 int bus, int devfn,
2271 struct device *dev, 2258 struct device *dev,
2272 struct dmar_domain *domain) 2259 struct dmar_domain *domain)
2273{ 2260{
2274 struct dmar_domain *found = NULL; 2261 struct dmar_domain *found = NULL;
2275 struct device_domain_info *info; 2262 struct device_domain_info *info;
2276 unsigned long flags; 2263 unsigned long flags;
2264 int ret;
2277 2265
2278 info = alloc_devinfo_mem(); 2266 info = alloc_devinfo_mem();
2279 if (!info) 2267 if (!info)
@@ -2290,12 +2278,16 @@ static struct dmar_domain *dmar_insert_dev_info(struct intel_iommu *iommu,
2290 spin_lock_irqsave(&device_domain_lock, flags); 2278 spin_lock_irqsave(&device_domain_lock, flags);
2291 if (dev) 2279 if (dev)
2292 found = find_domain(dev); 2280 found = find_domain(dev);
2293 else { 2281
2282 if (!found) {
2294 struct device_domain_info *info2; 2283 struct device_domain_info *info2;
2295 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn); 2284 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
2296 if (info2) 2285 if (info2) {
2297 found = info2->domain; 2286 found = info2->domain;
2287 info2->dev = dev;
2288 }
2298 } 2289 }
2290
2299 if (found) { 2291 if (found) {
2300 spin_unlock_irqrestore(&device_domain_lock, flags); 2292 spin_unlock_irqrestore(&device_domain_lock, flags);
2301 free_devinfo_mem(info); 2293 free_devinfo_mem(info);
@@ -2303,12 +2295,27 @@ static struct dmar_domain *dmar_insert_dev_info(struct intel_iommu *iommu,
2303 return found; 2295 return found;
2304 } 2296 }
2305 2297
2298 spin_lock(&iommu->lock);
2299 ret = domain_attach_iommu(domain, iommu);
2300 spin_unlock(&iommu->lock);
2301
2302 if (ret) {
2303 spin_unlock_irqrestore(&device_domain_lock, flags);
2304 return NULL;
2305 }
2306
2306 list_add(&info->link, &domain->devices); 2307 list_add(&info->link, &domain->devices);
2307 list_add(&info->global, &device_domain_list); 2308 list_add(&info->global, &device_domain_list);
2308 if (dev) 2309 if (dev)
2309 dev->archdata.iommu = info; 2310 dev->archdata.iommu = info;
2310 spin_unlock_irqrestore(&device_domain_lock, flags); 2311 spin_unlock_irqrestore(&device_domain_lock, flags);
2311 2312
2313 if (dev && domain_context_mapping(domain, dev)) {
2314 pr_err("Domain context map for %s failed\n", dev_name(dev));
2315 dmar_remove_one_dev_info(domain, dev);
2316 return NULL;
2317 }
2318
2312 return domain; 2319 return domain;
2313} 2320}
2314 2321
@@ -2321,10 +2328,10 @@ static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
2321/* domain is initialized */ 2328/* domain is initialized */
2322static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw) 2329static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
2323{ 2330{
2331 struct device_domain_info *info = NULL;
2324 struct dmar_domain *domain, *tmp; 2332 struct dmar_domain *domain, *tmp;
2325 struct intel_iommu *iommu; 2333 struct intel_iommu *iommu;
2326 struct device_domain_info *info; 2334 u16 req_id, dma_alias;
2327 u16 dma_alias;
2328 unsigned long flags; 2335 unsigned long flags;
2329 u8 bus, devfn; 2336 u8 bus, devfn;
2330 2337
@@ -2336,6 +2343,8 @@ static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
2336 if (!iommu) 2343 if (!iommu)
2337 return NULL; 2344 return NULL;
2338 2345
2346 req_id = ((u16)bus << 8) | devfn;
2347
2339 if (dev_is_pci(dev)) { 2348 if (dev_is_pci(dev)) {
2340 struct pci_dev *pdev = to_pci_dev(dev); 2349 struct pci_dev *pdev = to_pci_dev(dev);
2341 2350
@@ -2360,21 +2369,15 @@ static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
2360 domain = alloc_domain(0); 2369 domain = alloc_domain(0);
2361 if (!domain) 2370 if (!domain)
2362 return NULL; 2371 return NULL;
2363 domain->id = iommu_attach_domain(domain, iommu); 2372 if (domain_init(domain, iommu, gaw)) {
2364 if (domain->id < 0) {
2365 free_domain_mem(domain);
2366 return NULL;
2367 }
2368 domain_attach_iommu(domain, iommu);
2369 if (domain_init(domain, gaw)) {
2370 domain_exit(domain); 2373 domain_exit(domain);
2371 return NULL; 2374 return NULL;
2372 } 2375 }
2373 2376
2374 /* register PCI DMA alias device */ 2377 /* register PCI DMA alias device */
2375 if (dev_is_pci(dev)) { 2378 if (req_id != dma_alias && dev_is_pci(dev)) {
2376 tmp = dmar_insert_dev_info(iommu, PCI_BUS_NUM(dma_alias), 2379 tmp = dmar_insert_one_dev_info(iommu, PCI_BUS_NUM(dma_alias),
2377 dma_alias & 0xff, NULL, domain); 2380 dma_alias & 0xff, NULL, domain);
2378 2381
2379 if (!tmp || tmp != domain) { 2382 if (!tmp || tmp != domain) {
2380 domain_exit(domain); 2383 domain_exit(domain);
@@ -2386,7 +2389,7 @@ static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
2386 } 2389 }
2387 2390
2388found_domain: 2391found_domain:
2389 tmp = dmar_insert_dev_info(iommu, bus, devfn, dev, domain); 2392 tmp = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2390 2393
2391 if (!tmp || tmp != domain) { 2394 if (!tmp || tmp != domain) {
2392 domain_exit(domain); 2395 domain_exit(domain);
@@ -2414,8 +2417,7 @@ static int iommu_domain_identity_map(struct dmar_domain *domain,
2414 return -ENOMEM; 2417 return -ENOMEM;
2415 } 2418 }
2416 2419
2417 pr_debug("Mapping reserved region %llx-%llx for domain %d\n", 2420 pr_debug("Mapping reserved region %llx-%llx\n", start, end);
2418 start, end, domain->id);
2419 /* 2421 /*
2420 * RMRR range might have overlap with physical memory range, 2422 * RMRR range might have overlap with physical memory range,
2421 * clear it first 2423 * clear it first
@@ -2476,11 +2478,6 @@ static int iommu_prepare_identity_map(struct device *dev,
2476 if (ret) 2478 if (ret)
2477 goto error; 2479 goto error;
2478 2480
2479 /* context entry init */
2480 ret = domain_context_mapping(domain, dev, CONTEXT_TT_MULTI_LEVEL);
2481 if (ret)
2482 goto error;
2483
2484 return 0; 2481 return 0;
2485 2482
2486 error: 2483 error:
@@ -2526,37 +2523,18 @@ static int md_domain_init(struct dmar_domain *domain, int guest_width);
2526 2523
2527static int __init si_domain_init(int hw) 2524static int __init si_domain_init(int hw)
2528{ 2525{
2529 struct dmar_drhd_unit *drhd;
2530 struct intel_iommu *iommu;
2531 int nid, ret = 0; 2526 int nid, ret = 0;
2532 bool first = true;
2533 2527
2534 si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY); 2528 si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY);
2535 if (!si_domain) 2529 if (!si_domain)
2536 return -EFAULT; 2530 return -EFAULT;
2537 2531
2538 for_each_active_iommu(iommu, drhd) {
2539 ret = iommu_attach_domain(si_domain, iommu);
2540 if (ret < 0) {
2541 domain_exit(si_domain);
2542 return -EFAULT;
2543 } else if (first) {
2544 si_domain->id = ret;
2545 first = false;
2546 } else if (si_domain->id != ret) {
2547 domain_exit(si_domain);
2548 return -EFAULT;
2549 }
2550 domain_attach_iommu(si_domain, iommu);
2551 }
2552
2553 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) { 2532 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2554 domain_exit(si_domain); 2533 domain_exit(si_domain);
2555 return -EFAULT; 2534 return -EFAULT;
2556 } 2535 }
2557 2536
2558 pr_debug("Identity mapping domain is domain %d\n", 2537 pr_debug("Identity mapping domain allocated\n");
2559 si_domain->id);
2560 2538
2561 if (hw) 2539 if (hw)
2562 return 0; 2540 return 0;
@@ -2590,28 +2568,20 @@ static int identity_mapping(struct device *dev)
2590 return 0; 2568 return 0;
2591} 2569}
2592 2570
2593static int domain_add_dev_info(struct dmar_domain *domain, 2571static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev)
2594 struct device *dev, int translation)
2595{ 2572{
2596 struct dmar_domain *ndomain; 2573 struct dmar_domain *ndomain;
2597 struct intel_iommu *iommu; 2574 struct intel_iommu *iommu;
2598 u8 bus, devfn; 2575 u8 bus, devfn;
2599 int ret;
2600 2576
2601 iommu = device_to_iommu(dev, &bus, &devfn); 2577 iommu = device_to_iommu(dev, &bus, &devfn);
2602 if (!iommu) 2578 if (!iommu)
2603 return -ENODEV; 2579 return -ENODEV;
2604 2580
2605 ndomain = dmar_insert_dev_info(iommu, bus, devfn, dev, domain); 2581 ndomain = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2606 if (ndomain != domain) 2582 if (ndomain != domain)
2607 return -EBUSY; 2583 return -EBUSY;
2608 2584
2609 ret = domain_context_mapping(domain, dev, translation);
2610 if (ret) {
2611 domain_remove_one_dev_info(domain, dev);
2612 return ret;
2613 }
2614
2615 return 0; 2585 return 0;
2616} 2586}
2617 2587
@@ -2751,9 +2721,7 @@ static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw
2751 if (!iommu_should_identity_map(dev, 1)) 2721 if (!iommu_should_identity_map(dev, 1))
2752 return 0; 2722 return 0;
2753 2723
2754 ret = domain_add_dev_info(si_domain, dev, 2724 ret = domain_add_dev_info(si_domain, dev);
2755 hw ? CONTEXT_TT_PASS_THROUGH :
2756 CONTEXT_TT_MULTI_LEVEL);
2757 if (!ret) 2725 if (!ret)
2758 pr_info("%s identity mapping for device %s\n", 2726 pr_info("%s identity mapping for device %s\n",
2759 hw ? "Hardware" : "Software", dev_name(dev)); 2727 hw ? "Hardware" : "Software", dev_name(dev));
@@ -2839,15 +2807,18 @@ static void intel_iommu_init_qi(struct intel_iommu *iommu)
2839} 2807}
2840 2808
2841static int copy_context_table(struct intel_iommu *iommu, 2809static int copy_context_table(struct intel_iommu *iommu,
2842 struct root_entry *old_re, 2810 struct root_entry __iomem *old_re,
2843 struct context_entry **tbl, 2811 struct context_entry **tbl,
2844 int bus, bool ext) 2812 int bus, bool ext)
2845{ 2813{
2846 struct context_entry *old_ce = NULL, *new_ce = NULL, ce;
2847 int tbl_idx, pos = 0, idx, devfn, ret = 0, did; 2814 int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
2815 struct context_entry __iomem *old_ce = NULL;
2816 struct context_entry *new_ce = NULL, ce;
2817 struct root_entry re;
2848 phys_addr_t old_ce_phys; 2818 phys_addr_t old_ce_phys;
2849 2819
2850 tbl_idx = ext ? bus * 2 : bus; 2820 tbl_idx = ext ? bus * 2 : bus;
2821 memcpy_fromio(&re, old_re, sizeof(re));
2851 2822
2852 for (devfn = 0; devfn < 256; devfn++) { 2823 for (devfn = 0; devfn < 256; devfn++) {
2853 /* First calculate the correct index */ 2824 /* First calculate the correct index */
@@ -2867,9 +2838,9 @@ static int copy_context_table(struct intel_iommu *iommu,
2867 2838
2868 ret = 0; 2839 ret = 0;
2869 if (devfn < 0x80) 2840 if (devfn < 0x80)
2870 old_ce_phys = root_entry_lctp(old_re); 2841 old_ce_phys = root_entry_lctp(&re);
2871 else 2842 else
2872 old_ce_phys = root_entry_uctp(old_re); 2843 old_ce_phys = root_entry_uctp(&re);
2873 2844
2874 if (!old_ce_phys) { 2845 if (!old_ce_phys) {
2875 if (ext && devfn == 0) { 2846 if (ext && devfn == 0) {
@@ -2894,7 +2865,7 @@ static int copy_context_table(struct intel_iommu *iommu,
2894 } 2865 }
2895 2866
2896 /* Now copy the context entry */ 2867 /* Now copy the context entry */
2897 ce = old_ce[idx]; 2868 memcpy_fromio(&ce, old_ce + idx, sizeof(ce));
2898 2869
2899 if (!__context_present(&ce)) 2870 if (!__context_present(&ce))
2900 continue; 2871 continue;
@@ -2938,8 +2909,8 @@ out:
2938 2909
2939static int copy_translation_tables(struct intel_iommu *iommu) 2910static int copy_translation_tables(struct intel_iommu *iommu)
2940{ 2911{
2912 struct root_entry __iomem *old_rt;
2941 struct context_entry **ctxt_tbls; 2913 struct context_entry **ctxt_tbls;
2942 struct root_entry *old_rt;
2943 phys_addr_t old_rt_phys; 2914 phys_addr_t old_rt_phys;
2944 int ctxt_table_entries; 2915 int ctxt_table_entries;
2945 unsigned long flags; 2916 unsigned long flags;
@@ -3269,7 +3240,6 @@ static struct iova *intel_alloc_iova(struct device *dev,
3269static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev) 3240static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev)
3270{ 3241{
3271 struct dmar_domain *domain; 3242 struct dmar_domain *domain;
3272 int ret;
3273 3243
3274 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH); 3244 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
3275 if (!domain) { 3245 if (!domain) {
@@ -3278,16 +3248,6 @@ static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev)
3278 return NULL; 3248 return NULL;
3279 } 3249 }
3280 3250
3281 /* make sure context mapping is ok */
3282 if (unlikely(!domain_context_mapped(dev))) {
3283 ret = domain_context_mapping(domain, dev, CONTEXT_TT_MULTI_LEVEL);
3284 if (ret) {
3285 pr_err("Domain context map for %s failed\n",
3286 dev_name(dev));
3287 return NULL;
3288 }
3289 }
3290
3291 return domain; 3251 return domain;
3292} 3252}
3293 3253
@@ -3323,7 +3283,7 @@ static int iommu_no_mapping(struct device *dev)
3323 * 32 bit DMA is removed from si_domain and fall back 3283 * 32 bit DMA is removed from si_domain and fall back
3324 * to non-identity mapping. 3284 * to non-identity mapping.
3325 */ 3285 */
3326 domain_remove_one_dev_info(si_domain, dev); 3286 dmar_remove_one_dev_info(si_domain, dev);
3327 pr_info("32bit %s uses non-identity mapping\n", 3287 pr_info("32bit %s uses non-identity mapping\n",
3328 dev_name(dev)); 3288 dev_name(dev));
3329 return 0; 3289 return 0;
@@ -3335,10 +3295,7 @@ static int iommu_no_mapping(struct device *dev)
3335 */ 3295 */
3336 if (iommu_should_identity_map(dev, 0)) { 3296 if (iommu_should_identity_map(dev, 0)) {
3337 int ret; 3297 int ret;
3338 ret = domain_add_dev_info(si_domain, dev, 3298 ret = domain_add_dev_info(si_domain, dev);
3339 hw_pass_through ?
3340 CONTEXT_TT_PASS_THROUGH :
3341 CONTEXT_TT_MULTI_LEVEL);
3342 if (!ret) { 3299 if (!ret) {
3343 pr_info("64bit %s uses identity mapping\n", 3300 pr_info("64bit %s uses identity mapping\n",
3344 dev_name(dev)); 3301 dev_name(dev));
@@ -3399,7 +3356,9 @@ static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
3399 3356
3400 /* it's a non-present to present mapping. Only flush if caching mode */ 3357 /* it's a non-present to present mapping. Only flush if caching mode */
3401 if (cap_caching_mode(iommu->cap)) 3358 if (cap_caching_mode(iommu->cap))
3402 iommu_flush_iotlb_psi(iommu, domain->id, mm_to_dma_pfn(iova->pfn_lo), size, 0, 1); 3359 iommu_flush_iotlb_psi(iommu, domain,
3360 mm_to_dma_pfn(iova->pfn_lo),
3361 size, 0, 1);
3403 else 3362 else
3404 iommu_flush_write_buffer(iommu); 3363 iommu_flush_write_buffer(iommu);
3405 3364
@@ -3450,7 +3409,7 @@ static void flush_unmaps(void)
3450 3409
3451 /* On real hardware multiple invalidations are expensive */ 3410 /* On real hardware multiple invalidations are expensive */
3452 if (cap_caching_mode(iommu->cap)) 3411 if (cap_caching_mode(iommu->cap))
3453 iommu_flush_iotlb_psi(iommu, domain->id, 3412 iommu_flush_iotlb_psi(iommu, domain,
3454 iova->pfn_lo, iova_size(iova), 3413 iova->pfn_lo, iova_size(iova),
3455 !deferred_flush[i].freelist[j], 0); 3414 !deferred_flush[i].freelist[j], 0);
3456 else { 3415 else {
@@ -3534,7 +3493,7 @@ static void intel_unmap(struct device *dev, dma_addr_t dev_addr)
3534 freelist = domain_unmap(domain, start_pfn, last_pfn); 3493 freelist = domain_unmap(domain, start_pfn, last_pfn);
3535 3494
3536 if (intel_iommu_strict) { 3495 if (intel_iommu_strict) {
3537 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn, 3496 iommu_flush_iotlb_psi(iommu, domain, start_pfn,
3538 last_pfn - start_pfn + 1, !freelist, 0); 3497 last_pfn - start_pfn + 1, !freelist, 0);
3539 /* free iova */ 3498 /* free iova */
3540 __free_iova(&domain->iovad, iova); 3499 __free_iova(&domain->iovad, iova);
@@ -3692,7 +3651,7 @@ static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nele
3692 3651
3693 /* it's a non-present to present mapping. Only flush if caching mode */ 3652 /* it's a non-present to present mapping. Only flush if caching mode */
3694 if (cap_caching_mode(iommu->cap)) 3653 if (cap_caching_mode(iommu->cap))
3695 iommu_flush_iotlb_psi(iommu, domain->id, start_vpfn, size, 0, 1); 3654 iommu_flush_iotlb_psi(iommu, domain, start_vpfn, size, 0, 1);
3696 else 3655 else
3697 iommu_flush_write_buffer(iommu); 3656 iommu_flush_write_buffer(iommu);
3698 3657
@@ -4169,13 +4128,6 @@ static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
4169 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH); 4128 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4170 iommu_enable_translation(iommu); 4129 iommu_enable_translation(iommu);
4171 4130
4172 if (si_domain) {
4173 ret = iommu_attach_domain(si_domain, iommu);
4174 if (ret < 0 || si_domain->id != ret)
4175 goto disable_iommu;
4176 domain_attach_iommu(si_domain, iommu);
4177 }
4178
4179 iommu_disable_protect_mem_regions(iommu); 4131 iommu_disable_protect_mem_regions(iommu);
4180 return 0; 4132 return 0;
4181 4133
@@ -4337,11 +4289,9 @@ static int device_notifier(struct notifier_block *nb,
4337 if (!domain) 4289 if (!domain)
4338 return 0; 4290 return 0;
4339 4291
4340 down_read(&dmar_global_lock); 4292 dmar_remove_one_dev_info(domain, dev);
4341 domain_remove_one_dev_info(domain, dev);
4342 if (!domain_type_is_vm_or_si(domain) && list_empty(&domain->devices)) 4293 if (!domain_type_is_vm_or_si(domain) && list_empty(&domain->devices))
4343 domain_exit(domain); 4294 domain_exit(domain);
4344 up_read(&dmar_global_lock);
4345 4295
4346 return 0; 4296 return 0;
4347} 4297}
@@ -4398,7 +4348,7 @@ static int intel_iommu_memory_notifier(struct notifier_block *nb,
4398 4348
4399 rcu_read_lock(); 4349 rcu_read_lock();
4400 for_each_active_iommu(iommu, drhd) 4350 for_each_active_iommu(iommu, drhd)
4401 iommu_flush_iotlb_psi(iommu, si_domain->id, 4351 iommu_flush_iotlb_psi(iommu, si_domain,
4402 iova->pfn_lo, iova_size(iova), 4352 iova->pfn_lo, iova_size(iova),
4403 !freelist, 0); 4353 !freelist, 0);
4404 rcu_read_unlock(); 4354 rcu_read_unlock();
@@ -4457,11 +4407,32 @@ static ssize_t intel_iommu_show_ecap(struct device *dev,
4457} 4407}
4458static DEVICE_ATTR(ecap, S_IRUGO, intel_iommu_show_ecap, NULL); 4408static DEVICE_ATTR(ecap, S_IRUGO, intel_iommu_show_ecap, NULL);
4459 4409
4410static ssize_t intel_iommu_show_ndoms(struct device *dev,
4411 struct device_attribute *attr,
4412 char *buf)
4413{
4414 struct intel_iommu *iommu = dev_get_drvdata(dev);
4415 return sprintf(buf, "%ld\n", cap_ndoms(iommu->cap));
4416}
4417static DEVICE_ATTR(domains_supported, S_IRUGO, intel_iommu_show_ndoms, NULL);
4418
4419static ssize_t intel_iommu_show_ndoms_used(struct device *dev,
4420 struct device_attribute *attr,
4421 char *buf)
4422{
4423 struct intel_iommu *iommu = dev_get_drvdata(dev);
4424 return sprintf(buf, "%d\n", bitmap_weight(iommu->domain_ids,
4425 cap_ndoms(iommu->cap)));
4426}
4427static DEVICE_ATTR(domains_used, S_IRUGO, intel_iommu_show_ndoms_used, NULL);
4428
4460static struct attribute *intel_iommu_attrs[] = { 4429static struct attribute *intel_iommu_attrs[] = {
4461 &dev_attr_version.attr, 4430 &dev_attr_version.attr,
4462 &dev_attr_address.attr, 4431 &dev_attr_address.attr,
4463 &dev_attr_cap.attr, 4432 &dev_attr_cap.attr,
4464 &dev_attr_ecap.attr, 4433 &dev_attr_ecap.attr,
4434 &dev_attr_domains_supported.attr,
4435 &dev_attr_domains_used.attr,
4465 NULL, 4436 NULL,
4466}; 4437};
4467 4438
@@ -4541,7 +4512,7 @@ int __init intel_iommu_init(void)
4541 for_each_active_iommu(iommu, drhd) 4512 for_each_active_iommu(iommu, drhd)
4542 iommu->iommu_dev = iommu_device_create(NULL, iommu, 4513 iommu->iommu_dev = iommu_device_create(NULL, iommu,
4543 intel_iommu_groups, 4514 intel_iommu_groups,
4544 iommu->name); 4515 "%s", iommu->name);
4545 4516
4546 bus_set_iommu(&pci_bus_type, &intel_iommu_ops); 4517 bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
4547 bus_register_notifier(&pci_bus_type, &device_nb); 4518 bus_register_notifier(&pci_bus_type, &device_nb);
@@ -4561,11 +4532,11 @@ out_free_dmar:
4561 return ret; 4532 return ret;
4562} 4533}
4563 4534
4564static int iommu_detach_dev_cb(struct pci_dev *pdev, u16 alias, void *opaque) 4535static int domain_context_clear_one_cb(struct pci_dev *pdev, u16 alias, void *opaque)
4565{ 4536{
4566 struct intel_iommu *iommu = opaque; 4537 struct intel_iommu *iommu = opaque;
4567 4538
4568 iommu_detach_dev(iommu, PCI_BUS_NUM(alias), alias & 0xff); 4539 domain_context_clear_one(iommu, PCI_BUS_NUM(alias), alias & 0xff);
4569 return 0; 4540 return 0;
4570} 4541}
4571 4542
@@ -4575,63 +4546,50 @@ static int iommu_detach_dev_cb(struct pci_dev *pdev, u16 alias, void *opaque)
4575 * devices, unbinding the driver from any one of them will possibly leave 4546 * devices, unbinding the driver from any one of them will possibly leave
4576 * the others unable to operate. 4547 * the others unable to operate.
4577 */ 4548 */
4578static void iommu_detach_dependent_devices(struct intel_iommu *iommu, 4549static void domain_context_clear(struct intel_iommu *iommu, struct device *dev)
4579 struct device *dev)
4580{ 4550{
4581 if (!iommu || !dev || !dev_is_pci(dev)) 4551 if (!iommu || !dev || !dev_is_pci(dev))
4582 return; 4552 return;
4583 4553
4584 pci_for_each_dma_alias(to_pci_dev(dev), &iommu_detach_dev_cb, iommu); 4554 pci_for_each_dma_alias(to_pci_dev(dev), &domain_context_clear_one_cb, iommu);
4585} 4555}
4586 4556
4587static void domain_remove_one_dev_info(struct dmar_domain *domain, 4557static void __dmar_remove_one_dev_info(struct device_domain_info *info)
4588 struct device *dev)
4589{ 4558{
4590 struct device_domain_info *info, *tmp;
4591 struct intel_iommu *iommu; 4559 struct intel_iommu *iommu;
4592 unsigned long flags; 4560 unsigned long flags;
4593 bool found = false;
4594 u8 bus, devfn;
4595 4561
4596 iommu = device_to_iommu(dev, &bus, &devfn); 4562 assert_spin_locked(&device_domain_lock);
4597 if (!iommu) 4563
4564 if (WARN_ON(!info))
4598 return; 4565 return;
4599 4566
4600 spin_lock_irqsave(&device_domain_lock, flags); 4567 iommu = info->iommu;
4601 list_for_each_entry_safe(info, tmp, &domain->devices, link) {
4602 if (info->iommu == iommu && info->bus == bus &&
4603 info->devfn == devfn) {
4604 unlink_domain_info(info);
4605 spin_unlock_irqrestore(&device_domain_lock, flags);
4606 4568
4607 iommu_disable_dev_iotlb(info); 4569 if (info->dev) {
4608 iommu_detach_dev(iommu, info->bus, info->devfn); 4570 iommu_disable_dev_iotlb(info);
4609 iommu_detach_dependent_devices(iommu, dev); 4571 domain_context_clear(iommu, info->dev);
4610 free_devinfo_mem(info); 4572 }
4611 4573
4612 spin_lock_irqsave(&device_domain_lock, flags); 4574 unlink_domain_info(info);
4613 4575
4614 if (found) 4576 spin_lock_irqsave(&iommu->lock, flags);
4615 break; 4577 domain_detach_iommu(info->domain, iommu);
4616 else 4578 spin_unlock_irqrestore(&iommu->lock, flags);
4617 continue;
4618 }
4619 4579
4620 /* if there is no other devices under the same iommu 4580 free_devinfo_mem(info);
4621 * owned by this domain, clear this iommu in iommu_bmp 4581}
4622 * update iommu count and coherency
4623 */
4624 if (info->iommu == iommu)
4625 found = true;
4626 }
4627 4582
4628 spin_unlock_irqrestore(&device_domain_lock, flags); 4583static void dmar_remove_one_dev_info(struct dmar_domain *domain,
4584 struct device *dev)
4585{
4586 struct device_domain_info *info;
4587 unsigned long flags;
4629 4588
4630 if (found == 0) { 4589 spin_lock_irqsave(&device_domain_lock, flags);
4631 domain_detach_iommu(domain, iommu); 4590 info = dev->archdata.iommu;
4632 if (!domain_type_is_vm_or_si(domain)) 4591 __dmar_remove_one_dev_info(info);
4633 iommu_detach_domain(domain, iommu); 4592 spin_unlock_irqrestore(&device_domain_lock, flags);
4634 }
4635} 4593}
4636 4594
4637static int md_domain_init(struct dmar_domain *domain, int guest_width) 4595static int md_domain_init(struct dmar_domain *domain, int guest_width)
@@ -4712,10 +4670,9 @@ static int intel_iommu_attach_device(struct iommu_domain *domain,
4712 4670
4713 old_domain = find_domain(dev); 4671 old_domain = find_domain(dev);
4714 if (old_domain) { 4672 if (old_domain) {
4715 if (domain_type_is_vm_or_si(dmar_domain)) 4673 rcu_read_lock();
4716 domain_remove_one_dev_info(old_domain, dev); 4674 dmar_remove_one_dev_info(old_domain, dev);
4717 else 4675 rcu_read_unlock();
4718 domain_remove_dev_info(old_domain);
4719 4676
4720 if (!domain_type_is_vm_or_si(old_domain) && 4677 if (!domain_type_is_vm_or_si(old_domain) &&
4721 list_empty(&old_domain->devices)) 4678 list_empty(&old_domain->devices))
@@ -4755,13 +4712,13 @@ static int intel_iommu_attach_device(struct iommu_domain *domain,
4755 dmar_domain->agaw--; 4712 dmar_domain->agaw--;
4756 } 4713 }
4757 4714
4758 return domain_add_dev_info(dmar_domain, dev, CONTEXT_TT_MULTI_LEVEL); 4715 return domain_add_dev_info(dmar_domain, dev);
4759} 4716}
4760 4717
4761static void intel_iommu_detach_device(struct iommu_domain *domain, 4718static void intel_iommu_detach_device(struct iommu_domain *domain,
4762 struct device *dev) 4719 struct device *dev)
4763{ 4720{
4764 domain_remove_one_dev_info(to_dmar_domain(domain), dev); 4721 dmar_remove_one_dev_info(to_dmar_domain(domain), dev);
4765} 4722}
4766 4723
4767static int intel_iommu_map(struct iommu_domain *domain, 4724static int intel_iommu_map(struct iommu_domain *domain,
@@ -4810,12 +4767,11 @@ static size_t intel_iommu_unmap(struct iommu_domain *domain,
4810 struct intel_iommu *iommu; 4767 struct intel_iommu *iommu;
4811 unsigned long start_pfn, last_pfn; 4768 unsigned long start_pfn, last_pfn;
4812 unsigned int npages; 4769 unsigned int npages;
4813 int iommu_id, num, ndomains, level = 0; 4770 int iommu_id, level = 0;
4814 4771
4815 /* Cope with horrid API which requires us to unmap more than the 4772 /* Cope with horrid API which requires us to unmap more than the
4816 size argument if it happens to be a large-page mapping. */ 4773 size argument if it happens to be a large-page mapping. */
4817 if (!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level)) 4774 BUG_ON(!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level));
4818 BUG();
4819 4775
4820 if (size < VTD_PAGE_SIZE << level_to_offset_bits(level)) 4776 if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
4821 size = VTD_PAGE_SIZE << level_to_offset_bits(level); 4777 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
@@ -4827,19 +4783,11 @@ static size_t intel_iommu_unmap(struct iommu_domain *domain,
4827 4783
4828 npages = last_pfn - start_pfn + 1; 4784 npages = last_pfn - start_pfn + 1;
4829 4785
4830 for_each_set_bit(iommu_id, dmar_domain->iommu_bmp, g_num_of_iommus) { 4786 for_each_domain_iommu(iommu_id, dmar_domain) {
4831 iommu = g_iommus[iommu_id]; 4787 iommu = g_iommus[iommu_id];
4832
4833 /*
4834 * find bit position of dmar_domain
4835 */
4836 ndomains = cap_ndoms(iommu->cap);
4837 for_each_set_bit(num, iommu->domain_ids, ndomains) {
4838 if (iommu->domains[num] == dmar_domain)
4839 iommu_flush_iotlb_psi(iommu, num, start_pfn,
4840 npages, !freelist, 0);
4841 }
4842 4788
4789 iommu_flush_iotlb_psi(g_iommus[iommu_id], dmar_domain,
4790 start_pfn, npages, !freelist, 0);
4843 } 4791 }
4844 4792
4845 dma_free_pagelist(freelist); 4793 dma_free_pagelist(freelist);
diff --git a/drivers/iommu/intel_irq_remapping.c b/drivers/iommu/intel_irq_remapping.c
index f15692a410c7..9ec4e0d94ffd 100644
--- a/drivers/iommu/intel_irq_remapping.c
+++ b/drivers/iommu/intel_irq_remapping.c
@@ -384,7 +384,7 @@ static int set_msi_sid(struct irte *irte, struct pci_dev *dev)
384 384
385static int iommu_load_old_irte(struct intel_iommu *iommu) 385static int iommu_load_old_irte(struct intel_iommu *iommu)
386{ 386{
387 struct irte *old_ir_table; 387 struct irte __iomem *old_ir_table;
388 phys_addr_t irt_phys; 388 phys_addr_t irt_phys;
389 unsigned int i; 389 unsigned int i;
390 size_t size; 390 size_t size;
@@ -413,7 +413,7 @@ static int iommu_load_old_irte(struct intel_iommu *iommu)
413 return -ENOMEM; 413 return -ENOMEM;
414 414
415 /* Copy data over */ 415 /* Copy data over */
416 memcpy(iommu->ir_table->base, old_ir_table, size); 416 memcpy_fromio(iommu->ir_table->base, old_ir_table, size);
417 417
418 __iommu_flush_cache(iommu, iommu->ir_table->base, size); 418 __iommu_flush_cache(iommu, iommu->ir_table->base, size);
419 419
@@ -426,6 +426,8 @@ static int iommu_load_old_irte(struct intel_iommu *iommu)
426 bitmap_set(iommu->ir_table->bitmap, i, 1); 426 bitmap_set(iommu->ir_table->bitmap, i, 1);
427 } 427 }
428 428
429 iounmap(old_ir_table);
430
429 return 0; 431 return 0;
430} 432}
431 433
diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c
index 4e460216bd16..73c07482f487 100644
--- a/drivers/iommu/io-pgtable-arm.c
+++ b/drivers/iommu/io-pgtable-arm.c
@@ -26,6 +26,8 @@
26#include <linux/slab.h> 26#include <linux/slab.h>
27#include <linux/types.h> 27#include <linux/types.h>
28 28
29#include <asm/barrier.h>
30
29#include "io-pgtable.h" 31#include "io-pgtable.h"
30 32
31#define ARM_LPAE_MAX_ADDR_BITS 48 33#define ARM_LPAE_MAX_ADDR_BITS 48
@@ -200,20 +202,97 @@ typedef u64 arm_lpae_iopte;
200 202
201static bool selftest_running = false; 203static bool selftest_running = false;
202 204
205static dma_addr_t __arm_lpae_dma_addr(struct device *dev, void *pages)
206{
207 return phys_to_dma(dev, virt_to_phys(pages));
208}
209
210static void *__arm_lpae_alloc_pages(size_t size, gfp_t gfp,
211 struct io_pgtable_cfg *cfg)
212{
213 struct device *dev = cfg->iommu_dev;
214 dma_addr_t dma;
215 void *pages = alloc_pages_exact(size, gfp | __GFP_ZERO);
216
217 if (!pages)
218 return NULL;
219
220 if (!selftest_running) {
221 dma = dma_map_single(dev, pages, size, DMA_TO_DEVICE);
222 if (dma_mapping_error(dev, dma))
223 goto out_free;
224 /*
225 * We depend on the IOMMU being able to work with any physical
226 * address directly, so if the DMA layer suggests it can't by
227 * giving us back some translation, that bodes very badly...
228 */
229 if (dma != __arm_lpae_dma_addr(dev, pages))
230 goto out_unmap;
231 }
232
233 return pages;
234
235out_unmap:
236 dev_err(dev, "Cannot accommodate DMA translation for IOMMU page tables\n");
237 dma_unmap_single(dev, dma, size, DMA_TO_DEVICE);
238out_free:
239 free_pages_exact(pages, size);
240 return NULL;
241}
242
243static void __arm_lpae_free_pages(void *pages, size_t size,
244 struct io_pgtable_cfg *cfg)
245{
246 struct device *dev = cfg->iommu_dev;
247
248 if (!selftest_running)
249 dma_unmap_single(dev, __arm_lpae_dma_addr(dev, pages),
250 size, DMA_TO_DEVICE);
251 free_pages_exact(pages, size);
252}
253
254static void __arm_lpae_set_pte(arm_lpae_iopte *ptep, arm_lpae_iopte pte,
255 struct io_pgtable_cfg *cfg)
256{
257 struct device *dev = cfg->iommu_dev;
258
259 *ptep = pte;
260
261 if (!selftest_running)
262 dma_sync_single_for_device(dev, __arm_lpae_dma_addr(dev, ptep),
263 sizeof(pte), DMA_TO_DEVICE);
264}
265
266static int __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
267 unsigned long iova, size_t size, int lvl,
268 arm_lpae_iopte *ptep);
269
203static int arm_lpae_init_pte(struct arm_lpae_io_pgtable *data, 270static int arm_lpae_init_pte(struct arm_lpae_io_pgtable *data,
204 unsigned long iova, phys_addr_t paddr, 271 unsigned long iova, phys_addr_t paddr,
205 arm_lpae_iopte prot, int lvl, 272 arm_lpae_iopte prot, int lvl,
206 arm_lpae_iopte *ptep) 273 arm_lpae_iopte *ptep)
207{ 274{
208 arm_lpae_iopte pte = prot; 275 arm_lpae_iopte pte = prot;
276 struct io_pgtable_cfg *cfg = &data->iop.cfg;
209 277
210 /* We require an unmap first */
211 if (iopte_leaf(*ptep, lvl)) { 278 if (iopte_leaf(*ptep, lvl)) {
279 /* We require an unmap first */
212 WARN_ON(!selftest_running); 280 WARN_ON(!selftest_running);
213 return -EEXIST; 281 return -EEXIST;
282 } else if (iopte_type(*ptep, lvl) == ARM_LPAE_PTE_TYPE_TABLE) {
283 /*
284 * We need to unmap and free the old table before
285 * overwriting it with a block entry.
286 */
287 arm_lpae_iopte *tblp;
288 size_t sz = ARM_LPAE_BLOCK_SIZE(lvl, data);
289
290 tblp = ptep - ARM_LPAE_LVL_IDX(iova, lvl, data);
291 if (WARN_ON(__arm_lpae_unmap(data, iova, sz, lvl, tblp) != sz))
292 return -EINVAL;
214 } 293 }
215 294
216 if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_NS) 295 if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS)
217 pte |= ARM_LPAE_PTE_NS; 296 pte |= ARM_LPAE_PTE_NS;
218 297
219 if (lvl == ARM_LPAE_MAX_LEVELS - 1) 298 if (lvl == ARM_LPAE_MAX_LEVELS - 1)
@@ -224,8 +303,7 @@ static int arm_lpae_init_pte(struct arm_lpae_io_pgtable *data,
224 pte |= ARM_LPAE_PTE_AF | ARM_LPAE_PTE_SH_IS; 303 pte |= ARM_LPAE_PTE_AF | ARM_LPAE_PTE_SH_IS;
225 pte |= pfn_to_iopte(paddr >> data->pg_shift, data); 304 pte |= pfn_to_iopte(paddr >> data->pg_shift, data);
226 305
227 *ptep = pte; 306 __arm_lpae_set_pte(ptep, pte, cfg);
228 data->iop.cfg.tlb->flush_pgtable(ptep, sizeof(*ptep), data->iop.cookie);
229 return 0; 307 return 0;
230} 308}
231 309
@@ -234,14 +312,14 @@ static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova,
234 int lvl, arm_lpae_iopte *ptep) 312 int lvl, arm_lpae_iopte *ptep)
235{ 313{
236 arm_lpae_iopte *cptep, pte; 314 arm_lpae_iopte *cptep, pte;
237 void *cookie = data->iop.cookie;
238 size_t block_size = ARM_LPAE_BLOCK_SIZE(lvl, data); 315 size_t block_size = ARM_LPAE_BLOCK_SIZE(lvl, data);
316 struct io_pgtable_cfg *cfg = &data->iop.cfg;
239 317
240 /* Find our entry at the current level */ 318 /* Find our entry at the current level */
241 ptep += ARM_LPAE_LVL_IDX(iova, lvl, data); 319 ptep += ARM_LPAE_LVL_IDX(iova, lvl, data);
242 320
243 /* If we can install a leaf entry at this level, then do so */ 321 /* If we can install a leaf entry at this level, then do so */
244 if (size == block_size && (size & data->iop.cfg.pgsize_bitmap)) 322 if (size == block_size && (size & cfg->pgsize_bitmap))
245 return arm_lpae_init_pte(data, iova, paddr, prot, lvl, ptep); 323 return arm_lpae_init_pte(data, iova, paddr, prot, lvl, ptep);
246 324
247 /* We can't allocate tables at the final level */ 325 /* We can't allocate tables at the final level */
@@ -251,18 +329,15 @@ static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova,
251 /* Grab a pointer to the next level */ 329 /* Grab a pointer to the next level */
252 pte = *ptep; 330 pte = *ptep;
253 if (!pte) { 331 if (!pte) {
254 cptep = alloc_pages_exact(1UL << data->pg_shift, 332 cptep = __arm_lpae_alloc_pages(1UL << data->pg_shift,
255 GFP_ATOMIC | __GFP_ZERO); 333 GFP_ATOMIC, cfg);
256 if (!cptep) 334 if (!cptep)
257 return -ENOMEM; 335 return -ENOMEM;
258 336
259 data->iop.cfg.tlb->flush_pgtable(cptep, 1UL << data->pg_shift,
260 cookie);
261 pte = __pa(cptep) | ARM_LPAE_PTE_TYPE_TABLE; 337 pte = __pa(cptep) | ARM_LPAE_PTE_TYPE_TABLE;
262 if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_NS) 338 if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS)
263 pte |= ARM_LPAE_PTE_NSTABLE; 339 pte |= ARM_LPAE_PTE_NSTABLE;
264 *ptep = pte; 340 __arm_lpae_set_pte(ptep, pte, cfg);
265 data->iop.cfg.tlb->flush_pgtable(ptep, sizeof(*ptep), cookie);
266 } else { 341 } else {
267 cptep = iopte_deref(pte, data); 342 cptep = iopte_deref(pte, data);
268 } 343 }
@@ -309,7 +384,7 @@ static int arm_lpae_map(struct io_pgtable_ops *ops, unsigned long iova,
309{ 384{
310 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); 385 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops);
311 arm_lpae_iopte *ptep = data->pgd; 386 arm_lpae_iopte *ptep = data->pgd;
312 int lvl = ARM_LPAE_START_LVL(data); 387 int ret, lvl = ARM_LPAE_START_LVL(data);
313 arm_lpae_iopte prot; 388 arm_lpae_iopte prot;
314 389
315 /* If no access, then nothing to do */ 390 /* If no access, then nothing to do */
@@ -317,7 +392,14 @@ static int arm_lpae_map(struct io_pgtable_ops *ops, unsigned long iova,
317 return 0; 392 return 0;
318 393
319 prot = arm_lpae_prot_to_pte(data, iommu_prot); 394 prot = arm_lpae_prot_to_pte(data, iommu_prot);
320 return __arm_lpae_map(data, iova, paddr, size, prot, lvl, ptep); 395 ret = __arm_lpae_map(data, iova, paddr, size, prot, lvl, ptep);
396 /*
397 * Synchronise all PTE updates for the new mapping before there's
398 * a chance for anything to kick off a table walk for the new iova.
399 */
400 wmb();
401
402 return ret;
321} 403}
322 404
323static void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl, 405static void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl,
@@ -347,7 +429,7 @@ static void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl,
347 __arm_lpae_free_pgtable(data, lvl + 1, iopte_deref(pte, data)); 429 __arm_lpae_free_pgtable(data, lvl + 1, iopte_deref(pte, data));
348 } 430 }
349 431
350 free_pages_exact(start, table_size); 432 __arm_lpae_free_pages(start, table_size, &data->iop.cfg);
351} 433}
352 434
353static void arm_lpae_free_pgtable(struct io_pgtable *iop) 435static void arm_lpae_free_pgtable(struct io_pgtable *iop)
@@ -366,8 +448,7 @@ static int arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data,
366 unsigned long blk_start, blk_end; 448 unsigned long blk_start, blk_end;
367 phys_addr_t blk_paddr; 449 phys_addr_t blk_paddr;
368 arm_lpae_iopte table = 0; 450 arm_lpae_iopte table = 0;
369 void *cookie = data->iop.cookie; 451 struct io_pgtable_cfg *cfg = &data->iop.cfg;
370 const struct iommu_gather_ops *tlb = data->iop.cfg.tlb;
371 452
372 blk_start = iova & ~(blk_size - 1); 453 blk_start = iova & ~(blk_size - 1);
373 blk_end = blk_start + blk_size; 454 blk_end = blk_start + blk_size;
@@ -393,10 +474,9 @@ static int arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data,
393 } 474 }
394 } 475 }
395 476
396 *ptep = table; 477 __arm_lpae_set_pte(ptep, table, cfg);
397 tlb->flush_pgtable(ptep, sizeof(*ptep), cookie);
398 iova &= ~(blk_size - 1); 478 iova &= ~(blk_size - 1);
399 tlb->tlb_add_flush(iova, blk_size, true, cookie); 479 cfg->tlb->tlb_add_flush(iova, blk_size, true, data->iop.cookie);
400 return size; 480 return size;
401} 481}
402 482
@@ -418,13 +498,12 @@ static int __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
418 498
419 /* If the size matches this level, we're in the right place */ 499 /* If the size matches this level, we're in the right place */
420 if (size == blk_size) { 500 if (size == blk_size) {
421 *ptep = 0; 501 __arm_lpae_set_pte(ptep, 0, &data->iop.cfg);
422 tlb->flush_pgtable(ptep, sizeof(*ptep), cookie);
423 502
424 if (!iopte_leaf(pte, lvl)) { 503 if (!iopte_leaf(pte, lvl)) {
425 /* Also flush any partial walks */ 504 /* Also flush any partial walks */
426 tlb->tlb_add_flush(iova, size, false, cookie); 505 tlb->tlb_add_flush(iova, size, false, cookie);
427 tlb->tlb_sync(data->iop.cookie); 506 tlb->tlb_sync(cookie);
428 ptep = iopte_deref(pte, data); 507 ptep = iopte_deref(pte, data);
429 __arm_lpae_free_pgtable(data, lvl + 1, ptep); 508 __arm_lpae_free_pgtable(data, lvl + 1, ptep);
430 } else { 509 } else {
@@ -640,11 +719,12 @@ arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie)
640 cfg->arm_lpae_s1_cfg.mair[1] = 0; 719 cfg->arm_lpae_s1_cfg.mair[1] = 0;
641 720
642 /* Looking good; allocate a pgd */ 721 /* Looking good; allocate a pgd */
643 data->pgd = alloc_pages_exact(data->pgd_size, GFP_KERNEL | __GFP_ZERO); 722 data->pgd = __arm_lpae_alloc_pages(data->pgd_size, GFP_KERNEL, cfg);
644 if (!data->pgd) 723 if (!data->pgd)
645 goto out_free_data; 724 goto out_free_data;
646 725
647 cfg->tlb->flush_pgtable(data->pgd, data->pgd_size, cookie); 726 /* Ensure the empty pgd is visible before any actual TTBR write */
727 wmb();
648 728
649 /* TTBRs */ 729 /* TTBRs */
650 cfg->arm_lpae_s1_cfg.ttbr[0] = virt_to_phys(data->pgd); 730 cfg->arm_lpae_s1_cfg.ttbr[0] = virt_to_phys(data->pgd);
@@ -728,11 +808,12 @@ arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie)
728 cfg->arm_lpae_s2_cfg.vtcr = reg; 808 cfg->arm_lpae_s2_cfg.vtcr = reg;
729 809
730 /* Allocate pgd pages */ 810 /* Allocate pgd pages */
731 data->pgd = alloc_pages_exact(data->pgd_size, GFP_KERNEL | __GFP_ZERO); 811 data->pgd = __arm_lpae_alloc_pages(data->pgd_size, GFP_KERNEL, cfg);
732 if (!data->pgd) 812 if (!data->pgd)
733 goto out_free_data; 813 goto out_free_data;
734 814
735 cfg->tlb->flush_pgtable(data->pgd, data->pgd_size, cookie); 815 /* Ensure the empty pgd is visible before any actual TTBR write */
816 wmb();
736 817
737 /* VTTBR */ 818 /* VTTBR */
738 cfg->arm_lpae_s2_cfg.vttbr = virt_to_phys(data->pgd); 819 cfg->arm_lpae_s2_cfg.vttbr = virt_to_phys(data->pgd);
@@ -818,16 +899,10 @@ static void dummy_tlb_sync(void *cookie)
818 WARN_ON(cookie != cfg_cookie); 899 WARN_ON(cookie != cfg_cookie);
819} 900}
820 901
821static void dummy_flush_pgtable(void *ptr, size_t size, void *cookie)
822{
823 WARN_ON(cookie != cfg_cookie);
824}
825
826static struct iommu_gather_ops dummy_tlb_ops __initdata = { 902static struct iommu_gather_ops dummy_tlb_ops __initdata = {
827 .tlb_flush_all = dummy_tlb_flush_all, 903 .tlb_flush_all = dummy_tlb_flush_all,
828 .tlb_add_flush = dummy_tlb_add_flush, 904 .tlb_add_flush = dummy_tlb_add_flush,
829 .tlb_sync = dummy_tlb_sync, 905 .tlb_sync = dummy_tlb_sync,
830 .flush_pgtable = dummy_flush_pgtable,
831}; 906};
832 907
833static void __init arm_lpae_dump_ops(struct io_pgtable_ops *ops) 908static void __init arm_lpae_dump_ops(struct io_pgtable_ops *ops)
diff --git a/drivers/iommu/io-pgtable.c b/drivers/iommu/io-pgtable.c
index 6436fe24bc2f..6f2e319d4f04 100644
--- a/drivers/iommu/io-pgtable.c
+++ b/drivers/iommu/io-pgtable.c
@@ -24,11 +24,6 @@
24 24
25#include "io-pgtable.h" 25#include "io-pgtable.h"
26 26
27extern struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s1_init_fns;
28extern struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s2_init_fns;
29extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s1_init_fns;
30extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s2_init_fns;
31
32static const struct io_pgtable_init_fns * 27static const struct io_pgtable_init_fns *
33io_pgtable_init_table[IO_PGTABLE_NUM_FMTS] = 28io_pgtable_init_table[IO_PGTABLE_NUM_FMTS] =
34{ 29{
diff --git a/drivers/iommu/io-pgtable.h b/drivers/iommu/io-pgtable.h
index 10e32f69c668..ac9e2341a633 100644
--- a/drivers/iommu/io-pgtable.h
+++ b/drivers/iommu/io-pgtable.h
@@ -17,8 +17,9 @@ enum io_pgtable_fmt {
17 * 17 *
18 * @tlb_flush_all: Synchronously invalidate the entire TLB context. 18 * @tlb_flush_all: Synchronously invalidate the entire TLB context.
19 * @tlb_add_flush: Queue up a TLB invalidation for a virtual address range. 19 * @tlb_add_flush: Queue up a TLB invalidation for a virtual address range.
20 * @tlb_sync: Ensure any queue TLB invalidation has taken effect. 20 * @tlb_sync: Ensure any queued TLB invalidation has taken effect, and
21 * @flush_pgtable: Ensure page table updates are visible to the IOMMU. 21 * any corresponding page table updates are visible to the
22 * IOMMU.
22 * 23 *
23 * Note that these can all be called in atomic context and must therefore 24 * Note that these can all be called in atomic context and must therefore
24 * not block. 25 * not block.
@@ -28,7 +29,6 @@ struct iommu_gather_ops {
28 void (*tlb_add_flush)(unsigned long iova, size_t size, bool leaf, 29 void (*tlb_add_flush)(unsigned long iova, size_t size, bool leaf,
29 void *cookie); 30 void *cookie);
30 void (*tlb_sync)(void *cookie); 31 void (*tlb_sync)(void *cookie);
31 void (*flush_pgtable)(void *ptr, size_t size, void *cookie);
32}; 32};
33 33
34/** 34/**
@@ -41,6 +41,8 @@ struct iommu_gather_ops {
41 * @ias: Input address (iova) size, in bits. 41 * @ias: Input address (iova) size, in bits.
42 * @oas: Output address (paddr) size, in bits. 42 * @oas: Output address (paddr) size, in bits.
43 * @tlb: TLB management callbacks for this set of tables. 43 * @tlb: TLB management callbacks for this set of tables.
44 * @iommu_dev: The device representing the DMA configuration for the
45 * page table walker.
44 */ 46 */
45struct io_pgtable_cfg { 47struct io_pgtable_cfg {
46 #define IO_PGTABLE_QUIRK_ARM_NS (1 << 0) /* Set NS bit in PTEs */ 48 #define IO_PGTABLE_QUIRK_ARM_NS (1 << 0) /* Set NS bit in PTEs */
@@ -49,6 +51,7 @@ struct io_pgtable_cfg {
49 unsigned int ias; 51 unsigned int ias;
50 unsigned int oas; 52 unsigned int oas;
51 const struct iommu_gather_ops *tlb; 53 const struct iommu_gather_ops *tlb;
54 struct device *iommu_dev;
52 55
53 /* Low-level data specific to the table format */ 56 /* Low-level data specific to the table format */
54 union { 57 union {
@@ -140,4 +143,9 @@ struct io_pgtable_init_fns {
140 void (*free)(struct io_pgtable *iop); 143 void (*free)(struct io_pgtable *iop);
141}; 144};
142 145
146extern struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s1_init_fns;
147extern struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s2_init_fns;
148extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s1_init_fns;
149extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s2_init_fns;
150
143#endif /* __IO_PGTABLE_H */ 151#endif /* __IO_PGTABLE_H */
diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c
index 1a67c531a07e..8cf605fa9946 100644
--- a/drivers/iommu/ipmmu-vmsa.c
+++ b/drivers/iommu/ipmmu-vmsa.c
@@ -283,24 +283,10 @@ static void ipmmu_tlb_add_flush(unsigned long iova, size_t size, bool leaf,
283 /* The hardware doesn't support selective TLB flush. */ 283 /* The hardware doesn't support selective TLB flush. */
284} 284}
285 285
286static void ipmmu_flush_pgtable(void *ptr, size_t size, void *cookie)
287{
288 unsigned long offset = (unsigned long)ptr & ~PAGE_MASK;
289 struct ipmmu_vmsa_domain *domain = cookie;
290
291 /*
292 * TODO: Add support for coherent walk through CCI with DVM and remove
293 * cache handling.
294 */
295 dma_map_page(domain->mmu->dev, virt_to_page(ptr), offset, size,
296 DMA_TO_DEVICE);
297}
298
299static struct iommu_gather_ops ipmmu_gather_ops = { 286static struct iommu_gather_ops ipmmu_gather_ops = {
300 .tlb_flush_all = ipmmu_tlb_flush_all, 287 .tlb_flush_all = ipmmu_tlb_flush_all,
301 .tlb_add_flush = ipmmu_tlb_add_flush, 288 .tlb_add_flush = ipmmu_tlb_add_flush,
302 .tlb_sync = ipmmu_tlb_flush_all, 289 .tlb_sync = ipmmu_tlb_flush_all,
303 .flush_pgtable = ipmmu_flush_pgtable,
304}; 290};
305 291
306/* ----------------------------------------------------------------------------- 292/* -----------------------------------------------------------------------------
@@ -327,6 +313,11 @@ static int ipmmu_domain_init_context(struct ipmmu_vmsa_domain *domain)
327 domain->cfg.ias = 32; 313 domain->cfg.ias = 32;
328 domain->cfg.oas = 40; 314 domain->cfg.oas = 40;
329 domain->cfg.tlb = &ipmmu_gather_ops; 315 domain->cfg.tlb = &ipmmu_gather_ops;
316 /*
317 * TODO: Add support for coherent walk through CCI with DVM and remove
318 * cache handling. For now, delegate it to the io-pgtable code.
319 */
320 domain->cfg.iommu_dev = domain->mmu->dev;
330 321
331 domain->iop = alloc_io_pgtable_ops(ARM_32_LPAE_S1, &domain->cfg, 322 domain->iop = alloc_io_pgtable_ops(ARM_32_LPAE_S1, &domain->cfg,
332 domain); 323 domain);
diff --git a/drivers/iommu/irq_remapping.c b/drivers/iommu/irq_remapping.c
index 2d9993062ded..913455a5fd40 100644
--- a/drivers/iommu/irq_remapping.c
+++ b/drivers/iommu/irq_remapping.c
@@ -84,7 +84,7 @@ void set_irq_remapping_broken(void)
84bool irq_remapping_cap(enum irq_remap_cap cap) 84bool irq_remapping_cap(enum irq_remap_cap cap)
85{ 85{
86 if (!remap_ops || disable_irq_post) 86 if (!remap_ops || disable_irq_post)
87 return 0; 87 return false;
88 88
89 return (remap_ops->capability & (1 << cap)); 89 return (remap_ops->capability & (1 << cap));
90} 90}
diff --git a/drivers/iommu/msm_iommu.c b/drivers/iommu/msm_iommu.c
index 15a2063812fa..e321fa517a45 100644
--- a/drivers/iommu/msm_iommu.c
+++ b/drivers/iommu/msm_iommu.c
@@ -106,8 +106,8 @@ static int __flush_iotlb(struct iommu_domain *domain)
106#endif 106#endif
107 107
108 list_for_each_entry(ctx_drvdata, &priv->list_attached, attached_elm) { 108 list_for_each_entry(ctx_drvdata, &priv->list_attached, attached_elm) {
109 if (!ctx_drvdata->pdev || !ctx_drvdata->pdev->dev.parent) 109
110 BUG(); 110 BUG_ON(!ctx_drvdata->pdev || !ctx_drvdata->pdev->dev.parent);
111 111
112 iommu_drvdata = dev_get_drvdata(ctx_drvdata->pdev->dev.parent); 112 iommu_drvdata = dev_get_drvdata(ctx_drvdata->pdev->dev.parent);
113 BUG_ON(!iommu_drvdata); 113 BUG_ON(!iommu_drvdata);
diff --git a/drivers/iommu/of_iommu.c b/drivers/iommu/of_iommu.c
index 43429ab62228..60ba238090d9 100644
--- a/drivers/iommu/of_iommu.c
+++ b/drivers/iommu/of_iommu.c
@@ -141,10 +141,12 @@ struct iommu_ops *of_iommu_configure(struct device *dev,
141 struct iommu_ops *ops = NULL; 141 struct iommu_ops *ops = NULL;
142 int idx = 0; 142 int idx = 0;
143 143
144 if (dev_is_pci(dev)) { 144 /*
145 dev_err(dev, "IOMMU is currently not supported for PCI\n"); 145 * We can't do much for PCI devices without knowing how
146 * device IDs are wired up from the PCI bus to the IOMMU.
147 */
148 if (dev_is_pci(dev))
146 return NULL; 149 return NULL;
147 }
148 150
149 /* 151 /*
150 * We don't currently walk up the tree looking for a parent IOMMU. 152 * We don't currently walk up the tree looking for a parent IOMMU.
diff --git a/drivers/iommu/omap-iommu-debug.c b/drivers/iommu/omap-iommu-debug.c
index f3d20a2039d2..0717aa96ce39 100644
--- a/drivers/iommu/omap-iommu-debug.c
+++ b/drivers/iommu/omap-iommu-debug.c
@@ -14,6 +14,7 @@
14#include <linux/io.h> 14#include <linux/io.h>
15#include <linux/slab.h> 15#include <linux/slab.h>
16#include <linux/uaccess.h> 16#include <linux/uaccess.h>
17#include <linux/pm_runtime.h>
17#include <linux/debugfs.h> 18#include <linux/debugfs.h>
18#include <linux/platform_data/iommu-omap.h> 19#include <linux/platform_data/iommu-omap.h>
19 20
@@ -29,6 +30,59 @@ static inline bool is_omap_iommu_detached(struct omap_iommu *obj)
29 return !obj->domain; 30 return !obj->domain;
30} 31}
31 32
33#define pr_reg(name) \
34 do { \
35 ssize_t bytes; \
36 const char *str = "%20s: %08x\n"; \
37 const int maxcol = 32; \
38 bytes = snprintf(p, maxcol, str, __stringify(name), \
39 iommu_read_reg(obj, MMU_##name)); \
40 p += bytes; \
41 len -= bytes; \
42 if (len < maxcol) \
43 goto out; \
44 } while (0)
45
46static ssize_t
47omap2_iommu_dump_ctx(struct omap_iommu *obj, char *buf, ssize_t len)
48{
49 char *p = buf;
50
51 pr_reg(REVISION);
52 pr_reg(IRQSTATUS);
53 pr_reg(IRQENABLE);
54 pr_reg(WALKING_ST);
55 pr_reg(CNTL);
56 pr_reg(FAULT_AD);
57 pr_reg(TTB);
58 pr_reg(LOCK);
59 pr_reg(LD_TLB);
60 pr_reg(CAM);
61 pr_reg(RAM);
62 pr_reg(GFLUSH);
63 pr_reg(FLUSH_ENTRY);
64 pr_reg(READ_CAM);
65 pr_reg(READ_RAM);
66 pr_reg(EMU_FAULT_AD);
67out:
68 return p - buf;
69}
70
71static ssize_t omap_iommu_dump_ctx(struct omap_iommu *obj, char *buf,
72 ssize_t bytes)
73{
74 if (!obj || !buf)
75 return -EINVAL;
76
77 pm_runtime_get_sync(obj->dev);
78
79 bytes = omap2_iommu_dump_ctx(obj, buf, bytes);
80
81 pm_runtime_put_sync(obj->dev);
82
83 return bytes;
84}
85
32static ssize_t debug_read_regs(struct file *file, char __user *userbuf, 86static ssize_t debug_read_regs(struct file *file, char __user *userbuf,
33 size_t count, loff_t *ppos) 87 size_t count, loff_t *ppos)
34{ 88{
@@ -55,34 +109,71 @@ static ssize_t debug_read_regs(struct file *file, char __user *userbuf,
55 return bytes; 109 return bytes;
56} 110}
57 111
58static ssize_t debug_read_tlb(struct file *file, char __user *userbuf, 112static int
59 size_t count, loff_t *ppos) 113__dump_tlb_entries(struct omap_iommu *obj, struct cr_regs *crs, int num)
60{ 114{
61 struct omap_iommu *obj = file->private_data; 115 int i;
62 char *p, *buf; 116 struct iotlb_lock saved;
63 ssize_t bytes, rest; 117 struct cr_regs tmp;
118 struct cr_regs *p = crs;
119
120 pm_runtime_get_sync(obj->dev);
121 iotlb_lock_get(obj, &saved);
122
123 for_each_iotlb_cr(obj, num, i, tmp) {
124 if (!iotlb_cr_valid(&tmp))
125 continue;
126 *p++ = tmp;
127 }
128
129 iotlb_lock_set(obj, &saved);
130 pm_runtime_put_sync(obj->dev);
131
132 return p - crs;
133}
134
135static ssize_t iotlb_dump_cr(struct omap_iommu *obj, struct cr_regs *cr,
136 struct seq_file *s)
137{
138 return seq_printf(s, "%08x %08x %01x\n", cr->cam, cr->ram,
139 (cr->cam & MMU_CAM_P) ? 1 : 0);
140}
141
142static size_t omap_dump_tlb_entries(struct omap_iommu *obj, struct seq_file *s)
143{
144 int i, num;
145 struct cr_regs *cr;
146
147 num = obj->nr_tlb_entries;
148
149 cr = kcalloc(num, sizeof(*cr), GFP_KERNEL);
150 if (!cr)
151 return 0;
152
153 num = __dump_tlb_entries(obj, cr, num);
154 for (i = 0; i < num; i++)
155 iotlb_dump_cr(obj, cr + i, s);
156 kfree(cr);
157
158 return 0;
159}
160
161static int debug_read_tlb(struct seq_file *s, void *data)
162{
163 struct omap_iommu *obj = s->private;
64 164
65 if (is_omap_iommu_detached(obj)) 165 if (is_omap_iommu_detached(obj))
66 return -EPERM; 166 return -EPERM;
67 167
68 buf = kmalloc(count, GFP_KERNEL);
69 if (!buf)
70 return -ENOMEM;
71 p = buf;
72
73 mutex_lock(&iommu_debug_lock); 168 mutex_lock(&iommu_debug_lock);
74 169
75 p += sprintf(p, "%8s %8s\n", "cam:", "ram:"); 170 seq_printf(s, "%8s %8s\n", "cam:", "ram:");
76 p += sprintf(p, "-----------------------------------------\n"); 171 seq_puts(s, "-----------------------------------------\n");
77 rest = count - (p - buf); 172 omap_dump_tlb_entries(obj, s);
78 p += omap_dump_tlb_entries(obj, p, rest);
79
80 bytes = simple_read_from_buffer(userbuf, count, ppos, buf, p - buf);
81 173
82 mutex_unlock(&iommu_debug_lock); 174 mutex_unlock(&iommu_debug_lock);
83 kfree(buf);
84 175
85 return bytes; 176 return 0;
86} 177}
87 178
88static void dump_ioptable(struct seq_file *s) 179static void dump_ioptable(struct seq_file *s)
@@ -154,10 +245,10 @@ static int debug_read_pagetable(struct seq_file *s, void *data)
154 .open = simple_open, \ 245 .open = simple_open, \
155 .read = debug_read_##name, \ 246 .read = debug_read_##name, \
156 .llseek = generic_file_llseek, \ 247 .llseek = generic_file_llseek, \
157 }; 248 }
158 249
159DEBUG_FOPS_RO(regs); 250DEBUG_FOPS_RO(regs);
160DEBUG_FOPS_RO(tlb); 251DEBUG_SEQ_FOPS_RO(tlb);
161DEBUG_SEQ_FOPS_RO(pagetable); 252DEBUG_SEQ_FOPS_RO(pagetable);
162 253
163#define __DEBUG_ADD_FILE(attr, mode) \ 254#define __DEBUG_ADD_FILE(attr, mode) \
diff --git a/drivers/iommu/omap-iommu.c b/drivers/iommu/omap-iommu.c
index a22c33d6a486..36d0033c2ccb 100644
--- a/drivers/iommu/omap-iommu.c
+++ b/drivers/iommu/omap-iommu.c
@@ -12,7 +12,6 @@
12 */ 12 */
13 13
14#include <linux/err.h> 14#include <linux/err.h>
15#include <linux/module.h>
16#include <linux/slab.h> 15#include <linux/slab.h>
17#include <linux/interrupt.h> 16#include <linux/interrupt.h>
18#include <linux/ioport.h> 17#include <linux/ioport.h>
@@ -38,11 +37,6 @@
38#define to_iommu(dev) \ 37#define to_iommu(dev) \
39 ((struct omap_iommu *)platform_get_drvdata(to_platform_device(dev))) 38 ((struct omap_iommu *)platform_get_drvdata(to_platform_device(dev)))
40 39
41#define for_each_iotlb_cr(obj, n, __i, cr) \
42 for (__i = 0; \
43 (__i < (n)) && (cr = __iotlb_read_cr((obj), __i), true); \
44 __i++)
45
46/* bitmap of the page sizes currently supported */ 40/* bitmap of the page sizes currently supported */
47#define OMAP_IOMMU_PGSIZES (SZ_4K | SZ_64K | SZ_1M | SZ_16M) 41#define OMAP_IOMMU_PGSIZES (SZ_4K | SZ_64K | SZ_1M | SZ_16M)
48 42
@@ -72,11 +66,6 @@ struct omap_iommu_domain {
72#define MMU_LOCK_VICT(x) \ 66#define MMU_LOCK_VICT(x) \
73 ((x & MMU_LOCK_VICT_MASK) >> MMU_LOCK_VICT_SHIFT) 67 ((x & MMU_LOCK_VICT_MASK) >> MMU_LOCK_VICT_SHIFT)
74 68
75struct iotlb_lock {
76 short base;
77 short vict;
78};
79
80static struct platform_driver omap_iommu_driver; 69static struct platform_driver omap_iommu_driver;
81static struct kmem_cache *iopte_cachep; 70static struct kmem_cache *iopte_cachep;
82 71
@@ -213,14 +202,6 @@ static void iommu_disable(struct omap_iommu *obj)
213/* 202/*
214 * TLB operations 203 * TLB operations
215 */ 204 */
216static inline int iotlb_cr_valid(struct cr_regs *cr)
217{
218 if (!cr)
219 return -EINVAL;
220
221 return cr->cam & MMU_CAM_V;
222}
223
224static u32 iotlb_cr_to_virt(struct cr_regs *cr) 205static u32 iotlb_cr_to_virt(struct cr_regs *cr)
225{ 206{
226 u32 page_size = cr->cam & MMU_CAM_PGSZ_MASK; 207 u32 page_size = cr->cam & MMU_CAM_PGSZ_MASK;
@@ -260,7 +241,7 @@ static u32 iommu_report_fault(struct omap_iommu *obj, u32 *da)
260 return status; 241 return status;
261} 242}
262 243
263static void iotlb_lock_get(struct omap_iommu *obj, struct iotlb_lock *l) 244void iotlb_lock_get(struct omap_iommu *obj, struct iotlb_lock *l)
264{ 245{
265 u32 val; 246 u32 val;
266 247
@@ -268,10 +249,9 @@ static void iotlb_lock_get(struct omap_iommu *obj, struct iotlb_lock *l)
268 249
269 l->base = MMU_LOCK_BASE(val); 250 l->base = MMU_LOCK_BASE(val);
270 l->vict = MMU_LOCK_VICT(val); 251 l->vict = MMU_LOCK_VICT(val);
271
272} 252}
273 253
274static void iotlb_lock_set(struct omap_iommu *obj, struct iotlb_lock *l) 254void iotlb_lock_set(struct omap_iommu *obj, struct iotlb_lock *l)
275{ 255{
276 u32 val; 256 u32 val;
277 257
@@ -297,7 +277,7 @@ static void iotlb_load_cr(struct omap_iommu *obj, struct cr_regs *cr)
297} 277}
298 278
299/* only used in iotlb iteration for-loop */ 279/* only used in iotlb iteration for-loop */
300static struct cr_regs __iotlb_read_cr(struct omap_iommu *obj, int n) 280struct cr_regs __iotlb_read_cr(struct omap_iommu *obj, int n)
301{ 281{
302 struct cr_regs cr; 282 struct cr_regs cr;
303 struct iotlb_lock l; 283 struct iotlb_lock l;
@@ -468,129 +448,6 @@ static void flush_iotlb_all(struct omap_iommu *obj)
468 pm_runtime_put_sync(obj->dev); 448 pm_runtime_put_sync(obj->dev);
469} 449}
470 450
471#ifdef CONFIG_OMAP_IOMMU_DEBUG
472
473#define pr_reg(name) \
474 do { \
475 ssize_t bytes; \
476 const char *str = "%20s: %08x\n"; \
477 const int maxcol = 32; \
478 bytes = snprintf(p, maxcol, str, __stringify(name), \
479 iommu_read_reg(obj, MMU_##name)); \
480 p += bytes; \
481 len -= bytes; \
482 if (len < maxcol) \
483 goto out; \
484 } while (0)
485
486static ssize_t
487omap2_iommu_dump_ctx(struct omap_iommu *obj, char *buf, ssize_t len)
488{
489 char *p = buf;
490
491 pr_reg(REVISION);
492 pr_reg(IRQSTATUS);
493 pr_reg(IRQENABLE);
494 pr_reg(WALKING_ST);
495 pr_reg(CNTL);
496 pr_reg(FAULT_AD);
497 pr_reg(TTB);
498 pr_reg(LOCK);
499 pr_reg(LD_TLB);
500 pr_reg(CAM);
501 pr_reg(RAM);
502 pr_reg(GFLUSH);
503 pr_reg(FLUSH_ENTRY);
504 pr_reg(READ_CAM);
505 pr_reg(READ_RAM);
506 pr_reg(EMU_FAULT_AD);
507out:
508 return p - buf;
509}
510
511ssize_t omap_iommu_dump_ctx(struct omap_iommu *obj, char *buf, ssize_t bytes)
512{
513 if (!obj || !buf)
514 return -EINVAL;
515
516 pm_runtime_get_sync(obj->dev);
517
518 bytes = omap2_iommu_dump_ctx(obj, buf, bytes);
519
520 pm_runtime_put_sync(obj->dev);
521
522 return bytes;
523}
524
525static int
526__dump_tlb_entries(struct omap_iommu *obj, struct cr_regs *crs, int num)
527{
528 int i;
529 struct iotlb_lock saved;
530 struct cr_regs tmp;
531 struct cr_regs *p = crs;
532
533 pm_runtime_get_sync(obj->dev);
534 iotlb_lock_get(obj, &saved);
535
536 for_each_iotlb_cr(obj, num, i, tmp) {
537 if (!iotlb_cr_valid(&tmp))
538 continue;
539 *p++ = tmp;
540 }
541
542 iotlb_lock_set(obj, &saved);
543 pm_runtime_put_sync(obj->dev);
544
545 return p - crs;
546}
547
548/**
549 * iotlb_dump_cr - Dump an iommu tlb entry into buf
550 * @obj: target iommu
551 * @cr: contents of cam and ram register
552 * @buf: output buffer
553 **/
554static ssize_t iotlb_dump_cr(struct omap_iommu *obj, struct cr_regs *cr,
555 char *buf)
556{
557 char *p = buf;
558
559 /* FIXME: Need more detail analysis of cam/ram */
560 p += sprintf(p, "%08x %08x %01x\n", cr->cam, cr->ram,
561 (cr->cam & MMU_CAM_P) ? 1 : 0);
562
563 return p - buf;
564}
565
566/**
567 * omap_dump_tlb_entries - dump cr arrays to given buffer
568 * @obj: target iommu
569 * @buf: output buffer
570 **/
571size_t omap_dump_tlb_entries(struct omap_iommu *obj, char *buf, ssize_t bytes)
572{
573 int i, num;
574 struct cr_regs *cr;
575 char *p = buf;
576
577 num = bytes / sizeof(*cr);
578 num = min(obj->nr_tlb_entries, num);
579
580 cr = kcalloc(num, sizeof(*cr), GFP_KERNEL);
581 if (!cr)
582 return 0;
583
584 num = __dump_tlb_entries(obj, cr, num);
585 for (i = 0; i < num; i++)
586 p += iotlb_dump_cr(obj, cr + i, p);
587 kfree(cr);
588
589 return p - buf;
590}
591
592#endif /* CONFIG_OMAP_IOMMU_DEBUG */
593
594/* 451/*
595 * H/W pagetable operations 452 * H/W pagetable operations
596 */ 453 */
@@ -930,14 +787,14 @@ static irqreturn_t iommu_fault_handler(int irq, void *data)
930 787
931 if (!iopgd_is_table(*iopgd)) { 788 if (!iopgd_is_table(*iopgd)) {
932 dev_err(obj->dev, "%s: errs:0x%08x da:0x%08x pgd:0x%p *pgd:px%08x\n", 789 dev_err(obj->dev, "%s: errs:0x%08x da:0x%08x pgd:0x%p *pgd:px%08x\n",
933 obj->name, errs, da, iopgd, *iopgd); 790 obj->name, errs, da, iopgd, *iopgd);
934 return IRQ_NONE; 791 return IRQ_NONE;
935 } 792 }
936 793
937 iopte = iopte_offset(iopgd, da); 794 iopte = iopte_offset(iopgd, da);
938 795
939 dev_err(obj->dev, "%s: errs:0x%08x da:0x%08x pgd:0x%p *pgd:0x%08x pte:0x%p *pte:0x%08x\n", 796 dev_err(obj->dev, "%s: errs:0x%08x da:0x%08x pgd:0x%p *pgd:0x%08x pte:0x%p *pte:0x%08x\n",
940 obj->name, errs, da, iopgd, *iopgd, iopte, *iopte); 797 obj->name, errs, da, iopgd, *iopgd, iopte, *iopte);
941 798
942 return IRQ_NONE; 799 return IRQ_NONE;
943} 800}
@@ -963,9 +820,8 @@ static struct omap_iommu *omap_iommu_attach(const char *name, u32 *iopgd)
963 struct device *dev; 820 struct device *dev;
964 struct omap_iommu *obj; 821 struct omap_iommu *obj;
965 822
966 dev = driver_find_device(&omap_iommu_driver.driver, NULL, 823 dev = driver_find_device(&omap_iommu_driver.driver, NULL, (void *)name,
967 (void *)name, 824 device_match_by_alias);
968 device_match_by_alias);
969 if (!dev) 825 if (!dev)
970 return ERR_PTR(-ENODEV); 826 return ERR_PTR(-ENODEV);
971 827
@@ -1089,7 +945,6 @@ static const struct of_device_id omap_iommu_of_match[] = {
1089 { .compatible = "ti,dra7-iommu" }, 945 { .compatible = "ti,dra7-iommu" },
1090 {}, 946 {},
1091}; 947};
1092MODULE_DEVICE_TABLE(of, omap_iommu_of_match);
1093 948
1094static struct platform_driver omap_iommu_driver = { 949static struct platform_driver omap_iommu_driver = {
1095 .probe = omap_iommu_probe, 950 .probe = omap_iommu_probe,
@@ -1121,7 +976,7 @@ static u32 iotlb_init_entry(struct iotlb_entry *e, u32 da, u32 pa, int pgsz)
1121} 976}
1122 977
1123static int omap_iommu_map(struct iommu_domain *domain, unsigned long da, 978static int omap_iommu_map(struct iommu_domain *domain, unsigned long da,
1124 phys_addr_t pa, size_t bytes, int prot) 979 phys_addr_t pa, size_t bytes, int prot)
1125{ 980{
1126 struct omap_iommu_domain *omap_domain = to_omap_domain(domain); 981 struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
1127 struct omap_iommu *oiommu = omap_domain->iommu_dev; 982 struct omap_iommu *oiommu = omap_domain->iommu_dev;
@@ -1148,7 +1003,7 @@ static int omap_iommu_map(struct iommu_domain *domain, unsigned long da,
1148} 1003}
1149 1004
1150static size_t omap_iommu_unmap(struct iommu_domain *domain, unsigned long da, 1005static size_t omap_iommu_unmap(struct iommu_domain *domain, unsigned long da,
1151 size_t size) 1006 size_t size)
1152{ 1007{
1153 struct omap_iommu_domain *omap_domain = to_omap_domain(domain); 1008 struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
1154 struct omap_iommu *oiommu = omap_domain->iommu_dev; 1009 struct omap_iommu *oiommu = omap_domain->iommu_dev;
@@ -1199,7 +1054,7 @@ out:
1199} 1054}
1200 1055
1201static void _omap_iommu_detach_dev(struct omap_iommu_domain *omap_domain, 1056static void _omap_iommu_detach_dev(struct omap_iommu_domain *omap_domain,
1202 struct device *dev) 1057 struct device *dev)
1203{ 1058{
1204 struct omap_iommu *oiommu = dev_to_omap_iommu(dev); 1059 struct omap_iommu *oiommu = dev_to_omap_iommu(dev);
1205 struct omap_iommu_arch_data *arch_data = dev->archdata.iommu; 1060 struct omap_iommu_arch_data *arch_data = dev->archdata.iommu;
@@ -1220,7 +1075,7 @@ static void _omap_iommu_detach_dev(struct omap_iommu_domain *omap_domain,
1220} 1075}
1221 1076
1222static void omap_iommu_detach_dev(struct iommu_domain *domain, 1077static void omap_iommu_detach_dev(struct iommu_domain *domain,
1223 struct device *dev) 1078 struct device *dev)
1224{ 1079{
1225 struct omap_iommu_domain *omap_domain = to_omap_domain(domain); 1080 struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
1226 1081
@@ -1237,16 +1092,12 @@ static struct iommu_domain *omap_iommu_domain_alloc(unsigned type)
1237 return NULL; 1092 return NULL;
1238 1093
1239 omap_domain = kzalloc(sizeof(*omap_domain), GFP_KERNEL); 1094 omap_domain = kzalloc(sizeof(*omap_domain), GFP_KERNEL);
1240 if (!omap_domain) { 1095 if (!omap_domain)
1241 pr_err("kzalloc failed\n");
1242 goto out; 1096 goto out;
1243 }
1244 1097
1245 omap_domain->pgtable = kzalloc(IOPGD_TABLE_SIZE, GFP_KERNEL); 1098 omap_domain->pgtable = kzalloc(IOPGD_TABLE_SIZE, GFP_KERNEL);
1246 if (!omap_domain->pgtable) { 1099 if (!omap_domain->pgtable)
1247 pr_err("kzalloc failed\n");
1248 goto fail_nomem; 1100 goto fail_nomem;
1249 }
1250 1101
1251 /* 1102 /*
1252 * should never fail, but please keep this around to ensure 1103 * should never fail, but please keep this around to ensure
@@ -1285,7 +1136,7 @@ static void omap_iommu_domain_free(struct iommu_domain *domain)
1285} 1136}
1286 1137
1287static phys_addr_t omap_iommu_iova_to_phys(struct iommu_domain *domain, 1138static phys_addr_t omap_iommu_iova_to_phys(struct iommu_domain *domain,
1288 dma_addr_t da) 1139 dma_addr_t da)
1289{ 1140{
1290 struct omap_iommu_domain *omap_domain = to_omap_domain(domain); 1141 struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
1291 struct omap_iommu *oiommu = omap_domain->iommu_dev; 1142 struct omap_iommu *oiommu = omap_domain->iommu_dev;
@@ -1302,7 +1153,7 @@ static phys_addr_t omap_iommu_iova_to_phys(struct iommu_domain *domain,
1302 ret = omap_iommu_translate(*pte, da, IOLARGE_MASK); 1153 ret = omap_iommu_translate(*pte, da, IOLARGE_MASK);
1303 else 1154 else
1304 dev_err(dev, "bogus pte 0x%x, da 0x%llx", *pte, 1155 dev_err(dev, "bogus pte 0x%x, da 0x%llx", *pte,
1305 (unsigned long long)da); 1156 (unsigned long long)da);
1306 } else { 1157 } else {
1307 if (iopgd_is_section(*pgd)) 1158 if (iopgd_is_section(*pgd))
1308 ret = omap_iommu_translate(*pgd, da, IOSECTION_MASK); 1159 ret = omap_iommu_translate(*pgd, da, IOSECTION_MASK);
@@ -1310,7 +1161,7 @@ static phys_addr_t omap_iommu_iova_to_phys(struct iommu_domain *domain,
1310 ret = omap_iommu_translate(*pgd, da, IOSUPER_MASK); 1161 ret = omap_iommu_translate(*pgd, da, IOSUPER_MASK);
1311 else 1162 else
1312 dev_err(dev, "bogus pgd 0x%x, da 0x%llx", *pgd, 1163 dev_err(dev, "bogus pgd 0x%x, da 0x%llx", *pgd,
1313 (unsigned long long)da); 1164 (unsigned long long)da);
1314 } 1165 }
1315 1166
1316 return ret; 1167 return ret;
@@ -1405,20 +1256,5 @@ static int __init omap_iommu_init(void)
1405 1256
1406 return platform_driver_register(&omap_iommu_driver); 1257 return platform_driver_register(&omap_iommu_driver);
1407} 1258}
1408/* must be ready before omap3isp is probed */
1409subsys_initcall(omap_iommu_init); 1259subsys_initcall(omap_iommu_init);
1410 1260/* must be ready before omap3isp is probed */
1411static void __exit omap_iommu_exit(void)
1412{
1413 kmem_cache_destroy(iopte_cachep);
1414
1415 platform_driver_unregister(&omap_iommu_driver);
1416
1417 omap_iommu_debugfs_exit();
1418}
1419module_exit(omap_iommu_exit);
1420
1421MODULE_DESCRIPTION("omap iommu: tlb and pagetable primitives");
1422MODULE_ALIAS("platform:omap-iommu");
1423MODULE_AUTHOR("Hiroshi DOYU, Paul Mundt and Toshihiro Kobayashi");
1424MODULE_LICENSE("GPL v2");
diff --git a/drivers/iommu/omap-iommu.h b/drivers/iommu/omap-iommu.h
index d736630df3c8..a656df2f9e03 100644
--- a/drivers/iommu/omap-iommu.h
+++ b/drivers/iommu/omap-iommu.h
@@ -13,16 +13,18 @@
13#ifndef _OMAP_IOMMU_H 13#ifndef _OMAP_IOMMU_H
14#define _OMAP_IOMMU_H 14#define _OMAP_IOMMU_H
15 15
16#include <linux/bitops.h>
17
18#define for_each_iotlb_cr(obj, n, __i, cr) \
19 for (__i = 0; \
20 (__i < (n)) && (cr = __iotlb_read_cr((obj), __i), true); \
21 __i++)
22
16struct iotlb_entry { 23struct iotlb_entry {
17 u32 da; 24 u32 da;
18 u32 pa; 25 u32 pa;
19 u32 pgsz, prsvd, valid; 26 u32 pgsz, prsvd, valid;
20 union { 27 u32 endian, elsz, mixed;
21 u16 ap;
22 struct {
23 u32 endian, elsz, mixed;
24 };
25 };
26}; 28};
27 29
28struct omap_iommu { 30struct omap_iommu {
@@ -49,20 +51,13 @@ struct omap_iommu {
49}; 51};
50 52
51struct cr_regs { 53struct cr_regs {
52 union { 54 u32 cam;
53 struct { 55 u32 ram;
54 u16 cam_l; 56};
55 u16 cam_h; 57
56 }; 58struct iotlb_lock {
57 u32 cam; 59 short base;
58 }; 60 short vict;
59 union {
60 struct {
61 u16 ram_l;
62 u16 ram_h;
63 };
64 u32 ram;
65 };
66}; 61};
67 62
68/** 63/**
@@ -103,11 +98,11 @@ static inline struct omap_iommu *dev_to_omap_iommu(struct device *dev)
103 * MMU Register bit definitions 98 * MMU Register bit definitions
104 */ 99 */
105/* IRQSTATUS & IRQENABLE */ 100/* IRQSTATUS & IRQENABLE */
106#define MMU_IRQ_MULTIHITFAULT (1 << 4) 101#define MMU_IRQ_MULTIHITFAULT BIT(4)
107#define MMU_IRQ_TABLEWALKFAULT (1 << 3) 102#define MMU_IRQ_TABLEWALKFAULT BIT(3)
108#define MMU_IRQ_EMUMISS (1 << 2) 103#define MMU_IRQ_EMUMISS BIT(2)
109#define MMU_IRQ_TRANSLATIONFAULT (1 << 1) 104#define MMU_IRQ_TRANSLATIONFAULT BIT(1)
110#define MMU_IRQ_TLBMISS (1 << 0) 105#define MMU_IRQ_TLBMISS BIT(0)
111 106
112#define __MMU_IRQ_FAULT \ 107#define __MMU_IRQ_FAULT \
113 (MMU_IRQ_MULTIHITFAULT | MMU_IRQ_EMUMISS | MMU_IRQ_TRANSLATIONFAULT) 108 (MMU_IRQ_MULTIHITFAULT | MMU_IRQ_EMUMISS | MMU_IRQ_TRANSLATIONFAULT)
@@ -119,16 +114,16 @@ static inline struct omap_iommu *dev_to_omap_iommu(struct device *dev)
119/* MMU_CNTL */ 114/* MMU_CNTL */
120#define MMU_CNTL_SHIFT 1 115#define MMU_CNTL_SHIFT 1
121#define MMU_CNTL_MASK (7 << MMU_CNTL_SHIFT) 116#define MMU_CNTL_MASK (7 << MMU_CNTL_SHIFT)
122#define MMU_CNTL_EML_TLB (1 << 3) 117#define MMU_CNTL_EML_TLB BIT(3)
123#define MMU_CNTL_TWL_EN (1 << 2) 118#define MMU_CNTL_TWL_EN BIT(2)
124#define MMU_CNTL_MMU_EN (1 << 1) 119#define MMU_CNTL_MMU_EN BIT(1)
125 120
126/* CAM */ 121/* CAM */
127#define MMU_CAM_VATAG_SHIFT 12 122#define MMU_CAM_VATAG_SHIFT 12
128#define MMU_CAM_VATAG_MASK \ 123#define MMU_CAM_VATAG_MASK \
129 ((~0UL >> MMU_CAM_VATAG_SHIFT) << MMU_CAM_VATAG_SHIFT) 124 ((~0UL >> MMU_CAM_VATAG_SHIFT) << MMU_CAM_VATAG_SHIFT)
130#define MMU_CAM_P (1 << 3) 125#define MMU_CAM_P BIT(3)
131#define MMU_CAM_V (1 << 2) 126#define MMU_CAM_V BIT(2)
132#define MMU_CAM_PGSZ_MASK 3 127#define MMU_CAM_PGSZ_MASK 3
133#define MMU_CAM_PGSZ_1M (0 << 0) 128#define MMU_CAM_PGSZ_1M (0 << 0)
134#define MMU_CAM_PGSZ_64K (1 << 0) 129#define MMU_CAM_PGSZ_64K (1 << 0)
@@ -141,9 +136,9 @@ static inline struct omap_iommu *dev_to_omap_iommu(struct device *dev)
141 ((~0UL >> MMU_RAM_PADDR_SHIFT) << MMU_RAM_PADDR_SHIFT) 136 ((~0UL >> MMU_RAM_PADDR_SHIFT) << MMU_RAM_PADDR_SHIFT)
142 137
143#define MMU_RAM_ENDIAN_SHIFT 9 138#define MMU_RAM_ENDIAN_SHIFT 9
144#define MMU_RAM_ENDIAN_MASK (1 << MMU_RAM_ENDIAN_SHIFT) 139#define MMU_RAM_ENDIAN_MASK BIT(MMU_RAM_ENDIAN_SHIFT)
145#define MMU_RAM_ENDIAN_LITTLE (0 << MMU_RAM_ENDIAN_SHIFT) 140#define MMU_RAM_ENDIAN_LITTLE (0 << MMU_RAM_ENDIAN_SHIFT)
146#define MMU_RAM_ENDIAN_BIG (1 << MMU_RAM_ENDIAN_SHIFT) 141#define MMU_RAM_ENDIAN_BIG BIT(MMU_RAM_ENDIAN_SHIFT)
147 142
148#define MMU_RAM_ELSZ_SHIFT 7 143#define MMU_RAM_ELSZ_SHIFT 7
149#define MMU_RAM_ELSZ_MASK (3 << MMU_RAM_ELSZ_SHIFT) 144#define MMU_RAM_ELSZ_MASK (3 << MMU_RAM_ELSZ_SHIFT)
@@ -152,7 +147,7 @@ static inline struct omap_iommu *dev_to_omap_iommu(struct device *dev)
152#define MMU_RAM_ELSZ_32 (2 << MMU_RAM_ELSZ_SHIFT) 147#define MMU_RAM_ELSZ_32 (2 << MMU_RAM_ELSZ_SHIFT)
153#define MMU_RAM_ELSZ_NONE (3 << MMU_RAM_ELSZ_SHIFT) 148#define MMU_RAM_ELSZ_NONE (3 << MMU_RAM_ELSZ_SHIFT)
154#define MMU_RAM_MIXED_SHIFT 6 149#define MMU_RAM_MIXED_SHIFT 6
155#define MMU_RAM_MIXED_MASK (1 << MMU_RAM_MIXED_SHIFT) 150#define MMU_RAM_MIXED_MASK BIT(MMU_RAM_MIXED_SHIFT)
156#define MMU_RAM_MIXED MMU_RAM_MIXED_MASK 151#define MMU_RAM_MIXED MMU_RAM_MIXED_MASK
157 152
158#define MMU_GP_REG_BUS_ERR_BACK_EN 0x1 153#define MMU_GP_REG_BUS_ERR_BACK_EN 0x1
@@ -190,12 +185,12 @@ static inline struct omap_iommu *dev_to_omap_iommu(struct device *dev)
190/* 185/*
191 * global functions 186 * global functions
192 */ 187 */
193#ifdef CONFIG_OMAP_IOMMU_DEBUG
194extern ssize_t
195omap_iommu_dump_ctx(struct omap_iommu *obj, char *buf, ssize_t len);
196extern size_t
197omap_dump_tlb_entries(struct omap_iommu *obj, char *buf, ssize_t len);
198 188
189struct cr_regs __iotlb_read_cr(struct omap_iommu *obj, int n);
190void iotlb_lock_get(struct omap_iommu *obj, struct iotlb_lock *l);
191void iotlb_lock_set(struct omap_iommu *obj, struct iotlb_lock *l);
192
193#ifdef CONFIG_OMAP_IOMMU_DEBUG
199void omap_iommu_debugfs_init(void); 194void omap_iommu_debugfs_init(void);
200void omap_iommu_debugfs_exit(void); 195void omap_iommu_debugfs_exit(void);
201 196
@@ -222,4 +217,12 @@ static inline void iommu_write_reg(struct omap_iommu *obj, u32 val, size_t offs)
222 __raw_writel(val, obj->regbase + offs); 217 __raw_writel(val, obj->regbase + offs);
223} 218}
224 219
220static inline int iotlb_cr_valid(struct cr_regs *cr)
221{
222 if (!cr)
223 return -EINVAL;
224
225 return cr->cam & MMU_CAM_V;
226}
227
225#endif /* _OMAP_IOMMU_H */ 228#endif /* _OMAP_IOMMU_H */
diff --git a/drivers/iommu/omap-iopgtable.h b/drivers/iommu/omap-iopgtable.h
index f891683e3f05..01a315227bf0 100644
--- a/drivers/iommu/omap-iopgtable.h
+++ b/drivers/iommu/omap-iopgtable.h
@@ -10,25 +10,30 @@
10 * published by the Free Software Foundation. 10 * published by the Free Software Foundation.
11 */ 11 */
12 12
13#ifndef _OMAP_IOPGTABLE_H
14#define _OMAP_IOPGTABLE_H
15
16#include <linux/bitops.h>
17
13/* 18/*
14 * "L2 table" address mask and size definitions. 19 * "L2 table" address mask and size definitions.
15 */ 20 */
16#define IOPGD_SHIFT 20 21#define IOPGD_SHIFT 20
17#define IOPGD_SIZE (1UL << IOPGD_SHIFT) 22#define IOPGD_SIZE BIT(IOPGD_SHIFT)
18#define IOPGD_MASK (~(IOPGD_SIZE - 1)) 23#define IOPGD_MASK (~(IOPGD_SIZE - 1))
19 24
20/* 25/*
21 * "section" address mask and size definitions. 26 * "section" address mask and size definitions.
22 */ 27 */
23#define IOSECTION_SHIFT 20 28#define IOSECTION_SHIFT 20
24#define IOSECTION_SIZE (1UL << IOSECTION_SHIFT) 29#define IOSECTION_SIZE BIT(IOSECTION_SHIFT)
25#define IOSECTION_MASK (~(IOSECTION_SIZE - 1)) 30#define IOSECTION_MASK (~(IOSECTION_SIZE - 1))
26 31
27/* 32/*
28 * "supersection" address mask and size definitions. 33 * "supersection" address mask and size definitions.
29 */ 34 */
30#define IOSUPER_SHIFT 24 35#define IOSUPER_SHIFT 24
31#define IOSUPER_SIZE (1UL << IOSUPER_SHIFT) 36#define IOSUPER_SIZE BIT(IOSUPER_SHIFT)
32#define IOSUPER_MASK (~(IOSUPER_SIZE - 1)) 37#define IOSUPER_MASK (~(IOSUPER_SIZE - 1))
33 38
34#define PTRS_PER_IOPGD (1UL << (32 - IOPGD_SHIFT)) 39#define PTRS_PER_IOPGD (1UL << (32 - IOPGD_SHIFT))
@@ -38,14 +43,14 @@
38 * "small page" address mask and size definitions. 43 * "small page" address mask and size definitions.
39 */ 44 */
40#define IOPTE_SHIFT 12 45#define IOPTE_SHIFT 12
41#define IOPTE_SIZE (1UL << IOPTE_SHIFT) 46#define IOPTE_SIZE BIT(IOPTE_SHIFT)
42#define IOPTE_MASK (~(IOPTE_SIZE - 1)) 47#define IOPTE_MASK (~(IOPTE_SIZE - 1))
43 48
44/* 49/*
45 * "large page" address mask and size definitions. 50 * "large page" address mask and size definitions.
46 */ 51 */
47#define IOLARGE_SHIFT 16 52#define IOLARGE_SHIFT 16
48#define IOLARGE_SIZE (1UL << IOLARGE_SHIFT) 53#define IOLARGE_SIZE BIT(IOLARGE_SHIFT)
49#define IOLARGE_MASK (~(IOLARGE_SIZE - 1)) 54#define IOLARGE_MASK (~(IOLARGE_SIZE - 1))
50 55
51#define PTRS_PER_IOPTE (1UL << (IOPGD_SHIFT - IOPTE_SHIFT)) 56#define PTRS_PER_IOPTE (1UL << (IOPGD_SHIFT - IOPTE_SHIFT))
@@ -69,16 +74,16 @@ static inline phys_addr_t omap_iommu_translate(u32 d, u32 va, u32 mask)
69/* 74/*
70 * some descriptor attributes. 75 * some descriptor attributes.
71 */ 76 */
72#define IOPGD_TABLE (1 << 0) 77#define IOPGD_TABLE (1)
73#define IOPGD_SECTION (2 << 0) 78#define IOPGD_SECTION (2)
74#define IOPGD_SUPER (1 << 18 | 2 << 0) 79#define IOPGD_SUPER (BIT(18) | IOPGD_SECTION)
75 80
76#define iopgd_is_table(x) (((x) & 3) == IOPGD_TABLE) 81#define iopgd_is_table(x) (((x) & 3) == IOPGD_TABLE)
77#define iopgd_is_section(x) (((x) & (1 << 18 | 3)) == IOPGD_SECTION) 82#define iopgd_is_section(x) (((x) & (1 << 18 | 3)) == IOPGD_SECTION)
78#define iopgd_is_super(x) (((x) & (1 << 18 | 3)) == IOPGD_SUPER) 83#define iopgd_is_super(x) (((x) & (1 << 18 | 3)) == IOPGD_SUPER)
79 84
80#define IOPTE_SMALL (2 << 0) 85#define IOPTE_SMALL (2)
81#define IOPTE_LARGE (1 << 0) 86#define IOPTE_LARGE (1)
82 87
83#define iopte_is_small(x) (((x) & 2) == IOPTE_SMALL) 88#define iopte_is_small(x) (((x) & 2) == IOPTE_SMALL)
84#define iopte_is_large(x) (((x) & 3) == IOPTE_LARGE) 89#define iopte_is_large(x) (((x) & 3) == IOPTE_LARGE)
@@ -93,3 +98,5 @@ static inline phys_addr_t omap_iommu_translate(u32 d, u32 va, u32 mask)
93/* to find an entry in the second-level page table. */ 98/* to find an entry in the second-level page table. */
94#define iopte_index(da) (((da) >> IOPTE_SHIFT) & (PTRS_PER_IOPTE - 1)) 99#define iopte_index(da) (((da) >> IOPTE_SHIFT) & (PTRS_PER_IOPTE - 1))
95#define iopte_offset(iopgd, da) (iopgd_page_vaddr(iopgd) + iopte_index(da)) 100#define iopte_offset(iopgd, da) (iopgd_page_vaddr(iopgd) + iopte_index(da))
101
102#endif /* _OMAP_IOPGTABLE_H */
diff --git a/drivers/iommu/tegra-smmu.c b/drivers/iommu/tegra-smmu.c
index c1f2e521dc52..9305964250ac 100644
--- a/drivers/iommu/tegra-smmu.c
+++ b/drivers/iommu/tegra-smmu.c
@@ -27,6 +27,7 @@ struct tegra_smmu {
27 const struct tegra_smmu_soc *soc; 27 const struct tegra_smmu_soc *soc;
28 28
29 unsigned long pfn_mask; 29 unsigned long pfn_mask;
30 unsigned long tlb_mask;
30 31
31 unsigned long *asids; 32 unsigned long *asids;
32 struct mutex lock; 33 struct mutex lock;
@@ -40,8 +41,10 @@ struct tegra_smmu_as {
40 struct iommu_domain domain; 41 struct iommu_domain domain;
41 struct tegra_smmu *smmu; 42 struct tegra_smmu *smmu;
42 unsigned int use_count; 43 unsigned int use_count;
43 struct page *count; 44 u32 *count;
45 struct page **pts;
44 struct page *pd; 46 struct page *pd;
47 dma_addr_t pd_dma;
45 unsigned id; 48 unsigned id;
46 u32 attr; 49 u32 attr;
47}; 50};
@@ -68,7 +71,8 @@ static inline u32 smmu_readl(struct tegra_smmu *smmu, unsigned long offset)
68#define SMMU_TLB_CONFIG 0x14 71#define SMMU_TLB_CONFIG 0x14
69#define SMMU_TLB_CONFIG_HIT_UNDER_MISS (1 << 29) 72#define SMMU_TLB_CONFIG_HIT_UNDER_MISS (1 << 29)
70#define SMMU_TLB_CONFIG_ROUND_ROBIN_ARBITRATION (1 << 28) 73#define SMMU_TLB_CONFIG_ROUND_ROBIN_ARBITRATION (1 << 28)
71#define SMMU_TLB_CONFIG_ACTIVE_LINES(x) ((x) & 0x3f) 74#define SMMU_TLB_CONFIG_ACTIVE_LINES(smmu) \
75 ((smmu)->soc->num_tlb_lines & (smmu)->tlb_mask)
72 76
73#define SMMU_PTC_CONFIG 0x18 77#define SMMU_PTC_CONFIG 0x18
74#define SMMU_PTC_CONFIG_ENABLE (1 << 29) 78#define SMMU_PTC_CONFIG_ENABLE (1 << 29)
@@ -79,9 +83,9 @@ static inline u32 smmu_readl(struct tegra_smmu *smmu, unsigned long offset)
79#define SMMU_PTB_ASID_VALUE(x) ((x) & 0x7f) 83#define SMMU_PTB_ASID_VALUE(x) ((x) & 0x7f)
80 84
81#define SMMU_PTB_DATA 0x020 85#define SMMU_PTB_DATA 0x020
82#define SMMU_PTB_DATA_VALUE(page, attr) (page_to_phys(page) >> 12 | (attr)) 86#define SMMU_PTB_DATA_VALUE(dma, attr) ((dma) >> 12 | (attr))
83 87
84#define SMMU_MK_PDE(page, attr) (page_to_phys(page) >> SMMU_PTE_SHIFT | (attr)) 88#define SMMU_MK_PDE(dma, attr) ((dma) >> SMMU_PTE_SHIFT | (attr))
85 89
86#define SMMU_TLB_FLUSH 0x030 90#define SMMU_TLB_FLUSH 0x030
87#define SMMU_TLB_FLUSH_VA_MATCH_ALL (0 << 0) 91#define SMMU_TLB_FLUSH_VA_MATCH_ALL (0 << 0)
@@ -134,29 +138,49 @@ static inline u32 smmu_readl(struct tegra_smmu *smmu, unsigned long offset)
134#define SMMU_PTE_ATTR (SMMU_PTE_READABLE | SMMU_PTE_WRITABLE | \ 138#define SMMU_PTE_ATTR (SMMU_PTE_READABLE | SMMU_PTE_WRITABLE | \
135 SMMU_PTE_NONSECURE) 139 SMMU_PTE_NONSECURE)
136 140
137static inline void smmu_flush_ptc(struct tegra_smmu *smmu, struct page *page, 141static unsigned int iova_pd_index(unsigned long iova)
142{
143 return (iova >> SMMU_PDE_SHIFT) & (SMMU_NUM_PDE - 1);
144}
145
146static unsigned int iova_pt_index(unsigned long iova)
147{
148 return (iova >> SMMU_PTE_SHIFT) & (SMMU_NUM_PTE - 1);
149}
150
151static bool smmu_dma_addr_valid(struct tegra_smmu *smmu, dma_addr_t addr)
152{
153 addr >>= 12;
154 return (addr & smmu->pfn_mask) == addr;
155}
156
157static dma_addr_t smmu_pde_to_dma(u32 pde)
158{
159 return pde << 12;
160}
161
162static void smmu_flush_ptc_all(struct tegra_smmu *smmu)
163{
164 smmu_writel(smmu, SMMU_PTC_FLUSH_TYPE_ALL, SMMU_PTC_FLUSH);
165}
166
167static inline void smmu_flush_ptc(struct tegra_smmu *smmu, dma_addr_t dma,
138 unsigned long offset) 168 unsigned long offset)
139{ 169{
140 phys_addr_t phys = page ? page_to_phys(page) : 0;
141 u32 value; 170 u32 value;
142 171
143 if (page) { 172 offset &= ~(smmu->mc->soc->atom_size - 1);
144 offset &= ~(smmu->mc->soc->atom_size - 1);
145 173
146 if (smmu->mc->soc->num_address_bits > 32) { 174 if (smmu->mc->soc->num_address_bits > 32) {
147#ifdef CONFIG_PHYS_ADDR_T_64BIT 175#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
148 value = (phys >> 32) & SMMU_PTC_FLUSH_HI_MASK; 176 value = (dma >> 32) & SMMU_PTC_FLUSH_HI_MASK;
149#else 177#else
150 value = 0; 178 value = 0;
151#endif 179#endif
152 smmu_writel(smmu, value, SMMU_PTC_FLUSH_HI); 180 smmu_writel(smmu, value, SMMU_PTC_FLUSH_HI);
153 }
154
155 value = (phys + offset) | SMMU_PTC_FLUSH_TYPE_ADR;
156 } else {
157 value = SMMU_PTC_FLUSH_TYPE_ALL;
158 } 181 }
159 182
183 value = (dma + offset) | SMMU_PTC_FLUSH_TYPE_ADR;
160 smmu_writel(smmu, value, SMMU_PTC_FLUSH); 184 smmu_writel(smmu, value, SMMU_PTC_FLUSH);
161} 185}
162 186
@@ -236,8 +260,6 @@ static bool tegra_smmu_capable(enum iommu_cap cap)
236static struct iommu_domain *tegra_smmu_domain_alloc(unsigned type) 260static struct iommu_domain *tegra_smmu_domain_alloc(unsigned type)
237{ 261{
238 struct tegra_smmu_as *as; 262 struct tegra_smmu_as *as;
239 unsigned int i;
240 uint32_t *pd;
241 263
242 if (type != IOMMU_DOMAIN_UNMANAGED) 264 if (type != IOMMU_DOMAIN_UNMANAGED)
243 return NULL; 265 return NULL;
@@ -248,32 +270,26 @@ static struct iommu_domain *tegra_smmu_domain_alloc(unsigned type)
248 270
249 as->attr = SMMU_PD_READABLE | SMMU_PD_WRITABLE | SMMU_PD_NONSECURE; 271 as->attr = SMMU_PD_READABLE | SMMU_PD_WRITABLE | SMMU_PD_NONSECURE;
250 272
251 as->pd = alloc_page(GFP_KERNEL | __GFP_DMA); 273 as->pd = alloc_page(GFP_KERNEL | __GFP_DMA | __GFP_ZERO);
252 if (!as->pd) { 274 if (!as->pd) {
253 kfree(as); 275 kfree(as);
254 return NULL; 276 return NULL;
255 } 277 }
256 278
257 as->count = alloc_page(GFP_KERNEL); 279 as->count = kcalloc(SMMU_NUM_PDE, sizeof(u32), GFP_KERNEL);
258 if (!as->count) { 280 if (!as->count) {
259 __free_page(as->pd); 281 __free_page(as->pd);
260 kfree(as); 282 kfree(as);
261 return NULL; 283 return NULL;
262 } 284 }
263 285
264 /* clear PDEs */ 286 as->pts = kcalloc(SMMU_NUM_PDE, sizeof(*as->pts), GFP_KERNEL);
265 pd = page_address(as->pd); 287 if (!as->pts) {
266 SetPageReserved(as->pd); 288 kfree(as->count);
267 289 __free_page(as->pd);
268 for (i = 0; i < SMMU_NUM_PDE; i++) 290 kfree(as);
269 pd[i] = 0; 291 return NULL;
270 292 }
271 /* clear PDE usage counters */
272 pd = page_address(as->count);
273 SetPageReserved(as->count);
274
275 for (i = 0; i < SMMU_NUM_PDE; i++)
276 pd[i] = 0;
277 293
278 /* setup aperture */ 294 /* setup aperture */
279 as->domain.geometry.aperture_start = 0; 295 as->domain.geometry.aperture_start = 0;
@@ -288,7 +304,6 @@ static void tegra_smmu_domain_free(struct iommu_domain *domain)
288 struct tegra_smmu_as *as = to_smmu_as(domain); 304 struct tegra_smmu_as *as = to_smmu_as(domain);
289 305
290 /* TODO: free page directory and page tables */ 306 /* TODO: free page directory and page tables */
291 ClearPageReserved(as->pd);
292 307
293 kfree(as); 308 kfree(as);
294} 309}
@@ -376,16 +391,26 @@ static int tegra_smmu_as_prepare(struct tegra_smmu *smmu,
376 return 0; 391 return 0;
377 } 392 }
378 393
394 as->pd_dma = dma_map_page(smmu->dev, as->pd, 0, SMMU_SIZE_PD,
395 DMA_TO_DEVICE);
396 if (dma_mapping_error(smmu->dev, as->pd_dma))
397 return -ENOMEM;
398
399 /* We can't handle 64-bit DMA addresses */
400 if (!smmu_dma_addr_valid(smmu, as->pd_dma)) {
401 err = -ENOMEM;
402 goto err_unmap;
403 }
404
379 err = tegra_smmu_alloc_asid(smmu, &as->id); 405 err = tegra_smmu_alloc_asid(smmu, &as->id);
380 if (err < 0) 406 if (err < 0)
381 return err; 407 goto err_unmap;
382 408
383 smmu->soc->ops->flush_dcache(as->pd, 0, SMMU_SIZE_PD); 409 smmu_flush_ptc(smmu, as->pd_dma, 0);
384 smmu_flush_ptc(smmu, as->pd, 0);
385 smmu_flush_tlb_asid(smmu, as->id); 410 smmu_flush_tlb_asid(smmu, as->id);
386 411
387 smmu_writel(smmu, as->id & 0x7f, SMMU_PTB_ASID); 412 smmu_writel(smmu, as->id & 0x7f, SMMU_PTB_ASID);
388 value = SMMU_PTB_DATA_VALUE(as->pd, as->attr); 413 value = SMMU_PTB_DATA_VALUE(as->pd_dma, as->attr);
389 smmu_writel(smmu, value, SMMU_PTB_DATA); 414 smmu_writel(smmu, value, SMMU_PTB_DATA);
390 smmu_flush(smmu); 415 smmu_flush(smmu);
391 416
@@ -393,6 +418,10 @@ static int tegra_smmu_as_prepare(struct tegra_smmu *smmu,
393 as->use_count++; 418 as->use_count++;
394 419
395 return 0; 420 return 0;
421
422err_unmap:
423 dma_unmap_page(smmu->dev, as->pd_dma, SMMU_SIZE_PD, DMA_TO_DEVICE);
424 return err;
396} 425}
397 426
398static void tegra_smmu_as_unprepare(struct tegra_smmu *smmu, 427static void tegra_smmu_as_unprepare(struct tegra_smmu *smmu,
@@ -402,6 +431,9 @@ static void tegra_smmu_as_unprepare(struct tegra_smmu *smmu,
402 return; 431 return;
403 432
404 tegra_smmu_free_asid(smmu, as->id); 433 tegra_smmu_free_asid(smmu, as->id);
434
435 dma_unmap_page(smmu->dev, as->pd_dma, SMMU_SIZE_PD, DMA_TO_DEVICE);
436
405 as->smmu = NULL; 437 as->smmu = NULL;
406} 438}
407 439
@@ -465,96 +497,155 @@ static void tegra_smmu_detach_dev(struct iommu_domain *domain, struct device *de
465 } 497 }
466} 498}
467 499
500static void tegra_smmu_set_pde(struct tegra_smmu_as *as, unsigned long iova,
501 u32 value)
502{
503 unsigned int pd_index = iova_pd_index(iova);
504 struct tegra_smmu *smmu = as->smmu;
505 u32 *pd = page_address(as->pd);
506 unsigned long offset = pd_index * sizeof(*pd);
507
508 /* Set the page directory entry first */
509 pd[pd_index] = value;
510
511 /* The flush the page directory entry from caches */
512 dma_sync_single_range_for_device(smmu->dev, as->pd_dma, offset,
513 sizeof(*pd), DMA_TO_DEVICE);
514
515 /* And flush the iommu */
516 smmu_flush_ptc(smmu, as->pd_dma, offset);
517 smmu_flush_tlb_section(smmu, as->id, iova);
518 smmu_flush(smmu);
519}
520
521static u32 *tegra_smmu_pte_offset(struct page *pt_page, unsigned long iova)
522{
523 u32 *pt = page_address(pt_page);
524
525 return pt + iova_pt_index(iova);
526}
527
528static u32 *tegra_smmu_pte_lookup(struct tegra_smmu_as *as, unsigned long iova,
529 dma_addr_t *dmap)
530{
531 unsigned int pd_index = iova_pd_index(iova);
532 struct page *pt_page;
533 u32 *pd;
534
535 pt_page = as->pts[pd_index];
536 if (!pt_page)
537 return NULL;
538
539 pd = page_address(as->pd);
540 *dmap = smmu_pde_to_dma(pd[pd_index]);
541
542 return tegra_smmu_pte_offset(pt_page, iova);
543}
544
468static u32 *as_get_pte(struct tegra_smmu_as *as, dma_addr_t iova, 545static u32 *as_get_pte(struct tegra_smmu_as *as, dma_addr_t iova,
469 struct page **pagep) 546 dma_addr_t *dmap)
470{ 547{
471 u32 *pd = page_address(as->pd), *pt, *count; 548 unsigned int pde = iova_pd_index(iova);
472 u32 pde = (iova >> SMMU_PDE_SHIFT) & 0x3ff;
473 u32 pte = (iova >> SMMU_PTE_SHIFT) & 0x3ff;
474 struct tegra_smmu *smmu = as->smmu; 549 struct tegra_smmu *smmu = as->smmu;
475 struct page *page;
476 unsigned int i;
477 550
478 if (pd[pde] == 0) { 551 if (!as->pts[pde]) {
479 page = alloc_page(GFP_KERNEL | __GFP_DMA); 552 struct page *page;
553 dma_addr_t dma;
554
555 page = alloc_page(GFP_KERNEL | __GFP_DMA | __GFP_ZERO);
480 if (!page) 556 if (!page)
481 return NULL; 557 return NULL;
482 558
483 pt = page_address(page); 559 dma = dma_map_page(smmu->dev, page, 0, SMMU_SIZE_PT,
484 SetPageReserved(page); 560 DMA_TO_DEVICE);
561 if (dma_mapping_error(smmu->dev, dma)) {
562 __free_page(page);
563 return NULL;
564 }
485 565
486 for (i = 0; i < SMMU_NUM_PTE; i++) 566 if (!smmu_dma_addr_valid(smmu, dma)) {
487 pt[i] = 0; 567 dma_unmap_page(smmu->dev, dma, SMMU_SIZE_PT,
568 DMA_TO_DEVICE);
569 __free_page(page);
570 return NULL;
571 }
488 572
489 smmu->soc->ops->flush_dcache(page, 0, SMMU_SIZE_PT); 573 as->pts[pde] = page;
490 574
491 pd[pde] = SMMU_MK_PDE(page, SMMU_PDE_ATTR | SMMU_PDE_NEXT); 575 tegra_smmu_set_pde(as, iova, SMMU_MK_PDE(dma, SMMU_PDE_ATTR |
576 SMMU_PDE_NEXT));
492 577
493 smmu->soc->ops->flush_dcache(as->pd, pde << 2, 4); 578 *dmap = dma;
494 smmu_flush_ptc(smmu, as->pd, pde << 2);
495 smmu_flush_tlb_section(smmu, as->id, iova);
496 smmu_flush(smmu);
497 } else { 579 } else {
498 page = pfn_to_page(pd[pde] & smmu->pfn_mask); 580 u32 *pd = page_address(as->pd);
499 pt = page_address(page); 581
582 *dmap = smmu_pde_to_dma(pd[pde]);
500 } 583 }
501 584
502 *pagep = page; 585 return tegra_smmu_pte_offset(as->pts[pde], iova);
586}
503 587
504 /* Keep track of entries in this page table. */ 588static void tegra_smmu_pte_get_use(struct tegra_smmu_as *as, unsigned long iova)
505 count = page_address(as->count); 589{
506 if (pt[pte] == 0) 590 unsigned int pd_index = iova_pd_index(iova);
507 count[pde]++;
508 591
509 return &pt[pte]; 592 as->count[pd_index]++;
510} 593}
511 594
512static void as_put_pte(struct tegra_smmu_as *as, dma_addr_t iova) 595static void tegra_smmu_pte_put_use(struct tegra_smmu_as *as, unsigned long iova)
513{ 596{
514 u32 pde = (iova >> SMMU_PDE_SHIFT) & 0x3ff; 597 unsigned int pde = iova_pd_index(iova);
515 u32 pte = (iova >> SMMU_PTE_SHIFT) & 0x3ff; 598 struct page *page = as->pts[pde];
516 u32 *count = page_address(as->count);
517 u32 *pd = page_address(as->pd), *pt;
518 struct page *page;
519
520 page = pfn_to_page(pd[pde] & as->smmu->pfn_mask);
521 pt = page_address(page);
522 599
523 /* 600 /*
524 * When no entries in this page table are used anymore, return the 601 * When no entries in this page table are used anymore, return the
525 * memory page to the system. 602 * memory page to the system.
526 */ 603 */
527 if (pt[pte] != 0) { 604 if (--as->count[pde] == 0) {
528 if (--count[pde] == 0) { 605 struct tegra_smmu *smmu = as->smmu;
529 ClearPageReserved(page); 606 u32 *pd = page_address(as->pd);
530 __free_page(page); 607 dma_addr_t pte_dma = smmu_pde_to_dma(pd[pde]);
531 pd[pde] = 0; 608
532 } 609 tegra_smmu_set_pde(as, iova, 0);
533 610
534 pt[pte] = 0; 611 dma_unmap_page(smmu->dev, pte_dma, SMMU_SIZE_PT, DMA_TO_DEVICE);
612 __free_page(page);
613 as->pts[pde] = NULL;
535 } 614 }
536} 615}
537 616
617static void tegra_smmu_set_pte(struct tegra_smmu_as *as, unsigned long iova,
618 u32 *pte, dma_addr_t pte_dma, u32 val)
619{
620 struct tegra_smmu *smmu = as->smmu;
621 unsigned long offset = offset_in_page(pte);
622
623 *pte = val;
624
625 dma_sync_single_range_for_device(smmu->dev, pte_dma, offset,
626 4, DMA_TO_DEVICE);
627 smmu_flush_ptc(smmu, pte_dma, offset);
628 smmu_flush_tlb_group(smmu, as->id, iova);
629 smmu_flush(smmu);
630}
631
538static int tegra_smmu_map(struct iommu_domain *domain, unsigned long iova, 632static int tegra_smmu_map(struct iommu_domain *domain, unsigned long iova,
539 phys_addr_t paddr, size_t size, int prot) 633 phys_addr_t paddr, size_t size, int prot)
540{ 634{
541 struct tegra_smmu_as *as = to_smmu_as(domain); 635 struct tegra_smmu_as *as = to_smmu_as(domain);
542 struct tegra_smmu *smmu = as->smmu; 636 dma_addr_t pte_dma;
543 unsigned long offset;
544 struct page *page;
545 u32 *pte; 637 u32 *pte;
546 638
547 pte = as_get_pte(as, iova, &page); 639 pte = as_get_pte(as, iova, &pte_dma);
548 if (!pte) 640 if (!pte)
549 return -ENOMEM; 641 return -ENOMEM;
550 642
551 *pte = __phys_to_pfn(paddr) | SMMU_PTE_ATTR; 643 /* If we aren't overwriting a pre-existing entry, increment use */
552 offset = offset_in_page(pte); 644 if (*pte == 0)
645 tegra_smmu_pte_get_use(as, iova);
553 646
554 smmu->soc->ops->flush_dcache(page, offset, 4); 647 tegra_smmu_set_pte(as, iova, pte, pte_dma,
555 smmu_flush_ptc(smmu, page, offset); 648 __phys_to_pfn(paddr) | SMMU_PTE_ATTR);
556 smmu_flush_tlb_group(smmu, as->id, iova);
557 smmu_flush(smmu);
558 649
559 return 0; 650 return 0;
560} 651}
@@ -563,22 +654,15 @@ static size_t tegra_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
563 size_t size) 654 size_t size)
564{ 655{
565 struct tegra_smmu_as *as = to_smmu_as(domain); 656 struct tegra_smmu_as *as = to_smmu_as(domain);
566 struct tegra_smmu *smmu = as->smmu; 657 dma_addr_t pte_dma;
567 unsigned long offset;
568 struct page *page;
569 u32 *pte; 658 u32 *pte;
570 659
571 pte = as_get_pte(as, iova, &page); 660 pte = tegra_smmu_pte_lookup(as, iova, &pte_dma);
572 if (!pte) 661 if (!pte || !*pte)
573 return 0; 662 return 0;
574 663
575 offset = offset_in_page(pte); 664 tegra_smmu_set_pte(as, iova, pte, pte_dma, 0);
576 as_put_pte(as, iova); 665 tegra_smmu_pte_put_use(as, iova);
577
578 smmu->soc->ops->flush_dcache(page, offset, 4);
579 smmu_flush_ptc(smmu, page, offset);
580 smmu_flush_tlb_group(smmu, as->id, iova);
581 smmu_flush(smmu);
582 666
583 return size; 667 return size;
584} 668}
@@ -587,11 +671,14 @@ static phys_addr_t tegra_smmu_iova_to_phys(struct iommu_domain *domain,
587 dma_addr_t iova) 671 dma_addr_t iova)
588{ 672{
589 struct tegra_smmu_as *as = to_smmu_as(domain); 673 struct tegra_smmu_as *as = to_smmu_as(domain);
590 struct page *page;
591 unsigned long pfn; 674 unsigned long pfn;
675 dma_addr_t pte_dma;
592 u32 *pte; 676 u32 *pte;
593 677
594 pte = as_get_pte(as, iova, &page); 678 pte = tegra_smmu_pte_lookup(as, iova, &pte_dma);
679 if (!pte || !*pte)
680 return 0;
681
595 pfn = *pte & as->smmu->pfn_mask; 682 pfn = *pte & as->smmu->pfn_mask;
596 683
597 return PFN_PHYS(pfn); 684 return PFN_PHYS(pfn);
@@ -816,6 +903,9 @@ struct tegra_smmu *tegra_smmu_probe(struct device *dev,
816 smmu->pfn_mask = BIT_MASK(mc->soc->num_address_bits - PAGE_SHIFT) - 1; 903 smmu->pfn_mask = BIT_MASK(mc->soc->num_address_bits - PAGE_SHIFT) - 1;
817 dev_dbg(dev, "address bits: %u, PFN mask: %#lx\n", 904 dev_dbg(dev, "address bits: %u, PFN mask: %#lx\n",
818 mc->soc->num_address_bits, smmu->pfn_mask); 905 mc->soc->num_address_bits, smmu->pfn_mask);
906 smmu->tlb_mask = (smmu->soc->num_tlb_lines << 1) - 1;
907 dev_dbg(dev, "TLB lines: %u, mask: %#lx\n", smmu->soc->num_tlb_lines,
908 smmu->tlb_mask);
819 909
820 value = SMMU_PTC_CONFIG_ENABLE | SMMU_PTC_CONFIG_INDEX_MAP(0x3f); 910 value = SMMU_PTC_CONFIG_ENABLE | SMMU_PTC_CONFIG_INDEX_MAP(0x3f);
821 911
@@ -825,14 +915,14 @@ struct tegra_smmu *tegra_smmu_probe(struct device *dev,
825 smmu_writel(smmu, value, SMMU_PTC_CONFIG); 915 smmu_writel(smmu, value, SMMU_PTC_CONFIG);
826 916
827 value = SMMU_TLB_CONFIG_HIT_UNDER_MISS | 917 value = SMMU_TLB_CONFIG_HIT_UNDER_MISS |
828 SMMU_TLB_CONFIG_ACTIVE_LINES(0x20); 918 SMMU_TLB_CONFIG_ACTIVE_LINES(smmu);
829 919
830 if (soc->supports_round_robin_arbitration) 920 if (soc->supports_round_robin_arbitration)
831 value |= SMMU_TLB_CONFIG_ROUND_ROBIN_ARBITRATION; 921 value |= SMMU_TLB_CONFIG_ROUND_ROBIN_ARBITRATION;
832 922
833 smmu_writel(smmu, value, SMMU_TLB_CONFIG); 923 smmu_writel(smmu, value, SMMU_TLB_CONFIG);
834 924
835 smmu_flush_ptc(smmu, NULL, 0); 925 smmu_flush_ptc_all(smmu);
836 smmu_flush_tlb(smmu); 926 smmu_flush_tlb(smmu);
837 smmu_writel(smmu, SMMU_CONFIG_ENABLE, SMMU_CONFIG); 927 smmu_writel(smmu, SMMU_CONFIG_ENABLE, SMMU_CONFIG);
838 smmu_flush(smmu); 928 smmu_flush(smmu);
diff --git a/drivers/memory/tegra/tegra114.c b/drivers/memory/tegra/tegra114.c
index c8765db07a62..ba8fff3d66a6 100644
--- a/drivers/memory/tegra/tegra114.c
+++ b/drivers/memory/tegra/tegra114.c
@@ -9,8 +9,6 @@
9#include <linux/of.h> 9#include <linux/of.h>
10#include <linux/mm.h> 10#include <linux/mm.h>
11 11
12#include <asm/cacheflush.h>
13
14#include <dt-bindings/memory/tegra114-mc.h> 12#include <dt-bindings/memory/tegra114-mc.h>
15 13
16#include "mc.h" 14#include "mc.h"
@@ -914,20 +912,6 @@ static const struct tegra_smmu_swgroup tegra114_swgroups[] = {
914 { .name = "tsec", .swgroup = TEGRA_SWGROUP_TSEC, .reg = 0x294 }, 912 { .name = "tsec", .swgroup = TEGRA_SWGROUP_TSEC, .reg = 0x294 },
915}; 913};
916 914
917static void tegra114_flush_dcache(struct page *page, unsigned long offset,
918 size_t size)
919{
920 phys_addr_t phys = page_to_phys(page) + offset;
921 void *virt = page_address(page) + offset;
922
923 __cpuc_flush_dcache_area(virt, size);
924 outer_flush_range(phys, phys + size);
925}
926
927static const struct tegra_smmu_ops tegra114_smmu_ops = {
928 .flush_dcache = tegra114_flush_dcache,
929};
930
931static const struct tegra_smmu_soc tegra114_smmu_soc = { 915static const struct tegra_smmu_soc tegra114_smmu_soc = {
932 .clients = tegra114_mc_clients, 916 .clients = tegra114_mc_clients,
933 .num_clients = ARRAY_SIZE(tegra114_mc_clients), 917 .num_clients = ARRAY_SIZE(tegra114_mc_clients),
@@ -935,8 +919,8 @@ static const struct tegra_smmu_soc tegra114_smmu_soc = {
935 .num_swgroups = ARRAY_SIZE(tegra114_swgroups), 919 .num_swgroups = ARRAY_SIZE(tegra114_swgroups),
936 .supports_round_robin_arbitration = false, 920 .supports_round_robin_arbitration = false,
937 .supports_request_limit = false, 921 .supports_request_limit = false,
922 .num_tlb_lines = 32,
938 .num_asids = 4, 923 .num_asids = 4,
939 .ops = &tegra114_smmu_ops,
940}; 924};
941 925
942const struct tegra_mc_soc tegra114_mc_soc = { 926const struct tegra_mc_soc tegra114_mc_soc = {
diff --git a/drivers/memory/tegra/tegra124.c b/drivers/memory/tegra/tegra124.c
index 060fb3d7a23f..21e7255e3d96 100644
--- a/drivers/memory/tegra/tegra124.c
+++ b/drivers/memory/tegra/tegra124.c
@@ -9,8 +9,6 @@
9#include <linux/of.h> 9#include <linux/of.h>
10#include <linux/mm.h> 10#include <linux/mm.h>
11 11
12#include <asm/cacheflush.h>
13
14#include <dt-bindings/memory/tegra124-mc.h> 12#include <dt-bindings/memory/tegra124-mc.h>
15 13
16#include "mc.h" 14#include "mc.h"
@@ -1002,20 +1000,6 @@ static const struct tegra_smmu_swgroup tegra124_swgroups[] = {
1002}; 1000};
1003 1001
1004#ifdef CONFIG_ARCH_TEGRA_124_SOC 1002#ifdef CONFIG_ARCH_TEGRA_124_SOC
1005static void tegra124_flush_dcache(struct page *page, unsigned long offset,
1006 size_t size)
1007{
1008 phys_addr_t phys = page_to_phys(page) + offset;
1009 void *virt = page_address(page) + offset;
1010
1011 __cpuc_flush_dcache_area(virt, size);
1012 outer_flush_range(phys, phys + size);
1013}
1014
1015static const struct tegra_smmu_ops tegra124_smmu_ops = {
1016 .flush_dcache = tegra124_flush_dcache,
1017};
1018
1019static const struct tegra_smmu_soc tegra124_smmu_soc = { 1003static const struct tegra_smmu_soc tegra124_smmu_soc = {
1020 .clients = tegra124_mc_clients, 1004 .clients = tegra124_mc_clients,
1021 .num_clients = ARRAY_SIZE(tegra124_mc_clients), 1005 .num_clients = ARRAY_SIZE(tegra124_mc_clients),
@@ -1024,7 +1008,6 @@ static const struct tegra_smmu_soc tegra124_smmu_soc = {
1024 .supports_round_robin_arbitration = true, 1008 .supports_round_robin_arbitration = true,
1025 .supports_request_limit = true, 1009 .supports_request_limit = true,
1026 .num_asids = 128, 1010 .num_asids = 128,
1027 .ops = &tegra124_smmu_ops,
1028}; 1011};
1029 1012
1030const struct tegra_mc_soc tegra124_mc_soc = { 1013const struct tegra_mc_soc tegra124_mc_soc = {
@@ -1040,18 +1023,6 @@ const struct tegra_mc_soc tegra124_mc_soc = {
1040#endif /* CONFIG_ARCH_TEGRA_124_SOC */ 1023#endif /* CONFIG_ARCH_TEGRA_124_SOC */
1041 1024
1042#ifdef CONFIG_ARCH_TEGRA_132_SOC 1025#ifdef CONFIG_ARCH_TEGRA_132_SOC
1043static void tegra132_flush_dcache(struct page *page, unsigned long offset,
1044 size_t size)
1045{
1046 void *virt = page_address(page) + offset;
1047
1048 __flush_dcache_area(virt, size);
1049}
1050
1051static const struct tegra_smmu_ops tegra132_smmu_ops = {
1052 .flush_dcache = tegra132_flush_dcache,
1053};
1054
1055static const struct tegra_smmu_soc tegra132_smmu_soc = { 1026static const struct tegra_smmu_soc tegra132_smmu_soc = {
1056 .clients = tegra124_mc_clients, 1027 .clients = tegra124_mc_clients,
1057 .num_clients = ARRAY_SIZE(tegra124_mc_clients), 1028 .num_clients = ARRAY_SIZE(tegra124_mc_clients),
@@ -1059,8 +1030,8 @@ static const struct tegra_smmu_soc tegra132_smmu_soc = {
1059 .num_swgroups = ARRAY_SIZE(tegra124_swgroups), 1030 .num_swgroups = ARRAY_SIZE(tegra124_swgroups),
1060 .supports_round_robin_arbitration = true, 1031 .supports_round_robin_arbitration = true,
1061 .supports_request_limit = true, 1032 .supports_request_limit = true,
1033 .num_tlb_lines = 32,
1062 .num_asids = 128, 1034 .num_asids = 128,
1063 .ops = &tegra132_smmu_ops,
1064}; 1035};
1065 1036
1066const struct tegra_mc_soc tegra132_mc_soc = { 1037const struct tegra_mc_soc tegra132_mc_soc = {
diff --git a/drivers/memory/tegra/tegra30.c b/drivers/memory/tegra/tegra30.c
index 52e16c7b34f8..b44737840e70 100644
--- a/drivers/memory/tegra/tegra30.c
+++ b/drivers/memory/tegra/tegra30.c
@@ -9,8 +9,6 @@
9#include <linux/of.h> 9#include <linux/of.h>
10#include <linux/mm.h> 10#include <linux/mm.h>
11 11
12#include <asm/cacheflush.h>
13
14#include <dt-bindings/memory/tegra30-mc.h> 12#include <dt-bindings/memory/tegra30-mc.h>
15 13
16#include "mc.h" 14#include "mc.h"
@@ -936,20 +934,6 @@ static const struct tegra_smmu_swgroup tegra30_swgroups[] = {
936 { .name = "isp", .swgroup = TEGRA_SWGROUP_ISP, .reg = 0x258 }, 934 { .name = "isp", .swgroup = TEGRA_SWGROUP_ISP, .reg = 0x258 },
937}; 935};
938 936
939static void tegra30_flush_dcache(struct page *page, unsigned long offset,
940 size_t size)
941{
942 phys_addr_t phys = page_to_phys(page) + offset;
943 void *virt = page_address(page) + offset;
944
945 __cpuc_flush_dcache_area(virt, size);
946 outer_flush_range(phys, phys + size);
947}
948
949static const struct tegra_smmu_ops tegra30_smmu_ops = {
950 .flush_dcache = tegra30_flush_dcache,
951};
952
953static const struct tegra_smmu_soc tegra30_smmu_soc = { 937static const struct tegra_smmu_soc tegra30_smmu_soc = {
954 .clients = tegra30_mc_clients, 938 .clients = tegra30_mc_clients,
955 .num_clients = ARRAY_SIZE(tegra30_mc_clients), 939 .num_clients = ARRAY_SIZE(tegra30_mc_clients),
@@ -957,8 +941,8 @@ static const struct tegra_smmu_soc tegra30_smmu_soc = {
957 .num_swgroups = ARRAY_SIZE(tegra30_swgroups), 941 .num_swgroups = ARRAY_SIZE(tegra30_swgroups),
958 .supports_round_robin_arbitration = false, 942 .supports_round_robin_arbitration = false,
959 .supports_request_limit = false, 943 .supports_request_limit = false,
944 .num_tlb_lines = 16,
960 .num_asids = 4, 945 .num_asids = 4,
961 .ops = &tegra30_smmu_ops,
962}; 946};
963 947
964const struct tegra_mc_soc tegra30_mc_soc = { 948const struct tegra_mc_soc tegra30_mc_soc = {
diff --git a/include/linux/intel-iommu.h b/include/linux/intel-iommu.h
index d9a366d24e3b..6240063bdcac 100644
--- a/include/linux/intel-iommu.h
+++ b/include/linux/intel-iommu.h
@@ -344,7 +344,7 @@ struct intel_iommu {
344 344
345#ifdef CONFIG_INTEL_IOMMU 345#ifdef CONFIG_INTEL_IOMMU
346 unsigned long *domain_ids; /* bitmap of domains */ 346 unsigned long *domain_ids; /* bitmap of domains */
347 struct dmar_domain **domains; /* ptr to domains */ 347 struct dmar_domain ***domains; /* ptr to domains */
348 spinlock_t lock; /* protect context, domain ids */ 348 spinlock_t lock; /* protect context, domain ids */
349 struct root_entry *root_entry; /* virtual address */ 349 struct root_entry *root_entry; /* virtual address */
350 350
diff --git a/include/soc/tegra/mc.h b/include/soc/tegra/mc.h
index 370f2909ec19..44202ff897fd 100644
--- a/include/soc/tegra/mc.h
+++ b/include/soc/tegra/mc.h
@@ -51,11 +51,6 @@ struct tegra_smmu_swgroup {
51 unsigned int reg; 51 unsigned int reg;
52}; 52};
53 53
54struct tegra_smmu_ops {
55 void (*flush_dcache)(struct page *page, unsigned long offset,
56 size_t size);
57};
58
59struct tegra_smmu_soc { 54struct tegra_smmu_soc {
60 const struct tegra_mc_client *clients; 55 const struct tegra_mc_client *clients;
61 unsigned int num_clients; 56 unsigned int num_clients;
@@ -66,9 +61,8 @@ struct tegra_smmu_soc {
66 bool supports_round_robin_arbitration; 61 bool supports_round_robin_arbitration;
67 bool supports_request_limit; 62 bool supports_request_limit;
68 63
64 unsigned int num_tlb_lines;
69 unsigned int num_asids; 65 unsigned int num_asids;
70
71 const struct tegra_smmu_ops *ops;
72}; 66};
73 67
74struct tegra_mc; 68struct tegra_mc;
generated by cgit v1.2.2 (git 2.25.0) at 2025-04-11 07:00:15 -0400