aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/iommu
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/iommu')
-rw-r--r--drivers/iommu/Kconfig18
-rw-r--r--drivers/iommu/Makefile1
-rw-r--r--drivers/iommu/amd_iommu.c697
-rw-r--r--drivers/iommu/amd_iommu_init.c40
-rw-r--r--drivers/iommu/amd_iommu_proto.h2
-rw-r--r--drivers/iommu/amd_iommu_types.h11
-rw-r--r--drivers/iommu/amd_iommu_v2.c24
-rw-r--r--drivers/iommu/arm-smmu-v3.c2701
-rw-r--r--drivers/iommu/arm-smmu.c25
-rw-r--r--drivers/iommu/dmar.c28
-rw-r--r--drivers/iommu/exynos-iommu.c527
-rw-r--r--drivers/iommu/intel-iommu.c504
-rw-r--r--drivers/iommu/intel_irq_remapping.c252
-rw-r--r--drivers/iommu/iommu.c384
-rw-r--r--drivers/iommu/iova.c4
-rw-r--r--drivers/iommu/rockchip-iommu.c27
-rw-r--r--drivers/iommu/tegra-smmu.c109
17 files changed, 4314 insertions, 1040 deletions
diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig
index 1ae4e547b419..f1fb1d3ccc56 100644
--- a/drivers/iommu/Kconfig
+++ b/drivers/iommu/Kconfig
@@ -178,6 +178,9 @@ config OMAP_IOMMU
178 depends on ARM && MMU 178 depends on ARM && MMU
179 depends on ARCH_OMAP2PLUS || COMPILE_TEST 179 depends on ARCH_OMAP2PLUS || COMPILE_TEST
180 select IOMMU_API 180 select IOMMU_API
181 ---help---
182 The OMAP3 media platform drivers depend on iommu support,
183 if you need them say Y here.
181 184
182config OMAP_IOMMU_DEBUG 185config OMAP_IOMMU_DEBUG
183 bool "Export OMAP IOMMU internals in DebugFS" 186 bool "Export OMAP IOMMU internals in DebugFS"
@@ -219,7 +222,7 @@ config TEGRA_IOMMU_SMMU
219 select IOMMU_API 222 select IOMMU_API
220 help 223 help
221 This driver supports the IOMMU hardware (SMMU) found on NVIDIA Tegra 224 This driver supports the IOMMU hardware (SMMU) found on NVIDIA Tegra
222 SoCs (Tegra30 up to Tegra124). 225 SoCs (Tegra30 up to Tegra132).
223 226
224config EXYNOS_IOMMU 227config EXYNOS_IOMMU
225 bool "Exynos IOMMU Support" 228 bool "Exynos IOMMU Support"
@@ -339,6 +342,7 @@ config SPAPR_TCE_IOMMU
339 Enables bits of IOMMU API required by VFIO. The iommu_ops 342 Enables bits of IOMMU API required by VFIO. The iommu_ops
340 is not implemented as it is not necessary for VFIO. 343 is not implemented as it is not necessary for VFIO.
341 344
345# ARM IOMMU support
342config ARM_SMMU 346config ARM_SMMU
343 bool "ARM Ltd. System MMU (SMMU) Support" 347 bool "ARM Ltd. System MMU (SMMU) Support"
344 depends on (ARM64 || ARM) && MMU 348 depends on (ARM64 || ARM) && MMU
@@ -352,4 +356,16 @@ config ARM_SMMU
352 Say Y here if your SoC includes an IOMMU device implementing 356 Say Y here if your SoC includes an IOMMU device implementing
353 the ARM SMMU architecture. 357 the ARM SMMU architecture.
354 358
359config ARM_SMMU_V3
360 bool "ARM Ltd. System MMU Version 3 (SMMUv3) Support"
361 depends on ARM64 && PCI
362 select IOMMU_API
363 select IOMMU_IO_PGTABLE_LPAE
364 help
365 Support for implementations of the ARM System MMU architecture
366 version 3 providing translation support to a PCIe root complex.
367
368 Say Y here if your system includes an IOMMU device implementing
369 the ARM SMMUv3 architecture.
370
355endif # IOMMU_SUPPORT 371endif # IOMMU_SUPPORT
diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile
index 080ffab4ed1c..c6dcc513d711 100644
--- a/drivers/iommu/Makefile
+++ b/drivers/iommu/Makefile
@@ -9,6 +9,7 @@ obj-$(CONFIG_MSM_IOMMU) += msm_iommu.o msm_iommu_dev.o
9obj-$(CONFIG_AMD_IOMMU) += amd_iommu.o amd_iommu_init.o 9obj-$(CONFIG_AMD_IOMMU) += amd_iommu.o amd_iommu_init.o
10obj-$(CONFIG_AMD_IOMMU_V2) += amd_iommu_v2.o 10obj-$(CONFIG_AMD_IOMMU_V2) += amd_iommu_v2.o
11obj-$(CONFIG_ARM_SMMU) += arm-smmu.o 11obj-$(CONFIG_ARM_SMMU) += arm-smmu.o
12obj-$(CONFIG_ARM_SMMU_V3) += arm-smmu-v3.o
12obj-$(CONFIG_DMAR_TABLE) += dmar.o 13obj-$(CONFIG_DMAR_TABLE) += dmar.o
13obj-$(CONFIG_INTEL_IOMMU) += intel-iommu.o 14obj-$(CONFIG_INTEL_IOMMU) += intel-iommu.o
14obj-$(CONFIG_IPMMU_VMSA) += ipmmu-vmsa.o 15obj-$(CONFIG_IPMMU_VMSA) += ipmmu-vmsa.o
diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c
index fffea87a014f..658ee39e6569 100644
--- a/drivers/iommu/amd_iommu.c
+++ b/drivers/iommu/amd_iommu.c
@@ -65,10 +65,6 @@
65 65
66static DEFINE_RWLOCK(amd_iommu_devtable_lock); 66static DEFINE_RWLOCK(amd_iommu_devtable_lock);
67 67
68/* A list of preallocated protection domains */
69static LIST_HEAD(iommu_pd_list);
70static DEFINE_SPINLOCK(iommu_pd_list_lock);
71
72/* List of all available dev_data structures */ 68/* List of all available dev_data structures */
73static LIST_HEAD(dev_data_list); 69static LIST_HEAD(dev_data_list);
74static DEFINE_SPINLOCK(dev_data_list_lock); 70static DEFINE_SPINLOCK(dev_data_list_lock);
@@ -80,8 +76,6 @@ LIST_HEAD(hpet_map);
80 * Domain for untranslated devices - only allocated 76 * Domain for untranslated devices - only allocated
81 * if iommu=pt passed on kernel cmd line. 77 * if iommu=pt passed on kernel cmd line.
82 */ 78 */
83static struct protection_domain *pt_domain;
84
85static const struct iommu_ops amd_iommu_ops; 79static const struct iommu_ops amd_iommu_ops;
86 80
87static ATOMIC_NOTIFIER_HEAD(ppr_notifier); 81static ATOMIC_NOTIFIER_HEAD(ppr_notifier);
@@ -100,7 +94,7 @@ struct iommu_dev_data {
100 struct protection_domain *domain; /* Domain the device is bound to */ 94 struct protection_domain *domain; /* Domain the device is bound to */
101 u16 devid; /* PCI Device ID */ 95 u16 devid; /* PCI Device ID */
102 bool iommu_v2; /* Device can make use of IOMMUv2 */ 96 bool iommu_v2; /* Device can make use of IOMMUv2 */
103 bool passthrough; /* Default for device is pt_domain */ 97 bool passthrough; /* Device is identity mapped */
104 struct { 98 struct {
105 bool enabled; 99 bool enabled;
106 int qdep; 100 int qdep;
@@ -120,7 +114,7 @@ struct iommu_cmd {
120struct kmem_cache *amd_iommu_irq_cache; 114struct kmem_cache *amd_iommu_irq_cache;
121 115
122static void update_domain(struct protection_domain *domain); 116static void update_domain(struct protection_domain *domain);
123static int __init alloc_passthrough_domain(void); 117static int protection_domain_init(struct protection_domain *domain);
124 118
125/**************************************************************************** 119/****************************************************************************
126 * 120 *
@@ -235,31 +229,38 @@ static bool pdev_pri_erratum(struct pci_dev *pdev, u32 erratum)
235} 229}
236 230
237/* 231/*
238 * In this function the list of preallocated protection domains is traversed to 232 * This function actually applies the mapping to the page table of the
239 * find the domain for a specific device 233 * dma_ops domain.
240 */ 234 */
241static struct dma_ops_domain *find_protection_domain(u16 devid) 235static void alloc_unity_mapping(struct dma_ops_domain *dma_dom,
236 struct unity_map_entry *e)
242{ 237{
243 struct dma_ops_domain *entry, *ret = NULL; 238 u64 addr;
244 unsigned long flags;
245 u16 alias = amd_iommu_alias_table[devid];
246
247 if (list_empty(&iommu_pd_list))
248 return NULL;
249
250 spin_lock_irqsave(&iommu_pd_list_lock, flags);
251 239
252 list_for_each_entry(entry, &iommu_pd_list, list) { 240 for (addr = e->address_start; addr < e->address_end;
253 if (entry->target_dev == devid || 241 addr += PAGE_SIZE) {
254 entry->target_dev == alias) { 242 if (addr < dma_dom->aperture_size)
255 ret = entry; 243 __set_bit(addr >> PAGE_SHIFT,
256 break; 244 dma_dom->aperture[0]->bitmap);
257 }
258 } 245 }
246}
259 247
260 spin_unlock_irqrestore(&iommu_pd_list_lock, flags); 248/*
249 * Inits the unity mappings required for a specific device
250 */
251static void init_unity_mappings_for_device(struct device *dev,
252 struct dma_ops_domain *dma_dom)
253{
254 struct unity_map_entry *e;
255 u16 devid;
261 256
262 return ret; 257 devid = get_device_id(dev);
258
259 list_for_each_entry(e, &amd_iommu_unity_map, list) {
260 if (!(devid >= e->devid_start && devid <= e->devid_end))
261 continue;
262 alloc_unity_mapping(dma_dom, e);
263 }
263} 264}
264 265
265/* 266/*
@@ -291,11 +292,23 @@ static bool check_device(struct device *dev)
291 292
292static void init_iommu_group(struct device *dev) 293static void init_iommu_group(struct device *dev)
293{ 294{
295 struct dma_ops_domain *dma_domain;
296 struct iommu_domain *domain;
294 struct iommu_group *group; 297 struct iommu_group *group;
295 298
296 group = iommu_group_get_for_dev(dev); 299 group = iommu_group_get_for_dev(dev);
297 if (!IS_ERR(group)) 300 if (IS_ERR(group))
298 iommu_group_put(group); 301 return;
302
303 domain = iommu_group_default_domain(group);
304 if (!domain)
305 goto out;
306
307 dma_domain = to_pdomain(domain)->priv;
308
309 init_unity_mappings_for_device(dev, dma_domain);
310out:
311 iommu_group_put(group);
299} 312}
300 313
301static int __last_alias(struct pci_dev *pdev, u16 alias, void *data) 314static int __last_alias(struct pci_dev *pdev, u16 alias, void *data)
@@ -435,64 +448,15 @@ static void iommu_uninit_device(struct device *dev)
435 /* Unlink from alias, it may change if another device is re-plugged */ 448 /* Unlink from alias, it may change if another device is re-plugged */
436 dev_data->alias_data = NULL; 449 dev_data->alias_data = NULL;
437 450
451 /* Remove dma-ops */
452 dev->archdata.dma_ops = NULL;
453
438 /* 454 /*
439 * We keep dev_data around for unplugged devices and reuse it when the 455 * We keep dev_data around for unplugged devices and reuse it when the
440 * device is re-plugged - not doing so would introduce a ton of races. 456 * device is re-plugged - not doing so would introduce a ton of races.
441 */ 457 */
442} 458}
443 459
444void __init amd_iommu_uninit_devices(void)
445{
446 struct iommu_dev_data *dev_data, *n;
447 struct pci_dev *pdev = NULL;
448
449 for_each_pci_dev(pdev) {
450
451 if (!check_device(&pdev->dev))
452 continue;
453
454 iommu_uninit_device(&pdev->dev);
455 }
456
457 /* Free all of our dev_data structures */
458 list_for_each_entry_safe(dev_data, n, &dev_data_list, dev_data_list)
459 free_dev_data(dev_data);
460}
461
462int __init amd_iommu_init_devices(void)
463{
464 struct pci_dev *pdev = NULL;
465 int ret = 0;
466
467 for_each_pci_dev(pdev) {
468
469 if (!check_device(&pdev->dev))
470 continue;
471
472 ret = iommu_init_device(&pdev->dev);
473 if (ret == -ENOTSUPP)
474 iommu_ignore_device(&pdev->dev);
475 else if (ret)
476 goto out_free;
477 }
478
479 /*
480 * Initialize IOMMU groups only after iommu_init_device() has
481 * had a chance to populate any IVRS defined aliases.
482 */
483 for_each_pci_dev(pdev) {
484 if (check_device(&pdev->dev))
485 init_iommu_group(&pdev->dev);
486 }
487
488 return 0;
489
490out_free:
491
492 amd_iommu_uninit_devices();
493
494 return ret;
495}
496#ifdef CONFIG_AMD_IOMMU_STATS 460#ifdef CONFIG_AMD_IOMMU_STATS
497 461
498/* 462/*
@@ -1464,94 +1428,6 @@ static unsigned long iommu_unmap_page(struct protection_domain *dom,
1464 return unmapped; 1428 return unmapped;
1465} 1429}
1466 1430
1467/*
1468 * This function checks if a specific unity mapping entry is needed for
1469 * this specific IOMMU.
1470 */
1471static int iommu_for_unity_map(struct amd_iommu *iommu,
1472 struct unity_map_entry *entry)
1473{
1474 u16 bdf, i;
1475
1476 for (i = entry->devid_start; i <= entry->devid_end; ++i) {
1477 bdf = amd_iommu_alias_table[i];
1478 if (amd_iommu_rlookup_table[bdf] == iommu)
1479 return 1;
1480 }
1481
1482 return 0;
1483}
1484
1485/*
1486 * This function actually applies the mapping to the page table of the
1487 * dma_ops domain.
1488 */
1489static int dma_ops_unity_map(struct dma_ops_domain *dma_dom,
1490 struct unity_map_entry *e)
1491{
1492 u64 addr;
1493 int ret;
1494
1495 for (addr = e->address_start; addr < e->address_end;
1496 addr += PAGE_SIZE) {
1497 ret = iommu_map_page(&dma_dom->domain, addr, addr, e->prot,
1498 PAGE_SIZE);
1499 if (ret)
1500 return ret;
1501 /*
1502 * if unity mapping is in aperture range mark the page
1503 * as allocated in the aperture
1504 */
1505 if (addr < dma_dom->aperture_size)
1506 __set_bit(addr >> PAGE_SHIFT,
1507 dma_dom->aperture[0]->bitmap);
1508 }
1509
1510 return 0;
1511}
1512
1513/*
1514 * Init the unity mappings for a specific IOMMU in the system
1515 *
1516 * Basically iterates over all unity mapping entries and applies them to
1517 * the default domain DMA of that IOMMU if necessary.
1518 */
1519static int iommu_init_unity_mappings(struct amd_iommu *iommu)
1520{
1521 struct unity_map_entry *entry;
1522 int ret;
1523
1524 list_for_each_entry(entry, &amd_iommu_unity_map, list) {
1525 if (!iommu_for_unity_map(iommu, entry))
1526 continue;
1527 ret = dma_ops_unity_map(iommu->default_dom, entry);
1528 if (ret)
1529 return ret;
1530 }
1531
1532 return 0;
1533}
1534
1535/*
1536 * Inits the unity mappings required for a specific device
1537 */
1538static int init_unity_mappings_for_device(struct dma_ops_domain *dma_dom,
1539 u16 devid)
1540{
1541 struct unity_map_entry *e;
1542 int ret;
1543
1544 list_for_each_entry(e, &amd_iommu_unity_map, list) {
1545 if (!(devid >= e->devid_start && devid <= e->devid_end))
1546 continue;
1547 ret = dma_ops_unity_map(dma_dom, e);
1548 if (ret)
1549 return ret;
1550 }
1551
1552 return 0;
1553}
1554
1555/**************************************************************************** 1431/****************************************************************************
1556 * 1432 *
1557 * The next functions belong to the address allocator for the dma_ops 1433 * The next functions belong to the address allocator for the dma_ops
@@ -1705,14 +1581,16 @@ static unsigned long dma_ops_area_alloc(struct device *dev,
1705 unsigned long next_bit = dom->next_address % APERTURE_RANGE_SIZE; 1581 unsigned long next_bit = dom->next_address % APERTURE_RANGE_SIZE;
1706 int max_index = dom->aperture_size >> APERTURE_RANGE_SHIFT; 1582 int max_index = dom->aperture_size >> APERTURE_RANGE_SHIFT;
1707 int i = start >> APERTURE_RANGE_SHIFT; 1583 int i = start >> APERTURE_RANGE_SHIFT;
1708 unsigned long boundary_size; 1584 unsigned long boundary_size, mask;
1709 unsigned long address = -1; 1585 unsigned long address = -1;
1710 unsigned long limit; 1586 unsigned long limit;
1711 1587
1712 next_bit >>= PAGE_SHIFT; 1588 next_bit >>= PAGE_SHIFT;
1713 1589
1714 boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1, 1590 mask = dma_get_seg_boundary(dev);
1715 PAGE_SIZE) >> PAGE_SHIFT; 1591
1592 boundary_size = mask + 1 ? ALIGN(mask + 1, PAGE_SIZE) >> PAGE_SHIFT :
1593 1UL << (BITS_PER_LONG - PAGE_SHIFT);
1716 1594
1717 for (;i < max_index; ++i) { 1595 for (;i < max_index; ++i) {
1718 unsigned long offset = dom->aperture[i]->offset >> PAGE_SHIFT; 1596 unsigned long offset = dom->aperture[i]->offset >> PAGE_SHIFT;
@@ -1870,9 +1748,15 @@ static void free_pt_##LVL (unsigned long __pt) \
1870 pt = (u64 *)__pt; \ 1748 pt = (u64 *)__pt; \
1871 \ 1749 \
1872 for (i = 0; i < 512; ++i) { \ 1750 for (i = 0; i < 512; ++i) { \
1751 /* PTE present? */ \
1873 if (!IOMMU_PTE_PRESENT(pt[i])) \ 1752 if (!IOMMU_PTE_PRESENT(pt[i])) \
1874 continue; \ 1753 continue; \
1875 \ 1754 \
1755 /* Large PTE? */ \
1756 if (PM_PTE_LEVEL(pt[i]) == 0 || \
1757 PM_PTE_LEVEL(pt[i]) == 7) \
1758 continue; \
1759 \
1876 p = (unsigned long)IOMMU_PTE_PAGE(pt[i]); \ 1760 p = (unsigned long)IOMMU_PTE_PAGE(pt[i]); \
1877 FN(p); \ 1761 FN(p); \
1878 } \ 1762 } \
@@ -1995,12 +1879,9 @@ static struct dma_ops_domain *dma_ops_domain_alloc(void)
1995 if (!dma_dom) 1879 if (!dma_dom)
1996 return NULL; 1880 return NULL;
1997 1881
1998 spin_lock_init(&dma_dom->domain.lock); 1882 if (protection_domain_init(&dma_dom->domain))
1999
2000 dma_dom->domain.id = domain_id_alloc();
2001 if (dma_dom->domain.id == 0)
2002 goto free_dma_dom; 1883 goto free_dma_dom;
2003 INIT_LIST_HEAD(&dma_dom->domain.dev_list); 1884
2004 dma_dom->domain.mode = PAGE_MODE_2_LEVEL; 1885 dma_dom->domain.mode = PAGE_MODE_2_LEVEL;
2005 dma_dom->domain.pt_root = (void *)get_zeroed_page(GFP_KERNEL); 1886 dma_dom->domain.pt_root = (void *)get_zeroed_page(GFP_KERNEL);
2006 dma_dom->domain.flags = PD_DMA_OPS_MASK; 1887 dma_dom->domain.flags = PD_DMA_OPS_MASK;
@@ -2009,7 +1890,6 @@ static struct dma_ops_domain *dma_ops_domain_alloc(void)
2009 goto free_dma_dom; 1890 goto free_dma_dom;
2010 1891
2011 dma_dom->need_flush = false; 1892 dma_dom->need_flush = false;
2012 dma_dom->target_dev = 0xffff;
2013 1893
2014 add_domain_to_list(&dma_dom->domain); 1894 add_domain_to_list(&dma_dom->domain);
2015 1895
@@ -2284,15 +2164,17 @@ static int attach_device(struct device *dev,
2284 dev_data = get_dev_data(dev); 2164 dev_data = get_dev_data(dev);
2285 2165
2286 if (domain->flags & PD_IOMMUV2_MASK) { 2166 if (domain->flags & PD_IOMMUV2_MASK) {
2287 if (!dev_data->iommu_v2 || !dev_data->passthrough) 2167 if (!dev_data->passthrough)
2288 return -EINVAL; 2168 return -EINVAL;
2289 2169
2290 if (pdev_iommuv2_enable(pdev) != 0) 2170 if (dev_data->iommu_v2) {
2291 return -EINVAL; 2171 if (pdev_iommuv2_enable(pdev) != 0)
2172 return -EINVAL;
2292 2173
2293 dev_data->ats.enabled = true; 2174 dev_data->ats.enabled = true;
2294 dev_data->ats.qdep = pci_ats_queue_depth(pdev); 2175 dev_data->ats.qdep = pci_ats_queue_depth(pdev);
2295 dev_data->pri_tlp = pci_pri_tlp_required(pdev); 2176 dev_data->pri_tlp = pci_pri_tlp_required(pdev);
2177 }
2296 } else if (amd_iommu_iotlb_sup && 2178 } else if (amd_iommu_iotlb_sup &&
2297 pci_enable_ats(pdev, PAGE_SHIFT) == 0) { 2179 pci_enable_ats(pdev, PAGE_SHIFT) == 0) {
2298 dev_data->ats.enabled = true; 2180 dev_data->ats.enabled = true;
@@ -2338,15 +2220,6 @@ static void __detach_device(struct iommu_dev_data *dev_data)
2338 do_detach(head); 2220 do_detach(head);
2339 2221
2340 spin_unlock_irqrestore(&domain->lock, flags); 2222 spin_unlock_irqrestore(&domain->lock, flags);
2341
2342 /*
2343 * If we run in passthrough mode the device must be assigned to the
2344 * passthrough domain if it is detached from any other domain.
2345 * Make sure we can deassign from the pt_domain itself.
2346 */
2347 if (dev_data->passthrough &&
2348 (dev_data->domain == NULL && domain != pt_domain))
2349 __attach_device(dev_data, pt_domain);
2350} 2223}
2351 2224
2352/* 2225/*
@@ -2366,7 +2239,7 @@ static void detach_device(struct device *dev)
2366 __detach_device(dev_data); 2239 __detach_device(dev_data);
2367 write_unlock_irqrestore(&amd_iommu_devtable_lock, flags); 2240 write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
2368 2241
2369 if (domain->flags & PD_IOMMUV2_MASK) 2242 if (domain->flags & PD_IOMMUV2_MASK && dev_data->iommu_v2)
2370 pdev_iommuv2_disable(to_pci_dev(dev)); 2243 pdev_iommuv2_disable(to_pci_dev(dev));
2371 else if (dev_data->ats.enabled) 2244 else if (dev_data->ats.enabled)
2372 pci_disable_ats(to_pci_dev(dev)); 2245 pci_disable_ats(to_pci_dev(dev));
@@ -2374,110 +2247,65 @@ static void detach_device(struct device *dev)
2374 dev_data->ats.enabled = false; 2247 dev_data->ats.enabled = false;
2375} 2248}
2376 2249
2377/* 2250static int amd_iommu_add_device(struct device *dev)
2378 * Find out the protection domain structure for a given PCI device. This
2379 * will give us the pointer to the page table root for example.
2380 */
2381static struct protection_domain *domain_for_device(struct device *dev)
2382{
2383 struct iommu_dev_data *dev_data;
2384 struct protection_domain *dom = NULL;
2385 unsigned long flags;
2386
2387 dev_data = get_dev_data(dev);
2388
2389 if (dev_data->domain)
2390 return dev_data->domain;
2391
2392 if (dev_data->alias_data != NULL) {
2393 struct iommu_dev_data *alias_data = dev_data->alias_data;
2394
2395 read_lock_irqsave(&amd_iommu_devtable_lock, flags);
2396 if (alias_data->domain != NULL) {
2397 __attach_device(dev_data, alias_data->domain);
2398 dom = alias_data->domain;
2399 }
2400 read_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
2401 }
2402
2403 return dom;
2404}
2405
2406static int device_change_notifier(struct notifier_block *nb,
2407 unsigned long action, void *data)
2408{ 2251{
2409 struct dma_ops_domain *dma_domain;
2410 struct protection_domain *domain;
2411 struct iommu_dev_data *dev_data; 2252 struct iommu_dev_data *dev_data;
2412 struct device *dev = data; 2253 struct iommu_domain *domain;
2413 struct amd_iommu *iommu; 2254 struct amd_iommu *iommu;
2414 unsigned long flags;
2415 u16 devid; 2255 u16 devid;
2256 int ret;
2416 2257
2417 if (!check_device(dev)) 2258 if (!check_device(dev) || get_dev_data(dev))
2418 return 0; 2259 return 0;
2419 2260
2420 devid = get_device_id(dev); 2261 devid = get_device_id(dev);
2421 iommu = amd_iommu_rlookup_table[devid]; 2262 iommu = amd_iommu_rlookup_table[devid];
2422 dev_data = get_dev_data(dev);
2423
2424 switch (action) {
2425 case BUS_NOTIFY_ADD_DEVICE:
2426 2263
2427 iommu_init_device(dev); 2264 ret = iommu_init_device(dev);
2428 init_iommu_group(dev); 2265 if (ret) {
2266 if (ret != -ENOTSUPP)
2267 pr_err("Failed to initialize device %s - trying to proceed anyway\n",
2268 dev_name(dev));
2429 2269
2430 /* 2270 iommu_ignore_device(dev);
2431 * dev_data is still NULL and 2271 dev->archdata.dma_ops = &nommu_dma_ops;
2432 * got initialized in iommu_init_device 2272 goto out;
2433 */ 2273 }
2434 dev_data = get_dev_data(dev); 2274 init_iommu_group(dev);
2435 2275
2436 if (iommu_pass_through || dev_data->iommu_v2) { 2276 dev_data = get_dev_data(dev);
2437 dev_data->passthrough = true;
2438 attach_device(dev, pt_domain);
2439 break;
2440 }
2441 2277
2442 domain = domain_for_device(dev); 2278 BUG_ON(!dev_data);
2443 2279
2444 /* allocate a protection domain if a device is added */ 2280 if (iommu_pass_through || dev_data->iommu_v2)
2445 dma_domain = find_protection_domain(devid); 2281 iommu_request_dm_for_dev(dev);
2446 if (!dma_domain) {
2447 dma_domain = dma_ops_domain_alloc();
2448 if (!dma_domain)
2449 goto out;
2450 dma_domain->target_dev = devid;
2451
2452 spin_lock_irqsave(&iommu_pd_list_lock, flags);
2453 list_add_tail(&dma_domain->list, &iommu_pd_list);
2454 spin_unlock_irqrestore(&iommu_pd_list_lock, flags);
2455 }
2456 2282
2283 /* Domains are initialized for this device - have a look what we ended up with */
2284 domain = iommu_get_domain_for_dev(dev);
2285 if (domain->type == IOMMU_DOMAIN_IDENTITY)
2286 dev_data->passthrough = true;
2287 else
2457 dev->archdata.dma_ops = &amd_iommu_dma_ops; 2288 dev->archdata.dma_ops = &amd_iommu_dma_ops;
2458 2289
2459 break; 2290out:
2460 case BUS_NOTIFY_REMOVED_DEVICE:
2461
2462 iommu_uninit_device(dev);
2463
2464 default:
2465 goto out;
2466 }
2467
2468 iommu_completion_wait(iommu); 2291 iommu_completion_wait(iommu);
2469 2292
2470out:
2471 return 0; 2293 return 0;
2472} 2294}
2473 2295
2474static struct notifier_block device_nb = { 2296static void amd_iommu_remove_device(struct device *dev)
2475 .notifier_call = device_change_notifier,
2476};
2477
2478void amd_iommu_init_notifier(void)
2479{ 2297{
2480 bus_register_notifier(&pci_bus_type, &device_nb); 2298 struct amd_iommu *iommu;
2299 u16 devid;
2300
2301 if (!check_device(dev))
2302 return;
2303
2304 devid = get_device_id(dev);
2305 iommu = amd_iommu_rlookup_table[devid];
2306
2307 iommu_uninit_device(dev);
2308 iommu_completion_wait(iommu);
2481} 2309}
2482 2310
2483/***************************************************************************** 2311/*****************************************************************************
@@ -2496,28 +2324,20 @@ void amd_iommu_init_notifier(void)
2496static struct protection_domain *get_domain(struct device *dev) 2324static struct protection_domain *get_domain(struct device *dev)
2497{ 2325{
2498 struct protection_domain *domain; 2326 struct protection_domain *domain;
2499 struct dma_ops_domain *dma_dom; 2327 struct iommu_domain *io_domain;
2500 u16 devid = get_device_id(dev);
2501 2328
2502 if (!check_device(dev)) 2329 if (!check_device(dev))
2503 return ERR_PTR(-EINVAL); 2330 return ERR_PTR(-EINVAL);
2504 2331
2505 domain = domain_for_device(dev); 2332 io_domain = iommu_get_domain_for_dev(dev);
2506 if (domain != NULL && !dma_ops_domain(domain)) 2333 if (!io_domain)
2507 return ERR_PTR(-EBUSY); 2334 return NULL;
2508
2509 if (domain != NULL)
2510 return domain;
2511 2335
2512 /* Device not bound yet - bind it */ 2336 domain = to_pdomain(io_domain);
2513 dma_dom = find_protection_domain(devid); 2337 if (!dma_ops_domain(domain))
2514 if (!dma_dom) 2338 return ERR_PTR(-EBUSY);
2515 dma_dom = amd_iommu_rlookup_table[devid]->default_dom;
2516 attach_device(dev, &dma_dom->domain);
2517 DUMP_printk("Using protection domain %d for device %s\n",
2518 dma_dom->domain.id, dev_name(dev));
2519 2339
2520 return &dma_dom->domain; 2340 return domain;
2521} 2341}
2522 2342
2523static void update_device_table(struct protection_domain *domain) 2343static void update_device_table(struct protection_domain *domain)
@@ -3013,54 +2833,6 @@ static int amd_iommu_dma_supported(struct device *dev, u64 mask)
3013 return check_device(dev); 2833 return check_device(dev);
3014} 2834}
3015 2835
3016/*
3017 * The function for pre-allocating protection domains.
3018 *
3019 * If the driver core informs the DMA layer if a driver grabs a device
3020 * we don't need to preallocate the protection domains anymore.
3021 * For now we have to.
3022 */
3023static void __init prealloc_protection_domains(void)
3024{
3025 struct iommu_dev_data *dev_data;
3026 struct dma_ops_domain *dma_dom;
3027 struct pci_dev *dev = NULL;
3028 u16 devid;
3029
3030 for_each_pci_dev(dev) {
3031
3032 /* Do we handle this device? */
3033 if (!check_device(&dev->dev))
3034 continue;
3035
3036 dev_data = get_dev_data(&dev->dev);
3037 if (!amd_iommu_force_isolation && dev_data->iommu_v2) {
3038 /* Make sure passthrough domain is allocated */
3039 alloc_passthrough_domain();
3040 dev_data->passthrough = true;
3041 attach_device(&dev->dev, pt_domain);
3042 pr_info("AMD-Vi: Using passthrough domain for device %s\n",
3043 dev_name(&dev->dev));
3044 }
3045
3046 /* Is there already any domain for it? */
3047 if (domain_for_device(&dev->dev))
3048 continue;
3049
3050 devid = get_device_id(&dev->dev);
3051
3052 dma_dom = dma_ops_domain_alloc();
3053 if (!dma_dom)
3054 continue;
3055 init_unity_mappings_for_device(dma_dom, devid);
3056 dma_dom->target_dev = devid;
3057
3058 attach_device(&dev->dev, &dma_dom->domain);
3059
3060 list_add_tail(&dma_dom->list, &iommu_pd_list);
3061 }
3062}
3063
3064static struct dma_map_ops amd_iommu_dma_ops = { 2836static struct dma_map_ops amd_iommu_dma_ops = {
3065 .alloc = alloc_coherent, 2837 .alloc = alloc_coherent,
3066 .free = free_coherent, 2838 .free = free_coherent,
@@ -3071,75 +2843,24 @@ static struct dma_map_ops amd_iommu_dma_ops = {
3071 .dma_supported = amd_iommu_dma_supported, 2843 .dma_supported = amd_iommu_dma_supported,
3072}; 2844};
3073 2845
3074static unsigned device_dma_ops_init(void) 2846int __init amd_iommu_init_api(void)
3075{
3076 struct iommu_dev_data *dev_data;
3077 struct pci_dev *pdev = NULL;
3078 unsigned unhandled = 0;
3079
3080 for_each_pci_dev(pdev) {
3081 if (!check_device(&pdev->dev)) {
3082
3083 iommu_ignore_device(&pdev->dev);
3084
3085 unhandled += 1;
3086 continue;
3087 }
3088
3089 dev_data = get_dev_data(&pdev->dev);
3090
3091 if (!dev_data->passthrough)
3092 pdev->dev.archdata.dma_ops = &amd_iommu_dma_ops;
3093 else
3094 pdev->dev.archdata.dma_ops = &nommu_dma_ops;
3095 }
3096
3097 return unhandled;
3098}
3099
3100/*
3101 * The function which clues the AMD IOMMU driver into dma_ops.
3102 */
3103
3104void __init amd_iommu_init_api(void)
3105{ 2847{
3106 bus_set_iommu(&pci_bus_type, &amd_iommu_ops); 2848 return bus_set_iommu(&pci_bus_type, &amd_iommu_ops);
3107} 2849}
3108 2850
3109int __init amd_iommu_init_dma_ops(void) 2851int __init amd_iommu_init_dma_ops(void)
3110{ 2852{
3111 struct amd_iommu *iommu; 2853 swiotlb = iommu_pass_through ? 1 : 0;
3112 int ret, unhandled; 2854 iommu_detected = 1;
3113
3114 /*
3115 * first allocate a default protection domain for every IOMMU we
3116 * found in the system. Devices not assigned to any other
3117 * protection domain will be assigned to the default one.
3118 */
3119 for_each_iommu(iommu) {
3120 iommu->default_dom = dma_ops_domain_alloc();
3121 if (iommu->default_dom == NULL)
3122 return -ENOMEM;
3123 iommu->default_dom->domain.flags |= PD_DEFAULT_MASK;
3124 ret = iommu_init_unity_mappings(iommu);
3125 if (ret)
3126 goto free_domains;
3127 }
3128 2855
3129 /* 2856 /*
3130 * Pre-allocate the protection domains for each device. 2857 * In case we don't initialize SWIOTLB (actually the common case
2858 * when AMD IOMMU is enabled), make sure there are global
2859 * dma_ops set as a fall-back for devices not handled by this
2860 * driver (for example non-PCI devices).
3131 */ 2861 */
3132 prealloc_protection_domains(); 2862 if (!swiotlb)
3133 2863 dma_ops = &nommu_dma_ops;
3134 iommu_detected = 1;
3135 swiotlb = 0;
3136
3137 /* Make the driver finally visible to the drivers */
3138 unhandled = device_dma_ops_init();
3139 if (unhandled && max_pfn > MAX_DMA32_PFN) {
3140 /* There are unhandled devices - initialize swiotlb for them */
3141 swiotlb = 1;
3142 }
3143 2864
3144 amd_iommu_stats_init(); 2865 amd_iommu_stats_init();
3145 2866
@@ -3149,14 +2870,6 @@ int __init amd_iommu_init_dma_ops(void)
3149 pr_info("AMD-Vi: Lazy IO/TLB flushing enabled\n"); 2870 pr_info("AMD-Vi: Lazy IO/TLB flushing enabled\n");
3150 2871
3151 return 0; 2872 return 0;
3152
3153free_domains:
3154
3155 for_each_iommu(iommu) {
3156 dma_ops_domain_free(iommu->default_dom);
3157 }
3158
3159 return ret;
3160} 2873}
3161 2874
3162/***************************************************************************** 2875/*****************************************************************************
@@ -3198,6 +2911,18 @@ static void protection_domain_free(struct protection_domain *domain)
3198 kfree(domain); 2911 kfree(domain);
3199} 2912}
3200 2913
2914static int protection_domain_init(struct protection_domain *domain)
2915{
2916 spin_lock_init(&domain->lock);
2917 mutex_init(&domain->api_lock);
2918 domain->id = domain_id_alloc();
2919 if (!domain->id)
2920 return -ENOMEM;
2921 INIT_LIST_HEAD(&domain->dev_list);
2922
2923 return 0;
2924}
2925
3201static struct protection_domain *protection_domain_alloc(void) 2926static struct protection_domain *protection_domain_alloc(void)
3202{ 2927{
3203 struct protection_domain *domain; 2928 struct protection_domain *domain;
@@ -3206,12 +2931,8 @@ static struct protection_domain *protection_domain_alloc(void)
3206 if (!domain) 2931 if (!domain)
3207 return NULL; 2932 return NULL;
3208 2933
3209 spin_lock_init(&domain->lock); 2934 if (protection_domain_init(domain))
3210 mutex_init(&domain->api_lock);
3211 domain->id = domain_id_alloc();
3212 if (!domain->id)
3213 goto out_err; 2935 goto out_err;
3214 INIT_LIST_HEAD(&domain->dev_list);
3215 2936
3216 add_domain_to_list(domain); 2937 add_domain_to_list(domain);
3217 2938
@@ -3223,48 +2944,49 @@ out_err:
3223 return NULL; 2944 return NULL;
3224} 2945}
3225 2946
3226static int __init alloc_passthrough_domain(void)
3227{
3228 if (pt_domain != NULL)
3229 return 0;
3230
3231 /* allocate passthrough domain */
3232 pt_domain = protection_domain_alloc();
3233 if (!pt_domain)
3234 return -ENOMEM;
3235
3236 pt_domain->mode = PAGE_MODE_NONE;
3237
3238 return 0;
3239}
3240
3241static struct iommu_domain *amd_iommu_domain_alloc(unsigned type) 2947static struct iommu_domain *amd_iommu_domain_alloc(unsigned type)
3242{ 2948{
3243 struct protection_domain *pdomain; 2949 struct protection_domain *pdomain;
2950 struct dma_ops_domain *dma_domain;
3244 2951
3245 /* We only support unmanaged domains for now */ 2952 switch (type) {
3246 if (type != IOMMU_DOMAIN_UNMANAGED) 2953 case IOMMU_DOMAIN_UNMANAGED:
3247 return NULL; 2954 pdomain = protection_domain_alloc();
3248 2955 if (!pdomain)
3249 pdomain = protection_domain_alloc(); 2956 return NULL;
3250 if (!pdomain)
3251 goto out_free;
3252 2957
3253 pdomain->mode = PAGE_MODE_3_LEVEL; 2958 pdomain->mode = PAGE_MODE_3_LEVEL;
3254 pdomain->pt_root = (void *)get_zeroed_page(GFP_KERNEL); 2959 pdomain->pt_root = (void *)get_zeroed_page(GFP_KERNEL);
3255 if (!pdomain->pt_root) 2960 if (!pdomain->pt_root) {
3256 goto out_free; 2961 protection_domain_free(pdomain);
2962 return NULL;
2963 }
3257 2964
3258 pdomain->domain.geometry.aperture_start = 0; 2965 pdomain->domain.geometry.aperture_start = 0;
3259 pdomain->domain.geometry.aperture_end = ~0ULL; 2966 pdomain->domain.geometry.aperture_end = ~0ULL;
3260 pdomain->domain.geometry.force_aperture = true; 2967 pdomain->domain.geometry.force_aperture = true;
3261 2968
3262 return &pdomain->domain; 2969 break;
2970 case IOMMU_DOMAIN_DMA:
2971 dma_domain = dma_ops_domain_alloc();
2972 if (!dma_domain) {
2973 pr_err("AMD-Vi: Failed to allocate\n");
2974 return NULL;
2975 }
2976 pdomain = &dma_domain->domain;
2977 break;
2978 case IOMMU_DOMAIN_IDENTITY:
2979 pdomain = protection_domain_alloc();
2980 if (!pdomain)
2981 return NULL;
3263 2982
3264out_free: 2983 pdomain->mode = PAGE_MODE_NONE;
3265 protection_domain_free(pdomain); 2984 break;
2985 default:
2986 return NULL;
2987 }
3266 2988
3267 return NULL; 2989 return &pdomain->domain;
3268} 2990}
3269 2991
3270static void amd_iommu_domain_free(struct iommu_domain *dom) 2992static void amd_iommu_domain_free(struct iommu_domain *dom)
@@ -3414,6 +3136,47 @@ static bool amd_iommu_capable(enum iommu_cap cap)
3414 return false; 3136 return false;
3415} 3137}
3416 3138
3139static void amd_iommu_get_dm_regions(struct device *dev,
3140 struct list_head *head)
3141{
3142 struct unity_map_entry *entry;
3143 u16 devid;
3144
3145 devid = get_device_id(dev);
3146
3147 list_for_each_entry(entry, &amd_iommu_unity_map, list) {
3148 struct iommu_dm_region *region;
3149
3150 if (devid < entry->devid_start || devid > entry->devid_end)
3151 continue;
3152
3153 region = kzalloc(sizeof(*region), GFP_KERNEL);
3154 if (!region) {
3155 pr_err("Out of memory allocating dm-regions for %s\n",
3156 dev_name(dev));
3157 return;
3158 }
3159
3160 region->start = entry->address_start;
3161 region->length = entry->address_end - entry->address_start;
3162 if (entry->prot & IOMMU_PROT_IR)
3163 region->prot |= IOMMU_READ;
3164 if (entry->prot & IOMMU_PROT_IW)
3165 region->prot |= IOMMU_WRITE;
3166
3167 list_add_tail(&region->list, head);
3168 }
3169}
3170
3171static void amd_iommu_put_dm_regions(struct device *dev,
3172 struct list_head *head)
3173{
3174 struct iommu_dm_region *entry, *next;
3175
3176 list_for_each_entry_safe(entry, next, head, list)
3177 kfree(entry);
3178}
3179
3417static const struct iommu_ops amd_iommu_ops = { 3180static const struct iommu_ops amd_iommu_ops = {
3418 .capable = amd_iommu_capable, 3181 .capable = amd_iommu_capable,
3419 .domain_alloc = amd_iommu_domain_alloc, 3182 .domain_alloc = amd_iommu_domain_alloc,
@@ -3424,6 +3187,10 @@ static const struct iommu_ops amd_iommu_ops = {
3424 .unmap = amd_iommu_unmap, 3187 .unmap = amd_iommu_unmap,
3425 .map_sg = default_iommu_map_sg, 3188 .map_sg = default_iommu_map_sg,
3426 .iova_to_phys = amd_iommu_iova_to_phys, 3189 .iova_to_phys = amd_iommu_iova_to_phys,
3190 .add_device = amd_iommu_add_device,
3191 .remove_device = amd_iommu_remove_device,
3192 .get_dm_regions = amd_iommu_get_dm_regions,
3193 .put_dm_regions = amd_iommu_put_dm_regions,
3427 .pgsize_bitmap = AMD_IOMMU_PGSIZES, 3194 .pgsize_bitmap = AMD_IOMMU_PGSIZES,
3428}; 3195};
3429 3196
@@ -3437,33 +3204,6 @@ static const struct iommu_ops amd_iommu_ops = {
3437 * 3204 *
3438 *****************************************************************************/ 3205 *****************************************************************************/
3439 3206
3440int __init amd_iommu_init_passthrough(void)
3441{
3442 struct iommu_dev_data *dev_data;
3443 struct pci_dev *dev = NULL;
3444 int ret;
3445
3446 ret = alloc_passthrough_domain();
3447 if (ret)
3448 return ret;
3449
3450 for_each_pci_dev(dev) {
3451 if (!check_device(&dev->dev))
3452 continue;
3453
3454 dev_data = get_dev_data(&dev->dev);
3455 dev_data->passthrough = true;
3456
3457 attach_device(&dev->dev, pt_domain);
3458 }
3459
3460 amd_iommu_stats_init();
3461
3462 pr_info("AMD-Vi: Initialized for Passthrough Mode\n");
3463
3464 return 0;
3465}
3466
3467/* IOMMUv2 specific functions */ 3207/* IOMMUv2 specific functions */
3468int amd_iommu_register_ppr_notifier(struct notifier_block *nb) 3208int amd_iommu_register_ppr_notifier(struct notifier_block *nb)
3469{ 3209{
@@ -3578,7 +3318,12 @@ static int __flush_pasid(struct protection_domain *domain, int pasid,
3578 struct amd_iommu *iommu; 3318 struct amd_iommu *iommu;
3579 int qdep; 3319 int qdep;
3580 3320
3581 BUG_ON(!dev_data->ats.enabled); 3321 /*
3322 There might be non-IOMMUv2 capable devices in an IOMMUv2
3323 * domain.
3324 */
3325 if (!dev_data->ats.enabled)
3326 continue;
3582 3327
3583 qdep = dev_data->ats.qdep; 3328 qdep = dev_data->ats.qdep;
3584 iommu = amd_iommu_rlookup_table[dev_data->devid]; 3329 iommu = amd_iommu_rlookup_table[dev_data->devid];
diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c
index c17df04d7a7f..a24495eb4e26 100644
--- a/drivers/iommu/amd_iommu_init.c
+++ b/drivers/iommu/amd_iommu_init.c
@@ -226,6 +226,7 @@ static enum iommu_init_state init_state = IOMMU_START_STATE;
226 226
227static int amd_iommu_enable_interrupts(void); 227static int amd_iommu_enable_interrupts(void);
228static int __init iommu_go_to_state(enum iommu_init_state state); 228static int __init iommu_go_to_state(enum iommu_init_state state);
229static void init_device_table_dma(void);
229 230
230static inline void update_last_devid(u16 devid) 231static inline void update_last_devid(u16 devid)
231{ 232{
@@ -1389,9 +1390,15 @@ static int __init amd_iommu_init_pci(void)
1389 break; 1390 break;
1390 } 1391 }
1391 1392
1392 ret = amd_iommu_init_devices(); 1393 init_device_table_dma();
1394
1395 for_each_iommu(iommu)
1396 iommu_flush_all_caches(iommu);
1393 1397
1394 print_iommu_info(); 1398 ret = amd_iommu_init_api();
1399
1400 if (!ret)
1401 print_iommu_info();
1395 1402
1396 return ret; 1403 return ret;
1397} 1404}
@@ -1829,8 +1836,6 @@ static bool __init check_ioapic_information(void)
1829 1836
1830static void __init free_dma_resources(void) 1837static void __init free_dma_resources(void)
1831{ 1838{
1832 amd_iommu_uninit_devices();
1833
1834 free_pages((unsigned long)amd_iommu_pd_alloc_bitmap, 1839 free_pages((unsigned long)amd_iommu_pd_alloc_bitmap,
1835 get_order(MAX_DOMAIN_ID/8)); 1840 get_order(MAX_DOMAIN_ID/8));
1836 1841
@@ -2021,31 +2026,6 @@ static bool detect_ivrs(void)
2021 return true; 2026 return true;
2022} 2027}
2023 2028
2024static int amd_iommu_init_dma(void)
2025{
2026 struct amd_iommu *iommu;
2027 int ret;
2028
2029 if (iommu_pass_through)
2030 ret = amd_iommu_init_passthrough();
2031 else
2032 ret = amd_iommu_init_dma_ops();
2033
2034 if (ret)
2035 return ret;
2036
2037 init_device_table_dma();
2038
2039 for_each_iommu(iommu)
2040 iommu_flush_all_caches(iommu);
2041
2042 amd_iommu_init_api();
2043
2044 amd_iommu_init_notifier();
2045
2046 return 0;
2047}
2048
2049/**************************************************************************** 2029/****************************************************************************
2050 * 2030 *
2051 * AMD IOMMU Initialization State Machine 2031 * AMD IOMMU Initialization State Machine
@@ -2085,7 +2065,7 @@ static int __init state_next(void)
2085 init_state = ret ? IOMMU_INIT_ERROR : IOMMU_INTERRUPTS_EN; 2065 init_state = ret ? IOMMU_INIT_ERROR : IOMMU_INTERRUPTS_EN;
2086 break; 2066 break;
2087 case IOMMU_INTERRUPTS_EN: 2067 case IOMMU_INTERRUPTS_EN:
2088 ret = amd_iommu_init_dma(); 2068 ret = amd_iommu_init_dma_ops();
2089 init_state = ret ? IOMMU_INIT_ERROR : IOMMU_DMA_OPS; 2069 init_state = ret ? IOMMU_INIT_ERROR : IOMMU_DMA_OPS;
2090 break; 2070 break;
2091 case IOMMU_DMA_OPS: 2071 case IOMMU_DMA_OPS:
diff --git a/drivers/iommu/amd_iommu_proto.h b/drivers/iommu/amd_iommu_proto.h
index 0a21142d3639..0bd9eb374462 100644
--- a/drivers/iommu/amd_iommu_proto.h
+++ b/drivers/iommu/amd_iommu_proto.h
@@ -30,7 +30,7 @@ extern void amd_iommu_reset_cmd_buffer(struct amd_iommu *iommu);
30extern int amd_iommu_init_devices(void); 30extern int amd_iommu_init_devices(void);
31extern void amd_iommu_uninit_devices(void); 31extern void amd_iommu_uninit_devices(void);
32extern void amd_iommu_init_notifier(void); 32extern void amd_iommu_init_notifier(void);
33extern void amd_iommu_init_api(void); 33extern int amd_iommu_init_api(void);
34 34
35/* Needed for interrupt remapping */ 35/* Needed for interrupt remapping */
36extern int amd_iommu_prepare(void); 36extern int amd_iommu_prepare(void);
diff --git a/drivers/iommu/amd_iommu_types.h b/drivers/iommu/amd_iommu_types.h
index 6533e874c9d7..f65908841be0 100644
--- a/drivers/iommu/amd_iommu_types.h
+++ b/drivers/iommu/amd_iommu_types.h
@@ -447,8 +447,6 @@ struct aperture_range {
447 * Data container for a dma_ops specific protection domain 447 * Data container for a dma_ops specific protection domain
448 */ 448 */
449struct dma_ops_domain { 449struct dma_ops_domain {
450 struct list_head list;
451
452 /* generic protection domain information */ 450 /* generic protection domain information */
453 struct protection_domain domain; 451 struct protection_domain domain;
454 452
@@ -463,12 +461,6 @@ struct dma_ops_domain {
463 461
464 /* This will be set to true when TLB needs to be flushed */ 462 /* This will be set to true when TLB needs to be flushed */
465 bool need_flush; 463 bool need_flush;
466
467 /*
468 * if this is a preallocated domain, keep the device for which it was
469 * preallocated in this variable
470 */
471 u16 target_dev;
472}; 464};
473 465
474/* 466/*
@@ -553,9 +545,6 @@ struct amd_iommu {
553 /* if one, we need to send a completion wait command */ 545 /* if one, we need to send a completion wait command */
554 bool need_sync; 546 bool need_sync;
555 547
556 /* default dma_ops domain for that IOMMU */
557 struct dma_ops_domain *default_dom;
558
559 /* IOMMU sysfs device */ 548 /* IOMMU sysfs device */
560 struct device *iommu_dev; 549 struct device *iommu_dev;
561 550
diff --git a/drivers/iommu/amd_iommu_v2.c b/drivers/iommu/amd_iommu_v2.c
index 3465faf1809e..f7b875bb70d4 100644
--- a/drivers/iommu/amd_iommu_v2.c
+++ b/drivers/iommu/amd_iommu_v2.c
@@ -132,11 +132,19 @@ static struct device_state *get_device_state(u16 devid)
132 132
133static void free_device_state(struct device_state *dev_state) 133static void free_device_state(struct device_state *dev_state)
134{ 134{
135 struct iommu_group *group;
136
135 /* 137 /*
136 * First detach device from domain - No more PRI requests will arrive 138 * First detach device from domain - No more PRI requests will arrive
137 * from that device after it is unbound from the IOMMUv2 domain. 139 * from that device after it is unbound from the IOMMUv2 domain.
138 */ 140 */
139 iommu_detach_device(dev_state->domain, &dev_state->pdev->dev); 141 group = iommu_group_get(&dev_state->pdev->dev);
142 if (WARN_ON(!group))
143 return;
144
145 iommu_detach_group(dev_state->domain, group);
146
147 iommu_group_put(group);
140 148
141 /* Everything is down now, free the IOMMUv2 domain */ 149 /* Everything is down now, free the IOMMUv2 domain */
142 iommu_domain_free(dev_state->domain); 150 iommu_domain_free(dev_state->domain);
@@ -731,6 +739,7 @@ EXPORT_SYMBOL(amd_iommu_unbind_pasid);
731int amd_iommu_init_device(struct pci_dev *pdev, int pasids) 739int amd_iommu_init_device(struct pci_dev *pdev, int pasids)
732{ 740{
733 struct device_state *dev_state; 741 struct device_state *dev_state;
742 struct iommu_group *group;
734 unsigned long flags; 743 unsigned long flags;
735 int ret, tmp; 744 int ret, tmp;
736 u16 devid; 745 u16 devid;
@@ -776,10 +785,16 @@ int amd_iommu_init_device(struct pci_dev *pdev, int pasids)
776 if (ret) 785 if (ret)
777 goto out_free_domain; 786 goto out_free_domain;
778 787
779 ret = iommu_attach_device(dev_state->domain, &pdev->dev); 788 group = iommu_group_get(&pdev->dev);
780 if (ret != 0) 789 if (!group)
781 goto out_free_domain; 790 goto out_free_domain;
782 791
792 ret = iommu_attach_group(dev_state->domain, group);
793 if (ret != 0)
794 goto out_drop_group;
795
796 iommu_group_put(group);
797
783 spin_lock_irqsave(&state_lock, flags); 798 spin_lock_irqsave(&state_lock, flags);
784 799
785 if (__get_device_state(devid) != NULL) { 800 if (__get_device_state(devid) != NULL) {
@@ -794,6 +809,9 @@ int amd_iommu_init_device(struct pci_dev *pdev, int pasids)
794 809
795 return 0; 810 return 0;
796 811
812out_drop_group:
813 iommu_group_put(group);
814
797out_free_domain: 815out_free_domain:
798 iommu_domain_free(dev_state->domain); 816 iommu_domain_free(dev_state->domain);
799 817
diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c
new file mode 100644
index 000000000000..da902baaa794
--- /dev/null
+++ b/drivers/iommu/arm-smmu-v3.c
@@ -0,0 +1,2701 @@
1/*
2 * IOMMU API for ARM architected SMMUv3 implementations.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * along with this program. If not, see <http://www.gnu.org/licenses/>.
15 *
16 * Copyright (C) 2015 ARM Limited
17 *
18 * Author: Will Deacon <will.deacon@arm.com>
19 *
20 * This driver is powered by bad coffee and bombay mix.
21 */
22
23#include <linux/delay.h>
24#include <linux/err.h>
25#include <linux/interrupt.h>
26#include <linux/iommu.h>
27#include <linux/iopoll.h>
28#include <linux/module.h>
29#include <linux/of.h>
30#include <linux/of_address.h>
31#include <linux/pci.h>
32#include <linux/platform_device.h>
33
34#include "io-pgtable.h"
35
36/* MMIO registers */
37#define ARM_SMMU_IDR0 0x0
38#define IDR0_ST_LVL_SHIFT 27
39#define IDR0_ST_LVL_MASK 0x3
40#define IDR0_ST_LVL_2LVL (1 << IDR0_ST_LVL_SHIFT)
41#define IDR0_STALL_MODEL (3 << 24)
42#define IDR0_TTENDIAN_SHIFT 21
43#define IDR0_TTENDIAN_MASK 0x3
44#define IDR0_TTENDIAN_LE (2 << IDR0_TTENDIAN_SHIFT)
45#define IDR0_TTENDIAN_BE (3 << IDR0_TTENDIAN_SHIFT)
46#define IDR0_TTENDIAN_MIXED (0 << IDR0_TTENDIAN_SHIFT)
47#define IDR0_CD2L (1 << 19)
48#define IDR0_VMID16 (1 << 18)
49#define IDR0_PRI (1 << 16)
50#define IDR0_SEV (1 << 14)
51#define IDR0_MSI (1 << 13)
52#define IDR0_ASID16 (1 << 12)
53#define IDR0_ATS (1 << 10)
54#define IDR0_HYP (1 << 9)
55#define IDR0_COHACC (1 << 4)
56#define IDR0_TTF_SHIFT 2
57#define IDR0_TTF_MASK 0x3
58#define IDR0_TTF_AARCH64 (2 << IDR0_TTF_SHIFT)
59#define IDR0_S1P (1 << 1)
60#define IDR0_S2P (1 << 0)
61
62#define ARM_SMMU_IDR1 0x4
63#define IDR1_TABLES_PRESET (1 << 30)
64#define IDR1_QUEUES_PRESET (1 << 29)
65#define IDR1_REL (1 << 28)
66#define IDR1_CMDQ_SHIFT 21
67#define IDR1_CMDQ_MASK 0x1f
68#define IDR1_EVTQ_SHIFT 16
69#define IDR1_EVTQ_MASK 0x1f
70#define IDR1_PRIQ_SHIFT 11
71#define IDR1_PRIQ_MASK 0x1f
72#define IDR1_SSID_SHIFT 6
73#define IDR1_SSID_MASK 0x1f
74#define IDR1_SID_SHIFT 0
75#define IDR1_SID_MASK 0x3f
76
77#define ARM_SMMU_IDR5 0x14
78#define IDR5_STALL_MAX_SHIFT 16
79#define IDR5_STALL_MAX_MASK 0xffff
80#define IDR5_GRAN64K (1 << 6)
81#define IDR5_GRAN16K (1 << 5)
82#define IDR5_GRAN4K (1 << 4)
83#define IDR5_OAS_SHIFT 0
84#define IDR5_OAS_MASK 0x7
85#define IDR5_OAS_32_BIT (0 << IDR5_OAS_SHIFT)
86#define IDR5_OAS_36_BIT (1 << IDR5_OAS_SHIFT)
87#define IDR5_OAS_40_BIT (2 << IDR5_OAS_SHIFT)
88#define IDR5_OAS_42_BIT (3 << IDR5_OAS_SHIFT)
89#define IDR5_OAS_44_BIT (4 << IDR5_OAS_SHIFT)
90#define IDR5_OAS_48_BIT (5 << IDR5_OAS_SHIFT)
91
92#define ARM_SMMU_CR0 0x20
93#define CR0_CMDQEN (1 << 3)
94#define CR0_EVTQEN (1 << 2)
95#define CR0_PRIQEN (1 << 1)
96#define CR0_SMMUEN (1 << 0)
97
98#define ARM_SMMU_CR0ACK 0x24
99
100#define ARM_SMMU_CR1 0x28
101#define CR1_SH_NSH 0
102#define CR1_SH_OSH 2
103#define CR1_SH_ISH 3
104#define CR1_CACHE_NC 0
105#define CR1_CACHE_WB 1
106#define CR1_CACHE_WT 2
107#define CR1_TABLE_SH_SHIFT 10
108#define CR1_TABLE_OC_SHIFT 8
109#define CR1_TABLE_IC_SHIFT 6
110#define CR1_QUEUE_SH_SHIFT 4
111#define CR1_QUEUE_OC_SHIFT 2
112#define CR1_QUEUE_IC_SHIFT 0
113
114#define ARM_SMMU_CR2 0x2c
115#define CR2_PTM (1 << 2)
116#define CR2_RECINVSID (1 << 1)
117#define CR2_E2H (1 << 0)
118
119#define ARM_SMMU_IRQ_CTRL 0x50
120#define IRQ_CTRL_EVTQ_IRQEN (1 << 2)
121#define IRQ_CTRL_GERROR_IRQEN (1 << 0)
122
123#define ARM_SMMU_IRQ_CTRLACK 0x54
124
125#define ARM_SMMU_GERROR 0x60
126#define GERROR_SFM_ERR (1 << 8)
127#define GERROR_MSI_GERROR_ABT_ERR (1 << 7)
128#define GERROR_MSI_PRIQ_ABT_ERR (1 << 6)
129#define GERROR_MSI_EVTQ_ABT_ERR (1 << 5)
130#define GERROR_MSI_CMDQ_ABT_ERR (1 << 4)
131#define GERROR_PRIQ_ABT_ERR (1 << 3)
132#define GERROR_EVTQ_ABT_ERR (1 << 2)
133#define GERROR_CMDQ_ERR (1 << 0)
134#define GERROR_ERR_MASK 0xfd
135
136#define ARM_SMMU_GERRORN 0x64
137
138#define ARM_SMMU_GERROR_IRQ_CFG0 0x68
139#define ARM_SMMU_GERROR_IRQ_CFG1 0x70
140#define ARM_SMMU_GERROR_IRQ_CFG2 0x74
141
142#define ARM_SMMU_STRTAB_BASE 0x80
143#define STRTAB_BASE_RA (1UL << 62)
144#define STRTAB_BASE_ADDR_SHIFT 6
145#define STRTAB_BASE_ADDR_MASK 0x3ffffffffffUL
146
147#define ARM_SMMU_STRTAB_BASE_CFG 0x88
148#define STRTAB_BASE_CFG_LOG2SIZE_SHIFT 0
149#define STRTAB_BASE_CFG_LOG2SIZE_MASK 0x3f
150#define STRTAB_BASE_CFG_SPLIT_SHIFT 6
151#define STRTAB_BASE_CFG_SPLIT_MASK 0x1f
152#define STRTAB_BASE_CFG_FMT_SHIFT 16
153#define STRTAB_BASE_CFG_FMT_MASK 0x3
154#define STRTAB_BASE_CFG_FMT_LINEAR (0 << STRTAB_BASE_CFG_FMT_SHIFT)
155#define STRTAB_BASE_CFG_FMT_2LVL (1 << STRTAB_BASE_CFG_FMT_SHIFT)
156
157#define ARM_SMMU_CMDQ_BASE 0x90
158#define ARM_SMMU_CMDQ_PROD 0x98
159#define ARM_SMMU_CMDQ_CONS 0x9c
160
161#define ARM_SMMU_EVTQ_BASE 0xa0
162#define ARM_SMMU_EVTQ_PROD 0x100a8
163#define ARM_SMMU_EVTQ_CONS 0x100ac
164#define ARM_SMMU_EVTQ_IRQ_CFG0 0xb0
165#define ARM_SMMU_EVTQ_IRQ_CFG1 0xb8
166#define ARM_SMMU_EVTQ_IRQ_CFG2 0xbc
167
168#define ARM_SMMU_PRIQ_BASE 0xc0
169#define ARM_SMMU_PRIQ_PROD 0x100c8
170#define ARM_SMMU_PRIQ_CONS 0x100cc
171#define ARM_SMMU_PRIQ_IRQ_CFG0 0xd0
172#define ARM_SMMU_PRIQ_IRQ_CFG1 0xd8
173#define ARM_SMMU_PRIQ_IRQ_CFG2 0xdc
174
175/* Common MSI config fields */
176#define MSI_CFG0_SH_SHIFT 60
177#define MSI_CFG0_SH_NSH (0UL << MSI_CFG0_SH_SHIFT)
178#define MSI_CFG0_SH_OSH (2UL << MSI_CFG0_SH_SHIFT)
179#define MSI_CFG0_SH_ISH (3UL << MSI_CFG0_SH_SHIFT)
180#define MSI_CFG0_MEMATTR_SHIFT 56
181#define MSI_CFG0_MEMATTR_DEVICE_nGnRE (0x1 << MSI_CFG0_MEMATTR_SHIFT)
182#define MSI_CFG0_ADDR_SHIFT 2
183#define MSI_CFG0_ADDR_MASK 0x3fffffffffffUL
184
185#define Q_IDX(q, p) ((p) & ((1 << (q)->max_n_shift) - 1))
186#define Q_WRP(q, p) ((p) & (1 << (q)->max_n_shift))
187#define Q_OVERFLOW_FLAG (1 << 31)
188#define Q_OVF(q, p) ((p) & Q_OVERFLOW_FLAG)
189#define Q_ENT(q, p) ((q)->base + \
190 Q_IDX(q, p) * (q)->ent_dwords)
191
192#define Q_BASE_RWA (1UL << 62)
193#define Q_BASE_ADDR_SHIFT 5
194#define Q_BASE_ADDR_MASK 0xfffffffffffUL
195#define Q_BASE_LOG2SIZE_SHIFT 0
196#define Q_BASE_LOG2SIZE_MASK 0x1fUL
197
198/*
199 * Stream table.
200 *
201 * Linear: Enough to cover 1 << IDR1.SIDSIZE entries
202 * 2lvl: 128k L1 entries,
203 * 256 lazy entries per table (each table covers a PCI bus)
204 */
205#define STRTAB_L1_SZ_SHIFT 20
206#define STRTAB_SPLIT 8
207
208#define STRTAB_L1_DESC_DWORDS 1
209#define STRTAB_L1_DESC_SPAN_SHIFT 0
210#define STRTAB_L1_DESC_SPAN_MASK 0x1fUL
211#define STRTAB_L1_DESC_L2PTR_SHIFT 6
212#define STRTAB_L1_DESC_L2PTR_MASK 0x3ffffffffffUL
213
214#define STRTAB_STE_DWORDS 8
215#define STRTAB_STE_0_V (1UL << 0)
216#define STRTAB_STE_0_CFG_SHIFT 1
217#define STRTAB_STE_0_CFG_MASK 0x7UL
218#define STRTAB_STE_0_CFG_ABORT (0UL << STRTAB_STE_0_CFG_SHIFT)
219#define STRTAB_STE_0_CFG_BYPASS (4UL << STRTAB_STE_0_CFG_SHIFT)
220#define STRTAB_STE_0_CFG_S1_TRANS (5UL << STRTAB_STE_0_CFG_SHIFT)
221#define STRTAB_STE_0_CFG_S2_TRANS (6UL << STRTAB_STE_0_CFG_SHIFT)
222
223#define STRTAB_STE_0_S1FMT_SHIFT 4
224#define STRTAB_STE_0_S1FMT_LINEAR (0UL << STRTAB_STE_0_S1FMT_SHIFT)
225#define STRTAB_STE_0_S1CTXPTR_SHIFT 6
226#define STRTAB_STE_0_S1CTXPTR_MASK 0x3ffffffffffUL
227#define STRTAB_STE_0_S1CDMAX_SHIFT 59
228#define STRTAB_STE_0_S1CDMAX_MASK 0x1fUL
229
230#define STRTAB_STE_1_S1C_CACHE_NC 0UL
231#define STRTAB_STE_1_S1C_CACHE_WBRA 1UL
232#define STRTAB_STE_1_S1C_CACHE_WT 2UL
233#define STRTAB_STE_1_S1C_CACHE_WB 3UL
234#define STRTAB_STE_1_S1C_SH_NSH 0UL
235#define STRTAB_STE_1_S1C_SH_OSH 2UL
236#define STRTAB_STE_1_S1C_SH_ISH 3UL
237#define STRTAB_STE_1_S1CIR_SHIFT 2
238#define STRTAB_STE_1_S1COR_SHIFT 4
239#define STRTAB_STE_1_S1CSH_SHIFT 6
240
241#define STRTAB_STE_1_S1STALLD (1UL << 27)
242
243#define STRTAB_STE_1_EATS_ABT 0UL
244#define STRTAB_STE_1_EATS_TRANS 1UL
245#define STRTAB_STE_1_EATS_S1CHK 2UL
246#define STRTAB_STE_1_EATS_SHIFT 28
247
248#define STRTAB_STE_1_STRW_NSEL1 0UL
249#define STRTAB_STE_1_STRW_EL2 2UL
250#define STRTAB_STE_1_STRW_SHIFT 30
251
252#define STRTAB_STE_2_S2VMID_SHIFT 0
253#define STRTAB_STE_2_S2VMID_MASK 0xffffUL
254#define STRTAB_STE_2_VTCR_SHIFT 32
255#define STRTAB_STE_2_VTCR_MASK 0x7ffffUL
256#define STRTAB_STE_2_S2AA64 (1UL << 51)
257#define STRTAB_STE_2_S2ENDI (1UL << 52)
258#define STRTAB_STE_2_S2PTW (1UL << 54)
259#define STRTAB_STE_2_S2R (1UL << 58)
260
261#define STRTAB_STE_3_S2TTB_SHIFT 4
262#define STRTAB_STE_3_S2TTB_MASK 0xfffffffffffUL
263
264/* Context descriptor (stage-1 only) */
265#define CTXDESC_CD_DWORDS 8
266#define CTXDESC_CD_0_TCR_T0SZ_SHIFT 0
267#define ARM64_TCR_T0SZ_SHIFT 0
268#define ARM64_TCR_T0SZ_MASK 0x1fUL
269#define CTXDESC_CD_0_TCR_TG0_SHIFT 6
270#define ARM64_TCR_TG0_SHIFT 14
271#define ARM64_TCR_TG0_MASK 0x3UL
272#define CTXDESC_CD_0_TCR_IRGN0_SHIFT 8
273#define ARM64_TCR_IRGN0_SHIFT 8
274#define ARM64_TCR_IRGN0_MASK 0x3UL
275#define CTXDESC_CD_0_TCR_ORGN0_SHIFT 10
276#define ARM64_TCR_ORGN0_SHIFT 10
277#define ARM64_TCR_ORGN0_MASK 0x3UL
278#define CTXDESC_CD_0_TCR_SH0_SHIFT 12
279#define ARM64_TCR_SH0_SHIFT 12
280#define ARM64_TCR_SH0_MASK 0x3UL
281#define CTXDESC_CD_0_TCR_EPD0_SHIFT 14
282#define ARM64_TCR_EPD0_SHIFT 7
283#define ARM64_TCR_EPD0_MASK 0x1UL
284#define CTXDESC_CD_0_TCR_EPD1_SHIFT 30
285#define ARM64_TCR_EPD1_SHIFT 23
286#define ARM64_TCR_EPD1_MASK 0x1UL
287
288#define CTXDESC_CD_0_ENDI (1UL << 15)
289#define CTXDESC_CD_0_V (1UL << 31)
290
291#define CTXDESC_CD_0_TCR_IPS_SHIFT 32
292#define ARM64_TCR_IPS_SHIFT 32
293#define ARM64_TCR_IPS_MASK 0x7UL
294#define CTXDESC_CD_0_TCR_TBI0_SHIFT 38
295#define ARM64_TCR_TBI0_SHIFT 37
296#define ARM64_TCR_TBI0_MASK 0x1UL
297
298#define CTXDESC_CD_0_AA64 (1UL << 41)
299#define CTXDESC_CD_0_R (1UL << 45)
300#define CTXDESC_CD_0_A (1UL << 46)
301#define CTXDESC_CD_0_ASET_SHIFT 47
302#define CTXDESC_CD_0_ASET_SHARED (0UL << CTXDESC_CD_0_ASET_SHIFT)
303#define CTXDESC_CD_0_ASET_PRIVATE (1UL << CTXDESC_CD_0_ASET_SHIFT)
304#define CTXDESC_CD_0_ASID_SHIFT 48
305#define CTXDESC_CD_0_ASID_MASK 0xffffUL
306
307#define CTXDESC_CD_1_TTB0_SHIFT 4
308#define CTXDESC_CD_1_TTB0_MASK 0xfffffffffffUL
309
310#define CTXDESC_CD_3_MAIR_SHIFT 0
311
312/* Convert between AArch64 (CPU) TCR format and SMMU CD format */
313#define ARM_SMMU_TCR2CD(tcr, fld) \
314 (((tcr) >> ARM64_TCR_##fld##_SHIFT & ARM64_TCR_##fld##_MASK) \
315 << CTXDESC_CD_0_TCR_##fld##_SHIFT)
316
317/* Command queue */
318#define CMDQ_ENT_DWORDS 2
319#define CMDQ_MAX_SZ_SHIFT 8
320
321#define CMDQ_ERR_SHIFT 24
322#define CMDQ_ERR_MASK 0x7f
323#define CMDQ_ERR_CERROR_NONE_IDX 0
324#define CMDQ_ERR_CERROR_ILL_IDX 1
325#define CMDQ_ERR_CERROR_ABT_IDX 2
326
327#define CMDQ_0_OP_SHIFT 0
328#define CMDQ_0_OP_MASK 0xffUL
329#define CMDQ_0_SSV (1UL << 11)
330
331#define CMDQ_PREFETCH_0_SID_SHIFT 32
332#define CMDQ_PREFETCH_1_SIZE_SHIFT 0
333#define CMDQ_PREFETCH_1_ADDR_MASK ~0xfffUL
334
335#define CMDQ_CFGI_0_SID_SHIFT 32
336#define CMDQ_CFGI_0_SID_MASK 0xffffffffUL
337#define CMDQ_CFGI_1_LEAF (1UL << 0)
338#define CMDQ_CFGI_1_RANGE_SHIFT 0
339#define CMDQ_CFGI_1_RANGE_MASK 0x1fUL
340
341#define CMDQ_TLBI_0_VMID_SHIFT 32
342#define CMDQ_TLBI_0_ASID_SHIFT 48
343#define CMDQ_TLBI_1_LEAF (1UL << 0)
344#define CMDQ_TLBI_1_ADDR_MASK ~0xfffUL
345
346#define CMDQ_PRI_0_SSID_SHIFT 12
347#define CMDQ_PRI_0_SSID_MASK 0xfffffUL
348#define CMDQ_PRI_0_SID_SHIFT 32
349#define CMDQ_PRI_0_SID_MASK 0xffffffffUL
350#define CMDQ_PRI_1_GRPID_SHIFT 0
351#define CMDQ_PRI_1_GRPID_MASK 0x1ffUL
352#define CMDQ_PRI_1_RESP_SHIFT 12
353#define CMDQ_PRI_1_RESP_DENY (0UL << CMDQ_PRI_1_RESP_SHIFT)
354#define CMDQ_PRI_1_RESP_FAIL (1UL << CMDQ_PRI_1_RESP_SHIFT)
355#define CMDQ_PRI_1_RESP_SUCC (2UL << CMDQ_PRI_1_RESP_SHIFT)
356
357#define CMDQ_SYNC_0_CS_SHIFT 12
358#define CMDQ_SYNC_0_CS_NONE (0UL << CMDQ_SYNC_0_CS_SHIFT)
359#define CMDQ_SYNC_0_CS_SEV (2UL << CMDQ_SYNC_0_CS_SHIFT)
360
361/* Event queue */
362#define EVTQ_ENT_DWORDS 4
363#define EVTQ_MAX_SZ_SHIFT 7
364
365#define EVTQ_0_ID_SHIFT 0
366#define EVTQ_0_ID_MASK 0xffUL
367
368/* PRI queue */
369#define PRIQ_ENT_DWORDS 2
370#define PRIQ_MAX_SZ_SHIFT 8
371
372#define PRIQ_0_SID_SHIFT 0
373#define PRIQ_0_SID_MASK 0xffffffffUL
374#define PRIQ_0_SSID_SHIFT 32
375#define PRIQ_0_SSID_MASK 0xfffffUL
376#define PRIQ_0_OF (1UL << 57)
377#define PRIQ_0_PERM_PRIV (1UL << 58)
378#define PRIQ_0_PERM_EXEC (1UL << 59)
379#define PRIQ_0_PERM_READ (1UL << 60)
380#define PRIQ_0_PERM_WRITE (1UL << 61)
381#define PRIQ_0_PRG_LAST (1UL << 62)
382#define PRIQ_0_SSID_V (1UL << 63)
383
384#define PRIQ_1_PRG_IDX_SHIFT 0
385#define PRIQ_1_PRG_IDX_MASK 0x1ffUL
386#define PRIQ_1_ADDR_SHIFT 12
387#define PRIQ_1_ADDR_MASK 0xfffffffffffffUL
388
389/* High-level queue structures */
390#define ARM_SMMU_POLL_TIMEOUT_US 100
391
392static bool disable_bypass;
393module_param_named(disable_bypass, disable_bypass, bool, S_IRUGO);
394MODULE_PARM_DESC(disable_bypass,
395 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
396
397enum pri_resp {
398 PRI_RESP_DENY,
399 PRI_RESP_FAIL,
400 PRI_RESP_SUCC,
401};
402
403struct arm_smmu_cmdq_ent {
404 /* Common fields */
405 u8 opcode;
406 bool substream_valid;
407
408 /* Command-specific fields */
409 union {
410 #define CMDQ_OP_PREFETCH_CFG 0x1
411 struct {
412 u32 sid;
413 u8 size;
414 u64 addr;
415 } prefetch;
416
417 #define CMDQ_OP_CFGI_STE 0x3
418 #define CMDQ_OP_CFGI_ALL 0x4
419 struct {
420 u32 sid;
421 union {
422 bool leaf;
423 u8 span;
424 };
425 } cfgi;
426
427 #define CMDQ_OP_TLBI_NH_ASID 0x11
428 #define CMDQ_OP_TLBI_NH_VA 0x12
429 #define CMDQ_OP_TLBI_EL2_ALL 0x20
430 #define CMDQ_OP_TLBI_S12_VMALL 0x28
431 #define CMDQ_OP_TLBI_S2_IPA 0x2a
432 #define CMDQ_OP_TLBI_NSNH_ALL 0x30
433 struct {
434 u16 asid;
435 u16 vmid;
436 bool leaf;
437 u64 addr;
438 } tlbi;
439
440 #define CMDQ_OP_PRI_RESP 0x41
441 struct {
442 u32 sid;
443 u32 ssid;
444 u16 grpid;
445 enum pri_resp resp;
446 } pri;
447
448 #define CMDQ_OP_CMD_SYNC 0x46
449 };
450};
451
452struct arm_smmu_queue {
453 int irq; /* Wired interrupt */
454
455 __le64 *base;
456 dma_addr_t base_dma;
457 u64 q_base;
458
459 size_t ent_dwords;
460 u32 max_n_shift;
461 u32 prod;
462 u32 cons;
463
464 u32 __iomem *prod_reg;
465 u32 __iomem *cons_reg;
466};
467
468struct arm_smmu_cmdq {
469 struct arm_smmu_queue q;
470 spinlock_t lock;
471};
472
473struct arm_smmu_evtq {
474 struct arm_smmu_queue q;
475 u32 max_stalls;
476};
477
478struct arm_smmu_priq {
479 struct arm_smmu_queue q;
480};
481
482/* High-level stream table and context descriptor structures */
483struct arm_smmu_strtab_l1_desc {
484 u8 span;
485
486 __le64 *l2ptr;
487 dma_addr_t l2ptr_dma;
488};
489
490struct arm_smmu_s1_cfg {
491 __le64 *cdptr;
492 dma_addr_t cdptr_dma;
493
494 struct arm_smmu_ctx_desc {
495 u16 asid;
496 u64 ttbr;
497 u64 tcr;
498 u64 mair;
499 } cd;
500};
501
502struct arm_smmu_s2_cfg {
503 u16 vmid;
504 u64 vttbr;
505 u64 vtcr;
506};
507
508struct arm_smmu_strtab_ent {
509 bool valid;
510
511 bool bypass; /* Overrides s1/s2 config */
512 struct arm_smmu_s1_cfg *s1_cfg;
513 struct arm_smmu_s2_cfg *s2_cfg;
514};
515
516struct arm_smmu_strtab_cfg {
517 __le64 *strtab;
518 dma_addr_t strtab_dma;
519 struct arm_smmu_strtab_l1_desc *l1_desc;
520 unsigned int num_l1_ents;
521
522 u64 strtab_base;
523 u32 strtab_base_cfg;
524};
525
526/* An SMMUv3 instance */
527struct arm_smmu_device {
528 struct device *dev;
529 void __iomem *base;
530
531#define ARM_SMMU_FEAT_2_LVL_STRTAB (1 << 0)
532#define ARM_SMMU_FEAT_2_LVL_CDTAB (1 << 1)
533#define ARM_SMMU_FEAT_TT_LE (1 << 2)
534#define ARM_SMMU_FEAT_TT_BE (1 << 3)
535#define ARM_SMMU_FEAT_PRI (1 << 4)
536#define ARM_SMMU_FEAT_ATS (1 << 5)
537#define ARM_SMMU_FEAT_SEV (1 << 6)
538#define ARM_SMMU_FEAT_MSI (1 << 7)
539#define ARM_SMMU_FEAT_COHERENCY (1 << 8)
540#define ARM_SMMU_FEAT_TRANS_S1 (1 << 9)
541#define ARM_SMMU_FEAT_TRANS_S2 (1 << 10)
542#define ARM_SMMU_FEAT_STALLS (1 << 11)
543#define ARM_SMMU_FEAT_HYP (1 << 12)
544 u32 features;
545
546#define ARM_SMMU_OPT_SKIP_PREFETCH (1 << 0)
547 u32 options;
548
549 struct arm_smmu_cmdq cmdq;
550 struct arm_smmu_evtq evtq;
551 struct arm_smmu_priq priq;
552
553 int gerr_irq;
554
555 unsigned long ias; /* IPA */
556 unsigned long oas; /* PA */
557
558#define ARM_SMMU_MAX_ASIDS (1 << 16)
559 unsigned int asid_bits;
560 DECLARE_BITMAP(asid_map, ARM_SMMU_MAX_ASIDS);
561
562#define ARM_SMMU_MAX_VMIDS (1 << 16)
563 unsigned int vmid_bits;
564 DECLARE_BITMAP(vmid_map, ARM_SMMU_MAX_VMIDS);
565
566 unsigned int ssid_bits;
567 unsigned int sid_bits;
568
569 struct arm_smmu_strtab_cfg strtab_cfg;
570 struct list_head list;
571};
572
573/* SMMU private data for an IOMMU group */
574struct arm_smmu_group {
575 struct arm_smmu_device *smmu;
576 struct arm_smmu_domain *domain;
577 int num_sids;
578 u32 *sids;
579 struct arm_smmu_strtab_ent ste;
580};
581
582/* SMMU private data for an IOMMU domain */
583enum arm_smmu_domain_stage {
584 ARM_SMMU_DOMAIN_S1 = 0,
585 ARM_SMMU_DOMAIN_S2,
586 ARM_SMMU_DOMAIN_NESTED,
587};
588
589struct arm_smmu_domain {
590 struct arm_smmu_device *smmu;
591 struct mutex init_mutex; /* Protects smmu pointer */
592
593 struct io_pgtable_ops *pgtbl_ops;
594 spinlock_t pgtbl_lock;
595
596 enum arm_smmu_domain_stage stage;
597 union {
598 struct arm_smmu_s1_cfg s1_cfg;
599 struct arm_smmu_s2_cfg s2_cfg;
600 };
601
602 struct iommu_domain domain;
603};
604
605/* Our list of SMMU instances */
606static DEFINE_SPINLOCK(arm_smmu_devices_lock);
607static LIST_HEAD(arm_smmu_devices);
608
609struct arm_smmu_option_prop {
610 u32 opt;
611 const char *prop;
612};
613
614static struct arm_smmu_option_prop arm_smmu_options[] = {
615 { ARM_SMMU_OPT_SKIP_PREFETCH, "hisilicon,broken-prefetch-cmd" },
616 { 0, NULL},
617};
618
619static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
620{
621 return container_of(dom, struct arm_smmu_domain, domain);
622}
623
624static void parse_driver_options(struct arm_smmu_device *smmu)
625{
626 int i = 0;
627
628 do {
629 if (of_property_read_bool(smmu->dev->of_node,
630 arm_smmu_options[i].prop)) {
631 smmu->options |= arm_smmu_options[i].opt;
632 dev_notice(smmu->dev, "option %s\n",
633 arm_smmu_options[i].prop);
634 }
635 } while (arm_smmu_options[++i].opt);
636}
637
638/* Low-level queue manipulation functions */
639static bool queue_full(struct arm_smmu_queue *q)
640{
641 return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&
642 Q_WRP(q, q->prod) != Q_WRP(q, q->cons);
643}
644
645static bool queue_empty(struct arm_smmu_queue *q)
646{
647 return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&
648 Q_WRP(q, q->prod) == Q_WRP(q, q->cons);
649}
650
651static void queue_sync_cons(struct arm_smmu_queue *q)
652{
653 q->cons = readl_relaxed(q->cons_reg);
654}
655
656static void queue_inc_cons(struct arm_smmu_queue *q)
657{
658 u32 cons = (Q_WRP(q, q->cons) | Q_IDX(q, q->cons)) + 1;
659
660 q->cons = Q_OVF(q, q->cons) | Q_WRP(q, cons) | Q_IDX(q, cons);
661 writel(q->cons, q->cons_reg);
662}
663
664static int queue_sync_prod(struct arm_smmu_queue *q)
665{
666 int ret = 0;
667 u32 prod = readl_relaxed(q->prod_reg);
668
669 if (Q_OVF(q, prod) != Q_OVF(q, q->prod))
670 ret = -EOVERFLOW;
671
672 q->prod = prod;
673 return ret;
674}
675
676static void queue_inc_prod(struct arm_smmu_queue *q)
677{
678 u32 prod = (Q_WRP(q, q->prod) | Q_IDX(q, q->prod)) + 1;
679
680 q->prod = Q_OVF(q, q->prod) | Q_WRP(q, prod) | Q_IDX(q, prod);
681 writel(q->prod, q->prod_reg);
682}
683
684static bool __queue_cons_before(struct arm_smmu_queue *q, u32 until)
685{
686 if (Q_WRP(q, q->cons) == Q_WRP(q, until))
687 return Q_IDX(q, q->cons) < Q_IDX(q, until);
688
689 return Q_IDX(q, q->cons) >= Q_IDX(q, until);
690}
691
692static int queue_poll_cons(struct arm_smmu_queue *q, u32 until, bool wfe)
693{
694 ktime_t timeout = ktime_add_us(ktime_get(), ARM_SMMU_POLL_TIMEOUT_US);
695
696 while (queue_sync_cons(q), __queue_cons_before(q, until)) {
697 if (ktime_compare(ktime_get(), timeout) > 0)
698 return -ETIMEDOUT;
699
700 if (wfe) {
701 wfe();
702 } else {
703 cpu_relax();
704 udelay(1);
705 }
706 }
707
708 return 0;
709}
710
711static void queue_write(__le64 *dst, u64 *src, size_t n_dwords)
712{
713 int i;
714
715 for (i = 0; i < n_dwords; ++i)
716 *dst++ = cpu_to_le64(*src++);
717}
718
719static int queue_insert_raw(struct arm_smmu_queue *q, u64 *ent)
720{
721 if (queue_full(q))
722 return -ENOSPC;
723
724 queue_write(Q_ENT(q, q->prod), ent, q->ent_dwords);
725 queue_inc_prod(q);
726 return 0;
727}
728
729static void queue_read(__le64 *dst, u64 *src, size_t n_dwords)
730{
731 int i;
732
733 for (i = 0; i < n_dwords; ++i)
734 *dst++ = le64_to_cpu(*src++);
735}
736
737static int queue_remove_raw(struct arm_smmu_queue *q, u64 *ent)
738{
739 if (queue_empty(q))
740 return -EAGAIN;
741
742 queue_read(ent, Q_ENT(q, q->cons), q->ent_dwords);
743 queue_inc_cons(q);
744 return 0;
745}
746
747/* High-level queue accessors */
748static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct arm_smmu_cmdq_ent *ent)
749{
750 memset(cmd, 0, CMDQ_ENT_DWORDS << 3);
751 cmd[0] |= (ent->opcode & CMDQ_0_OP_MASK) << CMDQ_0_OP_SHIFT;
752
753 switch (ent->opcode) {
754 case CMDQ_OP_TLBI_EL2_ALL:
755 case CMDQ_OP_TLBI_NSNH_ALL:
756 break;
757 case CMDQ_OP_PREFETCH_CFG:
758 cmd[0] |= (u64)ent->prefetch.sid << CMDQ_PREFETCH_0_SID_SHIFT;
759 cmd[1] |= ent->prefetch.size << CMDQ_PREFETCH_1_SIZE_SHIFT;
760 cmd[1] |= ent->prefetch.addr & CMDQ_PREFETCH_1_ADDR_MASK;
761 break;
762 case CMDQ_OP_CFGI_STE:
763 cmd[0] |= (u64)ent->cfgi.sid << CMDQ_CFGI_0_SID_SHIFT;
764 cmd[1] |= ent->cfgi.leaf ? CMDQ_CFGI_1_LEAF : 0;
765 break;
766 case CMDQ_OP_CFGI_ALL:
767 /* Cover the entire SID range */
768 cmd[1] |= CMDQ_CFGI_1_RANGE_MASK << CMDQ_CFGI_1_RANGE_SHIFT;
769 break;
770 case CMDQ_OP_TLBI_NH_VA:
771 cmd[0] |= (u64)ent->tlbi.asid << CMDQ_TLBI_0_ASID_SHIFT;
772 /* Fallthrough */
773 case CMDQ_OP_TLBI_S2_IPA:
774 cmd[0] |= (u64)ent->tlbi.vmid << CMDQ_TLBI_0_VMID_SHIFT;
775 cmd[1] |= ent->tlbi.leaf ? CMDQ_TLBI_1_LEAF : 0;
776 cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_ADDR_MASK;
777 break;
778 case CMDQ_OP_TLBI_NH_ASID:
779 cmd[0] |= (u64)ent->tlbi.asid << CMDQ_TLBI_0_ASID_SHIFT;
780 /* Fallthrough */
781 case CMDQ_OP_TLBI_S12_VMALL:
782 cmd[0] |= (u64)ent->tlbi.vmid << CMDQ_TLBI_0_VMID_SHIFT;
783 break;
784 case CMDQ_OP_PRI_RESP:
785 cmd[0] |= ent->substream_valid ? CMDQ_0_SSV : 0;
786 cmd[0] |= ent->pri.ssid << CMDQ_PRI_0_SSID_SHIFT;
787 cmd[0] |= (u64)ent->pri.sid << CMDQ_PRI_0_SID_SHIFT;
788 cmd[1] |= ent->pri.grpid << CMDQ_PRI_1_GRPID_SHIFT;
789 switch (ent->pri.resp) {
790 case PRI_RESP_DENY:
791 cmd[1] |= CMDQ_PRI_1_RESP_DENY;
792 break;
793 case PRI_RESP_FAIL:
794 cmd[1] |= CMDQ_PRI_1_RESP_FAIL;
795 break;
796 case PRI_RESP_SUCC:
797 cmd[1] |= CMDQ_PRI_1_RESP_SUCC;
798 break;
799 default:
800 return -EINVAL;
801 }
802 break;
803 case CMDQ_OP_CMD_SYNC:
804 cmd[0] |= CMDQ_SYNC_0_CS_SEV;
805 break;
806 default:
807 return -ENOENT;
808 }
809
810 return 0;
811}
812
813static void arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu)
814{
815 static const char *cerror_str[] = {
816 [CMDQ_ERR_CERROR_NONE_IDX] = "No error",
817 [CMDQ_ERR_CERROR_ILL_IDX] = "Illegal command",
818 [CMDQ_ERR_CERROR_ABT_IDX] = "Abort on command fetch",
819 };
820
821 int i;
822 u64 cmd[CMDQ_ENT_DWORDS];
823 struct arm_smmu_queue *q = &smmu->cmdq.q;
824 u32 cons = readl_relaxed(q->cons_reg);
825 u32 idx = cons >> CMDQ_ERR_SHIFT & CMDQ_ERR_MASK;
826 struct arm_smmu_cmdq_ent cmd_sync = {
827 .opcode = CMDQ_OP_CMD_SYNC,
828 };
829
830 dev_err(smmu->dev, "CMDQ error (cons 0x%08x): %s\n", cons,
831 cerror_str[idx]);
832
833 switch (idx) {
834 case CMDQ_ERR_CERROR_ILL_IDX:
835 break;
836 case CMDQ_ERR_CERROR_ABT_IDX:
837 dev_err(smmu->dev, "retrying command fetch\n");
838 case CMDQ_ERR_CERROR_NONE_IDX:
839 return;
840 }
841
842 /*
843 * We may have concurrent producers, so we need to be careful
844 * not to touch any of the shadow cmdq state.
845 */
846 queue_read(cmd, Q_ENT(q, idx), q->ent_dwords);
847 dev_err(smmu->dev, "skipping command in error state:\n");
848 for (i = 0; i < ARRAY_SIZE(cmd); ++i)
849 dev_err(smmu->dev, "\t0x%016llx\n", (unsigned long long)cmd[i]);
850
851 /* Convert the erroneous command into a CMD_SYNC */
852 if (arm_smmu_cmdq_build_cmd(cmd, &cmd_sync)) {
853 dev_err(smmu->dev, "failed to convert to CMD_SYNC\n");
854 return;
855 }
856
857 queue_write(cmd, Q_ENT(q, idx), q->ent_dwords);
858}
859
860static void arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu,
861 struct arm_smmu_cmdq_ent *ent)
862{
863 u32 until;
864 u64 cmd[CMDQ_ENT_DWORDS];
865 bool wfe = !!(smmu->features & ARM_SMMU_FEAT_SEV);
866 struct arm_smmu_queue *q = &smmu->cmdq.q;
867
868 if (arm_smmu_cmdq_build_cmd(cmd, ent)) {
869 dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n",
870 ent->opcode);
871 return;
872 }
873
874 spin_lock(&smmu->cmdq.lock);
875 while (until = q->prod + 1, queue_insert_raw(q, cmd) == -ENOSPC) {
876 /*
877 * Keep the queue locked, otherwise the producer could wrap
878 * twice and we could see a future consumer pointer that looks
879 * like it's behind us.
880 */
881 if (queue_poll_cons(q, until, wfe))
882 dev_err_ratelimited(smmu->dev, "CMDQ timeout\n");
883 }
884
885 if (ent->opcode == CMDQ_OP_CMD_SYNC && queue_poll_cons(q, until, wfe))
886 dev_err_ratelimited(smmu->dev, "CMD_SYNC timeout\n");
887 spin_unlock(&smmu->cmdq.lock);
888}
889
890/* Context descriptor manipulation functions */
891static u64 arm_smmu_cpu_tcr_to_cd(u64 tcr)
892{
893 u64 val = 0;
894
895 /* Repack the TCR. Just care about TTBR0 for now */
896 val |= ARM_SMMU_TCR2CD(tcr, T0SZ);
897 val |= ARM_SMMU_TCR2CD(tcr, TG0);
898 val |= ARM_SMMU_TCR2CD(tcr, IRGN0);
899 val |= ARM_SMMU_TCR2CD(tcr, ORGN0);
900 val |= ARM_SMMU_TCR2CD(tcr, SH0);
901 val |= ARM_SMMU_TCR2CD(tcr, EPD0);
902 val |= ARM_SMMU_TCR2CD(tcr, EPD1);
903 val |= ARM_SMMU_TCR2CD(tcr, IPS);
904 val |= ARM_SMMU_TCR2CD(tcr, TBI0);
905
906 return val;
907}
908
909static void arm_smmu_write_ctx_desc(struct arm_smmu_device *smmu,
910 struct arm_smmu_s1_cfg *cfg)
911{
912 u64 val;
913
914 /*
915 * We don't need to issue any invalidation here, as we'll invalidate
916 * the STE when installing the new entry anyway.
917 */
918 val = arm_smmu_cpu_tcr_to_cd(cfg->cd.tcr) |
919#ifdef __BIG_ENDIAN
920 CTXDESC_CD_0_ENDI |
921#endif
922 CTXDESC_CD_0_R | CTXDESC_CD_0_A | CTXDESC_CD_0_ASET_PRIVATE |
923 CTXDESC_CD_0_AA64 | (u64)cfg->cd.asid << CTXDESC_CD_0_ASID_SHIFT |
924 CTXDESC_CD_0_V;
925 cfg->cdptr[0] = cpu_to_le64(val);
926
927 val = cfg->cd.ttbr & CTXDESC_CD_1_TTB0_MASK << CTXDESC_CD_1_TTB0_SHIFT;
928 cfg->cdptr[1] = cpu_to_le64(val);
929
930 cfg->cdptr[3] = cpu_to_le64(cfg->cd.mair << CTXDESC_CD_3_MAIR_SHIFT);
931}
932
933/* Stream table manipulation functions */
934static void
935arm_smmu_write_strtab_l1_desc(__le64 *dst, struct arm_smmu_strtab_l1_desc *desc)
936{
937 u64 val = 0;
938
939 val |= (desc->span & STRTAB_L1_DESC_SPAN_MASK)
940 << STRTAB_L1_DESC_SPAN_SHIFT;
941 val |= desc->l2ptr_dma &
942 STRTAB_L1_DESC_L2PTR_MASK << STRTAB_L1_DESC_L2PTR_SHIFT;
943
944 *dst = cpu_to_le64(val);
945}
946
947static void arm_smmu_sync_ste_for_sid(struct arm_smmu_device *smmu, u32 sid)
948{
949 struct arm_smmu_cmdq_ent cmd = {
950 .opcode = CMDQ_OP_CFGI_STE,
951 .cfgi = {
952 .sid = sid,
953 .leaf = true,
954 },
955 };
956
957 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
958 cmd.opcode = CMDQ_OP_CMD_SYNC;
959 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
960}
961
962static void arm_smmu_write_strtab_ent(struct arm_smmu_device *smmu, u32 sid,
963 __le64 *dst, struct arm_smmu_strtab_ent *ste)
964{
965 /*
966 * This is hideously complicated, but we only really care about
967 * three cases at the moment:
968 *
969 * 1. Invalid (all zero) -> bypass (init)
970 * 2. Bypass -> translation (attach)
971 * 3. Translation -> bypass (detach)
972 *
973 * Given that we can't update the STE atomically and the SMMU
974 * doesn't read the thing in a defined order, that leaves us
975 * with the following maintenance requirements:
976 *
977 * 1. Update Config, return (init time STEs aren't live)
978 * 2. Write everything apart from dword 0, sync, write dword 0, sync
979 * 3. Update Config, sync
980 */
981 u64 val = le64_to_cpu(dst[0]);
982 bool ste_live = false;
983 struct arm_smmu_cmdq_ent prefetch_cmd = {
984 .opcode = CMDQ_OP_PREFETCH_CFG,
985 .prefetch = {
986 .sid = sid,
987 },
988 };
989
990 if (val & STRTAB_STE_0_V) {
991 u64 cfg;
992
993 cfg = val & STRTAB_STE_0_CFG_MASK << STRTAB_STE_0_CFG_SHIFT;
994 switch (cfg) {
995 case STRTAB_STE_0_CFG_BYPASS:
996 break;
997 case STRTAB_STE_0_CFG_S1_TRANS:
998 case STRTAB_STE_0_CFG_S2_TRANS:
999 ste_live = true;
1000 break;
1001 default:
1002 BUG(); /* STE corruption */
1003 }
1004 }
1005
1006 /* Nuke the existing Config, as we're going to rewrite it */
1007 val &= ~(STRTAB_STE_0_CFG_MASK << STRTAB_STE_0_CFG_SHIFT);
1008
1009 if (ste->valid)
1010 val |= STRTAB_STE_0_V;
1011 else
1012 val &= ~STRTAB_STE_0_V;
1013
1014 if (ste->bypass) {
1015 val |= disable_bypass ? STRTAB_STE_0_CFG_ABORT
1016 : STRTAB_STE_0_CFG_BYPASS;
1017 dst[0] = cpu_to_le64(val);
1018 dst[2] = 0; /* Nuke the VMID */
1019 if (ste_live)
1020 arm_smmu_sync_ste_for_sid(smmu, sid);
1021 return;
1022 }
1023
1024 if (ste->s1_cfg) {
1025 BUG_ON(ste_live);
1026 dst[1] = cpu_to_le64(
1027 STRTAB_STE_1_S1C_CACHE_WBRA
1028 << STRTAB_STE_1_S1CIR_SHIFT |
1029 STRTAB_STE_1_S1C_CACHE_WBRA
1030 << STRTAB_STE_1_S1COR_SHIFT |
1031 STRTAB_STE_1_S1C_SH_ISH << STRTAB_STE_1_S1CSH_SHIFT |
1032 STRTAB_STE_1_S1STALLD |
1033#ifdef CONFIG_PCI_ATS
1034 STRTAB_STE_1_EATS_TRANS << STRTAB_STE_1_EATS_SHIFT |
1035#endif
1036 STRTAB_STE_1_STRW_NSEL1 << STRTAB_STE_1_STRW_SHIFT);
1037
1038 val |= (ste->s1_cfg->cdptr_dma & STRTAB_STE_0_S1CTXPTR_MASK
1039 << STRTAB_STE_0_S1CTXPTR_SHIFT) |
1040 STRTAB_STE_0_CFG_S1_TRANS;
1041
1042 }
1043
1044 if (ste->s2_cfg) {
1045 BUG_ON(ste_live);
1046 dst[2] = cpu_to_le64(
1047 ste->s2_cfg->vmid << STRTAB_STE_2_S2VMID_SHIFT |
1048 (ste->s2_cfg->vtcr & STRTAB_STE_2_VTCR_MASK)
1049 << STRTAB_STE_2_VTCR_SHIFT |
1050#ifdef __BIG_ENDIAN
1051 STRTAB_STE_2_S2ENDI |
1052#endif
1053 STRTAB_STE_2_S2PTW | STRTAB_STE_2_S2AA64 |
1054 STRTAB_STE_2_S2R);
1055
1056 dst[3] = cpu_to_le64(ste->s2_cfg->vttbr &
1057 STRTAB_STE_3_S2TTB_MASK << STRTAB_STE_3_S2TTB_SHIFT);
1058
1059 val |= STRTAB_STE_0_CFG_S2_TRANS;
1060 }
1061
1062 arm_smmu_sync_ste_for_sid(smmu, sid);
1063 dst[0] = cpu_to_le64(val);
1064 arm_smmu_sync_ste_for_sid(smmu, sid);
1065
1066 /* It's likely that we'll want to use the new STE soon */
1067 if (!(smmu->options & ARM_SMMU_OPT_SKIP_PREFETCH))
1068 arm_smmu_cmdq_issue_cmd(smmu, &prefetch_cmd);
1069}
1070
1071static void arm_smmu_init_bypass_stes(u64 *strtab, unsigned int nent)
1072{
1073 unsigned int i;
1074 struct arm_smmu_strtab_ent ste = {
1075 .valid = true,
1076 .bypass = true,
1077 };
1078
1079 for (i = 0; i < nent; ++i) {
1080 arm_smmu_write_strtab_ent(NULL, -1, strtab, &ste);
1081 strtab += STRTAB_STE_DWORDS;
1082 }
1083}
1084
1085static int arm_smmu_init_l2_strtab(struct arm_smmu_device *smmu, u32 sid)
1086{
1087 size_t size;
1088 void *strtab;
1089 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
1090 struct arm_smmu_strtab_l1_desc *desc = &cfg->l1_desc[sid >> STRTAB_SPLIT];
1091
1092 if (desc->l2ptr)
1093 return 0;
1094
1095 size = 1 << (STRTAB_SPLIT + ilog2(STRTAB_STE_DWORDS) + 3);
1096 strtab = &cfg->strtab[(sid >> STRTAB_SPLIT) * STRTAB_L1_DESC_DWORDS];
1097
1098 desc->span = STRTAB_SPLIT + 1;
1099 desc->l2ptr = dma_zalloc_coherent(smmu->dev, size, &desc->l2ptr_dma,
1100 GFP_KERNEL);
1101 if (!desc->l2ptr) {
1102 dev_err(smmu->dev,
1103 "failed to allocate l2 stream table for SID %u\n",
1104 sid);
1105 return -ENOMEM;
1106 }
1107
1108 arm_smmu_init_bypass_stes(desc->l2ptr, 1 << STRTAB_SPLIT);
1109 arm_smmu_write_strtab_l1_desc(strtab, desc);
1110 return 0;
1111}
1112
1113/* IRQ and event handlers */
1114static irqreturn_t arm_smmu_evtq_thread(int irq, void *dev)
1115{
1116 int i;
1117 struct arm_smmu_device *smmu = dev;
1118 struct arm_smmu_queue *q = &smmu->evtq.q;
1119 u64 evt[EVTQ_ENT_DWORDS];
1120
1121 while (!queue_remove_raw(q, evt)) {
1122 u8 id = evt[0] >> EVTQ_0_ID_SHIFT & EVTQ_0_ID_MASK;
1123
1124 dev_info(smmu->dev, "event 0x%02x received:\n", id);
1125 for (i = 0; i < ARRAY_SIZE(evt); ++i)
1126 dev_info(smmu->dev, "\t0x%016llx\n",
1127 (unsigned long long)evt[i]);
1128 }
1129
1130 /* Sync our overflow flag, as we believe we're up to speed */
1131 q->cons = Q_OVF(q, q->prod) | Q_WRP(q, q->cons) | Q_IDX(q, q->cons);
1132 return IRQ_HANDLED;
1133}
1134
1135static irqreturn_t arm_smmu_evtq_handler(int irq, void *dev)
1136{
1137 irqreturn_t ret = IRQ_WAKE_THREAD;
1138 struct arm_smmu_device *smmu = dev;
1139 struct arm_smmu_queue *q = &smmu->evtq.q;
1140
1141 /*
1142 * Not much we can do on overflow, so scream and pretend we're
1143 * trying harder.
1144 */
1145 if (queue_sync_prod(q) == -EOVERFLOW)
1146 dev_err(smmu->dev, "EVTQ overflow detected -- events lost\n");
1147 else if (queue_empty(q))
1148 ret = IRQ_NONE;
1149
1150 return ret;
1151}
1152
1153static irqreturn_t arm_smmu_priq_thread(int irq, void *dev)
1154{
1155 struct arm_smmu_device *smmu = dev;
1156 struct arm_smmu_queue *q = &smmu->priq.q;
1157 u64 evt[PRIQ_ENT_DWORDS];
1158
1159 while (!queue_remove_raw(q, evt)) {
1160 u32 sid, ssid;
1161 u16 grpid;
1162 bool ssv, last;
1163
1164 sid = evt[0] >> PRIQ_0_SID_SHIFT & PRIQ_0_SID_MASK;
1165 ssv = evt[0] & PRIQ_0_SSID_V;
1166 ssid = ssv ? evt[0] >> PRIQ_0_SSID_SHIFT & PRIQ_0_SSID_MASK : 0;
1167 last = evt[0] & PRIQ_0_PRG_LAST;
1168 grpid = evt[1] >> PRIQ_1_PRG_IDX_SHIFT & PRIQ_1_PRG_IDX_MASK;
1169
1170 dev_info(smmu->dev, "unexpected PRI request received:\n");
1171 dev_info(smmu->dev,
1172 "\tsid 0x%08x.0x%05x: [%u%s] %sprivileged %s%s%s access at iova 0x%016llx\n",
1173 sid, ssid, grpid, last ? "L" : "",
1174 evt[0] & PRIQ_0_PERM_PRIV ? "" : "un",
1175 evt[0] & PRIQ_0_PERM_READ ? "R" : "",
1176 evt[0] & PRIQ_0_PERM_WRITE ? "W" : "",
1177 evt[0] & PRIQ_0_PERM_EXEC ? "X" : "",
1178 evt[1] & PRIQ_1_ADDR_MASK << PRIQ_1_ADDR_SHIFT);
1179
1180 if (last) {
1181 struct arm_smmu_cmdq_ent cmd = {
1182 .opcode = CMDQ_OP_PRI_RESP,
1183 .substream_valid = ssv,
1184 .pri = {
1185 .sid = sid,
1186 .ssid = ssid,
1187 .grpid = grpid,
1188 .resp = PRI_RESP_DENY,
1189 },
1190 };
1191
1192 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1193 }
1194 }
1195
1196 /* Sync our overflow flag, as we believe we're up to speed */
1197 q->cons = Q_OVF(q, q->prod) | Q_WRP(q, q->cons) | Q_IDX(q, q->cons);
1198 return IRQ_HANDLED;
1199}
1200
1201static irqreturn_t arm_smmu_priq_handler(int irq, void *dev)
1202{
1203 irqreturn_t ret = IRQ_WAKE_THREAD;
1204 struct arm_smmu_device *smmu = dev;
1205 struct arm_smmu_queue *q = &smmu->priq.q;
1206
1207 /* PRIQ overflow indicates a programming error */
1208 if (queue_sync_prod(q) == -EOVERFLOW)
1209 dev_err(smmu->dev, "PRIQ overflow detected -- requests lost\n");
1210 else if (queue_empty(q))
1211 ret = IRQ_NONE;
1212
1213 return ret;
1214}
1215
1216static irqreturn_t arm_smmu_cmdq_sync_handler(int irq, void *dev)
1217{
1218 /* We don't actually use CMD_SYNC interrupts for anything */
1219 return IRQ_HANDLED;
1220}
1221
1222static int arm_smmu_device_disable(struct arm_smmu_device *smmu);
1223
1224static irqreturn_t arm_smmu_gerror_handler(int irq, void *dev)
1225{
1226 u32 gerror, gerrorn;
1227 struct arm_smmu_device *smmu = dev;
1228
1229 gerror = readl_relaxed(smmu->base + ARM_SMMU_GERROR);
1230 gerrorn = readl_relaxed(smmu->base + ARM_SMMU_GERRORN);
1231
1232 gerror ^= gerrorn;
1233 if (!(gerror & GERROR_ERR_MASK))
1234 return IRQ_NONE; /* No errors pending */
1235
1236 dev_warn(smmu->dev,
1237 "unexpected global error reported (0x%08x), this could be serious\n",
1238 gerror);
1239
1240 if (gerror & GERROR_SFM_ERR) {
1241 dev_err(smmu->dev, "device has entered Service Failure Mode!\n");
1242 arm_smmu_device_disable(smmu);
1243 }
1244
1245 if (gerror & GERROR_MSI_GERROR_ABT_ERR)
1246 dev_warn(smmu->dev, "GERROR MSI write aborted\n");
1247
1248 if (gerror & GERROR_MSI_PRIQ_ABT_ERR) {
1249 dev_warn(smmu->dev, "PRIQ MSI write aborted\n");
1250 arm_smmu_priq_handler(irq, smmu->dev);
1251 }
1252
1253 if (gerror & GERROR_MSI_EVTQ_ABT_ERR) {
1254 dev_warn(smmu->dev, "EVTQ MSI write aborted\n");
1255 arm_smmu_evtq_handler(irq, smmu->dev);
1256 }
1257
1258 if (gerror & GERROR_MSI_CMDQ_ABT_ERR) {
1259 dev_warn(smmu->dev, "CMDQ MSI write aborted\n");
1260 arm_smmu_cmdq_sync_handler(irq, smmu->dev);
1261 }
1262
1263 if (gerror & GERROR_PRIQ_ABT_ERR)
1264 dev_err(smmu->dev, "PRIQ write aborted -- events may have been lost\n");
1265
1266 if (gerror & GERROR_EVTQ_ABT_ERR)
1267 dev_err(smmu->dev, "EVTQ write aborted -- events may have been lost\n");
1268
1269 if (gerror & GERROR_CMDQ_ERR)
1270 arm_smmu_cmdq_skip_err(smmu);
1271
1272 writel(gerror, smmu->base + ARM_SMMU_GERRORN);
1273 return IRQ_HANDLED;
1274}
1275
1276/* IO_PGTABLE API */
1277static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu)
1278{
1279 struct arm_smmu_cmdq_ent cmd;
1280
1281 cmd.opcode = CMDQ_OP_CMD_SYNC;
1282 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1283}
1284
1285static void arm_smmu_tlb_sync(void *cookie)
1286{
1287 struct arm_smmu_domain *smmu_domain = cookie;
1288 __arm_smmu_tlb_sync(smmu_domain->smmu);
1289}
1290
1291static void arm_smmu_tlb_inv_context(void *cookie)
1292{
1293 struct arm_smmu_domain *smmu_domain = cookie;
1294 struct arm_smmu_device *smmu = smmu_domain->smmu;
1295 struct arm_smmu_cmdq_ent cmd;
1296
1297 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1298 cmd.opcode = CMDQ_OP_TLBI_NH_ASID;
1299 cmd.tlbi.asid = smmu_domain->s1_cfg.cd.asid;
1300 cmd.tlbi.vmid = 0;
1301 } else {
1302 cmd.opcode = CMDQ_OP_TLBI_S12_VMALL;
1303 cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid;
1304 }
1305
1306 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1307 __arm_smmu_tlb_sync(smmu);
1308}
1309
1310static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
1311 bool leaf, void *cookie)
1312{
1313 struct arm_smmu_domain *smmu_domain = cookie;
1314 struct arm_smmu_device *smmu = smmu_domain->smmu;
1315 struct arm_smmu_cmdq_ent cmd = {
1316 .tlbi = {
1317 .leaf = leaf,
1318 .addr = iova,
1319 },
1320 };
1321
1322 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1323 cmd.opcode = CMDQ_OP_TLBI_NH_VA;
1324 cmd.tlbi.asid = smmu_domain->s1_cfg.cd.asid;
1325 } else {
1326 cmd.opcode = CMDQ_OP_TLBI_S2_IPA;
1327 cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid;
1328 }
1329
1330 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1331}
1332
1333static void arm_smmu_flush_pgtable(void *addr, size_t size, void *cookie)
1334{
1335 struct arm_smmu_domain *smmu_domain = cookie;
1336 struct arm_smmu_device *smmu = smmu_domain->smmu;
1337 unsigned long offset = (unsigned long)addr & ~PAGE_MASK;
1338
1339 if (smmu->features & ARM_SMMU_FEAT_COHERENCY) {
1340 dsb(ishst);
1341 } else {
1342 dma_addr_t dma_addr;
1343 struct device *dev = smmu->dev;
1344
1345 dma_addr = dma_map_page(dev, virt_to_page(addr), offset, size,
1346 DMA_TO_DEVICE);
1347
1348 if (dma_mapping_error(dev, dma_addr))
1349 dev_err(dev, "failed to flush pgtable at %p\n", addr);
1350 else
1351 dma_unmap_page(dev, dma_addr, size, DMA_TO_DEVICE);
1352 }
1353}
1354
1355static struct iommu_gather_ops arm_smmu_gather_ops = {
1356 .tlb_flush_all = arm_smmu_tlb_inv_context,
1357 .tlb_add_flush = arm_smmu_tlb_inv_range_nosync,
1358 .tlb_sync = arm_smmu_tlb_sync,
1359 .flush_pgtable = arm_smmu_flush_pgtable,
1360};
1361
1362/* IOMMU API */
1363static bool arm_smmu_capable(enum iommu_cap cap)
1364{
1365 switch (cap) {
1366 case IOMMU_CAP_CACHE_COHERENCY:
1367 return true;
1368 case IOMMU_CAP_INTR_REMAP:
1369 return true; /* MSIs are just memory writes */
1370 case IOMMU_CAP_NOEXEC:
1371 return true;
1372 default:
1373 return false;
1374 }
1375}
1376
1377static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
1378{
1379 struct arm_smmu_domain *smmu_domain;
1380
1381 if (type != IOMMU_DOMAIN_UNMANAGED)
1382 return NULL;
1383
1384 /*
1385 * Allocate the domain and initialise some of its data structures.
1386 * We can't really do anything meaningful until we've added a
1387 * master.
1388 */
1389 smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
1390 if (!smmu_domain)
1391 return NULL;
1392
1393 mutex_init(&smmu_domain->init_mutex);
1394 spin_lock_init(&smmu_domain->pgtbl_lock);
1395 return &smmu_domain->domain;
1396}
1397
1398static int arm_smmu_bitmap_alloc(unsigned long *map, int span)
1399{
1400 int idx, size = 1 << span;
1401
1402 do {
1403 idx = find_first_zero_bit(map, size);
1404 if (idx == size)
1405 return -ENOSPC;
1406 } while (test_and_set_bit(idx, map));
1407
1408 return idx;
1409}
1410
1411static void arm_smmu_bitmap_free(unsigned long *map, int idx)
1412{
1413 clear_bit(idx, map);
1414}
1415
1416static void arm_smmu_domain_free(struct iommu_domain *domain)
1417{
1418 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1419 struct arm_smmu_device *smmu = smmu_domain->smmu;
1420
1421 free_io_pgtable_ops(smmu_domain->pgtbl_ops);
1422
1423 /* Free the CD and ASID, if we allocated them */
1424 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1425 struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
1426
1427 if (cfg->cdptr) {
1428 dma_free_coherent(smmu_domain->smmu->dev,
1429 CTXDESC_CD_DWORDS << 3,
1430 cfg->cdptr,
1431 cfg->cdptr_dma);
1432
1433 arm_smmu_bitmap_free(smmu->asid_map, cfg->cd.asid);
1434 }
1435 } else {
1436 struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
1437 if (cfg->vmid)
1438 arm_smmu_bitmap_free(smmu->vmid_map, cfg->vmid);
1439 }
1440
1441 kfree(smmu_domain);
1442}
1443
1444static int arm_smmu_domain_finalise_s1(struct arm_smmu_domain *smmu_domain,
1445 struct io_pgtable_cfg *pgtbl_cfg)
1446{
1447 int ret;
1448 u16 asid;
1449 struct arm_smmu_device *smmu = smmu_domain->smmu;
1450 struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
1451
1452 asid = arm_smmu_bitmap_alloc(smmu->asid_map, smmu->asid_bits);
1453 if (IS_ERR_VALUE(asid))
1454 return asid;
1455
1456 cfg->cdptr = dma_zalloc_coherent(smmu->dev, CTXDESC_CD_DWORDS << 3,
1457 &cfg->cdptr_dma, GFP_KERNEL);
1458 if (!cfg->cdptr) {
1459 dev_warn(smmu->dev, "failed to allocate context descriptor\n");
1460 goto out_free_asid;
1461 }
1462
1463 cfg->cd.asid = asid;
1464 cfg->cd.ttbr = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
1465 cfg->cd.tcr = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
1466 cfg->cd.mair = pgtbl_cfg->arm_lpae_s1_cfg.mair[0];
1467 return 0;
1468
1469out_free_asid:
1470 arm_smmu_bitmap_free(smmu->asid_map, asid);
1471 return ret;
1472}
1473
1474static int arm_smmu_domain_finalise_s2(struct arm_smmu_domain *smmu_domain,
1475 struct io_pgtable_cfg *pgtbl_cfg)
1476{
1477 u16 vmid;
1478 struct arm_smmu_device *smmu = smmu_domain->smmu;
1479 struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
1480
1481 vmid = arm_smmu_bitmap_alloc(smmu->vmid_map, smmu->vmid_bits);
1482 if (IS_ERR_VALUE(vmid))
1483 return vmid;
1484
1485 cfg->vmid = vmid;
1486 cfg->vttbr = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
1487 cfg->vtcr = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
1488 return 0;
1489}
1490
1491static struct iommu_ops arm_smmu_ops;
1492
1493static int arm_smmu_domain_finalise(struct iommu_domain *domain)
1494{
1495 int ret;
1496 unsigned long ias, oas;
1497 enum io_pgtable_fmt fmt;
1498 struct io_pgtable_cfg pgtbl_cfg;
1499 struct io_pgtable_ops *pgtbl_ops;
1500 int (*finalise_stage_fn)(struct arm_smmu_domain *,
1501 struct io_pgtable_cfg *);
1502 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1503 struct arm_smmu_device *smmu = smmu_domain->smmu;
1504
1505 /* Restrict the stage to what we can actually support */
1506 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
1507 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
1508 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
1509 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1510
1511 switch (smmu_domain->stage) {
1512 case ARM_SMMU_DOMAIN_S1:
1513 ias = VA_BITS;
1514 oas = smmu->ias;
1515 fmt = ARM_64_LPAE_S1;
1516 finalise_stage_fn = arm_smmu_domain_finalise_s1;
1517 break;
1518 case ARM_SMMU_DOMAIN_NESTED:
1519 case ARM_SMMU_DOMAIN_S2:
1520 ias = smmu->ias;
1521 oas = smmu->oas;
1522 fmt = ARM_64_LPAE_S2;
1523 finalise_stage_fn = arm_smmu_domain_finalise_s2;
1524 break;
1525 default:
1526 return -EINVAL;
1527 }
1528
1529 pgtbl_cfg = (struct io_pgtable_cfg) {
1530 .pgsize_bitmap = arm_smmu_ops.pgsize_bitmap,
1531 .ias = ias,
1532 .oas = oas,
1533 .tlb = &arm_smmu_gather_ops,
1534 };
1535
1536 pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
1537 if (!pgtbl_ops)
1538 return -ENOMEM;
1539
1540 arm_smmu_ops.pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
1541 smmu_domain->pgtbl_ops = pgtbl_ops;
1542
1543 ret = finalise_stage_fn(smmu_domain, &pgtbl_cfg);
1544 if (IS_ERR_VALUE(ret))
1545 free_io_pgtable_ops(pgtbl_ops);
1546
1547 return ret;
1548}
1549
1550static struct arm_smmu_group *arm_smmu_group_get(struct device *dev)
1551{
1552 struct iommu_group *group;
1553 struct arm_smmu_group *smmu_group;
1554
1555 group = iommu_group_get(dev);
1556 if (!group)
1557 return NULL;
1558
1559 smmu_group = iommu_group_get_iommudata(group);
1560 iommu_group_put(group);
1561 return smmu_group;
1562}
1563
1564static __le64 *arm_smmu_get_step_for_sid(struct arm_smmu_device *smmu, u32 sid)
1565{
1566 __le64 *step;
1567 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
1568
1569 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
1570 struct arm_smmu_strtab_l1_desc *l1_desc;
1571 int idx;
1572
1573 /* Two-level walk */
1574 idx = (sid >> STRTAB_SPLIT) * STRTAB_L1_DESC_DWORDS;
1575 l1_desc = &cfg->l1_desc[idx];
1576 idx = (sid & ((1 << STRTAB_SPLIT) - 1)) * STRTAB_STE_DWORDS;
1577 step = &l1_desc->l2ptr[idx];
1578 } else {
1579 /* Simple linear lookup */
1580 step = &cfg->strtab[sid * STRTAB_STE_DWORDS];
1581 }
1582
1583 return step;
1584}
1585
1586static int arm_smmu_install_ste_for_group(struct arm_smmu_group *smmu_group)
1587{
1588 int i;
1589 struct arm_smmu_domain *smmu_domain = smmu_group->domain;
1590 struct arm_smmu_strtab_ent *ste = &smmu_group->ste;
1591 struct arm_smmu_device *smmu = smmu_group->smmu;
1592
1593 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1594 ste->s1_cfg = &smmu_domain->s1_cfg;
1595 ste->s2_cfg = NULL;
1596 arm_smmu_write_ctx_desc(smmu, ste->s1_cfg);
1597 } else {
1598 ste->s1_cfg = NULL;
1599 ste->s2_cfg = &smmu_domain->s2_cfg;
1600 }
1601
1602 for (i = 0; i < smmu_group->num_sids; ++i) {
1603 u32 sid = smmu_group->sids[i];
1604 __le64 *step = arm_smmu_get_step_for_sid(smmu, sid);
1605
1606 arm_smmu_write_strtab_ent(smmu, sid, step, ste);
1607 }
1608
1609 return 0;
1610}
1611
1612static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1613{
1614 int ret = 0;
1615 struct arm_smmu_device *smmu;
1616 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1617 struct arm_smmu_group *smmu_group = arm_smmu_group_get(dev);
1618
1619 if (!smmu_group)
1620 return -ENOENT;
1621
1622 /* Already attached to a different domain? */
1623 if (smmu_group->domain && smmu_group->domain != smmu_domain)
1624 return -EEXIST;
1625
1626 smmu = smmu_group->smmu;
1627 mutex_lock(&smmu_domain->init_mutex);
1628
1629 if (!smmu_domain->smmu) {
1630 smmu_domain->smmu = smmu;
1631 ret = arm_smmu_domain_finalise(domain);
1632 if (ret) {
1633 smmu_domain->smmu = NULL;
1634 goto out_unlock;
1635 }
1636 } else if (smmu_domain->smmu != smmu) {
1637 dev_err(dev,
1638 "cannot attach to SMMU %s (upstream of %s)\n",
1639 dev_name(smmu_domain->smmu->dev),
1640 dev_name(smmu->dev));
1641 ret = -ENXIO;
1642 goto out_unlock;
1643 }
1644
1645 /* Group already attached to this domain? */
1646 if (smmu_group->domain)
1647 goto out_unlock;
1648
1649 smmu_group->domain = smmu_domain;
1650 smmu_group->ste.bypass = false;
1651
1652 ret = arm_smmu_install_ste_for_group(smmu_group);
1653 if (IS_ERR_VALUE(ret))
1654 smmu_group->domain = NULL;
1655
1656out_unlock:
1657 mutex_unlock(&smmu_domain->init_mutex);
1658 return ret;
1659}
1660
1661static void arm_smmu_detach_dev(struct iommu_domain *domain, struct device *dev)
1662{
1663 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1664 struct arm_smmu_group *smmu_group = arm_smmu_group_get(dev);
1665
1666 BUG_ON(!smmu_domain);
1667 BUG_ON(!smmu_group);
1668
1669 mutex_lock(&smmu_domain->init_mutex);
1670 BUG_ON(smmu_group->domain != smmu_domain);
1671
1672 smmu_group->ste.bypass = true;
1673 if (IS_ERR_VALUE(arm_smmu_install_ste_for_group(smmu_group)))
1674 dev_warn(dev, "failed to install bypass STE\n");
1675
1676 smmu_group->domain = NULL;
1677 mutex_unlock(&smmu_domain->init_mutex);
1678}
1679
1680static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
1681 phys_addr_t paddr, size_t size, int prot)
1682{
1683 int ret;
1684 unsigned long flags;
1685 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1686 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
1687
1688 if (!ops)
1689 return -ENODEV;
1690
1691 spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1692 ret = ops->map(ops, iova, paddr, size, prot);
1693 spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1694 return ret;
1695}
1696
1697static size_t
1698arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size)
1699{
1700 size_t ret;
1701 unsigned long flags;
1702 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1703 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
1704
1705 if (!ops)
1706 return 0;
1707
1708 spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1709 ret = ops->unmap(ops, iova, size);
1710 spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1711 return ret;
1712}
1713
1714static phys_addr_t
1715arm_smmu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova)
1716{
1717 phys_addr_t ret;
1718 unsigned long flags;
1719 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1720 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
1721
1722 if (!ops)
1723 return 0;
1724
1725 spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1726 ret = ops->iova_to_phys(ops, iova);
1727 spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1728
1729 return ret;
1730}
1731
1732static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *sidp)
1733{
1734 *(u32 *)sidp = alias;
1735 return 0; /* Continue walking */
1736}
1737
1738static void __arm_smmu_release_pci_iommudata(void *data)
1739{
1740 kfree(data);
1741}
1742
1743static struct arm_smmu_device *arm_smmu_get_for_pci_dev(struct pci_dev *pdev)
1744{
1745 struct device_node *of_node;
1746 struct arm_smmu_device *curr, *smmu = NULL;
1747 struct pci_bus *bus = pdev->bus;
1748
1749 /* Walk up to the root bus */
1750 while (!pci_is_root_bus(bus))
1751 bus = bus->parent;
1752
1753 /* Follow the "iommus" phandle from the host controller */
1754 of_node = of_parse_phandle(bus->bridge->parent->of_node, "iommus", 0);
1755 if (!of_node)
1756 return NULL;
1757
1758 /* See if we can find an SMMU corresponding to the phandle */
1759 spin_lock(&arm_smmu_devices_lock);
1760 list_for_each_entry(curr, &arm_smmu_devices, list) {
1761 if (curr->dev->of_node == of_node) {
1762 smmu = curr;
1763 break;
1764 }
1765 }
1766 spin_unlock(&arm_smmu_devices_lock);
1767 of_node_put(of_node);
1768 return smmu;
1769}
1770
1771static bool arm_smmu_sid_in_range(struct arm_smmu_device *smmu, u32 sid)
1772{
1773 unsigned long limit = smmu->strtab_cfg.num_l1_ents;
1774
1775 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
1776 limit *= 1UL << STRTAB_SPLIT;
1777
1778 return sid < limit;
1779}
1780
1781static int arm_smmu_add_device(struct device *dev)
1782{
1783 int i, ret;
1784 u32 sid, *sids;
1785 struct pci_dev *pdev;
1786 struct iommu_group *group;
1787 struct arm_smmu_group *smmu_group;
1788 struct arm_smmu_device *smmu;
1789
1790 /* We only support PCI, for now */
1791 if (!dev_is_pci(dev))
1792 return -ENODEV;
1793
1794 pdev = to_pci_dev(dev);
1795 group = iommu_group_get_for_dev(dev);
1796 if (IS_ERR(group))
1797 return PTR_ERR(group);
1798
1799 smmu_group = iommu_group_get_iommudata(group);
1800 if (!smmu_group) {
1801 smmu = arm_smmu_get_for_pci_dev(pdev);
1802 if (!smmu) {
1803 ret = -ENOENT;
1804 goto out_put_group;
1805 }
1806
1807 smmu_group = kzalloc(sizeof(*smmu_group), GFP_KERNEL);
1808 if (!smmu_group) {
1809 ret = -ENOMEM;
1810 goto out_put_group;
1811 }
1812
1813 smmu_group->ste.valid = true;
1814 smmu_group->smmu = smmu;
1815 iommu_group_set_iommudata(group, smmu_group,
1816 __arm_smmu_release_pci_iommudata);
1817 } else {
1818 smmu = smmu_group->smmu;
1819 }
1820
1821 /* Assume SID == RID until firmware tells us otherwise */
1822 pci_for_each_dma_alias(pdev, __arm_smmu_get_pci_sid, &sid);
1823 for (i = 0; i < smmu_group->num_sids; ++i) {
1824 /* If we already know about this SID, then we're done */
1825 if (smmu_group->sids[i] == sid)
1826 return 0;
1827 }
1828
1829 /* Check the SID is in range of the SMMU and our stream table */
1830 if (!arm_smmu_sid_in_range(smmu, sid)) {
1831 ret = -ERANGE;
1832 goto out_put_group;
1833 }
1834
1835 /* Ensure l2 strtab is initialised */
1836 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
1837 ret = arm_smmu_init_l2_strtab(smmu, sid);
1838 if (ret)
1839 goto out_put_group;
1840 }
1841
1842 /* Resize the SID array for the group */
1843 smmu_group->num_sids++;
1844 sids = krealloc(smmu_group->sids, smmu_group->num_sids * sizeof(*sids),
1845 GFP_KERNEL);
1846 if (!sids) {
1847 smmu_group->num_sids--;
1848 ret = -ENOMEM;
1849 goto out_put_group;
1850 }
1851
1852 /* Add the new SID */
1853 sids[smmu_group->num_sids - 1] = sid;
1854 smmu_group->sids = sids;
1855 return 0;
1856
1857out_put_group:
1858 iommu_group_put(group);
1859 return ret;
1860}
1861
1862static void arm_smmu_remove_device(struct device *dev)
1863{
1864 iommu_group_remove_device(dev);
1865}
1866
1867static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
1868 enum iommu_attr attr, void *data)
1869{
1870 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1871
1872 switch (attr) {
1873 case DOMAIN_ATTR_NESTING:
1874 *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
1875 return 0;
1876 default:
1877 return -ENODEV;
1878 }
1879}
1880
1881static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
1882 enum iommu_attr attr, void *data)
1883{
1884 int ret = 0;
1885 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1886
1887 mutex_lock(&smmu_domain->init_mutex);
1888
1889 switch (attr) {
1890 case DOMAIN_ATTR_NESTING:
1891 if (smmu_domain->smmu) {
1892 ret = -EPERM;
1893 goto out_unlock;
1894 }
1895
1896 if (*(int *)data)
1897 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1898 else
1899 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1900
1901 break;
1902 default:
1903 ret = -ENODEV;
1904 }
1905
1906out_unlock:
1907 mutex_unlock(&smmu_domain->init_mutex);
1908 return ret;
1909}
1910
1911static struct iommu_ops arm_smmu_ops = {
1912 .capable = arm_smmu_capable,
1913 .domain_alloc = arm_smmu_domain_alloc,
1914 .domain_free = arm_smmu_domain_free,
1915 .attach_dev = arm_smmu_attach_dev,
1916 .detach_dev = arm_smmu_detach_dev,
1917 .map = arm_smmu_map,
1918 .unmap = arm_smmu_unmap,
1919 .iova_to_phys = arm_smmu_iova_to_phys,
1920 .add_device = arm_smmu_add_device,
1921 .remove_device = arm_smmu_remove_device,
1922 .domain_get_attr = arm_smmu_domain_get_attr,
1923 .domain_set_attr = arm_smmu_domain_set_attr,
1924 .pgsize_bitmap = -1UL, /* Restricted during device attach */
1925};
1926
1927/* Probing and initialisation functions */
1928static int arm_smmu_init_one_queue(struct arm_smmu_device *smmu,
1929 struct arm_smmu_queue *q,
1930 unsigned long prod_off,
1931 unsigned long cons_off,
1932 size_t dwords)
1933{
1934 size_t qsz = ((1 << q->max_n_shift) * dwords) << 3;
1935
1936 q->base = dma_alloc_coherent(smmu->dev, qsz, &q->base_dma, GFP_KERNEL);
1937 if (!q->base) {
1938 dev_err(smmu->dev, "failed to allocate queue (0x%zx bytes)\n",
1939 qsz);
1940 return -ENOMEM;
1941 }
1942
1943 q->prod_reg = smmu->base + prod_off;
1944 q->cons_reg = smmu->base + cons_off;
1945 q->ent_dwords = dwords;
1946
1947 q->q_base = Q_BASE_RWA;
1948 q->q_base |= q->base_dma & Q_BASE_ADDR_MASK << Q_BASE_ADDR_SHIFT;
1949 q->q_base |= (q->max_n_shift & Q_BASE_LOG2SIZE_MASK)
1950 << Q_BASE_LOG2SIZE_SHIFT;
1951
1952 q->prod = q->cons = 0;
1953 return 0;
1954}
1955
1956static void arm_smmu_free_one_queue(struct arm_smmu_device *smmu,
1957 struct arm_smmu_queue *q)
1958{
1959 size_t qsz = ((1 << q->max_n_shift) * q->ent_dwords) << 3;
1960
1961 dma_free_coherent(smmu->dev, qsz, q->base, q->base_dma);
1962}
1963
1964static void arm_smmu_free_queues(struct arm_smmu_device *smmu)
1965{
1966 arm_smmu_free_one_queue(smmu, &smmu->cmdq.q);
1967 arm_smmu_free_one_queue(smmu, &smmu->evtq.q);
1968
1969 if (smmu->features & ARM_SMMU_FEAT_PRI)
1970 arm_smmu_free_one_queue(smmu, &smmu->priq.q);
1971}
1972
1973static int arm_smmu_init_queues(struct arm_smmu_device *smmu)
1974{
1975 int ret;
1976
1977 /* cmdq */
1978 spin_lock_init(&smmu->cmdq.lock);
1979 ret = arm_smmu_init_one_queue(smmu, &smmu->cmdq.q, ARM_SMMU_CMDQ_PROD,
1980 ARM_SMMU_CMDQ_CONS, CMDQ_ENT_DWORDS);
1981 if (ret)
1982 goto out;
1983
1984 /* evtq */
1985 ret = arm_smmu_init_one_queue(smmu, &smmu->evtq.q, ARM_SMMU_EVTQ_PROD,
1986 ARM_SMMU_EVTQ_CONS, EVTQ_ENT_DWORDS);
1987 if (ret)
1988 goto out_free_cmdq;
1989
1990 /* priq */
1991 if (!(smmu->features & ARM_SMMU_FEAT_PRI))
1992 return 0;
1993
1994 ret = arm_smmu_init_one_queue(smmu, &smmu->priq.q, ARM_SMMU_PRIQ_PROD,
1995 ARM_SMMU_PRIQ_CONS, PRIQ_ENT_DWORDS);
1996 if (ret)
1997 goto out_free_evtq;
1998
1999 return 0;
2000
2001out_free_evtq:
2002 arm_smmu_free_one_queue(smmu, &smmu->evtq.q);
2003out_free_cmdq:
2004 arm_smmu_free_one_queue(smmu, &smmu->cmdq.q);
2005out:
2006 return ret;
2007}
2008
2009static void arm_smmu_free_l2_strtab(struct arm_smmu_device *smmu)
2010{
2011 int i;
2012 size_t size;
2013 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2014
2015 size = 1 << (STRTAB_SPLIT + ilog2(STRTAB_STE_DWORDS) + 3);
2016 for (i = 0; i < cfg->num_l1_ents; ++i) {
2017 struct arm_smmu_strtab_l1_desc *desc = &cfg->l1_desc[i];
2018
2019 if (!desc->l2ptr)
2020 continue;
2021
2022 dma_free_coherent(smmu->dev, size, desc->l2ptr,
2023 desc->l2ptr_dma);
2024 }
2025}
2026
2027static int arm_smmu_init_l1_strtab(struct arm_smmu_device *smmu)
2028{
2029 unsigned int i;
2030 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2031 size_t size = sizeof(*cfg->l1_desc) * cfg->num_l1_ents;
2032 void *strtab = smmu->strtab_cfg.strtab;
2033
2034 cfg->l1_desc = devm_kzalloc(smmu->dev, size, GFP_KERNEL);
2035 if (!cfg->l1_desc) {
2036 dev_err(smmu->dev, "failed to allocate l1 stream table desc\n");
2037 return -ENOMEM;
2038 }
2039
2040 for (i = 0; i < cfg->num_l1_ents; ++i) {
2041 arm_smmu_write_strtab_l1_desc(strtab, &cfg->l1_desc[i]);
2042 strtab += STRTAB_L1_DESC_DWORDS << 3;
2043 }
2044
2045 return 0;
2046}
2047
2048static int arm_smmu_init_strtab_2lvl(struct arm_smmu_device *smmu)
2049{
2050 void *strtab;
2051 u64 reg;
2052 u32 size, l1size;
2053 int ret;
2054 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2055
2056 /* Calculate the L1 size, capped to the SIDSIZE */
2057 size = STRTAB_L1_SZ_SHIFT - (ilog2(STRTAB_L1_DESC_DWORDS) + 3);
2058 size = min(size, smmu->sid_bits - STRTAB_SPLIT);
2059 cfg->num_l1_ents = 1 << size;
2060
2061 size += STRTAB_SPLIT;
2062 if (size < smmu->sid_bits)
2063 dev_warn(smmu->dev,
2064 "2-level strtab only covers %u/%u bits of SID\n",
2065 size, smmu->sid_bits);
2066
2067 l1size = cfg->num_l1_ents * (STRTAB_L1_DESC_DWORDS << 3);
2068 strtab = dma_zalloc_coherent(smmu->dev, l1size, &cfg->strtab_dma,
2069 GFP_KERNEL);
2070 if (!strtab) {
2071 dev_err(smmu->dev,
2072 "failed to allocate l1 stream table (%u bytes)\n",
2073 size);
2074 return -ENOMEM;
2075 }
2076 cfg->strtab = strtab;
2077
2078 /* Configure strtab_base_cfg for 2 levels */
2079 reg = STRTAB_BASE_CFG_FMT_2LVL;
2080 reg |= (size & STRTAB_BASE_CFG_LOG2SIZE_MASK)
2081 << STRTAB_BASE_CFG_LOG2SIZE_SHIFT;
2082 reg |= (STRTAB_SPLIT & STRTAB_BASE_CFG_SPLIT_MASK)
2083 << STRTAB_BASE_CFG_SPLIT_SHIFT;
2084 cfg->strtab_base_cfg = reg;
2085
2086 ret = arm_smmu_init_l1_strtab(smmu);
2087 if (ret)
2088 dma_free_coherent(smmu->dev,
2089 l1size,
2090 strtab,
2091 cfg->strtab_dma);
2092 return ret;
2093}
2094
2095static int arm_smmu_init_strtab_linear(struct arm_smmu_device *smmu)
2096{
2097 void *strtab;
2098 u64 reg;
2099 u32 size;
2100 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2101
2102 size = (1 << smmu->sid_bits) * (STRTAB_STE_DWORDS << 3);
2103 strtab = dma_zalloc_coherent(smmu->dev, size, &cfg->strtab_dma,
2104 GFP_KERNEL);
2105 if (!strtab) {
2106 dev_err(smmu->dev,
2107 "failed to allocate linear stream table (%u bytes)\n",
2108 size);
2109 return -ENOMEM;
2110 }
2111 cfg->strtab = strtab;
2112 cfg->num_l1_ents = 1 << smmu->sid_bits;
2113
2114 /* Configure strtab_base_cfg for a linear table covering all SIDs */
2115 reg = STRTAB_BASE_CFG_FMT_LINEAR;
2116 reg |= (smmu->sid_bits & STRTAB_BASE_CFG_LOG2SIZE_MASK)
2117 << STRTAB_BASE_CFG_LOG2SIZE_SHIFT;
2118 cfg->strtab_base_cfg = reg;
2119
2120 arm_smmu_init_bypass_stes(strtab, cfg->num_l1_ents);
2121 return 0;
2122}
2123
2124static int arm_smmu_init_strtab(struct arm_smmu_device *smmu)
2125{
2126 u64 reg;
2127 int ret;
2128
2129 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
2130 ret = arm_smmu_init_strtab_2lvl(smmu);
2131 else
2132 ret = arm_smmu_init_strtab_linear(smmu);
2133
2134 if (ret)
2135 return ret;
2136
2137 /* Set the strtab base address */
2138 reg = smmu->strtab_cfg.strtab_dma &
2139 STRTAB_BASE_ADDR_MASK << STRTAB_BASE_ADDR_SHIFT;
2140 reg |= STRTAB_BASE_RA;
2141 smmu->strtab_cfg.strtab_base = reg;
2142
2143 /* Allocate the first VMID for stage-2 bypass STEs */
2144 set_bit(0, smmu->vmid_map);
2145 return 0;
2146}
2147
2148static void arm_smmu_free_strtab(struct arm_smmu_device *smmu)
2149{
2150 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2151 u32 size = cfg->num_l1_ents;
2152
2153 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
2154 arm_smmu_free_l2_strtab(smmu);
2155 size *= STRTAB_L1_DESC_DWORDS << 3;
2156 } else {
2157 size *= STRTAB_STE_DWORDS * 3;
2158 }
2159
2160 dma_free_coherent(smmu->dev, size, cfg->strtab, cfg->strtab_dma);
2161}
2162
2163static int arm_smmu_init_structures(struct arm_smmu_device *smmu)
2164{
2165 int ret;
2166
2167 ret = arm_smmu_init_queues(smmu);
2168 if (ret)
2169 return ret;
2170
2171 ret = arm_smmu_init_strtab(smmu);
2172 if (ret)
2173 goto out_free_queues;
2174
2175 return 0;
2176
2177out_free_queues:
2178 arm_smmu_free_queues(smmu);
2179 return ret;
2180}
2181
2182static void arm_smmu_free_structures(struct arm_smmu_device *smmu)
2183{
2184 arm_smmu_free_strtab(smmu);
2185 arm_smmu_free_queues(smmu);
2186}
2187
2188static int arm_smmu_write_reg_sync(struct arm_smmu_device *smmu, u32 val,
2189 unsigned int reg_off, unsigned int ack_off)
2190{
2191 u32 reg;
2192
2193 writel_relaxed(val, smmu->base + reg_off);
2194 return readl_relaxed_poll_timeout(smmu->base + ack_off, reg, reg == val,
2195 1, ARM_SMMU_POLL_TIMEOUT_US);
2196}
2197
2198static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu)
2199{
2200 int ret, irq;
2201
2202 /* Disable IRQs first */
2203 ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_IRQ_CTRL,
2204 ARM_SMMU_IRQ_CTRLACK);
2205 if (ret) {
2206 dev_err(smmu->dev, "failed to disable irqs\n");
2207 return ret;
2208 }
2209
2210 /* Clear the MSI address regs */
2211 writeq_relaxed(0, smmu->base + ARM_SMMU_GERROR_IRQ_CFG0);
2212 writeq_relaxed(0, smmu->base + ARM_SMMU_EVTQ_IRQ_CFG0);
2213
2214 /* Request wired interrupt lines */
2215 irq = smmu->evtq.q.irq;
2216 if (irq) {
2217 ret = devm_request_threaded_irq(smmu->dev, irq,
2218 arm_smmu_evtq_handler,
2219 arm_smmu_evtq_thread,
2220 0, "arm-smmu-v3-evtq", smmu);
2221 if (IS_ERR_VALUE(ret))
2222 dev_warn(smmu->dev, "failed to enable evtq irq\n");
2223 }
2224
2225 irq = smmu->cmdq.q.irq;
2226 if (irq) {
2227 ret = devm_request_irq(smmu->dev, irq,
2228 arm_smmu_cmdq_sync_handler, 0,
2229 "arm-smmu-v3-cmdq-sync", smmu);
2230 if (IS_ERR_VALUE(ret))
2231 dev_warn(smmu->dev, "failed to enable cmdq-sync irq\n");
2232 }
2233
2234 irq = smmu->gerr_irq;
2235 if (irq) {
2236 ret = devm_request_irq(smmu->dev, irq, arm_smmu_gerror_handler,
2237 0, "arm-smmu-v3-gerror", smmu);
2238 if (IS_ERR_VALUE(ret))
2239 dev_warn(smmu->dev, "failed to enable gerror irq\n");
2240 }
2241
2242 if (smmu->features & ARM_SMMU_FEAT_PRI) {
2243 writeq_relaxed(0, smmu->base + ARM_SMMU_PRIQ_IRQ_CFG0);
2244
2245 irq = smmu->priq.q.irq;
2246 if (irq) {
2247 ret = devm_request_threaded_irq(smmu->dev, irq,
2248 arm_smmu_priq_handler,
2249 arm_smmu_priq_thread,
2250 0, "arm-smmu-v3-priq",
2251 smmu);
2252 if (IS_ERR_VALUE(ret))
2253 dev_warn(smmu->dev,
2254 "failed to enable priq irq\n");
2255 }
2256 }
2257
2258 /* Enable interrupt generation on the SMMU */
2259 ret = arm_smmu_write_reg_sync(smmu,
2260 IRQ_CTRL_EVTQ_IRQEN |
2261 IRQ_CTRL_GERROR_IRQEN,
2262 ARM_SMMU_IRQ_CTRL, ARM_SMMU_IRQ_CTRLACK);
2263 if (ret)
2264 dev_warn(smmu->dev, "failed to enable irqs\n");
2265
2266 return 0;
2267}
2268
2269static int arm_smmu_device_disable(struct arm_smmu_device *smmu)
2270{
2271 int ret;
2272
2273 ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_CR0, ARM_SMMU_CR0ACK);
2274 if (ret)
2275 dev_err(smmu->dev, "failed to clear cr0\n");
2276
2277 return ret;
2278}
2279
2280static int arm_smmu_device_reset(struct arm_smmu_device *smmu)
2281{
2282 int ret;
2283 u32 reg, enables;
2284 struct arm_smmu_cmdq_ent cmd;
2285
2286 /* Clear CR0 and sync (disables SMMU and queue processing) */
2287 reg = readl_relaxed(smmu->base + ARM_SMMU_CR0);
2288 if (reg & CR0_SMMUEN)
2289 dev_warn(smmu->dev, "SMMU currently enabled! Resetting...\n");
2290
2291 ret = arm_smmu_device_disable(smmu);
2292 if (ret)
2293 return ret;
2294
2295 /* CR1 (table and queue memory attributes) */
2296 reg = (CR1_SH_ISH << CR1_TABLE_SH_SHIFT) |
2297 (CR1_CACHE_WB << CR1_TABLE_OC_SHIFT) |
2298 (CR1_CACHE_WB << CR1_TABLE_IC_SHIFT) |
2299 (CR1_SH_ISH << CR1_QUEUE_SH_SHIFT) |
2300 (CR1_CACHE_WB << CR1_QUEUE_OC_SHIFT) |
2301 (CR1_CACHE_WB << CR1_QUEUE_IC_SHIFT);
2302 writel_relaxed(reg, smmu->base + ARM_SMMU_CR1);
2303
2304 /* CR2 (random crap) */
2305 reg = CR2_PTM | CR2_RECINVSID | CR2_E2H;
2306 writel_relaxed(reg, smmu->base + ARM_SMMU_CR2);
2307
2308 /* Stream table */
2309 writeq_relaxed(smmu->strtab_cfg.strtab_base,
2310 smmu->base + ARM_SMMU_STRTAB_BASE);
2311 writel_relaxed(smmu->strtab_cfg.strtab_base_cfg,
2312 smmu->base + ARM_SMMU_STRTAB_BASE_CFG);
2313
2314 /* Command queue */
2315 writeq_relaxed(smmu->cmdq.q.q_base, smmu->base + ARM_SMMU_CMDQ_BASE);
2316 writel_relaxed(smmu->cmdq.q.prod, smmu->base + ARM_SMMU_CMDQ_PROD);
2317 writel_relaxed(smmu->cmdq.q.cons, smmu->base + ARM_SMMU_CMDQ_CONS);
2318
2319 enables = CR0_CMDQEN;
2320 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
2321 ARM_SMMU_CR0ACK);
2322 if (ret) {
2323 dev_err(smmu->dev, "failed to enable command queue\n");
2324 return ret;
2325 }
2326
2327 /* Invalidate any cached configuration */
2328 cmd.opcode = CMDQ_OP_CFGI_ALL;
2329 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
2330 cmd.opcode = CMDQ_OP_CMD_SYNC;
2331 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
2332
2333 /* Invalidate any stale TLB entries */
2334 if (smmu->features & ARM_SMMU_FEAT_HYP) {
2335 cmd.opcode = CMDQ_OP_TLBI_EL2_ALL;
2336 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
2337 }
2338
2339 cmd.opcode = CMDQ_OP_TLBI_NSNH_ALL;
2340 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
2341 cmd.opcode = CMDQ_OP_CMD_SYNC;
2342 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
2343
2344 /* Event queue */
2345 writeq_relaxed(smmu->evtq.q.q_base, smmu->base + ARM_SMMU_EVTQ_BASE);
2346 writel_relaxed(smmu->evtq.q.prod, smmu->base + ARM_SMMU_EVTQ_PROD);
2347 writel_relaxed(smmu->evtq.q.cons, smmu->base + ARM_SMMU_EVTQ_CONS);
2348
2349 enables |= CR0_EVTQEN;
2350 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
2351 ARM_SMMU_CR0ACK);
2352 if (ret) {
2353 dev_err(smmu->dev, "failed to enable event queue\n");
2354 return ret;
2355 }
2356
2357 /* PRI queue */
2358 if (smmu->features & ARM_SMMU_FEAT_PRI) {
2359 writeq_relaxed(smmu->priq.q.q_base,
2360 smmu->base + ARM_SMMU_PRIQ_BASE);
2361 writel_relaxed(smmu->priq.q.prod,
2362 smmu->base + ARM_SMMU_PRIQ_PROD);
2363 writel_relaxed(smmu->priq.q.cons,
2364 smmu->base + ARM_SMMU_PRIQ_CONS);
2365
2366 enables |= CR0_PRIQEN;
2367 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
2368 ARM_SMMU_CR0ACK);
2369 if (ret) {
2370 dev_err(smmu->dev, "failed to enable PRI queue\n");
2371 return ret;
2372 }
2373 }
2374
2375 ret = arm_smmu_setup_irqs(smmu);
2376 if (ret) {
2377 dev_err(smmu->dev, "failed to setup irqs\n");
2378 return ret;
2379 }
2380
2381 /* Enable the SMMU interface */
2382 enables |= CR0_SMMUEN;
2383 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
2384 ARM_SMMU_CR0ACK);
2385 if (ret) {
2386 dev_err(smmu->dev, "failed to enable SMMU interface\n");
2387 return ret;
2388 }
2389
2390 return 0;
2391}
2392
2393static int arm_smmu_device_probe(struct arm_smmu_device *smmu)
2394{
2395 u32 reg;
2396 bool coherent;
2397 unsigned long pgsize_bitmap = 0;
2398
2399 /* IDR0 */
2400 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR0);
2401
2402 /* 2-level structures */
2403 if ((reg & IDR0_ST_LVL_MASK << IDR0_ST_LVL_SHIFT) == IDR0_ST_LVL_2LVL)
2404 smmu->features |= ARM_SMMU_FEAT_2_LVL_STRTAB;
2405
2406 if (reg & IDR0_CD2L)
2407 smmu->features |= ARM_SMMU_FEAT_2_LVL_CDTAB;
2408
2409 /*
2410 * Translation table endianness.
2411 * We currently require the same endianness as the CPU, but this
2412 * could be changed later by adding a new IO_PGTABLE_QUIRK.
2413 */
2414 switch (reg & IDR0_TTENDIAN_MASK << IDR0_TTENDIAN_SHIFT) {
2415 case IDR0_TTENDIAN_MIXED:
2416 smmu->features |= ARM_SMMU_FEAT_TT_LE | ARM_SMMU_FEAT_TT_BE;
2417 break;
2418#ifdef __BIG_ENDIAN
2419 case IDR0_TTENDIAN_BE:
2420 smmu->features |= ARM_SMMU_FEAT_TT_BE;
2421 break;
2422#else
2423 case IDR0_TTENDIAN_LE:
2424 smmu->features |= ARM_SMMU_FEAT_TT_LE;
2425 break;
2426#endif
2427 default:
2428 dev_err(smmu->dev, "unknown/unsupported TT endianness!\n");
2429 return -ENXIO;
2430 }
2431
2432 /* Boolean feature flags */
2433 if (IS_ENABLED(CONFIG_PCI_PRI) && reg & IDR0_PRI)
2434 smmu->features |= ARM_SMMU_FEAT_PRI;
2435
2436 if (IS_ENABLED(CONFIG_PCI_ATS) && reg & IDR0_ATS)
2437 smmu->features |= ARM_SMMU_FEAT_ATS;
2438
2439 if (reg & IDR0_SEV)
2440 smmu->features |= ARM_SMMU_FEAT_SEV;
2441
2442 if (reg & IDR0_MSI)
2443 smmu->features |= ARM_SMMU_FEAT_MSI;
2444
2445 if (reg & IDR0_HYP)
2446 smmu->features |= ARM_SMMU_FEAT_HYP;
2447
2448 /*
2449 * The dma-coherent property is used in preference to the ID
2450 * register, but warn on mismatch.
2451 */
2452 coherent = of_dma_is_coherent(smmu->dev->of_node);
2453 if (coherent)
2454 smmu->features |= ARM_SMMU_FEAT_COHERENCY;
2455
2456 if (!!(reg & IDR0_COHACC) != coherent)
2457 dev_warn(smmu->dev, "IDR0.COHACC overridden by dma-coherent property (%s)\n",
2458 coherent ? "true" : "false");
2459
2460 if (reg & IDR0_STALL_MODEL)
2461 smmu->features |= ARM_SMMU_FEAT_STALLS;
2462
2463 if (reg & IDR0_S1P)
2464 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
2465
2466 if (reg & IDR0_S2P)
2467 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
2468
2469 if (!(reg & (IDR0_S1P | IDR0_S2P))) {
2470 dev_err(smmu->dev, "no translation support!\n");
2471 return -ENXIO;
2472 }
2473
2474 /* We only support the AArch64 table format at present */
2475 if ((reg & IDR0_TTF_MASK << IDR0_TTF_SHIFT) < IDR0_TTF_AARCH64) {
2476 dev_err(smmu->dev, "AArch64 table format not supported!\n");
2477 return -ENXIO;
2478 }
2479
2480 /* ASID/VMID sizes */
2481 smmu->asid_bits = reg & IDR0_ASID16 ? 16 : 8;
2482 smmu->vmid_bits = reg & IDR0_VMID16 ? 16 : 8;
2483
2484 /* IDR1 */
2485 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR1);
2486 if (reg & (IDR1_TABLES_PRESET | IDR1_QUEUES_PRESET | IDR1_REL)) {
2487 dev_err(smmu->dev, "embedded implementation not supported\n");
2488 return -ENXIO;
2489 }
2490
2491 /* Queue sizes, capped at 4k */
2492 smmu->cmdq.q.max_n_shift = min((u32)CMDQ_MAX_SZ_SHIFT,
2493 reg >> IDR1_CMDQ_SHIFT & IDR1_CMDQ_MASK);
2494 if (!smmu->cmdq.q.max_n_shift) {
2495 /* Odd alignment restrictions on the base, so ignore for now */
2496 dev_err(smmu->dev, "unit-length command queue not supported\n");
2497 return -ENXIO;
2498 }
2499
2500 smmu->evtq.q.max_n_shift = min((u32)EVTQ_MAX_SZ_SHIFT,
2501 reg >> IDR1_EVTQ_SHIFT & IDR1_EVTQ_MASK);
2502 smmu->priq.q.max_n_shift = min((u32)PRIQ_MAX_SZ_SHIFT,
2503 reg >> IDR1_PRIQ_SHIFT & IDR1_PRIQ_MASK);
2504
2505 /* SID/SSID sizes */
2506 smmu->ssid_bits = reg >> IDR1_SSID_SHIFT & IDR1_SSID_MASK;
2507 smmu->sid_bits = reg >> IDR1_SID_SHIFT & IDR1_SID_MASK;
2508
2509 /* IDR5 */
2510 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR5);
2511
2512 /* Maximum number of outstanding stalls */
2513 smmu->evtq.max_stalls = reg >> IDR5_STALL_MAX_SHIFT
2514 & IDR5_STALL_MAX_MASK;
2515
2516 /* Page sizes */
2517 if (reg & IDR5_GRAN64K)
2518 pgsize_bitmap |= SZ_64K | SZ_512M;
2519 if (reg & IDR5_GRAN16K)
2520 pgsize_bitmap |= SZ_16K | SZ_32M;
2521 if (reg & IDR5_GRAN4K)
2522 pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
2523
2524 arm_smmu_ops.pgsize_bitmap &= pgsize_bitmap;
2525
2526 /* Output address size */
2527 switch (reg & IDR5_OAS_MASK << IDR5_OAS_SHIFT) {
2528 case IDR5_OAS_32_BIT:
2529 smmu->oas = 32;
2530 break;
2531 case IDR5_OAS_36_BIT:
2532 smmu->oas = 36;
2533 break;
2534 case IDR5_OAS_40_BIT:
2535 smmu->oas = 40;
2536 break;
2537 case IDR5_OAS_42_BIT:
2538 smmu->oas = 42;
2539 break;
2540 case IDR5_OAS_44_BIT:
2541 smmu->oas = 44;
2542 break;
2543 case IDR5_OAS_48_BIT:
2544 smmu->oas = 48;
2545 break;
2546 default:
2547 dev_err(smmu->dev, "unknown output address size!\n");
2548 return -ENXIO;
2549 }
2550
2551 /* Set the DMA mask for our table walker */
2552 if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(smmu->oas)))
2553 dev_warn(smmu->dev,
2554 "failed to set DMA mask for table walker\n");
2555
2556 if (!smmu->ias)
2557 smmu->ias = smmu->oas;
2558
2559 dev_info(smmu->dev, "ias %lu-bit, oas %lu-bit (features 0x%08x)\n",
2560 smmu->ias, smmu->oas, smmu->features);
2561 return 0;
2562}
2563
2564static int arm_smmu_device_dt_probe(struct platform_device *pdev)
2565{
2566 int irq, ret;
2567 struct resource *res;
2568 struct arm_smmu_device *smmu;
2569 struct device *dev = &pdev->dev;
2570
2571 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
2572 if (!smmu) {
2573 dev_err(dev, "failed to allocate arm_smmu_device\n");
2574 return -ENOMEM;
2575 }
2576 smmu->dev = dev;
2577
2578 /* Base address */
2579 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2580 if (resource_size(res) + 1 < SZ_128K) {
2581 dev_err(dev, "MMIO region too small (%pr)\n", res);
2582 return -EINVAL;
2583 }
2584
2585 smmu->base = devm_ioremap_resource(dev, res);
2586 if (IS_ERR(smmu->base))
2587 return PTR_ERR(smmu->base);
2588
2589 /* Interrupt lines */
2590 irq = platform_get_irq_byname(pdev, "eventq");
2591 if (irq > 0)
2592 smmu->evtq.q.irq = irq;
2593
2594 irq = platform_get_irq_byname(pdev, "priq");
2595 if (irq > 0)
2596 smmu->priq.q.irq = irq;
2597
2598 irq = platform_get_irq_byname(pdev, "cmdq-sync");
2599 if (irq > 0)
2600 smmu->cmdq.q.irq = irq;
2601
2602 irq = platform_get_irq_byname(pdev, "gerror");
2603 if (irq > 0)
2604 smmu->gerr_irq = irq;
2605
2606 parse_driver_options(smmu);
2607
2608 /* Probe the h/w */
2609 ret = arm_smmu_device_probe(smmu);
2610 if (ret)
2611 return ret;
2612
2613 /* Initialise in-memory data structures */
2614 ret = arm_smmu_init_structures(smmu);
2615 if (ret)
2616 return ret;
2617
2618 /* Reset the device */
2619 ret = arm_smmu_device_reset(smmu);
2620 if (ret)
2621 goto out_free_structures;
2622
2623 /* Record our private device structure */
2624 INIT_LIST_HEAD(&smmu->list);
2625 spin_lock(&arm_smmu_devices_lock);
2626 list_add(&smmu->list, &arm_smmu_devices);
2627 spin_unlock(&arm_smmu_devices_lock);
2628 return 0;
2629
2630out_free_structures:
2631 arm_smmu_free_structures(smmu);
2632 return ret;
2633}
2634
2635static int arm_smmu_device_remove(struct platform_device *pdev)
2636{
2637 struct arm_smmu_device *curr, *smmu = NULL;
2638 struct device *dev = &pdev->dev;
2639
2640 spin_lock(&arm_smmu_devices_lock);
2641 list_for_each_entry(curr, &arm_smmu_devices, list) {
2642 if (curr->dev == dev) {
2643 smmu = curr;
2644 list_del(&smmu->list);
2645 break;
2646 }
2647 }
2648 spin_unlock(&arm_smmu_devices_lock);
2649
2650 if (!smmu)
2651 return -ENODEV;
2652
2653 arm_smmu_device_disable(smmu);
2654 arm_smmu_free_structures(smmu);
2655 return 0;
2656}
2657
2658static struct of_device_id arm_smmu_of_match[] = {
2659 { .compatible = "arm,smmu-v3", },
2660 { },
2661};
2662MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
2663
2664static struct platform_driver arm_smmu_driver = {
2665 .driver = {
2666 .name = "arm-smmu-v3",
2667 .of_match_table = of_match_ptr(arm_smmu_of_match),
2668 },
2669 .probe = arm_smmu_device_dt_probe,
2670 .remove = arm_smmu_device_remove,
2671};
2672
2673static int __init arm_smmu_init(void)
2674{
2675 struct device_node *np;
2676 int ret;
2677
2678 np = of_find_matching_node(NULL, arm_smmu_of_match);
2679 if (!np)
2680 return 0;
2681
2682 of_node_put(np);
2683
2684 ret = platform_driver_register(&arm_smmu_driver);
2685 if (ret)
2686 return ret;
2687
2688 return bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
2689}
2690
2691static void __exit arm_smmu_exit(void)
2692{
2693 return platform_driver_unregister(&arm_smmu_driver);
2694}
2695
2696subsys_initcall(arm_smmu_init);
2697module_exit(arm_smmu_exit);
2698
2699MODULE_DESCRIPTION("IOMMU API for ARM architected SMMUv3 implementations");
2700MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>");
2701MODULE_LICENSE("GPL v2");
diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c
index 66a803b9dd3a..4cd0c29cb585 100644
--- a/drivers/iommu/arm-smmu.c
+++ b/drivers/iommu/arm-smmu.c
@@ -202,8 +202,7 @@
202#define ARM_SMMU_CB_S1_TLBIVAL 0x620 202#define ARM_SMMU_CB_S1_TLBIVAL 0x620
203#define ARM_SMMU_CB_S2_TLBIIPAS2 0x630 203#define ARM_SMMU_CB_S2_TLBIIPAS2 0x630
204#define ARM_SMMU_CB_S2_TLBIIPAS2L 0x638 204#define ARM_SMMU_CB_S2_TLBIIPAS2L 0x638
205#define ARM_SMMU_CB_ATS1PR_LO 0x800 205#define ARM_SMMU_CB_ATS1PR 0x800
206#define ARM_SMMU_CB_ATS1PR_HI 0x804
207#define ARM_SMMU_CB_ATSR 0x8f0 206#define ARM_SMMU_CB_ATSR 0x8f0
208 207
209#define SCTLR_S1_ASIDPNE (1 << 12) 208#define SCTLR_S1_ASIDPNE (1 << 12)
@@ -247,7 +246,7 @@
247#define FSYNR0_WNR (1 << 4) 246#define FSYNR0_WNR (1 << 4)
248 247
249static int force_stage; 248static int force_stage;
250module_param_named(force_stage, force_stage, int, S_IRUGO | S_IWUSR); 249module_param_named(force_stage, force_stage, int, S_IRUGO);
251MODULE_PARM_DESC(force_stage, 250MODULE_PARM_DESC(force_stage,
252 "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation."); 251 "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
253 252
@@ -1229,18 +1228,18 @@ static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1229 void __iomem *cb_base; 1228 void __iomem *cb_base;
1230 u32 tmp; 1229 u32 tmp;
1231 u64 phys; 1230 u64 phys;
1231 unsigned long va;
1232 1232
1233 cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx); 1233 cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
1234 1234
1235 if (smmu->version == 1) { 1235 /* ATS1 registers can only be written atomically */
1236 u32 reg = iova & ~0xfff; 1236 va = iova & ~0xfffUL;
1237 writel_relaxed(reg, cb_base + ARM_SMMU_CB_ATS1PR_LO); 1237#ifdef CONFIG_64BIT
1238 } else { 1238 if (smmu->version == ARM_SMMU_V2)
1239 u32 reg = iova & ~0xfff; 1239 writeq_relaxed(va, cb_base + ARM_SMMU_CB_ATS1PR);
1240 writel_relaxed(reg, cb_base + ARM_SMMU_CB_ATS1PR_LO); 1240 else
1241 reg = ((u64)iova & ~0xfff) >> 32; 1241#endif
1242 writel_relaxed(reg, cb_base + ARM_SMMU_CB_ATS1PR_HI); 1242 writel_relaxed(va, cb_base + ARM_SMMU_CB_ATS1PR);
1243 }
1244 1243
1245 if (readl_poll_timeout_atomic(cb_base + ARM_SMMU_CB_ATSR, tmp, 1244 if (readl_poll_timeout_atomic(cb_base + ARM_SMMU_CB_ATSR, tmp,
1246 !(tmp & ATSR_ACTIVE), 5, 50)) { 1245 !(tmp & ATSR_ACTIVE), 5, 50)) {
@@ -1567,7 +1566,7 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1567 return -ENODEV; 1566 return -ENODEV;
1568 } 1567 }
1569 1568
1570 if ((id & ID0_S1TS) && ((smmu->version == 1) || (id & ID0_ATOSNS))) { 1569 if ((id & ID0_S1TS) && ((smmu->version == 1) || !(id & ID0_ATOSNS))) {
1571 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS; 1570 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1572 dev_notice(smmu->dev, "\taddress translation ops\n"); 1571 dev_notice(smmu->dev, "\taddress translation ops\n");
1573 } 1572 }
diff --git a/drivers/iommu/dmar.c b/drivers/iommu/dmar.c
index 536f2d8ea41a..c9db04d4ef39 100644
--- a/drivers/iommu/dmar.c
+++ b/drivers/iommu/dmar.c
@@ -26,7 +26,7 @@
26 * These routines are used by both DMA-remapping and Interrupt-remapping 26 * These routines are used by both DMA-remapping and Interrupt-remapping
27 */ 27 */
28 28
29#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt /* has to precede printk.h */ 29#define pr_fmt(fmt) "DMAR: " fmt
30 30
31#include <linux/pci.h> 31#include <linux/pci.h>
32#include <linux/dmar.h> 32#include <linux/dmar.h>
@@ -555,7 +555,7 @@ static int dmar_walk_remapping_entries(struct acpi_dmar_header *start,
555 break; 555 break;
556 } else if (next > end) { 556 } else if (next > end) {
557 /* Avoid passing table end */ 557 /* Avoid passing table end */
558 pr_warn(FW_BUG "record passes table end\n"); 558 pr_warn(FW_BUG "Record passes table end\n");
559 ret = -EINVAL; 559 ret = -EINVAL;
560 break; 560 break;
561 } 561 }
@@ -802,7 +802,7 @@ int __init dmar_table_init(void)
802 ret = parse_dmar_table(); 802 ret = parse_dmar_table();
803 if (ret < 0) { 803 if (ret < 0) {
804 if (ret != -ENODEV) 804 if (ret != -ENODEV)
805 pr_info("parse DMAR table failure.\n"); 805 pr_info("Parse DMAR table failure.\n");
806 } else if (list_empty(&dmar_drhd_units)) { 806 } else if (list_empty(&dmar_drhd_units)) {
807 pr_info("No DMAR devices found\n"); 807 pr_info("No DMAR devices found\n");
808 ret = -ENODEV; 808 ret = -ENODEV;
@@ -847,7 +847,7 @@ dmar_validate_one_drhd(struct acpi_dmar_header *entry, void *arg)
847 else 847 else
848 addr = early_ioremap(drhd->address, VTD_PAGE_SIZE); 848 addr = early_ioremap(drhd->address, VTD_PAGE_SIZE);
849 if (!addr) { 849 if (!addr) {
850 pr_warn("IOMMU: can't validate: %llx\n", drhd->address); 850 pr_warn("Can't validate DRHD address: %llx\n", drhd->address);
851 return -EINVAL; 851 return -EINVAL;
852 } 852 }
853 853
@@ -921,14 +921,14 @@ static int map_iommu(struct intel_iommu *iommu, u64 phys_addr)
921 iommu->reg_size = VTD_PAGE_SIZE; 921 iommu->reg_size = VTD_PAGE_SIZE;
922 922
923 if (!request_mem_region(iommu->reg_phys, iommu->reg_size, iommu->name)) { 923 if (!request_mem_region(iommu->reg_phys, iommu->reg_size, iommu->name)) {
924 pr_err("IOMMU: can't reserve memory\n"); 924 pr_err("Can't reserve memory\n");
925 err = -EBUSY; 925 err = -EBUSY;
926 goto out; 926 goto out;
927 } 927 }
928 928
929 iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size); 929 iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
930 if (!iommu->reg) { 930 if (!iommu->reg) {
931 pr_err("IOMMU: can't map the region\n"); 931 pr_err("Can't map the region\n");
932 err = -ENOMEM; 932 err = -ENOMEM;
933 goto release; 933 goto release;
934 } 934 }
@@ -952,13 +952,13 @@ static int map_iommu(struct intel_iommu *iommu, u64 phys_addr)
952 iommu->reg_size = map_size; 952 iommu->reg_size = map_size;
953 if (!request_mem_region(iommu->reg_phys, iommu->reg_size, 953 if (!request_mem_region(iommu->reg_phys, iommu->reg_size,
954 iommu->name)) { 954 iommu->name)) {
955 pr_err("IOMMU: can't reserve memory\n"); 955 pr_err("Can't reserve memory\n");
956 err = -EBUSY; 956 err = -EBUSY;
957 goto out; 957 goto out;
958 } 958 }
959 iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size); 959 iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
960 if (!iommu->reg) { 960 if (!iommu->reg) {
961 pr_err("IOMMU: can't map the region\n"); 961 pr_err("Can't map the region\n");
962 err = -ENOMEM; 962 err = -ENOMEM;
963 goto release; 963 goto release;
964 } 964 }
@@ -1014,14 +1014,14 @@ static int alloc_iommu(struct dmar_drhd_unit *drhd)
1014 return -ENOMEM; 1014 return -ENOMEM;
1015 1015
1016 if (dmar_alloc_seq_id(iommu) < 0) { 1016 if (dmar_alloc_seq_id(iommu) < 0) {
1017 pr_err("IOMMU: failed to allocate seq_id\n"); 1017 pr_err("Failed to allocate seq_id\n");
1018 err = -ENOSPC; 1018 err = -ENOSPC;
1019 goto error; 1019 goto error;
1020 } 1020 }
1021 1021
1022 err = map_iommu(iommu, drhd->reg_base_addr); 1022 err = map_iommu(iommu, drhd->reg_base_addr);
1023 if (err) { 1023 if (err) {
1024 pr_err("IOMMU: failed to map %s\n", iommu->name); 1024 pr_err("Failed to map %s\n", iommu->name);
1025 goto error_free_seq_id; 1025 goto error_free_seq_id;
1026 } 1026 }
1027 1027
@@ -1045,8 +1045,8 @@ static int alloc_iommu(struct dmar_drhd_unit *drhd)
1045 iommu->node = -1; 1045 iommu->node = -1;
1046 1046
1047 ver = readl(iommu->reg + DMAR_VER_REG); 1047 ver = readl(iommu->reg + DMAR_VER_REG);
1048 pr_info("IOMMU %d: reg_base_addr %llx ver %d:%d cap %llx ecap %llx\n", 1048 pr_info("%s: reg_base_addr %llx ver %d:%d cap %llx ecap %llx\n",
1049 iommu->seq_id, 1049 iommu->name,
1050 (unsigned long long)drhd->reg_base_addr, 1050 (unsigned long long)drhd->reg_base_addr,
1051 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver), 1051 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
1052 (unsigned long long)iommu->cap, 1052 (unsigned long long)iommu->cap,
@@ -1646,13 +1646,13 @@ int dmar_set_interrupt(struct intel_iommu *iommu)
1646 if (irq > 0) { 1646 if (irq > 0) {
1647 iommu->irq = irq; 1647 iommu->irq = irq;
1648 } else { 1648 } else {
1649 pr_err("IOMMU: no free vectors\n"); 1649 pr_err("No free IRQ vectors\n");
1650 return -EINVAL; 1650 return -EINVAL;
1651 } 1651 }
1652 1652
1653 ret = request_irq(irq, dmar_fault, IRQF_NO_THREAD, iommu->name, iommu); 1653 ret = request_irq(irq, dmar_fault, IRQF_NO_THREAD, iommu->name, iommu);
1654 if (ret) 1654 if (ret)
1655 pr_err("IOMMU: can't request irq\n"); 1655 pr_err("Can't request irq\n");
1656 return ret; 1656 return ret;
1657} 1657}
1658 1658
diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c
index 3e898504a7c4..97c41b8ab5d9 100644
--- a/drivers/iommu/exynos-iommu.c
+++ b/drivers/iommu/exynos-iommu.c
@@ -12,21 +12,22 @@
12#define DEBUG 12#define DEBUG
13#endif 13#endif
14 14
15#include <linux/io.h>
16#include <linux/interrupt.h>
17#include <linux/platform_device.h>
18#include <linux/slab.h>
19#include <linux/pm_runtime.h>
20#include <linux/clk.h> 15#include <linux/clk.h>
16#include <linux/dma-mapping.h>
21#include <linux/err.h> 17#include <linux/err.h>
22#include <linux/mm.h> 18#include <linux/io.h>
23#include <linux/iommu.h> 19#include <linux/iommu.h>
24#include <linux/errno.h> 20#include <linux/interrupt.h>
25#include <linux/list.h> 21#include <linux/list.h>
26#include <linux/memblock.h> 22#include <linux/of.h>
27#include <linux/export.h> 23#include <linux/of_iommu.h>
24#include <linux/of_platform.h>
25#include <linux/platform_device.h>
26#include <linux/pm_runtime.h>
27#include <linux/slab.h>
28 28
29#include <asm/cacheflush.h> 29#include <asm/cacheflush.h>
30#include <asm/dma-iommu.h>
30#include <asm/pgtable.h> 31#include <asm/pgtable.h>
31 32
32typedef u32 sysmmu_iova_t; 33typedef u32 sysmmu_iova_t;
@@ -184,35 +185,50 @@ static char *sysmmu_fault_name[SYSMMU_FAULTS_NUM] = {
184 "UNKNOWN FAULT" 185 "UNKNOWN FAULT"
185}; 186};
186 187
187/* attached to dev.archdata.iommu of the master device */ 188/*
189 * This structure is attached to dev.archdata.iommu of the master device
190 * on device add, contains a list of SYSMMU controllers defined by device tree,
191 * which are bound to given master device. It is usually referenced by 'owner'
192 * pointer.
193*/
188struct exynos_iommu_owner { 194struct exynos_iommu_owner {
189 struct list_head client; /* entry of exynos_iommu_domain.clients */ 195 struct list_head controllers; /* list of sysmmu_drvdata.owner_node */
190 struct device *dev;
191 struct device *sysmmu;
192 struct iommu_domain *domain;
193 void *vmm_data; /* IO virtual memory manager's data */
194 spinlock_t lock; /* Lock to preserve consistency of System MMU */
195}; 196};
196 197
198/*
199 * This structure exynos specific generalization of struct iommu_domain.
200 * It contains list of SYSMMU controllers from all master devices, which has
201 * been attached to this domain and page tables of IO address space defined by
202 * it. It is usually referenced by 'domain' pointer.
203 */
197struct exynos_iommu_domain { 204struct exynos_iommu_domain {
198 struct list_head clients; /* list of sysmmu_drvdata.node */ 205 struct list_head clients; /* list of sysmmu_drvdata.domain_node */
199 sysmmu_pte_t *pgtable; /* lv1 page table, 16KB */ 206 sysmmu_pte_t *pgtable; /* lv1 page table, 16KB */
200 short *lv2entcnt; /* free lv2 entry counter for each section */ 207 short *lv2entcnt; /* free lv2 entry counter for each section */
201 spinlock_t lock; /* lock for this structure */ 208 spinlock_t lock; /* lock for modyfying list of clients */
202 spinlock_t pgtablelock; /* lock for modifying page table @ pgtable */ 209 spinlock_t pgtablelock; /* lock for modifying page table @ pgtable */
203 struct iommu_domain domain; /* generic domain data structure */ 210 struct iommu_domain domain; /* generic domain data structure */
204}; 211};
205 212
213/*
214 * This structure hold all data of a single SYSMMU controller, this includes
215 * hw resources like registers and clocks, pointers and list nodes to connect
216 * it to all other structures, internal state and parameters read from device
217 * tree. It is usually referenced by 'data' pointer.
218 */
206struct sysmmu_drvdata { 219struct sysmmu_drvdata {
207 struct device *sysmmu; /* System MMU's device descriptor */ 220 struct device *sysmmu; /* SYSMMU controller device */
208 struct device *master; /* Owner of system MMU */ 221 struct device *master; /* master device (owner) */
209 void __iomem *sfrbase; 222 void __iomem *sfrbase; /* our registers */
210 struct clk *clk; 223 struct clk *clk; /* SYSMMU's clock */
211 struct clk *clk_master; 224 struct clk *clk_master; /* master's device clock */
212 int activations; 225 int activations; /* number of calls to sysmmu_enable */
213 spinlock_t lock; 226 spinlock_t lock; /* lock for modyfying state */
214 struct iommu_domain *domain; 227 struct exynos_iommu_domain *domain; /* domain we belong to */
215 phys_addr_t pgtable; 228 struct list_head domain_node; /* node for domain clients list */
229 struct list_head owner_node; /* node for owner controllers list */
230 phys_addr_t pgtable; /* assigned page table structure */
231 unsigned int version; /* our version */
216}; 232};
217 233
218static struct exynos_iommu_domain *to_exynos_domain(struct iommu_domain *dom) 234static struct exynos_iommu_domain *to_exynos_domain(struct iommu_domain *dom)
@@ -244,11 +260,6 @@ static void sysmmu_unblock(void __iomem *sfrbase)
244 __raw_writel(CTRL_ENABLE, sfrbase + REG_MMU_CTRL); 260 __raw_writel(CTRL_ENABLE, sfrbase + REG_MMU_CTRL);
245} 261}
246 262
247static unsigned int __raw_sysmmu_version(struct sysmmu_drvdata *data)
248{
249 return MMU_RAW_VER(__raw_readl(data->sfrbase + REG_MMU_VERSION));
250}
251
252static bool sysmmu_block(void __iomem *sfrbase) 263static bool sysmmu_block(void __iomem *sfrbase)
253{ 264{
254 int i = 120; 265 int i = 120;
@@ -345,7 +356,7 @@ static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id)
345 show_fault_information(dev_name(data->sysmmu), 356 show_fault_information(dev_name(data->sysmmu),
346 itype, base, addr); 357 itype, base, addr);
347 if (data->domain) 358 if (data->domain)
348 ret = report_iommu_fault(data->domain, 359 ret = report_iommu_fault(&data->domain->domain,
349 data->master, addr, itype); 360 data->master, addr, itype);
350 } 361 }
351 362
@@ -408,7 +419,7 @@ static void __sysmmu_init_config(struct sysmmu_drvdata *data)
408 unsigned int cfg = CFG_LRU | CFG_QOS(15); 419 unsigned int cfg = CFG_LRU | CFG_QOS(15);
409 unsigned int ver; 420 unsigned int ver;
410 421
411 ver = __raw_sysmmu_version(data); 422 ver = MMU_RAW_VER(__raw_readl(data->sfrbase + REG_MMU_VERSION));
412 if (MMU_MAJ_VER(ver) == 3) { 423 if (MMU_MAJ_VER(ver) == 3) {
413 if (MMU_MIN_VER(ver) >= 2) { 424 if (MMU_MIN_VER(ver) >= 2) {
414 cfg |= CFG_FLPDCACHE; 425 cfg |= CFG_FLPDCACHE;
@@ -422,6 +433,7 @@ static void __sysmmu_init_config(struct sysmmu_drvdata *data)
422 } 433 }
423 434
424 __raw_writel(cfg, data->sfrbase + REG_MMU_CFG); 435 __raw_writel(cfg, data->sfrbase + REG_MMU_CFG);
436 data->version = ver;
425} 437}
426 438
427static void __sysmmu_enable_nocount(struct sysmmu_drvdata *data) 439static void __sysmmu_enable_nocount(struct sysmmu_drvdata *data)
@@ -442,8 +454,8 @@ static void __sysmmu_enable_nocount(struct sysmmu_drvdata *data)
442 clk_disable(data->clk_master); 454 clk_disable(data->clk_master);
443} 455}
444 456
445static int __sysmmu_enable(struct sysmmu_drvdata *data, 457static int __sysmmu_enable(struct sysmmu_drvdata *data, phys_addr_t pgtable,
446 phys_addr_t pgtable, struct iommu_domain *domain) 458 struct exynos_iommu_domain *domain)
447{ 459{
448 int ret = 0; 460 int ret = 0;
449 unsigned long flags; 461 unsigned long flags;
@@ -470,77 +482,17 @@ static int __sysmmu_enable(struct sysmmu_drvdata *data,
470 return ret; 482 return ret;
471} 483}
472 484
473/* __exynos_sysmmu_enable: Enables System MMU
474 *
475 * returns -error if an error occurred and System MMU is not enabled,
476 * 0 if the System MMU has been just enabled and 1 if System MMU was already
477 * enabled before.
478 */
479static int __exynos_sysmmu_enable(struct device *dev, phys_addr_t pgtable,
480 struct iommu_domain *domain)
481{
482 int ret = 0;
483 unsigned long flags;
484 struct exynos_iommu_owner *owner = dev->archdata.iommu;
485 struct sysmmu_drvdata *data;
486
487 BUG_ON(!has_sysmmu(dev));
488
489 spin_lock_irqsave(&owner->lock, flags);
490
491 data = dev_get_drvdata(owner->sysmmu);
492
493 ret = __sysmmu_enable(data, pgtable, domain);
494 if (ret >= 0)
495 data->master = dev;
496
497 spin_unlock_irqrestore(&owner->lock, flags);
498
499 return ret;
500}
501
502int exynos_sysmmu_enable(struct device *dev, phys_addr_t pgtable)
503{
504 BUG_ON(!memblock_is_memory(pgtable));
505
506 return __exynos_sysmmu_enable(dev, pgtable, NULL);
507}
508
509static bool exynos_sysmmu_disable(struct device *dev)
510{
511 unsigned long flags;
512 bool disabled = true;
513 struct exynos_iommu_owner *owner = dev->archdata.iommu;
514 struct sysmmu_drvdata *data;
515
516 BUG_ON(!has_sysmmu(dev));
517
518 spin_lock_irqsave(&owner->lock, flags);
519
520 data = dev_get_drvdata(owner->sysmmu);
521
522 disabled = __sysmmu_disable(data);
523 if (disabled)
524 data->master = NULL;
525
526 spin_unlock_irqrestore(&owner->lock, flags);
527
528 return disabled;
529}
530
531static void __sysmmu_tlb_invalidate_flpdcache(struct sysmmu_drvdata *data, 485static void __sysmmu_tlb_invalidate_flpdcache(struct sysmmu_drvdata *data,
532 sysmmu_iova_t iova) 486 sysmmu_iova_t iova)
533{ 487{
534 if (__raw_sysmmu_version(data) == MAKE_MMU_VER(3, 3)) 488 if (data->version == MAKE_MMU_VER(3, 3))
535 __raw_writel(iova | 0x1, data->sfrbase + REG_MMU_FLUSH_ENTRY); 489 __raw_writel(iova | 0x1, data->sfrbase + REG_MMU_FLUSH_ENTRY);
536} 490}
537 491
538static void sysmmu_tlb_invalidate_flpdcache(struct device *dev, 492static void sysmmu_tlb_invalidate_flpdcache(struct sysmmu_drvdata *data,
539 sysmmu_iova_t iova) 493 sysmmu_iova_t iova)
540{ 494{
541 unsigned long flags; 495 unsigned long flags;
542 struct exynos_iommu_owner *owner = dev->archdata.iommu;
543 struct sysmmu_drvdata *data = dev_get_drvdata(owner->sysmmu);
544 496
545 if (!IS_ERR(data->clk_master)) 497 if (!IS_ERR(data->clk_master))
546 clk_enable(data->clk_master); 498 clk_enable(data->clk_master);
@@ -554,14 +506,10 @@ static void sysmmu_tlb_invalidate_flpdcache(struct device *dev,
554 clk_disable(data->clk_master); 506 clk_disable(data->clk_master);
555} 507}
556 508
557static void sysmmu_tlb_invalidate_entry(struct device *dev, sysmmu_iova_t iova, 509static void sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data,
558 size_t size) 510 sysmmu_iova_t iova, size_t size)
559{ 511{
560 struct exynos_iommu_owner *owner = dev->archdata.iommu;
561 unsigned long flags; 512 unsigned long flags;
562 struct sysmmu_drvdata *data;
563
564 data = dev_get_drvdata(owner->sysmmu);
565 513
566 spin_lock_irqsave(&data->lock, flags); 514 spin_lock_irqsave(&data->lock, flags);
567 if (is_sysmmu_active(data)) { 515 if (is_sysmmu_active(data)) {
@@ -580,7 +528,7 @@ static void sysmmu_tlb_invalidate_entry(struct device *dev, sysmmu_iova_t iova,
580 * 1MB page can be cached in one of all sets. 528 * 1MB page can be cached in one of all sets.
581 * 64KB page can be one of 16 consecutive sets. 529 * 64KB page can be one of 16 consecutive sets.
582 */ 530 */
583 if (MMU_MAJ_VER(__raw_sysmmu_version(data)) == 2) 531 if (MMU_MAJ_VER(data->version) == 2)
584 num_inv = min_t(unsigned int, size / PAGE_SIZE, 64); 532 num_inv = min_t(unsigned int, size / PAGE_SIZE, 64);
585 533
586 if (sysmmu_block(data->sfrbase)) { 534 if (sysmmu_block(data->sfrbase)) {
@@ -591,32 +539,8 @@ static void sysmmu_tlb_invalidate_entry(struct device *dev, sysmmu_iova_t iova,
591 if (!IS_ERR(data->clk_master)) 539 if (!IS_ERR(data->clk_master))
592 clk_disable(data->clk_master); 540 clk_disable(data->clk_master);
593 } else { 541 } else {
594 dev_dbg(dev, "disabled. Skipping TLB invalidation @ %#x\n", 542 dev_dbg(data->master,
595 iova); 543 "disabled. Skipping TLB invalidation @ %#x\n", iova);
596 }
597 spin_unlock_irqrestore(&data->lock, flags);
598}
599
600void exynos_sysmmu_tlb_invalidate(struct device *dev)
601{
602 struct exynos_iommu_owner *owner = dev->archdata.iommu;
603 unsigned long flags;
604 struct sysmmu_drvdata *data;
605
606 data = dev_get_drvdata(owner->sysmmu);
607
608 spin_lock_irqsave(&data->lock, flags);
609 if (is_sysmmu_active(data)) {
610 if (!IS_ERR(data->clk_master))
611 clk_enable(data->clk_master);
612 if (sysmmu_block(data->sfrbase)) {
613 __sysmmu_tlb_invalidate(data->sfrbase);
614 sysmmu_unblock(data->sfrbase);
615 }
616 if (!IS_ERR(data->clk_master))
617 clk_disable(data->clk_master);
618 } else {
619 dev_dbg(dev, "disabled. Skipping TLB invalidation\n");
620 } 544 }
621 spin_unlock_irqrestore(&data->lock, flags); 545 spin_unlock_irqrestore(&data->lock, flags);
622} 546}
@@ -682,6 +606,36 @@ static int __init exynos_sysmmu_probe(struct platform_device *pdev)
682 return 0; 606 return 0;
683} 607}
684 608
609#ifdef CONFIG_PM_SLEEP
610static int exynos_sysmmu_suspend(struct device *dev)
611{
612 struct sysmmu_drvdata *data = dev_get_drvdata(dev);
613
614 dev_dbg(dev, "suspend\n");
615 if (is_sysmmu_active(data)) {
616 __sysmmu_disable_nocount(data);
617 pm_runtime_put(dev);
618 }
619 return 0;
620}
621
622static int exynos_sysmmu_resume(struct device *dev)
623{
624 struct sysmmu_drvdata *data = dev_get_drvdata(dev);
625
626 dev_dbg(dev, "resume\n");
627 if (is_sysmmu_active(data)) {
628 pm_runtime_get_sync(dev);
629 __sysmmu_enable_nocount(data);
630 }
631 return 0;
632}
633#endif
634
635static const struct dev_pm_ops sysmmu_pm_ops = {
636 SET_LATE_SYSTEM_SLEEP_PM_OPS(exynos_sysmmu_suspend, exynos_sysmmu_resume)
637};
638
685static const struct of_device_id sysmmu_of_match[] __initconst = { 639static const struct of_device_id sysmmu_of_match[] __initconst = {
686 { .compatible = "samsung,exynos-sysmmu", }, 640 { .compatible = "samsung,exynos-sysmmu", },
687 { }, 641 { },
@@ -692,6 +646,7 @@ static struct platform_driver exynos_sysmmu_driver __refdata = {
692 .driver = { 646 .driver = {
693 .name = "exynos-sysmmu", 647 .name = "exynos-sysmmu",
694 .of_match_table = sysmmu_of_match, 648 .of_match_table = sysmmu_of_match,
649 .pm = &sysmmu_pm_ops,
695 } 650 }
696}; 651};
697 652
@@ -704,104 +659,108 @@ static inline void pgtable_flush(void *vastart, void *vaend)
704 659
705static struct iommu_domain *exynos_iommu_domain_alloc(unsigned type) 660static struct iommu_domain *exynos_iommu_domain_alloc(unsigned type)
706{ 661{
707 struct exynos_iommu_domain *exynos_domain; 662 struct exynos_iommu_domain *domain;
708 int i; 663 int i;
709 664
710 if (type != IOMMU_DOMAIN_UNMANAGED) 665 if (type != IOMMU_DOMAIN_UNMANAGED)
711 return NULL; 666 return NULL;
712 667
713 exynos_domain = kzalloc(sizeof(*exynos_domain), GFP_KERNEL); 668 domain = kzalloc(sizeof(*domain), GFP_KERNEL);
714 if (!exynos_domain) 669 if (!domain)
715 return NULL; 670 return NULL;
716 671
717 exynos_domain->pgtable = (sysmmu_pte_t *)__get_free_pages(GFP_KERNEL, 2); 672 domain->pgtable = (sysmmu_pte_t *)__get_free_pages(GFP_KERNEL, 2);
718 if (!exynos_domain->pgtable) 673 if (!domain->pgtable)
719 goto err_pgtable; 674 goto err_pgtable;
720 675
721 exynos_domain->lv2entcnt = (short *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 1); 676 domain->lv2entcnt = (short *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 1);
722 if (!exynos_domain->lv2entcnt) 677 if (!domain->lv2entcnt)
723 goto err_counter; 678 goto err_counter;
724 679
725 /* Workaround for System MMU v3.3 to prevent caching 1MiB mapping */ 680 /* Workaround for System MMU v3.3 to prevent caching 1MiB mapping */
726 for (i = 0; i < NUM_LV1ENTRIES; i += 8) { 681 for (i = 0; i < NUM_LV1ENTRIES; i += 8) {
727 exynos_domain->pgtable[i + 0] = ZERO_LV2LINK; 682 domain->pgtable[i + 0] = ZERO_LV2LINK;
728 exynos_domain->pgtable[i + 1] = ZERO_LV2LINK; 683 domain->pgtable[i + 1] = ZERO_LV2LINK;
729 exynos_domain->pgtable[i + 2] = ZERO_LV2LINK; 684 domain->pgtable[i + 2] = ZERO_LV2LINK;
730 exynos_domain->pgtable[i + 3] = ZERO_LV2LINK; 685 domain->pgtable[i + 3] = ZERO_LV2LINK;
731 exynos_domain->pgtable[i + 4] = ZERO_LV2LINK; 686 domain->pgtable[i + 4] = ZERO_LV2LINK;
732 exynos_domain->pgtable[i + 5] = ZERO_LV2LINK; 687 domain->pgtable[i + 5] = ZERO_LV2LINK;
733 exynos_domain->pgtable[i + 6] = ZERO_LV2LINK; 688 domain->pgtable[i + 6] = ZERO_LV2LINK;
734 exynos_domain->pgtable[i + 7] = ZERO_LV2LINK; 689 domain->pgtable[i + 7] = ZERO_LV2LINK;
735 } 690 }
736 691
737 pgtable_flush(exynos_domain->pgtable, exynos_domain->pgtable + NUM_LV1ENTRIES); 692 pgtable_flush(domain->pgtable, domain->pgtable + NUM_LV1ENTRIES);
738 693
739 spin_lock_init(&exynos_domain->lock); 694 spin_lock_init(&domain->lock);
740 spin_lock_init(&exynos_domain->pgtablelock); 695 spin_lock_init(&domain->pgtablelock);
741 INIT_LIST_HEAD(&exynos_domain->clients); 696 INIT_LIST_HEAD(&domain->clients);
742 697
743 exynos_domain->domain.geometry.aperture_start = 0; 698 domain->domain.geometry.aperture_start = 0;
744 exynos_domain->domain.geometry.aperture_end = ~0UL; 699 domain->domain.geometry.aperture_end = ~0UL;
745 exynos_domain->domain.geometry.force_aperture = true; 700 domain->domain.geometry.force_aperture = true;
746 701
747 return &exynos_domain->domain; 702 return &domain->domain;
748 703
749err_counter: 704err_counter:
750 free_pages((unsigned long)exynos_domain->pgtable, 2); 705 free_pages((unsigned long)domain->pgtable, 2);
751err_pgtable: 706err_pgtable:
752 kfree(exynos_domain); 707 kfree(domain);
753 return NULL; 708 return NULL;
754} 709}
755 710
756static void exynos_iommu_domain_free(struct iommu_domain *domain) 711static void exynos_iommu_domain_free(struct iommu_domain *iommu_domain)
757{ 712{
758 struct exynos_iommu_domain *priv = to_exynos_domain(domain); 713 struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain);
759 struct exynos_iommu_owner *owner; 714 struct sysmmu_drvdata *data, *next;
760 unsigned long flags; 715 unsigned long flags;
761 int i; 716 int i;
762 717
763 WARN_ON(!list_empty(&priv->clients)); 718 WARN_ON(!list_empty(&domain->clients));
764 719
765 spin_lock_irqsave(&priv->lock, flags); 720 spin_lock_irqsave(&domain->lock, flags);
766 721
767 list_for_each_entry(owner, &priv->clients, client) { 722 list_for_each_entry_safe(data, next, &domain->clients, domain_node) {
768 while (!exynos_sysmmu_disable(owner->dev)) 723 if (__sysmmu_disable(data))
769 ; /* until System MMU is actually disabled */ 724 data->master = NULL;
725 list_del_init(&data->domain_node);
770 } 726 }
771 727
772 while (!list_empty(&priv->clients)) 728 spin_unlock_irqrestore(&domain->lock, flags);
773 list_del_init(priv->clients.next);
774
775 spin_unlock_irqrestore(&priv->lock, flags);
776 729
777 for (i = 0; i < NUM_LV1ENTRIES; i++) 730 for (i = 0; i < NUM_LV1ENTRIES; i++)
778 if (lv1ent_page(priv->pgtable + i)) 731 if (lv1ent_page(domain->pgtable + i))
779 kmem_cache_free(lv2table_kmem_cache, 732 kmem_cache_free(lv2table_kmem_cache,
780 phys_to_virt(lv2table_base(priv->pgtable + i))); 733 phys_to_virt(lv2table_base(domain->pgtable + i)));
781 734
782 free_pages((unsigned long)priv->pgtable, 2); 735 free_pages((unsigned long)domain->pgtable, 2);
783 free_pages((unsigned long)priv->lv2entcnt, 1); 736 free_pages((unsigned long)domain->lv2entcnt, 1);
784 kfree(priv); 737 kfree(domain);
785} 738}
786 739
787static int exynos_iommu_attach_device(struct iommu_domain *domain, 740static int exynos_iommu_attach_device(struct iommu_domain *iommu_domain,
788 struct device *dev) 741 struct device *dev)
789{ 742{
790 struct exynos_iommu_owner *owner = dev->archdata.iommu; 743 struct exynos_iommu_owner *owner = dev->archdata.iommu;
791 struct exynos_iommu_domain *priv = to_exynos_domain(domain); 744 struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain);
792 phys_addr_t pagetable = virt_to_phys(priv->pgtable); 745 struct sysmmu_drvdata *data;
746 phys_addr_t pagetable = virt_to_phys(domain->pgtable);
793 unsigned long flags; 747 unsigned long flags;
794 int ret; 748 int ret = -ENODEV;
795 749
796 spin_lock_irqsave(&priv->lock, flags); 750 if (!has_sysmmu(dev))
751 return -ENODEV;
797 752
798 ret = __exynos_sysmmu_enable(dev, pagetable, domain); 753 list_for_each_entry(data, &owner->controllers, owner_node) {
799 if (ret == 0) { 754 pm_runtime_get_sync(data->sysmmu);
800 list_add_tail(&owner->client, &priv->clients); 755 ret = __sysmmu_enable(data, pagetable, domain);
801 owner->domain = domain; 756 if (ret >= 0) {
802 } 757 data->master = dev;
803 758
804 spin_unlock_irqrestore(&priv->lock, flags); 759 spin_lock_irqsave(&domain->lock, flags);
760 list_add_tail(&data->domain_node, &domain->clients);
761 spin_unlock_irqrestore(&domain->lock, flags);
762 }
763 }
805 764
806 if (ret < 0) { 765 if (ret < 0) {
807 dev_err(dev, "%s: Failed to attach IOMMU with pgtable %pa\n", 766 dev_err(dev, "%s: Failed to attach IOMMU with pgtable %pa\n",
@@ -815,36 +774,39 @@ static int exynos_iommu_attach_device(struct iommu_domain *domain,
815 return ret; 774 return ret;
816} 775}
817 776
818static void exynos_iommu_detach_device(struct iommu_domain *domain, 777static void exynos_iommu_detach_device(struct iommu_domain *iommu_domain,
819 struct device *dev) 778 struct device *dev)
820{ 779{
821 struct exynos_iommu_owner *owner; 780 struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain);
822 struct exynos_iommu_domain *priv = to_exynos_domain(domain); 781 phys_addr_t pagetable = virt_to_phys(domain->pgtable);
823 phys_addr_t pagetable = virt_to_phys(priv->pgtable); 782 struct sysmmu_drvdata *data, *next;
824 unsigned long flags; 783 unsigned long flags;
784 bool found = false;
825 785
826 spin_lock_irqsave(&priv->lock, flags); 786 if (!has_sysmmu(dev))
787 return;
827 788
828 list_for_each_entry(owner, &priv->clients, client) { 789 spin_lock_irqsave(&domain->lock, flags);
829 if (owner == dev->archdata.iommu) { 790 list_for_each_entry_safe(data, next, &domain->clients, domain_node) {
830 if (exynos_sysmmu_disable(dev)) { 791 if (data->master == dev) {
831 list_del_init(&owner->client); 792 if (__sysmmu_disable(data)) {
832 owner->domain = NULL; 793 data->master = NULL;
794 list_del_init(&data->domain_node);
833 } 795 }
834 break; 796 pm_runtime_put(data->sysmmu);
797 found = true;
835 } 798 }
836 } 799 }
800 spin_unlock_irqrestore(&domain->lock, flags);
837 801
838 spin_unlock_irqrestore(&priv->lock, flags); 802 if (found)
839
840 if (owner == dev->archdata.iommu)
841 dev_dbg(dev, "%s: Detached IOMMU with pgtable %pa\n", 803 dev_dbg(dev, "%s: Detached IOMMU with pgtable %pa\n",
842 __func__, &pagetable); 804 __func__, &pagetable);
843 else 805 else
844 dev_err(dev, "%s: No IOMMU is attached\n", __func__); 806 dev_err(dev, "%s: No IOMMU is attached\n", __func__);
845} 807}
846 808
847static sysmmu_pte_t *alloc_lv2entry(struct exynos_iommu_domain *priv, 809static sysmmu_pte_t *alloc_lv2entry(struct exynos_iommu_domain *domain,
848 sysmmu_pte_t *sent, sysmmu_iova_t iova, short *pgcounter) 810 sysmmu_pte_t *sent, sysmmu_iova_t iova, short *pgcounter)
849{ 811{
850 if (lv1ent_section(sent)) { 812 if (lv1ent_section(sent)) {
@@ -862,6 +824,7 @@ static sysmmu_pte_t *alloc_lv2entry(struct exynos_iommu_domain *priv,
862 return ERR_PTR(-ENOMEM); 824 return ERR_PTR(-ENOMEM);
863 825
864 *sent = mk_lv1ent_page(virt_to_phys(pent)); 826 *sent = mk_lv1ent_page(virt_to_phys(pent));
827 kmemleak_ignore(pent);
865 *pgcounter = NUM_LV2ENTRIES; 828 *pgcounter = NUM_LV2ENTRIES;
866 pgtable_flush(pent, pent + NUM_LV2ENTRIES); 829 pgtable_flush(pent, pent + NUM_LV2ENTRIES);
867 pgtable_flush(sent, sent + 1); 830 pgtable_flush(sent, sent + 1);
@@ -884,20 +847,19 @@ static sysmmu_pte_t *alloc_lv2entry(struct exynos_iommu_domain *priv,
884 * not currently mapped. 847 * not currently mapped.
885 */ 848 */
886 if (need_flush_flpd_cache) { 849 if (need_flush_flpd_cache) {
887 struct exynos_iommu_owner *owner; 850 struct sysmmu_drvdata *data;
888 851
889 spin_lock(&priv->lock); 852 spin_lock(&domain->lock);
890 list_for_each_entry(owner, &priv->clients, client) 853 list_for_each_entry(data, &domain->clients, domain_node)
891 sysmmu_tlb_invalidate_flpdcache( 854 sysmmu_tlb_invalidate_flpdcache(data, iova);
892 owner->dev, iova); 855 spin_unlock(&domain->lock);
893 spin_unlock(&priv->lock);
894 } 856 }
895 } 857 }
896 858
897 return page_entry(sent, iova); 859 return page_entry(sent, iova);
898} 860}
899 861
900static int lv1set_section(struct exynos_iommu_domain *priv, 862static int lv1set_section(struct exynos_iommu_domain *domain,
901 sysmmu_pte_t *sent, sysmmu_iova_t iova, 863 sysmmu_pte_t *sent, sysmmu_iova_t iova,
902 phys_addr_t paddr, short *pgcnt) 864 phys_addr_t paddr, short *pgcnt)
903{ 865{
@@ -922,17 +884,17 @@ static int lv1set_section(struct exynos_iommu_domain *priv,
922 884
923 pgtable_flush(sent, sent + 1); 885 pgtable_flush(sent, sent + 1);
924 886
925 spin_lock(&priv->lock); 887 spin_lock(&domain->lock);
926 if (lv1ent_page_zero(sent)) { 888 if (lv1ent_page_zero(sent)) {
927 struct exynos_iommu_owner *owner; 889 struct sysmmu_drvdata *data;
928 /* 890 /*
929 * Flushing FLPD cache in System MMU v3.3 that may cache a FLPD 891 * Flushing FLPD cache in System MMU v3.3 that may cache a FLPD
930 * entry by speculative prefetch of SLPD which has no mapping. 892 * entry by speculative prefetch of SLPD which has no mapping.
931 */ 893 */
932 list_for_each_entry(owner, &priv->clients, client) 894 list_for_each_entry(data, &domain->clients, domain_node)
933 sysmmu_tlb_invalidate_flpdcache(owner->dev, iova); 895 sysmmu_tlb_invalidate_flpdcache(data, iova);
934 } 896 }
935 spin_unlock(&priv->lock); 897 spin_unlock(&domain->lock);
936 898
937 return 0; 899 return 0;
938} 900}
@@ -992,74 +954,75 @@ static int lv2set_page(sysmmu_pte_t *pent, phys_addr_t paddr, size_t size,
992 * than or equal to 128KiB. 954 * than or equal to 128KiB.
993 * - Start address of an I/O virtual region must be aligned by 128KiB. 955 * - Start address of an I/O virtual region must be aligned by 128KiB.
994 */ 956 */
995static int exynos_iommu_map(struct iommu_domain *domain, unsigned long l_iova, 957static int exynos_iommu_map(struct iommu_domain *iommu_domain,
996 phys_addr_t paddr, size_t size, int prot) 958 unsigned long l_iova, phys_addr_t paddr, size_t size,
959 int prot)
997{ 960{
998 struct exynos_iommu_domain *priv = to_exynos_domain(domain); 961 struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain);
999 sysmmu_pte_t *entry; 962 sysmmu_pte_t *entry;
1000 sysmmu_iova_t iova = (sysmmu_iova_t)l_iova; 963 sysmmu_iova_t iova = (sysmmu_iova_t)l_iova;
1001 unsigned long flags; 964 unsigned long flags;
1002 int ret = -ENOMEM; 965 int ret = -ENOMEM;
1003 966
1004 BUG_ON(priv->pgtable == NULL); 967 BUG_ON(domain->pgtable == NULL);
1005 968
1006 spin_lock_irqsave(&priv->pgtablelock, flags); 969 spin_lock_irqsave(&domain->pgtablelock, flags);
1007 970
1008 entry = section_entry(priv->pgtable, iova); 971 entry = section_entry(domain->pgtable, iova);
1009 972
1010 if (size == SECT_SIZE) { 973 if (size == SECT_SIZE) {
1011 ret = lv1set_section(priv, entry, iova, paddr, 974 ret = lv1set_section(domain, entry, iova, paddr,
1012 &priv->lv2entcnt[lv1ent_offset(iova)]); 975 &domain->lv2entcnt[lv1ent_offset(iova)]);
1013 } else { 976 } else {
1014 sysmmu_pte_t *pent; 977 sysmmu_pte_t *pent;
1015 978
1016 pent = alloc_lv2entry(priv, entry, iova, 979 pent = alloc_lv2entry(domain, entry, iova,
1017 &priv->lv2entcnt[lv1ent_offset(iova)]); 980 &domain->lv2entcnt[lv1ent_offset(iova)]);
1018 981
1019 if (IS_ERR(pent)) 982 if (IS_ERR(pent))
1020 ret = PTR_ERR(pent); 983 ret = PTR_ERR(pent);
1021 else 984 else
1022 ret = lv2set_page(pent, paddr, size, 985 ret = lv2set_page(pent, paddr, size,
1023 &priv->lv2entcnt[lv1ent_offset(iova)]); 986 &domain->lv2entcnt[lv1ent_offset(iova)]);
1024 } 987 }
1025 988
1026 if (ret) 989 if (ret)
1027 pr_err("%s: Failed(%d) to map %#zx bytes @ %#x\n", 990 pr_err("%s: Failed(%d) to map %#zx bytes @ %#x\n",
1028 __func__, ret, size, iova); 991 __func__, ret, size, iova);
1029 992
1030 spin_unlock_irqrestore(&priv->pgtablelock, flags); 993 spin_unlock_irqrestore(&domain->pgtablelock, flags);
1031 994
1032 return ret; 995 return ret;
1033} 996}
1034 997
1035static void exynos_iommu_tlb_invalidate_entry(struct exynos_iommu_domain *priv, 998static void exynos_iommu_tlb_invalidate_entry(struct exynos_iommu_domain *domain,
1036 sysmmu_iova_t iova, size_t size) 999 sysmmu_iova_t iova, size_t size)
1037{ 1000{
1038 struct exynos_iommu_owner *owner; 1001 struct sysmmu_drvdata *data;
1039 unsigned long flags; 1002 unsigned long flags;
1040 1003
1041 spin_lock_irqsave(&priv->lock, flags); 1004 spin_lock_irqsave(&domain->lock, flags);
1042 1005
1043 list_for_each_entry(owner, &priv->clients, client) 1006 list_for_each_entry(data, &domain->clients, domain_node)
1044 sysmmu_tlb_invalidate_entry(owner->dev, iova, size); 1007 sysmmu_tlb_invalidate_entry(data, iova, size);
1045 1008
1046 spin_unlock_irqrestore(&priv->lock, flags); 1009 spin_unlock_irqrestore(&domain->lock, flags);
1047} 1010}
1048 1011
1049static size_t exynos_iommu_unmap(struct iommu_domain *domain, 1012static size_t exynos_iommu_unmap(struct iommu_domain *iommu_domain,
1050 unsigned long l_iova, size_t size) 1013 unsigned long l_iova, size_t size)
1051{ 1014{
1052 struct exynos_iommu_domain *priv = to_exynos_domain(domain); 1015 struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain);
1053 sysmmu_iova_t iova = (sysmmu_iova_t)l_iova; 1016 sysmmu_iova_t iova = (sysmmu_iova_t)l_iova;
1054 sysmmu_pte_t *ent; 1017 sysmmu_pte_t *ent;
1055 size_t err_pgsize; 1018 size_t err_pgsize;
1056 unsigned long flags; 1019 unsigned long flags;
1057 1020
1058 BUG_ON(priv->pgtable == NULL); 1021 BUG_ON(domain->pgtable == NULL);
1059 1022
1060 spin_lock_irqsave(&priv->pgtablelock, flags); 1023 spin_lock_irqsave(&domain->pgtablelock, flags);
1061 1024
1062 ent = section_entry(priv->pgtable, iova); 1025 ent = section_entry(domain->pgtable, iova);
1063 1026
1064 if (lv1ent_section(ent)) { 1027 if (lv1ent_section(ent)) {
1065 if (WARN_ON(size < SECT_SIZE)) { 1028 if (WARN_ON(size < SECT_SIZE)) {
@@ -1093,7 +1056,7 @@ static size_t exynos_iommu_unmap(struct iommu_domain *domain,
1093 *ent = 0; 1056 *ent = 0;
1094 size = SPAGE_SIZE; 1057 size = SPAGE_SIZE;
1095 pgtable_flush(ent, ent + 1); 1058 pgtable_flush(ent, ent + 1);
1096 priv->lv2entcnt[lv1ent_offset(iova)] += 1; 1059 domain->lv2entcnt[lv1ent_offset(iova)] += 1;
1097 goto done; 1060 goto done;
1098 } 1061 }
1099 1062
@@ -1107,15 +1070,15 @@ static size_t exynos_iommu_unmap(struct iommu_domain *domain,
1107 pgtable_flush(ent, ent + SPAGES_PER_LPAGE); 1070 pgtable_flush(ent, ent + SPAGES_PER_LPAGE);
1108 1071
1109 size = LPAGE_SIZE; 1072 size = LPAGE_SIZE;
1110 priv->lv2entcnt[lv1ent_offset(iova)] += SPAGES_PER_LPAGE; 1073 domain->lv2entcnt[lv1ent_offset(iova)] += SPAGES_PER_LPAGE;
1111done: 1074done:
1112 spin_unlock_irqrestore(&priv->pgtablelock, flags); 1075 spin_unlock_irqrestore(&domain->pgtablelock, flags);
1113 1076
1114 exynos_iommu_tlb_invalidate_entry(priv, iova, size); 1077 exynos_iommu_tlb_invalidate_entry(domain, iova, size);
1115 1078
1116 return size; 1079 return size;
1117err: 1080err:
1118 spin_unlock_irqrestore(&priv->pgtablelock, flags); 1081 spin_unlock_irqrestore(&domain->pgtablelock, flags);
1119 1082
1120 pr_err("%s: Failed: size(%#zx) @ %#x is smaller than page size %#zx\n", 1083 pr_err("%s: Failed: size(%#zx) @ %#x is smaller than page size %#zx\n",
1121 __func__, size, iova, err_pgsize); 1084 __func__, size, iova, err_pgsize);
@@ -1123,17 +1086,17 @@ err:
1123 return 0; 1086 return 0;
1124} 1087}
1125 1088
1126static phys_addr_t exynos_iommu_iova_to_phys(struct iommu_domain *domain, 1089static phys_addr_t exynos_iommu_iova_to_phys(struct iommu_domain *iommu_domain,
1127 dma_addr_t iova) 1090 dma_addr_t iova)
1128{ 1091{
1129 struct exynos_iommu_domain *priv = to_exynos_domain(domain); 1092 struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain);
1130 sysmmu_pte_t *entry; 1093 sysmmu_pte_t *entry;
1131 unsigned long flags; 1094 unsigned long flags;
1132 phys_addr_t phys = 0; 1095 phys_addr_t phys = 0;
1133 1096
1134 spin_lock_irqsave(&priv->pgtablelock, flags); 1097 spin_lock_irqsave(&domain->pgtablelock, flags);
1135 1098
1136 entry = section_entry(priv->pgtable, iova); 1099 entry = section_entry(domain->pgtable, iova);
1137 1100
1138 if (lv1ent_section(entry)) { 1101 if (lv1ent_section(entry)) {
1139 phys = section_phys(entry) + section_offs(iova); 1102 phys = section_phys(entry) + section_offs(iova);
@@ -1146,7 +1109,7 @@ static phys_addr_t exynos_iommu_iova_to_phys(struct iommu_domain *domain,
1146 phys = spage_phys(entry) + spage_offs(iova); 1109 phys = spage_phys(entry) + spage_offs(iova);
1147 } 1110 }
1148 1111
1149 spin_unlock_irqrestore(&priv->pgtablelock, flags); 1112 spin_unlock_irqrestore(&domain->pgtablelock, flags);
1150 1113
1151 return phys; 1114 return phys;
1152} 1115}
@@ -1156,6 +1119,9 @@ static int exynos_iommu_add_device(struct device *dev)
1156 struct iommu_group *group; 1119 struct iommu_group *group;
1157 int ret; 1120 int ret;
1158 1121
1122 if (!has_sysmmu(dev))
1123 return -ENODEV;
1124
1159 group = iommu_group_get(dev); 1125 group = iommu_group_get(dev);
1160 1126
1161 if (!group) { 1127 if (!group) {
@@ -1174,10 +1140,40 @@ static int exynos_iommu_add_device(struct device *dev)
1174 1140
1175static void exynos_iommu_remove_device(struct device *dev) 1141static void exynos_iommu_remove_device(struct device *dev)
1176{ 1142{
1143 if (!has_sysmmu(dev))
1144 return;
1145
1177 iommu_group_remove_device(dev); 1146 iommu_group_remove_device(dev);
1178} 1147}
1179 1148
1180static const struct iommu_ops exynos_iommu_ops = { 1149static int exynos_iommu_of_xlate(struct device *dev,
1150 struct of_phandle_args *spec)
1151{
1152 struct exynos_iommu_owner *owner = dev->archdata.iommu;
1153 struct platform_device *sysmmu = of_find_device_by_node(spec->np);
1154 struct sysmmu_drvdata *data;
1155
1156 if (!sysmmu)
1157 return -ENODEV;
1158
1159 data = platform_get_drvdata(sysmmu);
1160 if (!data)
1161 return -ENODEV;
1162
1163 if (!owner) {
1164 owner = kzalloc(sizeof(*owner), GFP_KERNEL);
1165 if (!owner)
1166 return -ENOMEM;
1167
1168 INIT_LIST_HEAD(&owner->controllers);
1169 dev->archdata.iommu = owner;
1170 }
1171
1172 list_add_tail(&data->owner_node, &owner->controllers);
1173 return 0;
1174}
1175
1176static struct iommu_ops exynos_iommu_ops = {
1181 .domain_alloc = exynos_iommu_domain_alloc, 1177 .domain_alloc = exynos_iommu_domain_alloc,
1182 .domain_free = exynos_iommu_domain_free, 1178 .domain_free = exynos_iommu_domain_free,
1183 .attach_dev = exynos_iommu_attach_device, 1179 .attach_dev = exynos_iommu_attach_device,
@@ -1189,19 +1185,15 @@ static const struct iommu_ops exynos_iommu_ops = {
1189 .add_device = exynos_iommu_add_device, 1185 .add_device = exynos_iommu_add_device,
1190 .remove_device = exynos_iommu_remove_device, 1186 .remove_device = exynos_iommu_remove_device,
1191 .pgsize_bitmap = SECT_SIZE | LPAGE_SIZE | SPAGE_SIZE, 1187 .pgsize_bitmap = SECT_SIZE | LPAGE_SIZE | SPAGE_SIZE,
1188 .of_xlate = exynos_iommu_of_xlate,
1192}; 1189};
1193 1190
1191static bool init_done;
1192
1194static int __init exynos_iommu_init(void) 1193static int __init exynos_iommu_init(void)
1195{ 1194{
1196 struct device_node *np;
1197 int ret; 1195 int ret;
1198 1196
1199 np = of_find_matching_node(NULL, sysmmu_of_match);
1200 if (!np)
1201 return 0;
1202
1203 of_node_put(np);
1204
1205 lv2table_kmem_cache = kmem_cache_create("exynos-iommu-lv2table", 1197 lv2table_kmem_cache = kmem_cache_create("exynos-iommu-lv2table",
1206 LV2TABLE_SIZE, LV2TABLE_SIZE, 0, NULL); 1198 LV2TABLE_SIZE, LV2TABLE_SIZE, 0, NULL);
1207 if (!lv2table_kmem_cache) { 1199 if (!lv2table_kmem_cache) {
@@ -1230,6 +1222,8 @@ static int __init exynos_iommu_init(void)
1230 goto err_set_iommu; 1222 goto err_set_iommu;
1231 } 1223 }
1232 1224
1225 init_done = true;
1226
1233 return 0; 1227 return 0;
1234err_set_iommu: 1228err_set_iommu:
1235 kmem_cache_free(lv2table_kmem_cache, zero_lv2_table); 1229 kmem_cache_free(lv2table_kmem_cache, zero_lv2_table);
@@ -1239,4 +1233,21 @@ err_reg_driver:
1239 kmem_cache_destroy(lv2table_kmem_cache); 1233 kmem_cache_destroy(lv2table_kmem_cache);
1240 return ret; 1234 return ret;
1241} 1235}
1242subsys_initcall(exynos_iommu_init); 1236
1237static int __init exynos_iommu_of_setup(struct device_node *np)
1238{
1239 struct platform_device *pdev;
1240
1241 if (!init_done)
1242 exynos_iommu_init();
1243
1244 pdev = of_platform_device_create(np, NULL, platform_bus_type.dev_root);
1245 if (IS_ERR(pdev))
1246 return PTR_ERR(pdev);
1247
1248 of_iommu_set_ops(np, &exynos_iommu_ops);
1249 return 0;
1250}
1251
1252IOMMU_OF_DECLARE(exynos_iommu_of, "samsung,exynos-sysmmu",
1253 exynos_iommu_of_setup);
diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c
index 5ecfaf29933a..0649b94f5958 100644
--- a/drivers/iommu/intel-iommu.c
+++ b/drivers/iommu/intel-iommu.c
@@ -15,8 +15,11 @@
15 * Shaohua Li <shaohua.li@intel.com>, 15 * Shaohua Li <shaohua.li@intel.com>,
16 * Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>, 16 * Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
17 * Fenghua Yu <fenghua.yu@intel.com> 17 * Fenghua Yu <fenghua.yu@intel.com>
18 * Joerg Roedel <jroedel@suse.de>
18 */ 19 */
19 20
21#define pr_fmt(fmt) "DMAR: " fmt
22
20#include <linux/init.h> 23#include <linux/init.h>
21#include <linux/bitmap.h> 24#include <linux/bitmap.h>
22#include <linux/debugfs.h> 25#include <linux/debugfs.h>
@@ -40,6 +43,7 @@
40#include <linux/pci-ats.h> 43#include <linux/pci-ats.h>
41#include <linux/memblock.h> 44#include <linux/memblock.h>
42#include <linux/dma-contiguous.h> 45#include <linux/dma-contiguous.h>
46#include <linux/crash_dump.h>
43#include <asm/irq_remapping.h> 47#include <asm/irq_remapping.h>
44#include <asm/cacheflush.h> 48#include <asm/cacheflush.h>
45#include <asm/iommu.h> 49#include <asm/iommu.h>
@@ -190,7 +194,29 @@ struct root_entry {
190}; 194};
191#define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry)) 195#define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
192 196
197/*
198 * Take a root_entry and return the Lower Context Table Pointer (LCTP)
199 * if marked present.
200 */
201static phys_addr_t root_entry_lctp(struct root_entry *re)
202{
203 if (!(re->lo & 1))
204 return 0;
205
206 return re->lo & VTD_PAGE_MASK;
207}
208
209/*
210 * Take a root_entry and return the Upper Context Table Pointer (UCTP)
211 * if marked present.
212 */
213static phys_addr_t root_entry_uctp(struct root_entry *re)
214{
215 if (!(re->hi & 1))
216 return 0;
193 217
218 return re->hi & VTD_PAGE_MASK;
219}
194/* 220/*
195 * low 64 bits: 221 * low 64 bits:
196 * 0: present 222 * 0: present
@@ -207,10 +233,38 @@ struct context_entry {
207 u64 hi; 233 u64 hi;
208}; 234};
209 235
210static inline bool context_present(struct context_entry *context) 236static inline void context_clear_pasid_enable(struct context_entry *context)
237{
238 context->lo &= ~(1ULL << 11);
239}
240
241static inline bool context_pasid_enabled(struct context_entry *context)
242{
243 return !!(context->lo & (1ULL << 11));
244}
245
246static inline void context_set_copied(struct context_entry *context)
247{
248 context->hi |= (1ull << 3);
249}
250
251static inline bool context_copied(struct context_entry *context)
252{
253 return !!(context->hi & (1ULL << 3));
254}
255
256static inline bool __context_present(struct context_entry *context)
211{ 257{
212 return (context->lo & 1); 258 return (context->lo & 1);
213} 259}
260
261static inline bool context_present(struct context_entry *context)
262{
263 return context_pasid_enabled(context) ?
264 __context_present(context) :
265 __context_present(context) && !context_copied(context);
266}
267
214static inline void context_set_present(struct context_entry *context) 268static inline void context_set_present(struct context_entry *context)
215{ 269{
216 context->lo |= 1; 270 context->lo |= 1;
@@ -247,6 +301,11 @@ static inline void context_set_domain_id(struct context_entry *context,
247 context->hi |= (value & ((1 << 16) - 1)) << 8; 301 context->hi |= (value & ((1 << 16) - 1)) << 8;
248} 302}
249 303
304static inline int context_domain_id(struct context_entry *c)
305{
306 return((c->hi >> 8) & 0xffff);
307}
308
250static inline void context_clear_entry(struct context_entry *context) 309static inline void context_clear_entry(struct context_entry *context)
251{ 310{
252 context->lo = 0; 311 context->lo = 0;
@@ -440,6 +499,25 @@ static LIST_HEAD(device_domain_list);
440 499
441static const struct iommu_ops intel_iommu_ops; 500static const struct iommu_ops intel_iommu_ops;
442 501
502static bool translation_pre_enabled(struct intel_iommu *iommu)
503{
504 return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED);
505}
506
507static void clear_translation_pre_enabled(struct intel_iommu *iommu)
508{
509 iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED;
510}
511
512static void init_translation_status(struct intel_iommu *iommu)
513{
514 u32 gsts;
515
516 gsts = readl(iommu->reg + DMAR_GSTS_REG);
517 if (gsts & DMA_GSTS_TES)
518 iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;
519}
520
443/* Convert generic 'struct iommu_domain to private struct dmar_domain */ 521/* Convert generic 'struct iommu_domain to private struct dmar_domain */
444static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom) 522static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)
445{ 523{
@@ -453,25 +531,21 @@ static int __init intel_iommu_setup(char *str)
453 while (*str) { 531 while (*str) {
454 if (!strncmp(str, "on", 2)) { 532 if (!strncmp(str, "on", 2)) {
455 dmar_disabled = 0; 533 dmar_disabled = 0;
456 printk(KERN_INFO "Intel-IOMMU: enabled\n"); 534 pr_info("IOMMU enabled\n");
457 } else if (!strncmp(str, "off", 3)) { 535 } else if (!strncmp(str, "off", 3)) {
458 dmar_disabled = 1; 536 dmar_disabled = 1;
459 printk(KERN_INFO "Intel-IOMMU: disabled\n"); 537 pr_info("IOMMU disabled\n");
460 } else if (!strncmp(str, "igfx_off", 8)) { 538 } else if (!strncmp(str, "igfx_off", 8)) {
461 dmar_map_gfx = 0; 539 dmar_map_gfx = 0;
462 printk(KERN_INFO 540 pr_info("Disable GFX device mapping\n");
463 "Intel-IOMMU: disable GFX device mapping\n");
464 } else if (!strncmp(str, "forcedac", 8)) { 541 } else if (!strncmp(str, "forcedac", 8)) {
465 printk(KERN_INFO 542 pr_info("Forcing DAC for PCI devices\n");
466 "Intel-IOMMU: Forcing DAC for PCI devices\n");
467 dmar_forcedac = 1; 543 dmar_forcedac = 1;
468 } else if (!strncmp(str, "strict", 6)) { 544 } else if (!strncmp(str, "strict", 6)) {
469 printk(KERN_INFO 545 pr_info("Disable batched IOTLB flush\n");
470 "Intel-IOMMU: disable batched IOTLB flush\n");
471 intel_iommu_strict = 1; 546 intel_iommu_strict = 1;
472 } else if (!strncmp(str, "sp_off", 6)) { 547 } else if (!strncmp(str, "sp_off", 6)) {
473 printk(KERN_INFO 548 pr_info("Disable supported super page\n");
474 "Intel-IOMMU: disable supported super page\n");
475 intel_iommu_superpage = 0; 549 intel_iommu_superpage = 0;
476 } else if (!strncmp(str, "ecs_off", 7)) { 550 } else if (!strncmp(str, "ecs_off", 7)) {
477 printk(KERN_INFO 551 printk(KERN_INFO
@@ -1132,7 +1206,7 @@ static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1132 1206
1133 root = (struct root_entry *)alloc_pgtable_page(iommu->node); 1207 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
1134 if (!root) { 1208 if (!root) {
1135 pr_err("IOMMU: allocating root entry for %s failed\n", 1209 pr_err("Allocating root entry for %s failed\n",
1136 iommu->name); 1210 iommu->name);
1137 return -ENOMEM; 1211 return -ENOMEM;
1138 } 1212 }
@@ -1270,9 +1344,9 @@ static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1270 1344
1271 /* check IOTLB invalidation granularity */ 1345 /* check IOTLB invalidation granularity */
1272 if (DMA_TLB_IAIG(val) == 0) 1346 if (DMA_TLB_IAIG(val) == 0)
1273 printk(KERN_ERR"IOMMU: flush IOTLB failed\n"); 1347 pr_err("Flush IOTLB failed\n");
1274 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type)) 1348 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1275 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n", 1349 pr_debug("TLB flush request %Lx, actual %Lx\n",
1276 (unsigned long long)DMA_TLB_IIRG(type), 1350 (unsigned long long)DMA_TLB_IIRG(type),
1277 (unsigned long long)DMA_TLB_IAIG(val)); 1351 (unsigned long long)DMA_TLB_IAIG(val));
1278} 1352}
@@ -1443,8 +1517,8 @@ static int iommu_init_domains(struct intel_iommu *iommu)
1443 unsigned long nlongs; 1517 unsigned long nlongs;
1444 1518
1445 ndomains = cap_ndoms(iommu->cap); 1519 ndomains = cap_ndoms(iommu->cap);
1446 pr_debug("IOMMU%d: Number of Domains supported <%ld>\n", 1520 pr_debug("%s: Number of Domains supported <%ld>\n",
1447 iommu->seq_id, ndomains); 1521 iommu->name, ndomains);
1448 nlongs = BITS_TO_LONGS(ndomains); 1522 nlongs = BITS_TO_LONGS(ndomains);
1449 1523
1450 spin_lock_init(&iommu->lock); 1524 spin_lock_init(&iommu->lock);
@@ -1454,15 +1528,15 @@ static int iommu_init_domains(struct intel_iommu *iommu)
1454 */ 1528 */
1455 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL); 1529 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1456 if (!iommu->domain_ids) { 1530 if (!iommu->domain_ids) {
1457 pr_err("IOMMU%d: allocating domain id array failed\n", 1531 pr_err("%s: Allocating domain id array failed\n",
1458 iommu->seq_id); 1532 iommu->name);
1459 return -ENOMEM; 1533 return -ENOMEM;
1460 } 1534 }
1461 iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *), 1535 iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
1462 GFP_KERNEL); 1536 GFP_KERNEL);
1463 if (!iommu->domains) { 1537 if (!iommu->domains) {
1464 pr_err("IOMMU%d: allocating domain array failed\n", 1538 pr_err("%s: Allocating domain array failed\n",
1465 iommu->seq_id); 1539 iommu->name);
1466 kfree(iommu->domain_ids); 1540 kfree(iommu->domain_ids);
1467 iommu->domain_ids = NULL; 1541 iommu->domain_ids = NULL;
1468 return -ENOMEM; 1542 return -ENOMEM;
@@ -1567,7 +1641,7 @@ static int iommu_attach_domain(struct dmar_domain *domain,
1567 num = __iommu_attach_domain(domain, iommu); 1641 num = __iommu_attach_domain(domain, iommu);
1568 spin_unlock_irqrestore(&iommu->lock, flags); 1642 spin_unlock_irqrestore(&iommu->lock, flags);
1569 if (num < 0) 1643 if (num < 0)
1570 pr_err("IOMMU: no free domain ids\n"); 1644 pr_err("%s: No free domain ids\n", iommu->name);
1571 1645
1572 return num; 1646 return num;
1573} 1647}
@@ -1659,7 +1733,7 @@ static int dmar_init_reserved_ranges(void)
1659 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START), 1733 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1660 IOVA_PFN(IOAPIC_RANGE_END)); 1734 IOVA_PFN(IOAPIC_RANGE_END));
1661 if (!iova) { 1735 if (!iova) {
1662 printk(KERN_ERR "Reserve IOAPIC range failed\n"); 1736 pr_err("Reserve IOAPIC range failed\n");
1663 return -ENODEV; 1737 return -ENODEV;
1664 } 1738 }
1665 1739
@@ -1675,7 +1749,7 @@ static int dmar_init_reserved_ranges(void)
1675 IOVA_PFN(r->start), 1749 IOVA_PFN(r->start),
1676 IOVA_PFN(r->end)); 1750 IOVA_PFN(r->end));
1677 if (!iova) { 1751 if (!iova) {
1678 printk(KERN_ERR "Reserve iova failed\n"); 1752 pr_err("Reserve iova failed\n");
1679 return -ENODEV; 1753 return -ENODEV;
1680 } 1754 }
1681 } 1755 }
@@ -1722,7 +1796,7 @@ static int domain_init(struct dmar_domain *domain, int guest_width)
1722 sagaw = cap_sagaw(iommu->cap); 1796 sagaw = cap_sagaw(iommu->cap);
1723 if (!test_bit(agaw, &sagaw)) { 1797 if (!test_bit(agaw, &sagaw)) {
1724 /* hardware doesn't support it, choose a bigger one */ 1798 /* hardware doesn't support it, choose a bigger one */
1725 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw); 1799 pr_debug("Hardware doesn't support agaw %d\n", agaw);
1726 agaw = find_next_bit(&sagaw, 5, agaw); 1800 agaw = find_next_bit(&sagaw, 5, agaw);
1727 if (agaw >= 5) 1801 if (agaw >= 5)
1728 return -ENODEV; 1802 return -ENODEV;
@@ -1756,8 +1830,9 @@ static int domain_init(struct dmar_domain *domain, int guest_width)
1756 1830
1757static void domain_exit(struct dmar_domain *domain) 1831static void domain_exit(struct dmar_domain *domain)
1758{ 1832{
1833 struct dmar_drhd_unit *drhd;
1834 struct intel_iommu *iommu;
1759 struct page *freelist = NULL; 1835 struct page *freelist = NULL;
1760 int i;
1761 1836
1762 /* Domain 0 is reserved, so dont process it */ 1837 /* Domain 0 is reserved, so dont process it */
1763 if (!domain) 1838 if (!domain)
@@ -1777,8 +1852,10 @@ static void domain_exit(struct dmar_domain *domain)
1777 1852
1778 /* clear attached or cached domains */ 1853 /* clear attached or cached domains */
1779 rcu_read_lock(); 1854 rcu_read_lock();
1780 for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus) 1855 for_each_active_iommu(iommu, drhd)
1781 iommu_detach_domain(domain, g_iommus[i]); 1856 if (domain_type_is_vm(domain) ||
1857 test_bit(iommu->seq_id, domain->iommu_bmp))
1858 iommu_detach_domain(domain, iommu);
1782 rcu_read_unlock(); 1859 rcu_read_unlock();
1783 1860
1784 dma_free_pagelist(freelist); 1861 dma_free_pagelist(freelist);
@@ -1815,6 +1892,8 @@ static int domain_context_mapping_one(struct dmar_domain *domain,
1815 return 0; 1892 return 0;
1816 } 1893 }
1817 1894
1895 context_clear_entry(context);
1896
1818 id = domain->id; 1897 id = domain->id;
1819 pgd = domain->pgd; 1898 pgd = domain->pgd;
1820 1899
@@ -1823,7 +1902,7 @@ static int domain_context_mapping_one(struct dmar_domain *domain,
1823 id = iommu_attach_vm_domain(domain, iommu); 1902 id = iommu_attach_vm_domain(domain, iommu);
1824 if (id < 0) { 1903 if (id < 0) {
1825 spin_unlock_irqrestore(&iommu->lock, flags); 1904 spin_unlock_irqrestore(&iommu->lock, flags);
1826 pr_err("IOMMU: no free domain ids\n"); 1905 pr_err("%s: No free domain ids\n", iommu->name);
1827 return -EFAULT; 1906 return -EFAULT;
1828 } 1907 }
1829 } 1908 }
@@ -2050,8 +2129,8 @@ static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2050 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval); 2129 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
2051 if (tmp) { 2130 if (tmp) {
2052 static int dumps = 5; 2131 static int dumps = 5;
2053 printk(KERN_CRIT "ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n", 2132 pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2054 iov_pfn, tmp, (unsigned long long)pteval); 2133 iov_pfn, tmp, (unsigned long long)pteval);
2055 if (dumps) { 2134 if (dumps) {
2056 dumps--; 2135 dumps--;
2057 debug_dma_dump_mappings(NULL); 2136 debug_dma_dump_mappings(NULL);
@@ -2323,7 +2402,7 @@ static int iommu_domain_identity_map(struct dmar_domain *domain,
2323 2402
2324 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn), 2403 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2325 dma_to_mm_pfn(last_vpfn))) { 2404 dma_to_mm_pfn(last_vpfn))) {
2326 printk(KERN_ERR "IOMMU: reserve iova failed\n"); 2405 pr_err("Reserving iova failed\n");
2327 return -ENOMEM; 2406 return -ENOMEM;
2328 } 2407 }
2329 2408
@@ -2356,15 +2435,14 @@ static int iommu_prepare_identity_map(struct device *dev,
2356 range which is reserved in E820, so which didn't get set 2435 range which is reserved in E820, so which didn't get set
2357 up to start with in si_domain */ 2436 up to start with in si_domain */
2358 if (domain == si_domain && hw_pass_through) { 2437 if (domain == si_domain && hw_pass_through) {
2359 printk("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n", 2438 pr_warn("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2360 dev_name(dev), start, end); 2439 dev_name(dev), start, end);
2361 return 0; 2440 return 0;
2362 } 2441 }
2363 2442
2364 printk(KERN_INFO 2443 pr_info("Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2365 "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n", 2444 dev_name(dev), start, end);
2366 dev_name(dev), start, end); 2445
2367
2368 if (end < start) { 2446 if (end < start) {
2369 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n" 2447 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2370 "BIOS vendor: %s; Ver: %s; Product Version: %s\n", 2448 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
@@ -2421,12 +2499,11 @@ static inline void iommu_prepare_isa(void)
2421 if (!pdev) 2499 if (!pdev)
2422 return; 2500 return;
2423 2501
2424 printk(KERN_INFO "IOMMU: Prepare 0-16MiB unity mapping for LPC\n"); 2502 pr_info("Prepare 0-16MiB unity mapping for LPC\n");
2425 ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1); 2503 ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
2426 2504
2427 if (ret) 2505 if (ret)
2428 printk(KERN_ERR "IOMMU: Failed to create 0-16MiB identity map; " 2506 pr_err("Failed to create 0-16MiB identity map - floppy might not work\n");
2429 "floppy might not work\n");
2430 2507
2431 pci_dev_put(pdev); 2508 pci_dev_put(pdev);
2432} 2509}
@@ -2470,7 +2547,7 @@ static int __init si_domain_init(int hw)
2470 return -EFAULT; 2547 return -EFAULT;
2471 } 2548 }
2472 2549
2473 pr_debug("IOMMU: identity mapping domain is domain %d\n", 2550 pr_debug("Identity mapping domain is domain %d\n",
2474 si_domain->id); 2551 si_domain->id);
2475 2552
2476 if (hw) 2553 if (hw)
@@ -2670,8 +2747,8 @@ static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw
2670 hw ? CONTEXT_TT_PASS_THROUGH : 2747 hw ? CONTEXT_TT_PASS_THROUGH :
2671 CONTEXT_TT_MULTI_LEVEL); 2748 CONTEXT_TT_MULTI_LEVEL);
2672 if (!ret) 2749 if (!ret)
2673 pr_info("IOMMU: %s identity mapping for device %s\n", 2750 pr_info("%s identity mapping for device %s\n",
2674 hw ? "hardware" : "software", dev_name(dev)); 2751 hw ? "Hardware" : "Software", dev_name(dev));
2675 else if (ret == -ENODEV) 2752 else if (ret == -ENODEV)
2676 /* device not associated with an iommu */ 2753 /* device not associated with an iommu */
2677 ret = 0; 2754 ret = 0;
@@ -2689,10 +2766,6 @@ static int __init iommu_prepare_static_identity_mapping(int hw)
2689 int i; 2766 int i;
2690 int ret = 0; 2767 int ret = 0;
2691 2768
2692 ret = si_domain_init(hw);
2693 if (ret)
2694 return -EFAULT;
2695
2696 for_each_pci_dev(pdev) { 2769 for_each_pci_dev(pdev) {
2697 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw); 2770 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
2698 if (ret) 2771 if (ret)
@@ -2706,7 +2779,7 @@ static int __init iommu_prepare_static_identity_mapping(int hw)
2706 2779
2707 if (dev->bus != &acpi_bus_type) 2780 if (dev->bus != &acpi_bus_type)
2708 continue; 2781 continue;
2709 2782
2710 adev= to_acpi_device(dev); 2783 adev= to_acpi_device(dev);
2711 mutex_lock(&adev->physical_node_lock); 2784 mutex_lock(&adev->physical_node_lock);
2712 list_for_each_entry(pn, &adev->physical_node_list, node) { 2785 list_for_each_entry(pn, &adev->physical_node_list, node) {
@@ -2748,19 +2821,200 @@ static void intel_iommu_init_qi(struct intel_iommu *iommu)
2748 */ 2821 */
2749 iommu->flush.flush_context = __iommu_flush_context; 2822 iommu->flush.flush_context = __iommu_flush_context;
2750 iommu->flush.flush_iotlb = __iommu_flush_iotlb; 2823 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
2751 pr_info("IOMMU: %s using Register based invalidation\n", 2824 pr_info("%s: Using Register based invalidation\n",
2752 iommu->name); 2825 iommu->name);
2753 } else { 2826 } else {
2754 iommu->flush.flush_context = qi_flush_context; 2827 iommu->flush.flush_context = qi_flush_context;
2755 iommu->flush.flush_iotlb = qi_flush_iotlb; 2828 iommu->flush.flush_iotlb = qi_flush_iotlb;
2756 pr_info("IOMMU: %s using Queued invalidation\n", iommu->name); 2829 pr_info("%s: Using Queued invalidation\n", iommu->name);
2757 } 2830 }
2758} 2831}
2759 2832
2833static int copy_context_table(struct intel_iommu *iommu,
2834 struct root_entry *old_re,
2835 struct context_entry **tbl,
2836 int bus, bool ext)
2837{
2838 struct context_entry *old_ce = NULL, *new_ce = NULL, ce;
2839 int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
2840 phys_addr_t old_ce_phys;
2841
2842 tbl_idx = ext ? bus * 2 : bus;
2843
2844 for (devfn = 0; devfn < 256; devfn++) {
2845 /* First calculate the correct index */
2846 idx = (ext ? devfn * 2 : devfn) % 256;
2847
2848 if (idx == 0) {
2849 /* First save what we may have and clean up */
2850 if (new_ce) {
2851 tbl[tbl_idx] = new_ce;
2852 __iommu_flush_cache(iommu, new_ce,
2853 VTD_PAGE_SIZE);
2854 pos = 1;
2855 }
2856
2857 if (old_ce)
2858 iounmap(old_ce);
2859
2860 ret = 0;
2861 if (devfn < 0x80)
2862 old_ce_phys = root_entry_lctp(old_re);
2863 else
2864 old_ce_phys = root_entry_uctp(old_re);
2865
2866 if (!old_ce_phys) {
2867 if (ext && devfn == 0) {
2868 /* No LCTP, try UCTP */
2869 devfn = 0x7f;
2870 continue;
2871 } else {
2872 goto out;
2873 }
2874 }
2875
2876 ret = -ENOMEM;
2877 old_ce = ioremap_cache(old_ce_phys, PAGE_SIZE);
2878 if (!old_ce)
2879 goto out;
2880
2881 new_ce = alloc_pgtable_page(iommu->node);
2882 if (!new_ce)
2883 goto out_unmap;
2884
2885 ret = 0;
2886 }
2887
2888 /* Now copy the context entry */
2889 ce = old_ce[idx];
2890
2891 if (!__context_present(&ce))
2892 continue;
2893
2894 did = context_domain_id(&ce);
2895 if (did >= 0 && did < cap_ndoms(iommu->cap))
2896 set_bit(did, iommu->domain_ids);
2897
2898 /*
2899 * We need a marker for copied context entries. This
2900 * marker needs to work for the old format as well as
2901 * for extended context entries.
2902 *
2903 * Bit 67 of the context entry is used. In the old
2904 * format this bit is available to software, in the
2905 * extended format it is the PGE bit, but PGE is ignored
2906 * by HW if PASIDs are disabled (and thus still
2907 * available).
2908 *
2909 * So disable PASIDs first and then mark the entry
2910 * copied. This means that we don't copy PASID
2911 * translations from the old kernel, but this is fine as
2912 * faults there are not fatal.
2913 */
2914 context_clear_pasid_enable(&ce);
2915 context_set_copied(&ce);
2916
2917 new_ce[idx] = ce;
2918 }
2919
2920 tbl[tbl_idx + pos] = new_ce;
2921
2922 __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);
2923
2924out_unmap:
2925 iounmap(old_ce);
2926
2927out:
2928 return ret;
2929}
2930
2931static int copy_translation_tables(struct intel_iommu *iommu)
2932{
2933 struct context_entry **ctxt_tbls;
2934 struct root_entry *old_rt;
2935 phys_addr_t old_rt_phys;
2936 int ctxt_table_entries;
2937 unsigned long flags;
2938 u64 rtaddr_reg;
2939 int bus, ret;
2940 bool new_ext, ext;
2941
2942 rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG);
2943 ext = !!(rtaddr_reg & DMA_RTADDR_RTT);
2944 new_ext = !!ecap_ecs(iommu->ecap);
2945
2946 /*
2947 * The RTT bit can only be changed when translation is disabled,
2948 * but disabling translation means to open a window for data
2949 * corruption. So bail out and don't copy anything if we would
2950 * have to change the bit.
2951 */
2952 if (new_ext != ext)
2953 return -EINVAL;
2954
2955 old_rt_phys = rtaddr_reg & VTD_PAGE_MASK;
2956 if (!old_rt_phys)
2957 return -EINVAL;
2958
2959 old_rt = ioremap_cache(old_rt_phys, PAGE_SIZE);
2960 if (!old_rt)
2961 return -ENOMEM;
2962
2963 /* This is too big for the stack - allocate it from slab */
2964 ctxt_table_entries = ext ? 512 : 256;
2965 ret = -ENOMEM;
2966 ctxt_tbls = kzalloc(ctxt_table_entries * sizeof(void *), GFP_KERNEL);
2967 if (!ctxt_tbls)
2968 goto out_unmap;
2969
2970 for (bus = 0; bus < 256; bus++) {
2971 ret = copy_context_table(iommu, &old_rt[bus],
2972 ctxt_tbls, bus, ext);
2973 if (ret) {
2974 pr_err("%s: Failed to copy context table for bus %d\n",
2975 iommu->name, bus);
2976 continue;
2977 }
2978 }
2979
2980 spin_lock_irqsave(&iommu->lock, flags);
2981
2982 /* Context tables are copied, now write them to the root_entry table */
2983 for (bus = 0; bus < 256; bus++) {
2984 int idx = ext ? bus * 2 : bus;
2985 u64 val;
2986
2987 if (ctxt_tbls[idx]) {
2988 val = virt_to_phys(ctxt_tbls[idx]) | 1;
2989 iommu->root_entry[bus].lo = val;
2990 }
2991
2992 if (!ext || !ctxt_tbls[idx + 1])
2993 continue;
2994
2995 val = virt_to_phys(ctxt_tbls[idx + 1]) | 1;
2996 iommu->root_entry[bus].hi = val;
2997 }
2998
2999 spin_unlock_irqrestore(&iommu->lock, flags);
3000
3001 kfree(ctxt_tbls);
3002
3003 __iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE);
3004
3005 ret = 0;
3006
3007out_unmap:
3008 iounmap(old_rt);
3009
3010 return ret;
3011}
3012
2760static int __init init_dmars(void) 3013static int __init init_dmars(void)
2761{ 3014{
2762 struct dmar_drhd_unit *drhd; 3015 struct dmar_drhd_unit *drhd;
2763 struct dmar_rmrr_unit *rmrr; 3016 struct dmar_rmrr_unit *rmrr;
3017 bool copied_tables = false;
2764 struct device *dev; 3018 struct device *dev;
2765 struct intel_iommu *iommu; 3019 struct intel_iommu *iommu;
2766 int i, ret; 3020 int i, ret;
@@ -2781,8 +3035,7 @@ static int __init init_dmars(void)
2781 g_num_of_iommus++; 3035 g_num_of_iommus++;
2782 continue; 3036 continue;
2783 } 3037 }
2784 printk_once(KERN_ERR "intel-iommu: exceeded %d IOMMUs\n", 3038 pr_err_once("Exceeded %d IOMMUs\n", DMAR_UNITS_SUPPORTED);
2785 DMAR_UNITS_SUPPORTED);
2786 } 3039 }
2787 3040
2788 /* Preallocate enough resources for IOMMU hot-addition */ 3041 /* Preallocate enough resources for IOMMU hot-addition */
@@ -2792,7 +3045,7 @@ static int __init init_dmars(void)
2792 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *), 3045 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
2793 GFP_KERNEL); 3046 GFP_KERNEL);
2794 if (!g_iommus) { 3047 if (!g_iommus) {
2795 printk(KERN_ERR "Allocating global iommu array failed\n"); 3048 pr_err("Allocating global iommu array failed\n");
2796 ret = -ENOMEM; 3049 ret = -ENOMEM;
2797 goto error; 3050 goto error;
2798 } 3051 }
@@ -2807,10 +3060,21 @@ static int __init init_dmars(void)
2807 for_each_active_iommu(iommu, drhd) { 3060 for_each_active_iommu(iommu, drhd) {
2808 g_iommus[iommu->seq_id] = iommu; 3061 g_iommus[iommu->seq_id] = iommu;
2809 3062
3063 intel_iommu_init_qi(iommu);
3064
2810 ret = iommu_init_domains(iommu); 3065 ret = iommu_init_domains(iommu);
2811 if (ret) 3066 if (ret)
2812 goto free_iommu; 3067 goto free_iommu;
2813 3068
3069 init_translation_status(iommu);
3070
3071 if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
3072 iommu_disable_translation(iommu);
3073 clear_translation_pre_enabled(iommu);
3074 pr_warn("Translation was enabled for %s but we are not in kdump mode\n",
3075 iommu->name);
3076 }
3077
2814 /* 3078 /*
2815 * TBD: 3079 * TBD:
2816 * we could share the same root & context tables 3080 * we could share the same root & context tables
@@ -2819,13 +3083,41 @@ static int __init init_dmars(void)
2819 ret = iommu_alloc_root_entry(iommu); 3083 ret = iommu_alloc_root_entry(iommu);
2820 if (ret) 3084 if (ret)
2821 goto free_iommu; 3085 goto free_iommu;
3086
3087 if (translation_pre_enabled(iommu)) {
3088 pr_info("Translation already enabled - trying to copy translation structures\n");
3089
3090 ret = copy_translation_tables(iommu);
3091 if (ret) {
3092 /*
3093 * We found the IOMMU with translation
3094 * enabled - but failed to copy over the
3095 * old root-entry table. Try to proceed
3096 * by disabling translation now and
3097 * allocating a clean root-entry table.
3098 * This might cause DMAR faults, but
3099 * probably the dump will still succeed.
3100 */
3101 pr_err("Failed to copy translation tables from previous kernel for %s\n",
3102 iommu->name);
3103 iommu_disable_translation(iommu);
3104 clear_translation_pre_enabled(iommu);
3105 } else {
3106 pr_info("Copied translation tables from previous kernel for %s\n",
3107 iommu->name);
3108 copied_tables = true;
3109 }
3110 }
3111
3112 iommu_flush_write_buffer(iommu);
3113 iommu_set_root_entry(iommu);
3114 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
3115 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3116
2822 if (!ecap_pass_through(iommu->ecap)) 3117 if (!ecap_pass_through(iommu->ecap))
2823 hw_pass_through = 0; 3118 hw_pass_through = 0;
2824 } 3119 }
2825 3120
2826 for_each_active_iommu(iommu, drhd)
2827 intel_iommu_init_qi(iommu);
2828
2829 if (iommu_pass_through) 3121 if (iommu_pass_through)
2830 iommu_identity_mapping |= IDENTMAP_ALL; 3122 iommu_identity_mapping |= IDENTMAP_ALL;
2831 3123
@@ -2833,9 +3125,24 @@ static int __init init_dmars(void)
2833 iommu_identity_mapping |= IDENTMAP_GFX; 3125 iommu_identity_mapping |= IDENTMAP_GFX;
2834#endif 3126#endif
2835 3127
3128 if (iommu_identity_mapping) {
3129 ret = si_domain_init(hw_pass_through);
3130 if (ret)
3131 goto free_iommu;
3132 }
3133
2836 check_tylersburg_isoch(); 3134 check_tylersburg_isoch();
2837 3135
2838 /* 3136 /*
3137 * If we copied translations from a previous kernel in the kdump
3138 * case, we can not assign the devices to domains now, as that
3139 * would eliminate the old mappings. So skip this part and defer
3140 * the assignment to device driver initialization time.
3141 */
3142 if (copied_tables)
3143 goto domains_done;
3144
3145 /*
2839 * If pass through is not set or not enabled, setup context entries for 3146 * If pass through is not set or not enabled, setup context entries for
2840 * identity mappings for rmrr, gfx, and isa and may fall back to static 3147 * identity mappings for rmrr, gfx, and isa and may fall back to static
2841 * identity mapping if iommu_identity_mapping is set. 3148 * identity mapping if iommu_identity_mapping is set.
@@ -2843,7 +3150,7 @@ static int __init init_dmars(void)
2843 if (iommu_identity_mapping) { 3150 if (iommu_identity_mapping) {
2844 ret = iommu_prepare_static_identity_mapping(hw_pass_through); 3151 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
2845 if (ret) { 3152 if (ret) {
2846 printk(KERN_CRIT "Failed to setup IOMMU pass-through\n"); 3153 pr_crit("Failed to setup IOMMU pass-through\n");
2847 goto free_iommu; 3154 goto free_iommu;
2848 } 3155 }
2849 } 3156 }
@@ -2861,20 +3168,21 @@ static int __init init_dmars(void)
2861 * endfor 3168 * endfor
2862 * endfor 3169 * endfor
2863 */ 3170 */
2864 printk(KERN_INFO "IOMMU: Setting RMRR:\n"); 3171 pr_info("Setting RMRR:\n");
2865 for_each_rmrr_units(rmrr) { 3172 for_each_rmrr_units(rmrr) {
2866 /* some BIOS lists non-exist devices in DMAR table. */ 3173 /* some BIOS lists non-exist devices in DMAR table. */
2867 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt, 3174 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
2868 i, dev) { 3175 i, dev) {
2869 ret = iommu_prepare_rmrr_dev(rmrr, dev); 3176 ret = iommu_prepare_rmrr_dev(rmrr, dev);
2870 if (ret) 3177 if (ret)
2871 printk(KERN_ERR 3178 pr_err("Mapping reserved region failed\n");
2872 "IOMMU: mapping reserved region failed\n");
2873 } 3179 }
2874 } 3180 }
2875 3181
2876 iommu_prepare_isa(); 3182 iommu_prepare_isa();
2877 3183
3184domains_done:
3185
2878 /* 3186 /*
2879 * for each drhd 3187 * for each drhd
2880 * enable fault log 3188 * enable fault log
@@ -2899,11 +3207,9 @@ static int __init init_dmars(void)
2899 if (ret) 3207 if (ret)
2900 goto free_iommu; 3208 goto free_iommu;
2901 3209
2902 iommu_set_root_entry(iommu); 3210 if (!translation_pre_enabled(iommu))
3211 iommu_enable_translation(iommu);
2903 3212
2904 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
2905 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
2906 iommu_enable_translation(iommu);
2907 iommu_disable_protect_mem_regions(iommu); 3213 iommu_disable_protect_mem_regions(iommu);
2908 } 3214 }
2909 3215
@@ -2944,7 +3250,7 @@ static struct iova *intel_alloc_iova(struct device *dev,
2944 } 3250 }
2945 iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1); 3251 iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
2946 if (unlikely(!iova)) { 3252 if (unlikely(!iova)) {
2947 printk(KERN_ERR "Allocating %ld-page iova for %s failed", 3253 pr_err("Allocating %ld-page iova for %s failed",
2948 nrpages, dev_name(dev)); 3254 nrpages, dev_name(dev));
2949 return NULL; 3255 return NULL;
2950 } 3256 }
@@ -2959,7 +3265,7 @@ static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev)
2959 3265
2960 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH); 3266 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
2961 if (!domain) { 3267 if (!domain) {
2962 printk(KERN_ERR "Allocating domain for %s failed", 3268 pr_err("Allocating domain for %s failed\n",
2963 dev_name(dev)); 3269 dev_name(dev));
2964 return NULL; 3270 return NULL;
2965 } 3271 }
@@ -2968,7 +3274,7 @@ static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev)
2968 if (unlikely(!domain_context_mapped(dev))) { 3274 if (unlikely(!domain_context_mapped(dev))) {
2969 ret = domain_context_mapping(domain, dev, CONTEXT_TT_MULTI_LEVEL); 3275 ret = domain_context_mapping(domain, dev, CONTEXT_TT_MULTI_LEVEL);
2970 if (ret) { 3276 if (ret) {
2971 printk(KERN_ERR "Domain context map for %s failed", 3277 pr_err("Domain context map for %s failed\n",
2972 dev_name(dev)); 3278 dev_name(dev));
2973 return NULL; 3279 return NULL;
2974 } 3280 }
@@ -3010,8 +3316,8 @@ static int iommu_no_mapping(struct device *dev)
3010 * to non-identity mapping. 3316 * to non-identity mapping.
3011 */ 3317 */
3012 domain_remove_one_dev_info(si_domain, dev); 3318 domain_remove_one_dev_info(si_domain, dev);
3013 printk(KERN_INFO "32bit %s uses non-identity mapping\n", 3319 pr_info("32bit %s uses non-identity mapping\n",
3014 dev_name(dev)); 3320 dev_name(dev));
3015 return 0; 3321 return 0;
3016 } 3322 }
3017 } else { 3323 } else {
@@ -3026,8 +3332,8 @@ static int iommu_no_mapping(struct device *dev)
3026 CONTEXT_TT_PASS_THROUGH : 3332 CONTEXT_TT_PASS_THROUGH :
3027 CONTEXT_TT_MULTI_LEVEL); 3333 CONTEXT_TT_MULTI_LEVEL);
3028 if (!ret) { 3334 if (!ret) {
3029 printk(KERN_INFO "64bit %s uses identity mapping\n", 3335 pr_info("64bit %s uses identity mapping\n",
3030 dev_name(dev)); 3336 dev_name(dev));
3031 return 1; 3337 return 1;
3032 } 3338 }
3033 } 3339 }
@@ -3096,7 +3402,7 @@ static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
3096error: 3402error:
3097 if (iova) 3403 if (iova)
3098 __free_iova(&domain->iovad, iova); 3404 __free_iova(&domain->iovad, iova);
3099 printk(KERN_ERR"Device %s request: %zx@%llx dir %d --- failed\n", 3405 pr_err("Device %s request: %zx@%llx dir %d --- failed\n",
3100 dev_name(dev), size, (unsigned long long)paddr, dir); 3406 dev_name(dev), size, (unsigned long long)paddr, dir);
3101 return 0; 3407 return 0;
3102} 3408}
@@ -3411,7 +3717,7 @@ static inline int iommu_domain_cache_init(void)
3411 3717
3412 NULL); 3718 NULL);
3413 if (!iommu_domain_cache) { 3719 if (!iommu_domain_cache) {
3414 printk(KERN_ERR "Couldn't create iommu_domain cache\n"); 3720 pr_err("Couldn't create iommu_domain cache\n");
3415 ret = -ENOMEM; 3721 ret = -ENOMEM;
3416 } 3722 }
3417 3723
@@ -3428,7 +3734,7 @@ static inline int iommu_devinfo_cache_init(void)
3428 SLAB_HWCACHE_ALIGN, 3734 SLAB_HWCACHE_ALIGN,
3429 NULL); 3735 NULL);
3430 if (!iommu_devinfo_cache) { 3736 if (!iommu_devinfo_cache) {
3431 printk(KERN_ERR "Couldn't create devinfo cache\n"); 3737 pr_err("Couldn't create devinfo cache\n");
3432 ret = -ENOMEM; 3738 ret = -ENOMEM;
3433 } 3739 }
3434 3740
@@ -3805,19 +4111,19 @@ static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
3805 return 0; 4111 return 0;
3806 4112
3807 if (hw_pass_through && !ecap_pass_through(iommu->ecap)) { 4113 if (hw_pass_through && !ecap_pass_through(iommu->ecap)) {
3808 pr_warn("IOMMU: %s doesn't support hardware pass through.\n", 4114 pr_warn("%s: Doesn't support hardware pass through.\n",
3809 iommu->name); 4115 iommu->name);
3810 return -ENXIO; 4116 return -ENXIO;
3811 } 4117 }
3812 if (!ecap_sc_support(iommu->ecap) && 4118 if (!ecap_sc_support(iommu->ecap) &&
3813 domain_update_iommu_snooping(iommu)) { 4119 domain_update_iommu_snooping(iommu)) {
3814 pr_warn("IOMMU: %s doesn't support snooping.\n", 4120 pr_warn("%s: Doesn't support snooping.\n",
3815 iommu->name); 4121 iommu->name);
3816 return -ENXIO; 4122 return -ENXIO;
3817 } 4123 }
3818 sp = domain_update_iommu_superpage(iommu) - 1; 4124 sp = domain_update_iommu_superpage(iommu) - 1;
3819 if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) { 4125 if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
3820 pr_warn("IOMMU: %s doesn't support large page.\n", 4126 pr_warn("%s: Doesn't support large page.\n",
3821 iommu->name); 4127 iommu->name);
3822 return -ENXIO; 4128 return -ENXIO;
3823 } 4129 }
@@ -4048,7 +4354,7 @@ static int intel_iommu_memory_notifier(struct notifier_block *nb,
4048 start = mhp->start_pfn << PAGE_SHIFT; 4354 start = mhp->start_pfn << PAGE_SHIFT;
4049 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1; 4355 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
4050 if (iommu_domain_identity_map(si_domain, start, end)) { 4356 if (iommu_domain_identity_map(si_domain, start, end)) {
4051 pr_warn("dmar: failed to build identity map for [%llx-%llx]\n", 4357 pr_warn("Failed to build identity map for [%llx-%llx]\n",
4052 start, end); 4358 start, end);
4053 return NOTIFY_BAD; 4359 return NOTIFY_BAD;
4054 } 4360 }
@@ -4066,7 +4372,7 @@ static int intel_iommu_memory_notifier(struct notifier_block *nb,
4066 4372
4067 iova = find_iova(&si_domain->iovad, start_vpfn); 4373 iova = find_iova(&si_domain->iovad, start_vpfn);
4068 if (iova == NULL) { 4374 if (iova == NULL) {
4069 pr_debug("dmar: failed get IOVA for PFN %lx\n", 4375 pr_debug("Failed get IOVA for PFN %lx\n",
4070 start_vpfn); 4376 start_vpfn);
4071 break; 4377 break;
4072 } 4378 }
@@ -4074,7 +4380,7 @@ static int intel_iommu_memory_notifier(struct notifier_block *nb,
4074 iova = split_and_remove_iova(&si_domain->iovad, iova, 4380 iova = split_and_remove_iova(&si_domain->iovad, iova,
4075 start_vpfn, last_vpfn); 4381 start_vpfn, last_vpfn);
4076 if (iova == NULL) { 4382 if (iova == NULL) {
4077 pr_warn("dmar: failed to split IOVA PFN [%lx-%lx]\n", 4383 pr_warn("Failed to split IOVA PFN [%lx-%lx]\n",
4078 start_vpfn, last_vpfn); 4384 start_vpfn, last_vpfn);
4079 return NOTIFY_BAD; 4385 return NOTIFY_BAD;
4080 } 4386 }
@@ -4183,13 +4489,6 @@ int __init intel_iommu_init(void)
4183 goto out_free_dmar; 4489 goto out_free_dmar;
4184 } 4490 }
4185 4491
4186 /*
4187 * Disable translation if already enabled prior to OS handover.
4188 */
4189 for_each_active_iommu(iommu, drhd)
4190 if (iommu->gcmd & DMA_GCMD_TE)
4191 iommu_disable_translation(iommu);
4192
4193 if (dmar_dev_scope_init() < 0) { 4492 if (dmar_dev_scope_init() < 0) {
4194 if (force_on) 4493 if (force_on)
4195 panic("tboot: Failed to initialize DMAR device scope\n"); 4494 panic("tboot: Failed to initialize DMAR device scope\n");
@@ -4200,10 +4499,10 @@ int __init intel_iommu_init(void)
4200 goto out_free_dmar; 4499 goto out_free_dmar;
4201 4500
4202 if (list_empty(&dmar_rmrr_units)) 4501 if (list_empty(&dmar_rmrr_units))
4203 printk(KERN_INFO "DMAR: No RMRR found\n"); 4502 pr_info("No RMRR found\n");
4204 4503
4205 if (list_empty(&dmar_atsr_units)) 4504 if (list_empty(&dmar_atsr_units))
4206 printk(KERN_INFO "DMAR: No ATSR found\n"); 4505 pr_info("No ATSR found\n");
4207 4506
4208 if (dmar_init_reserved_ranges()) { 4507 if (dmar_init_reserved_ranges()) {
4209 if (force_on) 4508 if (force_on)
@@ -4217,12 +4516,11 @@ int __init intel_iommu_init(void)
4217 if (ret) { 4516 if (ret) {
4218 if (force_on) 4517 if (force_on)
4219 panic("tboot: Failed to initialize DMARs\n"); 4518 panic("tboot: Failed to initialize DMARs\n");
4220 printk(KERN_ERR "IOMMU: dmar init failed\n"); 4519 pr_err("Initialization failed\n");
4221 goto out_free_reserved_range; 4520 goto out_free_reserved_range;
4222 } 4521 }
4223 up_write(&dmar_global_lock); 4522 up_write(&dmar_global_lock);
4224 printk(KERN_INFO 4523 pr_info("Intel(R) Virtualization Technology for Directed I/O\n");
4225 "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
4226 4524
4227 init_timer(&unmap_timer); 4525 init_timer(&unmap_timer);
4228#ifdef CONFIG_SWIOTLB 4526#ifdef CONFIG_SWIOTLB
@@ -4364,13 +4662,11 @@ static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
4364 4662
4365 dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE); 4663 dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE);
4366 if (!dmar_domain) { 4664 if (!dmar_domain) {
4367 printk(KERN_ERR 4665 pr_err("Can't allocate dmar_domain\n");
4368 "intel_iommu_domain_init: dmar_domain == NULL\n");
4369 return NULL; 4666 return NULL;
4370 } 4667 }
4371 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) { 4668 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
4372 printk(KERN_ERR 4669 pr_err("Domain initialization failed\n");
4373 "intel_iommu_domain_init() failed\n");
4374 domain_exit(dmar_domain); 4670 domain_exit(dmar_domain);
4375 return NULL; 4671 return NULL;
4376 } 4672 }
@@ -4429,7 +4725,7 @@ static int intel_iommu_attach_device(struct iommu_domain *domain,
4429 addr_width = cap_mgaw(iommu->cap); 4725 addr_width = cap_mgaw(iommu->cap);
4430 4726
4431 if (dmar_domain->max_addr > (1LL << addr_width)) { 4727 if (dmar_domain->max_addr > (1LL << addr_width)) {
4432 printk(KERN_ERR "%s: iommu width (%d) is not " 4728 pr_err("%s: iommu width (%d) is not "
4433 "sufficient for the mapped address (%llx)\n", 4729 "sufficient for the mapped address (%llx)\n",
4434 __func__, addr_width, dmar_domain->max_addr); 4730 __func__, addr_width, dmar_domain->max_addr);
4435 return -EFAULT; 4731 return -EFAULT;
@@ -4483,7 +4779,7 @@ static int intel_iommu_map(struct iommu_domain *domain,
4483 /* check if minimum agaw is sufficient for mapped address */ 4779 /* check if minimum agaw is sufficient for mapped address */
4484 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1; 4780 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
4485 if (end < max_addr) { 4781 if (end < max_addr) {
4486 printk(KERN_ERR "%s: iommu width (%d) is not " 4782 pr_err("%s: iommu width (%d) is not "
4487 "sufficient for the mapped address (%llx)\n", 4783 "sufficient for the mapped address (%llx)\n",
4488 __func__, dmar_domain->gaw, max_addr); 4784 __func__, dmar_domain->gaw, max_addr);
4489 return -EFAULT; 4785 return -EFAULT;
@@ -4624,7 +4920,7 @@ static const struct iommu_ops intel_iommu_ops = {
4624static void quirk_iommu_g4x_gfx(struct pci_dev *dev) 4920static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
4625{ 4921{
4626 /* G4x/GM45 integrated gfx dmar support is totally busted. */ 4922 /* G4x/GM45 integrated gfx dmar support is totally busted. */
4627 printk(KERN_INFO "DMAR: Disabling IOMMU for graphics on this chipset\n"); 4923 pr_info("Disabling IOMMU for graphics on this chipset\n");
4628 dmar_map_gfx = 0; 4924 dmar_map_gfx = 0;
4629} 4925}
4630 4926
@@ -4642,7 +4938,7 @@ static void quirk_iommu_rwbf(struct pci_dev *dev)
4642 * Mobile 4 Series Chipset neglects to set RWBF capability, 4938 * Mobile 4 Series Chipset neglects to set RWBF capability,
4643 * but needs it. Same seems to hold for the desktop versions. 4939 * but needs it. Same seems to hold for the desktop versions.
4644 */ 4940 */
4645 printk(KERN_INFO "DMAR: Forcing write-buffer flush capability\n"); 4941 pr_info("Forcing write-buffer flush capability\n");
4646 rwbf_quirk = 1; 4942 rwbf_quirk = 1;
4647} 4943}
4648 4944
@@ -4672,11 +4968,11 @@ static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
4672 return; 4968 return;
4673 4969
4674 if (!(ggc & GGC_MEMORY_VT_ENABLED)) { 4970 if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
4675 printk(KERN_INFO "DMAR: BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n"); 4971 pr_info("BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
4676 dmar_map_gfx = 0; 4972 dmar_map_gfx = 0;
4677 } else if (dmar_map_gfx) { 4973 } else if (dmar_map_gfx) {
4678 /* we have to ensure the gfx device is idle before we flush */ 4974 /* we have to ensure the gfx device is idle before we flush */
4679 printk(KERN_INFO "DMAR: Disabling batched IOTLB flush on Ironlake\n"); 4975 pr_info("Disabling batched IOTLB flush on Ironlake\n");
4680 intel_iommu_strict = 1; 4976 intel_iommu_strict = 1;
4681 } 4977 }
4682} 4978}
@@ -4738,7 +5034,7 @@ static void __init check_tylersburg_isoch(void)
4738 iommu_identity_mapping |= IDENTMAP_AZALIA; 5034 iommu_identity_mapping |= IDENTMAP_AZALIA;
4739 return; 5035 return;
4740 } 5036 }
4741 5037
4742 printk(KERN_WARNING "DMAR: Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n", 5038 pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
4743 vtisochctrl); 5039 vtisochctrl);
4744} 5040}
diff --git a/drivers/iommu/intel_irq_remapping.c b/drivers/iommu/intel_irq_remapping.c
index 80f1d1486247..f15692a410c7 100644
--- a/drivers/iommu/intel_irq_remapping.c
+++ b/drivers/iommu/intel_irq_remapping.c
@@ -1,3 +1,6 @@
1
2#define pr_fmt(fmt) "DMAR-IR: " fmt
3
1#include <linux/interrupt.h> 4#include <linux/interrupt.h>
2#include <linux/dmar.h> 5#include <linux/dmar.h>
3#include <linux/spinlock.h> 6#include <linux/spinlock.h>
@@ -9,6 +12,7 @@
9#include <linux/intel-iommu.h> 12#include <linux/intel-iommu.h>
10#include <linux/acpi.h> 13#include <linux/acpi.h>
11#include <linux/irqdomain.h> 14#include <linux/irqdomain.h>
15#include <linux/crash_dump.h>
12#include <asm/io_apic.h> 16#include <asm/io_apic.h>
13#include <asm/smp.h> 17#include <asm/smp.h>
14#include <asm/cpu.h> 18#include <asm/cpu.h>
@@ -74,8 +78,28 @@ static struct hpet_scope ir_hpet[MAX_HPET_TBS];
74static DEFINE_RAW_SPINLOCK(irq_2_ir_lock); 78static DEFINE_RAW_SPINLOCK(irq_2_ir_lock);
75static struct irq_domain_ops intel_ir_domain_ops; 79static struct irq_domain_ops intel_ir_domain_ops;
76 80
81static void iommu_disable_irq_remapping(struct intel_iommu *iommu);
77static int __init parse_ioapics_under_ir(void); 82static int __init parse_ioapics_under_ir(void);
78 83
84static bool ir_pre_enabled(struct intel_iommu *iommu)
85{
86 return (iommu->flags & VTD_FLAG_IRQ_REMAP_PRE_ENABLED);
87}
88
89static void clear_ir_pre_enabled(struct intel_iommu *iommu)
90{
91 iommu->flags &= ~VTD_FLAG_IRQ_REMAP_PRE_ENABLED;
92}
93
94static void init_ir_status(struct intel_iommu *iommu)
95{
96 u32 gsts;
97
98 gsts = readl(iommu->reg + DMAR_GSTS_REG);
99 if (gsts & DMA_GSTS_IRES)
100 iommu->flags |= VTD_FLAG_IRQ_REMAP_PRE_ENABLED;
101}
102
79static int alloc_irte(struct intel_iommu *iommu, int irq, 103static int alloc_irte(struct intel_iommu *iommu, int irq,
80 struct irq_2_iommu *irq_iommu, u16 count) 104 struct irq_2_iommu *irq_iommu, u16 count)
81{ 105{
@@ -93,8 +117,7 @@ static int alloc_irte(struct intel_iommu *iommu, int irq,
93 } 117 }
94 118
95 if (mask > ecap_max_handle_mask(iommu->ecap)) { 119 if (mask > ecap_max_handle_mask(iommu->ecap)) {
96 printk(KERN_ERR 120 pr_err("Requested mask %x exceeds the max invalidation handle"
97 "Requested mask %x exceeds the max invalidation handle"
98 " mask value %Lx\n", mask, 121 " mask value %Lx\n", mask,
99 ecap_max_handle_mask(iommu->ecap)); 122 ecap_max_handle_mask(iommu->ecap));
100 return -1; 123 return -1;
@@ -268,7 +291,7 @@ static int set_ioapic_sid(struct irte *irte, int apic)
268 up_read(&dmar_global_lock); 291 up_read(&dmar_global_lock);
269 292
270 if (sid == 0) { 293 if (sid == 0) {
271 pr_warning("Failed to set source-id of IOAPIC (%d)\n", apic); 294 pr_warn("Failed to set source-id of IOAPIC (%d)\n", apic);
272 return -1; 295 return -1;
273 } 296 }
274 297
@@ -295,7 +318,7 @@ static int set_hpet_sid(struct irte *irte, u8 id)
295 up_read(&dmar_global_lock); 318 up_read(&dmar_global_lock);
296 319
297 if (sid == 0) { 320 if (sid == 0) {
298 pr_warning("Failed to set source-id of HPET block (%d)\n", id); 321 pr_warn("Failed to set source-id of HPET block (%d)\n", id);
299 return -1; 322 return -1;
300 } 323 }
301 324
@@ -359,11 +382,59 @@ static int set_msi_sid(struct irte *irte, struct pci_dev *dev)
359 return 0; 382 return 0;
360} 383}
361 384
385static int iommu_load_old_irte(struct intel_iommu *iommu)
386{
387 struct irte *old_ir_table;
388 phys_addr_t irt_phys;
389 unsigned int i;
390 size_t size;
391 u64 irta;
392
393 if (!is_kdump_kernel()) {
394 pr_warn("IRQ remapping was enabled on %s but we are not in kdump mode\n",
395 iommu->name);
396 clear_ir_pre_enabled(iommu);
397 iommu_disable_irq_remapping(iommu);
398 return -EINVAL;
399 }
400
401 /* Check whether the old ir-table has the same size as ours */
402 irta = dmar_readq(iommu->reg + DMAR_IRTA_REG);
403 if ((irta & INTR_REMAP_TABLE_REG_SIZE_MASK)
404 != INTR_REMAP_TABLE_REG_SIZE)
405 return -EINVAL;
406
407 irt_phys = irta & VTD_PAGE_MASK;
408 size = INTR_REMAP_TABLE_ENTRIES*sizeof(struct irte);
409
410 /* Map the old IR table */
411 old_ir_table = ioremap_cache(irt_phys, size);
412 if (!old_ir_table)
413 return -ENOMEM;
414
415 /* Copy data over */
416 memcpy(iommu->ir_table->base, old_ir_table, size);
417
418 __iommu_flush_cache(iommu, iommu->ir_table->base, size);
419
420 /*
421 * Now check the table for used entries and mark those as
422 * allocated in the bitmap
423 */
424 for (i = 0; i < INTR_REMAP_TABLE_ENTRIES; i++) {
425 if (iommu->ir_table->base[i].present)
426 bitmap_set(iommu->ir_table->bitmap, i, 1);
427 }
428
429 return 0;
430}
431
432
362static void iommu_set_irq_remapping(struct intel_iommu *iommu, int mode) 433static void iommu_set_irq_remapping(struct intel_iommu *iommu, int mode)
363{ 434{
435 unsigned long flags;
364 u64 addr; 436 u64 addr;
365 u32 sts; 437 u32 sts;
366 unsigned long flags;
367 438
368 addr = virt_to_phys((void *)iommu->ir_table->base); 439 addr = virt_to_phys((void *)iommu->ir_table->base);
369 440
@@ -380,10 +451,16 @@ static void iommu_set_irq_remapping(struct intel_iommu *iommu, int mode)
380 raw_spin_unlock_irqrestore(&iommu->register_lock, flags); 451 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
381 452
382 /* 453 /*
383 * global invalidation of interrupt entry cache before enabling 454 * Global invalidation of interrupt entry cache to make sure the
384 * interrupt-remapping. 455 * hardware uses the new irq remapping table.
385 */ 456 */
386 qi_global_iec(iommu); 457 qi_global_iec(iommu);
458}
459
460static void iommu_enable_irq_remapping(struct intel_iommu *iommu)
461{
462 unsigned long flags;
463 u32 sts;
387 464
388 raw_spin_lock_irqsave(&iommu->register_lock, flags); 465 raw_spin_lock_irqsave(&iommu->register_lock, flags);
389 466
@@ -449,6 +526,37 @@ static int intel_setup_irq_remapping(struct intel_iommu *iommu)
449 ir_table->base = page_address(pages); 526 ir_table->base = page_address(pages);
450 ir_table->bitmap = bitmap; 527 ir_table->bitmap = bitmap;
451 iommu->ir_table = ir_table; 528 iommu->ir_table = ir_table;
529
530 /*
531 * If the queued invalidation is already initialized,
532 * shouldn't disable it.
533 */
534 if (!iommu->qi) {
535 /*
536 * Clear previous faults.
537 */
538 dmar_fault(-1, iommu);
539 dmar_disable_qi(iommu);
540
541 if (dmar_enable_qi(iommu)) {
542 pr_err("Failed to enable queued invalidation\n");
543 goto out_free_bitmap;
544 }
545 }
546
547 init_ir_status(iommu);
548
549 if (ir_pre_enabled(iommu)) {
550 if (iommu_load_old_irte(iommu))
551 pr_err("Failed to copy IR table for %s from previous kernel\n",
552 iommu->name);
553 else
554 pr_info("Copied IR table for %s from previous kernel\n",
555 iommu->name);
556 }
557
558 iommu_set_irq_remapping(iommu, eim_mode);
559
452 return 0; 560 return 0;
453 561
454out_free_bitmap: 562out_free_bitmap:
@@ -457,6 +565,9 @@ out_free_pages:
457 __free_pages(pages, INTR_REMAP_PAGE_ORDER); 565 __free_pages(pages, INTR_REMAP_PAGE_ORDER);
458out_free_table: 566out_free_table:
459 kfree(ir_table); 567 kfree(ir_table);
568
569 iommu->ir_table = NULL;
570
460 return -ENOMEM; 571 return -ENOMEM;
461} 572}
462 573
@@ -534,17 +645,17 @@ static void __init intel_cleanup_irq_remapping(void)
534 } 645 }
535 646
536 if (x2apic_supported()) 647 if (x2apic_supported())
537 pr_warn("Failed to enable irq remapping. You are vulnerable to irq-injection attacks.\n"); 648 pr_warn("Failed to enable irq remapping. You are vulnerable to irq-injection attacks.\n");
538} 649}
539 650
540static int __init intel_prepare_irq_remapping(void) 651static int __init intel_prepare_irq_remapping(void)
541{ 652{
542 struct dmar_drhd_unit *drhd; 653 struct dmar_drhd_unit *drhd;
543 struct intel_iommu *iommu; 654 struct intel_iommu *iommu;
655 int eim = 0;
544 656
545 if (irq_remap_broken) { 657 if (irq_remap_broken) {
546 printk(KERN_WARNING 658 pr_warn("This system BIOS has enabled interrupt remapping\n"
547 "This system BIOS has enabled interrupt remapping\n"
548 "on a chipset that contains an erratum making that\n" 659 "on a chipset that contains an erratum making that\n"
549 "feature unstable. To maintain system stability\n" 660 "feature unstable. To maintain system stability\n"
550 "interrupt remapping is being disabled. Please\n" 661 "interrupt remapping is being disabled. Please\n"
@@ -560,7 +671,7 @@ static int __init intel_prepare_irq_remapping(void)
560 return -ENODEV; 671 return -ENODEV;
561 672
562 if (parse_ioapics_under_ir() != 1) { 673 if (parse_ioapics_under_ir() != 1) {
563 printk(KERN_INFO "Not enabling interrupt remapping\n"); 674 pr_info("Not enabling interrupt remapping\n");
564 goto error; 675 goto error;
565 } 676 }
566 677
@@ -569,10 +680,34 @@ static int __init intel_prepare_irq_remapping(void)
569 if (!ecap_ir_support(iommu->ecap)) 680 if (!ecap_ir_support(iommu->ecap))
570 goto error; 681 goto error;
571 682
572 /* Do the allocations early */ 683 /* Detect remapping mode: lapic or x2apic */
573 for_each_iommu(iommu, drhd) 684 if (x2apic_supported()) {
574 if (intel_setup_irq_remapping(iommu)) 685 eim = !dmar_x2apic_optout();
686 if (!eim) {
687 pr_info("x2apic is disabled because BIOS sets x2apic opt out bit.");
688 pr_info("Use 'intremap=no_x2apic_optout' to override the BIOS setting.\n");
689 }
690 }
691
692 for_each_iommu(iommu, drhd) {
693 if (eim && !ecap_eim_support(iommu->ecap)) {
694 pr_info("%s does not support EIM\n", iommu->name);
695 eim = 0;
696 }
697 }
698
699 eim_mode = eim;
700 if (eim)
701 pr_info("Queued invalidation will be enabled to support x2apic and Intr-remapping.\n");
702
703 /* Do the initializations early */
704 for_each_iommu(iommu, drhd) {
705 if (intel_setup_irq_remapping(iommu)) {
706 pr_err("Failed to setup irq remapping for %s\n",
707 iommu->name);
575 goto error; 708 goto error;
709 }
710 }
576 711
577 return 0; 712 return 0;
578 713
@@ -606,68 +741,13 @@ static int __init intel_enable_irq_remapping(void)
606 struct dmar_drhd_unit *drhd; 741 struct dmar_drhd_unit *drhd;
607 struct intel_iommu *iommu; 742 struct intel_iommu *iommu;
608 bool setup = false; 743 bool setup = false;
609 int eim = 0;
610
611 if (x2apic_supported()) {
612 eim = !dmar_x2apic_optout();
613 if (!eim)
614 pr_info("x2apic is disabled because BIOS sets x2apic opt out bit. You can use 'intremap=no_x2apic_optout' to override the BIOS setting.\n");
615 }
616
617 for_each_iommu(iommu, drhd) {
618 /*
619 * If the queued invalidation is already initialized,
620 * shouldn't disable it.
621 */
622 if (iommu->qi)
623 continue;
624
625 /*
626 * Clear previous faults.
627 */
628 dmar_fault(-1, iommu);
629
630 /*
631 * Disable intr remapping and queued invalidation, if already
632 * enabled prior to OS handover.
633 */
634 iommu_disable_irq_remapping(iommu);
635
636 dmar_disable_qi(iommu);
637 }
638
639 /*
640 * check for the Interrupt-remapping support
641 */
642 for_each_iommu(iommu, drhd)
643 if (eim && !ecap_eim_support(iommu->ecap)) {
644 printk(KERN_INFO "DRHD %Lx: EIM not supported by DRHD, "
645 " ecap %Lx\n", drhd->reg_base_addr, iommu->ecap);
646 eim = 0;
647 }
648 eim_mode = eim;
649 if (eim)
650 pr_info("Queued invalidation will be enabled to support x2apic and Intr-remapping.\n");
651
652 /*
653 * Enable queued invalidation for all the DRHD's.
654 */
655 for_each_iommu(iommu, drhd) {
656 int ret = dmar_enable_qi(iommu);
657
658 if (ret) {
659 printk(KERN_ERR "DRHD %Lx: failed to enable queued, "
660 " invalidation, ecap %Lx, ret %d\n",
661 drhd->reg_base_addr, iommu->ecap, ret);
662 goto error;
663 }
664 }
665 744
666 /* 745 /*
667 * Setup Interrupt-remapping for all the DRHD's now. 746 * Setup Interrupt-remapping for all the DRHD's now.
668 */ 747 */
669 for_each_iommu(iommu, drhd) { 748 for_each_iommu(iommu, drhd) {
670 iommu_set_irq_remapping(iommu, eim); 749 if (!ir_pre_enabled(iommu))
750 iommu_enable_irq_remapping(iommu);
671 setup = true; 751 setup = true;
672 } 752 }
673 753
@@ -678,9 +758,9 @@ static int __init intel_enable_irq_remapping(void)
678 758
679 set_irq_posting_cap(); 759 set_irq_posting_cap();
680 760
681 pr_info("Enabled IRQ remapping in %s mode\n", eim ? "x2apic" : "xapic"); 761 pr_info("Enabled IRQ remapping in %s mode\n", eim_mode ? "x2apic" : "xapic");
682 762
683 return eim ? IRQ_REMAP_X2APIC_MODE : IRQ_REMAP_XAPIC_MODE; 763 return eim_mode ? IRQ_REMAP_X2APIC_MODE : IRQ_REMAP_XAPIC_MODE;
684 764
685error: 765error:
686 intel_cleanup_irq_remapping(); 766 intel_cleanup_irq_remapping();
@@ -905,6 +985,7 @@ static int reenable_irq_remapping(int eim)
905 985
906 /* Set up interrupt remapping for iommu.*/ 986 /* Set up interrupt remapping for iommu.*/
907 iommu_set_irq_remapping(iommu, eim); 987 iommu_set_irq_remapping(iommu, eim);
988 iommu_enable_irq_remapping(iommu);
908 setup = true; 989 setup = true;
909 } 990 }
910 991
@@ -1169,7 +1250,6 @@ static void intel_free_irq_resources(struct irq_domain *domain,
1169 struct irq_2_iommu *irq_iommu; 1250 struct irq_2_iommu *irq_iommu;
1170 unsigned long flags; 1251 unsigned long flags;
1171 int i; 1252 int i;
1172
1173 for (i = 0; i < nr_irqs; i++) { 1253 for (i = 0; i < nr_irqs; i++) {
1174 irq_data = irq_domain_get_irq_data(domain, virq + i); 1254 irq_data = irq_domain_get_irq_data(domain, virq + i);
1175 if (irq_data && irq_data->chip_data) { 1255 if (irq_data && irq_data->chip_data) {
@@ -1317,28 +1397,12 @@ static int dmar_ir_add(struct dmar_drhd_unit *dmaru, struct intel_iommu *iommu)
1317 /* Setup Interrupt-remapping now. */ 1397 /* Setup Interrupt-remapping now. */
1318 ret = intel_setup_irq_remapping(iommu); 1398 ret = intel_setup_irq_remapping(iommu);
1319 if (ret) { 1399 if (ret) {
1320 pr_err("DRHD %Lx: failed to allocate resource\n", 1400 pr_err("Failed to setup irq remapping for %s\n",
1321 iommu->reg_phys); 1401 iommu->name);
1322 ir_remove_ioapic_hpet_scope(iommu);
1323 return ret;
1324 }
1325
1326 if (!iommu->qi) {
1327 /* Clear previous faults. */
1328 dmar_fault(-1, iommu);
1329 iommu_disable_irq_remapping(iommu);
1330 dmar_disable_qi(iommu);
1331 }
1332
1333 /* Enable queued invalidation */
1334 ret = dmar_enable_qi(iommu);
1335 if (!ret) {
1336 iommu_set_irq_remapping(iommu, eim);
1337 } else {
1338 pr_err("DRHD %Lx: failed to enable queued invalidation, ecap %Lx, ret %d\n",
1339 iommu->reg_phys, iommu->ecap, ret);
1340 intel_teardown_irq_remapping(iommu); 1402 intel_teardown_irq_remapping(iommu);
1341 ir_remove_ioapic_hpet_scope(iommu); 1403 ir_remove_ioapic_hpet_scope(iommu);
1404 } else {
1405 iommu_enable_irq_remapping(iommu);
1342 } 1406 }
1343 1407
1344 return ret; 1408 return ret;
diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
index d4f527e56679..f286090931cc 100644
--- a/drivers/iommu/iommu.c
+++ b/drivers/iommu/iommu.c
@@ -16,7 +16,7 @@
16 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 16 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 */ 17 */
18 18
19#define pr_fmt(fmt) "%s: " fmt, __func__ 19#define pr_fmt(fmt) "iommu: " fmt
20 20
21#include <linux/device.h> 21#include <linux/device.h>
22#include <linux/kernel.h> 22#include <linux/kernel.h>
@@ -51,6 +51,8 @@ struct iommu_group {
51 void (*iommu_data_release)(void *iommu_data); 51 void (*iommu_data_release)(void *iommu_data);
52 char *name; 52 char *name;
53 int id; 53 int id;
54 struct iommu_domain *default_domain;
55 struct iommu_domain *domain;
54}; 56};
55 57
56struct iommu_device { 58struct iommu_device {
@@ -75,6 +77,15 @@ struct iommu_group_attribute iommu_group_attr_##_name = \
75#define to_iommu_group(_kobj) \ 77#define to_iommu_group(_kobj) \
76 container_of(_kobj, struct iommu_group, kobj) 78 container_of(_kobj, struct iommu_group, kobj)
77 79
80static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus,
81 unsigned type);
82static int __iommu_attach_device(struct iommu_domain *domain,
83 struct device *dev);
84static int __iommu_attach_group(struct iommu_domain *domain,
85 struct iommu_group *group);
86static void __iommu_detach_group(struct iommu_domain *domain,
87 struct iommu_group *group);
88
78static ssize_t iommu_group_attr_show(struct kobject *kobj, 89static ssize_t iommu_group_attr_show(struct kobject *kobj,
79 struct attribute *__attr, char *buf) 90 struct attribute *__attr, char *buf)
80{ 91{
@@ -128,6 +139,8 @@ static void iommu_group_release(struct kobject *kobj)
128{ 139{
129 struct iommu_group *group = to_iommu_group(kobj); 140 struct iommu_group *group = to_iommu_group(kobj);
130 141
142 pr_debug("Releasing group %d\n", group->id);
143
131 if (group->iommu_data_release) 144 if (group->iommu_data_release)
132 group->iommu_data_release(group->iommu_data); 145 group->iommu_data_release(group->iommu_data);
133 146
@@ -135,6 +148,9 @@ static void iommu_group_release(struct kobject *kobj)
135 ida_remove(&iommu_group_ida, group->id); 148 ida_remove(&iommu_group_ida, group->id);
136 mutex_unlock(&iommu_group_mutex); 149 mutex_unlock(&iommu_group_mutex);
137 150
151 if (group->default_domain)
152 iommu_domain_free(group->default_domain);
153
138 kfree(group->name); 154 kfree(group->name);
139 kfree(group); 155 kfree(group);
140} 156}
@@ -207,6 +223,8 @@ again:
207 */ 223 */
208 kobject_put(&group->kobj); 224 kobject_put(&group->kobj);
209 225
226 pr_debug("Allocated group %d\n", group->id);
227
210 return group; 228 return group;
211} 229}
212EXPORT_SYMBOL_GPL(iommu_group_alloc); 230EXPORT_SYMBOL_GPL(iommu_group_alloc);
@@ -307,6 +325,52 @@ int iommu_group_set_name(struct iommu_group *group, const char *name)
307} 325}
308EXPORT_SYMBOL_GPL(iommu_group_set_name); 326EXPORT_SYMBOL_GPL(iommu_group_set_name);
309 327
328static int iommu_group_create_direct_mappings(struct iommu_group *group,
329 struct device *dev)
330{
331 struct iommu_domain *domain = group->default_domain;
332 struct iommu_dm_region *entry;
333 struct list_head mappings;
334 unsigned long pg_size;
335 int ret = 0;
336
337 if (!domain || domain->type != IOMMU_DOMAIN_DMA)
338 return 0;
339
340 BUG_ON(!domain->ops->pgsize_bitmap);
341
342 pg_size = 1UL << __ffs(domain->ops->pgsize_bitmap);
343 INIT_LIST_HEAD(&mappings);
344
345 iommu_get_dm_regions(dev, &mappings);
346
347 /* We need to consider overlapping regions for different devices */
348 list_for_each_entry(entry, &mappings, list) {
349 dma_addr_t start, end, addr;
350
351 start = ALIGN(entry->start, pg_size);
352 end = ALIGN(entry->start + entry->length, pg_size);
353
354 for (addr = start; addr < end; addr += pg_size) {
355 phys_addr_t phys_addr;
356
357 phys_addr = iommu_iova_to_phys(domain, addr);
358 if (phys_addr)
359 continue;
360
361 ret = iommu_map(domain, addr, addr, pg_size, entry->prot);
362 if (ret)
363 goto out;
364 }
365
366 }
367
368out:
369 iommu_put_dm_regions(dev, &mappings);
370
371 return ret;
372}
373
310/** 374/**
311 * iommu_group_add_device - add a device to an iommu group 375 * iommu_group_add_device - add a device to an iommu group
312 * @group: the group into which to add the device (reference should be held) 376 * @group: the group into which to add the device (reference should be held)
@@ -363,8 +427,12 @@ rename:
363 427
364 dev->iommu_group = group; 428 dev->iommu_group = group;
365 429
430 iommu_group_create_direct_mappings(group, dev);
431
366 mutex_lock(&group->mutex); 432 mutex_lock(&group->mutex);
367 list_add_tail(&device->list, &group->devices); 433 list_add_tail(&device->list, &group->devices);
434 if (group->domain)
435 __iommu_attach_device(group->domain, dev);
368 mutex_unlock(&group->mutex); 436 mutex_unlock(&group->mutex);
369 437
370 /* Notify any listeners about change to group. */ 438 /* Notify any listeners about change to group. */
@@ -372,6 +440,9 @@ rename:
372 IOMMU_GROUP_NOTIFY_ADD_DEVICE, dev); 440 IOMMU_GROUP_NOTIFY_ADD_DEVICE, dev);
373 441
374 trace_add_device_to_group(group->id, dev); 442 trace_add_device_to_group(group->id, dev);
443
444 pr_info("Adding device %s to group %d\n", dev_name(dev), group->id);
445
375 return 0; 446 return 0;
376} 447}
377EXPORT_SYMBOL_GPL(iommu_group_add_device); 448EXPORT_SYMBOL_GPL(iommu_group_add_device);
@@ -388,6 +459,8 @@ void iommu_group_remove_device(struct device *dev)
388 struct iommu_group *group = dev->iommu_group; 459 struct iommu_group *group = dev->iommu_group;
389 struct iommu_device *tmp_device, *device = NULL; 460 struct iommu_device *tmp_device, *device = NULL;
390 461
462 pr_info("Removing device %s from group %d\n", dev_name(dev), group->id);
463
391 /* Pre-notify listeners that a device is being removed. */ 464 /* Pre-notify listeners that a device is being removed. */
392 blocking_notifier_call_chain(&group->notifier, 465 blocking_notifier_call_chain(&group->notifier,
393 IOMMU_GROUP_NOTIFY_DEL_DEVICE, dev); 466 IOMMU_GROUP_NOTIFY_DEL_DEVICE, dev);
@@ -417,6 +490,17 @@ void iommu_group_remove_device(struct device *dev)
417} 490}
418EXPORT_SYMBOL_GPL(iommu_group_remove_device); 491EXPORT_SYMBOL_GPL(iommu_group_remove_device);
419 492
493static int iommu_group_device_count(struct iommu_group *group)
494{
495 struct iommu_device *entry;
496 int ret = 0;
497
498 list_for_each_entry(entry, &group->devices, list)
499 ret++;
500
501 return ret;
502}
503
420/** 504/**
421 * iommu_group_for_each_dev - iterate over each device in the group 505 * iommu_group_for_each_dev - iterate over each device in the group
422 * @group: the group 506 * @group: the group
@@ -428,19 +512,30 @@ EXPORT_SYMBOL_GPL(iommu_group_remove_device);
428 * The group->mutex is held across callbacks, which will block calls to 512 * The group->mutex is held across callbacks, which will block calls to
429 * iommu_group_add/remove_device. 513 * iommu_group_add/remove_device.
430 */ 514 */
431int iommu_group_for_each_dev(struct iommu_group *group, void *data, 515static int __iommu_group_for_each_dev(struct iommu_group *group, void *data,
432 int (*fn)(struct device *, void *)) 516 int (*fn)(struct device *, void *))
433{ 517{
434 struct iommu_device *device; 518 struct iommu_device *device;
435 int ret = 0; 519 int ret = 0;
436 520
437 mutex_lock(&group->mutex);
438 list_for_each_entry(device, &group->devices, list) { 521 list_for_each_entry(device, &group->devices, list) {
439 ret = fn(device->dev, data); 522 ret = fn(device->dev, data);
440 if (ret) 523 if (ret)
441 break; 524 break;
442 } 525 }
526 return ret;
527}
528
529
530int iommu_group_for_each_dev(struct iommu_group *group, void *data,
531 int (*fn)(struct device *, void *))
532{
533 int ret;
534
535 mutex_lock(&group->mutex);
536 ret = __iommu_group_for_each_dev(group, data, fn);
443 mutex_unlock(&group->mutex); 537 mutex_unlock(&group->mutex);
538
444 return ret; 539 return ret;
445} 540}
446EXPORT_SYMBOL_GPL(iommu_group_for_each_dev); 541EXPORT_SYMBOL_GPL(iommu_group_for_each_dev);
@@ -692,7 +787,19 @@ static struct iommu_group *iommu_group_get_for_pci_dev(struct pci_dev *pdev)
692 return group; 787 return group;
693 788
694 /* No shared group found, allocate new */ 789 /* No shared group found, allocate new */
695 return iommu_group_alloc(); 790 group = iommu_group_alloc();
791 if (IS_ERR(group))
792 return NULL;
793
794 /*
795 * Try to allocate a default domain - needs support from the
796 * IOMMU driver.
797 */
798 group->default_domain = __iommu_domain_alloc(pdev->dev.bus,
799 IOMMU_DOMAIN_DMA);
800 group->domain = group->default_domain;
801
802 return group;
696} 803}
697 804
698/** 805/**
@@ -731,17 +838,42 @@ struct iommu_group *iommu_group_get_for_dev(struct device *dev)
731 return group; 838 return group;
732} 839}
733 840
841struct iommu_domain *iommu_group_default_domain(struct iommu_group *group)
842{
843 return group->default_domain;
844}
845
734static int add_iommu_group(struct device *dev, void *data) 846static int add_iommu_group(struct device *dev, void *data)
735{ 847{
736 struct iommu_callback_data *cb = data; 848 struct iommu_callback_data *cb = data;
737 const struct iommu_ops *ops = cb->ops; 849 const struct iommu_ops *ops = cb->ops;
850 int ret;
738 851
739 if (!ops->add_device) 852 if (!ops->add_device)
740 return 0; 853 return 0;
741 854
742 WARN_ON(dev->iommu_group); 855 WARN_ON(dev->iommu_group);
743 856
744 ops->add_device(dev); 857 ret = ops->add_device(dev);
858
859 /*
860 * We ignore -ENODEV errors for now, as they just mean that the
861 * device is not translated by an IOMMU. We still care about
862 * other errors and fail to initialize when they happen.
863 */
864 if (ret == -ENODEV)
865 ret = 0;
866
867 return ret;
868}
869
870static int remove_iommu_group(struct device *dev, void *data)
871{
872 struct iommu_callback_data *cb = data;
873 const struct iommu_ops *ops = cb->ops;
874
875 if (ops->remove_device && dev->iommu_group)
876 ops->remove_device(dev);
745 877
746 return 0; 878 return 0;
747} 879}
@@ -761,7 +893,7 @@ static int iommu_bus_notifier(struct notifier_block *nb,
761 if (action == BUS_NOTIFY_ADD_DEVICE) { 893 if (action == BUS_NOTIFY_ADD_DEVICE) {
762 if (ops->add_device) 894 if (ops->add_device)
763 return ops->add_device(dev); 895 return ops->add_device(dev);
764 } else if (action == BUS_NOTIFY_DEL_DEVICE) { 896 } else if (action == BUS_NOTIFY_REMOVED_DEVICE) {
765 if (ops->remove_device && dev->iommu_group) { 897 if (ops->remove_device && dev->iommu_group) {
766 ops->remove_device(dev); 898 ops->remove_device(dev);
767 return 0; 899 return 0;
@@ -814,19 +946,25 @@ static int iommu_bus_init(struct bus_type *bus, const struct iommu_ops *ops)
814 nb->notifier_call = iommu_bus_notifier; 946 nb->notifier_call = iommu_bus_notifier;
815 947
816 err = bus_register_notifier(bus, nb); 948 err = bus_register_notifier(bus, nb);
817 if (err) { 949 if (err)
818 kfree(nb); 950 goto out_free;
819 return err;
820 }
821 951
822 err = bus_for_each_dev(bus, NULL, &cb, add_iommu_group); 952 err = bus_for_each_dev(bus, NULL, &cb, add_iommu_group);
823 if (err) { 953 if (err)
824 bus_unregister_notifier(bus, nb); 954 goto out_err;
825 kfree(nb); 955
826 return err;
827 }
828 956
829 return 0; 957 return 0;
958
959out_err:
960 /* Clean up */
961 bus_for_each_dev(bus, NULL, &cb, remove_iommu_group);
962 bus_unregister_notifier(bus, nb);
963
964out_free:
965 kfree(nb);
966
967 return err;
830} 968}
831 969
832/** 970/**
@@ -898,22 +1036,28 @@ void iommu_set_fault_handler(struct iommu_domain *domain,
898} 1036}
899EXPORT_SYMBOL_GPL(iommu_set_fault_handler); 1037EXPORT_SYMBOL_GPL(iommu_set_fault_handler);
900 1038
901struct iommu_domain *iommu_domain_alloc(struct bus_type *bus) 1039static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus,
1040 unsigned type)
902{ 1041{
903 struct iommu_domain *domain; 1042 struct iommu_domain *domain;
904 1043
905 if (bus == NULL || bus->iommu_ops == NULL) 1044 if (bus == NULL || bus->iommu_ops == NULL)
906 return NULL; 1045 return NULL;
907 1046
908 domain = bus->iommu_ops->domain_alloc(IOMMU_DOMAIN_UNMANAGED); 1047 domain = bus->iommu_ops->domain_alloc(type);
909 if (!domain) 1048 if (!domain)
910 return NULL; 1049 return NULL;
911 1050
912 domain->ops = bus->iommu_ops; 1051 domain->ops = bus->iommu_ops;
913 domain->type = IOMMU_DOMAIN_UNMANAGED; 1052 domain->type = type;
914 1053
915 return domain; 1054 return domain;
916} 1055}
1056
1057struct iommu_domain *iommu_domain_alloc(struct bus_type *bus)
1058{
1059 return __iommu_domain_alloc(bus, IOMMU_DOMAIN_UNMANAGED);
1060}
917EXPORT_SYMBOL_GPL(iommu_domain_alloc); 1061EXPORT_SYMBOL_GPL(iommu_domain_alloc);
918 1062
919void iommu_domain_free(struct iommu_domain *domain) 1063void iommu_domain_free(struct iommu_domain *domain)
@@ -922,7 +1066,8 @@ void iommu_domain_free(struct iommu_domain *domain)
922} 1066}
923EXPORT_SYMBOL_GPL(iommu_domain_free); 1067EXPORT_SYMBOL_GPL(iommu_domain_free);
924 1068
925int iommu_attach_device(struct iommu_domain *domain, struct device *dev) 1069static int __iommu_attach_device(struct iommu_domain *domain,
1070 struct device *dev)
926{ 1071{
927 int ret; 1072 int ret;
928 if (unlikely(domain->ops->attach_dev == NULL)) 1073 if (unlikely(domain->ops->attach_dev == NULL))
@@ -933,9 +1078,38 @@ int iommu_attach_device(struct iommu_domain *domain, struct device *dev)
933 trace_attach_device_to_domain(dev); 1078 trace_attach_device_to_domain(dev);
934 return ret; 1079 return ret;
935} 1080}
1081
1082int iommu_attach_device(struct iommu_domain *domain, struct device *dev)
1083{
1084 struct iommu_group *group;
1085 int ret;
1086
1087 group = iommu_group_get(dev);
1088 /* FIXME: Remove this when groups a mandatory for iommu drivers */
1089 if (group == NULL)
1090 return __iommu_attach_device(domain, dev);
1091
1092 /*
1093 * We have a group - lock it to make sure the device-count doesn't
1094 * change while we are attaching
1095 */
1096 mutex_lock(&group->mutex);
1097 ret = -EINVAL;
1098 if (iommu_group_device_count(group) != 1)
1099 goto out_unlock;
1100
1101 ret = __iommu_attach_group(domain, group);
1102
1103out_unlock:
1104 mutex_unlock(&group->mutex);
1105 iommu_group_put(group);
1106
1107 return ret;
1108}
936EXPORT_SYMBOL_GPL(iommu_attach_device); 1109EXPORT_SYMBOL_GPL(iommu_attach_device);
937 1110
938void iommu_detach_device(struct iommu_domain *domain, struct device *dev) 1111static void __iommu_detach_device(struct iommu_domain *domain,
1112 struct device *dev)
939{ 1113{
940 if (unlikely(domain->ops->detach_dev == NULL)) 1114 if (unlikely(domain->ops->detach_dev == NULL))
941 return; 1115 return;
@@ -943,8 +1117,48 @@ void iommu_detach_device(struct iommu_domain *domain, struct device *dev)
943 domain->ops->detach_dev(domain, dev); 1117 domain->ops->detach_dev(domain, dev);
944 trace_detach_device_from_domain(dev); 1118 trace_detach_device_from_domain(dev);
945} 1119}
1120
1121void iommu_detach_device(struct iommu_domain *domain, struct device *dev)
1122{
1123 struct iommu_group *group;
1124
1125 group = iommu_group_get(dev);
1126 /* FIXME: Remove this when groups a mandatory for iommu drivers */
1127 if (group == NULL)
1128 return __iommu_detach_device(domain, dev);
1129
1130 mutex_lock(&group->mutex);
1131 if (iommu_group_device_count(group) != 1) {
1132 WARN_ON(1);
1133 goto out_unlock;
1134 }
1135
1136 __iommu_detach_group(domain, group);
1137
1138out_unlock:
1139 mutex_unlock(&group->mutex);
1140 iommu_group_put(group);
1141}
946EXPORT_SYMBOL_GPL(iommu_detach_device); 1142EXPORT_SYMBOL_GPL(iommu_detach_device);
947 1143
1144struct iommu_domain *iommu_get_domain_for_dev(struct device *dev)
1145{
1146 struct iommu_domain *domain;
1147 struct iommu_group *group;
1148
1149 group = iommu_group_get(dev);
1150 /* FIXME: Remove this when groups a mandatory for iommu drivers */
1151 if (group == NULL)
1152 return NULL;
1153
1154 domain = group->domain;
1155
1156 iommu_group_put(group);
1157
1158 return domain;
1159}
1160EXPORT_SYMBOL_GPL(iommu_get_domain_for_dev);
1161
948/* 1162/*
949 * IOMMU groups are really the natrual working unit of the IOMMU, but 1163 * IOMMU groups are really the natrual working unit of the IOMMU, but
950 * the IOMMU API works on domains and devices. Bridge that gap by 1164 * the IOMMU API works on domains and devices. Bridge that gap by
@@ -959,13 +1173,34 @@ static int iommu_group_do_attach_device(struct device *dev, void *data)
959{ 1173{
960 struct iommu_domain *domain = data; 1174 struct iommu_domain *domain = data;
961 1175
962 return iommu_attach_device(domain, dev); 1176 return __iommu_attach_device(domain, dev);
1177}
1178
1179static int __iommu_attach_group(struct iommu_domain *domain,
1180 struct iommu_group *group)
1181{
1182 int ret;
1183
1184 if (group->default_domain && group->domain != group->default_domain)
1185 return -EBUSY;
1186
1187 ret = __iommu_group_for_each_dev(group, domain,
1188 iommu_group_do_attach_device);
1189 if (ret == 0)
1190 group->domain = domain;
1191
1192 return ret;
963} 1193}
964 1194
965int iommu_attach_group(struct iommu_domain *domain, struct iommu_group *group) 1195int iommu_attach_group(struct iommu_domain *domain, struct iommu_group *group)
966{ 1196{
967 return iommu_group_for_each_dev(group, domain, 1197 int ret;
968 iommu_group_do_attach_device); 1198
1199 mutex_lock(&group->mutex);
1200 ret = __iommu_attach_group(domain, group);
1201 mutex_unlock(&group->mutex);
1202
1203 return ret;
969} 1204}
970EXPORT_SYMBOL_GPL(iommu_attach_group); 1205EXPORT_SYMBOL_GPL(iommu_attach_group);
971 1206
@@ -973,14 +1208,40 @@ static int iommu_group_do_detach_device(struct device *dev, void *data)
973{ 1208{
974 struct iommu_domain *domain = data; 1209 struct iommu_domain *domain = data;
975 1210
976 iommu_detach_device(domain, dev); 1211 __iommu_detach_device(domain, dev);
977 1212
978 return 0; 1213 return 0;
979} 1214}
980 1215
1216static void __iommu_detach_group(struct iommu_domain *domain,
1217 struct iommu_group *group)
1218{
1219 int ret;
1220
1221 if (!group->default_domain) {
1222 __iommu_group_for_each_dev(group, domain,
1223 iommu_group_do_detach_device);
1224 group->domain = NULL;
1225 return;
1226 }
1227
1228 if (group->domain == group->default_domain)
1229 return;
1230
1231 /* Detach by re-attaching to the default domain */
1232 ret = __iommu_group_for_each_dev(group, group->default_domain,
1233 iommu_group_do_attach_device);
1234 if (ret != 0)
1235 WARN_ON(1);
1236 else
1237 group->domain = group->default_domain;
1238}
1239
981void iommu_detach_group(struct iommu_domain *domain, struct iommu_group *group) 1240void iommu_detach_group(struct iommu_domain *domain, struct iommu_group *group)
982{ 1241{
983 iommu_group_for_each_dev(group, domain, iommu_group_do_detach_device); 1242 mutex_lock(&group->mutex);
1243 __iommu_detach_group(domain, group);
1244 mutex_unlock(&group->mutex);
984} 1245}
985EXPORT_SYMBOL_GPL(iommu_detach_group); 1246EXPORT_SYMBOL_GPL(iommu_detach_group);
986 1247
@@ -1207,7 +1468,7 @@ static int __init iommu_init(void)
1207 1468
1208 return 0; 1469 return 0;
1209} 1470}
1210arch_initcall(iommu_init); 1471core_initcall(iommu_init);
1211 1472
1212int iommu_domain_get_attr(struct iommu_domain *domain, 1473int iommu_domain_get_attr(struct iommu_domain *domain,
1213 enum iommu_attr attr, void *data) 1474 enum iommu_attr attr, void *data)
@@ -1273,3 +1534,72 @@ int iommu_domain_set_attr(struct iommu_domain *domain,
1273 return ret; 1534 return ret;
1274} 1535}
1275EXPORT_SYMBOL_GPL(iommu_domain_set_attr); 1536EXPORT_SYMBOL_GPL(iommu_domain_set_attr);
1537
1538void iommu_get_dm_regions(struct device *dev, struct list_head *list)
1539{
1540 const struct iommu_ops *ops = dev->bus->iommu_ops;
1541
1542 if (ops && ops->get_dm_regions)
1543 ops->get_dm_regions(dev, list);
1544}
1545
1546void iommu_put_dm_regions(struct device *dev, struct list_head *list)
1547{
1548 const struct iommu_ops *ops = dev->bus->iommu_ops;
1549
1550 if (ops && ops->put_dm_regions)
1551 ops->put_dm_regions(dev, list);
1552}
1553
1554/* Request that a device is direct mapped by the IOMMU */
1555int iommu_request_dm_for_dev(struct device *dev)
1556{
1557 struct iommu_domain *dm_domain;
1558 struct iommu_group *group;
1559 int ret;
1560
1561 /* Device must already be in a group before calling this function */
1562 group = iommu_group_get_for_dev(dev);
1563 if (IS_ERR(group))
1564 return PTR_ERR(group);
1565
1566 mutex_lock(&group->mutex);
1567
1568 /* Check if the default domain is already direct mapped */
1569 ret = 0;
1570 if (group->default_domain &&
1571 group->default_domain->type == IOMMU_DOMAIN_IDENTITY)
1572 goto out;
1573
1574 /* Don't change mappings of existing devices */
1575 ret = -EBUSY;
1576 if (iommu_group_device_count(group) != 1)
1577 goto out;
1578
1579 /* Allocate a direct mapped domain */
1580 ret = -ENOMEM;
1581 dm_domain = __iommu_domain_alloc(dev->bus, IOMMU_DOMAIN_IDENTITY);
1582 if (!dm_domain)
1583 goto out;
1584
1585 /* Attach the device to the domain */
1586 ret = __iommu_attach_group(dm_domain, group);
1587 if (ret) {
1588 iommu_domain_free(dm_domain);
1589 goto out;
1590 }
1591
1592 /* Make the direct mapped domain the default for this group */
1593 if (group->default_domain)
1594 iommu_domain_free(group->default_domain);
1595 group->default_domain = dm_domain;
1596
1597 pr_info("Using direct mapping for device %s\n", dev_name(dev));
1598
1599 ret = 0;
1600out:
1601 mutex_unlock(&group->mutex);
1602 iommu_group_put(group);
1603
1604 return ret;
1605}
diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c
index 9dd8208312c2..b7c3d923f3e1 100644
--- a/drivers/iommu/iova.c
+++ b/drivers/iommu/iova.c
@@ -227,6 +227,7 @@ iova_insert_rbtree(struct rb_root *root, struct iova *iova)
227 /* Figure out where to put new node */ 227 /* Figure out where to put new node */
228 while (*new) { 228 while (*new) {
229 struct iova *this = container_of(*new, struct iova, node); 229 struct iova *this = container_of(*new, struct iova, node);
230
230 parent = *new; 231 parent = *new;
231 232
232 if (iova->pfn_lo < this->pfn_lo) 233 if (iova->pfn_lo < this->pfn_lo)
@@ -350,6 +351,7 @@ void
350free_iova(struct iova_domain *iovad, unsigned long pfn) 351free_iova(struct iova_domain *iovad, unsigned long pfn)
351{ 352{
352 struct iova *iova = find_iova(iovad, pfn); 353 struct iova *iova = find_iova(iovad, pfn);
354
353 if (iova) 355 if (iova)
354 __free_iova(iovad, iova); 356 __free_iova(iovad, iova);
355 357
@@ -369,6 +371,7 @@ void put_iova_domain(struct iova_domain *iovad)
369 node = rb_first(&iovad->rbroot); 371 node = rb_first(&iovad->rbroot);
370 while (node) { 372 while (node) {
371 struct iova *iova = container_of(node, struct iova, node); 373 struct iova *iova = container_of(node, struct iova, node);
374
372 rb_erase(node, &iovad->rbroot); 375 rb_erase(node, &iovad->rbroot);
373 free_iova_mem(iova); 376 free_iova_mem(iova);
374 node = rb_first(&iovad->rbroot); 377 node = rb_first(&iovad->rbroot);
@@ -482,6 +485,7 @@ copy_reserved_iova(struct iova_domain *from, struct iova_domain *to)
482 for (node = rb_first(&from->rbroot); node; node = rb_next(node)) { 485 for (node = rb_first(&from->rbroot); node; node = rb_next(node)) {
483 struct iova *iova = container_of(node, struct iova, node); 486 struct iova *iova = container_of(node, struct iova, node);
484 struct iova *new_iova; 487 struct iova *new_iova;
488
485 new_iova = reserve_iova(to, iova->pfn_lo, iova->pfn_hi); 489 new_iova = reserve_iova(to, iova->pfn_lo, iova->pfn_hi);
486 if (!new_iova) 490 if (!new_iova)
487 printk(KERN_ERR "Reserve iova range %lx@%lx failed\n", 491 printk(KERN_ERR "Reserve iova range %lx@%lx failed\n",
diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c
index cab214544237..ebf0adb8e7ea 100644
--- a/drivers/iommu/rockchip-iommu.c
+++ b/drivers/iommu/rockchip-iommu.c
@@ -551,6 +551,15 @@ static void rk_iommu_zap_iova(struct rk_iommu_domain *rk_domain,
551 spin_unlock_irqrestore(&rk_domain->iommus_lock, flags); 551 spin_unlock_irqrestore(&rk_domain->iommus_lock, flags);
552} 552}
553 553
554static void rk_iommu_zap_iova_first_last(struct rk_iommu_domain *rk_domain,
555 dma_addr_t iova, size_t size)
556{
557 rk_iommu_zap_iova(rk_domain, iova, SPAGE_SIZE);
558 if (size > SPAGE_SIZE)
559 rk_iommu_zap_iova(rk_domain, iova + size - SPAGE_SIZE,
560 SPAGE_SIZE);
561}
562
554static u32 *rk_dte_get_page_table(struct rk_iommu_domain *rk_domain, 563static u32 *rk_dte_get_page_table(struct rk_iommu_domain *rk_domain,
555 dma_addr_t iova) 564 dma_addr_t iova)
556{ 565{
@@ -575,12 +584,6 @@ static u32 *rk_dte_get_page_table(struct rk_iommu_domain *rk_domain,
575 rk_table_flush(page_table, NUM_PT_ENTRIES); 584 rk_table_flush(page_table, NUM_PT_ENTRIES);
576 rk_table_flush(dte_addr, 1); 585 rk_table_flush(dte_addr, 1);
577 586
578 /*
579 * Zap the first iova of newly allocated page table so iommu evicts
580 * old cached value of new dte from the iotlb.
581 */
582 rk_iommu_zap_iova(rk_domain, iova, SPAGE_SIZE);
583
584done: 587done:
585 pt_phys = rk_dte_pt_address(dte); 588 pt_phys = rk_dte_pt_address(dte);
586 return (u32 *)phys_to_virt(pt_phys); 589 return (u32 *)phys_to_virt(pt_phys);
@@ -630,6 +633,14 @@ static int rk_iommu_map_iova(struct rk_iommu_domain *rk_domain, u32 *pte_addr,
630 633
631 rk_table_flush(pte_addr, pte_count); 634 rk_table_flush(pte_addr, pte_count);
632 635
636 /*
637 * Zap the first and last iova to evict from iotlb any previously
638 * mapped cachelines holding stale values for its dte and pte.
639 * We only zap the first and last iova, since only they could have
640 * dte or pte shared with an existing mapping.
641 */
642 rk_iommu_zap_iova_first_last(rk_domain, iova, size);
643
633 return 0; 644 return 0;
634unwind: 645unwind:
635 /* Unmap the range of iovas that we just mapped */ 646 /* Unmap the range of iovas that we just mapped */
@@ -774,7 +785,7 @@ static int rk_iommu_attach_device(struct iommu_domain *domain,
774 list_add_tail(&iommu->node, &rk_domain->iommus); 785 list_add_tail(&iommu->node, &rk_domain->iommus);
775 spin_unlock_irqrestore(&rk_domain->iommus_lock, flags); 786 spin_unlock_irqrestore(&rk_domain->iommus_lock, flags);
776 787
777 dev_info(dev, "Attached to iommu domain\n"); 788 dev_dbg(dev, "Attached to iommu domain\n");
778 789
779 rk_iommu_disable_stall(iommu); 790 rk_iommu_disable_stall(iommu);
780 791
@@ -808,7 +819,7 @@ static void rk_iommu_detach_device(struct iommu_domain *domain,
808 819
809 iommu->domain = NULL; 820 iommu->domain = NULL;
810 821
811 dev_info(dev, "Detached from iommu domain\n"); 822 dev_dbg(dev, "Detached from iommu domain\n");
812} 823}
813 824
814static struct iommu_domain *rk_iommu_domain_alloc(unsigned type) 825static struct iommu_domain *rk_iommu_domain_alloc(unsigned type)
diff --git a/drivers/iommu/tegra-smmu.c b/drivers/iommu/tegra-smmu.c
index c845d99ecf6b..c1f2e521dc52 100644
--- a/drivers/iommu/tegra-smmu.c
+++ b/drivers/iommu/tegra-smmu.c
@@ -7,6 +7,7 @@
7 */ 7 */
8 8
9#include <linux/bitops.h> 9#include <linux/bitops.h>
10#include <linux/debugfs.h>
10#include <linux/err.h> 11#include <linux/err.h>
11#include <linux/iommu.h> 12#include <linux/iommu.h>
12#include <linux/kernel.h> 13#include <linux/kernel.h>
@@ -31,6 +32,8 @@ struct tegra_smmu {
31 struct mutex lock; 32 struct mutex lock;
32 33
33 struct list_head list; 34 struct list_head list;
35
36 struct dentry *debugfs;
34}; 37};
35 38
36struct tegra_smmu_as { 39struct tegra_smmu_as {
@@ -673,6 +676,103 @@ static void tegra_smmu_ahb_enable(void)
673 } 676 }
674} 677}
675 678
679static int tegra_smmu_swgroups_show(struct seq_file *s, void *data)
680{
681 struct tegra_smmu *smmu = s->private;
682 unsigned int i;
683 u32 value;
684
685 seq_printf(s, "swgroup enabled ASID\n");
686 seq_printf(s, "------------------------\n");
687
688 for (i = 0; i < smmu->soc->num_swgroups; i++) {
689 const struct tegra_smmu_swgroup *group = &smmu->soc->swgroups[i];
690 const char *status;
691 unsigned int asid;
692
693 value = smmu_readl(smmu, group->reg);
694
695 if (value & SMMU_ASID_ENABLE)
696 status = "yes";
697 else
698 status = "no";
699
700 asid = value & SMMU_ASID_MASK;
701
702 seq_printf(s, "%-9s %-7s %#04x\n", group->name, status,
703 asid);
704 }
705
706 return 0;
707}
708
709static int tegra_smmu_swgroups_open(struct inode *inode, struct file *file)
710{
711 return single_open(file, tegra_smmu_swgroups_show, inode->i_private);
712}
713
714static const struct file_operations tegra_smmu_swgroups_fops = {
715 .open = tegra_smmu_swgroups_open,
716 .read = seq_read,
717 .llseek = seq_lseek,
718 .release = single_release,
719};
720
721static int tegra_smmu_clients_show(struct seq_file *s, void *data)
722{
723 struct tegra_smmu *smmu = s->private;
724 unsigned int i;
725 u32 value;
726
727 seq_printf(s, "client enabled\n");
728 seq_printf(s, "--------------------\n");
729
730 for (i = 0; i < smmu->soc->num_clients; i++) {
731 const struct tegra_mc_client *client = &smmu->soc->clients[i];
732 const char *status;
733
734 value = smmu_readl(smmu, client->smmu.reg);
735
736 if (value & BIT(client->smmu.bit))
737 status = "yes";
738 else
739 status = "no";
740
741 seq_printf(s, "%-12s %s\n", client->name, status);
742 }
743
744 return 0;
745}
746
747static int tegra_smmu_clients_open(struct inode *inode, struct file *file)
748{
749 return single_open(file, tegra_smmu_clients_show, inode->i_private);
750}
751
752static const struct file_operations tegra_smmu_clients_fops = {
753 .open = tegra_smmu_clients_open,
754 .read = seq_read,
755 .llseek = seq_lseek,
756 .release = single_release,
757};
758
759static void tegra_smmu_debugfs_init(struct tegra_smmu *smmu)
760{
761 smmu->debugfs = debugfs_create_dir("smmu", NULL);
762 if (!smmu->debugfs)
763 return;
764
765 debugfs_create_file("swgroups", S_IRUGO, smmu->debugfs, smmu,
766 &tegra_smmu_swgroups_fops);
767 debugfs_create_file("clients", S_IRUGO, smmu->debugfs, smmu,
768 &tegra_smmu_clients_fops);
769}
770
771static void tegra_smmu_debugfs_exit(struct tegra_smmu *smmu)
772{
773 debugfs_remove_recursive(smmu->debugfs);
774}
775
676struct tegra_smmu *tegra_smmu_probe(struct device *dev, 776struct tegra_smmu *tegra_smmu_probe(struct device *dev,
677 const struct tegra_smmu_soc *soc, 777 const struct tegra_smmu_soc *soc,
678 struct tegra_mc *mc) 778 struct tegra_mc *mc)
@@ -743,5 +843,14 @@ struct tegra_smmu *tegra_smmu_probe(struct device *dev,
743 if (err < 0) 843 if (err < 0)
744 return ERR_PTR(err); 844 return ERR_PTR(err);
745 845
846 if (IS_ENABLED(CONFIG_DEBUG_FS))
847 tegra_smmu_debugfs_init(smmu);
848
746 return smmu; 849 return smmu;
747} 850}
851
852void tegra_smmu_remove(struct tegra_smmu *smmu)
853{
854 if (IS_ENABLED(CONFIG_DEBUG_FS))
855 tegra_smmu_debugfs_exit(smmu);
856}
generated by cgit v1.2.2 (git 2.25.0) at 2026-01-09 00:48:49 -0500