aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/iommu
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/iommu')
-rw-r--r--drivers/iommu/amd_iommu.c125
-rw-r--r--drivers/iommu/amd_iommu_init.c21
-rw-r--r--drivers/iommu/amd_iommu_types.h21
-rw-r--r--drivers/iommu/arm-smmu.c331
-rw-r--r--drivers/iommu/dmar.c3
-rw-r--r--drivers/iommu/exynos-iommu.c51
-rw-r--r--drivers/iommu/fsl_pamu_domain.c15
-rw-r--r--drivers/iommu/intel-iommu.c21
-rw-r--r--drivers/iommu/intel_irq_remapping.c4
-rw-r--r--drivers/iommu/iommu.c218
-rw-r--r--drivers/iommu/irq_remapping.c11
-rw-r--r--drivers/iommu/irq_remapping.h2
-rw-r--r--drivers/iommu/msm_iommu.c7
-rw-r--r--drivers/iommu/omap-iommu.c27
-rw-r--r--drivers/iommu/omap-iommu.h1
-rw-r--r--drivers/iommu/tegra-gart.c9
-rw-r--r--drivers/iommu/tegra-smmu.c9
17 files changed, 497 insertions, 379 deletions
diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c
index 18405314168b..505a9adac2d5 100644
--- a/drivers/iommu/amd_iommu.c
+++ b/drivers/iommu/amd_iommu.c
@@ -88,6 +88,27 @@ int amd_iommu_max_glx_val = -1;
88static struct dma_map_ops amd_iommu_dma_ops; 88static struct dma_map_ops amd_iommu_dma_ops;
89 89
90/* 90/*
91 * This struct contains device specific data for the IOMMU
92 */
93struct iommu_dev_data {
94 struct list_head list; /* For domain->dev_list */
95 struct list_head dev_data_list; /* For global dev_data_list */
96 struct list_head alias_list; /* Link alias-groups together */
97 struct iommu_dev_data *alias_data;/* The alias dev_data */
98 struct protection_domain *domain; /* Domain the device is bound to */
99 u16 devid; /* PCI Device ID */
100 bool iommu_v2; /* Device can make use of IOMMUv2 */
101 bool passthrough; /* Default for device is pt_domain */
102 struct {
103 bool enabled;
104 int qdep;
105 } ats; /* ATS state */
106 bool pri_tlp; /* PASID TLB required for
107 PPR completions */
108 u32 errata; /* Bitmap for errata to apply */
109};
110
111/*
91 * general struct to manage commands send to an IOMMU 112 * general struct to manage commands send to an IOMMU
92 */ 113 */
93struct iommu_cmd { 114struct iommu_cmd {
@@ -114,8 +135,9 @@ static struct iommu_dev_data *alloc_dev_data(u16 devid)
114 if (!dev_data) 135 if (!dev_data)
115 return NULL; 136 return NULL;
116 137
138 INIT_LIST_HEAD(&dev_data->alias_list);
139
117 dev_data->devid = devid; 140 dev_data->devid = devid;
118 atomic_set(&dev_data->bind, 0);
119 141
120 spin_lock_irqsave(&dev_data_list_lock, flags); 142 spin_lock_irqsave(&dev_data_list_lock, flags);
121 list_add_tail(&dev_data->dev_data_list, &dev_data_list); 143 list_add_tail(&dev_data->dev_data_list, &dev_data_list);
@@ -260,17 +282,13 @@ static bool check_device(struct device *dev)
260 return true; 282 return true;
261} 283}
262 284
263static int init_iommu_group(struct device *dev) 285static void init_iommu_group(struct device *dev)
264{ 286{
265 struct iommu_group *group; 287 struct iommu_group *group;
266 288
267 group = iommu_group_get_for_dev(dev); 289 group = iommu_group_get_for_dev(dev);
268 290 if (!IS_ERR(group))
269 if (IS_ERR(group)) 291 iommu_group_put(group);
270 return PTR_ERR(group);
271
272 iommu_group_put(group);
273 return 0;
274} 292}
275 293
276static int __last_alias(struct pci_dev *pdev, u16 alias, void *data) 294static int __last_alias(struct pci_dev *pdev, u16 alias, void *data)
@@ -340,7 +358,6 @@ static int iommu_init_device(struct device *dev)
340 struct pci_dev *pdev = to_pci_dev(dev); 358 struct pci_dev *pdev = to_pci_dev(dev);
341 struct iommu_dev_data *dev_data; 359 struct iommu_dev_data *dev_data;
342 u16 alias; 360 u16 alias;
343 int ret;
344 361
345 if (dev->archdata.iommu) 362 if (dev->archdata.iommu)
346 return 0; 363 return 0;
@@ -362,12 +379,9 @@ static int iommu_init_device(struct device *dev)
362 return -ENOTSUPP; 379 return -ENOTSUPP;
363 } 380 }
364 dev_data->alias_data = alias_data; 381 dev_data->alias_data = alias_data;
365 }
366 382
367 ret = init_iommu_group(dev); 383 /* Add device to the alias_list */
368 if (ret) { 384 list_add(&dev_data->alias_list, &alias_data->alias_list);
369 free_dev_data(dev_data);
370 return ret;
371 } 385 }
372 386
373 if (pci_iommuv2_capable(pdev)) { 387 if (pci_iommuv2_capable(pdev)) {
@@ -455,6 +469,15 @@ int __init amd_iommu_init_devices(void)
455 goto out_free; 469 goto out_free;
456 } 470 }
457 471
472 /*
473 * Initialize IOMMU groups only after iommu_init_device() has
474 * had a chance to populate any IVRS defined aliases.
475 */
476 for_each_pci_dev(pdev) {
477 if (check_device(&pdev->dev))
478 init_iommu_group(&pdev->dev);
479 }
480
458 return 0; 481 return 0;
459 482
460out_free: 483out_free:
@@ -1368,6 +1391,9 @@ static int iommu_map_page(struct protection_domain *dom,
1368 count = PAGE_SIZE_PTE_COUNT(page_size); 1391 count = PAGE_SIZE_PTE_COUNT(page_size);
1369 pte = alloc_pte(dom, bus_addr, page_size, NULL, GFP_KERNEL); 1392 pte = alloc_pte(dom, bus_addr, page_size, NULL, GFP_KERNEL);
1370 1393
1394 if (!pte)
1395 return -ENOMEM;
1396
1371 for (i = 0; i < count; ++i) 1397 for (i = 0; i < count; ++i)
1372 if (IOMMU_PTE_PRESENT(pte[i])) 1398 if (IOMMU_PTE_PRESENT(pte[i]))
1373 return -EBUSY; 1399 return -EBUSY;
@@ -2122,35 +2148,29 @@ static void do_detach(struct iommu_dev_data *dev_data)
2122static int __attach_device(struct iommu_dev_data *dev_data, 2148static int __attach_device(struct iommu_dev_data *dev_data,
2123 struct protection_domain *domain) 2149 struct protection_domain *domain)
2124{ 2150{
2151 struct iommu_dev_data *head, *entry;
2125 int ret; 2152 int ret;
2126 2153
2127 /* lock domain */ 2154 /* lock domain */
2128 spin_lock(&domain->lock); 2155 spin_lock(&domain->lock);
2129 2156
2130 if (dev_data->alias_data != NULL) { 2157 head = dev_data;
2131 struct iommu_dev_data *alias_data = dev_data->alias_data;
2132 2158
2133 /* Some sanity checks */ 2159 if (head->alias_data != NULL)
2134 ret = -EBUSY; 2160 head = head->alias_data;
2135 if (alias_data->domain != NULL &&
2136 alias_data->domain != domain)
2137 goto out_unlock;
2138 2161
2139 if (dev_data->domain != NULL && 2162 /* Now we have the root of the alias group, if any */
2140 dev_data->domain != domain)
2141 goto out_unlock;
2142 2163
2143 /* Do real assignment */ 2164 ret = -EBUSY;
2144 if (alias_data->domain == NULL) 2165 if (head->domain != NULL)
2145 do_attach(alias_data, domain); 2166 goto out_unlock;
2146
2147 atomic_inc(&alias_data->bind);
2148 }
2149 2167
2150 if (dev_data->domain == NULL) 2168 /* Attach alias group root */
2151 do_attach(dev_data, domain); 2169 do_attach(head, domain);
2152 2170
2153 atomic_inc(&dev_data->bind); 2171 /* Attach other devices in the alias group */
2172 list_for_each_entry(entry, &head->alias_list, alias_list)
2173 do_attach(entry, domain);
2154 2174
2155 ret = 0; 2175 ret = 0;
2156 2176
@@ -2298,6 +2318,7 @@ static int attach_device(struct device *dev,
2298 */ 2318 */
2299static void __detach_device(struct iommu_dev_data *dev_data) 2319static void __detach_device(struct iommu_dev_data *dev_data)
2300{ 2320{
2321 struct iommu_dev_data *head, *entry;
2301 struct protection_domain *domain; 2322 struct protection_domain *domain;
2302 unsigned long flags; 2323 unsigned long flags;
2303 2324
@@ -2307,15 +2328,14 @@ static void __detach_device(struct iommu_dev_data *dev_data)
2307 2328
2308 spin_lock_irqsave(&domain->lock, flags); 2329 spin_lock_irqsave(&domain->lock, flags);
2309 2330
2310 if (dev_data->alias_data != NULL) { 2331 head = dev_data;
2311 struct iommu_dev_data *alias_data = dev_data->alias_data; 2332 if (head->alias_data != NULL)
2333 head = head->alias_data;
2312 2334
2313 if (atomic_dec_and_test(&alias_data->bind)) 2335 list_for_each_entry(entry, &head->alias_list, alias_list)
2314 do_detach(alias_data); 2336 do_detach(entry);
2315 }
2316 2337
2317 if (atomic_dec_and_test(&dev_data->bind)) 2338 do_detach(head);
2318 do_detach(dev_data);
2319 2339
2320 spin_unlock_irqrestore(&domain->lock, flags); 2340 spin_unlock_irqrestore(&domain->lock, flags);
2321 2341
@@ -2415,6 +2435,7 @@ static int device_change_notifier(struct notifier_block *nb,
2415 case BUS_NOTIFY_ADD_DEVICE: 2435 case BUS_NOTIFY_ADD_DEVICE:
2416 2436
2417 iommu_init_device(dev); 2437 iommu_init_device(dev);
2438 init_iommu_group(dev);
2418 2439
2419 /* 2440 /*
2420 * dev_data is still NULL and 2441 * dev_data is still NULL and
@@ -3149,14 +3170,15 @@ free_domains:
3149 3170
3150static void cleanup_domain(struct protection_domain *domain) 3171static void cleanup_domain(struct protection_domain *domain)
3151{ 3172{
3152 struct iommu_dev_data *dev_data, *next; 3173 struct iommu_dev_data *entry;
3153 unsigned long flags; 3174 unsigned long flags;
3154 3175
3155 write_lock_irqsave(&amd_iommu_devtable_lock, flags); 3176 write_lock_irqsave(&amd_iommu_devtable_lock, flags);
3156 3177
3157 list_for_each_entry_safe(dev_data, next, &domain->dev_list, list) { 3178 while (!list_empty(&domain->dev_list)) {
3158 __detach_device(dev_data); 3179 entry = list_first_entry(&domain->dev_list,
3159 atomic_set(&dev_data->bind, 0); 3180 struct iommu_dev_data, list);
3181 __detach_device(entry);
3160 } 3182 }
3161 3183
3162 write_unlock_irqrestore(&amd_iommu_devtable_lock, flags); 3184 write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
@@ -3382,20 +3404,20 @@ static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom,
3382 return paddr; 3404 return paddr;
3383} 3405}
3384 3406
3385static int amd_iommu_domain_has_cap(struct iommu_domain *domain, 3407static bool amd_iommu_capable(enum iommu_cap cap)
3386 unsigned long cap)
3387{ 3408{
3388 switch (cap) { 3409 switch (cap) {
3389 case IOMMU_CAP_CACHE_COHERENCY: 3410 case IOMMU_CAP_CACHE_COHERENCY:
3390 return 1; 3411 return true;
3391 case IOMMU_CAP_INTR_REMAP: 3412 case IOMMU_CAP_INTR_REMAP:
3392 return irq_remapping_enabled; 3413 return (irq_remapping_enabled == 1);
3393 } 3414 }
3394 3415
3395 return 0; 3416 return false;
3396} 3417}
3397 3418
3398static const struct iommu_ops amd_iommu_ops = { 3419static const struct iommu_ops amd_iommu_ops = {
3420 .capable = amd_iommu_capable,
3399 .domain_init = amd_iommu_domain_init, 3421 .domain_init = amd_iommu_domain_init,
3400 .domain_destroy = amd_iommu_domain_destroy, 3422 .domain_destroy = amd_iommu_domain_destroy,
3401 .attach_dev = amd_iommu_attach_device, 3423 .attach_dev = amd_iommu_attach_device,
@@ -3403,7 +3425,6 @@ static const struct iommu_ops amd_iommu_ops = {
3403 .map = amd_iommu_map, 3425 .map = amd_iommu_map,
3404 .unmap = amd_iommu_unmap, 3426 .unmap = amd_iommu_unmap,
3405 .iova_to_phys = amd_iommu_iova_to_phys, 3427 .iova_to_phys = amd_iommu_iova_to_phys,
3406 .domain_has_cap = amd_iommu_domain_has_cap,
3407 .pgsize_bitmap = AMD_IOMMU_PGSIZES, 3428 .pgsize_bitmap = AMD_IOMMU_PGSIZES,
3408}; 3429};
3409 3430
@@ -4233,7 +4254,7 @@ static int msi_setup_irq(struct pci_dev *pdev, unsigned int irq,
4233 return 0; 4254 return 0;
4234} 4255}
4235 4256
4236static int setup_hpet_msi(unsigned int irq, unsigned int id) 4257static int alloc_hpet_msi(unsigned int irq, unsigned int id)
4237{ 4258{
4238 struct irq_2_irte *irte_info; 4259 struct irq_2_irte *irte_info;
4239 struct irq_cfg *cfg; 4260 struct irq_cfg *cfg;
@@ -4272,6 +4293,6 @@ struct irq_remap_ops amd_iommu_irq_ops = {
4272 .compose_msi_msg = compose_msi_msg, 4293 .compose_msi_msg = compose_msi_msg,
4273 .msi_alloc_irq = msi_alloc_irq, 4294 .msi_alloc_irq = msi_alloc_irq,
4274 .msi_setup_irq = msi_setup_irq, 4295 .msi_setup_irq = msi_setup_irq,
4275 .setup_hpet_msi = setup_hpet_msi, 4296 .alloc_hpet_msi = alloc_hpet_msi,
4276}; 4297};
4277#endif 4298#endif
diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c
index 3783e0b44df6..b0522f15730f 100644
--- a/drivers/iommu/amd_iommu_init.c
+++ b/drivers/iommu/amd_iommu_init.c
@@ -712,7 +712,7 @@ static void __init set_dev_entry_from_acpi(struct amd_iommu *iommu,
712 set_iommu_for_device(iommu, devid); 712 set_iommu_for_device(iommu, devid);
713} 713}
714 714
715static int __init add_special_device(u8 type, u8 id, u16 devid, bool cmd_line) 715static int __init add_special_device(u8 type, u8 id, u16 *devid, bool cmd_line)
716{ 716{
717 struct devid_map *entry; 717 struct devid_map *entry;
718 struct list_head *list; 718 struct list_head *list;
@@ -731,6 +731,8 @@ static int __init add_special_device(u8 type, u8 id, u16 devid, bool cmd_line)
731 pr_info("AMD-Vi: Command-line override present for %s id %d - ignoring\n", 731 pr_info("AMD-Vi: Command-line override present for %s id %d - ignoring\n",
732 type == IVHD_SPECIAL_IOAPIC ? "IOAPIC" : "HPET", id); 732 type == IVHD_SPECIAL_IOAPIC ? "IOAPIC" : "HPET", id);
733 733
734 *devid = entry->devid;
735
734 return 0; 736 return 0;
735 } 737 }
736 738
@@ -739,7 +741,7 @@ static int __init add_special_device(u8 type, u8 id, u16 devid, bool cmd_line)
739 return -ENOMEM; 741 return -ENOMEM;
740 742
741 entry->id = id; 743 entry->id = id;
742 entry->devid = devid; 744 entry->devid = *devid;
743 entry->cmd_line = cmd_line; 745 entry->cmd_line = cmd_line;
744 746
745 list_add_tail(&entry->list, list); 747 list_add_tail(&entry->list, list);
@@ -754,7 +756,7 @@ static int __init add_early_maps(void)
754 for (i = 0; i < early_ioapic_map_size; ++i) { 756 for (i = 0; i < early_ioapic_map_size; ++i) {
755 ret = add_special_device(IVHD_SPECIAL_IOAPIC, 757 ret = add_special_device(IVHD_SPECIAL_IOAPIC,
756 early_ioapic_map[i].id, 758 early_ioapic_map[i].id,
757 early_ioapic_map[i].devid, 759 &early_ioapic_map[i].devid,
758 early_ioapic_map[i].cmd_line); 760 early_ioapic_map[i].cmd_line);
759 if (ret) 761 if (ret)
760 return ret; 762 return ret;
@@ -763,7 +765,7 @@ static int __init add_early_maps(void)
763 for (i = 0; i < early_hpet_map_size; ++i) { 765 for (i = 0; i < early_hpet_map_size; ++i) {
764 ret = add_special_device(IVHD_SPECIAL_HPET, 766 ret = add_special_device(IVHD_SPECIAL_HPET,
765 early_hpet_map[i].id, 767 early_hpet_map[i].id,
766 early_hpet_map[i].devid, 768 &early_hpet_map[i].devid,
767 early_hpet_map[i].cmd_line); 769 early_hpet_map[i].cmd_line);
768 if (ret) 770 if (ret)
769 return ret; 771 return ret;
@@ -978,10 +980,17 @@ static int __init init_iommu_from_acpi(struct amd_iommu *iommu,
978 PCI_SLOT(devid), 980 PCI_SLOT(devid),
979 PCI_FUNC(devid)); 981 PCI_FUNC(devid));
980 982
981 set_dev_entry_from_acpi(iommu, devid, e->flags, 0); 983 ret = add_special_device(type, handle, &devid, false);
982 ret = add_special_device(type, handle, devid, false);
983 if (ret) 984 if (ret)
984 return ret; 985 return ret;
986
987 /*
988 * add_special_device might update the devid in case a
989 * command-line override is present. So call
990 * set_dev_entry_from_acpi after add_special_device.
991 */
992 set_dev_entry_from_acpi(iommu, devid, e->flags, 0);
993
985 break; 994 break;
986 } 995 }
987 default: 996 default:
diff --git a/drivers/iommu/amd_iommu_types.h b/drivers/iommu/amd_iommu_types.h
index 8e43b7cba133..cec51a8ba844 100644
--- a/drivers/iommu/amd_iommu_types.h
+++ b/drivers/iommu/amd_iommu_types.h
@@ -418,27 +418,6 @@ struct protection_domain {
418}; 418};
419 419
420/* 420/*
421 * This struct contains device specific data for the IOMMU
422 */
423struct iommu_dev_data {
424 struct list_head list; /* For domain->dev_list */
425 struct list_head dev_data_list; /* For global dev_data_list */
426 struct iommu_dev_data *alias_data;/* The alias dev_data */
427 struct protection_domain *domain; /* Domain the device is bound to */
428 atomic_t bind; /* Domain attach reference count */
429 u16 devid; /* PCI Device ID */
430 bool iommu_v2; /* Device can make use of IOMMUv2 */
431 bool passthrough; /* Default for device is pt_domain */
432 struct {
433 bool enabled;
434 int qdep;
435 } ats; /* ATS state */
436 bool pri_tlp; /* PASID TLB required for
437 PPR completions */
438 u32 errata; /* Bitmap for errata to apply */
439};
440
441/*
442 * For dynamic growth the aperture size is split into ranges of 128MB of 421 * For dynamic growth the aperture size is split into ranges of 128MB of
443 * DMA address space each. This struct represents one such range. 422 * DMA address space each. This struct represents one such range.
444 */ 423 */
diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c
index ca18d6d42a9b..60558f794922 100644
--- a/drivers/iommu/arm-smmu.c
+++ b/drivers/iommu/arm-smmu.c
@@ -24,7 +24,7 @@
24 * - v7/v8 long-descriptor format 24 * - v7/v8 long-descriptor format
25 * - Non-secure access to the SMMU 25 * - Non-secure access to the SMMU
26 * - 4k and 64k pages, with contiguous pte hints. 26 * - 4k and 64k pages, with contiguous pte hints.
27 * - Up to 42-bit addressing (dependent on VA_BITS) 27 * - Up to 48-bit addressing (dependent on VA_BITS)
28 * - Context fault reporting 28 * - Context fault reporting
29 */ 29 */
30 30
@@ -59,7 +59,7 @@
59 59
60/* SMMU global address space */ 60/* SMMU global address space */
61#define ARM_SMMU_GR0(smmu) ((smmu)->base) 61#define ARM_SMMU_GR0(smmu) ((smmu)->base)
62#define ARM_SMMU_GR1(smmu) ((smmu)->base + (smmu)->pagesize) 62#define ARM_SMMU_GR1(smmu) ((smmu)->base + (1 << (smmu)->pgshift))
63 63
64/* 64/*
65 * SMMU global address space with conditional offset to access secure 65 * SMMU global address space with conditional offset to access secure
@@ -146,6 +146,8 @@
146#define ID0_CTTW (1 << 14) 146#define ID0_CTTW (1 << 14)
147#define ID0_NUMIRPT_SHIFT 16 147#define ID0_NUMIRPT_SHIFT 16
148#define ID0_NUMIRPT_MASK 0xff 148#define ID0_NUMIRPT_MASK 0xff
149#define ID0_NUMSIDB_SHIFT 9
150#define ID0_NUMSIDB_MASK 0xf
149#define ID0_NUMSMRG_SHIFT 0 151#define ID0_NUMSMRG_SHIFT 0
150#define ID0_NUMSMRG_MASK 0xff 152#define ID0_NUMSMRG_MASK 0xff
151 153
@@ -222,7 +224,7 @@
222 224
223/* Translation context bank */ 225/* Translation context bank */
224#define ARM_SMMU_CB_BASE(smmu) ((smmu)->base + ((smmu)->size >> 1)) 226#define ARM_SMMU_CB_BASE(smmu) ((smmu)->base + ((smmu)->size >> 1))
225#define ARM_SMMU_CB(smmu, n) ((n) * (smmu)->pagesize) 227#define ARM_SMMU_CB(smmu, n) ((n) * (1 << (smmu)->pgshift))
226 228
227#define ARM_SMMU_CB_SCTLR 0x0 229#define ARM_SMMU_CB_SCTLR 0x0
228#define ARM_SMMU_CB_RESUME 0x8 230#define ARM_SMMU_CB_RESUME 0x8
@@ -324,6 +326,16 @@
324 326
325#define FSYNR0_WNR (1 << 4) 327#define FSYNR0_WNR (1 << 4)
326 328
329static int force_stage;
330module_param_named(force_stage, force_stage, int, S_IRUGO | S_IWUSR);
331MODULE_PARM_DESC(force_stage,
332 "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
333
334enum arm_smmu_arch_version {
335 ARM_SMMU_V1 = 1,
336 ARM_SMMU_V2,
337};
338
327struct arm_smmu_smr { 339struct arm_smmu_smr {
328 u8 idx; 340 u8 idx;
329 u16 mask; 341 u16 mask;
@@ -347,7 +359,7 @@ struct arm_smmu_device {
347 359
348 void __iomem *base; 360 void __iomem *base;
349 unsigned long size; 361 unsigned long size;
350 unsigned long pagesize; 362 unsigned long pgshift;
351 363
352#define ARM_SMMU_FEAT_COHERENT_WALK (1 << 0) 364#define ARM_SMMU_FEAT_COHERENT_WALK (1 << 0)
353#define ARM_SMMU_FEAT_STREAM_MATCH (1 << 1) 365#define ARM_SMMU_FEAT_STREAM_MATCH (1 << 1)
@@ -358,7 +370,7 @@ struct arm_smmu_device {
358 370
359#define ARM_SMMU_OPT_SECURE_CFG_ACCESS (1 << 0) 371#define ARM_SMMU_OPT_SECURE_CFG_ACCESS (1 << 0)
360 u32 options; 372 u32 options;
361 int version; 373 enum arm_smmu_arch_version version;
362 374
363 u32 num_context_banks; 375 u32 num_context_banks;
364 u32 num_s2_context_banks; 376 u32 num_s2_context_banks;
@@ -368,8 +380,9 @@ struct arm_smmu_device {
368 u32 num_mapping_groups; 380 u32 num_mapping_groups;
369 DECLARE_BITMAP(smr_map, ARM_SMMU_MAX_SMRS); 381 DECLARE_BITMAP(smr_map, ARM_SMMU_MAX_SMRS);
370 382
371 unsigned long input_size; 383 unsigned long s1_input_size;
372 unsigned long s1_output_size; 384 unsigned long s1_output_size;
385 unsigned long s2_input_size;
373 unsigned long s2_output_size; 386 unsigned long s2_output_size;
374 387
375 u32 num_global_irqs; 388 u32 num_global_irqs;
@@ -424,17 +437,17 @@ static void parse_driver_options(struct arm_smmu_device *smmu)
424 } while (arm_smmu_options[++i].opt); 437 } while (arm_smmu_options[++i].opt);
425} 438}
426 439
427static struct device *dev_get_master_dev(struct device *dev) 440static struct device_node *dev_get_dev_node(struct device *dev)
428{ 441{
429 if (dev_is_pci(dev)) { 442 if (dev_is_pci(dev)) {
430 struct pci_bus *bus = to_pci_dev(dev)->bus; 443 struct pci_bus *bus = to_pci_dev(dev)->bus;
431 444
432 while (!pci_is_root_bus(bus)) 445 while (!pci_is_root_bus(bus))
433 bus = bus->parent; 446 bus = bus->parent;
434 return bus->bridge->parent; 447 return bus->bridge->parent->of_node;
435 } 448 }
436 449
437 return dev; 450 return dev->of_node;
438} 451}
439 452
440static struct arm_smmu_master *find_smmu_master(struct arm_smmu_device *smmu, 453static struct arm_smmu_master *find_smmu_master(struct arm_smmu_device *smmu,
@@ -459,15 +472,17 @@ static struct arm_smmu_master *find_smmu_master(struct arm_smmu_device *smmu,
459} 472}
460 473
461static struct arm_smmu_master_cfg * 474static struct arm_smmu_master_cfg *
462find_smmu_master_cfg(struct arm_smmu_device *smmu, struct device *dev) 475find_smmu_master_cfg(struct device *dev)
463{ 476{
464 struct arm_smmu_master *master; 477 struct arm_smmu_master_cfg *cfg = NULL;
478 struct iommu_group *group = iommu_group_get(dev);
465 479
466 if (dev_is_pci(dev)) 480 if (group) {
467 return dev->archdata.iommu; 481 cfg = iommu_group_get_iommudata(group);
482 iommu_group_put(group);
483 }
468 484
469 master = find_smmu_master(smmu, dev->of_node); 485 return cfg;
470 return master ? &master->cfg : NULL;
471} 486}
472 487
473static int insert_smmu_master(struct arm_smmu_device *smmu, 488static int insert_smmu_master(struct arm_smmu_device *smmu,
@@ -524,9 +539,18 @@ static int register_smmu_master(struct arm_smmu_device *smmu,
524 master->of_node = masterspec->np; 539 master->of_node = masterspec->np;
525 master->cfg.num_streamids = masterspec->args_count; 540 master->cfg.num_streamids = masterspec->args_count;
526 541
527 for (i = 0; i < master->cfg.num_streamids; ++i) 542 for (i = 0; i < master->cfg.num_streamids; ++i) {
528 master->cfg.streamids[i] = masterspec->args[i]; 543 u16 streamid = masterspec->args[i];
529 544
545 if (!(smmu->features & ARM_SMMU_FEAT_STREAM_MATCH) &&
546 (streamid >= smmu->num_mapping_groups)) {
547 dev_err(dev,
548 "stream ID for master device %s greater than maximum allowed (%d)\n",
549 masterspec->np->name, smmu->num_mapping_groups);
550 return -ERANGE;
551 }
552 master->cfg.streamids[i] = streamid;
553 }
530 return insert_smmu_master(smmu, master); 554 return insert_smmu_master(smmu, master);
531} 555}
532 556
@@ -534,7 +558,7 @@ static struct arm_smmu_device *find_smmu_for_device(struct device *dev)
534{ 558{
535 struct arm_smmu_device *smmu; 559 struct arm_smmu_device *smmu;
536 struct arm_smmu_master *master = NULL; 560 struct arm_smmu_master *master = NULL;
537 struct device_node *dev_node = dev_get_master_dev(dev)->of_node; 561 struct device_node *dev_node = dev_get_dev_node(dev);
538 562
539 spin_lock(&arm_smmu_devices_lock); 563 spin_lock(&arm_smmu_devices_lock);
540 list_for_each_entry(smmu, &arm_smmu_devices, list) { 564 list_for_each_entry(smmu, &arm_smmu_devices, list) {
@@ -623,7 +647,7 @@ static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
623 647
624 if (fsr & FSR_IGN) 648 if (fsr & FSR_IGN)
625 dev_err_ratelimited(smmu->dev, 649 dev_err_ratelimited(smmu->dev,
626 "Unexpected context fault (fsr 0x%u)\n", 650 "Unexpected context fault (fsr 0x%x)\n",
627 fsr); 651 fsr);
628 652
629 fsynr = readl_relaxed(cb_base + ARM_SMMU_CB_FSYNR0); 653 fsynr = readl_relaxed(cb_base + ARM_SMMU_CB_FSYNR0);
@@ -718,7 +742,7 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain)
718 742
719 /* CBAR */ 743 /* CBAR */
720 reg = cfg->cbar; 744 reg = cfg->cbar;
721 if (smmu->version == 1) 745 if (smmu->version == ARM_SMMU_V1)
722 reg |= cfg->irptndx << CBAR_IRPTNDX_SHIFT; 746 reg |= cfg->irptndx << CBAR_IRPTNDX_SHIFT;
723 747
724 /* 748 /*
@@ -733,7 +757,7 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain)
733 } 757 }
734 writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(cfg->cbndx)); 758 writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(cfg->cbndx));
735 759
736 if (smmu->version > 1) { 760 if (smmu->version > ARM_SMMU_V1) {
737 /* CBA2R */ 761 /* CBA2R */
738#ifdef CONFIG_64BIT 762#ifdef CONFIG_64BIT
739 reg = CBA2R_RW64_64BIT; 763 reg = CBA2R_RW64_64BIT;
@@ -744,7 +768,7 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain)
744 gr1_base + ARM_SMMU_GR1_CBA2R(cfg->cbndx)); 768 gr1_base + ARM_SMMU_GR1_CBA2R(cfg->cbndx));
745 769
746 /* TTBCR2 */ 770 /* TTBCR2 */
747 switch (smmu->input_size) { 771 switch (smmu->s1_input_size) {
748 case 32: 772 case 32:
749 reg = (TTBCR2_ADDR_32 << TTBCR2_SEP_SHIFT); 773 reg = (TTBCR2_ADDR_32 << TTBCR2_SEP_SHIFT);
750 break; 774 break;
@@ -752,6 +776,7 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain)
752 reg = (TTBCR2_ADDR_36 << TTBCR2_SEP_SHIFT); 776 reg = (TTBCR2_ADDR_36 << TTBCR2_SEP_SHIFT);
753 break; 777 break;
754 case 39: 778 case 39:
779 case 40:
755 reg = (TTBCR2_ADDR_40 << TTBCR2_SEP_SHIFT); 780 reg = (TTBCR2_ADDR_40 << TTBCR2_SEP_SHIFT);
756 break; 781 break;
757 case 42: 782 case 42:
@@ -773,6 +798,7 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain)
773 reg |= (TTBCR2_ADDR_36 << TTBCR2_PASIZE_SHIFT); 798 reg |= (TTBCR2_ADDR_36 << TTBCR2_PASIZE_SHIFT);
774 break; 799 break;
775 case 39: 800 case 39:
801 case 40:
776 reg |= (TTBCR2_ADDR_40 << TTBCR2_PASIZE_SHIFT); 802 reg |= (TTBCR2_ADDR_40 << TTBCR2_PASIZE_SHIFT);
777 break; 803 break;
778 case 42: 804 case 42:
@@ -804,14 +830,14 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain)
804 * TTBCR 830 * TTBCR
805 * We use long descriptor, with inner-shareable WBWA tables in TTBR0. 831 * We use long descriptor, with inner-shareable WBWA tables in TTBR0.
806 */ 832 */
807 if (smmu->version > 1) { 833 if (smmu->version > ARM_SMMU_V1) {
808 if (PAGE_SIZE == SZ_4K) 834 if (PAGE_SIZE == SZ_4K)
809 reg = TTBCR_TG0_4K; 835 reg = TTBCR_TG0_4K;
810 else 836 else
811 reg = TTBCR_TG0_64K; 837 reg = TTBCR_TG0_64K;
812 838
813 if (!stage1) { 839 if (!stage1) {
814 reg |= (64 - smmu->s1_output_size) << TTBCR_T0SZ_SHIFT; 840 reg |= (64 - smmu->s2_input_size) << TTBCR_T0SZ_SHIFT;
815 841
816 switch (smmu->s2_output_size) { 842 switch (smmu->s2_output_size) {
817 case 32: 843 case 32:
@@ -834,7 +860,7 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain)
834 break; 860 break;
835 } 861 }
836 } else { 862 } else {
837 reg |= (64 - smmu->input_size) << TTBCR_T0SZ_SHIFT; 863 reg |= (64 - smmu->s1_input_size) << TTBCR_T0SZ_SHIFT;
838 } 864 }
839 } else { 865 } else {
840 reg = 0; 866 reg = 0;
@@ -843,8 +869,11 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain)
843 reg |= TTBCR_EAE | 869 reg |= TTBCR_EAE |
844 (TTBCR_SH_IS << TTBCR_SH0_SHIFT) | 870 (TTBCR_SH_IS << TTBCR_SH0_SHIFT) |
845 (TTBCR_RGN_WBWA << TTBCR_ORGN0_SHIFT) | 871 (TTBCR_RGN_WBWA << TTBCR_ORGN0_SHIFT) |
846 (TTBCR_RGN_WBWA << TTBCR_IRGN0_SHIFT) | 872 (TTBCR_RGN_WBWA << TTBCR_IRGN0_SHIFT);
847 (TTBCR_SL0_LVL_1 << TTBCR_SL0_SHIFT); 873
874 if (!stage1)
875 reg |= (TTBCR_SL0_LVL_1 << TTBCR_SL0_SHIFT);
876
848 writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR); 877 writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR);
849 878
850 /* MAIR0 (stage-1 only) */ 879 /* MAIR0 (stage-1 only) */
@@ -868,10 +897,15 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain)
868static int arm_smmu_init_domain_context(struct iommu_domain *domain, 897static int arm_smmu_init_domain_context(struct iommu_domain *domain,
869 struct arm_smmu_device *smmu) 898 struct arm_smmu_device *smmu)
870{ 899{
871 int irq, ret, start; 900 int irq, start, ret = 0;
901 unsigned long flags;
872 struct arm_smmu_domain *smmu_domain = domain->priv; 902 struct arm_smmu_domain *smmu_domain = domain->priv;
873 struct arm_smmu_cfg *cfg = &smmu_domain->cfg; 903 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
874 904
905 spin_lock_irqsave(&smmu_domain->lock, flags);
906 if (smmu_domain->smmu)
907 goto out_unlock;
908
875 if (smmu->features & ARM_SMMU_FEAT_TRANS_NESTED) { 909 if (smmu->features & ARM_SMMU_FEAT_TRANS_NESTED) {
876 /* 910 /*
877 * We will likely want to change this if/when KVM gets 911 * We will likely want to change this if/when KVM gets
@@ -890,16 +924,20 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain,
890 ret = __arm_smmu_alloc_bitmap(smmu->context_map, start, 924 ret = __arm_smmu_alloc_bitmap(smmu->context_map, start,
891 smmu->num_context_banks); 925 smmu->num_context_banks);
892 if (IS_ERR_VALUE(ret)) 926 if (IS_ERR_VALUE(ret))
893 return ret; 927 goto out_unlock;
894 928
895 cfg->cbndx = ret; 929 cfg->cbndx = ret;
896 if (smmu->version == 1) { 930 if (smmu->version == ARM_SMMU_V1) {
897 cfg->irptndx = atomic_inc_return(&smmu->irptndx); 931 cfg->irptndx = atomic_inc_return(&smmu->irptndx);
898 cfg->irptndx %= smmu->num_context_irqs; 932 cfg->irptndx %= smmu->num_context_irqs;
899 } else { 933 } else {
900 cfg->irptndx = cfg->cbndx; 934 cfg->irptndx = cfg->cbndx;
901 } 935 }
902 936
937 ACCESS_ONCE(smmu_domain->smmu) = smmu;
938 arm_smmu_init_context_bank(smmu_domain);
939 spin_unlock_irqrestore(&smmu_domain->lock, flags);
940
903 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx]; 941 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
904 ret = request_irq(irq, arm_smmu_context_fault, IRQF_SHARED, 942 ret = request_irq(irq, arm_smmu_context_fault, IRQF_SHARED,
905 "arm-smmu-context-fault", domain); 943 "arm-smmu-context-fault", domain);
@@ -907,15 +945,12 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain,
907 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n", 945 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
908 cfg->irptndx, irq); 946 cfg->irptndx, irq);
909 cfg->irptndx = INVALID_IRPTNDX; 947 cfg->irptndx = INVALID_IRPTNDX;
910 goto out_free_context;
911 } 948 }
912 949
913 smmu_domain->smmu = smmu;
914 arm_smmu_init_context_bank(smmu_domain);
915 return 0; 950 return 0;
916 951
917out_free_context: 952out_unlock:
918 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx); 953 spin_unlock_irqrestore(&smmu_domain->lock, flags);
919 return ret; 954 return ret;
920} 955}
921 956
@@ -975,7 +1010,6 @@ static void arm_smmu_free_ptes(pmd_t *pmd)
975{ 1010{
976 pgtable_t table = pmd_pgtable(*pmd); 1011 pgtable_t table = pmd_pgtable(*pmd);
977 1012
978 pgtable_page_dtor(table);
979 __free_page(table); 1013 __free_page(table);
980} 1014}
981 1015
@@ -1108,6 +1142,9 @@ static void arm_smmu_master_free_smrs(struct arm_smmu_device *smmu,
1108 void __iomem *gr0_base = ARM_SMMU_GR0(smmu); 1142 void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1109 struct arm_smmu_smr *smrs = cfg->smrs; 1143 struct arm_smmu_smr *smrs = cfg->smrs;
1110 1144
1145 if (!smrs)
1146 return;
1147
1111 /* Invalidate the SMRs before freeing back to the allocator */ 1148 /* Invalidate the SMRs before freeing back to the allocator */
1112 for (i = 0; i < cfg->num_streamids; ++i) { 1149 for (i = 0; i < cfg->num_streamids; ++i) {
1113 u8 idx = smrs[i].idx; 1150 u8 idx = smrs[i].idx;
@@ -1120,20 +1157,6 @@ static void arm_smmu_master_free_smrs(struct arm_smmu_device *smmu,
1120 kfree(smrs); 1157 kfree(smrs);
1121} 1158}
1122 1159
1123static void arm_smmu_bypass_stream_mapping(struct arm_smmu_device *smmu,
1124 struct arm_smmu_master_cfg *cfg)
1125{
1126 int i;
1127 void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1128
1129 for (i = 0; i < cfg->num_streamids; ++i) {
1130 u16 sid = cfg->streamids[i];
1131
1132 writel_relaxed(S2CR_TYPE_BYPASS,
1133 gr0_base + ARM_SMMU_GR0_S2CR(sid));
1134 }
1135}
1136
1137static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain, 1160static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
1138 struct arm_smmu_master_cfg *cfg) 1161 struct arm_smmu_master_cfg *cfg)
1139{ 1162{
@@ -1141,9 +1164,10 @@ static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
1141 struct arm_smmu_device *smmu = smmu_domain->smmu; 1164 struct arm_smmu_device *smmu = smmu_domain->smmu;
1142 void __iomem *gr0_base = ARM_SMMU_GR0(smmu); 1165 void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1143 1166
1167 /* Devices in an IOMMU group may already be configured */
1144 ret = arm_smmu_master_configure_smrs(smmu, cfg); 1168 ret = arm_smmu_master_configure_smrs(smmu, cfg);
1145 if (ret) 1169 if (ret)
1146 return ret; 1170 return ret == -EEXIST ? 0 : ret;
1147 1171
1148 for (i = 0; i < cfg->num_streamids; ++i) { 1172 for (i = 0; i < cfg->num_streamids; ++i) {
1149 u32 idx, s2cr; 1173 u32 idx, s2cr;
@@ -1160,58 +1184,75 @@ static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
1160static void arm_smmu_domain_remove_master(struct arm_smmu_domain *smmu_domain, 1184static void arm_smmu_domain_remove_master(struct arm_smmu_domain *smmu_domain,
1161 struct arm_smmu_master_cfg *cfg) 1185 struct arm_smmu_master_cfg *cfg)
1162{ 1186{
1187 int i;
1163 struct arm_smmu_device *smmu = smmu_domain->smmu; 1188 struct arm_smmu_device *smmu = smmu_domain->smmu;
1189 void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1190
1191 /* An IOMMU group is torn down by the first device to be removed */
1192 if ((smmu->features & ARM_SMMU_FEAT_STREAM_MATCH) && !cfg->smrs)
1193 return;
1164 1194
1165 /* 1195 /*
1166 * We *must* clear the S2CR first, because freeing the SMR means 1196 * We *must* clear the S2CR first, because freeing the SMR means
1167 * that it can be re-allocated immediately. 1197 * that it can be re-allocated immediately.
1168 */ 1198 */
1169 arm_smmu_bypass_stream_mapping(smmu, cfg); 1199 for (i = 0; i < cfg->num_streamids; ++i) {
1200 u32 idx = cfg->smrs ? cfg->smrs[i].idx : cfg->streamids[i];
1201
1202 writel_relaxed(S2CR_TYPE_BYPASS,
1203 gr0_base + ARM_SMMU_GR0_S2CR(idx));
1204 }
1205
1170 arm_smmu_master_free_smrs(smmu, cfg); 1206 arm_smmu_master_free_smrs(smmu, cfg);
1171} 1207}
1172 1208
1173static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) 1209static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1174{ 1210{
1175 int ret = -EINVAL; 1211 int ret;
1176 struct arm_smmu_domain *smmu_domain = domain->priv; 1212 struct arm_smmu_domain *smmu_domain = domain->priv;
1177 struct arm_smmu_device *smmu; 1213 struct arm_smmu_device *smmu, *dom_smmu;
1178 struct arm_smmu_master_cfg *cfg; 1214 struct arm_smmu_master_cfg *cfg;
1179 unsigned long flags;
1180 1215
1181 smmu = dev_get_master_dev(dev)->archdata.iommu; 1216 smmu = find_smmu_for_device(dev);
1182 if (!smmu) { 1217 if (!smmu) {
1183 dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n"); 1218 dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n");
1184 return -ENXIO; 1219 return -ENXIO;
1185 } 1220 }
1186 1221
1222 if (dev->archdata.iommu) {
1223 dev_err(dev, "already attached to IOMMU domain\n");
1224 return -EEXIST;
1225 }
1226
1187 /* 1227 /*
1188 * Sanity check the domain. We don't support domains across 1228 * Sanity check the domain. We don't support domains across
1189 * different SMMUs. 1229 * different SMMUs.
1190 */ 1230 */
1191 spin_lock_irqsave(&smmu_domain->lock, flags); 1231 dom_smmu = ACCESS_ONCE(smmu_domain->smmu);
1192 if (!smmu_domain->smmu) { 1232 if (!dom_smmu) {
1193 /* Now that we have a master, we can finalise the domain */ 1233 /* Now that we have a master, we can finalise the domain */
1194 ret = arm_smmu_init_domain_context(domain, smmu); 1234 ret = arm_smmu_init_domain_context(domain, smmu);
1195 if (IS_ERR_VALUE(ret)) 1235 if (IS_ERR_VALUE(ret))
1196 goto err_unlock; 1236 return ret;
1197 } else if (smmu_domain->smmu != smmu) { 1237
1238 dom_smmu = smmu_domain->smmu;
1239 }
1240
1241 if (dom_smmu != smmu) {
1198 dev_err(dev, 1242 dev_err(dev,
1199 "cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n", 1243 "cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n",
1200 dev_name(smmu_domain->smmu->dev), 1244 dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev));
1201 dev_name(smmu->dev)); 1245 return -EINVAL;
1202 goto err_unlock;
1203 } 1246 }
1204 spin_unlock_irqrestore(&smmu_domain->lock, flags);
1205 1247
1206 /* Looks ok, so add the device to the domain */ 1248 /* Looks ok, so add the device to the domain */
1207 cfg = find_smmu_master_cfg(smmu_domain->smmu, dev); 1249 cfg = find_smmu_master_cfg(dev);
1208 if (!cfg) 1250 if (!cfg)
1209 return -ENODEV; 1251 return -ENODEV;
1210 1252
1211 return arm_smmu_domain_add_master(smmu_domain, cfg); 1253 ret = arm_smmu_domain_add_master(smmu_domain, cfg);
1212 1254 if (!ret)
1213err_unlock: 1255 dev->archdata.iommu = domain;
1214 spin_unlock_irqrestore(&smmu_domain->lock, flags);
1215 return ret; 1256 return ret;
1216} 1257}
1217 1258
@@ -1220,9 +1261,12 @@ static void arm_smmu_detach_dev(struct iommu_domain *domain, struct device *dev)
1220 struct arm_smmu_domain *smmu_domain = domain->priv; 1261 struct arm_smmu_domain *smmu_domain = domain->priv;
1221 struct arm_smmu_master_cfg *cfg; 1262 struct arm_smmu_master_cfg *cfg;
1222 1263
1223 cfg = find_smmu_master_cfg(smmu_domain->smmu, dev); 1264 cfg = find_smmu_master_cfg(dev);
1224 if (cfg) 1265 if (!cfg)
1225 arm_smmu_domain_remove_master(smmu_domain, cfg); 1266 return;
1267
1268 dev->archdata.iommu = NULL;
1269 arm_smmu_domain_remove_master(smmu_domain, cfg);
1226} 1270}
1227 1271
1228static bool arm_smmu_pte_is_contiguous_range(unsigned long addr, 1272static bool arm_smmu_pte_is_contiguous_range(unsigned long addr,
@@ -1247,10 +1291,6 @@ static int arm_smmu_alloc_init_pte(struct arm_smmu_device *smmu, pmd_t *pmd,
1247 return -ENOMEM; 1291 return -ENOMEM;
1248 1292
1249 arm_smmu_flush_pgtable(smmu, page_address(table), PAGE_SIZE); 1293 arm_smmu_flush_pgtable(smmu, page_address(table), PAGE_SIZE);
1250 if (!pgtable_page_ctor(table)) {
1251 __free_page(table);
1252 return -ENOMEM;
1253 }
1254 pmd_populate(NULL, pmd, table); 1294 pmd_populate(NULL, pmd, table);
1255 arm_smmu_flush_pgtable(smmu, pmd, sizeof(*pmd)); 1295 arm_smmu_flush_pgtable(smmu, pmd, sizeof(*pmd));
1256 } 1296 }
@@ -1368,6 +1408,7 @@ static int arm_smmu_alloc_init_pmd(struct arm_smmu_device *smmu, pud_t *pud,
1368 ret = arm_smmu_alloc_init_pte(smmu, pmd, addr, next, pfn, 1408 ret = arm_smmu_alloc_init_pte(smmu, pmd, addr, next, pfn,
1369 prot, stage); 1409 prot, stage);
1370 phys += next - addr; 1410 phys += next - addr;
1411 pfn = __phys_to_pfn(phys);
1371 } while (pmd++, addr = next, addr < end); 1412 } while (pmd++, addr = next, addr < end);
1372 1413
1373 return ret; 1414 return ret;
@@ -1420,9 +1461,11 @@ static int arm_smmu_handle_mapping(struct arm_smmu_domain *smmu_domain,
1420 1461
1421 if (cfg->cbar == CBAR_TYPE_S2_TRANS) { 1462 if (cfg->cbar == CBAR_TYPE_S2_TRANS) {
1422 stage = 2; 1463 stage = 2;
1464 input_mask = (1ULL << smmu->s2_input_size) - 1;
1423 output_mask = (1ULL << smmu->s2_output_size) - 1; 1465 output_mask = (1ULL << smmu->s2_output_size) - 1;
1424 } else { 1466 } else {
1425 stage = 1; 1467 stage = 1;
1468 input_mask = (1ULL << smmu->s1_input_size) - 1;
1426 output_mask = (1ULL << smmu->s1_output_size) - 1; 1469 output_mask = (1ULL << smmu->s1_output_size) - 1;
1427 } 1470 }
1428 1471
@@ -1432,7 +1475,6 @@ static int arm_smmu_handle_mapping(struct arm_smmu_domain *smmu_domain,
1432 if (size & ~PAGE_MASK) 1475 if (size & ~PAGE_MASK)
1433 return -EINVAL; 1476 return -EINVAL;
1434 1477
1435 input_mask = (1ULL << smmu->input_size) - 1;
1436 if ((phys_addr_t)iova & ~input_mask) 1478 if ((phys_addr_t)iova & ~input_mask)
1437 return -ERANGE; 1479 return -ERANGE;
1438 1480
@@ -1515,20 +1557,19 @@ static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1515 return __pfn_to_phys(pte_pfn(pte)) | (iova & ~PAGE_MASK); 1557 return __pfn_to_phys(pte_pfn(pte)) | (iova & ~PAGE_MASK);
1516} 1558}
1517 1559
1518static int arm_smmu_domain_has_cap(struct iommu_domain *domain, 1560static bool arm_smmu_capable(enum iommu_cap cap)
1519 unsigned long cap)
1520{ 1561{
1521 struct arm_smmu_domain *smmu_domain = domain->priv;
1522 struct arm_smmu_device *smmu = smmu_domain->smmu;
1523 u32 features = smmu ? smmu->features : 0;
1524
1525 switch (cap) { 1562 switch (cap) {
1526 case IOMMU_CAP_CACHE_COHERENCY: 1563 case IOMMU_CAP_CACHE_COHERENCY:
1527 return features & ARM_SMMU_FEAT_COHERENT_WALK; 1564 /*
1565 * Return true here as the SMMU can always send out coherent
1566 * requests.
1567 */
1568 return true;
1528 case IOMMU_CAP_INTR_REMAP: 1569 case IOMMU_CAP_INTR_REMAP:
1529 return 1; /* MSIs are just memory writes */ 1570 return true; /* MSIs are just memory writes */
1530 default: 1571 default:
1531 return 0; 1572 return false;
1532 } 1573 }
1533} 1574}
1534 1575
@@ -1538,17 +1579,19 @@ static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
1538 return 0; /* Continue walking */ 1579 return 0; /* Continue walking */
1539} 1580}
1540 1581
1582static void __arm_smmu_release_pci_iommudata(void *data)
1583{
1584 kfree(data);
1585}
1586
1541static int arm_smmu_add_device(struct device *dev) 1587static int arm_smmu_add_device(struct device *dev)
1542{ 1588{
1543 struct arm_smmu_device *smmu; 1589 struct arm_smmu_device *smmu;
1590 struct arm_smmu_master_cfg *cfg;
1544 struct iommu_group *group; 1591 struct iommu_group *group;
1592 void (*releasefn)(void *) = NULL;
1545 int ret; 1593 int ret;
1546 1594
1547 if (dev->archdata.iommu) {
1548 dev_warn(dev, "IOMMU driver already assigned to device\n");
1549 return -EINVAL;
1550 }
1551
1552 smmu = find_smmu_for_device(dev); 1595 smmu = find_smmu_for_device(dev);
1553 if (!smmu) 1596 if (!smmu)
1554 return -ENODEV; 1597 return -ENODEV;
@@ -1560,7 +1603,6 @@ static int arm_smmu_add_device(struct device *dev)
1560 } 1603 }
1561 1604
1562 if (dev_is_pci(dev)) { 1605 if (dev_is_pci(dev)) {
1563 struct arm_smmu_master_cfg *cfg;
1564 struct pci_dev *pdev = to_pci_dev(dev); 1606 struct pci_dev *pdev = to_pci_dev(dev);
1565 1607
1566 cfg = kzalloc(sizeof(*cfg), GFP_KERNEL); 1608 cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
@@ -1576,11 +1618,20 @@ static int arm_smmu_add_device(struct device *dev)
1576 */ 1618 */
1577 pci_for_each_dma_alias(pdev, __arm_smmu_get_pci_sid, 1619 pci_for_each_dma_alias(pdev, __arm_smmu_get_pci_sid,
1578 &cfg->streamids[0]); 1620 &cfg->streamids[0]);
1579 dev->archdata.iommu = cfg; 1621 releasefn = __arm_smmu_release_pci_iommudata;
1580 } else { 1622 } else {
1581 dev->archdata.iommu = smmu; 1623 struct arm_smmu_master *master;
1624
1625 master = find_smmu_master(smmu, dev->of_node);
1626 if (!master) {
1627 ret = -ENODEV;
1628 goto out_put_group;
1629 }
1630
1631 cfg = &master->cfg;
1582 } 1632 }
1583 1633
1634 iommu_group_set_iommudata(group, cfg, releasefn);
1584 ret = iommu_group_add_device(group, dev); 1635 ret = iommu_group_add_device(group, dev);
1585 1636
1586out_put_group: 1637out_put_group:
@@ -1590,14 +1641,11 @@ out_put_group:
1590 1641
1591static void arm_smmu_remove_device(struct device *dev) 1642static void arm_smmu_remove_device(struct device *dev)
1592{ 1643{
1593 if (dev_is_pci(dev))
1594 kfree(dev->archdata.iommu);
1595
1596 dev->archdata.iommu = NULL;
1597 iommu_group_remove_device(dev); 1644 iommu_group_remove_device(dev);
1598} 1645}
1599 1646
1600static const struct iommu_ops arm_smmu_ops = { 1647static const struct iommu_ops arm_smmu_ops = {
1648 .capable = arm_smmu_capable,
1601 .domain_init = arm_smmu_domain_init, 1649 .domain_init = arm_smmu_domain_init,
1602 .domain_destroy = arm_smmu_domain_destroy, 1650 .domain_destroy = arm_smmu_domain_destroy,
1603 .attach_dev = arm_smmu_attach_dev, 1651 .attach_dev = arm_smmu_attach_dev,
@@ -1605,7 +1653,6 @@ static const struct iommu_ops arm_smmu_ops = {
1605 .map = arm_smmu_map, 1653 .map = arm_smmu_map,
1606 .unmap = arm_smmu_unmap, 1654 .unmap = arm_smmu_unmap,
1607 .iova_to_phys = arm_smmu_iova_to_phys, 1655 .iova_to_phys = arm_smmu_iova_to_phys,
1608 .domain_has_cap = arm_smmu_domain_has_cap,
1609 .add_device = arm_smmu_add_device, 1656 .add_device = arm_smmu_add_device,
1610 .remove_device = arm_smmu_remove_device, 1657 .remove_device = arm_smmu_remove_device,
1611 .pgsize_bitmap = (SECTION_SIZE | 1658 .pgsize_bitmap = (SECTION_SIZE |
@@ -1626,7 +1673,7 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1626 1673
1627 /* Mark all SMRn as invalid and all S2CRn as bypass */ 1674 /* Mark all SMRn as invalid and all S2CRn as bypass */
1628 for (i = 0; i < smmu->num_mapping_groups; ++i) { 1675 for (i = 0; i < smmu->num_mapping_groups; ++i) {
1629 writel_relaxed(~SMR_VALID, gr0_base + ARM_SMMU_GR0_SMR(i)); 1676 writel_relaxed(0, gr0_base + ARM_SMMU_GR0_SMR(i));
1630 writel_relaxed(S2CR_TYPE_BYPASS, 1677 writel_relaxed(S2CR_TYPE_BYPASS,
1631 gr0_base + ARM_SMMU_GR0_S2CR(i)); 1678 gr0_base + ARM_SMMU_GR0_S2CR(i));
1632 } 1679 }
@@ -1691,10 +1738,6 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1691 u32 id; 1738 u32 id;
1692 1739
1693 dev_notice(smmu->dev, "probing hardware configuration...\n"); 1740 dev_notice(smmu->dev, "probing hardware configuration...\n");
1694
1695 /* Primecell ID */
1696 id = readl_relaxed(gr0_base + ARM_SMMU_GR0_PIDR2);
1697 smmu->version = ((id >> PIDR2_ARCH_SHIFT) & PIDR2_ARCH_MASK) + 1;
1698 dev_notice(smmu->dev, "SMMUv%d with:\n", smmu->version); 1741 dev_notice(smmu->dev, "SMMUv%d with:\n", smmu->version);
1699 1742
1700 /* ID0 */ 1743 /* ID0 */
@@ -1705,6 +1748,13 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1705 return -ENODEV; 1748 return -ENODEV;
1706 } 1749 }
1707#endif 1750#endif
1751
1752 /* Restrict available stages based on module parameter */
1753 if (force_stage == 1)
1754 id &= ~(ID0_S2TS | ID0_NTS);
1755 else if (force_stage == 2)
1756 id &= ~(ID0_S1TS | ID0_NTS);
1757
1708 if (id & ID0_S1TS) { 1758 if (id & ID0_S1TS) {
1709 smmu->features |= ARM_SMMU_FEAT_TRANS_S1; 1759 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1710 dev_notice(smmu->dev, "\tstage 1 translation\n"); 1760 dev_notice(smmu->dev, "\tstage 1 translation\n");
@@ -1721,8 +1771,7 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1721 } 1771 }
1722 1772
1723 if (!(smmu->features & 1773 if (!(smmu->features &
1724 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2 | 1774 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1725 ARM_SMMU_FEAT_TRANS_NESTED))) {
1726 dev_err(smmu->dev, "\tno translation support!\n"); 1775 dev_err(smmu->dev, "\tno translation support!\n");
1727 return -ENODEV; 1776 return -ENODEV;
1728 } 1777 }
@@ -1761,16 +1810,19 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1761 dev_notice(smmu->dev, 1810 dev_notice(smmu->dev,
1762 "\tstream matching with %u register groups, mask 0x%x", 1811 "\tstream matching with %u register groups, mask 0x%x",
1763 smmu->num_mapping_groups, mask); 1812 smmu->num_mapping_groups, mask);
1813 } else {
1814 smmu->num_mapping_groups = (id >> ID0_NUMSIDB_SHIFT) &
1815 ID0_NUMSIDB_MASK;
1764 } 1816 }
1765 1817
1766 /* ID1 */ 1818 /* ID1 */
1767 id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID1); 1819 id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID1);
1768 smmu->pagesize = (id & ID1_PAGESIZE) ? SZ_64K : SZ_4K; 1820 smmu->pgshift = (id & ID1_PAGESIZE) ? 16 : 12;
1769 1821
1770 /* Check for size mismatch of SMMU address space from mapped region */ 1822 /* Check for size mismatch of SMMU address space from mapped region */
1771 size = 1 << 1823 size = 1 <<
1772 (((id >> ID1_NUMPAGENDXB_SHIFT) & ID1_NUMPAGENDXB_MASK) + 1); 1824 (((id >> ID1_NUMPAGENDXB_SHIFT) & ID1_NUMPAGENDXB_MASK) + 1);
1773 size *= (smmu->pagesize << 1); 1825 size *= 2 << smmu->pgshift;
1774 if (smmu->size != size) 1826 if (smmu->size != size)
1775 dev_warn(smmu->dev, 1827 dev_warn(smmu->dev,
1776 "SMMU address space size (0x%lx) differs from mapped region size (0x%lx)!\n", 1828 "SMMU address space size (0x%lx) differs from mapped region size (0x%lx)!\n",
@@ -1789,23 +1841,21 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1789 /* ID2 */ 1841 /* ID2 */
1790 id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID2); 1842 id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID2);
1791 size = arm_smmu_id_size_to_bits((id >> ID2_IAS_SHIFT) & ID2_IAS_MASK); 1843 size = arm_smmu_id_size_to_bits((id >> ID2_IAS_SHIFT) & ID2_IAS_MASK);
1844 smmu->s1_output_size = min_t(unsigned long, PHYS_MASK_SHIFT, size);
1792 1845
1793 /* 1846 /* Stage-2 input size limited due to pgd allocation (PTRS_PER_PGD) */
1794 * Stage-1 output limited by stage-2 input size due to pgd
1795 * allocation (PTRS_PER_PGD).
1796 */
1797#ifdef CONFIG_64BIT 1847#ifdef CONFIG_64BIT
1798 smmu->s1_output_size = min_t(unsigned long, VA_BITS, size); 1848 smmu->s2_input_size = min_t(unsigned long, VA_BITS, size);
1799#else 1849#else
1800 smmu->s1_output_size = min(32UL, size); 1850 smmu->s2_input_size = min(32UL, size);
1801#endif 1851#endif
1802 1852
1803 /* The stage-2 output mask is also applied for bypass */ 1853 /* The stage-2 output mask is also applied for bypass */
1804 size = arm_smmu_id_size_to_bits((id >> ID2_OAS_SHIFT) & ID2_OAS_MASK); 1854 size = arm_smmu_id_size_to_bits((id >> ID2_OAS_SHIFT) & ID2_OAS_MASK);
1805 smmu->s2_output_size = min_t(unsigned long, PHYS_MASK_SHIFT, size); 1855 smmu->s2_output_size = min_t(unsigned long, PHYS_MASK_SHIFT, size);
1806 1856
1807 if (smmu->version == 1) { 1857 if (smmu->version == ARM_SMMU_V1) {
1808 smmu->input_size = 32; 1858 smmu->s1_input_size = 32;
1809 } else { 1859 } else {
1810#ifdef CONFIG_64BIT 1860#ifdef CONFIG_64BIT
1811 size = (id >> ID2_UBS_SHIFT) & ID2_UBS_MASK; 1861 size = (id >> ID2_UBS_SHIFT) & ID2_UBS_MASK;
@@ -1813,7 +1863,7 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1813#else 1863#else
1814 size = 32; 1864 size = 32;
1815#endif 1865#endif
1816 smmu->input_size = size; 1866 smmu->s1_input_size = size;
1817 1867
1818 if ((PAGE_SIZE == SZ_4K && !(id & ID2_PTFS_4K)) || 1868 if ((PAGE_SIZE == SZ_4K && !(id & ID2_PTFS_4K)) ||
1819 (PAGE_SIZE == SZ_64K && !(id & ID2_PTFS_64K)) || 1869 (PAGE_SIZE == SZ_64K && !(id & ID2_PTFS_64K)) ||
@@ -1824,15 +1874,30 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1824 } 1874 }
1825 } 1875 }
1826 1876
1827 dev_notice(smmu->dev, 1877 if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
1828 "\t%lu-bit VA, %lu-bit IPA, %lu-bit PA\n", 1878 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
1829 smmu->input_size, smmu->s1_output_size, 1879 smmu->s1_input_size, smmu->s1_output_size);
1830 smmu->s2_output_size); 1880
1881 if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
1882 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
1883 smmu->s2_input_size, smmu->s2_output_size);
1884
1831 return 0; 1885 return 0;
1832} 1886}
1833 1887
1888static const struct of_device_id arm_smmu_of_match[] = {
1889 { .compatible = "arm,smmu-v1", .data = (void *)ARM_SMMU_V1 },
1890 { .compatible = "arm,smmu-v2", .data = (void *)ARM_SMMU_V2 },
1891 { .compatible = "arm,mmu-400", .data = (void *)ARM_SMMU_V1 },
1892 { .compatible = "arm,mmu-401", .data = (void *)ARM_SMMU_V1 },
1893 { .compatible = "arm,mmu-500", .data = (void *)ARM_SMMU_V2 },
1894 { },
1895};
1896MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
1897
1834static int arm_smmu_device_dt_probe(struct platform_device *pdev) 1898static int arm_smmu_device_dt_probe(struct platform_device *pdev)
1835{ 1899{
1900 const struct of_device_id *of_id;
1836 struct resource *res; 1901 struct resource *res;
1837 struct arm_smmu_device *smmu; 1902 struct arm_smmu_device *smmu;
1838 struct device *dev = &pdev->dev; 1903 struct device *dev = &pdev->dev;
@@ -1847,6 +1912,9 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev)
1847 } 1912 }
1848 smmu->dev = dev; 1913 smmu->dev = dev;
1849 1914
1915 of_id = of_match_node(arm_smmu_of_match, dev->of_node);
1916 smmu->version = (enum arm_smmu_arch_version)of_id->data;
1917
1850 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1918 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1851 smmu->base = devm_ioremap_resource(dev, res); 1919 smmu->base = devm_ioremap_resource(dev, res);
1852 if (IS_ERR(smmu->base)) 1920 if (IS_ERR(smmu->base))
@@ -1889,6 +1957,10 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev)
1889 smmu->irqs[i] = irq; 1957 smmu->irqs[i] = irq;
1890 } 1958 }
1891 1959
1960 err = arm_smmu_device_cfg_probe(smmu);
1961 if (err)
1962 return err;
1963
1892 i = 0; 1964 i = 0;
1893 smmu->masters = RB_ROOT; 1965 smmu->masters = RB_ROOT;
1894 while (!of_parse_phandle_with_args(dev->of_node, "mmu-masters", 1966 while (!of_parse_phandle_with_args(dev->of_node, "mmu-masters",
@@ -1905,13 +1977,9 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev)
1905 } 1977 }
1906 dev_notice(dev, "registered %d master devices\n", i); 1978 dev_notice(dev, "registered %d master devices\n", i);
1907 1979
1908 err = arm_smmu_device_cfg_probe(smmu);
1909 if (err)
1910 goto out_put_masters;
1911
1912 parse_driver_options(smmu); 1980 parse_driver_options(smmu);
1913 1981
1914 if (smmu->version > 1 && 1982 if (smmu->version > ARM_SMMU_V1 &&
1915 smmu->num_context_banks != smmu->num_context_irqs) { 1983 smmu->num_context_banks != smmu->num_context_irqs) {
1916 dev_err(dev, 1984 dev_err(dev,
1917 "found only %d context interrupt(s) but %d required\n", 1985 "found only %d context interrupt(s) but %d required\n",
@@ -1992,17 +2060,6 @@ static int arm_smmu_device_remove(struct platform_device *pdev)
1992 return 0; 2060 return 0;
1993} 2061}
1994 2062
1995#ifdef CONFIG_OF
1996static struct of_device_id arm_smmu_of_match[] = {
1997 { .compatible = "arm,smmu-v1", },
1998 { .compatible = "arm,smmu-v2", },
1999 { .compatible = "arm,mmu-400", },
2000 { .compatible = "arm,mmu-500", },
2001 { },
2002};
2003MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
2004#endif
2005
2006static struct platform_driver arm_smmu_driver = { 2063static struct platform_driver arm_smmu_driver = {
2007 .driver = { 2064 .driver = {
2008 .owner = THIS_MODULE, 2065 .owner = THIS_MODULE,
diff --git a/drivers/iommu/dmar.c b/drivers/iommu/dmar.c
index b3774ef3f255..c5c61cabd6e3 100644
--- a/drivers/iommu/dmar.c
+++ b/drivers/iommu/dmar.c
@@ -695,8 +695,7 @@ static int __init dmar_acpi_dev_scope_init(void)
695 andd->device_name); 695 andd->device_name);
696 continue; 696 continue;
697 } 697 }
698 acpi_bus_get_device(h, &adev); 698 if (acpi_bus_get_device(h, &adev)) {
699 if (!adev) {
700 pr_err("Failed to get device for ACPI object %s\n", 699 pr_err("Failed to get device for ACPI object %s\n",
701 andd->device_name); 700 andd->device_name);
702 continue; 701 continue;
diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c
index d037e87a1fe5..74233186f6f7 100644
--- a/drivers/iommu/exynos-iommu.c
+++ b/drivers/iommu/exynos-iommu.c
@@ -32,7 +32,7 @@
32typedef u32 sysmmu_iova_t; 32typedef u32 sysmmu_iova_t;
33typedef u32 sysmmu_pte_t; 33typedef u32 sysmmu_pte_t;
34 34
35/* We does not consider super section mapping (16MB) */ 35/* We do not consider super section mapping (16MB) */
36#define SECT_ORDER 20 36#define SECT_ORDER 20
37#define LPAGE_ORDER 16 37#define LPAGE_ORDER 16
38#define SPAGE_ORDER 12 38#define SPAGE_ORDER 12
@@ -307,7 +307,7 @@ static void show_fault_information(const char *name,
307 307
308static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id) 308static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id)
309{ 309{
310 /* SYSMMU is in blocked when interrupt occurred. */ 310 /* SYSMMU is in blocked state when interrupt occurred. */
311 struct sysmmu_drvdata *data = dev_id; 311 struct sysmmu_drvdata *data = dev_id;
312 enum exynos_sysmmu_inttype itype; 312 enum exynos_sysmmu_inttype itype;
313 sysmmu_iova_t addr = -1; 313 sysmmu_iova_t addr = -1;
@@ -567,8 +567,8 @@ static void sysmmu_tlb_invalidate_entry(struct device *dev, sysmmu_iova_t iova,
567 /* 567 /*
568 * L2TLB invalidation required 568 * L2TLB invalidation required
569 * 4KB page: 1 invalidation 569 * 4KB page: 1 invalidation
570 * 64KB page: 16 invalidation 570 * 64KB page: 16 invalidations
571 * 1MB page: 64 invalidation 571 * 1MB page: 64 invalidations
572 * because it is set-associative TLB 572 * because it is set-associative TLB
573 * with 8-way and 64 sets. 573 * with 8-way and 64 sets.
574 * 1MB page can be cached in one of all sets. 574 * 1MB page can be cached in one of all sets.
@@ -714,7 +714,7 @@ static int exynos_iommu_domain_init(struct iommu_domain *domain)
714 if (!priv->lv2entcnt) 714 if (!priv->lv2entcnt)
715 goto err_counter; 715 goto err_counter;
716 716
717 /* w/a of System MMU v3.3 to prevent caching 1MiB mapping */ 717 /* Workaround for System MMU v3.3 to prevent caching 1MiB mapping */
718 for (i = 0; i < NUM_LV1ENTRIES; i += 8) { 718 for (i = 0; i < NUM_LV1ENTRIES; i += 8) {
719 priv->pgtable[i + 0] = ZERO_LV2LINK; 719 priv->pgtable[i + 0] = ZERO_LV2LINK;
720 priv->pgtable[i + 1] = ZERO_LV2LINK; 720 priv->pgtable[i + 1] = ZERO_LV2LINK;
@@ -861,14 +861,14 @@ static sysmmu_pte_t *alloc_lv2entry(struct exynos_iommu_domain *priv,
861 pgtable_flush(sent, sent + 1); 861 pgtable_flush(sent, sent + 1);
862 862
863 /* 863 /*
864 * If pretched SLPD is a fault SLPD in zero_l2_table, FLPD cache 864 * If pre-fetched SLPD is a faulty SLPD in zero_l2_table,
865 * may caches the address of zero_l2_table. This function 865 * FLPD cache may cache the address of zero_l2_table. This
866 * replaces the zero_l2_table with new L2 page table to write 866 * function replaces the zero_l2_table with new L2 page table
867 * valid mappings. 867 * to write valid mappings.
868 * Accessing the valid area may cause page fault since FLPD 868 * Accessing the valid area may cause page fault since FLPD
869 * cache may still caches zero_l2_table for the valid area 869 * cache may still cache zero_l2_table for the valid area
870 * instead of new L2 page table that have the mapping 870 * instead of new L2 page table that has the mapping
871 * information of the valid area 871 * information of the valid area.
872 * Thus any replacement of zero_l2_table with other valid L2 872 * Thus any replacement of zero_l2_table with other valid L2
873 * page table must involve FLPD cache invalidation for System 873 * page table must involve FLPD cache invalidation for System
874 * MMU v3.3. 874 * MMU v3.3.
@@ -963,27 +963,27 @@ static int lv2set_page(sysmmu_pte_t *pent, phys_addr_t paddr, size_t size,
963/* 963/*
964 * *CAUTION* to the I/O virtual memory managers that support exynos-iommu: 964 * *CAUTION* to the I/O virtual memory managers that support exynos-iommu:
965 * 965 *
966 * System MMU v3.x have an advanced logic to improve address translation 966 * System MMU v3.x has advanced logic to improve address translation
967 * performance with caching more page table entries by a page table walk. 967 * performance with caching more page table entries by a page table walk.
968 * However, the logic has a bug that caching fault page table entries and System 968 * However, the logic has a bug that while caching faulty page table entries,
969 * MMU reports page fault if the cached fault entry is hit even though the fault 969 * System MMU reports page fault if the cached fault entry is hit even though
970 * entry is updated to a valid entry after the entry is cached. 970 * the fault entry is updated to a valid entry after the entry is cached.
971 * To prevent caching fault page table entries which may be updated to valid 971 * To prevent caching faulty page table entries which may be updated to valid
972 * entries later, the virtual memory manager should care about the w/a about the 972 * entries later, the virtual memory manager should care about the workaround
973 * problem. The followings describe w/a. 973 * for the problem. The following describes the workaround.
974 * 974 *
975 * Any two consecutive I/O virtual address regions must have a hole of 128KiB 975 * Any two consecutive I/O virtual address regions must have a hole of 128KiB
976 * in maximum to prevent misbehavior of System MMU 3.x. (w/a of h/w bug) 976 * at maximum to prevent misbehavior of System MMU 3.x (workaround for h/w bug).
977 * 977 *
978 * Precisely, any start address of I/O virtual region must be aligned by 978 * Precisely, any start address of I/O virtual region must be aligned with
979 * the following sizes for System MMU v3.1 and v3.2. 979 * the following sizes for System MMU v3.1 and v3.2.
980 * System MMU v3.1: 128KiB 980 * System MMU v3.1: 128KiB
981 * System MMU v3.2: 256KiB 981 * System MMU v3.2: 256KiB
982 * 982 *
983 * Because System MMU v3.3 caches page table entries more aggressively, it needs 983 * Because System MMU v3.3 caches page table entries more aggressively, it needs
984 * more w/a. 984 * more workarounds.
985 * - Any two consecutive I/O virtual regions must be have a hole of larger size 985 * - Any two consecutive I/O virtual regions must have a hole of size larger
986 * than or equal size to 128KiB. 986 * than or equal to 128KiB.
987 * - Start address of an I/O virtual region must be aligned by 128KiB. 987 * - Start address of an I/O virtual region must be aligned by 128KiB.
988 */ 988 */
989static int exynos_iommu_map(struct iommu_domain *domain, unsigned long l_iova, 989static int exynos_iommu_map(struct iommu_domain *domain, unsigned long l_iova,
@@ -1061,7 +1061,8 @@ static size_t exynos_iommu_unmap(struct iommu_domain *domain,
1061 goto err; 1061 goto err;
1062 } 1062 }
1063 1063
1064 *ent = ZERO_LV2LINK; /* w/a for h/w bug in Sysmem MMU v3.3 */ 1064 /* workaround for h/w bug in System MMU v3.3 */
1065 *ent = ZERO_LV2LINK;
1065 pgtable_flush(ent, ent + 1); 1066 pgtable_flush(ent, ent + 1);
1066 size = SECT_SIZE; 1067 size = SECT_SIZE;
1067 goto done; 1068 goto done;
diff --git a/drivers/iommu/fsl_pamu_domain.c b/drivers/iommu/fsl_pamu_domain.c
index 61d1dafa242d..c828f80d48b0 100644
--- a/drivers/iommu/fsl_pamu_domain.c
+++ b/drivers/iommu/fsl_pamu_domain.c
@@ -411,8 +411,7 @@ static phys_addr_t fsl_pamu_iova_to_phys(struct iommu_domain *domain,
411 return get_phys_addr(dma_domain, iova); 411 return get_phys_addr(dma_domain, iova);
412} 412}
413 413
414static int fsl_pamu_domain_has_cap(struct iommu_domain *domain, 414static bool fsl_pamu_capable(enum iommu_cap cap)
415 unsigned long cap)
416{ 415{
417 return cap == IOMMU_CAP_CACHE_COHERENCY; 416 return cap == IOMMU_CAP_CACHE_COHERENCY;
418} 417}
@@ -984,7 +983,7 @@ static int fsl_pamu_add_device(struct device *dev)
984 struct iommu_group *group = ERR_PTR(-ENODEV); 983 struct iommu_group *group = ERR_PTR(-ENODEV);
985 struct pci_dev *pdev; 984 struct pci_dev *pdev;
986 const u32 *prop; 985 const u32 *prop;
987 int ret, len; 986 int ret = 0, len;
988 987
989 /* 988 /*
990 * For platform devices we allocate a separate group for 989 * For platform devices we allocate a separate group for
@@ -1007,7 +1006,13 @@ static int fsl_pamu_add_device(struct device *dev)
1007 if (IS_ERR(group)) 1006 if (IS_ERR(group))
1008 return PTR_ERR(group); 1007 return PTR_ERR(group);
1009 1008
1010 ret = iommu_group_add_device(group, dev); 1009 /*
1010 * Check if device has already been added to an iommu group.
1011 * Group could have already been created for a PCI device in
1012 * the iommu_group_get_for_dev path.
1013 */
1014 if (!dev->iommu_group)
1015 ret = iommu_group_add_device(group, dev);
1011 1016
1012 iommu_group_put(group); 1017 iommu_group_put(group);
1013 return ret; 1018 return ret;
@@ -1074,6 +1079,7 @@ static u32 fsl_pamu_get_windows(struct iommu_domain *domain)
1074} 1079}
1075 1080
1076static const struct iommu_ops fsl_pamu_ops = { 1081static const struct iommu_ops fsl_pamu_ops = {
1082 .capable = fsl_pamu_capable,
1077 .domain_init = fsl_pamu_domain_init, 1083 .domain_init = fsl_pamu_domain_init,
1078 .domain_destroy = fsl_pamu_domain_destroy, 1084 .domain_destroy = fsl_pamu_domain_destroy,
1079 .attach_dev = fsl_pamu_attach_device, 1085 .attach_dev = fsl_pamu_attach_device,
@@ -1083,7 +1089,6 @@ static const struct iommu_ops fsl_pamu_ops = {
1083 .domain_get_windows = fsl_pamu_get_windows, 1089 .domain_get_windows = fsl_pamu_get_windows,
1084 .domain_set_windows = fsl_pamu_set_windows, 1090 .domain_set_windows = fsl_pamu_set_windows,
1085 .iova_to_phys = fsl_pamu_iova_to_phys, 1091 .iova_to_phys = fsl_pamu_iova_to_phys,
1086 .domain_has_cap = fsl_pamu_domain_has_cap,
1087 .domain_set_attr = fsl_pamu_set_domain_attr, 1092 .domain_set_attr = fsl_pamu_set_domain_attr,
1088 .domain_get_attr = fsl_pamu_get_domain_attr, 1093 .domain_get_attr = fsl_pamu_get_domain_attr,
1089 .add_device = fsl_pamu_add_device, 1094 .add_device = fsl_pamu_add_device,
diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c
index eaf825ac7d28..a27d6cb1a793 100644
--- a/drivers/iommu/intel-iommu.c
+++ b/drivers/iommu/intel-iommu.c
@@ -3868,6 +3868,14 @@ static int device_notifier(struct notifier_block *nb,
3868 if (action != BUS_NOTIFY_REMOVED_DEVICE) 3868 if (action != BUS_NOTIFY_REMOVED_DEVICE)
3869 return 0; 3869 return 0;
3870 3870
3871 /*
3872 * If the device is still attached to a device driver we can't
3873 * tear down the domain yet as DMA mappings may still be in use.
3874 * Wait for the BUS_NOTIFY_UNBOUND_DRIVER event to do that.
3875 */
3876 if (action == BUS_NOTIFY_DEL_DEVICE && dev->driver != NULL)
3877 return 0;
3878
3871 domain = find_domain(dev); 3879 domain = find_domain(dev);
3872 if (!domain) 3880 if (!domain)
3873 return 0; 3881 return 0;
@@ -4406,17 +4414,14 @@ static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
4406 return phys; 4414 return phys;
4407} 4415}
4408 4416
4409static int intel_iommu_domain_has_cap(struct iommu_domain *domain, 4417static bool intel_iommu_capable(enum iommu_cap cap)
4410 unsigned long cap)
4411{ 4418{
4412 struct dmar_domain *dmar_domain = domain->priv;
4413
4414 if (cap == IOMMU_CAP_CACHE_COHERENCY) 4419 if (cap == IOMMU_CAP_CACHE_COHERENCY)
4415 return dmar_domain->iommu_snooping; 4420 return domain_update_iommu_snooping(NULL) == 1;
4416 if (cap == IOMMU_CAP_INTR_REMAP) 4421 if (cap == IOMMU_CAP_INTR_REMAP)
4417 return irq_remapping_enabled; 4422 return irq_remapping_enabled == 1;
4418 4423
4419 return 0; 4424 return false;
4420} 4425}
4421 4426
4422static int intel_iommu_add_device(struct device *dev) 4427static int intel_iommu_add_device(struct device *dev)
@@ -4455,6 +4460,7 @@ static void intel_iommu_remove_device(struct device *dev)
4455} 4460}
4456 4461
4457static const struct iommu_ops intel_iommu_ops = { 4462static const struct iommu_ops intel_iommu_ops = {
4463 .capable = intel_iommu_capable,
4458 .domain_init = intel_iommu_domain_init, 4464 .domain_init = intel_iommu_domain_init,
4459 .domain_destroy = intel_iommu_domain_destroy, 4465 .domain_destroy = intel_iommu_domain_destroy,
4460 .attach_dev = intel_iommu_attach_device, 4466 .attach_dev = intel_iommu_attach_device,
@@ -4462,7 +4468,6 @@ static const struct iommu_ops intel_iommu_ops = {
4462 .map = intel_iommu_map, 4468 .map = intel_iommu_map,
4463 .unmap = intel_iommu_unmap, 4469 .unmap = intel_iommu_unmap,
4464 .iova_to_phys = intel_iommu_iova_to_phys, 4470 .iova_to_phys = intel_iommu_iova_to_phys,
4465 .domain_has_cap = intel_iommu_domain_has_cap,
4466 .add_device = intel_iommu_add_device, 4471 .add_device = intel_iommu_add_device,
4467 .remove_device = intel_iommu_remove_device, 4472 .remove_device = intel_iommu_remove_device,
4468 .pgsize_bitmap = INTEL_IOMMU_PGSIZES, 4473 .pgsize_bitmap = INTEL_IOMMU_PGSIZES,
diff --git a/drivers/iommu/intel_irq_remapping.c b/drivers/iommu/intel_irq_remapping.c
index a872874c2565..7c80661b35c1 100644
--- a/drivers/iommu/intel_irq_remapping.c
+++ b/drivers/iommu/intel_irq_remapping.c
@@ -1138,7 +1138,7 @@ static int intel_msi_setup_irq(struct pci_dev *pdev, unsigned int irq,
1138 return ret; 1138 return ret;
1139} 1139}
1140 1140
1141static int intel_setup_hpet_msi(unsigned int irq, unsigned int id) 1141static int intel_alloc_hpet_msi(unsigned int irq, unsigned int id)
1142{ 1142{
1143 int ret = -1; 1143 int ret = -1;
1144 struct intel_iommu *iommu; 1144 struct intel_iommu *iommu;
@@ -1169,5 +1169,5 @@ struct irq_remap_ops intel_irq_remap_ops = {
1169 .compose_msi_msg = intel_compose_msi_msg, 1169 .compose_msi_msg = intel_compose_msi_msg,
1170 .msi_alloc_irq = intel_msi_alloc_irq, 1170 .msi_alloc_irq = intel_msi_alloc_irq,
1171 .msi_setup_irq = intel_msi_setup_irq, 1171 .msi_setup_irq = intel_msi_setup_irq,
1172 .setup_hpet_msi = intel_setup_hpet_msi, 1172 .alloc_hpet_msi = intel_alloc_hpet_msi,
1173}; 1173};
diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
index 169836020208..ed8b04867b1f 100644
--- a/drivers/iommu/iommu.c
+++ b/drivers/iommu/iommu.c
@@ -30,6 +30,7 @@
30#include <linux/notifier.h> 30#include <linux/notifier.h>
31#include <linux/err.h> 31#include <linux/err.h>
32#include <linux/pci.h> 32#include <linux/pci.h>
33#include <linux/bitops.h>
33#include <trace/events/iommu.h> 34#include <trace/events/iommu.h>
34 35
35static struct kset *iommu_group_kset; 36static struct kset *iommu_group_kset;
@@ -519,6 +520,9 @@ int iommu_group_id(struct iommu_group *group)
519} 520}
520EXPORT_SYMBOL_GPL(iommu_group_id); 521EXPORT_SYMBOL_GPL(iommu_group_id);
521 522
523static struct iommu_group *get_pci_alias_group(struct pci_dev *pdev,
524 unsigned long *devfns);
525
522/* 526/*
523 * To consider a PCI device isolated, we require ACS to support Source 527 * To consider a PCI device isolated, we require ACS to support Source
524 * Validation, Request Redirection, Completer Redirection, and Upstream 528 * Validation, Request Redirection, Completer Redirection, and Upstream
@@ -529,6 +533,86 @@ EXPORT_SYMBOL_GPL(iommu_group_id);
529 */ 533 */
530#define REQ_ACS_FLAGS (PCI_ACS_SV | PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_UF) 534#define REQ_ACS_FLAGS (PCI_ACS_SV | PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_UF)
531 535
536/*
537 * For multifunction devices which are not isolated from each other, find
538 * all the other non-isolated functions and look for existing groups. For
539 * each function, we also need to look for aliases to or from other devices
540 * that may already have a group.
541 */
542static struct iommu_group *get_pci_function_alias_group(struct pci_dev *pdev,
543 unsigned long *devfns)
544{
545 struct pci_dev *tmp = NULL;
546 struct iommu_group *group;
547
548 if (!pdev->multifunction || pci_acs_enabled(pdev, REQ_ACS_FLAGS))
549 return NULL;
550
551 for_each_pci_dev(tmp) {
552 if (tmp == pdev || tmp->bus != pdev->bus ||
553 PCI_SLOT(tmp->devfn) != PCI_SLOT(pdev->devfn) ||
554 pci_acs_enabled(tmp, REQ_ACS_FLAGS))
555 continue;
556
557 group = get_pci_alias_group(tmp, devfns);
558 if (group) {
559 pci_dev_put(tmp);
560 return group;
561 }
562 }
563
564 return NULL;
565}
566
567/*
568 * Look for aliases to or from the given device for exisiting groups. The
569 * dma_alias_devfn only supports aliases on the same bus, therefore the search
570 * space is quite small (especially since we're really only looking at pcie
571 * device, and therefore only expect multiple slots on the root complex or
572 * downstream switch ports). It's conceivable though that a pair of
573 * multifunction devices could have aliases between them that would cause a
574 * loop. To prevent this, we use a bitmap to track where we've been.
575 */
576static struct iommu_group *get_pci_alias_group(struct pci_dev *pdev,
577 unsigned long *devfns)
578{
579 struct pci_dev *tmp = NULL;
580 struct iommu_group *group;
581
582 if (test_and_set_bit(pdev->devfn & 0xff, devfns))
583 return NULL;
584
585 group = iommu_group_get(&pdev->dev);
586 if (group)
587 return group;
588
589 for_each_pci_dev(tmp) {
590 if (tmp == pdev || tmp->bus != pdev->bus)
591 continue;
592
593 /* We alias them or they alias us */
594 if (((pdev->dev_flags & PCI_DEV_FLAGS_DMA_ALIAS_DEVFN) &&
595 pdev->dma_alias_devfn == tmp->devfn) ||
596 ((tmp->dev_flags & PCI_DEV_FLAGS_DMA_ALIAS_DEVFN) &&
597 tmp->dma_alias_devfn == pdev->devfn)) {
598
599 group = get_pci_alias_group(tmp, devfns);
600 if (group) {
601 pci_dev_put(tmp);
602 return group;
603 }
604
605 group = get_pci_function_alias_group(tmp, devfns);
606 if (group) {
607 pci_dev_put(tmp);
608 return group;
609 }
610 }
611 }
612
613 return NULL;
614}
615
532struct group_for_pci_data { 616struct group_for_pci_data {
533 struct pci_dev *pdev; 617 struct pci_dev *pdev;
534 struct iommu_group *group; 618 struct iommu_group *group;
@@ -557,7 +641,7 @@ static struct iommu_group *iommu_group_get_for_pci_dev(struct pci_dev *pdev)
557 struct group_for_pci_data data; 641 struct group_for_pci_data data;
558 struct pci_bus *bus; 642 struct pci_bus *bus;
559 struct iommu_group *group = NULL; 643 struct iommu_group *group = NULL;
560 struct pci_dev *tmp; 644 u64 devfns[4] = { 0 };
561 645
562 /* 646 /*
563 * Find the upstream DMA alias for the device. A device must not 647 * Find the upstream DMA alias for the device. A device must not
@@ -591,76 +675,21 @@ static struct iommu_group *iommu_group_get_for_pci_dev(struct pci_dev *pdev)
591 } 675 }
592 676
593 /* 677 /*
594 * Next we need to consider DMA alias quirks. If one device aliases 678 * Look for existing groups on device aliases. If we alias another
595 * to another, they should be grouped together. It's theoretically 679 * device or another device aliases us, use the same group.
596 * possible that aliases could create chains of devices where each
597 * device aliases another device. If we then factor in multifunction
598 * ACS grouping requirements, each alias could incorporate a new slot
599 * with multiple functions, each with aliases. This is all extremely
600 * unlikely as DMA alias quirks are typically only used for PCIe
601 * devices where we usually have a single slot per bus. Furthermore,
602 * the alias quirk is usually to another function within the slot
603 * (and ACS multifunction is not supported) or to a different slot
604 * that doesn't physically exist. The likely scenario is therefore
605 * that everything on the bus gets grouped together. To reduce the
606 * problem space, share the IOMMU group for all devices on the bus
607 * if a DMA alias quirk is present on the bus.
608 */
609 tmp = NULL;
610 for_each_pci_dev(tmp) {
611 if (tmp->bus != pdev->bus ||
612 !(tmp->dev_flags & PCI_DEV_FLAGS_DMA_ALIAS_DEVFN))
613 continue;
614
615 pci_dev_put(tmp);
616 tmp = NULL;
617
618 /* We have an alias quirk, search for an existing group */
619 for_each_pci_dev(tmp) {
620 struct iommu_group *group_tmp;
621
622 if (tmp->bus != pdev->bus)
623 continue;
624
625 group_tmp = iommu_group_get(&tmp->dev);
626 if (!group) {
627 group = group_tmp;
628 continue;
629 }
630
631 if (group_tmp) {
632 WARN_ON(group != group_tmp);
633 iommu_group_put(group_tmp);
634 }
635 }
636
637 return group ? group : iommu_group_alloc();
638 }
639
640 /*
641 * Non-multifunction devices or multifunction devices supporting
642 * ACS get their own group.
643 */ 680 */
644 if (!pdev->multifunction || pci_acs_enabled(pdev, REQ_ACS_FLAGS)) 681 group = get_pci_alias_group(pdev, (unsigned long *)devfns);
645 return iommu_group_alloc(); 682 if (group)
683 return group;
646 684
647 /* 685 /*
648 * Multifunction devices not supporting ACS share a group with other 686 * Look for existing groups on non-isolated functions on the same
649 * similar devices in the same slot. 687 * slot and aliases of those funcions, if any. No need to clear
688 * the search bitmap, the tested devfns are still valid.
650 */ 689 */
651 tmp = NULL; 690 group = get_pci_function_alias_group(pdev, (unsigned long *)devfns);
652 for_each_pci_dev(tmp) { 691 if (group)
653 if (tmp == pdev || tmp->bus != pdev->bus || 692 return group;
654 PCI_SLOT(tmp->devfn) != PCI_SLOT(pdev->devfn) ||
655 pci_acs_enabled(tmp, REQ_ACS_FLAGS))
656 continue;
657
658 group = iommu_group_get(&tmp->dev);
659 if (group) {
660 pci_dev_put(tmp);
661 return group;
662 }
663 }
664 693
665 /* No shared group found, allocate new */ 694 /* No shared group found, allocate new */
666 return iommu_group_alloc(); 695 return iommu_group_alloc();
@@ -678,15 +707,17 @@ static struct iommu_group *iommu_group_get_for_pci_dev(struct pci_dev *pdev)
678 */ 707 */
679struct iommu_group *iommu_group_get_for_dev(struct device *dev) 708struct iommu_group *iommu_group_get_for_dev(struct device *dev)
680{ 709{
681 struct iommu_group *group = ERR_PTR(-EIO); 710 struct iommu_group *group;
682 int ret; 711 int ret;
683 712
684 group = iommu_group_get(dev); 713 group = iommu_group_get(dev);
685 if (group) 714 if (group)
686 return group; 715 return group;
687 716
688 if (dev_is_pci(dev)) 717 if (!dev_is_pci(dev))
689 group = iommu_group_get_for_pci_dev(to_pci_dev(dev)); 718 return ERR_PTR(-EINVAL);
719
720 group = iommu_group_get_for_pci_dev(to_pci_dev(dev));
690 721
691 if (IS_ERR(group)) 722 if (IS_ERR(group))
692 return group; 723 return group;
@@ -768,18 +799,26 @@ static int iommu_bus_notifier(struct notifier_block *nb,
768 return 0; 799 return 0;
769} 800}
770 801
771static struct notifier_block iommu_bus_nb = { 802static int iommu_bus_init(struct bus_type *bus, const struct iommu_ops *ops)
772 .notifier_call = iommu_bus_notifier,
773};
774
775static void iommu_bus_init(struct bus_type *bus, const struct iommu_ops *ops)
776{ 803{
804 int err;
805 struct notifier_block *nb;
777 struct iommu_callback_data cb = { 806 struct iommu_callback_data cb = {
778 .ops = ops, 807 .ops = ops,
779 }; 808 };
780 809
781 bus_register_notifier(bus, &iommu_bus_nb); 810 nb = kzalloc(sizeof(struct notifier_block), GFP_KERNEL);
782 bus_for_each_dev(bus, NULL, &cb, add_iommu_group); 811 if (!nb)
812 return -ENOMEM;
813
814 nb->notifier_call = iommu_bus_notifier;
815
816 err = bus_register_notifier(bus, nb);
817 if (err) {
818 kfree(nb);
819 return err;
820 }
821 return bus_for_each_dev(bus, NULL, &cb, add_iommu_group);
783} 822}
784 823
785/** 824/**
@@ -803,9 +842,7 @@ int bus_set_iommu(struct bus_type *bus, const struct iommu_ops *ops)
803 bus->iommu_ops = ops; 842 bus->iommu_ops = ops;
804 843
805 /* Do IOMMU specific setup for this bus-type */ 844 /* Do IOMMU specific setup for this bus-type */
806 iommu_bus_init(bus, ops); 845 return iommu_bus_init(bus, ops);
807
808 return 0;
809} 846}
810EXPORT_SYMBOL_GPL(bus_set_iommu); 847EXPORT_SYMBOL_GPL(bus_set_iommu);
811 848
@@ -815,6 +852,15 @@ bool iommu_present(struct bus_type *bus)
815} 852}
816EXPORT_SYMBOL_GPL(iommu_present); 853EXPORT_SYMBOL_GPL(iommu_present);
817 854
855bool iommu_capable(struct bus_type *bus, enum iommu_cap cap)
856{
857 if (!bus->iommu_ops || !bus->iommu_ops->capable)
858 return false;
859
860 return bus->iommu_ops->capable(cap);
861}
862EXPORT_SYMBOL_GPL(iommu_capable);
863
818/** 864/**
819 * iommu_set_fault_handler() - set a fault handler for an iommu domain 865 * iommu_set_fault_handler() - set a fault handler for an iommu domain
820 * @domain: iommu domain 866 * @domain: iommu domain
@@ -945,16 +991,6 @@ phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova)
945} 991}
946EXPORT_SYMBOL_GPL(iommu_iova_to_phys); 992EXPORT_SYMBOL_GPL(iommu_iova_to_phys);
947 993
948int iommu_domain_has_cap(struct iommu_domain *domain,
949 unsigned long cap)
950{
951 if (unlikely(domain->ops->domain_has_cap == NULL))
952 return 0;
953
954 return domain->ops->domain_has_cap(domain, cap);
955}
956EXPORT_SYMBOL_GPL(iommu_domain_has_cap);
957
958static size_t iommu_pgsize(struct iommu_domain *domain, 994static size_t iommu_pgsize(struct iommu_domain *domain,
959 unsigned long addr_merge, size_t size) 995 unsigned long addr_merge, size_t size)
960{ 996{
@@ -995,7 +1031,7 @@ int iommu_map(struct iommu_domain *domain, unsigned long iova,
995 size_t orig_size = size; 1031 size_t orig_size = size;
996 int ret = 0; 1032 int ret = 0;
997 1033
998 if (unlikely(domain->ops->unmap == NULL || 1034 if (unlikely(domain->ops->map == NULL ||
999 domain->ops->pgsize_bitmap == 0UL)) 1035 domain->ops->pgsize_bitmap == 0UL))
1000 return -ENODEV; 1036 return -ENODEV;
1001 1037
diff --git a/drivers/iommu/irq_remapping.c b/drivers/iommu/irq_remapping.c
index 33c439524080..74a1767c89b5 100644
--- a/drivers/iommu/irq_remapping.c
+++ b/drivers/iommu/irq_remapping.c
@@ -12,6 +12,7 @@
12#include <asm/processor.h> 12#include <asm/processor.h>
13#include <asm/x86_init.h> 13#include <asm/x86_init.h>
14#include <asm/apic.h> 14#include <asm/apic.h>
15#include <asm/hpet.h>
15 16
16#include "irq_remapping.h" 17#include "irq_remapping.h"
17 18
@@ -345,10 +346,16 @@ static int msi_setup_remapped_irq(struct pci_dev *pdev, unsigned int irq,
345 346
346int setup_hpet_msi_remapped(unsigned int irq, unsigned int id) 347int setup_hpet_msi_remapped(unsigned int irq, unsigned int id)
347{ 348{
348 if (!remap_ops || !remap_ops->setup_hpet_msi) 349 int ret;
350
351 if (!remap_ops || !remap_ops->alloc_hpet_msi)
349 return -ENODEV; 352 return -ENODEV;
350 353
351 return remap_ops->setup_hpet_msi(irq, id); 354 ret = remap_ops->alloc_hpet_msi(irq, id);
355 if (ret)
356 return -EINVAL;
357
358 return default_setup_hpet_msi(irq, id);
352} 359}
353 360
354void panic_if_irq_remap(const char *msg) 361void panic_if_irq_remap(const char *msg)
diff --git a/drivers/iommu/irq_remapping.h b/drivers/iommu/irq_remapping.h
index 90c4dae5a46b..fde250f86e60 100644
--- a/drivers/iommu/irq_remapping.h
+++ b/drivers/iommu/irq_remapping.h
@@ -80,7 +80,7 @@ struct irq_remap_ops {
80 int (*msi_setup_irq)(struct pci_dev *, unsigned int, int, int); 80 int (*msi_setup_irq)(struct pci_dev *, unsigned int, int, int);
81 81
82 /* Setup interrupt remapping for an HPET MSI */ 82 /* Setup interrupt remapping for an HPET MSI */
83 int (*setup_hpet_msi)(unsigned int, unsigned int); 83 int (*alloc_hpet_msi)(unsigned int, unsigned int);
84}; 84};
85 85
86extern struct irq_remap_ops intel_irq_remap_ops; 86extern struct irq_remap_ops intel_irq_remap_ops;
diff --git a/drivers/iommu/msm_iommu.c b/drivers/iommu/msm_iommu.c
index 49f41d6e02f1..6e3dcc289d59 100644
--- a/drivers/iommu/msm_iommu.c
+++ b/drivers/iommu/msm_iommu.c
@@ -603,10 +603,9 @@ fail:
603 return ret; 603 return ret;
604} 604}
605 605
606static int msm_iommu_domain_has_cap(struct iommu_domain *domain, 606static bool msm_iommu_capable(enum iommu_cap cap)
607 unsigned long cap)
608{ 607{
609 return 0; 608 return false;
610} 609}
611 610
612static void print_ctx_regs(void __iomem *base, int ctx) 611static void print_ctx_regs(void __iomem *base, int ctx)
@@ -675,6 +674,7 @@ fail:
675} 674}
676 675
677static const struct iommu_ops msm_iommu_ops = { 676static const struct iommu_ops msm_iommu_ops = {
677 .capable = msm_iommu_capable,
678 .domain_init = msm_iommu_domain_init, 678 .domain_init = msm_iommu_domain_init,
679 .domain_destroy = msm_iommu_domain_destroy, 679 .domain_destroy = msm_iommu_domain_destroy,
680 .attach_dev = msm_iommu_attach_dev, 680 .attach_dev = msm_iommu_attach_dev,
@@ -682,7 +682,6 @@ static const struct iommu_ops msm_iommu_ops = {
682 .map = msm_iommu_map, 682 .map = msm_iommu_map,
683 .unmap = msm_iommu_unmap, 683 .unmap = msm_iommu_unmap,
684 .iova_to_phys = msm_iommu_iova_to_phys, 684 .iova_to_phys = msm_iommu_iova_to_phys,
685 .domain_has_cap = msm_iommu_domain_has_cap,
686 .pgsize_bitmap = MSM_IOMMU_PGSIZES, 685 .pgsize_bitmap = MSM_IOMMU_PGSIZES,
687}; 686};
688 687
diff --git a/drivers/iommu/omap-iommu.c b/drivers/iommu/omap-iommu.c
index e202b0c24120..36278870e84a 100644
--- a/drivers/iommu/omap-iommu.c
+++ b/drivers/iommu/omap-iommu.c
@@ -26,6 +26,7 @@
26#include <linux/of.h> 26#include <linux/of.h>
27#include <linux/of_iommu.h> 27#include <linux/of_iommu.h>
28#include <linux/of_irq.h> 28#include <linux/of_irq.h>
29#include <linux/of_platform.h>
29 30
30#include <asm/cacheflush.h> 31#include <asm/cacheflush.h>
31 32
@@ -892,19 +893,11 @@ static struct omap_iommu *omap_iommu_attach(const char *name, u32 *iopgd)
892 goto err_enable; 893 goto err_enable;
893 flush_iotlb_all(obj); 894 flush_iotlb_all(obj);
894 895
895 if (!try_module_get(obj->owner)) {
896 err = -ENODEV;
897 goto err_module;
898 }
899
900 spin_unlock(&obj->iommu_lock); 896 spin_unlock(&obj->iommu_lock);
901 897
902 dev_dbg(obj->dev, "%s: %s\n", __func__, obj->name); 898 dev_dbg(obj->dev, "%s: %s\n", __func__, obj->name);
903 return obj; 899 return obj;
904 900
905err_module:
906 if (obj->refcount == 1)
907 iommu_disable(obj);
908err_enable: 901err_enable:
909 obj->refcount--; 902 obj->refcount--;
910 spin_unlock(&obj->iommu_lock); 903 spin_unlock(&obj->iommu_lock);
@@ -925,8 +918,6 @@ static void omap_iommu_detach(struct omap_iommu *obj)
925 if (--obj->refcount == 0) 918 if (--obj->refcount == 0)
926 iommu_disable(obj); 919 iommu_disable(obj);
927 920
928 module_put(obj->owner);
929
930 obj->iopgd = NULL; 921 obj->iopgd = NULL;
931 922
932 spin_unlock(&obj->iommu_lock); 923 spin_unlock(&obj->iommu_lock);
@@ -1006,7 +997,7 @@ static int omap_iommu_remove(struct platform_device *pdev)
1006 return 0; 997 return 0;
1007} 998}
1008 999
1009static struct of_device_id omap_iommu_of_match[] = { 1000static const struct of_device_id omap_iommu_of_match[] = {
1010 { .compatible = "ti,omap2-iommu" }, 1001 { .compatible = "ti,omap2-iommu" },
1011 { .compatible = "ti,omap4-iommu" }, 1002 { .compatible = "ti,omap4-iommu" },
1012 { .compatible = "ti,dra7-iommu" }, 1003 { .compatible = "ti,dra7-iommu" },
@@ -1091,6 +1082,11 @@ omap_iommu_attach_dev(struct iommu_domain *domain, struct device *dev)
1091 struct omap_iommu_arch_data *arch_data = dev->archdata.iommu; 1082 struct omap_iommu_arch_data *arch_data = dev->archdata.iommu;
1092 int ret = 0; 1083 int ret = 0;
1093 1084
1085 if (!arch_data || !arch_data->name) {
1086 dev_err(dev, "device doesn't have an associated iommu\n");
1087 return -EINVAL;
1088 }
1089
1094 spin_lock(&omap_domain->lock); 1090 spin_lock(&omap_domain->lock);
1095 1091
1096 /* only a single device is supported per domain for now */ 1092 /* only a single device is supported per domain for now */
@@ -1239,6 +1235,7 @@ static int omap_iommu_add_device(struct device *dev)
1239{ 1235{
1240 struct omap_iommu_arch_data *arch_data; 1236 struct omap_iommu_arch_data *arch_data;
1241 struct device_node *np; 1237 struct device_node *np;
1238 struct platform_device *pdev;
1242 1239
1243 /* 1240 /*
1244 * Allocate the archdata iommu structure for DT-based devices. 1241 * Allocate the archdata iommu structure for DT-based devices.
@@ -1253,13 +1250,19 @@ static int omap_iommu_add_device(struct device *dev)
1253 if (!np) 1250 if (!np)
1254 return 0; 1251 return 0;
1255 1252
1253 pdev = of_find_device_by_node(np);
1254 if (WARN_ON(!pdev)) {
1255 of_node_put(np);
1256 return -EINVAL;
1257 }
1258
1256 arch_data = kzalloc(sizeof(*arch_data), GFP_KERNEL); 1259 arch_data = kzalloc(sizeof(*arch_data), GFP_KERNEL);
1257 if (!arch_data) { 1260 if (!arch_data) {
1258 of_node_put(np); 1261 of_node_put(np);
1259 return -ENOMEM; 1262 return -ENOMEM;
1260 } 1263 }
1261 1264
1262 arch_data->name = kstrdup(dev_name(dev), GFP_KERNEL); 1265 arch_data->name = kstrdup(dev_name(&pdev->dev), GFP_KERNEL);
1263 dev->archdata.iommu = arch_data; 1266 dev->archdata.iommu = arch_data;
1264 1267
1265 of_node_put(np); 1268 of_node_put(np);
diff --git a/drivers/iommu/omap-iommu.h b/drivers/iommu/omap-iommu.h
index 1275a822934b..4f1b68c08c15 100644
--- a/drivers/iommu/omap-iommu.h
+++ b/drivers/iommu/omap-iommu.h
@@ -28,7 +28,6 @@ struct iotlb_entry {
28 28
29struct omap_iommu { 29struct omap_iommu {
30 const char *name; 30 const char *name;
31 struct module *owner;
32 void __iomem *regbase; 31 void __iomem *regbase;
33 struct device *dev; 32 struct device *dev;
34 void *isr_priv; 33 void *isr_priv;
diff --git a/drivers/iommu/tegra-gart.c b/drivers/iommu/tegra-gart.c
index b10a8ecede8e..a6d76abf2c06 100644
--- a/drivers/iommu/tegra-gart.c
+++ b/drivers/iommu/tegra-gart.c
@@ -303,13 +303,13 @@ static phys_addr_t gart_iommu_iova_to_phys(struct iommu_domain *domain,
303 return pa; 303 return pa;
304} 304}
305 305
306static int gart_iommu_domain_has_cap(struct iommu_domain *domain, 306static bool gart_iommu_capable(enum iommu_cap cap)
307 unsigned long cap)
308{ 307{
309 return 0; 308 return false;
310} 309}
311 310
312static const struct iommu_ops gart_iommu_ops = { 311static const struct iommu_ops gart_iommu_ops = {
312 .capable = gart_iommu_capable,
313 .domain_init = gart_iommu_domain_init, 313 .domain_init = gart_iommu_domain_init,
314 .domain_destroy = gart_iommu_domain_destroy, 314 .domain_destroy = gart_iommu_domain_destroy,
315 .attach_dev = gart_iommu_attach_dev, 315 .attach_dev = gart_iommu_attach_dev,
@@ -317,7 +317,6 @@ static const struct iommu_ops gart_iommu_ops = {
317 .map = gart_iommu_map, 317 .map = gart_iommu_map,
318 .unmap = gart_iommu_unmap, 318 .unmap = gart_iommu_unmap,
319 .iova_to_phys = gart_iommu_iova_to_phys, 319 .iova_to_phys = gart_iommu_iova_to_phys,
320 .domain_has_cap = gart_iommu_domain_has_cap,
321 .pgsize_bitmap = GART_IOMMU_PGSIZES, 320 .pgsize_bitmap = GART_IOMMU_PGSIZES,
322}; 321};
323 322
@@ -416,7 +415,7 @@ static const struct dev_pm_ops tegra_gart_pm_ops = {
416 .resume = tegra_gart_resume, 415 .resume = tegra_gart_resume,
417}; 416};
418 417
419static struct of_device_id tegra_gart_of_match[] = { 418static const struct of_device_id tegra_gart_of_match[] = {
420 { .compatible = "nvidia,tegra20-gart", }, 419 { .compatible = "nvidia,tegra20-gart", },
421 { }, 420 { },
422}; 421};
diff --git a/drivers/iommu/tegra-smmu.c b/drivers/iommu/tegra-smmu.c
index 3ded3894623c..3afdf43f732a 100644
--- a/drivers/iommu/tegra-smmu.c
+++ b/drivers/iommu/tegra-smmu.c
@@ -780,10 +780,9 @@ static phys_addr_t smmu_iommu_iova_to_phys(struct iommu_domain *domain,
780 return PFN_PHYS(pfn); 780 return PFN_PHYS(pfn);
781} 781}
782 782
783static int smmu_iommu_domain_has_cap(struct iommu_domain *domain, 783static bool smmu_iommu_capable(enum iommu_cap cap)
784 unsigned long cap)
785{ 784{
786 return 0; 785 return false;
787} 786}
788 787
789static int smmu_iommu_attach_dev(struct iommu_domain *domain, 788static int smmu_iommu_attach_dev(struct iommu_domain *domain,
@@ -949,6 +948,7 @@ static void smmu_iommu_domain_destroy(struct iommu_domain *domain)
949} 948}
950 949
951static const struct iommu_ops smmu_iommu_ops = { 950static const struct iommu_ops smmu_iommu_ops = {
951 .capable = smmu_iommu_capable,
952 .domain_init = smmu_iommu_domain_init, 952 .domain_init = smmu_iommu_domain_init,
953 .domain_destroy = smmu_iommu_domain_destroy, 953 .domain_destroy = smmu_iommu_domain_destroy,
954 .attach_dev = smmu_iommu_attach_dev, 954 .attach_dev = smmu_iommu_attach_dev,
@@ -956,7 +956,6 @@ static const struct iommu_ops smmu_iommu_ops = {
956 .map = smmu_iommu_map, 956 .map = smmu_iommu_map,
957 .unmap = smmu_iommu_unmap, 957 .unmap = smmu_iommu_unmap,
958 .iova_to_phys = smmu_iommu_iova_to_phys, 958 .iova_to_phys = smmu_iommu_iova_to_phys,
959 .domain_has_cap = smmu_iommu_domain_has_cap,
960 .pgsize_bitmap = SMMU_IOMMU_PGSIZES, 959 .pgsize_bitmap = SMMU_IOMMU_PGSIZES,
961}; 960};
962 961
@@ -1260,7 +1259,7 @@ static const struct dev_pm_ops tegra_smmu_pm_ops = {
1260 .resume = tegra_smmu_resume, 1259 .resume = tegra_smmu_resume,
1261}; 1260};
1262 1261
1263static struct of_device_id tegra_smmu_of_match[] = { 1262static const struct of_device_id tegra_smmu_of_match[] = {
1264 { .compatible = "nvidia,tegra30-smmu", }, 1263 { .compatible = "nvidia,tegra30-smmu", },
1265 { }, 1264 { },
1266}; 1265};
generated by cgit v1.2.2 (git 2.25.0) at 2025-01-29 22:14:54 -0500