Merge tag 'iommu-updates-v4.7' of git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu

Pull IOMMU updates from Joerg Roedel:
 "The updates include:

   - rate limiting for the VT-d fault handler

   - remove statistics code from the AMD IOMMU driver.  It is unused and
     should be replaced by something more generic if needed

   - per-domain pagesize-bitmaps in IOMMU core code to support systems
     with different types of IOMMUs

   - support for ACPI devices in the AMD IOMMU driver

   - 4GB mode support for Mediatek IOMMU driver

   - ARM-SMMU updates from Will Deacon:
      - support for 64k pages with SMMUv1 implementations (e.g MMU-401)
      - remove open-coded 64-bit MMIO accessors
      - initial support for 16-bit VMIDs, as supported by some ThunderX
        SMMU implementations
      - a couple of errata workarounds for silicon in the field

   - various fixes here and there"

* tag 'iommu-updates-v4.7' of git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu: (44 commits)
  iommu/arm-smmu: Use per-domain page sizes.
  iommu/amd: Remove statistics code
  iommu/dma: Finish optimising higher-order allocations
  iommu: Allow selecting page sizes per domain
  iommu: of: enforce const-ness of struct iommu_ops
  iommu: remove unused priv field from struct iommu_ops
  iommu/dma: Implement scatterlist segment merging
  iommu/arm-smmu: Clear cache lock bit of ACR
  iommu/arm-smmu: Support SMMUv1 64KB supplement
  iommu/arm-smmu: Decouple context format from kernel config
  iommu/arm-smmu: Tidy up 64-bit/atomic I/O accesses
  io-64-nonatomic: Add relaxed accessor variants
  iommu/arm-smmu: Work around MMU-500 prefetch errata
  iommu/arm-smmu: Convert ThunderX workaround to new method
  iommu/arm-smmu: Differentiate specific implementations
  iommu/arm-smmu: Workaround for ThunderX erratum #27704
  iommu/arm-smmu: Add support for 16 bit VMID
  iommu/amd: Move get_device_id() and friends to beginning of file
  iommu/amd: Don't use IS_ERR_VALUE to check integer values
  iommu/amd: Signedness bug in acpihid_device_group()
  ...

1 files changed, 149 insertions, 118 deletions

diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c
index 3839fd2865a6..634f636393d5 100644
--- a/drivers/iommu/amd_iommu.c
+++ b/drivers/iommu/amd_iommu.c
@@ -19,6 +19,8 @@
 
 #include <linux/ratelimit.h>
 #include <linux/pci.h>
+#include <linux/acpi.h>
+#include <linux/amba/bus.h>
 #include <linux/pci-ats.h>
 #include <linux/bitmap.h>
 #include <linux/slab.h>
@@ -72,6 +74,7 @@ static DEFINE_SPINLOCK(dev_data_list_lock);
 
 LIST_HEAD(ioapic_map);
 LIST_HEAD(hpet_map);
+LIST_HEAD(acpihid_map);
 
 /*
  * Domain for untranslated devices - only allocated
@@ -162,18 +165,65 @@ struct dma_ops_domain {
  *
  ****************************************************************************/
 
-static struct protection_domain *to_pdomain(struct iommu_domain *dom)
+static inline int match_hid_uid(struct device *dev,
+                                struct acpihid_map_entry *entry)
 {
-        return container_of(dom, struct protection_domain, domain);
+        const char *hid, *uid;
+
+        hid = acpi_device_hid(ACPI_COMPANION(dev));
+        uid = acpi_device_uid(ACPI_COMPANION(dev));
+
+        if (!hid || !(*hid))
+                return -ENODEV;
+
+        if (!uid || !(*uid))
+                return strcmp(hid, entry->hid);
+
+        if (!(*entry->uid))
+                return strcmp(hid, entry->hid);
+
+        return (strcmp(hid, entry->hid) || strcmp(uid, entry->uid));
 }
 
-static inline u16 get_device_id(struct device *dev)
+static inline u16 get_pci_device_id(struct device *dev)
 {
         struct pci_dev *pdev = to_pci_dev(dev);
 
         return PCI_DEVID(pdev->bus->number, pdev->devfn);
 }
 
+static inline int get_acpihid_device_id(struct device *dev,
+                                        struct acpihid_map_entry **entry)
+{
+        struct acpihid_map_entry *p;
+
+        list_for_each_entry(p, &acpihid_map, list) {
+                if (!match_hid_uid(dev, p)) {
+                        if (entry)
+                                *entry = p;
+                        return p->devid;
+                }
+        }
+        return -EINVAL;
+}
+
+static inline int get_device_id(struct device *dev)
+{
+        int devid;
+
+        if (dev_is_pci(dev))
+                devid = get_pci_device_id(dev);
+        else
+                devid = get_acpihid_device_id(dev, NULL);
+
+        return devid;
+}
+
+static struct protection_domain *to_pdomain(struct iommu_domain *dom)
+{
+        return container_of(dom, struct protection_domain, domain);
+}
+
 static struct iommu_dev_data *alloc_dev_data(u16 devid)
 {
         struct iommu_dev_data *dev_data;
@@ -222,6 +272,7 @@ static u16 get_alias(struct device *dev)
         struct pci_dev *pdev = to_pci_dev(dev);
         u16 devid, ivrs_alias, pci_alias;
 
+        /* The callers make sure that get_device_id() does not fail here */
         devid = get_device_id(dev);
         ivrs_alias = amd_iommu_alias_table[devid];
         pci_for_each_dma_alias(pdev, __last_alias, &pci_alias);
@@ -289,6 +340,29 @@ static struct iommu_dev_data *get_dev_data(struct device *dev)
         return dev->archdata.iommu;
 }
 
+/*
+* Find or create an IOMMU group for a acpihid device.
+*/
+static struct iommu_group *acpihid_device_group(struct device *dev)
+{
+        struct acpihid_map_entry *p, *entry = NULL;
+        int devid;
+
+        devid = get_acpihid_device_id(dev, &entry);
+        if (devid < 0)
+                return ERR_PTR(devid);
+
+        list_for_each_entry(p, &acpihid_map, list) {
+                if ((devid == p->devid) && p->group)
+                        entry->group = p->group;
+        }
+
+        if (!entry->group)
+                entry->group = generic_device_group(dev);
+
+        return entry->group;
+}
+
 static bool pci_iommuv2_capable(struct pci_dev *pdev)
 {
         static const int caps[] = {
@@ -340,9 +414,11 @@ static void init_unity_mappings_for_device(struct device *dev,
                                            struct dma_ops_domain *dma_dom)
 {
         struct unity_map_entry *e;
-        u16 devid;
+        int devid;
 
         devid = get_device_id(dev);
+        if (devid < 0)
+                return;
 
         list_for_each_entry(e, &amd_iommu_unity_map, list) {
                 if (!(devid >= e->devid_start && devid <= e->devid_end))
@@ -357,16 +433,14 @@ static void init_unity_mappings_for_device(struct device *dev,
  */
 static bool check_device(struct device *dev)
 {
-        u16 devid;
+        int devid;
 
         if (!dev || !dev->dma_mask)
                 return false;
 
-        /* No PCI device */
-        if (!dev_is_pci(dev))
-                return false;
-
         devid = get_device_id(dev);
+        if (devid < 0)
+                return false;
 
         /* Out of our scope? */
         if (devid > amd_iommu_last_bdf)
@@ -401,22 +475,26 @@ out:
 
 static int iommu_init_device(struct device *dev)
 {
-        struct pci_dev *pdev = to_pci_dev(dev);
         struct iommu_dev_data *dev_data;
+        int devid;
 
         if (dev->archdata.iommu)
                 return 0;
 
-        dev_data = find_dev_data(get_device_id(dev));
+        devid = get_device_id(dev);
+        if (devid < 0)
+                return devid;
+
+        dev_data = find_dev_data(devid);
         if (!dev_data)
                 return -ENOMEM;
 
         dev_data->alias = get_alias(dev);
 
-        if (pci_iommuv2_capable(pdev)) {
+        if (dev_is_pci(dev) && pci_iommuv2_capable(to_pci_dev(dev))) {
                 struct amd_iommu *iommu;
 
-                iommu              = amd_iommu_rlookup_table[dev_data->devid];
+                iommu = amd_iommu_rlookup_table[dev_data->devid];
                 dev_data->iommu_v2 = iommu->is_iommu_v2;
         }
 
@@ -430,9 +508,13 @@ static int iommu_init_device(struct device *dev)
 
 static void iommu_ignore_device(struct device *dev)
 {
-        u16 devid, alias;
+        u16 alias;
+        int devid;
 
         devid = get_device_id(dev);
+        if (devid < 0)
+                return;
+
         alias = get_alias(dev);
 
         memset(&amd_iommu_dev_table[devid], 0, sizeof(struct dev_table_entry));
@@ -444,8 +526,14 @@ static void iommu_ignore_device(struct device *dev)
 
 static void iommu_uninit_device(struct device *dev)
 {
-        struct iommu_dev_data *dev_data = search_dev_data(get_device_id(dev));
+        int devid;
+        struct iommu_dev_data *dev_data;
 
+        devid = get_device_id(dev);
+        if (devid < 0)
+                return;
+
+        dev_data = search_dev_data(devid);
         if (!dev_data)
                 return;
 
@@ -466,70 +554,6 @@ static void iommu_uninit_device(struct device *dev)
          */
 }
 
-#ifdef CONFIG_AMD_IOMMU_STATS
-
-/*
- * Initialization code for statistics collection
- */
-
-DECLARE_STATS_COUNTER(compl_wait);
-DECLARE_STATS_COUNTER(cnt_map_single);
-DECLARE_STATS_COUNTER(cnt_unmap_single);
-DECLARE_STATS_COUNTER(cnt_map_sg);
-DECLARE_STATS_COUNTER(cnt_unmap_sg);
-DECLARE_STATS_COUNTER(cnt_alloc_coherent);
-DECLARE_STATS_COUNTER(cnt_free_coherent);
-DECLARE_STATS_COUNTER(cross_page);
-DECLARE_STATS_COUNTER(domain_flush_single);
-DECLARE_STATS_COUNTER(domain_flush_all);
-DECLARE_STATS_COUNTER(alloced_io_mem);
-DECLARE_STATS_COUNTER(total_map_requests);
-DECLARE_STATS_COUNTER(complete_ppr);
-DECLARE_STATS_COUNTER(invalidate_iotlb);
-DECLARE_STATS_COUNTER(invalidate_iotlb_all);
-DECLARE_STATS_COUNTER(pri_requests);
-
-static struct dentry *stats_dir;
-static struct dentry *de_fflush;
-
-static void amd_iommu_stats_add(struct __iommu_counter *cnt)
-{
-        if (stats_dir == NULL)
-                return;
-
-        cnt->dent = debugfs_create_u64(cnt->name, 0444, stats_dir,
-                                       &cnt->value);
-}
-
-static void amd_iommu_stats_init(void)
-{
-        stats_dir = debugfs_create_dir("amd-iommu", NULL);
-        if (stats_dir == NULL)
-                return;
-
-        de_fflush  = debugfs_create_bool("fullflush", 0444, stats_dir,
-                                         &amd_iommu_unmap_flush);
-
-        amd_iommu_stats_add(&compl_wait);
-        amd_iommu_stats_add(&cnt_map_single);
-        amd_iommu_stats_add(&cnt_unmap_single);
-        amd_iommu_stats_add(&cnt_map_sg);
-        amd_iommu_stats_add(&cnt_unmap_sg);
-        amd_iommu_stats_add(&cnt_alloc_coherent);
-        amd_iommu_stats_add(&cnt_free_coherent);
-        amd_iommu_stats_add(&cross_page);
-        amd_iommu_stats_add(&domain_flush_single);
-        amd_iommu_stats_add(&domain_flush_all);
-        amd_iommu_stats_add(&alloced_io_mem);
-        amd_iommu_stats_add(&total_map_requests);
-        amd_iommu_stats_add(&complete_ppr);
-        amd_iommu_stats_add(&invalidate_iotlb);
-        amd_iommu_stats_add(&invalidate_iotlb_all);
-        amd_iommu_stats_add(&pri_requests);
-}
-
-#endif
-
 /****************************************************************************
  *
  * Interrupt handling functions
@@ -652,8 +676,6 @@ static void iommu_handle_ppr_entry(struct amd_iommu *iommu, u64 *raw)
 {
         struct amd_iommu_fault fault;
 
-        INC_STATS_COUNTER(pri_requests);
-
         if (PPR_REQ_TYPE(raw[0]) != PPR_REQ_FAULT) {
                 pr_err_ratelimited("AMD-Vi: Unknown PPR request received\n");
                 return;
@@ -2283,13 +2305,17 @@ static bool pci_pri_tlp_required(struct pci_dev *pdev)
 static int attach_device(struct device *dev,
                          struct protection_domain *domain)
 {
-        struct pci_dev *pdev = to_pci_dev(dev);
+        struct pci_dev *pdev;
         struct iommu_dev_data *dev_data;
         unsigned long flags;
         int ret;
 
         dev_data = get_dev_data(dev);
 
+        if (!dev_is_pci(dev))
+                goto skip_ats_check;
+
+        pdev = to_pci_dev(dev);
         if (domain->flags & PD_IOMMUV2_MASK) {
                 if (!dev_data->passthrough)
                         return -EINVAL;
@@ -2308,6 +2334,7 @@ static int attach_device(struct device *dev,
                 dev_data->ats.qdep    = pci_ats_queue_depth(pdev);
         }
 
+skip_ats_check:
         write_lock_irqsave(&amd_iommu_devtable_lock, flags);
         ret = __attach_device(dev_data, domain);
         write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
@@ -2364,6 +2391,9 @@ static void detach_device(struct device *dev)
         __detach_device(dev_data);
         write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
 
+        if (!dev_is_pci(dev))
+                return;
+
         if (domain->flags & PD_IOMMUV2_MASK && dev_data->iommu_v2)
                 pdev_iommuv2_disable(to_pci_dev(dev));
         else if (dev_data->ats.enabled)
@@ -2377,13 +2407,15 @@ static int amd_iommu_add_device(struct device *dev)
         struct iommu_dev_data *dev_data;
         struct iommu_domain *domain;
         struct amd_iommu *iommu;
-        u16 devid;
-        int ret;
+        int ret, devid;
 
         if (!check_device(dev) || get_dev_data(dev))
                 return 0;
 
         devid = get_device_id(dev);
+        if (devid < 0)
+                return devid;
+
         iommu = amd_iommu_rlookup_table[devid];
 
         ret = iommu_init_device(dev);
@@ -2421,18 +2453,29 @@ out:
 static void amd_iommu_remove_device(struct device *dev)
 {
         struct amd_iommu *iommu;
-        u16 devid;
+        int devid;
 
         if (!check_device(dev))
                 return;
 
         devid = get_device_id(dev);
+        if (devid < 0)
+                return;
+
         iommu = amd_iommu_rlookup_table[devid];
 
         iommu_uninit_device(dev);
         iommu_completion_wait(iommu);
 }
 
+static struct iommu_group *amd_iommu_device_group(struct device *dev)
+{
+        if (dev_is_pci(dev))
+                return pci_device_group(dev);
+
+        return acpihid_device_group(dev);
+}
+
 /*****************************************************************************
  *
  * The next functions belong to the dma_ops mapping/unmapping code.
@@ -2597,11 +2640,6 @@ static dma_addr_t __map_single(struct device *dev,
         pages = iommu_num_pages(paddr, size, PAGE_SIZE);
         paddr &= PAGE_MASK;
 
-        INC_STATS_COUNTER(total_map_requests);
-
-        if (pages > 1)
-                INC_STATS_COUNTER(cross_page);
-
         if (align)
                 align_mask = (1UL << get_order(size)) - 1;
 
@@ -2622,8 +2660,6 @@ static dma_addr_t __map_single(struct device *dev,
         }
         address += offset;
 
-        ADD_STATS_COUNTER(alloced_io_mem, size);
-
         if (unlikely(amd_iommu_np_cache)) {
                 domain_flush_pages(&dma_dom->domain, address, size);
                 domain_flush_complete(&dma_dom->domain);
@@ -2671,8 +2707,6 @@ static void __unmap_single(struct dma_ops_domain *dma_dom,
                 start += PAGE_SIZE;
         }
 
-        SUB_STATS_COUNTER(alloced_io_mem, size);
-
         dma_ops_free_addresses(dma_dom, dma_addr, pages);
 }
 
@@ -2688,8 +2722,6 @@ static dma_addr_t map_page(struct device *dev, struct page *page,
         struct protection_domain *domain;
         u64 dma_mask;
 
-        INC_STATS_COUNTER(cnt_map_single);
-
         domain = get_domain(dev);
         if (PTR_ERR(domain) == -EINVAL)
                 return (dma_addr_t)paddr;
@@ -2710,8 +2742,6 @@ static void unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size,
 {
         struct protection_domain *domain;
 
-        INC_STATS_COUNTER(cnt_unmap_single);
-
         domain = get_domain(dev);
         if (IS_ERR(domain))
                 return;
@@ -2734,8 +2764,6 @@ static int map_sg(struct device *dev, struct scatterlist *sglist,
         int mapped_elems = 0;
         u64 dma_mask;
 
-        INC_STATS_COUNTER(cnt_map_sg);
-
         domain = get_domain(dev);
         if (IS_ERR(domain))
                 return 0;
@@ -2781,8 +2809,6 @@ static void unmap_sg(struct device *dev, struct scatterlist *sglist,
         struct scatterlist *s;
         int i;
 
-        INC_STATS_COUNTER(cnt_unmap_sg);
-
         domain = get_domain(dev);
         if (IS_ERR(domain))
                 return;
@@ -2805,8 +2831,6 @@ static void *alloc_coherent(struct device *dev, size_t size,
         struct protection_domain *domain;
         struct page *page;
 
-        INC_STATS_COUNTER(cnt_alloc_coherent);
-
         domain = get_domain(dev);
         if (PTR_ERR(domain) == -EINVAL) {
                 page = alloc_pages(flag, get_order(size));
@@ -2860,8 +2884,6 @@ static void free_coherent(struct device *dev, size_t size,
         struct protection_domain *domain;
         struct page *page;
 
-        INC_STATS_COUNTER(cnt_free_coherent);
-
         page = virt_to_page(virt_addr);
         size = PAGE_ALIGN(size);
 
@@ -2926,7 +2948,17 @@ static struct dma_map_ops amd_iommu_dma_ops = {
 
 int __init amd_iommu_init_api(void)
 {
-        return bus_set_iommu(&pci_bus_type, &amd_iommu_ops);
+        int err = 0;
+
+        err = bus_set_iommu(&pci_bus_type, &amd_iommu_ops);
+        if (err)
+                return err;
+#ifdef CONFIG_ARM_AMBA
+        err = bus_set_iommu(&amba_bustype, &amd_iommu_ops);
+        if (err)
+                return err;
+#endif
+        return 0;
 }
 
 int __init amd_iommu_init_dma_ops(void)
@@ -2943,8 +2975,6 @@ int __init amd_iommu_init_dma_ops(void)
         if (!swiotlb)
                 dma_ops = &nommu_dma_ops;
 
-        amd_iommu_stats_init();
-
         if (amd_iommu_unmap_flush)
                 pr_info("AMD-Vi: IO/TLB flush on unmap enabled\n");
         else
@@ -3098,12 +3128,14 @@ static void amd_iommu_detach_device(struct iommu_domain *dom,
 {
         struct iommu_dev_data *dev_data = dev->archdata.iommu;
         struct amd_iommu *iommu;
-        u16 devid;
+        int devid;
 
         if (!check_device(dev))
                 return;
 
         devid = get_device_id(dev);
+        if (devid < 0)
+                return;
 
         if (dev_data->domain != NULL)
                 detach_device(dev);
@@ -3221,9 +3253,11 @@ static void amd_iommu_get_dm_regions(struct device *dev,
                                      struct list_head *head)
 {
         struct unity_map_entry *entry;
-        u16 devid;
+        int devid;
 
         devid = get_device_id(dev);
+        if (devid < 0)
+                return;
 
         list_for_each_entry(entry, &amd_iommu_unity_map, list) {
                 struct iommu_dm_region *region;
@@ -3270,7 +3304,7 @@ static const struct iommu_ops amd_iommu_ops = {
         .iova_to_phys = amd_iommu_iova_to_phys,
         .add_device = amd_iommu_add_device,
         .remove_device = amd_iommu_remove_device,
-        .device_group = pci_device_group,
+        .device_group = amd_iommu_device_group,
         .get_dm_regions = amd_iommu_get_dm_regions,
         .put_dm_regions = amd_iommu_put_dm_regions,
         .pgsize_bitmap  = AMD_IOMMU_PGSIZES,
@@ -3431,8 +3465,6 @@ out:
 static int __amd_iommu_flush_page(struct protection_domain *domain, int pasid,
                                   u64 address)
 {
-        INC_STATS_COUNTER(invalidate_iotlb);
-
         return __flush_pasid(domain, pasid, address, false);
 }
 
@@ -3453,8 +3485,6 @@ EXPORT_SYMBOL(amd_iommu_flush_page);
 
 static int __amd_iommu_flush_tlb(struct protection_domain *domain, int pasid)
 {
-        INC_STATS_COUNTER(invalidate_iotlb_all);
-
         return __flush_pasid(domain, pasid, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,
                              true);
 }
@@ -3574,8 +3604,6 @@ int amd_iommu_complete_ppr(struct pci_dev *pdev, int pasid,
         struct amd_iommu *iommu;
         struct iommu_cmd cmd;
 
-        INC_STATS_COUNTER(complete_ppr);
-
         dev_data = get_dev_data(&pdev->dev);
         iommu    = amd_iommu_rlookup_table[dev_data->devid];
 
@@ -3925,6 +3953,9 @@ static struct irq_domain *get_irq_domain(struct irq_alloc_info *info)
         case X86_IRQ_ALLOC_TYPE_MSI:
         case X86_IRQ_ALLOC_TYPE_MSIX:
                 devid = get_device_id(&info->msi_dev->dev);
+                if (devid < 0)
+                        return NULL;
+
                 iommu = amd_iommu_rlookup_table[devid];
                 if (iommu)
                         return iommu->msi_domain;