Merge tag 'iommu-updates-v5.3' of git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu

Pull iommu updates from Joerg Roedel:

 - Make the dma-iommu code more generic so that it can be used outside
   of the ARM context with other IOMMU drivers. Goal is to make use of
   it on x86 too.

 - Generic IOMMU domain support for the Intel VT-d driver. This driver
   now makes more use of common IOMMU code to allocate default domains
   for the devices it handles.

 - An IOMMU fault reporting API to userspace. With that the IOMMU fault
   handling can be done in user-space, for example to forward the faults
   to a VM.

 - Better handling for reserved regions requested by the firmware. These
   can be 'relaxed' now, meaning that those don't prevent a device being
   attached to a VM.

 - Suspend/Resume support for the Renesas IOMMU driver.

 - Added support for dumping SVA related fields of the DMAR table in the
   Intel VT-d driver via debugfs.

 - A pile of smaller fixes and cleanups.

* tag 'iommu-updates-v5.3' of git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu: (90 commits)
  iommu/omap: No need to check return value of debugfs_create functions
  iommu/arm-smmu-v3: Invalidate ATC when detaching a device
  iommu/arm-smmu-v3: Fix compilation when CONFIG_CMA=n
  iommu/vt-d: Cleanup unused variable
  iommu/amd: Flush not present cache in iommu_map_page
  iommu/amd: Only free resources once on init error
  iommu/amd: Move gart fallback to amd_iommu_init
  iommu/amd: Make iommu_disable safer
  iommu/io-pgtable: Support non-coherent page tables
  iommu/io-pgtable: Replace IO_PGTABLE_QUIRK_NO_DMA with specific flag
  iommu/io-pgtable-arm: Add support to use system cache
  iommu/arm-smmu-v3: Increase maximum size of queues
  iommu/vt-d: Silence a variable set but not used
  iommu/vt-d: Remove an unused variable "length"
  iommu: Fix integer truncation
  iommu: Add padding to struct iommu_fault
  iommu/vt-d: Consolidate domain_init() to avoid duplication
  iommu/vt-d: Cleanup after delegating DMA domain to generic iommu
  iommu/vt-d: Fix suspicious RCU usage in probe_acpi_namespace_devices()
  iommu/vt-d: Allow DMA domain attaching to rmrr locked device
  ...

6 files changed, 124 insertions, 53 deletions

diff --git a/include/linux/device.h b/include/linux/device.h
index 4a295e324ac5..b6ff25d9fff4 100644
--- a/include/linux/device.h
+++ b/include/linux/device.h
@@ -42,6 +42,7 @@ struct iommu_ops;
 struct iommu_group;
 struct iommu_fwspec;
 struct dev_pin_info;
+struct iommu_param;
 
 struct bus_attribute {
         struct attribute        attr;
@@ -961,6 +962,7 @@ struct dev_links_info {
  *              device (i.e. the bus driver that discovered the device).
  * @iommu_group: IOMMU group the device belongs to.
  * @iommu_fwspec: IOMMU-specific properties supplied by firmware.
+ * @iommu_param: Per device generic IOMMU runtime data
  *
  * @offline_disabled: If set, the device is permanently online.
  * @offline:    Set after successful invocation of bus type's .offline().
@@ -1054,6 +1056,7 @@ struct device {
         void    (*release)(struct device *dev);
         struct iommu_group      *iommu_group;
         struct iommu_fwspec     *iommu_fwspec;
+        struct iommu_param      *iommu_param;
 
         bool                    offline_disabled:1;
         bool                    offline:1;
diff --git a/include/linux/dma-iommu.h b/include/linux/dma-iommu.h
index 37258c8b2063..2112f21f73d8 100644
--- a/include/linux/dma-iommu.h
+++ b/include/linux/dma-iommu.h
@@ -5,59 +5,21 @@
 #ifndef __DMA_IOMMU_H
 #define __DMA_IOMMU_H
 
-#ifdef __KERNEL__
+#include <linux/errno.h>
 #include <linux/types.h>
-#include <asm/errno.h>
 
 #ifdef CONFIG_IOMMU_DMA
 #include <linux/dma-mapping.h>
 #include <linux/iommu.h>
 #include <linux/msi.h>
 
-int iommu_dma_init(void);
-
 /* Domain management interface for IOMMU drivers */
 int iommu_get_dma_cookie(struct iommu_domain *domain);
 int iommu_get_msi_cookie(struct iommu_domain *domain, dma_addr_t base);
 void iommu_put_dma_cookie(struct iommu_domain *domain);
 
 /* Setup call for arch DMA mapping code */
-int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base,
-                u64 size, struct device *dev);
-
-/* General helpers for DMA-API <-> IOMMU-API interaction */
-int dma_info_to_prot(enum dma_data_direction dir, bool coherent,
-                     unsigned long attrs);
-
-/*
- * These implement the bulk of the relevant DMA mapping callbacks, but require
- * the arch code to take care of attributes and cache maintenance
- */
-struct page **iommu_dma_alloc(struct device *dev, size_t size, gfp_t gfp,
-                unsigned long attrs, int prot, dma_addr_t *handle,
-                void (*flush_page)(struct device *, const void *, phys_addr_t));
-void iommu_dma_free(struct device *dev, struct page **pages, size_t size,
-                dma_addr_t *handle);
-
-int iommu_dma_mmap(struct page **pages, size_t size, struct vm_area_struct *vma);
-
-dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page,
-                unsigned long offset, size_t size, int prot);
-int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg,
-                int nents, int prot);
-
-/*
- * Arch code with no special attribute handling may use these
- * directly as DMA mapping callbacks for simplicity
- */
-void iommu_dma_unmap_page(struct device *dev, dma_addr_t handle, size_t size,
-                enum dma_data_direction dir, unsigned long attrs);
-void iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
-                enum dma_data_direction dir, unsigned long attrs);
-dma_addr_t iommu_dma_map_resource(struct device *dev, phys_addr_t phys,
-                size_t size, enum dma_data_direction dir, unsigned long attrs);
-void iommu_dma_unmap_resource(struct device *dev, dma_addr_t handle,
-                size_t size, enum dma_data_direction dir, unsigned long attrs);
+void iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 size);
 
 /* The DMA API isn't _quite_ the whole story, though... */
 /*
@@ -75,16 +37,16 @@ void iommu_dma_compose_msi_msg(struct msi_desc *desc,
 
 void iommu_dma_get_resv_regions(struct device *dev, struct list_head *list);
 
-#else
+#else /* CONFIG_IOMMU_DMA */
 
 struct iommu_domain;
 struct msi_desc;
 struct msi_msg;
 struct device;
 
-static inline int iommu_dma_init(void)
+static inline void iommu_setup_dma_ops(struct device *dev, u64 dma_base,
+                u64 size)
 {
-        return 0;
 }
 
 static inline int iommu_get_dma_cookie(struct iommu_domain *domain)
@@ -117,5 +79,4 @@ static inline void iommu_dma_get_resv_regions(struct device *dev, struct list_he
 }
 
 #endif  /* CONFIG_IOMMU_DMA */
-#endif  /* __KERNEL__ */
 #endif  /* __DMA_IOMMU_H */
diff --git a/include/linux/intel-iommu.h b/include/linux/intel-iommu.h
index 6a8dd4af0147..f2ae8a006ff8 100644
--- a/include/linux/intel-iommu.h
+++ b/include/linux/intel-iommu.h
@@ -435,6 +435,12 @@ enum {
 #define VTD_FLAG_TRANS_PRE_ENABLED      (1 << 0)
 #define VTD_FLAG_IRQ_REMAP_PRE_ENABLED  (1 << 1)
 
+extern int intel_iommu_sm;
+
+#define sm_supported(iommu)     (intel_iommu_sm && ecap_smts((iommu)->ecap))
+#define pasid_supported(iommu)  (sm_supported(iommu) &&                 \
+                                 ecap_pasid((iommu)->ecap))
+
 struct pasid_entry;
 struct pasid_state_entry;
 struct page_req_dsc;
@@ -642,7 +648,6 @@ extern int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu);
 
 extern int dmar_ir_support(void);
 
-struct dmar_domain *get_valid_domain_for_dev(struct device *dev);
 void *alloc_pgtable_page(int node);
 void free_pgtable_page(void *vaddr);
 struct intel_iommu *domain_get_iommu(struct dmar_domain *domain);
diff --git a/include/linux/intel-svm.h b/include/linux/intel-svm.h
index 54ffcc6a322e..94f047a8a845 100644
--- a/include/linux/intel-svm.h
+++ b/include/linux/intel-svm.h
@@ -49,7 +49,7 @@ struct svm_dev_ops {
 
 /**
  * intel_svm_bind_mm() - Bind the current process to a PASID
- * @dev:        Device to be granted acccess
+ * @dev:        Device to be granted access
  * @pasid:      Address for allocated PASID
  * @flags:      Flags. Later for requesting supervisor mode, etc.
  * @ops:        Callbacks to device driver
diff --git a/include/linux/io-pgtable.h b/include/linux/io-pgtable.h
index 76969a564831..b5a450a3bb47 100644
--- a/include/linux/io-pgtable.h
+++ b/include/linux/io-pgtable.h
@@ -44,6 +44,8 @@ struct iommu_gather_ops {
  *                 tables.
  * @ias:           Input address (iova) size, in bits.
  * @oas:           Output address (paddr) size, in bits.
+ * @coherent_walk  A flag to indicate whether or not page table walks made
+ *                 by the IOMMU are coherent with the CPU caches.
  * @tlb:           TLB management callbacks for this set of tables.
  * @iommu_dev:     The device representing the DMA configuration for the
  *                 page table walker.
@@ -68,11 +70,6 @@ struct io_pgtable_cfg {
          *      when the SoC is in "4GB mode" and they can only access the high
          *      remap of DRAM (0x1_00000000 to 0x1_ffffffff).
          *
-         * IO_PGTABLE_QUIRK_NO_DMA: Guarantees that the tables will only ever
-         *      be accessed by a fully cache-coherent IOMMU or CPU (e.g. for a
-         *      software-emulated IOMMU), such that pagetable updates need not
-         *      be treated as explicit DMA data.
-         *
          * IO_PGTABLE_QUIRK_NON_STRICT: Skip issuing synchronous leaf TLBIs
          *      on unmap, for DMA domains using the flush queue mechanism for
          *      delayed invalidation.
@@ -81,12 +78,12 @@ struct io_pgtable_cfg {
         #define IO_PGTABLE_QUIRK_NO_PERMS       BIT(1)
         #define IO_PGTABLE_QUIRK_TLBI_ON_MAP    BIT(2)
         #define IO_PGTABLE_QUIRK_ARM_MTK_4GB    BIT(3)
-        #define IO_PGTABLE_QUIRK_NO_DMA         BIT(4)
-        #define IO_PGTABLE_QUIRK_NON_STRICT     BIT(5)
+        #define IO_PGTABLE_QUIRK_NON_STRICT     BIT(4)
         unsigned long                   quirks;
         unsigned long                   pgsize_bitmap;
         unsigned int                    ias;
         unsigned int                    oas;
+        bool                            coherent_walk;
         const struct iommu_gather_ops   *tlb;
         struct device                   *iommu_dev;
 
diff --git a/include/linux/iommu.h b/include/linux/iommu.h
index e552c3b63f6f..fdc355ccc570 100644
--- a/include/linux/iommu.h
+++ b/include/linux/iommu.h
@@ -13,6 +13,7 @@
 #include <linux/errno.h>
 #include <linux/err.h>
 #include <linux/of.h>
+#include <uapi/linux/iommu.h>
 
 #define IOMMU_READ      (1 << 0)
 #define IOMMU_WRITE     (1 << 1)
@@ -29,6 +30,12 @@
  * if the IOMMU page table format is equivalent.
  */
 #define IOMMU_PRIV      (1 << 5)
+/*
+ * Non-coherent masters on few Qualcomm SoCs can use this page protection flag
+ * to set correct cacheability attributes to use an outer level of cache -
+ * last level cache, aka system cache.
+ */
+#define IOMMU_QCOM_SYS_CACHE    (1 << 6)
 
 struct iommu_ops;
 struct iommu_group;
@@ -37,6 +44,7 @@ struct device;
 struct iommu_domain;
 struct notifier_block;
 struct iommu_sva;
+struct iommu_fault_event;
 
 /* iommu fault flags */
 #define IOMMU_FAULT_READ        0x0
@@ -46,6 +54,7 @@ typedef int (*iommu_fault_handler_t)(struct iommu_domain *,
                         struct device *, unsigned long, int, void *);
 typedef int (*iommu_mm_exit_handler_t)(struct device *dev, struct iommu_sva *,
                                        void *);
+typedef int (*iommu_dev_fault_handler_t)(struct iommu_fault *, void *);
 
 struct iommu_domain_geometry {
         dma_addr_t aperture_start; /* First address that can be mapped    */
@@ -123,6 +132,12 @@ enum iommu_attr {
 enum iommu_resv_type {
         /* Memory regions which must be mapped 1:1 at all times */
         IOMMU_RESV_DIRECT,
+        /*
+         * Memory regions which are advertised to be 1:1 but are
+         * commonly considered relaxable in some conditions,
+         * for instance in device assignment use case (USB, Graphics)
+         */
+        IOMMU_RESV_DIRECT_RELAXABLE,
         /* Arbitrary "never map this or give it to a device" address ranges */
         IOMMU_RESV_RESERVED,
         /* Hardware MSI region (untranslated) */
@@ -212,6 +227,7 @@ struct iommu_sva_ops {
  * @sva_bind: Bind process address space to device
  * @sva_unbind: Unbind process address space from device
  * @sva_get_pasid: Get PASID associated to a SVA handle
+ * @page_response: handle page request response
  * @pgsize_bitmap: bitmap of all possible supported page sizes
  */
 struct iommu_ops {
@@ -272,6 +288,10 @@ struct iommu_ops {
         void (*sva_unbind)(struct iommu_sva *handle);
         int (*sva_get_pasid)(struct iommu_sva *handle);
 
+        int (*page_response)(struct device *dev,
+                             struct iommu_fault_event *evt,
+                             struct iommu_page_response *msg);
+
         unsigned long pgsize_bitmap;
 };
 
@@ -289,6 +309,48 @@ struct iommu_device {
         struct device *dev;
 };
 
+/**
+ * struct iommu_fault_event - Generic fault event
+ *
+ * Can represent recoverable faults such as a page requests or
+ * unrecoverable faults such as DMA or IRQ remapping faults.
+ *
+ * @fault: fault descriptor
+ * @list: pending fault event list, used for tracking responses
+ */
+struct iommu_fault_event {
+        struct iommu_fault fault;
+        struct list_head list;
+};
+
+/**
+ * struct iommu_fault_param - per-device IOMMU fault data
+ * @handler: Callback function to handle IOMMU faults at device level
+ * @data: handler private data
+ * @faults: holds the pending faults which needs response
+ * @lock: protect pending faults list
+ */
+struct iommu_fault_param {
+        iommu_dev_fault_handler_t handler;
+        void *data;
+        struct list_head faults;
+        struct mutex lock;
+};
+
+/**
+ * struct iommu_param - collection of per-device IOMMU data
+ *
+ * @fault_param: IOMMU detected device fault reporting data
+ *
+ * TODO: migrate other per device data pointers under iommu_dev_data, e.g.
+ *      struct iommu_group      *iommu_group;
+ *      struct iommu_fwspec     *iommu_fwspec;
+ */
+struct iommu_param {
+        struct mutex lock;
+        struct iommu_fault_param *fault_param;
+};
+
 int  iommu_device_register(struct iommu_device *iommu);
 void iommu_device_unregister(struct iommu_device *iommu);
 int  iommu_device_sysfs_add(struct iommu_device *iommu,
@@ -350,6 +412,7 @@ extern void iommu_set_fault_handler(struct iommu_domain *domain,
 extern void iommu_get_resv_regions(struct device *dev, struct list_head *list);
 extern void iommu_put_resv_regions(struct device *dev, struct list_head *list);
 extern int iommu_request_dm_for_dev(struct device *dev);
+extern int iommu_request_dma_domain_for_dev(struct device *dev);
 extern struct iommu_resv_region *
 iommu_alloc_resv_region(phys_addr_t start, size_t length, int prot,
                         enum iommu_resv_type type);
@@ -378,6 +441,17 @@ extern int iommu_group_register_notifier(struct iommu_group *group,
                                          struct notifier_block *nb);
 extern int iommu_group_unregister_notifier(struct iommu_group *group,
                                            struct notifier_block *nb);
+extern int iommu_register_device_fault_handler(struct device *dev,
+                                        iommu_dev_fault_handler_t handler,
+                                        void *data);
+
+extern int iommu_unregister_device_fault_handler(struct device *dev);
+
+extern int iommu_report_device_fault(struct device *dev,
+                                     struct iommu_fault_event *evt);
+extern int iommu_page_response(struct device *dev,
+                               struct iommu_page_response *msg);
+
 extern int iommu_group_id(struct iommu_group *group);
 extern struct iommu_group *iommu_group_get_for_dev(struct device *dev);
 extern struct iommu_domain *iommu_group_default_domain(struct iommu_group *);
@@ -492,6 +566,7 @@ struct iommu_ops {};
 struct iommu_group {};
 struct iommu_fwspec {};
 struct iommu_device {};
+struct iommu_fault_param {};
 
 static inline bool iommu_present(struct bus_type *bus)
 {
@@ -614,6 +689,11 @@ static inline int iommu_request_dm_for_dev(struct device *dev)
         return -ENODEV;
 }
 
+static inline int iommu_request_dma_domain_for_dev(struct device *dev)
+{
+        return -ENODEV;
+}
+
 static inline int iommu_attach_group(struct iommu_domain *domain,
                                      struct iommu_group *group)
 {
@@ -685,6 +765,31 @@ static inline int iommu_group_unregister_notifier(struct iommu_group *group,
         return 0;
 }
 
+static inline
+int iommu_register_device_fault_handler(struct device *dev,
+                                        iommu_dev_fault_handler_t handler,
+                                        void *data)
+{
+        return -ENODEV;
+}
+
+static inline int iommu_unregister_device_fault_handler(struct device *dev)
+{
+        return 0;
+}
+
+static inline
+int iommu_report_device_fault(struct device *dev, struct iommu_fault_event *evt)
+{
+        return -ENODEV;
+}
+
+static inline int iommu_page_response(struct device *dev,
+                                      struct iommu_page_response *msg)
+{
+        return -ENODEV;
+}
+
 static inline int iommu_group_id(struct iommu_group *group)
 {
         return -ENODEV;