dma-mapping: bypass indirect calls for dma-direct

Avoid expensive indirect calls in the fast path DMA mapping
operations by directly calling the dma_direct_* ops if we are using
the directly mapped DMA operations.

Signed-off-by: Christoph Hellwig <hch@lst.de>
Acked-by: Jesper Dangaard Brouer <brouer@redhat.com>
Tested-by: Jesper Dangaard Brouer <brouer@redhat.com>
Tested-by: Tony Luck <tony.luck@intel.com>

1 files changed, 97 insertions, 14 deletions

diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h
index 269ee27fc3d9..f422aec0f53c 100644
--- a/include/linux/dma-mapping.h
+++ b/include/linux/dma-mapping.h
@@ -134,7 +134,6 @@ struct dma_map_ops {
 
 #define DMA_MAPPING_ERROR               (~(dma_addr_t)0)
 
-extern const struct dma_map_ops dma_direct_ops;
 extern const struct dma_map_ops dma_virt_ops;
 extern const struct dma_map_ops dma_dummy_ops;
 
@@ -222,6 +221,69 @@ static inline const struct dma_map_ops *get_dma_ops(struct device *dev)
 }
 #endif
 
+static inline bool dma_is_direct(const struct dma_map_ops *ops)
+{
+        return likely(!ops);
+}
+
+/*
+ * All the dma_direct_* declarations are here just for the indirect call bypass,
+ * and must not be used directly drivers!
+ */
+dma_addr_t dma_direct_map_page(struct device *dev, struct page *page,
+                unsigned long offset, size_t size, enum dma_data_direction dir,
+                unsigned long attrs);
+int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, int nents,
+                enum dma_data_direction dir, unsigned long attrs);
+
+#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \
+    defined(CONFIG_SWIOTLB)
+void dma_direct_sync_single_for_device(struct device *dev,
+                dma_addr_t addr, size_t size, enum dma_data_direction dir);
+void dma_direct_sync_sg_for_device(struct device *dev,
+                struct scatterlist *sgl, int nents, enum dma_data_direction dir);
+#else
+static inline void dma_direct_sync_single_for_device(struct device *dev,
+                dma_addr_t addr, size_t size, enum dma_data_direction dir)
+{
+}
+static inline void dma_direct_sync_sg_for_device(struct device *dev,
+                struct scatterlist *sgl, int nents, enum dma_data_direction dir)
+{
+}
+#endif
+
+#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
+    defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL) || \
+    defined(CONFIG_SWIOTLB)
+void dma_direct_unmap_page(struct device *dev, dma_addr_t addr,
+                size_t size, enum dma_data_direction dir, unsigned long attrs);
+void dma_direct_unmap_sg(struct device *dev, struct scatterlist *sgl,
+                int nents, enum dma_data_direction dir, unsigned long attrs);
+void dma_direct_sync_single_for_cpu(struct device *dev,
+                dma_addr_t addr, size_t size, enum dma_data_direction dir);
+void dma_direct_sync_sg_for_cpu(struct device *dev,
+                struct scatterlist *sgl, int nents, enum dma_data_direction dir);
+#else
+static inline void dma_direct_unmap_page(struct device *dev, dma_addr_t addr,
+                size_t size, enum dma_data_direction dir, unsigned long attrs)
+{
+}
+static inline void dma_direct_unmap_sg(struct device *dev,
+                struct scatterlist *sgl, int nents, enum dma_data_direction dir,
+                unsigned long attrs)
+{
+}
+static inline void dma_direct_sync_single_for_cpu(struct device *dev,
+                dma_addr_t addr, size_t size, enum dma_data_direction dir)
+{
+}
+static inline void dma_direct_sync_sg_for_cpu(struct device *dev,
+                struct scatterlist *sgl, int nents, enum dma_data_direction dir)
+{
+}
+#endif
+
 static inline dma_addr_t dma_map_single_attrs(struct device *dev, void *ptr,
                                               size_t size,
                                               enum dma_data_direction dir,
@@ -232,9 +294,12 @@ static inline dma_addr_t dma_map_single_attrs(struct device *dev, void *ptr,
 
         BUG_ON(!valid_dma_direction(dir));
         debug_dma_map_single(dev, ptr, size);
-        addr = ops->map_page(dev, virt_to_page(ptr),
-                             offset_in_page(ptr), size,
-                             dir, attrs);
+        if (dma_is_direct(ops))
+                addr = dma_direct_map_page(dev, virt_to_page(ptr),
+                                offset_in_page(ptr), size, dir, attrs);
+        else
+                addr = ops->map_page(dev, virt_to_page(ptr),
+                                offset_in_page(ptr), size, dir, attrs);
         debug_dma_map_page(dev, virt_to_page(ptr),
                            offset_in_page(ptr), size,
                            dir, addr, true);
@@ -249,7 +314,9 @@ static inline void dma_unmap_single_attrs(struct device *dev, dma_addr_t addr,
         const struct dma_map_ops *ops = get_dma_ops(dev);
 
         BUG_ON(!valid_dma_direction(dir));
-        if (ops->unmap_page)
+        if (dma_is_direct(ops))
+                dma_direct_unmap_page(dev, addr, size, dir, attrs);
+        else if (ops->unmap_page)
                 ops->unmap_page(dev, addr, size, dir, attrs);
         debug_dma_unmap_page(dev, addr, size, dir, true);
 }
@@ -272,7 +339,10 @@ static inline int dma_map_sg_attrs(struct device *dev, struct scatterlist *sg,
         int ents;
 
         BUG_ON(!valid_dma_direction(dir));
-        ents = ops->map_sg(dev, sg, nents, dir, attrs);
+        if (dma_is_direct(ops))
+                ents = dma_direct_map_sg(dev, sg, nents, dir, attrs);
+        else
+                ents = ops->map_sg(dev, sg, nents, dir, attrs);
         BUG_ON(ents < 0);
         debug_dma_map_sg(dev, sg, nents, ents, dir);
 
@@ -287,7 +357,9 @@ static inline void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg
 
         BUG_ON(!valid_dma_direction(dir));
         debug_dma_unmap_sg(dev, sg, nents, dir);
-        if (ops->unmap_sg)
+        if (dma_is_direct(ops))
+                dma_direct_unmap_sg(dev, sg, nents, dir, attrs);
+        else if (ops->unmap_sg)
                 ops->unmap_sg(dev, sg, nents, dir, attrs);
 }
 
@@ -301,7 +373,10 @@ static inline dma_addr_t dma_map_page_attrs(struct device *dev,
         dma_addr_t addr;
 
         BUG_ON(!valid_dma_direction(dir));
-        addr = ops->map_page(dev, page, offset, size, dir, attrs);
+        if (dma_is_direct(ops))
+                addr = dma_direct_map_page(dev, page, offset, size, dir, attrs);
+        else
+                addr = ops->map_page(dev, page, offset, size, dir, attrs);
         debug_dma_map_page(dev, page, offset, size, dir, addr, false);
 
         return addr;
@@ -322,7 +397,7 @@ static inline dma_addr_t dma_map_resource(struct device *dev,
         BUG_ON(pfn_valid(PHYS_PFN(phys_addr)));
 
         addr = phys_addr;
-        if (ops->map_resource)
+        if (ops && ops->map_resource)
                 addr = ops->map_resource(dev, phys_addr, size, dir, attrs);
 
         debug_dma_map_resource(dev, phys_addr, size, dir, addr);
@@ -337,7 +412,7 @@ static inline void dma_unmap_resource(struct device *dev, dma_addr_t addr,
         const struct dma_map_ops *ops = get_dma_ops(dev);
 
         BUG_ON(!valid_dma_direction(dir));
-        if (ops->unmap_resource)
+        if (ops && ops->unmap_resource)
                 ops->unmap_resource(dev, addr, size, dir, attrs);
         debug_dma_unmap_resource(dev, addr, size, dir);
 }
@@ -349,7 +424,9 @@ static inline void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr,
         const struct dma_map_ops *ops = get_dma_ops(dev);
 
         BUG_ON(!valid_dma_direction(dir));
-        if (ops->sync_single_for_cpu)
+        if (dma_is_direct(ops))
+                dma_direct_sync_single_for_cpu(dev, addr, size, dir);
+        else if (ops->sync_single_for_cpu)
                 ops->sync_single_for_cpu(dev, addr, size, dir);
         debug_dma_sync_single_for_cpu(dev, addr, size, dir);
 }
@@ -368,7 +445,9 @@ static inline void dma_sync_single_for_device(struct device *dev,
         const struct dma_map_ops *ops = get_dma_ops(dev);
 
         BUG_ON(!valid_dma_direction(dir));
-        if (ops->sync_single_for_device)
+        if (dma_is_direct(ops))
+                dma_direct_sync_single_for_device(dev, addr, size, dir);
+        else if (ops->sync_single_for_device)
                 ops->sync_single_for_device(dev, addr, size, dir);
         debug_dma_sync_single_for_device(dev, addr, size, dir);
 }
@@ -387,7 +466,9 @@ dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
         const struct dma_map_ops *ops = get_dma_ops(dev);
 
         BUG_ON(!valid_dma_direction(dir));
-        if (ops->sync_sg_for_cpu)
+        if (dma_is_direct(ops))
+                dma_direct_sync_sg_for_cpu(dev, sg, nelems, dir);
+        else if (ops->sync_sg_for_cpu)
                 ops->sync_sg_for_cpu(dev, sg, nelems, dir);
         debug_dma_sync_sg_for_cpu(dev, sg, nelems, dir);
 }
@@ -399,7 +480,9 @@ dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
         const struct dma_map_ops *ops = get_dma_ops(dev);
 
         BUG_ON(!valid_dma_direction(dir));
-        if (ops->sync_sg_for_device)
+        if (dma_is_direct(ops))
+                dma_direct_sync_sg_for_device(dev, sg, nelems, dir);
+        else if (ops->sync_sg_for_device)
                 ops->sync_sg_for_device(dev, sg, nelems, dir);
         debug_dma_sync_sg_for_device(dev, sg, nelems, dir);