IB: Add DMA mapping functions to allow device drivers to interpose

The QLogic InfiniPath HCAs use programmed I/O instead of HW DMA.
This patch allows a verbs device driver to interpose on DMA mapping
function calls in order to avoid relying on bus_to_virt() and
phys_to_virt() to undo the mappings created by dma_map_single(),
dma_map_sg(), etc.

Signed-off-by: Ralph Campbell <ralph.campbell@qlogic.com>
Signed-off-by: Roland Dreier <rolandd@cisco.com>

1 files changed, 253 insertions, 0 deletions

diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h
index 8eacc3510993..fd2353fa7e12 100644
--- a/include/rdma/ib_verbs.h
+++ b/include/rdma/ib_verbs.h
@@ -43,6 +43,8 @@
 
 #include <linux/types.h>
 #include <linux/device.h>
+#include <linux/mm.h>
+#include <linux/dma-mapping.h>
 
 #include <asm/atomic.h>
 #include <asm/scatterlist.h>
@@ -848,6 +850,49 @@ struct ib_cache {
         u8                     *lmc_cache;
 };
 
+struct ib_dma_mapping_ops {
+        int             (*mapping_error)(struct ib_device *dev,
+                                         u64 dma_addr);
+        u64             (*map_single)(struct ib_device *dev,
+                                      void *ptr, size_t size,
+                                      enum dma_data_direction direction);
+        void            (*unmap_single)(struct ib_device *dev,
+                                        u64 addr, size_t size,
+                                        enum dma_data_direction direction);
+        u64             (*map_page)(struct ib_device *dev,
+                                    struct page *page, unsigned long offset,
+                                    size_t size,
+                                    enum dma_data_direction direction);
+        void            (*unmap_page)(struct ib_device *dev,
+                                      u64 addr, size_t size,
+                                      enum dma_data_direction direction);
+        int             (*map_sg)(struct ib_device *dev,
+                                  struct scatterlist *sg, int nents,
+                                  enum dma_data_direction direction);
+        void            (*unmap_sg)(struct ib_device *dev,
+                                    struct scatterlist *sg, int nents,
+                                    enum dma_data_direction direction);
+        u64             (*dma_address)(struct ib_device *dev,
+                                       struct scatterlist *sg);
+        unsigned int    (*dma_len)(struct ib_device *dev,
+                                   struct scatterlist *sg);
+        void            (*sync_single_for_cpu)(struct ib_device *dev,
+                                               u64 dma_handle,
+                                               size_t size,
+                                               enum dma_data_direction dir);
+        void            (*sync_single_for_device)(struct ib_device *dev,
+                                                  u64 dma_handle,
+                                                  size_t size,
+                                                  enum dma_data_direction dir);
+        void            *(*alloc_coherent)(struct ib_device *dev,
+                                           size_t size,
+                                           u64 *dma_handle,
+                                           gfp_t flag);
+        void            (*free_coherent)(struct ib_device *dev,
+                                         size_t size, void *cpu_addr,
+                                         u64 dma_handle);
+};
+
 struct iw_cm_verbs;
 
 struct ib_device {
@@ -992,6 +1037,8 @@ struct ib_device {
                                                   struct ib_mad *in_mad,
                                                   struct ib_mad *out_mad);
 
+        struct ib_dma_mapping_ops   *dma_ops;
+
         struct module               *owner;
         struct class_device          class_dev;
         struct kobject               ports_parent;
@@ -1395,10 +1442,216 @@ static inline int ib_req_ncomp_notif(struct ib_cq *cq, int wc_cnt)
  *   usable for DMA.
  * @pd: The protection domain associated with the memory region.
  * @mr_access_flags: Specifies the memory access rights.
+ *
+ * Note that the ib_dma_*() functions defined below must be used
+ * to create/destroy addresses used with the Lkey or Rkey returned
+ * by ib_get_dma_mr().
  */
 struct ib_mr *ib_get_dma_mr(struct ib_pd *pd, int mr_access_flags);
 
 /**
+ * ib_dma_mapping_error - check a DMA addr for error
+ * @dev: The device for which the dma_addr was created
+ * @dma_addr: The DMA address to check
+ */
+static inline int ib_dma_mapping_error(struct ib_device *dev, u64 dma_addr)
+{
+        return dev->dma_ops ?
+                dev->dma_ops->mapping_error(dev, dma_addr) :
+                dma_mapping_error(dma_addr);
+}
+
+/**
+ * ib_dma_map_single - Map a kernel virtual address to DMA address
+ * @dev: The device for which the dma_addr is to be created
+ * @cpu_addr: The kernel virtual address
+ * @size: The size of the region in bytes
+ * @direction: The direction of the DMA
+ */
+static inline u64 ib_dma_map_single(struct ib_device *dev,
+                                    void *cpu_addr, size_t size,
+                                    enum dma_data_direction direction)
+{
+        return dev->dma_ops ?
+                dev->dma_ops->map_single(dev, cpu_addr, size, direction) :
+                dma_map_single(dev->dma_device, cpu_addr, size, direction);
+}
+
+/**
+ * ib_dma_unmap_single - Destroy a mapping created by ib_dma_map_single()
+ * @dev: The device for which the DMA address was created
+ * @addr: The DMA address
+ * @size: The size of the region in bytes
+ * @direction: The direction of the DMA
+ */
+static inline void ib_dma_unmap_single(struct ib_device *dev,
+                                       u64 addr, size_t size,
+                                       enum dma_data_direction direction)
+{
+        dev->dma_ops ?
+                dev->dma_ops->unmap_single(dev, addr, size, direction) :
+                dma_unmap_single(dev->dma_device, addr, size, direction);
+}
+
+/**
+ * ib_dma_map_page - Map a physical page to DMA address
+ * @dev: The device for which the dma_addr is to be created
+ * @page: The page to be mapped
+ * @offset: The offset within the page
+ * @size: The size of the region in bytes
+ * @direction: The direction of the DMA
+ */
+static inline u64 ib_dma_map_page(struct ib_device *dev,
+                                  struct page *page,
+                                  unsigned long offset,
+                                  size_t size,
+                                         enum dma_data_direction direction)
+{
+        return dev->dma_ops ?
+                dev->dma_ops->map_page(dev, page, offset, size, direction) :
+                dma_map_page(dev->dma_device, page, offset, size, direction);
+}
+
+/**
+ * ib_dma_unmap_page - Destroy a mapping created by ib_dma_map_page()
+ * @dev: The device for which the DMA address was created
+ * @addr: The DMA address
+ * @size: The size of the region in bytes
+ * @direction: The direction of the DMA
+ */
+static inline void ib_dma_unmap_page(struct ib_device *dev,
+                                     u64 addr, size_t size,
+                                     enum dma_data_direction direction)
+{
+        dev->dma_ops ?
+                dev->dma_ops->unmap_page(dev, addr, size, direction) :
+                dma_unmap_page(dev->dma_device, addr, size, direction);
+}
+
+/**
+ * ib_dma_map_sg - Map a scatter/gather list to DMA addresses
+ * @dev: The device for which the DMA addresses are to be created
+ * @sg: The array of scatter/gather entries
+ * @nents: The number of scatter/gather entries
+ * @direction: The direction of the DMA
+ */
+static inline int ib_dma_map_sg(struct ib_device *dev,
+                                struct scatterlist *sg, int nents,
+                                enum dma_data_direction direction)
+{
+        return dev->dma_ops ?
+                dev->dma_ops->map_sg(dev, sg, nents, direction) :
+                dma_map_sg(dev->dma_device, sg, nents, direction);
+}
+
+/**
+ * ib_dma_unmap_sg - Unmap a scatter/gather list of DMA addresses
+ * @dev: The device for which the DMA addresses were created
+ * @sg: The array of scatter/gather entries
+ * @nents: The number of scatter/gather entries
+ * @direction: The direction of the DMA
+ */
+static inline void ib_dma_unmap_sg(struct ib_device *dev,
+                                   struct scatterlist *sg, int nents,
+                                   enum dma_data_direction direction)
+{
+        dev->dma_ops ?
+                dev->dma_ops->unmap_sg(dev, sg, nents, direction) :
+                dma_unmap_sg(dev->dma_device, sg, nents, direction);
+}
+
+/**
+ * ib_sg_dma_address - Return the DMA address from a scatter/gather entry
+ * @dev: The device for which the DMA addresses were created
+ * @sg: The scatter/gather entry
+ */
+static inline u64 ib_sg_dma_address(struct ib_device *dev,
+                                    struct scatterlist *sg)
+{
+        return dev->dma_ops ?
+                dev->dma_ops->dma_address(dev, sg) : sg_dma_address(sg);
+}
+
+/**
+ * ib_sg_dma_len - Return the DMA length from a scatter/gather entry
+ * @dev: The device for which the DMA addresses were created
+ * @sg: The scatter/gather entry
+ */
+static inline unsigned int ib_sg_dma_len(struct ib_device *dev,
+                                         struct scatterlist *sg)
+{
+        return dev->dma_ops ?
+                dev->dma_ops->dma_len(dev, sg) : sg_dma_len(sg);
+}
+
+/**
+ * ib_dma_sync_single_for_cpu - Prepare DMA region to be accessed by CPU
+ * @dev: The device for which the DMA address was created
+ * @addr: The DMA address
+ * @size: The size of the region in bytes
+ * @dir: The direction of the DMA
+ */
+static inline void ib_dma_sync_single_for_cpu(struct ib_device *dev,
+                                              u64 addr,
+                                              size_t size,
+                                              enum dma_data_direction dir)
+{
+        dev->dma_ops ?
+                dev->dma_ops->sync_single_for_cpu(dev, addr, size, dir) :
+                dma_sync_single_for_cpu(dev->dma_device, addr, size, dir);
+}
+
+/**
+ * ib_dma_sync_single_for_device - Prepare DMA region to be accessed by device
+ * @dev: The device for which the DMA address was created
+ * @addr: The DMA address
+ * @size: The size of the region in bytes
+ * @dir: The direction of the DMA
+ */
+static inline void ib_dma_sync_single_for_device(struct ib_device *dev,
+                                                 u64 addr,
+                                                 size_t size,
+                                                 enum dma_data_direction dir)
+{
+        dev->dma_ops ?
+                dev->dma_ops->sync_single_for_device(dev, addr, size, dir) :
+                dma_sync_single_for_device(dev->dma_device, addr, size, dir);
+}
+
+/**
+ * ib_dma_alloc_coherent - Allocate memory and map it for DMA
+ * @dev: The device for which the DMA address is requested
+ * @size: The size of the region to allocate in bytes
+ * @dma_handle: A pointer for returning the DMA address of the region
+ * @flag: memory allocator flags
+ */
+static inline void *ib_dma_alloc_coherent(struct ib_device *dev,
+                                           size_t size,
+                                           u64 *dma_handle,
+                                           gfp_t flag)
+{
+        return dev->dma_ops ?
+                dev->dma_ops->alloc_coherent(dev, size, dma_handle, flag) :
+                dma_alloc_coherent(dev->dma_device, size, dma_handle, flag);
+}
+
+/**
+ * ib_dma_free_coherent - Free memory allocated by ib_dma_alloc_coherent()
+ * @dev: The device for which the DMA addresses were allocated
+ * @size: The size of the region
+ * @cpu_addr: the address returned by ib_dma_alloc_coherent()
+ * @dma_handle: the DMA address returned by ib_dma_alloc_coherent()
+ */
+static inline void ib_dma_free_coherent(struct ib_device *dev,
+                                        size_t size, void *cpu_addr,
+                                        u64 dma_handle)
+{
+        dev->dma_ops ?
+                dev->dma_ops->free_coherent(dev, size, cpu_addr, dma_handle) :
+                dma_free_coherent(dev->dma_device, size, cpu_addr, dma_handle);
+}
+
+/**
  * ib_reg_phys_mr - Prepares a virtually addressed memory region for use
  *   by an HCA.
  * @pd: The protection domain associated assigned to the registered region.