1 files changed, 86 insertions, 63 deletions

diff --git a/lib/swiotlb.c b/lib/swiotlb.c
index 025922807e6e..d568894df8cc 100644
--- a/lib/swiotlb.c
+++ b/lib/swiotlb.c
@@ -31,6 +31,7 @@
 
 #include <linux/init.h>
 #include <linux/bootmem.h>
+#include <linux/iommu-helper.h>
 
 #define OFFSET(val,align) ((unsigned long)      \
                            ( (val) & ( (align) - 1)))
@@ -282,15 +283,6 @@ address_needs_mapping(struct device *hwdev, dma_addr_t addr)
         return (addr & ~mask) != 0;
 }
 
-static inline unsigned int is_span_boundary(unsigned int index,
-                                            unsigned int nslots,
-                                            unsigned long offset_slots,
-                                            unsigned long max_slots)
-{
-        unsigned long offset = (offset_slots + index) & (max_slots - 1);
-        return offset + nslots > max_slots;
-}
-
 /*
  * Allocates bounce buffer and returns its kernel virtual address.
  */
@@ -331,56 +323,53 @@ map_single(struct device *hwdev, char *buffer, size_t size, int dir)
          * request and allocate a buffer from that IO TLB pool.
          */
         spin_lock_irqsave(&io_tlb_lock, flags);
-        {
-                index = ALIGN(io_tlb_index, stride);
-                if (index >= io_tlb_nslabs)
-                        index = 0;
-                wrap = index;
-
-                do {
-                        while (is_span_boundary(index, nslots, offset_slots,
-                                                max_slots)) {
-                                index += stride;
-                                if (index >= io_tlb_nslabs)
-                                        index = 0;
-                                if (index == wrap)
-                                        goto not_found;
-                        }
-
-                        /*
-                         * If we find a slot that indicates we have 'nslots'
-                         * number of contiguous buffers, we allocate the
-                         * buffers from that slot and mark the entries as '0'
-                         * indicating unavailable.
-                         */
-                        if (io_tlb_list[index] >= nslots) {
-                                int count = 0;
-
-                                for (i = index; i < (int) (index + nslots); i++)
-                                        io_tlb_list[i] = 0;
-                                for (i = index - 1; (OFFSET(i, IO_TLB_SEGSIZE) != IO_TLB_SEGSIZE -1) && io_tlb_list[i]; i--)
-                                        io_tlb_list[i] = ++count;
-                                dma_addr = io_tlb_start + (index << IO_TLB_SHIFT);
-
-                                /*
-                                 * Update the indices to avoid searching in
-                                 * the next round.
-                                 */
-                                io_tlb_index = ((index + nslots) < io_tlb_nslabs
-                                                ? (index + nslots) : 0);
-
-                                goto found;
-                        }
+        index = ALIGN(io_tlb_index, stride);
+        if (index >= io_tlb_nslabs)
+                index = 0;
+        wrap = index;
+
+        do {
+                while (iommu_is_span_boundary(index, nslots, offset_slots,
+                                              max_slots)) {
                         index += stride;
                         if (index >= io_tlb_nslabs)
                                 index = 0;
-                } while (index != wrap);
+                        if (index == wrap)
+                                goto not_found;
+                }
 
-  not_found:
-                spin_unlock_irqrestore(&io_tlb_lock, flags);
-                return NULL;
-        }
-  found:
+                /*
+                 * If we find a slot that indicates we have 'nslots' number of
+                 * contiguous buffers, we allocate the buffers from that slot
+                 * and mark the entries as '0' indicating unavailable.
+                 */
+                if (io_tlb_list[index] >= nslots) {
+                        int count = 0;
+
+                        for (i = index; i < (int) (index + nslots); i++)
+                                io_tlb_list[i] = 0;
+                        for (i = index - 1; (OFFSET(i, IO_TLB_SEGSIZE) != IO_TLB_SEGSIZE - 1) && io_tlb_list[i]; i--)
+                                io_tlb_list[i] = ++count;
+                        dma_addr = io_tlb_start + (index << IO_TLB_SHIFT);
+
+                        /*
+                         * Update the indices to avoid searching in the next
+                         * round.
+                         */
+                        io_tlb_index = ((index + nslots) < io_tlb_nslabs
+                                        ? (index + nslots) : 0);
+
+                        goto found;
+                }
+                index += stride;
+                if (index >= io_tlb_nslabs)
+                        index = 0;
+        } while (index != wrap);
+
+not_found:
+        spin_unlock_irqrestore(&io_tlb_lock, flags);
+        return NULL;
+found:
         spin_unlock_irqrestore(&io_tlb_lock, flags);
 
         /*
@@ -566,7 +555,8 @@ swiotlb_full(struct device *dev, size_t size, int dir, int do_panic)
  * either swiotlb_unmap_single or swiotlb_dma_sync_single is performed.
  */
 dma_addr_t
-swiotlb_map_single(struct device *hwdev, void *ptr, size_t size, int dir)
+swiotlb_map_single_attrs(struct device *hwdev, void *ptr, size_t size,
+                         int dir, struct dma_attrs *attrs)
 {
         dma_addr_t dev_addr = virt_to_bus(ptr);
         void *map;
@@ -599,6 +589,13 @@ swiotlb_map_single(struct device *hwdev, void *ptr, size_t size, int dir)
 
         return dev_addr;
 }
+EXPORT_SYMBOL(swiotlb_map_single_attrs);
+
+dma_addr_t
+swiotlb_map_single(struct device *hwdev, void *ptr, size_t size, int dir)
+{
+        return swiotlb_map_single_attrs(hwdev, ptr, size, dir, NULL);
+}
 
 /*
  * Unmap a single streaming mode DMA translation.  The dma_addr and size must
@@ -609,8 +606,8 @@ swiotlb_map_single(struct device *hwdev, void *ptr, size_t size, int dir)
  * whatever the device wrote there.
  */
 void
-swiotlb_unmap_single(struct device *hwdev, dma_addr_t dev_addr, size_t size,
-                     int dir)
+swiotlb_unmap_single_attrs(struct device *hwdev, dma_addr_t dev_addr,
+                           size_t size, int dir, struct dma_attrs *attrs)
 {
         char *dma_addr = bus_to_virt(dev_addr);
 
@@ -620,7 +617,14 @@ swiotlb_unmap_single(struct device *hwdev, dma_addr_t dev_addr, size_t size,
         else if (dir == DMA_FROM_DEVICE)
                 dma_mark_clean(dma_addr, size);
 }
+EXPORT_SYMBOL(swiotlb_unmap_single_attrs);
 
+void
+swiotlb_unmap_single(struct device *hwdev, dma_addr_t dev_addr, size_t size,
+                     int dir)
+{
+        return swiotlb_unmap_single_attrs(hwdev, dev_addr, size, dir, NULL);
+}
 /*
  * Make physical memory consistent for a single streaming mode DMA translation
  * after a transfer.
@@ -691,6 +695,8 @@ swiotlb_sync_single_range_for_device(struct device *hwdev, dma_addr_t dev_addr,
                                   SYNC_FOR_DEVICE);
 }
 
+void swiotlb_unmap_sg_attrs(struct device *, struct scatterlist *, int, int,
+                            struct dma_attrs *);
 /*
  * Map a set of buffers described by scatterlist in streaming mode for DMA.
  * This is the scatter-gather version of the above swiotlb_map_single
@@ -708,8 +714,8 @@ swiotlb_sync_single_range_for_device(struct device *hwdev, dma_addr_t dev_addr,
  * same here.
  */
 int
-swiotlb_map_sg(struct device *hwdev, struct scatterlist *sgl, int nelems,
-               int dir)
+swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl, int nelems,
+                     int dir, struct dma_attrs *attrs)
 {
         struct scatterlist *sg;
         void *addr;
@@ -727,7 +733,8 @@ swiotlb_map_sg(struct device *hwdev, struct scatterlist *sgl, int nelems,
                                 /* Don't panic here, we expect map_sg users
                                    to do proper error handling. */
                                 swiotlb_full(hwdev, sg->length, dir, 0);
-                                swiotlb_unmap_sg(hwdev, sgl, i, dir);
+                                swiotlb_unmap_sg_attrs(hwdev, sgl, i, dir,
+                                                       attrs);
                                 sgl[0].dma_length = 0;
                                 return 0;
                         }
@@ -738,14 +745,22 @@ swiotlb_map_sg(struct device *hwdev, struct scatterlist *sgl, int nelems,
         }
         return nelems;
 }
+EXPORT_SYMBOL(swiotlb_map_sg_attrs);
+
+int
+swiotlb_map_sg(struct device *hwdev, struct scatterlist *sgl, int nelems,
+               int dir)
+{
+        return swiotlb_map_sg_attrs(hwdev, sgl, nelems, dir, NULL);
+}
 
 /*
  * Unmap a set of streaming mode DMA translations.  Again, cpu read rules
  * concerning calls here are the same as for swiotlb_unmap_single() above.
  */
 void
-swiotlb_unmap_sg(struct device *hwdev, struct scatterlist *sgl, int nelems,
-                 int dir)
+swiotlb_unmap_sg_attrs(struct device *hwdev, struct scatterlist *sgl,
+                       int nelems, int dir, struct dma_attrs *attrs)
 {
         struct scatterlist *sg;
         int i;
@@ -760,6 +775,14 @@ swiotlb_unmap_sg(struct device *hwdev, struct scatterlist *sgl, int nelems,
                         dma_mark_clean(SG_ENT_VIRT_ADDRESS(sg), sg->dma_length);
         }
 }
+EXPORT_SYMBOL(swiotlb_unmap_sg_attrs);
+
+void
+swiotlb_unmap_sg(struct device *hwdev, struct scatterlist *sgl, int nelems,
+                 int dir)
+{
+        return swiotlb_unmap_sg_attrs(hwdev, sgl, nelems, dir, NULL);
+}
 
 /*
  * Make physical memory consistent for a set of streaming mode DMA translations