7 files changed, 85 insertions, 69 deletions

diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile
index 6e9c1f320acf..c611ad64137f 100644
--- a/arch/x86/kernel/Makefile
+++ b/arch/x86/kernel/Makefile
@@ -105,7 +105,7 @@ obj-$(CONFIG_MICROCODE)			+= microcode.o
 
 obj-$(CONFIG_X86_CHECK_BIOS_CORRUPTION) += check.o
 
-obj-$(CONFIG_SWIOTLB)                   += pci-swiotlb_64.o # NB rename without _64
+obj-$(CONFIG_SWIOTLB)                   += pci-swiotlb.o
 
 ###
 # 64 bit specific files
diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c
index 5113c080f0c4..c5962fe3796f 100644
--- a/arch/x86/kernel/amd_iommu.c
+++ b/arch/x86/kernel/amd_iommu.c
@@ -22,10 +22,9 @@
 #include <linux/bitops.h>
 #include <linux/debugfs.h>
 #include <linux/scatterlist.h>
+#include <linux/dma-mapping.h>
 #include <linux/iommu-helper.h>
-#ifdef CONFIG_IOMMU_API
 #include <linux/iommu.h>
-#endif
 #include <asm/proto.h>
 #include <asm/iommu.h>
 #include <asm/gart.h>
@@ -1297,8 +1296,10 @@ static void __unmap_single(struct amd_iommu *iommu,
 /*
  * The exported map_single function for dma_ops.
  */
-static dma_addr_t map_single(struct device *dev, phys_addr_t paddr,
-                             size_t size, int dir)
+static dma_addr_t map_page(struct device *dev, struct page *page,
+                           unsigned long offset, size_t size,
+                           enum dma_data_direction dir,
+                           struct dma_attrs *attrs)
 {
         unsigned long flags;
         struct amd_iommu *iommu;
@@ -1306,6 +1307,7 @@ static dma_addr_t map_single(struct device *dev, phys_addr_t paddr,
         u16 devid;
         dma_addr_t addr;
         u64 dma_mask;
+        phys_addr_t paddr = page_to_phys(page) + offset;
 
         INC_STATS_COUNTER(cnt_map_single);
 
@@ -1340,8 +1342,8 @@ out:
 /*
  * The exported unmap_single function for dma_ops.
  */
-static void unmap_single(struct device *dev, dma_addr_t dma_addr,
-                         size_t size, int dir)
+static void unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size,
+                       enum dma_data_direction dir, struct dma_attrs *attrs)
 {
         unsigned long flags;
         struct amd_iommu *iommu;
@@ -1390,7 +1392,8 @@ static int map_sg_no_iommu(struct device *dev, struct scatterlist *sglist,
  * lists).
  */
 static int map_sg(struct device *dev, struct scatterlist *sglist,
-                  int nelems, int dir)
+                  int nelems, enum dma_data_direction dir,
+                  struct dma_attrs *attrs)
 {
         unsigned long flags;
         struct amd_iommu *iommu;
@@ -1457,7 +1460,8 @@ unmap:
  * lists).
  */
 static void unmap_sg(struct device *dev, struct scatterlist *sglist,
-                     int nelems, int dir)
+                     int nelems, enum dma_data_direction dir,
+                     struct dma_attrs *attrs)
 {
         unsigned long flags;
         struct amd_iommu *iommu;
@@ -1644,11 +1648,11 @@ static void prealloc_protection_domains(void)
         }
 }
 
-static struct dma_mapping_ops amd_iommu_dma_ops = {
+static struct dma_map_ops amd_iommu_dma_ops = {
         .alloc_coherent = alloc_coherent,
         .free_coherent = free_coherent,
-        .map_single = map_single,
-        .unmap_single = unmap_single,
+        .map_page = map_page,
+        .unmap_page = unmap_page,
         .map_sg = map_sg,
         .unmap_sg = unmap_sg,
         .dma_supported = amd_iommu_dma_supported,
diff --git a/arch/x86/kernel/pci-calgary_64.c b/arch/x86/kernel/pci-calgary_64.c
index d28bbdc35e4e..755c21e906f3 100644
--- a/arch/x86/kernel/pci-calgary_64.c
+++ b/arch/x86/kernel/pci-calgary_64.c
@@ -380,8 +380,9 @@ static inline struct iommu_table *find_iommu_table(struct device *dev)
         return tbl;
 }
 
-static void calgary_unmap_sg(struct device *dev,
-        struct scatterlist *sglist, int nelems, int direction)
+static void calgary_unmap_sg(struct device *dev, struct scatterlist *sglist,
+                             int nelems,enum dma_data_direction dir,
+                             struct dma_attrs *attrs)
 {
         struct iommu_table *tbl = find_iommu_table(dev);
         struct scatterlist *s;
@@ -404,7 +405,8 @@ static void calgary_unmap_sg(struct device *dev,
 }
 
 static int calgary_map_sg(struct device *dev, struct scatterlist *sg,
-        int nelems, int direction)
+                          int nelems, enum dma_data_direction dir,
+                          struct dma_attrs *attrs)
 {
         struct iommu_table *tbl = find_iommu_table(dev);
         struct scatterlist *s;
@@ -429,15 +431,14 @@ static int calgary_map_sg(struct device *dev, struct scatterlist *sg,
                 s->dma_address = (entry << PAGE_SHIFT) | s->offset;
 
                 /* insert into HW table */
-                tce_build(tbl, entry, npages, vaddr & PAGE_MASK,
-                          direction);
+                tce_build(tbl, entry, npages, vaddr & PAGE_MASK, dir);
 
                 s->dma_length = s->length;
         }
 
         return nelems;
 error:
-        calgary_unmap_sg(dev, sg, nelems, direction);
+        calgary_unmap_sg(dev, sg, nelems, dir, NULL);
         for_each_sg(sg, s, nelems, i) {
                 sg->dma_address = bad_dma_address;
                 sg->dma_length = 0;
@@ -445,10 +446,12 @@ error:
         return 0;
 }
 
-static dma_addr_t calgary_map_single(struct device *dev, phys_addr_t paddr,
-        size_t size, int direction)
+static dma_addr_t calgary_map_page(struct device *dev, struct page *page,
+                                   unsigned long offset, size_t size,
+                                   enum dma_data_direction dir,
+                                   struct dma_attrs *attrs)
 {
-        void *vaddr = phys_to_virt(paddr);
+        void *vaddr = page_address(page) + offset;
         unsigned long uaddr;
         unsigned int npages;
         struct iommu_table *tbl = find_iommu_table(dev);
@@ -456,17 +459,18 @@ static dma_addr_t calgary_map_single(struct device *dev, phys_addr_t paddr,
         uaddr = (unsigned long)vaddr;
         npages = iommu_num_pages(uaddr, size, PAGE_SIZE);
 
-        return iommu_alloc(dev, tbl, vaddr, npages, direction);
+        return iommu_alloc(dev, tbl, vaddr, npages, dir);
 }
 
-static void calgary_unmap_single(struct device *dev, dma_addr_t dma_handle,
-        size_t size, int direction)
+static void calgary_unmap_page(struct device *dev, dma_addr_t dma_addr,
+                               size_t size, enum dma_data_direction dir,
+                               struct dma_attrs *attrs)
 {
         struct iommu_table *tbl = find_iommu_table(dev);
         unsigned int npages;
 
-        npages = iommu_num_pages(dma_handle, size, PAGE_SIZE);
-        iommu_free(tbl, dma_handle, npages);
+        npages = iommu_num_pages(dma_addr, size, PAGE_SIZE);
+        iommu_free(tbl, dma_addr, npages);
 }
 
 static void* calgary_alloc_coherent(struct device *dev, size_t size,
@@ -515,13 +519,13 @@ static void calgary_free_coherent(struct device *dev, size_t size,
         free_pages((unsigned long)vaddr, get_order(size));
 }
 
-static struct dma_mapping_ops calgary_dma_ops = {
+static struct dma_map_ops calgary_dma_ops = {
         .alloc_coherent = calgary_alloc_coherent,
         .free_coherent = calgary_free_coherent,
-        .map_single = calgary_map_single,
-        .unmap_single = calgary_unmap_single,
         .map_sg = calgary_map_sg,
         .unmap_sg = calgary_unmap_sg,
+        .map_page = calgary_map_page,
+        .unmap_page = calgary_unmap_page,
 };
 
 static inline void __iomem * busno_to_bbar(unsigned char num)
diff --git a/arch/x86/kernel/pci-dma.c b/arch/x86/kernel/pci-dma.c
index b25428533141..c7c4776ff630 100644
--- a/arch/x86/kernel/pci-dma.c
+++ b/arch/x86/kernel/pci-dma.c
@@ -1,4 +1,5 @@
 #include <linux/dma-mapping.h>
+#include <linux/dma-debug.h>
 #include <linux/dmar.h>
 #include <linux/bootmem.h>
 #include <linux/pci.h>
@@ -12,7 +13,7 @@
 
 static int forbid_dac __read_mostly;
 
-struct dma_mapping_ops *dma_ops;
+struct dma_map_ops *dma_ops;
 EXPORT_SYMBOL(dma_ops);
 
 static int iommu_sac_force __read_mostly;
@@ -44,6 +45,9 @@ struct device x86_dma_fallback_dev = {
 };
 EXPORT_SYMBOL(x86_dma_fallback_dev);
 
+/* Number of entries preallocated for DMA-API debugging */
+#define PREALLOC_DMA_DEBUG_ENTRIES       32768
+
 int dma_set_mask(struct device *dev, u64 mask)
 {
         if (!dev->dma_mask || !dma_supported(dev, mask))
@@ -224,7 +228,7 @@ early_param("iommu", iommu_setup);
 
 int dma_supported(struct device *dev, u64 mask)
 {
-        struct dma_mapping_ops *ops = get_dma_ops(dev);
+        struct dma_map_ops *ops = get_dma_ops(dev);
 
 #ifdef CONFIG_PCI
         if (mask > 0xffffffff && forbid_dac > 0) {
@@ -265,6 +269,12 @@ EXPORT_SYMBOL(dma_supported);
 
 static int __init pci_iommu_init(void)
 {
+        dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
+
+#ifdef CONFIG_PCI
+        dma_debug_add_bus(&pci_bus_type);
+#endif
+
         calgary_iommu_init();
 
         intel_iommu_init();
diff --git a/arch/x86/kernel/pci-gart_64.c b/arch/x86/kernel/pci-gart_64.c
index d5768b1af080..b284b58c035c 100644
--- a/arch/x86/kernel/pci-gart_64.c
+++ b/arch/x86/kernel/pci-gart_64.c
@@ -255,10 +255,13 @@ static dma_addr_t dma_map_area(struct device *dev, dma_addr_t phys_mem,
 }
 
 /* Map a single area into the IOMMU */
-static dma_addr_t
-gart_map_single(struct device *dev, phys_addr_t paddr, size_t size, int dir)
+static dma_addr_t gart_map_page(struct device *dev, struct page *page,
+                                unsigned long offset, size_t size,
+                                enum dma_data_direction dir,
+                                struct dma_attrs *attrs)
 {
         unsigned long bus;
+        phys_addr_t paddr = page_to_phys(page) + offset;
 
         if (!dev)
                 dev = &x86_dma_fallback_dev;
@@ -275,8 +278,9 @@ gart_map_single(struct device *dev, phys_addr_t paddr, size_t size, int dir)
 /*
  * Free a DMA mapping.
  */
-static void gart_unmap_single(struct device *dev, dma_addr_t dma_addr,
-                              size_t size, int direction)
+static void gart_unmap_page(struct device *dev, dma_addr_t dma_addr,
+                            size_t size, enum dma_data_direction dir,
+                            struct dma_attrs *attrs)
 {
         unsigned long iommu_page;
         int npages;
@@ -298,8 +302,8 @@ static void gart_unmap_single(struct device *dev, dma_addr_t dma_addr,
 /*
  * Wrapper for pci_unmap_single working with scatterlists.
  */
-static void
-gart_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, int dir)
+static void gart_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
+                          enum dma_data_direction dir, struct dma_attrs *attrs)
 {
         struct scatterlist *s;
         int i;
@@ -307,7 +311,7 @@ gart_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, int dir)
         for_each_sg(sg, s, nents, i) {
                 if (!s->dma_length || !s->length)
                         break;
-                gart_unmap_single(dev, s->dma_address, s->dma_length, dir);
+                gart_unmap_page(dev, s->dma_address, s->dma_length, dir, NULL);
         }
 }
 
@@ -329,7 +333,7 @@ static int dma_map_sg_nonforce(struct device *dev, struct scatterlist *sg,
                         addr = dma_map_area(dev, addr, s->length, dir, 0);
                         if (addr == bad_dma_address) {
                                 if (i > 0)
-                                        gart_unmap_sg(dev, sg, i, dir);
+                                        gart_unmap_sg(dev, sg, i, dir, NULL);
                                 nents = 0;
                                 sg[0].dma_length = 0;
                                 break;
@@ -400,8 +404,8 @@ dma_map_cont(struct device *dev, struct scatterlist *start, int nelems,
  * DMA map all entries in a scatterlist.
  * Merge chunks that have page aligned sizes into a continuous mapping.
  */
-static int
-gart_map_sg(struct device *dev, struct scatterlist *sg, int nents, int dir)
+static int gart_map_sg(struct device *dev, struct scatterlist *sg, int nents,
+                       enum dma_data_direction dir, struct dma_attrs *attrs)
 {
         struct scatterlist *s, *ps, *start_sg, *sgmap;
         int need = 0, nextneed, i, out, start;
@@ -468,7 +472,7 @@ gart_map_sg(struct device *dev, struct scatterlist *sg, int nents, int dir)
 
 error:
         flush_gart();
-        gart_unmap_sg(dev, sg, out, dir);
+        gart_unmap_sg(dev, sg, out, dir, NULL);
 
         /* When it was forced or merged try again in a dumb way */
         if (force_iommu || iommu_merge) {
@@ -521,7 +525,7 @@ static void
 gart_free_coherent(struct device *dev, size_t size, void *vaddr,
                    dma_addr_t dma_addr)
 {
-        gart_unmap_single(dev, dma_addr, size, DMA_BIDIRECTIONAL);
+        gart_unmap_page(dev, dma_addr, size, DMA_BIDIRECTIONAL, NULL);
         free_pages((unsigned long)vaddr, get_order(size));
 }
 
@@ -707,11 +711,11 @@ static __init int init_k8_gatt(struct agp_kern_info *info)
         return -1;
 }
 
-static struct dma_mapping_ops gart_dma_ops = {
-        .map_single                     = gart_map_single,
-        .unmap_single                   = gart_unmap_single,
+static struct dma_map_ops gart_dma_ops = {
         .map_sg                         = gart_map_sg,
         .unmap_sg                       = gart_unmap_sg,
+        .map_page                       = gart_map_page,
+        .unmap_page                     = gart_unmap_page,
         .alloc_coherent                 = gart_alloc_coherent,
         .free_coherent                  = gart_free_coherent,
 };
diff --git a/arch/x86/kernel/pci-nommu.c b/arch/x86/kernel/pci-nommu.c
index 8b02a3936d42..c6d703b39326 100644
--- a/arch/x86/kernel/pci-nommu.c
+++ b/arch/x86/kernel/pci-nommu.c
@@ -25,19 +25,19 @@ check_addr(char *name, struct device *hwdev, dma_addr_t bus, size_t size)
         return 1;
 }
 
-static dma_addr_t
-nommu_map_single(struct device *hwdev, phys_addr_t paddr, size_t size,
-               int direction)
+static dma_addr_t nommu_map_page(struct device *dev, struct page *page,
+                                 unsigned long offset, size_t size,
+                                 enum dma_data_direction dir,
+                                 struct dma_attrs *attrs)
 {
-        dma_addr_t bus = paddr;
+        dma_addr_t bus = page_to_phys(page) + offset;
         WARN_ON(size == 0);
-        if (!check_addr("map_single", hwdev, bus, size))
-                                return bad_dma_address;
+        if (!check_addr("map_single", dev, bus, size))
+                return bad_dma_address;
         flush_write_buffers();
         return bus;
 }
 
-
 /* Map a set of buffers described by scatterlist in streaming
  * mode for DMA.  This is the scatter-gather version of the
  * above pci_map_single interface.  Here the scatter gather list
@@ -54,7 +54,8 @@ nommu_map_single(struct device *hwdev, phys_addr_t paddr, size_t size,
  * the same here.
  */
 static int nommu_map_sg(struct device *hwdev, struct scatterlist *sg,
-               int nents, int direction)
+                        int nents, enum dma_data_direction dir,
+                        struct dma_attrs *attrs)
 {
         struct scatterlist *s;
         int i;
@@ -78,11 +79,11 @@ static void nommu_free_coherent(struct device *dev, size_t size, void *vaddr,
         free_pages((unsigned long)vaddr, get_order(size));
 }
 
-struct dma_mapping_ops nommu_dma_ops = {
+struct dma_map_ops nommu_dma_ops = {
         .alloc_coherent = dma_generic_alloc_coherent,
         .free_coherent  = nommu_free_coherent,
-        .map_single     = nommu_map_single,
         .map_sg         = nommu_map_sg,
+        .map_page       = nommu_map_page,
         .is_phys        = 1,
 };
 
diff --git a/arch/x86/kernel/pci-swiotlb_64.c b/arch/x86/kernel/pci-swiotlb.c
index d59c91747665..34f12e9996ed 100644
--- a/arch/x86/kernel/pci-swiotlb_64.c
+++ b/arch/x86/kernel/pci-swiotlb.c
@@ -33,18 +33,11 @@ phys_addr_t swiotlb_bus_to_phys(dma_addr_t baddr)
         return baddr;
 }
 
-int __weak swiotlb_arch_range_needs_mapping(void *ptr, size_t size)
+int __weak swiotlb_arch_range_needs_mapping(phys_addr_t paddr, size_t size)
 {
         return 0;
 }
 
-static dma_addr_t
-swiotlb_map_single_phys(struct device *hwdev, phys_addr_t paddr, size_t size,
-                        int direction)
-{
-        return swiotlb_map_single(hwdev, phys_to_virt(paddr), size, direction);
-}
-
 static void *x86_swiotlb_alloc_coherent(struct device *hwdev, size_t size,
                                         dma_addr_t *dma_handle, gfp_t flags)
 {
@@ -57,20 +50,20 @@ static void *x86_swiotlb_alloc_coherent(struct device *hwdev, size_t size,
         return swiotlb_alloc_coherent(hwdev, size, dma_handle, flags);
 }
 
-struct dma_mapping_ops swiotlb_dma_ops = {
+struct dma_map_ops swiotlb_dma_ops = {
         .mapping_error = swiotlb_dma_mapping_error,
         .alloc_coherent = x86_swiotlb_alloc_coherent,
         .free_coherent = swiotlb_free_coherent,
-        .map_single = swiotlb_map_single_phys,
-        .unmap_single = swiotlb_unmap_single,
         .sync_single_for_cpu = swiotlb_sync_single_for_cpu,
         .sync_single_for_device = swiotlb_sync_single_for_device,
         .sync_single_range_for_cpu = swiotlb_sync_single_range_for_cpu,
         .sync_single_range_for_device = swiotlb_sync_single_range_for_device,
         .sync_sg_for_cpu = swiotlb_sync_sg_for_cpu,
         .sync_sg_for_device = swiotlb_sync_sg_for_device,
-        .map_sg = swiotlb_map_sg,
-        .unmap_sg = swiotlb_unmap_sg,
+        .map_sg = swiotlb_map_sg_attrs,
+        .unmap_sg = swiotlb_unmap_sg_attrs,
+        .map_page = swiotlb_map_page,
+        .unmap_page = swiotlb_unmap_page,
         .dma_supported = NULL,
 };