11 files changed, 181 insertions, 166 deletions

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 34bc3a89228b..45161b816313 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -40,6 +40,7 @@ config X86
         select HAVE_GENERIC_DMA_COHERENT if X86_32
         select HAVE_EFFICIENT_UNALIGNED_ACCESS
         select USER_STACKTRACE_SUPPORT
+        select HAVE_DMA_API_DEBUG
         select HAVE_KERNEL_GZIP
         select HAVE_KERNEL_BZIP2
         select HAVE_KERNEL_LZMA
diff --git a/arch/x86/include/asm/device.h b/arch/x86/include/asm/device.h
index 3c034f48fdb0..4994a20acbcb 100644
--- a/arch/x86/include/asm/device.h
+++ b/arch/x86/include/asm/device.h
@@ -6,7 +6,7 @@ struct dev_archdata {
         void    *acpi_handle;
 #endif
 #ifdef CONFIG_X86_64
-struct dma_mapping_ops *dma_ops;
+struct dma_map_ops *dma_ops;
 #endif
 #ifdef CONFIG_DMAR
         void *iommu; /* hook for IOMMU specific extension */
diff --git a/arch/x86/include/asm/dma-mapping.h b/arch/x86/include/asm/dma-mapping.h
index 132a134d12f2..cea7b74963e9 100644
--- a/arch/x86/include/asm/dma-mapping.h
+++ b/arch/x86/include/asm/dma-mapping.h
@@ -7,6 +7,8 @@
  */
 
 #include <linux/scatterlist.h>
+#include <linux/dma-debug.h>
+#include <linux/dma-attrs.h>
 #include <asm/io.h>
 #include <asm/swiotlb.h>
 #include <asm-generic/dma-coherent.h>
@@ -16,47 +18,9 @@ extern int iommu_merge;
 extern struct device x86_dma_fallback_dev;
 extern int panic_on_overflow;
 
-struct dma_mapping_ops {
-        int             (*mapping_error)(struct device *dev,
-                                         dma_addr_t dma_addr);
-        void*           (*alloc_coherent)(struct device *dev, size_t size,
-                                dma_addr_t *dma_handle, gfp_t gfp);
-        void            (*free_coherent)(struct device *dev, size_t size,
-                                void *vaddr, dma_addr_t dma_handle);
-        dma_addr_t      (*map_single)(struct device *hwdev, phys_addr_t ptr,
-                                size_t size, int direction);
-        void            (*unmap_single)(struct device *dev, dma_addr_t addr,
-                                size_t size, int direction);
-        void            (*sync_single_for_cpu)(struct device *hwdev,
-                                dma_addr_t dma_handle, size_t size,
-                                int direction);
-        void            (*sync_single_for_device)(struct device *hwdev,
-                                dma_addr_t dma_handle, size_t size,
-                                int direction);
-        void            (*sync_single_range_for_cpu)(struct device *hwdev,
-                                dma_addr_t dma_handle, unsigned long offset,
-                                size_t size, int direction);
-        void            (*sync_single_range_for_device)(struct device *hwdev,
-                                dma_addr_t dma_handle, unsigned long offset,
-                                size_t size, int direction);
-        void            (*sync_sg_for_cpu)(struct device *hwdev,
-                                struct scatterlist *sg, int nelems,
-                                int direction);
-        void            (*sync_sg_for_device)(struct device *hwdev,
-                                struct scatterlist *sg, int nelems,
-                                int direction);
-        int             (*map_sg)(struct device *hwdev, struct scatterlist *sg,
-                                int nents, int direction);
-        void            (*unmap_sg)(struct device *hwdev,
-                                struct scatterlist *sg, int nents,
-                                int direction);
-        int             (*dma_supported)(struct device *hwdev, u64 mask);
-        int             is_phys;
-};
-
-extern struct dma_mapping_ops *dma_ops;
-
-static inline struct dma_mapping_ops *get_dma_ops(struct device *dev)
+extern struct dma_map_ops *dma_ops;
+
+static inline struct dma_map_ops *get_dma_ops(struct device *dev)
 {
 #ifdef CONFIG_X86_32
         return dma_ops;
@@ -71,7 +35,7 @@ static inline struct dma_mapping_ops *get_dma_ops(struct device *dev)
 /* Make sure we keep the same behaviour */
 static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
 {
-        struct dma_mapping_ops *ops = get_dma_ops(dev);
+        struct dma_map_ops *ops = get_dma_ops(dev);
         if (ops->mapping_error)
                 return ops->mapping_error(dev, dma_addr);
 
@@ -90,137 +54,167 @@ extern void *dma_generic_alloc_coherent(struct device *dev, size_t size,
 
 static inline dma_addr_t
 dma_map_single(struct device *hwdev, void *ptr, size_t size,
-               int direction)
+               enum dma_data_direction dir)
 {
-        struct dma_mapping_ops *ops = get_dma_ops(hwdev);
-
-        BUG_ON(!valid_dma_direction(direction));
-        return ops->map_single(hwdev, virt_to_phys(ptr), size, direction);
+        struct dma_map_ops *ops = get_dma_ops(hwdev);
+        dma_addr_t addr;
+
+        BUG_ON(!valid_dma_direction(dir));
+        addr = ops->map_page(hwdev, virt_to_page(ptr),
+                             (unsigned long)ptr & ~PAGE_MASK, size,
+                             dir, NULL);
+        debug_dma_map_page(hwdev, virt_to_page(ptr),
+                           (unsigned long)ptr & ~PAGE_MASK, size,
+                           dir, addr, true);
+        return addr;
 }
 
 static inline void
 dma_unmap_single(struct device *dev, dma_addr_t addr, size_t size,
-                 int direction)
+                 enum dma_data_direction dir)
 {
-        struct dma_mapping_ops *ops = get_dma_ops(dev);
+        struct dma_map_ops *ops = get_dma_ops(dev);
 
-        BUG_ON(!valid_dma_direction(direction));
-        if (ops->unmap_single)
-                ops->unmap_single(dev, addr, size, direction);
+        BUG_ON(!valid_dma_direction(dir));
+        if (ops->unmap_page)
+                ops->unmap_page(dev, addr, size, dir, NULL);
+        debug_dma_unmap_page(dev, addr, size, dir, true);
 }
 
 static inline int
 dma_map_sg(struct device *hwdev, struct scatterlist *sg,
-           int nents, int direction)
+           int nents, enum dma_data_direction dir)
 {
-        struct dma_mapping_ops *ops = get_dma_ops(hwdev);
+        struct dma_map_ops *ops = get_dma_ops(hwdev);
+        int ents;
+
+        BUG_ON(!valid_dma_direction(dir));
+        ents = ops->map_sg(hwdev, sg, nents, dir, NULL);
+        debug_dma_map_sg(hwdev, sg, nents, ents, dir);
 
-        BUG_ON(!valid_dma_direction(direction));
-        return ops->map_sg(hwdev, sg, nents, direction);
+        return ents;
 }
 
 static inline void
 dma_unmap_sg(struct device *hwdev, struct scatterlist *sg, int nents,
-             int direction)
+             enum dma_data_direction dir)
 {
-        struct dma_mapping_ops *ops = get_dma_ops(hwdev);
+        struct dma_map_ops *ops = get_dma_ops(hwdev);
 
-        BUG_ON(!valid_dma_direction(direction));
+        BUG_ON(!valid_dma_direction(dir));
+        debug_dma_unmap_sg(hwdev, sg, nents, dir);
         if (ops->unmap_sg)
-                ops->unmap_sg(hwdev, sg, nents, direction);
+                ops->unmap_sg(hwdev, sg, nents, dir, NULL);
 }
 
 static inline void
 dma_sync_single_for_cpu(struct device *hwdev, dma_addr_t dma_handle,
-                        size_t size, int direction)
+                        size_t size, enum dma_data_direction dir)
 {
-        struct dma_mapping_ops *ops = get_dma_ops(hwdev);
+        struct dma_map_ops *ops = get_dma_ops(hwdev);
 
-        BUG_ON(!valid_dma_direction(direction));
+        BUG_ON(!valid_dma_direction(dir));
         if (ops->sync_single_for_cpu)
-                ops->sync_single_for_cpu(hwdev, dma_handle, size, direction);
+                ops->sync_single_for_cpu(hwdev, dma_handle, size, dir);
+        debug_dma_sync_single_for_cpu(hwdev, dma_handle, size, dir);
         flush_write_buffers();
 }
 
 static inline void
 dma_sync_single_for_device(struct device *hwdev, dma_addr_t dma_handle,
-                           size_t size, int direction)
+                           size_t size, enum dma_data_direction dir)
 {
-        struct dma_mapping_ops *ops = get_dma_ops(hwdev);
+        struct dma_map_ops *ops = get_dma_ops(hwdev);
 
-        BUG_ON(!valid_dma_direction(direction));
+        BUG_ON(!valid_dma_direction(dir));
         if (ops->sync_single_for_device)
-                ops->sync_single_for_device(hwdev, dma_handle, size, direction);
+                ops->sync_single_for_device(hwdev, dma_handle, size, dir);
+        debug_dma_sync_single_for_device(hwdev, dma_handle, size, dir);
         flush_write_buffers();
 }
 
 static inline void
 dma_sync_single_range_for_cpu(struct device *hwdev, dma_addr_t dma_handle,
-                              unsigned long offset, size_t size, int direction)
+                              unsigned long offset, size_t size,
+                              enum dma_data_direction dir)
 {
-        struct dma_mapping_ops *ops = get_dma_ops(hwdev);
+        struct dma_map_ops *ops = get_dma_ops(hwdev);
 
-        BUG_ON(!valid_dma_direction(direction));
+        BUG_ON(!valid_dma_direction(dir));
         if (ops->sync_single_range_for_cpu)
                 ops->sync_single_range_for_cpu(hwdev, dma_handle, offset,
-                                               size, direction);
+                                               size, dir);
+        debug_dma_sync_single_range_for_cpu(hwdev, dma_handle,
+                                            offset, size, dir);
         flush_write_buffers();
 }
 
 static inline void
 dma_sync_single_range_for_device(struct device *hwdev, dma_addr_t dma_handle,
                                  unsigned long offset, size_t size,
-                                 int direction)
+                                 enum dma_data_direction dir)
 {
-        struct dma_mapping_ops *ops = get_dma_ops(hwdev);
+        struct dma_map_ops *ops = get_dma_ops(hwdev);
 
-        BUG_ON(!valid_dma_direction(direction));
+        BUG_ON(!valid_dma_direction(dir));
         if (ops->sync_single_range_for_device)
                 ops->sync_single_range_for_device(hwdev, dma_handle,
-                                                  offset, size, direction);
+                                                  offset, size, dir);
+        debug_dma_sync_single_range_for_device(hwdev, dma_handle,
+                                               offset, size, dir);
         flush_write_buffers();
 }
 
 static inline void
 dma_sync_sg_for_cpu(struct device *hwdev, struct scatterlist *sg,
-                    int nelems, int direction)
+                    int nelems, enum dma_data_direction dir)
 {
-        struct dma_mapping_ops *ops = get_dma_ops(hwdev);
+        struct dma_map_ops *ops = get_dma_ops(hwdev);
 
-        BUG_ON(!valid_dma_direction(direction));
+        BUG_ON(!valid_dma_direction(dir));
         if (ops->sync_sg_for_cpu)
-                ops->sync_sg_for_cpu(hwdev, sg, nelems, direction);
+                ops->sync_sg_for_cpu(hwdev, sg, nelems, dir);
+        debug_dma_sync_sg_for_cpu(hwdev, sg, nelems, dir);
         flush_write_buffers();
 }
 
 static inline void
 dma_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg,
-                       int nelems, int direction)
+                       int nelems, enum dma_data_direction dir)
 {
-        struct dma_mapping_ops *ops = get_dma_ops(hwdev);
+        struct dma_map_ops *ops = get_dma_ops(hwdev);
 
-        BUG_ON(!valid_dma_direction(direction));
+        BUG_ON(!valid_dma_direction(dir));
         if (ops->sync_sg_for_device)
-                ops->sync_sg_for_device(hwdev, sg, nelems, direction);
+                ops->sync_sg_for_device(hwdev, sg, nelems, dir);
+        debug_dma_sync_sg_for_device(hwdev, sg, nelems, dir);
 
         flush_write_buffers();
 }
 
 static inline dma_addr_t dma_map_page(struct device *dev, struct page *page,
                                       size_t offset, size_t size,
-                                      int direction)
+                                      enum dma_data_direction dir)
 {
-        struct dma_mapping_ops *ops = get_dma_ops(dev);
+        struct dma_map_ops *ops = get_dma_ops(dev);
+        dma_addr_t addr;
 
-        BUG_ON(!valid_dma_direction(direction));
-        return ops->map_single(dev, page_to_phys(page) + offset,
-                               size, direction);
+        BUG_ON(!valid_dma_direction(dir));
+        addr = ops->map_page(dev, page, offset, size, dir, NULL);
+        debug_dma_map_page(dev, page, offset, size, dir, addr, false);
+
+        return addr;
 }
 
 static inline void dma_unmap_page(struct device *dev, dma_addr_t addr,
-                                  size_t size, int direction)
+                                  size_t size, enum dma_data_direction dir)
 {
-        dma_unmap_single(dev, addr, size, direction);
+        struct dma_map_ops *ops = get_dma_ops(dev);
+
+        BUG_ON(!valid_dma_direction(dir));
+        if (ops->unmap_page)
+                ops->unmap_page(dev, addr, size, dir, NULL);
+        debug_dma_unmap_page(dev, addr, size, dir, false);
 }
 
 static inline void
@@ -266,7 +260,7 @@ static inline void *
 dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle,
                 gfp_t gfp)
 {
-        struct dma_mapping_ops *ops = get_dma_ops(dev);
+        struct dma_map_ops *ops = get_dma_ops(dev);
         void *memory;
 
         gfp &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32);
@@ -285,20 +279,24 @@ dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle,
         if (!ops->alloc_coherent)
                 return NULL;
 
-        return ops->alloc_coherent(dev, size, dma_handle,
-                                   dma_alloc_coherent_gfp_flags(dev, gfp));
+        memory = ops->alloc_coherent(dev, size, dma_handle,
+                                     dma_alloc_coherent_gfp_flags(dev, gfp));
+        debug_dma_alloc_coherent(dev, size, *dma_handle, memory);
+
+        return memory;
 }
 
 static inline void dma_free_coherent(struct device *dev, size_t size,
                                      void *vaddr, dma_addr_t bus)
 {
-        struct dma_mapping_ops *ops = get_dma_ops(dev);
+        struct dma_map_ops *ops = get_dma_ops(dev);
 
         WARN_ON(irqs_disabled());       /* for portability */
 
         if (dma_release_from_coherent(dev, get_order(size), vaddr))
                 return;
 
+        debug_dma_free_coherent(dev, size, vaddr, bus);
         if (ops->free_coherent)
                 ops->free_coherent(dev, size, vaddr, bus);
 }
diff --git a/arch/x86/include/asm/iommu.h b/arch/x86/include/asm/iommu.h
index a6ee9e6f530f..af326a2975b5 100644
--- a/arch/x86/include/asm/iommu.h
+++ b/arch/x86/include/asm/iommu.h
@@ -3,7 +3,7 @@
 
 extern void pci_iommu_shutdown(void);
 extern void no_iommu_init(void);
-extern struct dma_mapping_ops nommu_dma_ops;
+extern struct dma_map_ops nommu_dma_ops;
 extern int force_iommu, no_iommu;
 extern int iommu_detected;
 
diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile
index 6e9c1f320acf..c611ad64137f 100644
--- a/arch/x86/kernel/Makefile
+++ b/arch/x86/kernel/Makefile
@@ -105,7 +105,7 @@ obj-$(CONFIG_MICROCODE)			+= microcode.o
 
 obj-$(CONFIG_X86_CHECK_BIOS_CORRUPTION) += check.o
 
-obj-$(CONFIG_SWIOTLB)                   += pci-swiotlb_64.o # NB rename without _64
+obj-$(CONFIG_SWIOTLB)                   += pci-swiotlb.o
 
 ###
 # 64 bit specific files
diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c
index 5113c080f0c4..c5962fe3796f 100644
--- a/arch/x86/kernel/amd_iommu.c
+++ b/arch/x86/kernel/amd_iommu.c
@@ -22,10 +22,9 @@
 #include <linux/bitops.h>
 #include <linux/debugfs.h>
 #include <linux/scatterlist.h>
+#include <linux/dma-mapping.h>
 #include <linux/iommu-helper.h>
-#ifdef CONFIG_IOMMU_API
 #include <linux/iommu.h>
-#endif
 #include <asm/proto.h>
 #include <asm/iommu.h>
 #include <asm/gart.h>
@@ -1297,8 +1296,10 @@ static void __unmap_single(struct amd_iommu *iommu,
 /*
  * The exported map_single function for dma_ops.
  */
-static dma_addr_t map_single(struct device *dev, phys_addr_t paddr,
-                             size_t size, int dir)
+static dma_addr_t map_page(struct device *dev, struct page *page,
+                           unsigned long offset, size_t size,
+                           enum dma_data_direction dir,
+                           struct dma_attrs *attrs)
 {
         unsigned long flags;
         struct amd_iommu *iommu;
@@ -1306,6 +1307,7 @@ static dma_addr_t map_single(struct device *dev, phys_addr_t paddr,
         u16 devid;
         dma_addr_t addr;
         u64 dma_mask;
+        phys_addr_t paddr = page_to_phys(page) + offset;
 
         INC_STATS_COUNTER(cnt_map_single);
 
@@ -1340,8 +1342,8 @@ out:
 /*
  * The exported unmap_single function for dma_ops.
  */
-static void unmap_single(struct device *dev, dma_addr_t dma_addr,
-                         size_t size, int dir)
+static void unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size,
+                       enum dma_data_direction dir, struct dma_attrs *attrs)
 {
         unsigned long flags;
         struct amd_iommu *iommu;
@@ -1390,7 +1392,8 @@ static int map_sg_no_iommu(struct device *dev, struct scatterlist *sglist,
  * lists).
  */
 static int map_sg(struct device *dev, struct scatterlist *sglist,
-                  int nelems, int dir)
+                  int nelems, enum dma_data_direction dir,
+                  struct dma_attrs *attrs)
 {
         unsigned long flags;
         struct amd_iommu *iommu;
@@ -1457,7 +1460,8 @@ unmap:
  * lists).
  */
 static void unmap_sg(struct device *dev, struct scatterlist *sglist,
-                     int nelems, int dir)
+                     int nelems, enum dma_data_direction dir,
+                     struct dma_attrs *attrs)
 {
         unsigned long flags;
         struct amd_iommu *iommu;
@@ -1644,11 +1648,11 @@ static void prealloc_protection_domains(void)
         }
 }
 
-static struct dma_mapping_ops amd_iommu_dma_ops = {
+static struct dma_map_ops amd_iommu_dma_ops = {
         .alloc_coherent = alloc_coherent,
         .free_coherent = free_coherent,
-        .map_single = map_single,
-        .unmap_single = unmap_single,
+        .map_page = map_page,
+        .unmap_page = unmap_page,
         .map_sg = map_sg,
         .unmap_sg = unmap_sg,
         .dma_supported = amd_iommu_dma_supported,
diff --git a/arch/x86/kernel/pci-calgary_64.c b/arch/x86/kernel/pci-calgary_64.c
index d28bbdc35e4e..755c21e906f3 100644
--- a/arch/x86/kernel/pci-calgary_64.c
+++ b/arch/x86/kernel/pci-calgary_64.c
@@ -380,8 +380,9 @@ static inline struct iommu_table *find_iommu_table(struct device *dev)
         return tbl;
 }
 
-static void calgary_unmap_sg(struct device *dev,
-        struct scatterlist *sglist, int nelems, int direction)
+static void calgary_unmap_sg(struct device *dev, struct scatterlist *sglist,
+                             int nelems,enum dma_data_direction dir,
+                             struct dma_attrs *attrs)
 {
         struct iommu_table *tbl = find_iommu_table(dev);
         struct scatterlist *s;
@@ -404,7 +405,8 @@ static void calgary_unmap_sg(struct device *dev,
 }
 
 static int calgary_map_sg(struct device *dev, struct scatterlist *sg,
-        int nelems, int direction)
+                          int nelems, enum dma_data_direction dir,
+                          struct dma_attrs *attrs)
 {
         struct iommu_table *tbl = find_iommu_table(dev);
         struct scatterlist *s;
@@ -429,15 +431,14 @@ static int calgary_map_sg(struct device *dev, struct scatterlist *sg,
                 s->dma_address = (entry << PAGE_SHIFT) | s->offset;
 
                 /* insert into HW table */
-                tce_build(tbl, entry, npages, vaddr & PAGE_MASK,
-                          direction);
+                tce_build(tbl, entry, npages, vaddr & PAGE_MASK, dir);
 
                 s->dma_length = s->length;
         }
 
         return nelems;
 error:
-        calgary_unmap_sg(dev, sg, nelems, direction);
+        calgary_unmap_sg(dev, sg, nelems, dir, NULL);
         for_each_sg(sg, s, nelems, i) {
                 sg->dma_address = bad_dma_address;
                 sg->dma_length = 0;
@@ -445,10 +446,12 @@ error:
         return 0;
 }
 
-static dma_addr_t calgary_map_single(struct device *dev, phys_addr_t paddr,
-        size_t size, int direction)
+static dma_addr_t calgary_map_page(struct device *dev, struct page *page,
+                                   unsigned long offset, size_t size,
+                                   enum dma_data_direction dir,
+                                   struct dma_attrs *attrs)
 {
-        void *vaddr = phys_to_virt(paddr);
+        void *vaddr = page_address(page) + offset;
         unsigned long uaddr;
         unsigned int npages;
         struct iommu_table *tbl = find_iommu_table(dev);
@@ -456,17 +459,18 @@ static dma_addr_t calgary_map_single(struct device *dev, phys_addr_t paddr,
         uaddr = (unsigned long)vaddr;
         npages = iommu_num_pages(uaddr, size, PAGE_SIZE);
 
-        return iommu_alloc(dev, tbl, vaddr, npages, direction);
+        return iommu_alloc(dev, tbl, vaddr, npages, dir);
 }
 
-static void calgary_unmap_single(struct device *dev, dma_addr_t dma_handle,
-        size_t size, int direction)
+static void calgary_unmap_page(struct device *dev, dma_addr_t dma_addr,
+                               size_t size, enum dma_data_direction dir,
+                               struct dma_attrs *attrs)
 {
         struct iommu_table *tbl = find_iommu_table(dev);
         unsigned int npages;
 
-        npages = iommu_num_pages(dma_handle, size, PAGE_SIZE);
-        iommu_free(tbl, dma_handle, npages);
+        npages = iommu_num_pages(dma_addr, size, PAGE_SIZE);
+        iommu_free(tbl, dma_addr, npages);
 }
 
 static void* calgary_alloc_coherent(struct device *dev, size_t size,
@@ -515,13 +519,13 @@ static void calgary_free_coherent(struct device *dev, size_t size,
         free_pages((unsigned long)vaddr, get_order(size));
 }
 
-static struct dma_mapping_ops calgary_dma_ops = {
+static struct dma_map_ops calgary_dma_ops = {
         .alloc_coherent = calgary_alloc_coherent,
         .free_coherent = calgary_free_coherent,
-        .map_single = calgary_map_single,
-        .unmap_single = calgary_unmap_single,
         .map_sg = calgary_map_sg,
         .unmap_sg = calgary_unmap_sg,
+        .map_page = calgary_map_page,
+        .unmap_page = calgary_unmap_page,
 };
 
 static inline void __iomem * busno_to_bbar(unsigned char num)
diff --git a/arch/x86/kernel/pci-dma.c b/arch/x86/kernel/pci-dma.c
index b25428533141..c7c4776ff630 100644
--- a/arch/x86/kernel/pci-dma.c
+++ b/arch/x86/kernel/pci-dma.c
@@ -1,4 +1,5 @@
 #include <linux/dma-mapping.h>
+#include <linux/dma-debug.h>
 #include <linux/dmar.h>
 #include <linux/bootmem.h>
 #include <linux/pci.h>
@@ -12,7 +13,7 @@
 
 static int forbid_dac __read_mostly;
 
-struct dma_mapping_ops *dma_ops;
+struct dma_map_ops *dma_ops;
 EXPORT_SYMBOL(dma_ops);
 
 static int iommu_sac_force __read_mostly;
@@ -44,6 +45,9 @@ struct device x86_dma_fallback_dev = {
 };
 EXPORT_SYMBOL(x86_dma_fallback_dev);
 
+/* Number of entries preallocated for DMA-API debugging */
+#define PREALLOC_DMA_DEBUG_ENTRIES       32768
+
 int dma_set_mask(struct device *dev, u64 mask)
 {
         if (!dev->dma_mask || !dma_supported(dev, mask))
@@ -224,7 +228,7 @@ early_param("iommu", iommu_setup);
 
 int dma_supported(struct device *dev, u64 mask)
 {
-        struct dma_mapping_ops *ops = get_dma_ops(dev);
+        struct dma_map_ops *ops = get_dma_ops(dev);
 
 #ifdef CONFIG_PCI
         if (mask > 0xffffffff && forbid_dac > 0) {
@@ -265,6 +269,12 @@ EXPORT_SYMBOL(dma_supported);
 
 static int __init pci_iommu_init(void)
 {
+        dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
+
+#ifdef CONFIG_PCI
+        dma_debug_add_bus(&pci_bus_type);
+#endif
+
         calgary_iommu_init();
 
         intel_iommu_init();
diff --git a/arch/x86/kernel/pci-gart_64.c b/arch/x86/kernel/pci-gart_64.c
index d5768b1af080..b284b58c035c 100644
--- a/arch/x86/kernel/pci-gart_64.c
+++ b/arch/x86/kernel/pci-gart_64.c
@@ -255,10 +255,13 @@ static dma_addr_t dma_map_area(struct device *dev, dma_addr_t phys_mem,
 }
 
 /* Map a single area into the IOMMU */
-static dma_addr_t
-gart_map_single(struct device *dev, phys_addr_t paddr, size_t size, int dir)
+static dma_addr_t gart_map_page(struct device *dev, struct page *page,
+                                unsigned long offset, size_t size,
+                                enum dma_data_direction dir,
+                                struct dma_attrs *attrs)
 {
         unsigned long bus;
+        phys_addr_t paddr = page_to_phys(page) + offset;
 
         if (!dev)
                 dev = &x86_dma_fallback_dev;
@@ -275,8 +278,9 @@ gart_map_single(struct device *dev, phys_addr_t paddr, size_t size, int dir)
 /*
  * Free a DMA mapping.
  */
-static void gart_unmap_single(struct device *dev, dma_addr_t dma_addr,
-                              size_t size, int direction)
+static void gart_unmap_page(struct device *dev, dma_addr_t dma_addr,
+                            size_t size, enum dma_data_direction dir,
+                            struct dma_attrs *attrs)
 {
         unsigned long iommu_page;
         int npages;
@@ -298,8 +302,8 @@ static void gart_unmap_single(struct device *dev, dma_addr_t dma_addr,
 /*
  * Wrapper for pci_unmap_single working with scatterlists.
  */
-static void
-gart_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, int dir)
+static void gart_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
+                          enum dma_data_direction dir, struct dma_attrs *attrs)
 {
         struct scatterlist *s;
         int i;
@@ -307,7 +311,7 @@ gart_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, int dir)
         for_each_sg(sg, s, nents, i) {
                 if (!s->dma_length || !s->length)
                         break;
-                gart_unmap_single(dev, s->dma_address, s->dma_length, dir);
+                gart_unmap_page(dev, s->dma_address, s->dma_length, dir, NULL);
         }
 }
 
@@ -329,7 +333,7 @@ static int dma_map_sg_nonforce(struct device *dev, struct scatterlist *sg,
                         addr = dma_map_area(dev, addr, s->length, dir, 0);
                         if (addr == bad_dma_address) {
                                 if (i > 0)
-                                        gart_unmap_sg(dev, sg, i, dir);
+                                        gart_unmap_sg(dev, sg, i, dir, NULL);
                                 nents = 0;
                                 sg[0].dma_length = 0;
                                 break;
@@ -400,8 +404,8 @@ dma_map_cont(struct device *dev, struct scatterlist *start, int nelems,
  * DMA map all entries in a scatterlist.
  * Merge chunks that have page aligned sizes into a continuous mapping.
  */
-static int
-gart_map_sg(struct device *dev, struct scatterlist *sg, int nents, int dir)
+static int gart_map_sg(struct device *dev, struct scatterlist *sg, int nents,
+                       enum dma_data_direction dir, struct dma_attrs *attrs)
 {
         struct scatterlist *s, *ps, *start_sg, *sgmap;
         int need = 0, nextneed, i, out, start;
@@ -468,7 +472,7 @@ gart_map_sg(struct device *dev, struct scatterlist *sg, int nents, int dir)
 
 error:
         flush_gart();
-        gart_unmap_sg(dev, sg, out, dir);
+        gart_unmap_sg(dev, sg, out, dir, NULL);
 
         /* When it was forced or merged try again in a dumb way */
         if (force_iommu || iommu_merge) {
@@ -521,7 +525,7 @@ static void
 gart_free_coherent(struct device *dev, size_t size, void *vaddr,
                    dma_addr_t dma_addr)
 {
-        gart_unmap_single(dev, dma_addr, size, DMA_BIDIRECTIONAL);
+        gart_unmap_page(dev, dma_addr, size, DMA_BIDIRECTIONAL, NULL);
         free_pages((unsigned long)vaddr, get_order(size));
 }
 
@@ -707,11 +711,11 @@ static __init int init_k8_gatt(struct agp_kern_info *info)
         return -1;
 }
 
-static struct dma_mapping_ops gart_dma_ops = {
-        .map_single                     = gart_map_single,
-        .unmap_single                   = gart_unmap_single,
+static struct dma_map_ops gart_dma_ops = {
         .map_sg                         = gart_map_sg,
         .unmap_sg                       = gart_unmap_sg,
+        .map_page                       = gart_map_page,
+        .unmap_page                     = gart_unmap_page,
         .alloc_coherent                 = gart_alloc_coherent,
         .free_coherent                  = gart_free_coherent,
 };
diff --git a/arch/x86/kernel/pci-nommu.c b/arch/x86/kernel/pci-nommu.c
index 8b02a3936d42..c6d703b39326 100644
--- a/arch/x86/kernel/pci-nommu.c
+++ b/arch/x86/kernel/pci-nommu.c
@@ -25,19 +25,19 @@ check_addr(char *name, struct device *hwdev, dma_addr_t bus, size_t size)
         return 1;
 }
 
-static dma_addr_t
-nommu_map_single(struct device *hwdev, phys_addr_t paddr, size_t size,
-               int direction)
+static dma_addr_t nommu_map_page(struct device *dev, struct page *page,
+                                 unsigned long offset, size_t size,
+                                 enum dma_data_direction dir,
+                                 struct dma_attrs *attrs)
 {
-        dma_addr_t bus = paddr;
+        dma_addr_t bus = page_to_phys(page) + offset;
         WARN_ON(size == 0);
-        if (!check_addr("map_single", hwdev, bus, size))
-                                return bad_dma_address;
+        if (!check_addr("map_single", dev, bus, size))
+                return bad_dma_address;
         flush_write_buffers();
         return bus;
 }
 
-
 /* Map a set of buffers described by scatterlist in streaming
  * mode for DMA.  This is the scatter-gather version of the
  * above pci_map_single interface.  Here the scatter gather list
@@ -54,7 +54,8 @@ nommu_map_single(struct device *hwdev, phys_addr_t paddr, size_t size,
  * the same here.
  */
 static int nommu_map_sg(struct device *hwdev, struct scatterlist *sg,
-               int nents, int direction)
+                        int nents, enum dma_data_direction dir,
+                        struct dma_attrs *attrs)
 {
         struct scatterlist *s;
         int i;
@@ -78,11 +79,11 @@ static void nommu_free_coherent(struct device *dev, size_t size, void *vaddr,
         free_pages((unsigned long)vaddr, get_order(size));
 }
 
-struct dma_mapping_ops nommu_dma_ops = {
+struct dma_map_ops nommu_dma_ops = {
         .alloc_coherent = dma_generic_alloc_coherent,
         .free_coherent  = nommu_free_coherent,
-        .map_single     = nommu_map_single,
         .map_sg         = nommu_map_sg,
+        .map_page       = nommu_map_page,
         .is_phys        = 1,
 };
 
diff --git a/arch/x86/kernel/pci-swiotlb_64.c b/arch/x86/kernel/pci-swiotlb.c
index d59c91747665..34f12e9996ed 100644
--- a/arch/x86/kernel/pci-swiotlb_64.c
+++ b/arch/x86/kernel/pci-swiotlb.c
@@ -33,18 +33,11 @@ phys_addr_t swiotlb_bus_to_phys(dma_addr_t baddr)
         return baddr;
 }
 
-int __weak swiotlb_arch_range_needs_mapping(void *ptr, size_t size)
+int __weak swiotlb_arch_range_needs_mapping(phys_addr_t paddr, size_t size)
 {
         return 0;
 }
 
-static dma_addr_t
-swiotlb_map_single_phys(struct device *hwdev, phys_addr_t paddr, size_t size,
-                        int direction)
-{
-        return swiotlb_map_single(hwdev, phys_to_virt(paddr), size, direction);
-}
-
 static void *x86_swiotlb_alloc_coherent(struct device *hwdev, size_t size,
                                         dma_addr_t *dma_handle, gfp_t flags)
 {
@@ -57,20 +50,20 @@ static void *x86_swiotlb_alloc_coherent(struct device *hwdev, size_t size,
         return swiotlb_alloc_coherent(hwdev, size, dma_handle, flags);
 }
 
-struct dma_mapping_ops swiotlb_dma_ops = {
+struct dma_map_ops swiotlb_dma_ops = {
         .mapping_error = swiotlb_dma_mapping_error,
         .alloc_coherent = x86_swiotlb_alloc_coherent,
         .free_coherent = swiotlb_free_coherent,
-        .map_single = swiotlb_map_single_phys,
-        .unmap_single = swiotlb_unmap_single,
         .sync_single_for_cpu = swiotlb_sync_single_for_cpu,
         .sync_single_for_device = swiotlb_sync_single_for_device,
         .sync_single_range_for_cpu = swiotlb_sync_single_range_for_cpu,
         .sync_single_range_for_device = swiotlb_sync_single_range_for_device,
         .sync_sg_for_cpu = swiotlb_sync_sg_for_cpu,
         .sync_sg_for_device = swiotlb_sync_sg_for_device,
-        .map_sg = swiotlb_map_sg,
-        .unmap_sg = swiotlb_unmap_sg,
+        .map_sg = swiotlb_map_sg_attrs,
+        .unmap_sg = swiotlb_unmap_sg_attrs,
+        .map_page = swiotlb_map_page,
+        .unmap_page = swiotlb_unmap_page,
         .dma_supported = NULL,
 };