8 files changed, 135 insertions, 170 deletions

diff --git a/arch/ia64/include/asm/dma-mapping.h b/arch/ia64/include/asm/dma-mapping.h
index 9f0df9bd46b7..06ff1ba21465 100644
--- a/arch/ia64/include/asm/dma-mapping.h
+++ b/arch/ia64/include/asm/dma-mapping.h
@@ -8,7 +8,9 @@
 #include <asm/machvec.h>
 #include <linux/scatterlist.h>
 
-#define dma_alloc_coherent      platform_dma_alloc_coherent
+#define dma_alloc_coherent(dev, size, handle, gfp)      \
+        platform_dma_alloc_coherent(dev, size, handle, (gfp) | GFP_DMA)
+
 /* coherent mem. is cheap */
 static inline void *
 dma_alloc_noncoherent(struct device *dev, size_t size, dma_addr_t *dma_handle,
diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c
index 69b4d060b21c..01c68c38840d 100644
--- a/arch/x86/kernel/amd_iommu.c
+++ b/arch/x86/kernel/amd_iommu.c
@@ -1038,8 +1038,6 @@ out:
 
 /*
  * The exported free_coherent function for dma_ops.
- * FIXME: fix the generic x86 DMA layer so that it actually calls that
- *        function.
  */
 static void free_coherent(struct device *dev, size_t size,
                           void *virt_addr, dma_addr_t dma_addr)
diff --git a/arch/x86/kernel/early-quirks.c b/arch/x86/kernel/early-quirks.c
index 4353cf5e6fac..24bb5faf5efa 100644
--- a/arch/x86/kernel/early-quirks.c
+++ b/arch/x86/kernel/early-quirks.c
@@ -95,6 +95,20 @@ static void __init nvidia_bugs(int num, int slot, int func)
 
 }
 
+#ifdef CONFIG_DMAR
+static void __init intel_g33_dmar(int num, int slot, int func)
+{
+        struct acpi_table_header *dmar_tbl;
+        acpi_status status;
+
+        status = acpi_get_table(ACPI_SIG_DMAR, 0, &dmar_tbl);
+        if (ACPI_SUCCESS(status)) {
+                printk(KERN_INFO "BIOS BUG: DMAR advertised on Intel G31/G33 chipset -- ignoring\n");
+                dmar_disabled = 1;
+        }
+}
+#endif
+
 #define QFLAG_APPLY_ONCE        0x1
 #define QFLAG_APPLIED           0x2
 #define QFLAG_DONE              (QFLAG_APPLY_ONCE|QFLAG_APPLIED)
@@ -114,6 +128,10 @@ static struct chipset early_qrk[] __initdata = {
           PCI_CLASS_BRIDGE_PCI, PCI_ANY_ID, QFLAG_APPLY_ONCE, via_bugs },
         { PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_K8_NB,
           PCI_CLASS_BRIDGE_HOST, PCI_ANY_ID, 0, fix_hypertransport_config },
+#ifdef CONFIG_DMAR
+        { PCI_VENDOR_ID_INTEL, 0x29c0,
+          PCI_CLASS_BRIDGE_HOST, PCI_ANY_ID, 0, intel_g33_dmar },
+#endif
         {}
 };
 
diff --git a/arch/x86/kernel/k8.c b/arch/x86/kernel/k8.c
index 7377ccb21335..304d8bad6559 100644
--- a/arch/x86/kernel/k8.c
+++ b/arch/x86/kernel/k8.c
@@ -16,8 +16,9 @@ EXPORT_SYMBOL(num_k8_northbridges);
 static u32 *flush_words;
 
 struct pci_device_id k8_nb_ids[] = {
-        { PCI_DEVICE(PCI_VENDOR_ID_AMD, 0x1103) },
-        { PCI_DEVICE(PCI_VENDOR_ID_AMD, 0x1203) },
+        { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_K8_NB_MISC) },
+        { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_10H_NB_MISC) },
+        { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_11H_NB_MISC) },
         {}
 };
 EXPORT_SYMBOL(k8_nb_ids);
diff --git a/arch/x86/kernel/pci-calgary_64.c b/arch/x86/kernel/pci-calgary_64.c
index dcdac6c826e9..8415d92853c4 100644
--- a/arch/x86/kernel/pci-calgary_64.c
+++ b/arch/x86/kernel/pci-calgary_64.c
@@ -510,8 +510,22 @@ error:
         return ret;
 }
 
+static void calgary_free_coherent(struct device *dev, size_t size,
+                                  void *vaddr, dma_addr_t dma_handle)
+{
+        unsigned int npages;
+        struct iommu_table *tbl = find_iommu_table(dev);
+
+        size = PAGE_ALIGN(size);
+        npages = size >> PAGE_SHIFT;
+
+        iommu_free(tbl, dma_handle, npages);
+        free_pages((unsigned long)vaddr, get_order(size));
+}
+
 static struct dma_mapping_ops calgary_dma_ops = {
         .alloc_coherent = calgary_alloc_coherent,
+        .free_coherent = calgary_free_coherent,
         .map_single = calgary_map_single,
         .unmap_single = calgary_unmap_single,
         .map_sg = calgary_map_sg,
diff --git a/arch/x86/kernel/pci-dma.c b/arch/x86/kernel/pci-dma.c
index 87d4d6964ec2..0a1408abcc62 100644
--- a/arch/x86/kernel/pci-dma.c
+++ b/arch/x86/kernel/pci-dma.c
@@ -41,11 +41,12 @@ EXPORT_SYMBOL(bad_dma_address);
 /* Dummy device used for NULL arguments (normally ISA). Better would
    be probably a smaller DMA mask, but this is bug-to-bug compatible
    to older i386. */
-struct device fallback_dev = {
+struct device x86_dma_fallback_dev = {
         .bus_id = "fallback device",
         .coherent_dma_mask = DMA_32BIT_MASK,
-        .dma_mask = &fallback_dev.coherent_dma_mask,
+        .dma_mask = &x86_dma_fallback_dev.coherent_dma_mask,
 };
+EXPORT_SYMBOL(x86_dma_fallback_dev);
 
 int dma_set_mask(struct device *dev, u64 mask)
 {
@@ -241,147 +242,6 @@ int dma_supported(struct device *dev, u64 mask)
 }
 EXPORT_SYMBOL(dma_supported);
 
-/* Allocate DMA memory on node near device */
-static noinline struct page *
-dma_alloc_pages(struct device *dev, gfp_t gfp, unsigned order)
-{
-        int node;
-
-        node = dev_to_node(dev);
-
-        return alloc_pages_node(node, gfp, order);
-}
-
-/*
- * Allocate memory for a coherent mapping.
- */
-void *
-dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle,
-                   gfp_t gfp)
-{
-        struct dma_mapping_ops *ops = get_dma_ops(dev);
-        void *memory = NULL;
-        struct page *page;
-        unsigned long dma_mask = 0;
-        dma_addr_t bus;
-        int noretry = 0;
-
-        /* ignore region specifiers */
-        gfp &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32);
-
-        if (dma_alloc_from_coherent(dev, size, dma_handle, &memory))
-                return memory;
-
-        if (!dev) {
-                dev = &fallback_dev;
-                gfp |= GFP_DMA;
-        }
-        dma_mask = dev->coherent_dma_mask;
-        if (dma_mask == 0)
-                dma_mask = (gfp & GFP_DMA) ? DMA_24BIT_MASK : DMA_32BIT_MASK;
-
-        /* Device not DMA able */
-        if (dev->dma_mask == NULL)
-                return NULL;
-
-        /* Don't invoke OOM killer or retry in lower 16MB DMA zone */
-        if (gfp & __GFP_DMA)
-                noretry = 1;
-
-#ifdef CONFIG_X86_64
-        /* Why <=? Even when the mask is smaller than 4GB it is often
-           larger than 16MB and in this case we have a chance of
-           finding fitting memory in the next higher zone first. If
-           not retry with true GFP_DMA. -AK */
-        if (dma_mask <= DMA_32BIT_MASK && !(gfp & GFP_DMA)) {
-                gfp |= GFP_DMA32;
-                if (dma_mask < DMA_32BIT_MASK)
-                        noretry = 1;
-        }
-#endif
-
- again:
-        page = dma_alloc_pages(dev,
-                noretry ? gfp | __GFP_NORETRY : gfp, get_order(size));
-        if (page == NULL)
-                return NULL;
-
-        {
-                int high, mmu;
-                bus = page_to_phys(page);
-                memory = page_address(page);
-                high = (bus + size) >= dma_mask;
-                mmu = high;
-                if (force_iommu && !(gfp & GFP_DMA))
-                        mmu = 1;
-                else if (high) {
-                        free_pages((unsigned long)memory,
-                                   get_order(size));
-
-                        /* Don't use the 16MB ZONE_DMA unless absolutely
-                           needed. It's better to use remapping first. */
-                        if (dma_mask < DMA_32BIT_MASK && !(gfp & GFP_DMA)) {
-                                gfp = (gfp & ~GFP_DMA32) | GFP_DMA;
-                                goto again;
-                        }
-
-                        /* Let low level make its own zone decisions */
-                        gfp &= ~(GFP_DMA32|GFP_DMA);
-
-                        if (ops->alloc_coherent)
-                                return ops->alloc_coherent(dev, size,
-                                                           dma_handle, gfp);
-                        return NULL;
-                }
-
-                memset(memory, 0, size);
-                if (!mmu) {
-                        *dma_handle = bus;
-                        return memory;
-                }
-        }
-
-        if (ops->alloc_coherent) {
-                free_pages((unsigned long)memory, get_order(size));
-                gfp &= ~(GFP_DMA|GFP_DMA32);
-                return ops->alloc_coherent(dev, size, dma_handle, gfp);
-        }
-
-        if (ops->map_simple) {
-                *dma_handle = ops->map_simple(dev, virt_to_phys(memory),
-                                              size,
-                                              PCI_DMA_BIDIRECTIONAL);
-                if (*dma_handle != bad_dma_address)
-                        return memory;
-        }
-
-        if (panic_on_overflow)
-                panic("dma_alloc_coherent: IOMMU overflow by %lu bytes\n",
-                      (unsigned long)size);
-        free_pages((unsigned long)memory, get_order(size));
-        return NULL;
-}
-EXPORT_SYMBOL(dma_alloc_coherent);
-
-/*
- * Unmap coherent memory.
- * The caller must ensure that the device has finished accessing the mapping.
- */
-void dma_free_coherent(struct device *dev, size_t size,
-                         void *vaddr, dma_addr_t bus)
-{
-        struct dma_mapping_ops *ops = get_dma_ops(dev);
-
-        int order = get_order(size);
-        WARN_ON(irqs_disabled());       /* for portability */
-        if (dma_release_from_coherent(dev, order, vaddr))
-                return;
-        if (ops->unmap_single)
-                ops->unmap_single(dev, bus, size, 0);
-        free_pages((unsigned long)vaddr, order);
-}
-EXPORT_SYMBOL(dma_free_coherent);
-
 static int __init pci_iommu_init(void)
 {
         calgary_iommu_init();
diff --git a/arch/x86/kernel/pci-gart_64.c b/arch/x86/kernel/pci-gart_64.c
index 49285f8fd4d5..0b99d4a06f74 100644
--- a/arch/x86/kernel/pci-gart_64.c
+++ b/arch/x86/kernel/pci-gart_64.c
@@ -82,7 +82,8 @@ AGPEXTERN __u32 *agp_gatt_table;
 static unsigned long next_bit;  /* protected by iommu_bitmap_lock */
 static int need_flush;          /* global flush state. set for each gart wrap */
 
-static unsigned long alloc_iommu(struct device *dev, int size)
+static unsigned long alloc_iommu(struct device *dev, int size,
+                                 unsigned long align_mask)
 {
         unsigned long offset, flags;
         unsigned long boundary_size;
@@ -90,16 +91,17 @@ static unsigned long alloc_iommu(struct device *dev, int size)
 
         base_index = ALIGN(iommu_bus_base & dma_get_seg_boundary(dev),
                            PAGE_SIZE) >> PAGE_SHIFT;
-        boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,
+        boundary_size = ALIGN((unsigned long long)dma_get_seg_boundary(dev) + 1,
                               PAGE_SIZE) >> PAGE_SHIFT;
 
         spin_lock_irqsave(&iommu_bitmap_lock, flags);
         offset = iommu_area_alloc(iommu_gart_bitmap, iommu_pages, next_bit,
-                                  size, base_index, boundary_size, 0);
+                                  size, base_index, boundary_size, align_mask);
         if (offset == -1) {
                 need_flush = 1;
                 offset = iommu_area_alloc(iommu_gart_bitmap, iommu_pages, 0,
-                                          size, base_index, boundary_size, 0);
+                                          size, base_index, boundary_size,
+                                          align_mask);
         }
         if (offset != -1) {
                 next_bit = offset+size;
@@ -236,10 +238,10 @@ nonforced_iommu(struct device *dev, unsigned long addr, size_t size)
  * Caller needs to check if the iommu is needed and flush.
  */
 static dma_addr_t dma_map_area(struct device *dev, dma_addr_t phys_mem,
-                                size_t size, int dir)
+                                size_t size, int dir, unsigned long align_mask)
 {
         unsigned long npages = iommu_num_pages(phys_mem, size);
-        unsigned long iommu_page = alloc_iommu(dev, npages);
+        unsigned long iommu_page = alloc_iommu(dev, npages, align_mask);
         int i;
 
         if (iommu_page == -1) {
@@ -259,16 +261,6 @@ static dma_addr_t dma_map_area(struct device *dev, dma_addr_t phys_mem,
         return iommu_bus_base + iommu_page*PAGE_SIZE + (phys_mem & ~PAGE_MASK);
 }
 
-static dma_addr_t
-gart_map_simple(struct device *dev, phys_addr_t paddr, size_t size, int dir)
-{
-        dma_addr_t map = dma_map_area(dev, paddr, size, dir);
-
-        flush_gart();
-
-        return map;
-}
-
 /* Map a single area into the IOMMU */
 static dma_addr_t
 gart_map_single(struct device *dev, phys_addr_t paddr, size_t size, int dir)
@@ -276,12 +268,13 @@ gart_map_single(struct device *dev, phys_addr_t paddr, size_t size, int dir)
         unsigned long bus;
 
         if (!dev)
-                dev = &fallback_dev;
+                dev = &x86_dma_fallback_dev;
 
         if (!need_iommu(dev, paddr, size))
                 return paddr;
 
-        bus = gart_map_simple(dev, paddr, size, dir);
+        bus = dma_map_area(dev, paddr, size, dir, 0);
+        flush_gart();
 
         return bus;
 }
@@ -340,7 +333,7 @@ static int dma_map_sg_nonforce(struct device *dev, struct scatterlist *sg,
                 unsigned long addr = sg_phys(s);
 
                 if (nonforced_iommu(dev, addr, s->length)) {
-                        addr = dma_map_area(dev, addr, s->length, dir);
+                        addr = dma_map_area(dev, addr, s->length, dir, 0);
                         if (addr == bad_dma_address) {
                                 if (i > 0)
                                         gart_unmap_sg(dev, sg, i, dir);
@@ -362,7 +355,7 @@ static int __dma_map_cont(struct device *dev, struct scatterlist *start,
                           int nelems, struct scatterlist *sout,
                           unsigned long pages)
 {
-        unsigned long iommu_start = alloc_iommu(dev, pages);
+        unsigned long iommu_start = alloc_iommu(dev, pages, 0);
         unsigned long iommu_page = iommu_start;
         struct scatterlist *s;
         int i;
@@ -427,7 +420,7 @@ gart_map_sg(struct device *dev, struct scatterlist *sg, int nents, int dir)
                 return 0;
 
         if (!dev)
-                dev = &fallback_dev;
+                dev = &x86_dma_fallback_dev;
 
         out = 0;
         start = 0;
@@ -499,6 +492,41 @@ error:
         return 0;
 }
 
+/* allocate and map a coherent mapping */
+static void *
+gart_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_addr,
+                    gfp_t flag)
+{
+        void *vaddr;
+        unsigned long align_mask;
+
+        vaddr = (void *)__get_free_pages(flag | __GFP_ZERO, get_order(size));
+        if (!vaddr)
+                return NULL;
+
+        align_mask = (1UL << get_order(size)) - 1;
+
+        *dma_addr = dma_map_area(dev, __pa(vaddr), size, DMA_BIDIRECTIONAL,
+                                 align_mask);
+        flush_gart();
+
+        if (*dma_addr != bad_dma_address)
+                return vaddr;
+
+        free_pages((unsigned long)vaddr, get_order(size));
+
+        return NULL;
+}
+
+/* free a coherent mapping */
+static void
+gart_free_coherent(struct device *dev, size_t size, void *vaddr,
+                   dma_addr_t dma_addr)
+{
+        gart_unmap_single(dev, dma_addr, size, DMA_BIDIRECTIONAL);
+        free_pages((unsigned long)vaddr, get_order(size));
+}
+
 static int no_agp;
 
 static __init unsigned long check_iommu_size(unsigned long aper, u64 aper_size)
@@ -691,7 +719,6 @@ extern int agp_amd64_init(void);
 
 static struct dma_mapping_ops gart_dma_ops = {
         .map_single                     = gart_map_single,
-        .map_simple                     = gart_map_simple,
         .unmap_single                   = gart_unmap_single,
         .sync_single_for_cpu            = NULL,
         .sync_single_for_device         = NULL,
@@ -701,6 +728,8 @@ static struct dma_mapping_ops gart_dma_ops = {
         .sync_sg_for_device             = NULL,
         .map_sg                         = gart_map_sg,
         .unmap_sg                       = gart_unmap_sg,
+        .alloc_coherent                 = gart_alloc_coherent,
+        .free_coherent                  = gart_free_coherent,
 };
 
 void gart_iommu_shutdown(void)
diff --git a/arch/x86/kernel/pci-nommu.c b/arch/x86/kernel/pci-nommu.c
index 3f91f71cdc3e..8e398b56f50b 100644
--- a/arch/x86/kernel/pci-nommu.c
+++ b/arch/x86/kernel/pci-nommu.c
@@ -72,7 +72,50 @@ static int nommu_map_sg(struct device *hwdev, struct scatterlist *sg,
         return nents;
 }
 
+static void *
+nommu_alloc_coherent(struct device *hwdev, size_t size,
+                     dma_addr_t *dma_addr, gfp_t gfp)
+{
+        unsigned long dma_mask;
+        int node;
+        struct page *page;
+
+        dma_mask = dma_alloc_coherent_mask(hwdev, gfp);
+
+        gfp |= __GFP_ZERO;
+
+        node = dev_to_node(hwdev);
+again:
+        page = alloc_pages_node(node, gfp, get_order(size));
+        if (!page)
+                return NULL;
+
+        if ((page_to_phys(page) + size > dma_mask) && !(gfp & GFP_DMA)) {
+                free_pages((unsigned long)page_address(page), get_order(size));
+                gfp |= GFP_DMA;
+                goto again;
+        }
+
+        *dma_addr = page_to_phys(page);
+        if (check_addr("alloc_coherent", hwdev, *dma_addr, size)) {
+                flush_write_buffers();
+                return page_address(page);
+        }
+
+        free_pages((unsigned long)page_address(page), get_order(size));
+
+        return NULL;
+}
+
+static void nommu_free_coherent(struct device *dev, size_t size, void *vaddr,
+                                dma_addr_t dma_addr)
+{
+        free_pages((unsigned long)vaddr, get_order(size));
+}
+
 struct dma_mapping_ops nommu_dma_ops = {
+        .alloc_coherent = nommu_alloc_coherent,
+        .free_coherent = nommu_free_coherent,
         .map_single = nommu_map_single,
         .map_sg = nommu_map_sg,
         .is_phys = 1,