23 files changed, 254 insertions, 117 deletions

diff --git a/arch/alpha/include/asm/agp.h b/arch/alpha/include/asm/agp.h
index 26c179135293..a94d48b8677f 100644
--- a/arch/alpha/include/asm/agp.h
+++ b/arch/alpha/include/asm/agp.h
@@ -9,10 +9,6 @@
 #define unmap_page_from_agp(page) 
 #define flush_agp_cache() mb()
 
-/* Convert a physical address to an address suitable for the GART. */
-#define phys_to_gart(x) (x)
-#define gart_to_phys(x) (x)
-
 /* GATT allocation. Returns/accepts GATT kernel virtual address. */
 #define alloc_gatt_pages(order)         \
         ((char *)__get_free_pages(GFP_KERNEL, (order)))
diff --git a/arch/ia64/include/asm/agp.h b/arch/ia64/include/asm/agp.h
index c11fdd8ab4d7..01d09c401c5c 100644
--- a/arch/ia64/include/asm/agp.h
+++ b/arch/ia64/include/asm/agp.h
@@ -17,10 +17,6 @@
 #define unmap_page_from_agp(page)       /* nothing */
 #define flush_agp_cache()               mb()
 
-/* Convert a physical address to an address suitable for the GART. */
-#define phys_to_gart(x) (x)
-#define gart_to_phys(x) (x)
-
 /* GATT allocation. Returns/accepts GATT kernel virtual address. */
 #define alloc_gatt_pages(order)         \
         ((char *)__get_free_pages(GFP_KERNEL, (order)))
diff --git a/arch/parisc/include/asm/agp.h b/arch/parisc/include/asm/agp.h
index 9651660da639..d226ffa8fc12 100644
--- a/arch/parisc/include/asm/agp.h
+++ b/arch/parisc/include/asm/agp.h
@@ -11,10 +11,6 @@
 #define unmap_page_from_agp(page)       /* nothing */
 #define flush_agp_cache()               mb()
 
-/* Convert a physical address to an address suitable for the GART. */
-#define phys_to_gart(x) (x)
-#define gart_to_phys(x) (x)
-
 /* GATT allocation. Returns/accepts GATT kernel virtual address. */
 #define alloc_gatt_pages(order)         \
         ((char *)__get_free_pages(GFP_KERNEL, (order)))
diff --git a/arch/powerpc/include/asm/agp.h b/arch/powerpc/include/asm/agp.h
index 86455c4c31ee..416e12c2d505 100644
--- a/arch/powerpc/include/asm/agp.h
+++ b/arch/powerpc/include/asm/agp.h
@@ -8,10 +8,6 @@
 #define unmap_page_from_agp(page)
 #define flush_agp_cache() mb()
 
-/* Convert a physical address to an address suitable for the GART. */
-#define phys_to_gart(x) (x)
-#define gart_to_phys(x) (x)
-
 /* GATT allocation. Returns/accepts GATT kernel virtual address. */
 #define alloc_gatt_pages(order)         \
         ((char *)__get_free_pages(GFP_KERNEL, (order)))
diff --git a/arch/sparc/include/asm/agp.h b/arch/sparc/include/asm/agp.h
index c2456870b05c..70f52c1661bc 100644
--- a/arch/sparc/include/asm/agp.h
+++ b/arch/sparc/include/asm/agp.h
@@ -7,10 +7,6 @@
 #define unmap_page_from_agp(page)
 #define flush_agp_cache() mb()
 
-/* Convert a physical address to an address suitable for the GART. */
-#define phys_to_gart(x) (x)
-#define gart_to_phys(x) (x)
-
 /* GATT allocation. Returns/accepts GATT kernel virtual address. */
 #define alloc_gatt_pages(order)         \
         ((char *)__get_free_pages(GFP_KERNEL, (order)))
diff --git a/arch/x86/include/asm/agp.h b/arch/x86/include/asm/agp.h
index 9825cd64c9b6..eec2a70d4376 100644
--- a/arch/x86/include/asm/agp.h
+++ b/arch/x86/include/asm/agp.h
@@ -22,10 +22,6 @@
  */
 #define flush_agp_cache() wbinvd()
 
-/* Convert a physical address to an address suitable for the GART. */
-#define phys_to_gart(x) (x)
-#define gart_to_phys(x) (x)
-
 /* GATT allocation. Returns/accepts GATT kernel virtual address. */
 #define alloc_gatt_pages(order)         \
         ((char *)__get_free_pages(GFP_KERNEL, (order)))
diff --git a/drivers/char/agp/agp.h b/drivers/char/agp/agp.h
index 178e2e9e9f09..d6f36c004d9b 100644
--- a/drivers/char/agp/agp.h
+++ b/drivers/char/agp/agp.h
@@ -107,7 +107,7 @@ struct agp_bridge_driver {
         void (*agp_enable)(struct agp_bridge_data *, u32);
         void (*cleanup)(void);
         void (*tlb_flush)(struct agp_memory *);
-        unsigned long (*mask_memory)(struct agp_bridge_data *, struct page *, int);
+        unsigned long (*mask_memory)(struct agp_bridge_data *, dma_addr_t, int);
         void (*cache_flush)(void);
         int (*create_gatt_table)(struct agp_bridge_data *);
         int (*free_gatt_table)(struct agp_bridge_data *);
@@ -121,6 +121,11 @@ struct agp_bridge_driver {
         void (*agp_destroy_pages)(struct agp_memory *);
         int (*agp_type_to_mask_type) (struct agp_bridge_data *, int);
         void (*chipset_flush)(struct agp_bridge_data *);
+
+        int (*agp_map_page)(struct page *page, dma_addr_t *ret);
+        void (*agp_unmap_page)(struct page *page, dma_addr_t dma);
+        int (*agp_map_memory)(struct agp_memory *mem);
+        void (*agp_unmap_memory)(struct agp_memory *mem);
 };
 
 struct agp_bridge_data {
@@ -134,7 +139,8 @@ struct agp_bridge_data {
         u32 __iomem *gatt_table;
         u32 *gatt_table_real;
         unsigned long scratch_page;
-        unsigned long scratch_page_real;
+        struct page *scratch_page_page;
+        dma_addr_t scratch_page_dma;
         unsigned long gart_bus_addr;
         unsigned long gatt_bus_addr;
         u32 mode;
@@ -291,7 +297,7 @@ int agp_3_5_enable(struct agp_bridge_data *bridge);
 void global_cache_flush(void);
 void get_agp_version(struct agp_bridge_data *bridge);
 unsigned long agp_generic_mask_memory(struct agp_bridge_data *bridge,
-                                      struct page *page, int type);
+                                      dma_addr_t phys, int type);
 int agp_generic_type_to_mask_type(struct agp_bridge_data *bridge,
                                   int type);
 struct agp_bridge_data *agp_generic_find_bridge(struct pci_dev *pdev);
@@ -312,9 +318,6 @@ void agp3_generic_cleanup(void);
 #define AGP_GENERIC_SIZES_ENTRIES 11
 extern const struct aper_size_info_16 agp3_generic_sizes[];
 
-#define virt_to_gart(x) (phys_to_gart(virt_to_phys(x)))
-#define gart_to_virt(x) (phys_to_virt(gart_to_phys(x)))
-
 extern int agp_off;
 extern int agp_try_unsupported_boot;
 
diff --git a/drivers/char/agp/ali-agp.c b/drivers/char/agp/ali-agp.c
index 201ef3ffd484..d2ce68f27e4b 100644
--- a/drivers/char/agp/ali-agp.c
+++ b/drivers/char/agp/ali-agp.c
@@ -152,7 +152,7 @@ static struct page *m1541_alloc_page(struct agp_bridge_data *bridge)
         pci_read_config_dword(agp_bridge->dev, ALI_CACHE_FLUSH_CTRL, &temp);
         pci_write_config_dword(agp_bridge->dev, ALI_CACHE_FLUSH_CTRL,
                         (((temp & ALI_CACHE_FLUSH_ADDR_MASK) |
-                          phys_to_gart(page_to_phys(page))) | ALI_CACHE_FLUSH_EN ));
+                          page_to_phys(page)) | ALI_CACHE_FLUSH_EN ));
         return page;
 }
 
@@ -180,7 +180,7 @@ static void m1541_destroy_page(struct page *page, int flags)
                 pci_read_config_dword(agp_bridge->dev, ALI_CACHE_FLUSH_CTRL, &temp);
                 pci_write_config_dword(agp_bridge->dev, ALI_CACHE_FLUSH_CTRL,
                                        (((temp & ALI_CACHE_FLUSH_ADDR_MASK) |
-                                         phys_to_gart(page_to_phys(page))) | ALI_CACHE_FLUSH_EN));
+                                         page_to_phys(page)) | ALI_CACHE_FLUSH_EN));
         }
         agp_generic_destroy_page(page, flags);
 }
diff --git a/drivers/char/agp/amd-k7-agp.c b/drivers/char/agp/amd-k7-agp.c
index ba9bde71eaaf..73dbf40c874d 100644
--- a/drivers/char/agp/amd-k7-agp.c
+++ b/drivers/char/agp/amd-k7-agp.c
@@ -44,7 +44,7 @@ static int amd_create_page_map(struct amd_page_map *page_map)
 #ifndef CONFIG_X86
         SetPageReserved(virt_to_page(page_map->real));
         global_cache_flush();
-        page_map->remapped = ioremap_nocache(virt_to_gart(page_map->real),
+        page_map->remapped = ioremap_nocache(virt_to_phys(page_map->real),
                                             PAGE_SIZE);
         if (page_map->remapped == NULL) {
                 ClearPageReserved(virt_to_page(page_map->real));
@@ -160,7 +160,7 @@ static int amd_create_gatt_table(struct agp_bridge_data *bridge)
 
         agp_bridge->gatt_table_real = (u32 *)page_dir.real;
         agp_bridge->gatt_table = (u32 __iomem *)page_dir.remapped;
-        agp_bridge->gatt_bus_addr = virt_to_gart(page_dir.real);
+        agp_bridge->gatt_bus_addr = virt_to_phys(page_dir.real);
 
         /* Get the address for the gart region.
          * This is a bus address even on the alpha, b/c its
@@ -173,7 +173,7 @@ static int amd_create_gatt_table(struct agp_bridge_data *bridge)
 
         /* Calculate the agp offset */
         for (i = 0; i < value->num_entries / 1024; i++, addr += 0x00400000) {
-                writel(virt_to_gart(amd_irongate_private.gatt_pages[i]->real) | 1,
+                writel(virt_to_phys(amd_irongate_private.gatt_pages[i]->real) | 1,
                         page_dir.remapped+GET_PAGE_DIR_OFF(addr));
                 readl(page_dir.remapped+GET_PAGE_DIR_OFF(addr));        /* PCI Posting. */
         }
@@ -325,7 +325,9 @@ static int amd_insert_memory(struct agp_memory *mem, off_t pg_start, int type)
                 addr = (j * PAGE_SIZE) + agp_bridge->gart_bus_addr;
                 cur_gatt = GET_GATT(addr);
                 writel(agp_generic_mask_memory(agp_bridge,
-                        mem->pages[i], mem->type), cur_gatt+GET_GATT_OFF(addr));
+                                               page_to_phys(mem->pages[i]),
+                                               mem->type),
+                       cur_gatt+GET_GATT_OFF(addr));
                 readl(cur_gatt+GET_GATT_OFF(addr));     /* PCI Posting. */
         }
         amd_irongate_tlbflush(mem);
diff --git a/drivers/char/agp/amd64-agp.c b/drivers/char/agp/amd64-agp.c
index 3bf5dda90f4a..2fb2e6cc322a 100644
--- a/drivers/char/agp/amd64-agp.c
+++ b/drivers/char/agp/amd64-agp.c
@@ -79,7 +79,8 @@ static int amd64_insert_memory(struct agp_memory *mem, off_t pg_start, int type)
 
         for (i = 0, j = pg_start; i < mem->page_count; i++, j++) {
                 tmp = agp_bridge->driver->mask_memory(agp_bridge,
-                        mem->pages[i], mask_type);
+                                                      page_to_phys(mem->pages[i]),
+                                                      mask_type);
 
                 BUG_ON(tmp & 0xffffff0000000ffcULL);
                 pte = (tmp & 0x000000ff00000000ULL) >> 28;
@@ -177,7 +178,7 @@ static const struct aper_size_info_32 amd_8151_sizes[7] =
 
 static int amd_8151_configure(void)
 {
-        unsigned long gatt_bus = virt_to_gart(agp_bridge->gatt_table_real);
+        unsigned long gatt_bus = virt_to_phys(agp_bridge->gatt_table_real);
         int i;
 
         /* Configure AGP regs in each x86-64 host bridge. */
@@ -557,7 +558,7 @@ static void __devexit agp_amd64_remove(struct pci_dev *pdev)
 {
         struct agp_bridge_data *bridge = pci_get_drvdata(pdev);
 
-        release_mem_region(virt_to_gart(bridge->gatt_table_real),
+        release_mem_region(virt_to_phys(bridge->gatt_table_real),
                            amd64_aperture_sizes[bridge->aperture_size_idx].size);
         agp_remove_bridge(bridge);
         agp_put_bridge(bridge);
diff --git a/drivers/char/agp/ati-agp.c b/drivers/char/agp/ati-agp.c
index 33656e144cc5..3b2ecbe86ebe 100644
--- a/drivers/char/agp/ati-agp.c
+++ b/drivers/char/agp/ati-agp.c
@@ -302,7 +302,8 @@ static int ati_insert_memory(struct agp_memory * mem,
                 addr = (j * PAGE_SIZE) + agp_bridge->gart_bus_addr;
                 cur_gatt = GET_GATT(addr);
                 writel(agp_bridge->driver->mask_memory(agp_bridge,      
-                                                       mem->pages[i], mem->type),
+                                                       page_to_phys(mem->pages[i]),
+                                                       mem->type),
                        cur_gatt+GET_GATT_OFF(addr));
         }
         readl(GET_GATT(agp_bridge->gart_bus_addr)); /* PCI posting */
@@ -359,7 +360,7 @@ static int ati_create_gatt_table(struct agp_bridge_data *bridge)
 
         agp_bridge->gatt_table_real = (u32 *)page_dir.real;
         agp_bridge->gatt_table = (u32 __iomem *) page_dir.remapped;
-        agp_bridge->gatt_bus_addr = virt_to_gart(page_dir.real);
+        agp_bridge->gatt_bus_addr = virt_to_phys(page_dir.real);
 
         /* Write out the size register */
         current_size = A_SIZE_LVL2(agp_bridge->current_size);
@@ -389,7 +390,7 @@ static int ati_create_gatt_table(struct agp_bridge_data *bridge)
 
         /* Calculate the agp offset */
         for (i = 0; i < value->num_entries / 1024; i++, addr += 0x00400000) {
-                writel(virt_to_gart(ati_generic_private.gatt_pages[i]->real) | 1,
+                writel(virt_to_phys(ati_generic_private.gatt_pages[i]->real) | 1,
                         page_dir.remapped+GET_PAGE_DIR_OFF(addr));
                 readl(page_dir.remapped+GET_PAGE_DIR_OFF(addr));        /* PCI Posting. */
         }
diff --git a/drivers/char/agp/backend.c b/drivers/char/agp/backend.c
index cfa5a649dfe7..ad87753f6de4 100644
--- a/drivers/char/agp/backend.c
+++ b/drivers/char/agp/backend.c
@@ -149,9 +149,21 @@ static int agp_backend_initialize(struct agp_bridge_data *bridge)
                         return -ENOMEM;
                 }
 
-                bridge->scratch_page_real = phys_to_gart(page_to_phys(page));
-                bridge->scratch_page =
-                    bridge->driver->mask_memory(bridge, page, 0);
+                bridge->scratch_page_page = page;
+                if (bridge->driver->agp_map_page) {
+                        if (bridge->driver->agp_map_page(page,
+                                                         &bridge->scratch_page_dma)) {
+                                dev_err(&bridge->dev->dev,
+                                        "unable to dma-map scratch page\n");
+                                rc = -ENOMEM;
+                                goto err_out_nounmap;
+                        }
+                } else {
+                        bridge->scratch_page_dma = page_to_phys(page);
+                }
+
+                bridge->scratch_page = bridge->driver->mask_memory(bridge,
+                                                   bridge->scratch_page_dma, 0);
         }
 
         size_value = bridge->driver->fetch_size();
@@ -191,8 +203,14 @@ static int agp_backend_initialize(struct agp_bridge_data *bridge)
         return 0;
 
 err_out:
+        if (bridge->driver->needs_scratch_page &&
+            bridge->driver->agp_unmap_page) {
+                bridge->driver->agp_unmap_page(bridge->scratch_page_page,
+                                               bridge->scratch_page_dma);
+        }
+err_out_nounmap:
         if (bridge->driver->needs_scratch_page) {
-                void *va = gart_to_virt(bridge->scratch_page_real);
+                void *va = page_address(bridge->scratch_page_page);
 
                 bridge->driver->agp_destroy_page(va, AGP_PAGE_DESTROY_UNMAP);
                 bridge->driver->agp_destroy_page(va, AGP_PAGE_DESTROY_FREE);
@@ -219,7 +237,11 @@ static void agp_backend_cleanup(struct agp_bridge_data *bridge)
 
         if (bridge->driver->agp_destroy_page &&
             bridge->driver->needs_scratch_page) {
-                void *va = gart_to_virt(bridge->scratch_page_real);
+                void *va = page_address(bridge->scratch_page_page);
+
+                if (bridge->driver->agp_unmap_page)
+                        bridge->driver->agp_unmap_page(bridge->scratch_page_page,
+                                                       bridge->scratch_page_dma);
 
                 bridge->driver->agp_destroy_page(va, AGP_PAGE_DESTROY_UNMAP);
                 bridge->driver->agp_destroy_page(va, AGP_PAGE_DESTROY_FREE);
diff --git a/drivers/char/agp/efficeon-agp.c b/drivers/char/agp/efficeon-agp.c
index 35d50f2861b6..793f39ea9618 100644
--- a/drivers/char/agp/efficeon-agp.c
+++ b/drivers/char/agp/efficeon-agp.c
@@ -67,7 +67,7 @@ static const struct gatt_mask efficeon_generic_masks[] =
 /* This function does the same thing as mask_memory() for this chipset... */
 static inline unsigned long efficeon_mask_memory(struct page *page)
 {
-        unsigned long addr = phys_to_gart(page_to_phys(page));
+        unsigned long addr = page_to_phys(page);
         return addr | 0x00000001;
 }
 
@@ -226,7 +226,7 @@ static int efficeon_create_gatt_table(struct agp_bridge_data *bridge)
 
                 efficeon_private.l1_table[index] = page;
 
-                value = virt_to_gart((unsigned long *)page) | pati | present | index;
+                value = virt_to_phys((unsigned long *)page) | pati | present | index;
 
                 pci_write_config_dword(agp_bridge->dev,
                         EFFICEON_ATTPAGE, value);
diff --git a/drivers/char/agp/generic.c b/drivers/char/agp/generic.c
index 1e8b461b91f1..c50543966eb2 100644
--- a/drivers/char/agp/generic.c
+++ b/drivers/char/agp/generic.c
@@ -437,6 +437,12 @@ int agp_bind_memory(struct agp_memory *curr, off_t pg_start)
                 curr->bridge->driver->cache_flush();
                 curr->is_flushed = true;
         }
+
+        if (curr->bridge->driver->agp_map_memory) {
+                ret_val = curr->bridge->driver->agp_map_memory(curr);
+                if (ret_val)
+                        return ret_val;
+        }
         ret_val = curr->bridge->driver->insert_memory(curr, pg_start, curr->type);
 
         if (ret_val != 0)
@@ -478,6 +484,9 @@ int agp_unbind_memory(struct agp_memory *curr)
         if (ret_val != 0)
                 return ret_val;
 
+        if (curr->bridge->driver->agp_unmap_memory)
+                curr->bridge->driver->agp_unmap_memory(curr);
+
         curr->is_bound = false;
         curr->pg_start = 0;
         spin_lock(&curr->bridge->mapped_lock);
@@ -979,7 +988,7 @@ int agp_generic_create_gatt_table(struct agp_bridge_data *bridge)
         set_memory_uc((unsigned long)table, 1 << page_order);
         bridge->gatt_table = (void *)table;
 #else
-        bridge->gatt_table = ioremap_nocache(virt_to_gart(table),
+        bridge->gatt_table = ioremap_nocache(virt_to_phys(table),
                                         (PAGE_SIZE * (1 << page_order)));
         bridge->driver->cache_flush();
 #endif
@@ -992,7 +1001,7 @@ int agp_generic_create_gatt_table(struct agp_bridge_data *bridge)
 
                 return -ENOMEM;
         }
-        bridge->gatt_bus_addr = virt_to_gart(bridge->gatt_table_real);
+        bridge->gatt_bus_addr = virt_to_phys(bridge->gatt_table_real);
 
         /* AK: bogus, should encode addresses > 4GB */
         for (i = 0; i < num_entries; i++) {
@@ -1132,7 +1141,9 @@ int agp_generic_insert_memory(struct agp_memory * mem, off_t pg_start, int type)
         }
 
         for (i = 0, j = pg_start; i < mem->page_count; i++, j++) {
-                writel(bridge->driver->mask_memory(bridge, mem->pages[i], mask_type),
+                writel(bridge->driver->mask_memory(bridge,
+                                                   page_to_phys(mem->pages[i]),
+                                                   mask_type),
                        bridge->gatt_table+j);
         }
         readl(bridge->gatt_table+j-1);  /* PCI Posting. */
@@ -1347,9 +1358,8 @@ void global_cache_flush(void)
 EXPORT_SYMBOL(global_cache_flush);
 
 unsigned long agp_generic_mask_memory(struct agp_bridge_data *bridge,
-                                      struct page *page, int type)
+                                      dma_addr_t addr, int type)
 {
-        unsigned long addr = phys_to_gart(page_to_phys(page));
         /* memory type is ignored in the generic routine */
         if (bridge->driver->masks)
                 return addr | bridge->driver->masks[0].mask;
diff --git a/drivers/char/agp/hp-agp.c b/drivers/char/agp/hp-agp.c
index 8f3d4c184914..501e293e5ad0 100644
--- a/drivers/char/agp/hp-agp.c
+++ b/drivers/char/agp/hp-agp.c
@@ -107,7 +107,7 @@ static int __init hp_zx1_ioc_shared(void)
         hp->gart_size = HP_ZX1_GART_SIZE;
         hp->gatt_entries = hp->gart_size / hp->io_page_size;
 
-        hp->io_pdir = gart_to_virt(readq(hp->ioc_regs+HP_ZX1_PDIR_BASE));
+        hp->io_pdir = phys_to_virt(readq(hp->ioc_regs+HP_ZX1_PDIR_BASE));
         hp->gatt = &hp->io_pdir[HP_ZX1_IOVA_TO_PDIR(hp->gart_base)];
 
         if (hp->gatt[0] != HP_ZX1_SBA_IOMMU_COOKIE) {
@@ -246,7 +246,7 @@ hp_zx1_configure (void)
         agp_bridge->mode = readl(hp->lba_regs+hp->lba_cap_offset+PCI_AGP_STATUS);
 
         if (hp->io_pdir_owner) {
-                writel(virt_to_gart(hp->io_pdir), hp->ioc_regs+HP_ZX1_PDIR_BASE);
+                writel(virt_to_phys(hp->io_pdir), hp->ioc_regs+HP_ZX1_PDIR_BASE);
                 readl(hp->ioc_regs+HP_ZX1_PDIR_BASE);
                 writel(hp->io_tlb_ps, hp->ioc_regs+HP_ZX1_TCNFG);
                 readl(hp->ioc_regs+HP_ZX1_TCNFG);
@@ -394,10 +394,8 @@ hp_zx1_remove_memory (struct agp_memory *mem, off_t pg_start, int type)
 }
 
 static unsigned long
-hp_zx1_mask_memory (struct agp_bridge_data *bridge,
-                    struct page *page, int type)
+hp_zx1_mask_memory (struct agp_bridge_data *bridge, dma_addr_t addr, int type)
 {
-        unsigned long addr = phys_to_gart(page_to_phys(page));
         return HP_ZX1_PDIR_VALID_BIT | addr;
 }
 
diff --git a/drivers/char/agp/i460-agp.c b/drivers/char/agp/i460-agp.c
index 60cc35bb5db7..e763d3312ce7 100644
--- a/drivers/char/agp/i460-agp.c
+++ b/drivers/char/agp/i460-agp.c
@@ -61,7 +61,7 @@
 #define WR_FLUSH_GATT(index)    RD_GATT(index)
 
 static unsigned long i460_mask_memory (struct agp_bridge_data *bridge,
-                                       unsigned long addr, int type);
+                                       dma_addr_t addr, int type);
 
 static struct {
         void *gatt;                             /* ioremap'd GATT area */
@@ -325,7 +325,7 @@ static int i460_insert_memory_small_io_page (struct agp_memory *mem,
 
         io_page_size = 1UL << I460_IO_PAGE_SHIFT;
         for (i = 0, j = io_pg_start; i < mem->page_count; i++) {
-                paddr = phys_to_gart(page_to_phys(mem->pages[i]));
+                paddr = page_to_phys(mem->pages[i]);
                 for (k = 0; k < I460_IOPAGES_PER_KPAGE; k++, j++, paddr += io_page_size)
                         WR_GATT(j, i460_mask_memory(agp_bridge, paddr, mem->type));
         }
@@ -382,7 +382,7 @@ static int i460_alloc_large_page (struct lp_desc *lp)
                 return -ENOMEM;
         }
 
-        lp->paddr = phys_to_gart(page_to_phys(lp->page));
+        lp->paddr = page_to_phys(lp->page);
         lp->refcount = 0;
         atomic_add(I460_KPAGES_PER_IOPAGE, &agp_bridge->current_memory_agp);
         return 0;
@@ -546,20 +546,13 @@ static void i460_destroy_page (struct page *page, int flags)
 #endif /* I460_LARGE_IO_PAGES */
 
 static unsigned long i460_mask_memory (struct agp_bridge_data *bridge,
-                                       unsigned long addr, int type)
+                                       dma_addr_t addr, int type)
 {
         /* Make sure the returned address is a valid GATT entry */
         return bridge->driver->masks[0].mask
                 | (((addr & ~((1 << I460_IO_PAGE_SHIFT) - 1)) & 0xfffff000) >> 12);
 }
 
-static unsigned long i460_page_mask_memory(struct agp_bridge_data *bridge,
-                                           struct page *page, int type)
-{
-        unsigned long addr = phys_to_gart(page_to_phys(page));
-        return i460_mask_memory(bridge, addr, type);
-}
-
 const struct agp_bridge_driver intel_i460_driver = {
         .owner                  = THIS_MODULE,
         .aperture_sizes         = i460_sizes,
@@ -569,7 +562,7 @@ const struct agp_bridge_driver intel_i460_driver = {
         .fetch_size             = i460_fetch_size,
         .cleanup                = i460_cleanup,
         .tlb_flush              = i460_tlb_flush,
-        .mask_memory            = i460_page_mask_memory,
+        .mask_memory            = i460_mask_memory,
         .masks                  = i460_masks,
         .agp_enable             = agp_generic_enable,
         .cache_flush            = global_cache_flush,
diff --git a/drivers/char/agp/intel-agp.c b/drivers/char/agp/intel-agp.c
index c58557790585..1540e693d91e 100644
--- a/drivers/char/agp/intel-agp.c
+++ b/drivers/char/agp/intel-agp.c
@@ -10,6 +10,16 @@
 #include <linux/agp_backend.h>
 #include "agp.h"
 
+/*
+ * If we have Intel graphics, we're not going to have anything other than
+ * an Intel IOMMU. So make the correct use of the PCI DMA API contingent
+ * on the Intel IOMMU support (CONFIG_DMAR).
+ * Only newer chipsets need to bother with this, of course.
+ */
+#ifdef CONFIG_DMAR
+#define USE_PCI_DMA_API 1
+#endif
+
 #define PCI_DEVICE_ID_INTEL_E7221_HB    0x2588
 #define PCI_DEVICE_ID_INTEL_E7221_IG    0x258a
 #define PCI_DEVICE_ID_INTEL_82946GZ_HB      0x2970
@@ -172,6 +182,123 @@ static struct _intel_private {
         int resource_valid;
 } intel_private;
 
+#ifdef USE_PCI_DMA_API
+static int intel_agp_map_page(struct page *page, dma_addr_t *ret)
+{
+        *ret = pci_map_page(intel_private.pcidev, page, 0,
+                            PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
+        if (pci_dma_mapping_error(intel_private.pcidev, *ret))
+                return -EINVAL;
+        return 0;
+}
+
+static void intel_agp_unmap_page(struct page *page, dma_addr_t dma)
+{
+        pci_unmap_page(intel_private.pcidev, dma,
+                       PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
+}
+
+static void intel_agp_free_sglist(struct agp_memory *mem)
+{
+        struct sg_table st;
+
+        st.sgl = mem->sg_list;
+        st.orig_nents = st.nents = mem->page_count;
+
+        sg_free_table(&st);
+
+        mem->sg_list = NULL;
+        mem->num_sg = 0;
+}
+
+static int intel_agp_map_memory(struct agp_memory *mem)
+{
+        struct sg_table st;
+        struct scatterlist *sg;
+        int i;
+
+        DBG("try mapping %lu pages\n", (unsigned long)mem->page_count);
+
+        if (sg_alloc_table(&st, mem->page_count, GFP_KERNEL))
+                return -ENOMEM;
+
+        mem->sg_list = sg = st.sgl;
+
+        for (i = 0 ; i < mem->page_count; i++, sg = sg_next(sg))
+                sg_set_page(sg, mem->pages[i], PAGE_SIZE, 0);
+
+        mem->num_sg = pci_map_sg(intel_private.pcidev, mem->sg_list,
+                                 mem->page_count, PCI_DMA_BIDIRECTIONAL);
+        if (unlikely(!mem->num_sg)) {
+                intel_agp_free_sglist(mem);
+                return -ENOMEM;
+        }
+        return 0;
+}
+
+static void intel_agp_unmap_memory(struct agp_memory *mem)
+{
+        DBG("try unmapping %lu pages\n", (unsigned long)mem->page_count);
+
+        pci_unmap_sg(intel_private.pcidev, mem->sg_list,
+                     mem->page_count, PCI_DMA_BIDIRECTIONAL);
+        intel_agp_free_sglist(mem);
+}
+
+static void intel_agp_insert_sg_entries(struct agp_memory *mem,
+                                        off_t pg_start, int mask_type)
+{
+        struct scatterlist *sg;
+        int i, j;
+
+        j = pg_start;
+
+        WARN_ON(!mem->num_sg);
+
+        if (mem->num_sg == mem->page_count) {
+                for_each_sg(mem->sg_list, sg, mem->page_count, i) {
+                        writel(agp_bridge->driver->mask_memory(agp_bridge,
+                                        sg_dma_address(sg), mask_type),
+                                        intel_private.gtt+j);
+                        j++;
+                }
+        } else {
+                /* sg may merge pages, but we have to seperate
+                 * per-page addr for GTT */
+                unsigned int len, m;
+
+                for_each_sg(mem->sg_list, sg, mem->num_sg, i) {
+                        len = sg_dma_len(sg) / PAGE_SIZE;
+                        for (m = 0; m < len; m++) {
+                                writel(agp_bridge->driver->mask_memory(agp_bridge,
+                                                                       sg_dma_address(sg) + m * PAGE_SIZE,
+                                                                       mask_type),
+                                       intel_private.gtt+j);
+                                j++;
+                        }
+                }
+        }
+        readl(intel_private.gtt+j-1);
+}
+
+#else
+
+static void intel_agp_insert_sg_entries(struct agp_memory *mem,
+                                        off_t pg_start, int mask_type)
+{
+        int i, j;
+
+        for (i = 0, j = pg_start; i < mem->page_count; i++, j++) {
+                writel(agp_bridge->driver->mask_memory(agp_bridge,
+                                page_to_phys(mem->pages[i]), mask_type),
+                       intel_private.gtt+j);
+        }
+
+        readl(intel_private.gtt+j-1);
+}
+
+#endif
+
 static int intel_i810_fetch_size(void)
 {
         u32 smram_miscc;
@@ -345,8 +472,7 @@ static int intel_i810_insert_entries(struct agp_memory *mem, off_t pg_start,
                         global_cache_flush();
                 for (i = 0, j = pg_start; i < mem->page_count; i++, j++) {
                         writel(agp_bridge->driver->mask_memory(agp_bridge,
-                                                               mem->pages[i],
-                                                               mask_type),
+                                        page_to_phys(mem->pages[i]), mask_type),
                                intel_private.registers+I810_PTE_BASE+(j*4));
                 }
                 readl(intel_private.registers+I810_PTE_BASE+((j-1)*4));
@@ -463,9 +589,8 @@ static void intel_i810_free_by_type(struct agp_memory *curr)
 }
 
 static unsigned long intel_i810_mask_memory(struct agp_bridge_data *bridge,
-                                            struct page *page, int type)
+                                            dma_addr_t addr, int type)
 {
-        unsigned long addr = phys_to_gart(page_to_phys(page));
         /* Type checking must be done elsewhere */
         return addr | bridge->driver->masks[type].mask;
 }
@@ -853,7 +978,7 @@ static int intel_i830_insert_entries(struct agp_memory *mem, off_t pg_start,
 
         for (i = 0, j = pg_start; i < mem->page_count; i++, j++) {
                 writel(agp_bridge->driver->mask_memory(agp_bridge,
-                                                       mem->pages[i], mask_type),
+                                page_to_phys(mem->pages[i]), mask_type),
                        intel_private.registers+I810_PTE_BASE+(j*4));
         }
         readl(intel_private.registers+I810_PTE_BASE+((j-1)*4));
@@ -1017,6 +1142,12 @@ static int intel_i915_configure(void)
 
         intel_i9xx_setup_flush();
 
+#ifdef USE_PCI_DMA_API 
+        if (pci_set_dma_mask(intel_private.pcidev, DMA_BIT_MASK(36)))
+                dev_err(&intel_private.pcidev->dev,
+                        "set gfx device dma mask 36bit failed!\n");
+#endif
+
         return 0;
 }
 
@@ -1041,7 +1172,7 @@ static void intel_i915_chipset_flush(struct agp_bridge_data *bridge)
 static int intel_i915_insert_entries(struct agp_memory *mem, off_t pg_start,
                                      int type)
 {
-        int i, j, num_entries;
+        int num_entries;
         void *temp;
         int ret = -EINVAL;
         int mask_type;
@@ -1065,7 +1196,7 @@ static int intel_i915_insert_entries(struct agp_memory *mem, off_t pg_start,
         if ((pg_start + mem->page_count) > num_entries)
                 goto out_err;
 
-        /* The i915 can't check the GTT for entries since its read only,
+        /* The i915 can't check the GTT for entries since it's read only;
          * depend on the caller to make the correct offset decisions.
          */
 
@@ -1081,12 +1212,7 @@ static int intel_i915_insert_entries(struct agp_memory *mem, off_t pg_start,
         if (!mem->is_flushed)
                 global_cache_flush();
 
-        for (i = 0, j = pg_start; i < mem->page_count; i++, j++) {
-                writel(agp_bridge->driver->mask_memory(agp_bridge,
-                                                       mem->pages[i], mask_type), intel_private.gtt+j);
-        }
-
-        readl(intel_private.gtt+j-1);
+        intel_agp_insert_sg_entries(mem, pg_start, mask_type);
         agp_bridge->driver->tlb_flush(mem);
 
  out:
@@ -1198,9 +1324,8 @@ static int intel_i915_create_gatt_table(struct agp_bridge_data *bridge)
  * this conditional.
  */
 static unsigned long intel_i965_mask_memory(struct agp_bridge_data *bridge,
-                                            struct page *page, int type)
+                                            dma_addr_t addr, int type)
 {
-        dma_addr_t addr = phys_to_gart(page_to_phys(page));
         /* Shift high bits down */
         addr |= (addr >> 28) & 0xf0;
 
@@ -2006,6 +2131,12 @@ static const struct agp_bridge_driver intel_915_driver = {
         .agp_destroy_pages      = agp_generic_destroy_pages,
         .agp_type_to_mask_type  = intel_i830_type_to_mask_type,
         .chipset_flush          = intel_i915_chipset_flush,
+#ifdef USE_PCI_DMA_API
+        .agp_map_page           = intel_agp_map_page,
+        .agp_unmap_page         = intel_agp_unmap_page,
+        .agp_map_memory         = intel_agp_map_memory,
+        .agp_unmap_memory       = intel_agp_unmap_memory,
+#endif
 };
 
 static const struct agp_bridge_driver intel_i965_driver = {
@@ -2034,6 +2165,12 @@ static const struct agp_bridge_driver intel_i965_driver = {
         .agp_destroy_pages      = agp_generic_destroy_pages,
         .agp_type_to_mask_type  = intel_i830_type_to_mask_type,
         .chipset_flush          = intel_i915_chipset_flush,
+#ifdef USE_PCI_DMA_API
+        .agp_map_page           = intel_agp_map_page,
+        .agp_unmap_page         = intel_agp_unmap_page,
+        .agp_map_memory         = intel_agp_map_memory,
+        .agp_unmap_memory       = intel_agp_unmap_memory,
+#endif
 };
 
 static const struct agp_bridge_driver intel_7505_driver = {
@@ -2088,6 +2225,12 @@ static const struct agp_bridge_driver intel_g33_driver = {
         .agp_destroy_pages      = agp_generic_destroy_pages,
         .agp_type_to_mask_type  = intel_i830_type_to_mask_type,
         .chipset_flush          = intel_i915_chipset_flush,
+#ifdef USE_PCI_DMA_API
+        .agp_map_page           = intel_agp_map_page,
+        .agp_unmap_page         = intel_agp_unmap_page,
+        .agp_map_memory         = intel_agp_map_memory,
+        .agp_unmap_memory       = intel_agp_unmap_memory,
+#endif
 };
 
 static int find_gmch(u16 device)
@@ -2313,15 +2456,6 @@ static int agp_intel_resume(struct pci_dev *pdev)
         struct agp_bridge_data *bridge = pci_get_drvdata(pdev);
         int ret_val;
 
-        pci_restore_state(pdev);
-
-        /* We should restore our graphics device's config space,
-         * as host bridge (00:00) resumes before graphics device (02:00),
-         * then our access to its pci space can work right.
-         */
-        if (intel_private.pcidev)
-                pci_restore_state(intel_private.pcidev);
-
         if (bridge->driver == &intel_generic_driver)
                 intel_configure();
         else if (bridge->driver == &intel_850_driver)
diff --git a/drivers/char/agp/nvidia-agp.c b/drivers/char/agp/nvidia-agp.c
index 263d71dd441c..7e36d2b4f9d4 100644
--- a/drivers/char/agp/nvidia-agp.c
+++ b/drivers/char/agp/nvidia-agp.c
@@ -225,7 +225,7 @@ static int nvidia_insert_memory(struct agp_memory *mem, off_t pg_start, int type
         }
         for (i = 0, j = pg_start; i < mem->page_count; i++, j++) {
                 writel(agp_bridge->driver->mask_memory(agp_bridge,
-                        mem->pages[i], mask_type),
+                               page_to_phys(mem->pages[i]), mask_type),
                         agp_bridge->gatt_table+nvidia_private.pg_offset+j);
         }
 
diff --git a/drivers/char/agp/parisc-agp.c b/drivers/char/agp/parisc-agp.c
index e077701ae3d9..60ab75104da9 100644
--- a/drivers/char/agp/parisc-agp.c
+++ b/drivers/char/agp/parisc-agp.c
@@ -32,7 +32,7 @@
 #define AGP8X_MODE              (1 << AGP8X_MODE_BIT)
 
 static unsigned long
-parisc_agp_mask_memory(struct agp_bridge_data *bridge, unsigned long addr,
+parisc_agp_mask_memory(struct agp_bridge_data *bridge, dma_addr_t addr,
                        int type);
 
 static struct _parisc_agp_info {
@@ -189,20 +189,12 @@ parisc_agp_remove_memory(struct agp_memory *mem, off_t pg_start, int type)
 }
 
 static unsigned long
-parisc_agp_mask_memory(struct agp_bridge_data *bridge, unsigned long addr,
+parisc_agp_mask_memory(struct agp_bridge_data *bridge, dma_addr_t addr,
                        int type)
 {
         return SBA_PDIR_VALID_BIT | addr;
 }
 
-static unsigned long
-parisc_agp_page_mask_memory(struct agp_bridge_data *bridge, struct page *page,
-                            int type)
-{
-        unsigned long addr = phys_to_gart(page_to_phys(page));
-        return SBA_PDIR_VALID_BIT | addr;
-}
-
 static void
 parisc_agp_enable(struct agp_bridge_data *bridge, u32 mode)
 {
diff --git a/drivers/char/agp/sgi-agp.c b/drivers/char/agp/sgi-agp.c
index d3ea2e4226b5..0d426ae39c85 100644
--- a/drivers/char/agp/sgi-agp.c
+++ b/drivers/char/agp/sgi-agp.c
@@ -70,10 +70,9 @@ static void sgi_tioca_tlbflush(struct agp_memory *mem)
  * entry.
  */
 static unsigned long
-sgi_tioca_mask_memory(struct agp_bridge_data *bridge,
-                      struct page *page, int type)
+sgi_tioca_mask_memory(struct agp_bridge_data *bridge, dma_addr_t addr,
+                      int type)
 {
-        unsigned long addr = phys_to_gart(page_to_phys(page));
         return tioca_physpage_to_gart(addr);
 }
 
@@ -190,7 +189,8 @@ static int sgi_tioca_insert_memory(struct agp_memory *mem, off_t pg_start,
 
         for (i = 0, j = pg_start; i < mem->page_count; i++, j++) {
                 table[j] =
-                    bridge->driver->mask_memory(bridge, mem->pages[i],
+                    bridge->driver->mask_memory(bridge,
+                                                page_to_phys(mem->pages[i]),
                                                 mem->type);
         }
 
diff --git a/drivers/char/agp/sworks-agp.c b/drivers/char/agp/sworks-agp.c
index b964a2199329..13acaaf64edb 100644
--- a/drivers/char/agp/sworks-agp.c
+++ b/drivers/char/agp/sworks-agp.c
@@ -155,7 +155,7 @@ static int serverworks_create_gatt_table(struct agp_bridge_data *bridge)
         /* Create a fake scratch directory */
         for (i = 0; i < 1024; i++) {
                 writel(agp_bridge->scratch_page, serverworks_private.scratch_dir.remapped+i);
-                writel(virt_to_gart(serverworks_private.scratch_dir.real) | 1, page_dir.remapped+i);
+                writel(virt_to_phys(serverworks_private.scratch_dir.real) | 1, page_dir.remapped+i);
         }
 
         retval = serverworks_create_gatt_pages(value->num_entries / 1024);
@@ -167,7 +167,7 @@ static int serverworks_create_gatt_table(struct agp_bridge_data *bridge)
 
         agp_bridge->gatt_table_real = (u32 *)page_dir.real;
         agp_bridge->gatt_table = (u32 __iomem *)page_dir.remapped;
-        agp_bridge->gatt_bus_addr = virt_to_gart(page_dir.real);
+        agp_bridge->gatt_bus_addr = virt_to_phys(page_dir.real);
 
         /* Get the address for the gart region.
          * This is a bus address even on the alpha, b/c its
@@ -179,7 +179,7 @@ static int serverworks_create_gatt_table(struct agp_bridge_data *bridge)
 
         /* Calculate the agp offset */
         for (i = 0; i < value->num_entries / 1024; i++)
-                writel(virt_to_gart(serverworks_private.gatt_pages[i]->real)|1, page_dir.remapped+i);
+                writel(virt_to_phys(serverworks_private.gatt_pages[i]->real)|1, page_dir.remapped+i);
 
         return 0;
 }
@@ -349,7 +349,9 @@ static int serverworks_insert_memory(struct agp_memory *mem,
         for (i = 0, j = pg_start; i < mem->page_count; i++, j++) {
                 addr = (j * PAGE_SIZE) + agp_bridge->gart_bus_addr;
                 cur_gatt = SVRWRKS_GET_GATT(addr);
-                writel(agp_bridge->driver->mask_memory(agp_bridge, mem->pages[i], mem->type), cur_gatt+GET_GATT_OFF(addr));
+                writel(agp_bridge->driver->mask_memory(agp_bridge, 
+                                page_to_phys(mem->pages[i]), mem->type),
+                       cur_gatt+GET_GATT_OFF(addr));
         }
         serverworks_tlbflush(mem);
         return 0;
diff --git a/drivers/char/agp/uninorth-agp.c b/drivers/char/agp/uninorth-agp.c
index f192c3b9ad41..4317a5588daf 100644
--- a/drivers/char/agp/uninorth-agp.c
+++ b/drivers/char/agp/uninorth-agp.c
@@ -135,7 +135,7 @@ static int uninorth_configure(void)
         if (is_u3) {
                 pci_write_config_dword(agp_bridge->dev,
                                        UNI_N_CFG_GART_DUMMY_PAGE,
-                                       agp_bridge->scratch_page_real >> 12);
+                                       page_to_phys(agp_bridge->scratch_page_page) >> 12);
         }
 
         return 0;
@@ -431,7 +431,7 @@ static int uninorth_create_gatt_table(struct agp_bridge_data *bridge)
 
         bridge->gatt_table_real = (u32 *) table;
         bridge->gatt_table = (u32 *)table;
-        bridge->gatt_bus_addr = virt_to_gart(table);
+        bridge->gatt_bus_addr = virt_to_phys(table);
 
         for (i = 0; i < num_entries; i++)
                 bridge->gatt_table[i] = 0;
diff --git a/include/linux/agp_backend.h b/include/linux/agp_backend.h
index 76fa794fdac0..880130f7311f 100644
--- a/include/linux/agp_backend.h
+++ b/include/linux/agp_backend.h
@@ -79,9 +79,12 @@ struct agp_memory {
         u32 physical;
         bool is_bound;
         bool is_flushed;
-        bool vmalloc_flag;
+        bool vmalloc_flag;
         /* list of agp_memory mapped to the aperture */
         struct list_head mapped_list;
+        /* DMA-mapped addresses */
+        struct scatterlist *sg_list;
+        int num_sg;
 };
 
 #define AGP_NORMAL_MEMORY 0