12 files changed, 119 insertions, 36 deletions

diff --git a/drivers/gpu/drm/nouveau/nouveau_mem.c b/drivers/gpu/drm/nouveau/nouveau_mem.c
index 123969dd4f56..2b4e5e912110 100644
--- a/drivers/gpu/drm/nouveau/nouveau_mem.c
+++ b/drivers/gpu/drm/nouveau/nouveau_mem.c
@@ -409,6 +409,7 @@ nouveau_mem_vram_init(struct drm_device *dev)
         if (ret)
                 return ret;
 
+        dev_priv->ttm.bdev.dev = dev->dev;
         ret = ttm_bo_device_init(&dev_priv->ttm.bdev,
                                  dev_priv->ttm.bo_global_ref.ref.object,
                                  &nouveau_bo_driver, DRM_FILE_PAGE_OFFSET,
diff --git a/drivers/gpu/drm/nouveau/nouveau_sgdma.c b/drivers/gpu/drm/nouveau/nouveau_sgdma.c
index 9a250eb53098..07b115184b87 100644
--- a/drivers/gpu/drm/nouveau/nouveau_sgdma.c
+++ b/drivers/gpu/drm/nouveau/nouveau_sgdma.c
@@ -12,6 +12,7 @@ struct nouveau_sgdma_be {
         struct drm_device *dev;
 
         dma_addr_t *pages;
+        bool *ttm_alloced;
         unsigned nr_pages;
 
         u64 offset;
@@ -20,7 +21,8 @@ struct nouveau_sgdma_be {
 
 static int
 nouveau_sgdma_populate(struct ttm_backend *be, unsigned long num_pages,
-                       struct page **pages, struct page *dummy_read_page)
+                       struct page **pages, struct page *dummy_read_page,
+                       dma_addr_t *dma_addrs)
 {
         struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)be;
         struct drm_device *dev = nvbe->dev;
@@ -34,15 +36,25 @@ nouveau_sgdma_populate(struct ttm_backend *be, unsigned long num_pages,
         if (!nvbe->pages)
                 return -ENOMEM;
 
+        nvbe->ttm_alloced = kmalloc(sizeof(bool) * num_pages, GFP_KERNEL);
+        if (!nvbe->ttm_alloced)
+                return -ENOMEM;
+
         nvbe->nr_pages = 0;
         while (num_pages--) {
-                nvbe->pages[nvbe->nr_pages] =
-                        pci_map_page(dev->pdev, pages[nvbe->nr_pages], 0,
+                if (dma_addrs[nvbe->nr_pages] != DMA_ERROR_CODE) {
+                        nvbe->pages[nvbe->nr_pages] =
+                                        dma_addrs[nvbe->nr_pages];
+                        nvbe->ttm_alloced[nvbe->nr_pages] = true;
+                } else {
+                        nvbe->pages[nvbe->nr_pages] =
+                                pci_map_page(dev->pdev, pages[nvbe->nr_pages], 0,
                                      PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
-                if (pci_dma_mapping_error(dev->pdev,
-                                          nvbe->pages[nvbe->nr_pages])) {
-                        be->func->clear(be);
-                        return -EFAULT;
+                        if (pci_dma_mapping_error(dev->pdev,
+                                                  nvbe->pages[nvbe->nr_pages])) {
+                                be->func->clear(be);
+                                return -EFAULT;
+                        }
                 }
 
                 nvbe->nr_pages++;
@@ -65,11 +77,14 @@ nouveau_sgdma_clear(struct ttm_backend *be)
                         be->func->unbind(be);
 
                 while (nvbe->nr_pages--) {
-                        pci_unmap_page(dev->pdev, nvbe->pages[nvbe->nr_pages],
+                        if (!nvbe->ttm_alloced[nvbe->nr_pages])
+                                pci_unmap_page(dev->pdev, nvbe->pages[nvbe->nr_pages],
                                        PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
                 }
                 kfree(nvbe->pages);
+                kfree(nvbe->ttm_alloced);
                 nvbe->pages = NULL;
+                nvbe->ttm_alloced = NULL;
                 nvbe->nr_pages = 0;
         }
 }
diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h
index a4605362c528..82aa59941aa1 100644
--- a/drivers/gpu/drm/radeon/radeon.h
+++ b/drivers/gpu/drm/radeon/radeon.h
@@ -328,6 +328,7 @@ struct radeon_gart {
         union radeon_gart_table         table;
         struct page                     **pages;
         dma_addr_t                      *pages_addr;
+        bool                            *ttm_alloced;
         bool                            ready;
 };
 
@@ -340,7 +341,8 @@ void radeon_gart_fini(struct radeon_device *rdev);
 void radeon_gart_unbind(struct radeon_device *rdev, unsigned offset,
                         int pages);
 int radeon_gart_bind(struct radeon_device *rdev, unsigned offset,
-                     int pages, struct page **pagelist);
+                     int pages, struct page **pagelist,
+                     dma_addr_t *dma_addr);
 
 
 /*
diff --git a/drivers/gpu/drm/radeon/radeon_gart.c b/drivers/gpu/drm/radeon/radeon_gart.c
index 65016117d95f..a6b0fed7bae9 100644
--- a/drivers/gpu/drm/radeon/radeon_gart.c
+++ b/drivers/gpu/drm/radeon/radeon_gart.c
@@ -149,8 +149,9 @@ void radeon_gart_unbind(struct radeon_device *rdev, unsigned offset,
         p = t / (PAGE_SIZE / RADEON_GPU_PAGE_SIZE);
         for (i = 0; i < pages; i++, p++) {
                 if (rdev->gart.pages[p]) {
-                        pci_unmap_page(rdev->pdev, rdev->gart.pages_addr[p],
-                                       PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
+                        if (!rdev->gart.ttm_alloced[p])
+                                pci_unmap_page(rdev->pdev, rdev->gart.pages_addr[p],
+                                                PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
                         rdev->gart.pages[p] = NULL;
                         rdev->gart.pages_addr[p] = rdev->dummy_page.addr;
                         page_base = rdev->gart.pages_addr[p];
@@ -165,7 +166,7 @@ void radeon_gart_unbind(struct radeon_device *rdev, unsigned offset,
 }
 
 int radeon_gart_bind(struct radeon_device *rdev, unsigned offset,
-                     int pages, struct page **pagelist)
+                     int pages, struct page **pagelist, dma_addr_t *dma_addr)
 {
         unsigned t;
         unsigned p;
@@ -180,15 +181,22 @@ int radeon_gart_bind(struct radeon_device *rdev, unsigned offset,
         p = t / (PAGE_SIZE / RADEON_GPU_PAGE_SIZE);
 
         for (i = 0; i < pages; i++, p++) {
-                /* we need to support large memory configurations */
-                /* assume that unbind have already been call on the range */
-                rdev->gart.pages_addr[p] = pci_map_page(rdev->pdev, pagelist[i],
+                /* On TTM path, we only use the DMA API if TTM_PAGE_FLAG_DMA32
+                 * is requested. */
+                if (dma_addr[i] != DMA_ERROR_CODE) {
+                        rdev->gart.ttm_alloced[p] = true;
+                        rdev->gart.pages_addr[p] = dma_addr[i];
+                } else {
+                        /* we need to support large memory configurations */
+                        /* assume that unbind have already been call on the range */
+                        rdev->gart.pages_addr[p] = pci_map_page(rdev->pdev, pagelist[i],
                                                         0, PAGE_SIZE,
                                                         PCI_DMA_BIDIRECTIONAL);
-                if (pci_dma_mapping_error(rdev->pdev, rdev->gart.pages_addr[p])) {
-                        /* FIXME: failed to map page (return -ENOMEM?) */
-                        radeon_gart_unbind(rdev, offset, pages);
-                        return -ENOMEM;
+                        if (pci_dma_mapping_error(rdev->pdev, rdev->gart.pages_addr[p])) {
+                                /* FIXME: failed to map page (return -ENOMEM?) */
+                                radeon_gart_unbind(rdev, offset, pages);
+                                return -ENOMEM;
+                        }
                 }
                 rdev->gart.pages[p] = pagelist[i];
                 page_base = rdev->gart.pages_addr[p];
@@ -251,6 +259,12 @@ int radeon_gart_init(struct radeon_device *rdev)
                 radeon_gart_fini(rdev);
                 return -ENOMEM;
         }
+        rdev->gart.ttm_alloced = kzalloc(sizeof(bool) *
+                                         rdev->gart.num_cpu_pages, GFP_KERNEL);
+        if (rdev->gart.ttm_alloced == NULL) {
+                radeon_gart_fini(rdev);
+                return -ENOMEM;
+        }
         /* set GART entry to point to the dummy page by default */
         for (i = 0; i < rdev->gart.num_cpu_pages; i++) {
                 rdev->gart.pages_addr[i] = rdev->dummy_page.addr;
@@ -267,6 +281,8 @@ void radeon_gart_fini(struct radeon_device *rdev)
         rdev->gart.ready = false;
         kfree(rdev->gart.pages);
         kfree(rdev->gart.pages_addr);
+        kfree(rdev->gart.ttm_alloced);
         rdev->gart.pages = NULL;
         rdev->gart.pages_addr = NULL;
+        rdev->gart.ttm_alloced = NULL;
 }
diff --git a/drivers/gpu/drm/radeon/radeon_ttm.c b/drivers/gpu/drm/radeon/radeon_ttm.c
index 1272e4b6a1d4..c345e899e881 100644
--- a/drivers/gpu/drm/radeon/radeon_ttm.c
+++ b/drivers/gpu/drm/radeon/radeon_ttm.c
@@ -513,6 +513,7 @@ int radeon_ttm_init(struct radeon_device *rdev)
         if (r) {
                 return r;
         }
+        rdev->mman.bdev.dev = rdev->dev;
         /* No others user of address space so set it to 0 */
         r = ttm_bo_device_init(&rdev->mman.bdev,
                                rdev->mman.bo_global_ref.ref.object,
@@ -647,6 +648,7 @@ struct radeon_ttm_backend {
         unsigned long                   num_pages;
         struct page                     **pages;
         struct page                     *dummy_read_page;
+        dma_addr_t                      *dma_addrs;
         bool                            populated;
         bool                            bound;
         unsigned                        offset;
@@ -655,12 +657,14 @@ struct radeon_ttm_backend {
 static int radeon_ttm_backend_populate(struct ttm_backend *backend,
                                        unsigned long num_pages,
                                        struct page **pages,
-                                       struct page *dummy_read_page)
+                                       struct page *dummy_read_page,
+                                       dma_addr_t *dma_addrs)
 {
         struct radeon_ttm_backend *gtt;
 
         gtt = container_of(backend, struct radeon_ttm_backend, backend);
         gtt->pages = pages;
+        gtt->dma_addrs = dma_addrs;
         gtt->num_pages = num_pages;
         gtt->dummy_read_page = dummy_read_page;
         gtt->populated = true;
@@ -673,6 +677,7 @@ static void radeon_ttm_backend_clear(struct ttm_backend *backend)
 
         gtt = container_of(backend, struct radeon_ttm_backend, backend);
         gtt->pages = NULL;
+        gtt->dma_addrs = NULL;
         gtt->num_pages = 0;
         gtt->dummy_read_page = NULL;
         gtt->populated = false;
@@ -693,7 +698,7 @@ static int radeon_ttm_backend_bind(struct ttm_backend *backend,
                      gtt->num_pages, bo_mem, backend);
         }
         r = radeon_gart_bind(gtt->rdev, gtt->offset,
-                             gtt->num_pages, gtt->pages);
+                             gtt->num_pages, gtt->pages, gtt->dma_addrs);
         if (r) {
                 DRM_ERROR("failed to bind %lu pages at 0x%08X\n",
                           gtt->num_pages, gtt->offset);
diff --git a/drivers/gpu/drm/ttm/ttm_agp_backend.c b/drivers/gpu/drm/ttm/ttm_agp_backend.c
index f999e36f30b4..1c4a72f681c1 100644
--- a/drivers/gpu/drm/ttm/ttm_agp_backend.c
+++ b/drivers/gpu/drm/ttm/ttm_agp_backend.c
@@ -47,7 +47,8 @@ struct ttm_agp_backend {
 
 static int ttm_agp_populate(struct ttm_backend *backend,
                             unsigned long num_pages, struct page **pages,
-                            struct page *dummy_read_page)
+                            struct page *dummy_read_page,
+                            dma_addr_t *dma_addrs)
 {
         struct ttm_agp_backend *agp_be =
             container_of(backend, struct ttm_agp_backend, backend);
diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc.c b/drivers/gpu/drm/ttm/ttm_page_alloc.c
index b1e02fffd3cc..35849dbf3ab5 100644
--- a/drivers/gpu/drm/ttm/ttm_page_alloc.c
+++ b/drivers/gpu/drm/ttm/ttm_page_alloc.c
@@ -38,6 +38,7 @@
 #include <linux/mm.h>
 #include <linux/seq_file.h> /* for seq_printf */
 #include <linux/slab.h>
+#include <linux/dma-mapping.h>
 
 #include <asm/atomic.h>
 
@@ -662,7 +663,8 @@ out:
  * cached pages.
  */
 int ttm_get_pages(struct list_head *pages, int flags,
-                enum ttm_caching_state cstate, unsigned count)
+                  enum ttm_caching_state cstate, unsigned count,
+                  dma_addr_t *dma_address, struct device *dev)
 {
         struct ttm_page_pool *pool = ttm_get_pool(flags, cstate);
         struct page *p = NULL;
@@ -681,14 +683,22 @@ int ttm_get_pages(struct list_head *pages, int flags,
                         gfp_flags |= GFP_HIGHUSER;
 
                 for (r = 0; r < count; ++r) {
-                        p = alloc_page(gfp_flags);
+                        if ((flags & TTM_PAGE_FLAG_DMA32) && dma_address) {
+                                void *addr;
+                                addr = dma_alloc_coherent(dev, PAGE_SIZE,
+                                                          &dma_address[r],
+                                                          gfp_flags);
+                                if (addr == NULL)
+                                        return -ENOMEM;
+                                p = virt_to_page(addr);
+                        } else
+                                p = alloc_page(gfp_flags);
                         if (!p) {
 
                                 printk(KERN_ERR TTM_PFX
                                        "Unable to allocate page.");
                                 return -ENOMEM;
                         }
-
                         list_add(&p->lru, pages);
                 }
                 return 0;
@@ -720,7 +730,7 @@ int ttm_get_pages(struct list_head *pages, int flags,
                         printk(KERN_ERR TTM_PFX
                                "Failed to allocate extra pages "
                                "for large request.");
-                        ttm_put_pages(pages, 0, flags, cstate);
+                        ttm_put_pages(pages, 0, flags, cstate, NULL, NULL);
                         return r;
                 }
         }
@@ -731,17 +741,30 @@ int ttm_get_pages(struct list_head *pages, int flags,
 
 /* Put all pages in pages list to correct pool to wait for reuse */
 void ttm_put_pages(struct list_head *pages, unsigned page_count, int flags,
-                enum ttm_caching_state cstate)
+                   enum ttm_caching_state cstate, dma_addr_t *dma_address,
+                   struct device *dev)
 {
         unsigned long irq_flags;
         struct ttm_page_pool *pool = ttm_get_pool(flags, cstate);
         struct page *p, *tmp;
+        unsigned r;
 
         if (pool == NULL) {
                 /* No pool for this memory type so free the pages */
 
+                r = page_count-1;
                 list_for_each_entry_safe(p, tmp, pages, lru) {
-                        __free_page(p);
+                        if ((flags & TTM_PAGE_FLAG_DMA32) && dma_address) {
+                                void *addr = page_address(p);
+                                WARN_ON(!addr || !dma_address[r]);
+                                if (addr)
+                                        dma_free_coherent(dev, PAGE_SIZE,
+                                                          addr,
+                                                          dma_address[r]);
+                                dma_address[r] = 0;
+                        } else
+                                __free_page(p);
+                        r--;
                 }
                 /* Make the pages list empty */
                 INIT_LIST_HEAD(pages);
diff --git a/drivers/gpu/drm/ttm/ttm_tt.c b/drivers/gpu/drm/ttm/ttm_tt.c
index af789dc869b9..0f8fc9ff0c53 100644
--- a/drivers/gpu/drm/ttm/ttm_tt.c
+++ b/drivers/gpu/drm/ttm/ttm_tt.c
@@ -49,12 +49,16 @@ static int ttm_tt_swapin(struct ttm_tt *ttm);
 static void ttm_tt_alloc_page_directory(struct ttm_tt *ttm)
 {
         ttm->pages = drm_calloc_large(ttm->num_pages, sizeof(*ttm->pages));
+        ttm->dma_address = drm_calloc_large(ttm->num_pages,
+                                            sizeof(*ttm->dma_address));
 }
 
 static void ttm_tt_free_page_directory(struct ttm_tt *ttm)
 {
         drm_free_large(ttm->pages);
         ttm->pages = NULL;
+        drm_free_large(ttm->dma_address);
+        ttm->dma_address = NULL;
 }
 
 static void ttm_tt_free_user_pages(struct ttm_tt *ttm)
@@ -105,7 +109,8 @@ static struct page *__ttm_tt_get_page(struct ttm_tt *ttm, int index)
 
                 INIT_LIST_HEAD(&h);
 
-                ret = ttm_get_pages(&h, ttm->page_flags, ttm->caching_state, 1);
+                ret = ttm_get_pages(&h, ttm->page_flags, ttm->caching_state, 1,
+                                    &ttm->dma_address[index], ttm->be->bdev->dev);
 
                 if (ret != 0)
                         return NULL;
@@ -164,7 +169,7 @@ int ttm_tt_populate(struct ttm_tt *ttm)
         }
 
         be->func->populate(be, ttm->num_pages, ttm->pages,
-                           ttm->dummy_read_page);
+                           ttm->dummy_read_page, ttm->dma_address);
         ttm->state = tt_unbound;
         return 0;
 }
@@ -298,7 +303,8 @@ static void ttm_tt_free_alloced_pages(struct ttm_tt *ttm)
                         count++;
                 }
         }
-        ttm_put_pages(&h, count, ttm->page_flags, ttm->caching_state);
+        ttm_put_pages(&h, count, ttm->page_flags, ttm->caching_state,
+                      ttm->dma_address, ttm->be->bdev->dev);
         ttm->state = tt_unpopulated;
         ttm->first_himem_page = ttm->num_pages;
         ttm->last_lomem_page = -1;
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c b/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c
index 80bc37b274e7..87e43e0733bf 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c
@@ -102,7 +102,8 @@ struct vmw_ttm_backend {
 
 static int vmw_ttm_populate(struct ttm_backend *backend,
                             unsigned long num_pages, struct page **pages,
-                            struct page *dummy_read_page)
+                            struct page *dummy_read_page,
+                            dma_addr_t *dma_addrs)
 {
         struct vmw_ttm_backend *vmw_be =
             container_of(backend, struct vmw_ttm_backend, backend);
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
index 96949b93d920..df04661e2b93 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
@@ -322,7 +322,7 @@ static int vmw_driver_load(struct drm_device *dev, unsigned long chipset)
         ttm_lock_set_kill(&dev_priv->fbdev_master.lock, false, SIGTERM);
         dev_priv->active_master = &dev_priv->fbdev_master;
 
-
+        dev_priv->bdev.dev = dev->dev;
         ret = ttm_bo_device_init(&dev_priv->bdev,
                                  dev_priv->bo_global_ref.ref.object,
                                  &vmw_bo_driver, VMWGFX_FILE_PAGE_OFFSET,
diff --git a/include/drm/ttm/ttm_bo_driver.h b/include/drm/ttm/ttm_bo_driver.h
index 1da8af6ac884..38ff06822609 100644
--- a/include/drm/ttm/ttm_bo_driver.h
+++ b/include/drm/ttm/ttm_bo_driver.h
@@ -50,13 +50,15 @@ struct ttm_backend_func {
          * @pages: Array of pointers to ttm pages.
          * @dummy_read_page: Page to be used instead of NULL pages in the
          * array @pages.
+         * @dma_addrs: Array of DMA (bus) address of the ttm pages.
          *
          * Populate the backend with ttm pages. Depending on the backend,
          * it may or may not copy the @pages array.
          */
         int (*populate) (struct ttm_backend *backend,
                          unsigned long num_pages, struct page **pages,
-                         struct page *dummy_read_page);
+                         struct page *dummy_read_page,
+                         dma_addr_t *dma_addrs);
         /**
          * struct ttm_backend_func member clear
          *
@@ -149,6 +151,7 @@ enum ttm_caching_state {
  * @swap_storage: Pointer to shmem struct file for swap storage.
  * @caching_state: The current caching state of the pages.
  * @state: The current binding state of the pages.
+ * @dma_address: The DMA (bus) addresses of the pages (if TTM_PAGE_FLAG_DMA32)
  *
  * This is a structure holding the pages, caching- and aperture binding
  * status for a buffer object that isn't backed by fixed (VRAM / AGP)
@@ -173,6 +176,7 @@ struct ttm_tt {
                 tt_unbound,
                 tt_unpopulated,
         } state;
+        dma_addr_t *dma_address;
 };
 
 #define TTM_MEMTYPE_FLAG_FIXED         (1 << 0) /* Fixed (on-card) PCI memory */
@@ -547,6 +551,7 @@ struct ttm_bo_device {
         struct list_head device_list;
         struct ttm_bo_global *glob;
         struct ttm_bo_driver *driver;
+        struct device *dev;
         rwlock_t vm_lock;
         struct ttm_mem_type_manager man[TTM_NUM_MEM_TYPES];
         spinlock_t fence_lock;
diff --git a/include/drm/ttm/ttm_page_alloc.h b/include/drm/ttm/ttm_page_alloc.h
index 116821448c38..ccb6b7a240e2 100644
--- a/include/drm/ttm/ttm_page_alloc.h
+++ b/include/drm/ttm/ttm_page_alloc.h
@@ -36,11 +36,15 @@
  * @flags: ttm flags for page allocation.
  * @cstate: ttm caching state for the page.
  * @count: number of pages to allocate.
+ * @dma_address: The DMA (bus) address of pages (if TTM_PAGE_FLAG_DMA32 set).
+ * @dev: struct device for appropiate DMA accounting.
  */
 int ttm_get_pages(struct list_head *pages,
                   int flags,
                   enum ttm_caching_state cstate,
-                  unsigned count);
+                  unsigned count,
+                  dma_addr_t *dma_address,
+                  struct device *dev);
 /**
  * Put linked list of pages to pool.
  *
@@ -49,11 +53,15 @@ int ttm_get_pages(struct list_head *pages,
  * count.
  * @flags: ttm flags for page allocation.
  * @cstate: ttm caching state.
+ * @dma_address: The DMA (bus) address of pages (if TTM_PAGE_FLAG_DMA32 set).
+ * @dev: struct device for appropiate DMA accounting.
  */
 void ttm_put_pages(struct list_head *pages,
                    unsigned page_count,
                    int flags,
-                   enum ttm_caching_state cstate);
+                   enum ttm_caching_state cstate,
+                   dma_addr_t *dma_address,
+                   struct device *dev);
 /**
  * Initialize pool allocator.
  */