aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/gpu/drm/ttm
diff options
context:
space:
mode:
authorDave Airlie <airlied@redhat.com>2010-04-20 00:15:09 -0400
committerDave Airlie <airlied@redhat.com>2010-04-20 00:15:09 -0400
commit7547a917fa5f3b2406f52c7dcf7ec9ad3c8532eb (patch)
tree59b0d0e9b6c251c4df5799b93395454592004d57 /drivers/gpu/drm/ttm
parenta8089e849a32c5b6bfd6c88dbd09c0ea4a779b71 (diff)
parent6b8b1786a8c29ce6e32298b93ac8d4a18a2b11c4 (diff)
Merge branch 'drm-ttm-unmappable' into drm-core-next
* drm-ttm-unmappable: drm/radeon/kms: enable use of unmappable VRAM V2 drm/ttm: remove io_ field from TTM V6 drm/vmwgfx: add support for new TTM fault callback V5 drm/nouveau/kms: add support for new TTM fault callback V5 drm/radeon/kms: add support for new fault callback V7 drm/ttm: ttm_fault callback to allow driver to handle bo placement V6 drm/ttm: split no_wait argument in 2 GPU or reserve wait Conflicts: drivers/gpu/drm/nouveau/nouveau_bo.c
Diffstat (limited to 'drivers/gpu/drm/ttm')
-rw-r--r--drivers/gpu/drm/ttm/ttm_bo.c84
-rw-r--r--drivers/gpu/drm/ttm/ttm_bo_util.c122
-rw-r--r--drivers/gpu/drm/ttm/ttm_bo_vm.c41
3 files changed, 114 insertions, 133 deletions
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c
index dd47b2a9a791..3b5b094b1397 100644
--- a/drivers/gpu/drm/ttm/ttm_bo.c
+++ b/drivers/gpu/drm/ttm/ttm_bo.c
@@ -79,8 +79,6 @@ static void ttm_mem_type_debug(struct ttm_bo_device *bdev, int mem_type)
79 printk(KERN_ERR TTM_PFX " use_type: %d\n", man->use_type); 79 printk(KERN_ERR TTM_PFX " use_type: %d\n", man->use_type);
80 printk(KERN_ERR TTM_PFX " flags: 0x%08X\n", man->flags); 80 printk(KERN_ERR TTM_PFX " flags: 0x%08X\n", man->flags);
81 printk(KERN_ERR TTM_PFX " gpu_offset: 0x%08lX\n", man->gpu_offset); 81 printk(KERN_ERR TTM_PFX " gpu_offset: 0x%08lX\n", man->gpu_offset);
82 printk(KERN_ERR TTM_PFX " io_offset: 0x%08lX\n", man->io_offset);
83 printk(KERN_ERR TTM_PFX " io_size: %ld\n", man->io_size);
84 printk(KERN_ERR TTM_PFX " size: %llu\n", man->size); 82 printk(KERN_ERR TTM_PFX " size: %llu\n", man->size);
85 printk(KERN_ERR TTM_PFX " available_caching: 0x%08X\n", 83 printk(KERN_ERR TTM_PFX " available_caching: 0x%08X\n",
86 man->available_caching); 84 man->available_caching);
@@ -357,7 +355,8 @@ static int ttm_bo_add_ttm(struct ttm_buffer_object *bo, bool zero_alloc)
357 355
358static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, 356static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
359 struct ttm_mem_reg *mem, 357 struct ttm_mem_reg *mem,
360 bool evict, bool interruptible, bool no_wait) 358 bool evict, bool interruptible,
359 bool no_wait_reserve, bool no_wait_gpu)
361{ 360{
362 struct ttm_bo_device *bdev = bo->bdev; 361 struct ttm_bo_device *bdev = bo->bdev;
363 bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem); 362 bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem);
@@ -402,12 +401,12 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
402 401
403 if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) && 402 if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) &&
404 !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) 403 !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED))
405 ret = ttm_bo_move_ttm(bo, evict, no_wait, mem); 404 ret = ttm_bo_move_ttm(bo, evict, no_wait_reserve, no_wait_gpu, mem);
406 else if (bdev->driver->move) 405 else if (bdev->driver->move)
407 ret = bdev->driver->move(bo, evict, interruptible, 406 ret = bdev->driver->move(bo, evict, interruptible,
408 no_wait, mem); 407 no_wait_reserve, no_wait_gpu, mem);
409 else 408 else
410 ret = ttm_bo_move_memcpy(bo, evict, no_wait, mem); 409 ret = ttm_bo_move_memcpy(bo, evict, no_wait_reserve, no_wait_gpu, mem);
411 410
412 if (ret) 411 if (ret)
413 goto out_err; 412 goto out_err;
@@ -606,7 +605,7 @@ void ttm_bo_unref(struct ttm_buffer_object **p_bo)
606EXPORT_SYMBOL(ttm_bo_unref); 605EXPORT_SYMBOL(ttm_bo_unref);
607 606
608static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible, 607static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible,
609 bool no_wait) 608 bool no_wait_reserve, bool no_wait_gpu)
610{ 609{
611 struct ttm_bo_device *bdev = bo->bdev; 610 struct ttm_bo_device *bdev = bo->bdev;
612 struct ttm_bo_global *glob = bo->glob; 611 struct ttm_bo_global *glob = bo->glob;
@@ -615,7 +614,7 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible,
615 int ret = 0; 614 int ret = 0;
616 615
617 spin_lock(&bo->lock); 616 spin_lock(&bo->lock);
618 ret = ttm_bo_wait(bo, false, interruptible, no_wait); 617 ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu);
619 spin_unlock(&bo->lock); 618 spin_unlock(&bo->lock);
620 619
621 if (unlikely(ret != 0)) { 620 if (unlikely(ret != 0)) {
@@ -631,6 +630,7 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible,
631 630
632 evict_mem = bo->mem; 631 evict_mem = bo->mem;
633 evict_mem.mm_node = NULL; 632 evict_mem.mm_node = NULL;
633 evict_mem.bus.io_reserved = false;
634 634
635 placement.fpfn = 0; 635 placement.fpfn = 0;
636 placement.lpfn = 0; 636 placement.lpfn = 0;
@@ -638,7 +638,7 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible,
638 placement.num_busy_placement = 0; 638 placement.num_busy_placement = 0;
639 bdev->driver->evict_flags(bo, &placement); 639 bdev->driver->evict_flags(bo, &placement);
640 ret = ttm_bo_mem_space(bo, &placement, &evict_mem, interruptible, 640 ret = ttm_bo_mem_space(bo, &placement, &evict_mem, interruptible,
641 no_wait); 641 no_wait_reserve, no_wait_gpu);
642 if (ret) { 642 if (ret) {
643 if (ret != -ERESTARTSYS) { 643 if (ret != -ERESTARTSYS) {
644 printk(KERN_ERR TTM_PFX 644 printk(KERN_ERR TTM_PFX
@@ -650,7 +650,7 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible,
650 } 650 }
651 651
652 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, interruptible, 652 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, interruptible,
653 no_wait); 653 no_wait_reserve, no_wait_gpu);
654 if (ret) { 654 if (ret) {
655 if (ret != -ERESTARTSYS) 655 if (ret != -ERESTARTSYS)
656 printk(KERN_ERR TTM_PFX "Buffer eviction failed\n"); 656 printk(KERN_ERR TTM_PFX "Buffer eviction failed\n");
@@ -670,7 +670,8 @@ out:
670 670
671static int ttm_mem_evict_first(struct ttm_bo_device *bdev, 671static int ttm_mem_evict_first(struct ttm_bo_device *bdev,
672 uint32_t mem_type, 672 uint32_t mem_type,
673 bool interruptible, bool no_wait) 673 bool interruptible, bool no_wait_reserve,
674 bool no_wait_gpu)
674{ 675{
675 struct ttm_bo_global *glob = bdev->glob; 676 struct ttm_bo_global *glob = bdev->glob;
676 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 677 struct ttm_mem_type_manager *man = &bdev->man[mem_type];
@@ -687,11 +688,11 @@ retry:
687 bo = list_first_entry(&man->lru, struct ttm_buffer_object, lru); 688 bo = list_first_entry(&man->lru, struct ttm_buffer_object, lru);
688 kref_get(&bo->list_kref); 689 kref_get(&bo->list_kref);
689 690
690 ret = ttm_bo_reserve_locked(bo, false, true, false, 0); 691 ret = ttm_bo_reserve_locked(bo, false, no_wait_reserve, false, 0);
691 692
692 if (unlikely(ret == -EBUSY)) { 693 if (unlikely(ret == -EBUSY)) {
693 spin_unlock(&glob->lru_lock); 694 spin_unlock(&glob->lru_lock);
694 if (likely(!no_wait)) 695 if (likely(!no_wait_gpu))
695 ret = ttm_bo_wait_unreserved(bo, interruptible); 696 ret = ttm_bo_wait_unreserved(bo, interruptible);
696 697
697 kref_put(&bo->list_kref, ttm_bo_release_list); 698 kref_put(&bo->list_kref, ttm_bo_release_list);
@@ -713,7 +714,7 @@ retry:
713 while (put_count--) 714 while (put_count--)
714 kref_put(&bo->list_kref, ttm_bo_ref_bug); 715 kref_put(&bo->list_kref, ttm_bo_ref_bug);
715 716
716 ret = ttm_bo_evict(bo, interruptible, no_wait); 717 ret = ttm_bo_evict(bo, interruptible, no_wait_reserve, no_wait_gpu);
717 ttm_bo_unreserve(bo); 718 ttm_bo_unreserve(bo);
718 719
719 kref_put(&bo->list_kref, ttm_bo_release_list); 720 kref_put(&bo->list_kref, ttm_bo_release_list);
@@ -764,7 +765,9 @@ static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo,
764 uint32_t mem_type, 765 uint32_t mem_type,
765 struct ttm_placement *placement, 766 struct ttm_placement *placement,
766 struct ttm_mem_reg *mem, 767 struct ttm_mem_reg *mem,
767 bool interruptible, bool no_wait) 768 bool interruptible,
769 bool no_wait_reserve,
770 bool no_wait_gpu)
768{ 771{
769 struct ttm_bo_device *bdev = bo->bdev; 772 struct ttm_bo_device *bdev = bo->bdev;
770 struct ttm_bo_global *glob = bdev->glob; 773 struct ttm_bo_global *glob = bdev->glob;
@@ -785,7 +788,7 @@ static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo,
785 } 788 }
786 spin_unlock(&glob->lru_lock); 789 spin_unlock(&glob->lru_lock);
787 ret = ttm_mem_evict_first(bdev, mem_type, interruptible, 790 ret = ttm_mem_evict_first(bdev, mem_type, interruptible,
788 no_wait); 791 no_wait_reserve, no_wait_gpu);
789 if (unlikely(ret != 0)) 792 if (unlikely(ret != 0))
790 return ret; 793 return ret;
791 } while (1); 794 } while (1);
@@ -855,7 +858,8 @@ static bool ttm_bo_mt_compatible(struct ttm_mem_type_manager *man,
855int ttm_bo_mem_space(struct ttm_buffer_object *bo, 858int ttm_bo_mem_space(struct ttm_buffer_object *bo,
856 struct ttm_placement *placement, 859 struct ttm_placement *placement,
857 struct ttm_mem_reg *mem, 860 struct ttm_mem_reg *mem,
858 bool interruptible, bool no_wait) 861 bool interruptible, bool no_wait_reserve,
862 bool no_wait_gpu)
859{ 863{
860 struct ttm_bo_device *bdev = bo->bdev; 864 struct ttm_bo_device *bdev = bo->bdev;
861 struct ttm_mem_type_manager *man; 865 struct ttm_mem_type_manager *man;
@@ -952,7 +956,7 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo,
952 } 956 }
953 957
954 ret = ttm_bo_mem_force_space(bo, mem_type, placement, mem, 958 ret = ttm_bo_mem_force_space(bo, mem_type, placement, mem,
955 interruptible, no_wait); 959 interruptible, no_wait_reserve, no_wait_gpu);
956 if (ret == 0 && mem->mm_node) { 960 if (ret == 0 && mem->mm_node) {
957 mem->placement = cur_flags; 961 mem->placement = cur_flags;
958 mem->mm_node->private = bo; 962 mem->mm_node->private = bo;
@@ -978,7 +982,8 @@ EXPORT_SYMBOL(ttm_bo_wait_cpu);
978 982
979int ttm_bo_move_buffer(struct ttm_buffer_object *bo, 983int ttm_bo_move_buffer(struct ttm_buffer_object *bo,
980 struct ttm_placement *placement, 984 struct ttm_placement *placement,
981 bool interruptible, bool no_wait) 985 bool interruptible, bool no_wait_reserve,
986 bool no_wait_gpu)
982{ 987{
983 struct ttm_bo_global *glob = bo->glob; 988 struct ttm_bo_global *glob = bo->glob;
984 int ret = 0; 989 int ret = 0;
@@ -992,20 +997,21 @@ int ttm_bo_move_buffer(struct ttm_buffer_object *bo,
992 * instead of doing it here. 997 * instead of doing it here.
993 */ 998 */
994 spin_lock(&bo->lock); 999 spin_lock(&bo->lock);
995 ret = ttm_bo_wait(bo, false, interruptible, no_wait); 1000 ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu);
996 spin_unlock(&bo->lock); 1001 spin_unlock(&bo->lock);
997 if (ret) 1002 if (ret)
998 return ret; 1003 return ret;
999 mem.num_pages = bo->num_pages; 1004 mem.num_pages = bo->num_pages;
1000 mem.size = mem.num_pages << PAGE_SHIFT; 1005 mem.size = mem.num_pages << PAGE_SHIFT;
1001 mem.page_alignment = bo->mem.page_alignment; 1006 mem.page_alignment = bo->mem.page_alignment;
1007 mem.bus.io_reserved = false;
1002 /* 1008 /*
1003 * Determine where to move the buffer. 1009 * Determine where to move the buffer.
1004 */ 1010 */
1005 ret = ttm_bo_mem_space(bo, placement, &mem, interruptible, no_wait); 1011 ret = ttm_bo_mem_space(bo, placement, &mem, interruptible, no_wait_reserve, no_wait_gpu);
1006 if (ret) 1012 if (ret)
1007 goto out_unlock; 1013 goto out_unlock;
1008 ret = ttm_bo_handle_move_mem(bo, &mem, false, interruptible, no_wait); 1014 ret = ttm_bo_handle_move_mem(bo, &mem, false, interruptible, no_wait_reserve, no_wait_gpu);
1009out_unlock: 1015out_unlock:
1010 if (ret && mem.mm_node) { 1016 if (ret && mem.mm_node) {
1011 spin_lock(&glob->lru_lock); 1017 spin_lock(&glob->lru_lock);
@@ -1039,7 +1045,8 @@ static int ttm_bo_mem_compat(struct ttm_placement *placement,
1039 1045
1040int ttm_bo_validate(struct ttm_buffer_object *bo, 1046int ttm_bo_validate(struct ttm_buffer_object *bo,
1041 struct ttm_placement *placement, 1047 struct ttm_placement *placement,
1042 bool interruptible, bool no_wait) 1048 bool interruptible, bool no_wait_reserve,
1049 bool no_wait_gpu)
1043{ 1050{
1044 int ret; 1051 int ret;
1045 1052
@@ -1054,7 +1061,7 @@ int ttm_bo_validate(struct ttm_buffer_object *bo,
1054 */ 1061 */
1055 ret = ttm_bo_mem_compat(placement, &bo->mem); 1062 ret = ttm_bo_mem_compat(placement, &bo->mem);
1056 if (ret < 0) { 1063 if (ret < 0) {
1057 ret = ttm_bo_move_buffer(bo, placement, interruptible, no_wait); 1064 ret = ttm_bo_move_buffer(bo, placement, interruptible, no_wait_reserve, no_wait_gpu);
1058 if (ret) 1065 if (ret)
1059 return ret; 1066 return ret;
1060 } else { 1067 } else {
@@ -1153,6 +1160,7 @@ int ttm_bo_init(struct ttm_bo_device *bdev,
1153 bo->mem.num_pages = bo->num_pages; 1160 bo->mem.num_pages = bo->num_pages;
1154 bo->mem.mm_node = NULL; 1161 bo->mem.mm_node = NULL;
1155 bo->mem.page_alignment = page_alignment; 1162 bo->mem.page_alignment = page_alignment;
1163 bo->mem.bus.io_reserved = false;
1156 bo->buffer_start = buffer_start & PAGE_MASK; 1164 bo->buffer_start = buffer_start & PAGE_MASK;
1157 bo->priv_flags = 0; 1165 bo->priv_flags = 0;
1158 bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED); 1166 bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED);
@@ -1175,7 +1183,7 @@ int ttm_bo_init(struct ttm_bo_device *bdev,
1175 goto out_err; 1183 goto out_err;
1176 } 1184 }
1177 1185
1178 ret = ttm_bo_validate(bo, placement, interruptible, false); 1186 ret = ttm_bo_validate(bo, placement, interruptible, false, false);
1179 if (ret) 1187 if (ret)
1180 goto out_err; 1188 goto out_err;
1181 1189
@@ -1249,7 +1257,7 @@ static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev,
1249 spin_lock(&glob->lru_lock); 1257 spin_lock(&glob->lru_lock);
1250 while (!list_empty(&man->lru)) { 1258 while (!list_empty(&man->lru)) {
1251 spin_unlock(&glob->lru_lock); 1259 spin_unlock(&glob->lru_lock);
1252 ret = ttm_mem_evict_first(bdev, mem_type, false, false); 1260 ret = ttm_mem_evict_first(bdev, mem_type, false, false, false);
1253 if (ret) { 1261 if (ret) {
1254 if (allow_errors) { 1262 if (allow_errors) {
1255 return ret; 1263 return ret;
@@ -1553,26 +1561,6 @@ bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
1553 return true; 1561 return true;
1554} 1562}
1555 1563
1556int ttm_bo_pci_offset(struct ttm_bo_device *bdev,
1557 struct ttm_mem_reg *mem,
1558 unsigned long *bus_base,
1559 unsigned long *bus_offset, unsigned long *bus_size)
1560{
1561 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
1562
1563 *bus_size = 0;
1564 if (!(man->flags & TTM_MEMTYPE_FLAG_MAPPABLE))
1565 return -EINVAL;
1566
1567 if (ttm_mem_reg_is_pci(bdev, mem)) {
1568 *bus_offset = mem->mm_node->start << PAGE_SHIFT;
1569 *bus_size = mem->num_pages << PAGE_SHIFT;
1570 *bus_base = man->io_offset;
1571 }
1572
1573 return 0;
1574}
1575
1576void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo) 1564void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo)
1577{ 1565{
1578 struct ttm_bo_device *bdev = bo->bdev; 1566 struct ttm_bo_device *bdev = bo->bdev;
@@ -1581,8 +1569,8 @@ void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo)
1581 1569
1582 if (!bdev->dev_mapping) 1570 if (!bdev->dev_mapping)
1583 return; 1571 return;
1584
1585 unmap_mapping_range(bdev->dev_mapping, offset, holelen, 1); 1572 unmap_mapping_range(bdev->dev_mapping, offset, holelen, 1);
1573 ttm_mem_io_free(bdev, &bo->mem);
1586} 1574}
1587EXPORT_SYMBOL(ttm_bo_unmap_virtual); 1575EXPORT_SYMBOL(ttm_bo_unmap_virtual);
1588 1576
@@ -1839,7 +1827,7 @@ static int ttm_bo_swapout(struct ttm_mem_shrink *shrink)
1839 evict_mem.mem_type = TTM_PL_SYSTEM; 1827 evict_mem.mem_type = TTM_PL_SYSTEM;
1840 1828
1841 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, 1829 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true,
1842 false, false); 1830 false, false, false);
1843 if (unlikely(ret != 0)) 1831 if (unlikely(ret != 0))
1844 goto out; 1832 goto out;
1845 } 1833 }
diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c
index d764e82e799b..a37a94872a14 100644
--- a/drivers/gpu/drm/ttm/ttm_bo_util.c
+++ b/drivers/gpu/drm/ttm/ttm_bo_util.c
@@ -50,7 +50,8 @@ void ttm_bo_free_old_node(struct ttm_buffer_object *bo)
50} 50}
51 51
52int ttm_bo_move_ttm(struct ttm_buffer_object *bo, 52int ttm_bo_move_ttm(struct ttm_buffer_object *bo,
53 bool evict, bool no_wait, struct ttm_mem_reg *new_mem) 53 bool evict, bool no_wait_reserve,
54 bool no_wait_gpu, struct ttm_mem_reg *new_mem)
54{ 55{
55 struct ttm_tt *ttm = bo->ttm; 56 struct ttm_tt *ttm = bo->ttm;
56 struct ttm_mem_reg *old_mem = &bo->mem; 57 struct ttm_mem_reg *old_mem = &bo->mem;
@@ -81,30 +82,51 @@ int ttm_bo_move_ttm(struct ttm_buffer_object *bo,
81} 82}
82EXPORT_SYMBOL(ttm_bo_move_ttm); 83EXPORT_SYMBOL(ttm_bo_move_ttm);
83 84
85int ttm_mem_io_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
86{
87 int ret;
88
89 if (!mem->bus.io_reserved) {
90 mem->bus.io_reserved = true;
91 ret = bdev->driver->io_mem_reserve(bdev, mem);
92 if (unlikely(ret != 0))
93 return ret;
94 }
95 return 0;
96}
97
98void ttm_mem_io_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
99{
100 if (bdev->driver->io_mem_reserve) {
101 if (mem->bus.io_reserved) {
102 mem->bus.io_reserved = false;
103 bdev->driver->io_mem_free(bdev, mem);
104 }
105 }
106}
107
84int ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem, 108int ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem,
85 void **virtual) 109 void **virtual)
86{ 110{
87 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
88 unsigned long bus_offset;
89 unsigned long bus_size;
90 unsigned long bus_base;
91 int ret; 111 int ret;
92 void *addr; 112 void *addr;
93 113
94 *virtual = NULL; 114 *virtual = NULL;
95 ret = ttm_bo_pci_offset(bdev, mem, &bus_base, &bus_offset, &bus_size); 115 ret = ttm_mem_io_reserve(bdev, mem);
96 if (ret || bus_size == 0) 116 if (ret)
97 return ret; 117 return ret;
98 118
99 if (!(man->flags & TTM_MEMTYPE_FLAG_NEEDS_IOREMAP)) 119 if (mem->bus.addr) {
100 addr = (void *)(((u8 *) man->io_addr) + bus_offset); 120 addr = mem->bus.addr;
101 else { 121 } else {
102 if (mem->placement & TTM_PL_FLAG_WC) 122 if (mem->placement & TTM_PL_FLAG_WC)
103 addr = ioremap_wc(bus_base + bus_offset, bus_size); 123 addr = ioremap_wc(mem->bus.base + mem->bus.offset, mem->bus.size);
104 else 124 else
105 addr = ioremap_nocache(bus_base + bus_offset, bus_size); 125 addr = ioremap_nocache(mem->bus.base + mem->bus.offset, mem->bus.size);
106 if (!addr) 126 if (!addr) {
127 ttm_mem_io_free(bdev, mem);
107 return -ENOMEM; 128 return -ENOMEM;
129 }
108 } 130 }
109 *virtual = addr; 131 *virtual = addr;
110 return 0; 132 return 0;
@@ -117,8 +139,9 @@ void ttm_mem_reg_iounmap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem,
117 139
118 man = &bdev->man[mem->mem_type]; 140 man = &bdev->man[mem->mem_type];
119 141
120 if (virtual && (man->flags & TTM_MEMTYPE_FLAG_NEEDS_IOREMAP)) 142 if (virtual && mem->bus.addr == NULL)
121 iounmap(virtual); 143 iounmap(virtual);
144 ttm_mem_io_free(bdev, mem);
122} 145}
123 146
124static int ttm_copy_io_page(void *dst, void *src, unsigned long page) 147static int ttm_copy_io_page(void *dst, void *src, unsigned long page)
@@ -208,7 +231,8 @@ static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst,
208} 231}
209 232
210int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, 233int ttm_bo_move_memcpy(struct ttm_buffer_object *bo,
211 bool evict, bool no_wait, struct ttm_mem_reg *new_mem) 234 bool evict, bool no_wait_reserve, bool no_wait_gpu,
235 struct ttm_mem_reg *new_mem)
212{ 236{
213 struct ttm_bo_device *bdev = bo->bdev; 237 struct ttm_bo_device *bdev = bo->bdev;
214 struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type]; 238 struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type];
@@ -369,26 +393,23 @@ pgprot_t ttm_io_prot(uint32_t caching_flags, pgprot_t tmp)
369EXPORT_SYMBOL(ttm_io_prot); 393EXPORT_SYMBOL(ttm_io_prot);
370 394
371static int ttm_bo_ioremap(struct ttm_buffer_object *bo, 395static int ttm_bo_ioremap(struct ttm_buffer_object *bo,
372 unsigned long bus_base, 396 unsigned long offset,
373 unsigned long bus_offset, 397 unsigned long size,
374 unsigned long bus_size,
375 struct ttm_bo_kmap_obj *map) 398 struct ttm_bo_kmap_obj *map)
376{ 399{
377 struct ttm_bo_device *bdev = bo->bdev;
378 struct ttm_mem_reg *mem = &bo->mem; 400 struct ttm_mem_reg *mem = &bo->mem;
379 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
380 401
381 if (!(man->flags & TTM_MEMTYPE_FLAG_NEEDS_IOREMAP)) { 402 if (bo->mem.bus.addr) {
382 map->bo_kmap_type = ttm_bo_map_premapped; 403 map->bo_kmap_type = ttm_bo_map_premapped;
383 map->virtual = (void *)(((u8 *) man->io_addr) + bus_offset); 404 map->virtual = (void *)(((u8 *)bo->mem.bus.addr) + offset);
384 } else { 405 } else {
385 map->bo_kmap_type = ttm_bo_map_iomap; 406 map->bo_kmap_type = ttm_bo_map_iomap;
386 if (mem->placement & TTM_PL_FLAG_WC) 407 if (mem->placement & TTM_PL_FLAG_WC)
387 map->virtual = ioremap_wc(bus_base + bus_offset, 408 map->virtual = ioremap_wc(bo->mem.bus.base + bo->mem.bus.offset + offset,
388 bus_size); 409 size);
389 else 410 else
390 map->virtual = ioremap_nocache(bus_base + bus_offset, 411 map->virtual = ioremap_nocache(bo->mem.bus.base + bo->mem.bus.offset + offset,
391 bus_size); 412 size);
392 } 413 }
393 return (!map->virtual) ? -ENOMEM : 0; 414 return (!map->virtual) ? -ENOMEM : 0;
394} 415}
@@ -441,13 +462,12 @@ int ttm_bo_kmap(struct ttm_buffer_object *bo,
441 unsigned long start_page, unsigned long num_pages, 462 unsigned long start_page, unsigned long num_pages,
442 struct ttm_bo_kmap_obj *map) 463 struct ttm_bo_kmap_obj *map)
443{ 464{
465 unsigned long offset, size;
444 int ret; 466 int ret;
445 unsigned long bus_base;
446 unsigned long bus_offset;
447 unsigned long bus_size;
448 467
449 BUG_ON(!list_empty(&bo->swap)); 468 BUG_ON(!list_empty(&bo->swap));
450 map->virtual = NULL; 469 map->virtual = NULL;
470 map->bo = bo;
451 if (num_pages > bo->num_pages) 471 if (num_pages > bo->num_pages)
452 return -EINVAL; 472 return -EINVAL;
453 if (start_page > bo->num_pages) 473 if (start_page > bo->num_pages)
@@ -456,16 +476,15 @@ int ttm_bo_kmap(struct ttm_buffer_object *bo,
456 if (num_pages > 1 && !DRM_SUSER(DRM_CURPROC)) 476 if (num_pages > 1 && !DRM_SUSER(DRM_CURPROC))
457 return -EPERM; 477 return -EPERM;
458#endif 478#endif
459 ret = ttm_bo_pci_offset(bo->bdev, &bo->mem, &bus_base, 479 ret = ttm_mem_io_reserve(bo->bdev, &bo->mem);
460 &bus_offset, &bus_size);
461 if (ret) 480 if (ret)
462 return ret; 481 return ret;
463 if (bus_size == 0) { 482 if (!bo->mem.bus.is_iomem) {
464 return ttm_bo_kmap_ttm(bo, start_page, num_pages, map); 483 return ttm_bo_kmap_ttm(bo, start_page, num_pages, map);
465 } else { 484 } else {
466 bus_offset += start_page << PAGE_SHIFT; 485 offset = start_page << PAGE_SHIFT;
467 bus_size = num_pages << PAGE_SHIFT; 486 size = num_pages << PAGE_SHIFT;
468 return ttm_bo_ioremap(bo, bus_base, bus_offset, bus_size, map); 487 return ttm_bo_ioremap(bo, offset, size, map);
469 } 488 }
470} 489}
471EXPORT_SYMBOL(ttm_bo_kmap); 490EXPORT_SYMBOL(ttm_bo_kmap);
@@ -477,6 +496,7 @@ void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map)
477 switch (map->bo_kmap_type) { 496 switch (map->bo_kmap_type) {
478 case ttm_bo_map_iomap: 497 case ttm_bo_map_iomap:
479 iounmap(map->virtual); 498 iounmap(map->virtual);
499 ttm_mem_io_free(map->bo->bdev, &map->bo->mem);
480 break; 500 break;
481 case ttm_bo_map_vmap: 501 case ttm_bo_map_vmap:
482 vunmap(map->virtual); 502 vunmap(map->virtual);
@@ -494,39 +514,11 @@ void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map)
494} 514}
495EXPORT_SYMBOL(ttm_bo_kunmap); 515EXPORT_SYMBOL(ttm_bo_kunmap);
496 516
497int ttm_bo_pfn_prot(struct ttm_buffer_object *bo,
498 unsigned long dst_offset,
499 unsigned long *pfn, pgprot_t *prot)
500{
501 struct ttm_mem_reg *mem = &bo->mem;
502 struct ttm_bo_device *bdev = bo->bdev;
503 unsigned long bus_offset;
504 unsigned long bus_size;
505 unsigned long bus_base;
506 int ret;
507 ret = ttm_bo_pci_offset(bdev, mem, &bus_base, &bus_offset,
508 &bus_size);
509 if (ret)
510 return -EINVAL;
511 if (bus_size != 0)
512 *pfn = (bus_base + bus_offset + dst_offset) >> PAGE_SHIFT;
513 else
514 if (!bo->ttm)
515 return -EINVAL;
516 else
517 *pfn = page_to_pfn(ttm_tt_get_page(bo->ttm,
518 dst_offset >>
519 PAGE_SHIFT));
520 *prot = (mem->placement & TTM_PL_FLAG_CACHED) ?
521 PAGE_KERNEL : ttm_io_prot(mem->placement, PAGE_KERNEL);
522
523 return 0;
524}
525
526int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo, 517int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo,
527 void *sync_obj, 518 void *sync_obj,
528 void *sync_obj_arg, 519 void *sync_obj_arg,
529 bool evict, bool no_wait, 520 bool evict, bool no_wait_reserve,
521 bool no_wait_gpu,
530 struct ttm_mem_reg *new_mem) 522 struct ttm_mem_reg *new_mem)
531{ 523{
532 struct ttm_bo_device *bdev = bo->bdev; 524 struct ttm_bo_device *bdev = bo->bdev;
diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c
index 668dbe8b8dd3..fe6cb77899f4 100644
--- a/drivers/gpu/drm/ttm/ttm_bo_vm.c
+++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c
@@ -74,9 +74,6 @@ static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
74 struct ttm_buffer_object *bo = (struct ttm_buffer_object *) 74 struct ttm_buffer_object *bo = (struct ttm_buffer_object *)
75 vma->vm_private_data; 75 vma->vm_private_data;
76 struct ttm_bo_device *bdev = bo->bdev; 76 struct ttm_bo_device *bdev = bo->bdev;
77 unsigned long bus_base;
78 unsigned long bus_offset;
79 unsigned long bus_size;
80 unsigned long page_offset; 77 unsigned long page_offset;
81 unsigned long page_last; 78 unsigned long page_last;
82 unsigned long pfn; 79 unsigned long pfn;
@@ -84,7 +81,6 @@ static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
84 struct page *page; 81 struct page *page;
85 int ret; 82 int ret;
86 int i; 83 int i;
87 bool is_iomem;
88 unsigned long address = (unsigned long)vmf->virtual_address; 84 unsigned long address = (unsigned long)vmf->virtual_address;
89 int retval = VM_FAULT_NOPAGE; 85 int retval = VM_FAULT_NOPAGE;
90 86
@@ -101,8 +97,21 @@ static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
101 return VM_FAULT_NOPAGE; 97 return VM_FAULT_NOPAGE;
102 } 98 }
103 99
104 if (bdev->driver->fault_reserve_notify) 100 if (bdev->driver->fault_reserve_notify) {
105 bdev->driver->fault_reserve_notify(bo); 101 ret = bdev->driver->fault_reserve_notify(bo);
102 switch (ret) {
103 case 0:
104 break;
105 case -EBUSY:
106 set_need_resched();
107 case -ERESTARTSYS:
108 retval = VM_FAULT_NOPAGE;
109 goto out_unlock;
110 default:
111 retval = VM_FAULT_SIGBUS;
112 goto out_unlock;
113 }
114 }
106 115
107 /* 116 /*
108 * Wait for buffer data in transit, due to a pipelined 117 * Wait for buffer data in transit, due to a pipelined
@@ -122,15 +131,12 @@ static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
122 spin_unlock(&bo->lock); 131 spin_unlock(&bo->lock);
123 132
124 133
125 ret = ttm_bo_pci_offset(bdev, &bo->mem, &bus_base, &bus_offset, 134 ret = ttm_mem_io_reserve(bdev, &bo->mem);
126 &bus_size); 135 if (ret) {
127 if (unlikely(ret != 0)) {
128 retval = VM_FAULT_SIGBUS; 136 retval = VM_FAULT_SIGBUS;
129 goto out_unlock; 137 goto out_unlock;
130 } 138 }
131 139
132 is_iomem = (bus_size != 0);
133
134 page_offset = ((address - vma->vm_start) >> PAGE_SHIFT) + 140 page_offset = ((address - vma->vm_start) >> PAGE_SHIFT) +
135 bo->vm_node->start - vma->vm_pgoff; 141 bo->vm_node->start - vma->vm_pgoff;
136 page_last = ((vma->vm_end - vma->vm_start) >> PAGE_SHIFT) + 142 page_last = ((vma->vm_end - vma->vm_start) >> PAGE_SHIFT) +
@@ -154,8 +160,7 @@ static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
154 * vma->vm_page_prot when the object changes caching policy, with 160 * vma->vm_page_prot when the object changes caching policy, with
155 * the correct locks held. 161 * the correct locks held.
156 */ 162 */
157 163 if (bo->mem.bus.is_iomem) {
158 if (is_iomem) {
159 vma->vm_page_prot = ttm_io_prot(bo->mem.placement, 164 vma->vm_page_prot = ttm_io_prot(bo->mem.placement,
160 vma->vm_page_prot); 165 vma->vm_page_prot);
161 } else { 166 } else {
@@ -171,10 +176,8 @@ static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
171 */ 176 */
172 177
173 for (i = 0; i < TTM_BO_VM_NUM_PREFAULT; ++i) { 178 for (i = 0; i < TTM_BO_VM_NUM_PREFAULT; ++i) {
174 179 if (bo->mem.bus.is_iomem)
175 if (is_iomem) 180 pfn = ((bo->mem.bus.base + bo->mem.bus.offset) >> PAGE_SHIFT) + page_offset;
176 pfn = ((bus_base + bus_offset) >> PAGE_SHIFT) +
177 page_offset;
178 else { 181 else {
179 page = ttm_tt_get_page(ttm, page_offset); 182 page = ttm_tt_get_page(ttm, page_offset);
180 if (unlikely(!page && i == 0)) { 183 if (unlikely(!page && i == 0)) {
@@ -198,7 +201,6 @@ static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
198 retval = 201 retval =
199 (ret == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS; 202 (ret == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS;
200 goto out_unlock; 203 goto out_unlock;
201
202 } 204 }
203 205
204 address += PAGE_SIZE; 206 address += PAGE_SIZE;
@@ -221,8 +223,7 @@ static void ttm_bo_vm_open(struct vm_area_struct *vma)
221 223
222static void ttm_bo_vm_close(struct vm_area_struct *vma) 224static void ttm_bo_vm_close(struct vm_area_struct *vma)
223{ 225{
224 struct ttm_buffer_object *bo = 226 struct ttm_buffer_object *bo = (struct ttm_buffer_object *)vma->vm_private_data;
225 (struct ttm_buffer_object *)vma->vm_private_data;
226 227
227 ttm_bo_unref(&bo); 228 ttm_bo_unref(&bo);
228 vma->vm_private_data = NULL; 229 vma->vm_private_data = NULL;