diff options
author | Dave Airlie <airlied@redhat.com> | 2010-04-20 00:15:09 -0400 |
---|---|---|
committer | Dave Airlie <airlied@redhat.com> | 2010-04-20 00:15:09 -0400 |
commit | 7547a917fa5f3b2406f52c7dcf7ec9ad3c8532eb (patch) | |
tree | 59b0d0e9b6c251c4df5799b93395454592004d57 /drivers/gpu/drm/ttm | |
parent | a8089e849a32c5b6bfd6c88dbd09c0ea4a779b71 (diff) | |
parent | 6b8b1786a8c29ce6e32298b93ac8d4a18a2b11c4 (diff) |
Merge branch 'drm-ttm-unmappable' into drm-core-next
* drm-ttm-unmappable:
drm/radeon/kms: enable use of unmappable VRAM V2
drm/ttm: remove io_ field from TTM V6
drm/vmwgfx: add support for new TTM fault callback V5
drm/nouveau/kms: add support for new TTM fault callback V5
drm/radeon/kms: add support for new fault callback V7
drm/ttm: ttm_fault callback to allow driver to handle bo placement V6
drm/ttm: split no_wait argument in 2 GPU or reserve wait
Conflicts:
drivers/gpu/drm/nouveau/nouveau_bo.c
Diffstat (limited to 'drivers/gpu/drm/ttm')
-rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo.c | 84 | ||||
-rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo_util.c | 122 | ||||
-rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo_vm.c | 41 |
3 files changed, 114 insertions, 133 deletions
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index dd47b2a9a791..3b5b094b1397 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c | |||
@@ -79,8 +79,6 @@ static void ttm_mem_type_debug(struct ttm_bo_device *bdev, int mem_type) | |||
79 | printk(KERN_ERR TTM_PFX " use_type: %d\n", man->use_type); | 79 | printk(KERN_ERR TTM_PFX " use_type: %d\n", man->use_type); |
80 | printk(KERN_ERR TTM_PFX " flags: 0x%08X\n", man->flags); | 80 | printk(KERN_ERR TTM_PFX " flags: 0x%08X\n", man->flags); |
81 | printk(KERN_ERR TTM_PFX " gpu_offset: 0x%08lX\n", man->gpu_offset); | 81 | printk(KERN_ERR TTM_PFX " gpu_offset: 0x%08lX\n", man->gpu_offset); |
82 | printk(KERN_ERR TTM_PFX " io_offset: 0x%08lX\n", man->io_offset); | ||
83 | printk(KERN_ERR TTM_PFX " io_size: %ld\n", man->io_size); | ||
84 | printk(KERN_ERR TTM_PFX " size: %llu\n", man->size); | 82 | printk(KERN_ERR TTM_PFX " size: %llu\n", man->size); |
85 | printk(KERN_ERR TTM_PFX " available_caching: 0x%08X\n", | 83 | printk(KERN_ERR TTM_PFX " available_caching: 0x%08X\n", |
86 | man->available_caching); | 84 | man->available_caching); |
@@ -357,7 +355,8 @@ static int ttm_bo_add_ttm(struct ttm_buffer_object *bo, bool zero_alloc) | |||
357 | 355 | ||
358 | static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, | 356 | static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, |
359 | struct ttm_mem_reg *mem, | 357 | struct ttm_mem_reg *mem, |
360 | bool evict, bool interruptible, bool no_wait) | 358 | bool evict, bool interruptible, |
359 | bool no_wait_reserve, bool no_wait_gpu) | ||
361 | { | 360 | { |
362 | struct ttm_bo_device *bdev = bo->bdev; | 361 | struct ttm_bo_device *bdev = bo->bdev; |
363 | bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem); | 362 | bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem); |
@@ -402,12 +401,12 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, | |||
402 | 401 | ||
403 | if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) && | 402 | if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) && |
404 | !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) | 403 | !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) |
405 | ret = ttm_bo_move_ttm(bo, evict, no_wait, mem); | 404 | ret = ttm_bo_move_ttm(bo, evict, no_wait_reserve, no_wait_gpu, mem); |
406 | else if (bdev->driver->move) | 405 | else if (bdev->driver->move) |
407 | ret = bdev->driver->move(bo, evict, interruptible, | 406 | ret = bdev->driver->move(bo, evict, interruptible, |
408 | no_wait, mem); | 407 | no_wait_reserve, no_wait_gpu, mem); |
409 | else | 408 | else |
410 | ret = ttm_bo_move_memcpy(bo, evict, no_wait, mem); | 409 | ret = ttm_bo_move_memcpy(bo, evict, no_wait_reserve, no_wait_gpu, mem); |
411 | 410 | ||
412 | if (ret) | 411 | if (ret) |
413 | goto out_err; | 412 | goto out_err; |
@@ -606,7 +605,7 @@ void ttm_bo_unref(struct ttm_buffer_object **p_bo) | |||
606 | EXPORT_SYMBOL(ttm_bo_unref); | 605 | EXPORT_SYMBOL(ttm_bo_unref); |
607 | 606 | ||
608 | static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible, | 607 | static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible, |
609 | bool no_wait) | 608 | bool no_wait_reserve, bool no_wait_gpu) |
610 | { | 609 | { |
611 | struct ttm_bo_device *bdev = bo->bdev; | 610 | struct ttm_bo_device *bdev = bo->bdev; |
612 | struct ttm_bo_global *glob = bo->glob; | 611 | struct ttm_bo_global *glob = bo->glob; |
@@ -615,7 +614,7 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible, | |||
615 | int ret = 0; | 614 | int ret = 0; |
616 | 615 | ||
617 | spin_lock(&bo->lock); | 616 | spin_lock(&bo->lock); |
618 | ret = ttm_bo_wait(bo, false, interruptible, no_wait); | 617 | ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu); |
619 | spin_unlock(&bo->lock); | 618 | spin_unlock(&bo->lock); |
620 | 619 | ||
621 | if (unlikely(ret != 0)) { | 620 | if (unlikely(ret != 0)) { |
@@ -631,6 +630,7 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible, | |||
631 | 630 | ||
632 | evict_mem = bo->mem; | 631 | evict_mem = bo->mem; |
633 | evict_mem.mm_node = NULL; | 632 | evict_mem.mm_node = NULL; |
633 | evict_mem.bus.io_reserved = false; | ||
634 | 634 | ||
635 | placement.fpfn = 0; | 635 | placement.fpfn = 0; |
636 | placement.lpfn = 0; | 636 | placement.lpfn = 0; |
@@ -638,7 +638,7 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible, | |||
638 | placement.num_busy_placement = 0; | 638 | placement.num_busy_placement = 0; |
639 | bdev->driver->evict_flags(bo, &placement); | 639 | bdev->driver->evict_flags(bo, &placement); |
640 | ret = ttm_bo_mem_space(bo, &placement, &evict_mem, interruptible, | 640 | ret = ttm_bo_mem_space(bo, &placement, &evict_mem, interruptible, |
641 | no_wait); | 641 | no_wait_reserve, no_wait_gpu); |
642 | if (ret) { | 642 | if (ret) { |
643 | if (ret != -ERESTARTSYS) { | 643 | if (ret != -ERESTARTSYS) { |
644 | printk(KERN_ERR TTM_PFX | 644 | printk(KERN_ERR TTM_PFX |
@@ -650,7 +650,7 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible, | |||
650 | } | 650 | } |
651 | 651 | ||
652 | ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, interruptible, | 652 | ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, interruptible, |
653 | no_wait); | 653 | no_wait_reserve, no_wait_gpu); |
654 | if (ret) { | 654 | if (ret) { |
655 | if (ret != -ERESTARTSYS) | 655 | if (ret != -ERESTARTSYS) |
656 | printk(KERN_ERR TTM_PFX "Buffer eviction failed\n"); | 656 | printk(KERN_ERR TTM_PFX "Buffer eviction failed\n"); |
@@ -670,7 +670,8 @@ out: | |||
670 | 670 | ||
671 | static int ttm_mem_evict_first(struct ttm_bo_device *bdev, | 671 | static int ttm_mem_evict_first(struct ttm_bo_device *bdev, |
672 | uint32_t mem_type, | 672 | uint32_t mem_type, |
673 | bool interruptible, bool no_wait) | 673 | bool interruptible, bool no_wait_reserve, |
674 | bool no_wait_gpu) | ||
674 | { | 675 | { |
675 | struct ttm_bo_global *glob = bdev->glob; | 676 | struct ttm_bo_global *glob = bdev->glob; |
676 | struct ttm_mem_type_manager *man = &bdev->man[mem_type]; | 677 | struct ttm_mem_type_manager *man = &bdev->man[mem_type]; |
@@ -687,11 +688,11 @@ retry: | |||
687 | bo = list_first_entry(&man->lru, struct ttm_buffer_object, lru); | 688 | bo = list_first_entry(&man->lru, struct ttm_buffer_object, lru); |
688 | kref_get(&bo->list_kref); | 689 | kref_get(&bo->list_kref); |
689 | 690 | ||
690 | ret = ttm_bo_reserve_locked(bo, false, true, false, 0); | 691 | ret = ttm_bo_reserve_locked(bo, false, no_wait_reserve, false, 0); |
691 | 692 | ||
692 | if (unlikely(ret == -EBUSY)) { | 693 | if (unlikely(ret == -EBUSY)) { |
693 | spin_unlock(&glob->lru_lock); | 694 | spin_unlock(&glob->lru_lock); |
694 | if (likely(!no_wait)) | 695 | if (likely(!no_wait_gpu)) |
695 | ret = ttm_bo_wait_unreserved(bo, interruptible); | 696 | ret = ttm_bo_wait_unreserved(bo, interruptible); |
696 | 697 | ||
697 | kref_put(&bo->list_kref, ttm_bo_release_list); | 698 | kref_put(&bo->list_kref, ttm_bo_release_list); |
@@ -713,7 +714,7 @@ retry: | |||
713 | while (put_count--) | 714 | while (put_count--) |
714 | kref_put(&bo->list_kref, ttm_bo_ref_bug); | 715 | kref_put(&bo->list_kref, ttm_bo_ref_bug); |
715 | 716 | ||
716 | ret = ttm_bo_evict(bo, interruptible, no_wait); | 717 | ret = ttm_bo_evict(bo, interruptible, no_wait_reserve, no_wait_gpu); |
717 | ttm_bo_unreserve(bo); | 718 | ttm_bo_unreserve(bo); |
718 | 719 | ||
719 | kref_put(&bo->list_kref, ttm_bo_release_list); | 720 | kref_put(&bo->list_kref, ttm_bo_release_list); |
@@ -764,7 +765,9 @@ static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, | |||
764 | uint32_t mem_type, | 765 | uint32_t mem_type, |
765 | struct ttm_placement *placement, | 766 | struct ttm_placement *placement, |
766 | struct ttm_mem_reg *mem, | 767 | struct ttm_mem_reg *mem, |
767 | bool interruptible, bool no_wait) | 768 | bool interruptible, |
769 | bool no_wait_reserve, | ||
770 | bool no_wait_gpu) | ||
768 | { | 771 | { |
769 | struct ttm_bo_device *bdev = bo->bdev; | 772 | struct ttm_bo_device *bdev = bo->bdev; |
770 | struct ttm_bo_global *glob = bdev->glob; | 773 | struct ttm_bo_global *glob = bdev->glob; |
@@ -785,7 +788,7 @@ static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, | |||
785 | } | 788 | } |
786 | spin_unlock(&glob->lru_lock); | 789 | spin_unlock(&glob->lru_lock); |
787 | ret = ttm_mem_evict_first(bdev, mem_type, interruptible, | 790 | ret = ttm_mem_evict_first(bdev, mem_type, interruptible, |
788 | no_wait); | 791 | no_wait_reserve, no_wait_gpu); |
789 | if (unlikely(ret != 0)) | 792 | if (unlikely(ret != 0)) |
790 | return ret; | 793 | return ret; |
791 | } while (1); | 794 | } while (1); |
@@ -855,7 +858,8 @@ static bool ttm_bo_mt_compatible(struct ttm_mem_type_manager *man, | |||
855 | int ttm_bo_mem_space(struct ttm_buffer_object *bo, | 858 | int ttm_bo_mem_space(struct ttm_buffer_object *bo, |
856 | struct ttm_placement *placement, | 859 | struct ttm_placement *placement, |
857 | struct ttm_mem_reg *mem, | 860 | struct ttm_mem_reg *mem, |
858 | bool interruptible, bool no_wait) | 861 | bool interruptible, bool no_wait_reserve, |
862 | bool no_wait_gpu) | ||
859 | { | 863 | { |
860 | struct ttm_bo_device *bdev = bo->bdev; | 864 | struct ttm_bo_device *bdev = bo->bdev; |
861 | struct ttm_mem_type_manager *man; | 865 | struct ttm_mem_type_manager *man; |
@@ -952,7 +956,7 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo, | |||
952 | } | 956 | } |
953 | 957 | ||
954 | ret = ttm_bo_mem_force_space(bo, mem_type, placement, mem, | 958 | ret = ttm_bo_mem_force_space(bo, mem_type, placement, mem, |
955 | interruptible, no_wait); | 959 | interruptible, no_wait_reserve, no_wait_gpu); |
956 | if (ret == 0 && mem->mm_node) { | 960 | if (ret == 0 && mem->mm_node) { |
957 | mem->placement = cur_flags; | 961 | mem->placement = cur_flags; |
958 | mem->mm_node->private = bo; | 962 | mem->mm_node->private = bo; |
@@ -978,7 +982,8 @@ EXPORT_SYMBOL(ttm_bo_wait_cpu); | |||
978 | 982 | ||
979 | int ttm_bo_move_buffer(struct ttm_buffer_object *bo, | 983 | int ttm_bo_move_buffer(struct ttm_buffer_object *bo, |
980 | struct ttm_placement *placement, | 984 | struct ttm_placement *placement, |
981 | bool interruptible, bool no_wait) | 985 | bool interruptible, bool no_wait_reserve, |
986 | bool no_wait_gpu) | ||
982 | { | 987 | { |
983 | struct ttm_bo_global *glob = bo->glob; | 988 | struct ttm_bo_global *glob = bo->glob; |
984 | int ret = 0; | 989 | int ret = 0; |
@@ -992,20 +997,21 @@ int ttm_bo_move_buffer(struct ttm_buffer_object *bo, | |||
992 | * instead of doing it here. | 997 | * instead of doing it here. |
993 | */ | 998 | */ |
994 | spin_lock(&bo->lock); | 999 | spin_lock(&bo->lock); |
995 | ret = ttm_bo_wait(bo, false, interruptible, no_wait); | 1000 | ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu); |
996 | spin_unlock(&bo->lock); | 1001 | spin_unlock(&bo->lock); |
997 | if (ret) | 1002 | if (ret) |
998 | return ret; | 1003 | return ret; |
999 | mem.num_pages = bo->num_pages; | 1004 | mem.num_pages = bo->num_pages; |
1000 | mem.size = mem.num_pages << PAGE_SHIFT; | 1005 | mem.size = mem.num_pages << PAGE_SHIFT; |
1001 | mem.page_alignment = bo->mem.page_alignment; | 1006 | mem.page_alignment = bo->mem.page_alignment; |
1007 | mem.bus.io_reserved = false; | ||
1002 | /* | 1008 | /* |
1003 | * Determine where to move the buffer. | 1009 | * Determine where to move the buffer. |
1004 | */ | 1010 | */ |
1005 | ret = ttm_bo_mem_space(bo, placement, &mem, interruptible, no_wait); | 1011 | ret = ttm_bo_mem_space(bo, placement, &mem, interruptible, no_wait_reserve, no_wait_gpu); |
1006 | if (ret) | 1012 | if (ret) |
1007 | goto out_unlock; | 1013 | goto out_unlock; |
1008 | ret = ttm_bo_handle_move_mem(bo, &mem, false, interruptible, no_wait); | 1014 | ret = ttm_bo_handle_move_mem(bo, &mem, false, interruptible, no_wait_reserve, no_wait_gpu); |
1009 | out_unlock: | 1015 | out_unlock: |
1010 | if (ret && mem.mm_node) { | 1016 | if (ret && mem.mm_node) { |
1011 | spin_lock(&glob->lru_lock); | 1017 | spin_lock(&glob->lru_lock); |
@@ -1039,7 +1045,8 @@ static int ttm_bo_mem_compat(struct ttm_placement *placement, | |||
1039 | 1045 | ||
1040 | int ttm_bo_validate(struct ttm_buffer_object *bo, | 1046 | int ttm_bo_validate(struct ttm_buffer_object *bo, |
1041 | struct ttm_placement *placement, | 1047 | struct ttm_placement *placement, |
1042 | bool interruptible, bool no_wait) | 1048 | bool interruptible, bool no_wait_reserve, |
1049 | bool no_wait_gpu) | ||
1043 | { | 1050 | { |
1044 | int ret; | 1051 | int ret; |
1045 | 1052 | ||
@@ -1054,7 +1061,7 @@ int ttm_bo_validate(struct ttm_buffer_object *bo, | |||
1054 | */ | 1061 | */ |
1055 | ret = ttm_bo_mem_compat(placement, &bo->mem); | 1062 | ret = ttm_bo_mem_compat(placement, &bo->mem); |
1056 | if (ret < 0) { | 1063 | if (ret < 0) { |
1057 | ret = ttm_bo_move_buffer(bo, placement, interruptible, no_wait); | 1064 | ret = ttm_bo_move_buffer(bo, placement, interruptible, no_wait_reserve, no_wait_gpu); |
1058 | if (ret) | 1065 | if (ret) |
1059 | return ret; | 1066 | return ret; |
1060 | } else { | 1067 | } else { |
@@ -1153,6 +1160,7 @@ int ttm_bo_init(struct ttm_bo_device *bdev, | |||
1153 | bo->mem.num_pages = bo->num_pages; | 1160 | bo->mem.num_pages = bo->num_pages; |
1154 | bo->mem.mm_node = NULL; | 1161 | bo->mem.mm_node = NULL; |
1155 | bo->mem.page_alignment = page_alignment; | 1162 | bo->mem.page_alignment = page_alignment; |
1163 | bo->mem.bus.io_reserved = false; | ||
1156 | bo->buffer_start = buffer_start & PAGE_MASK; | 1164 | bo->buffer_start = buffer_start & PAGE_MASK; |
1157 | bo->priv_flags = 0; | 1165 | bo->priv_flags = 0; |
1158 | bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED); | 1166 | bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED); |
@@ -1175,7 +1183,7 @@ int ttm_bo_init(struct ttm_bo_device *bdev, | |||
1175 | goto out_err; | 1183 | goto out_err; |
1176 | } | 1184 | } |
1177 | 1185 | ||
1178 | ret = ttm_bo_validate(bo, placement, interruptible, false); | 1186 | ret = ttm_bo_validate(bo, placement, interruptible, false, false); |
1179 | if (ret) | 1187 | if (ret) |
1180 | goto out_err; | 1188 | goto out_err; |
1181 | 1189 | ||
@@ -1249,7 +1257,7 @@ static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev, | |||
1249 | spin_lock(&glob->lru_lock); | 1257 | spin_lock(&glob->lru_lock); |
1250 | while (!list_empty(&man->lru)) { | 1258 | while (!list_empty(&man->lru)) { |
1251 | spin_unlock(&glob->lru_lock); | 1259 | spin_unlock(&glob->lru_lock); |
1252 | ret = ttm_mem_evict_first(bdev, mem_type, false, false); | 1260 | ret = ttm_mem_evict_first(bdev, mem_type, false, false, false); |
1253 | if (ret) { | 1261 | if (ret) { |
1254 | if (allow_errors) { | 1262 | if (allow_errors) { |
1255 | return ret; | 1263 | return ret; |
@@ -1553,26 +1561,6 @@ bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) | |||
1553 | return true; | 1561 | return true; |
1554 | } | 1562 | } |
1555 | 1563 | ||
1556 | int ttm_bo_pci_offset(struct ttm_bo_device *bdev, | ||
1557 | struct ttm_mem_reg *mem, | ||
1558 | unsigned long *bus_base, | ||
1559 | unsigned long *bus_offset, unsigned long *bus_size) | ||
1560 | { | ||
1561 | struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; | ||
1562 | |||
1563 | *bus_size = 0; | ||
1564 | if (!(man->flags & TTM_MEMTYPE_FLAG_MAPPABLE)) | ||
1565 | return -EINVAL; | ||
1566 | |||
1567 | if (ttm_mem_reg_is_pci(bdev, mem)) { | ||
1568 | *bus_offset = mem->mm_node->start << PAGE_SHIFT; | ||
1569 | *bus_size = mem->num_pages << PAGE_SHIFT; | ||
1570 | *bus_base = man->io_offset; | ||
1571 | } | ||
1572 | |||
1573 | return 0; | ||
1574 | } | ||
1575 | |||
1576 | void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo) | 1564 | void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo) |
1577 | { | 1565 | { |
1578 | struct ttm_bo_device *bdev = bo->bdev; | 1566 | struct ttm_bo_device *bdev = bo->bdev; |
@@ -1581,8 +1569,8 @@ void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo) | |||
1581 | 1569 | ||
1582 | if (!bdev->dev_mapping) | 1570 | if (!bdev->dev_mapping) |
1583 | return; | 1571 | return; |
1584 | |||
1585 | unmap_mapping_range(bdev->dev_mapping, offset, holelen, 1); | 1572 | unmap_mapping_range(bdev->dev_mapping, offset, holelen, 1); |
1573 | ttm_mem_io_free(bdev, &bo->mem); | ||
1586 | } | 1574 | } |
1587 | EXPORT_SYMBOL(ttm_bo_unmap_virtual); | 1575 | EXPORT_SYMBOL(ttm_bo_unmap_virtual); |
1588 | 1576 | ||
@@ -1839,7 +1827,7 @@ static int ttm_bo_swapout(struct ttm_mem_shrink *shrink) | |||
1839 | evict_mem.mem_type = TTM_PL_SYSTEM; | 1827 | evict_mem.mem_type = TTM_PL_SYSTEM; |
1840 | 1828 | ||
1841 | ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, | 1829 | ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, |
1842 | false, false); | 1830 | false, false, false); |
1843 | if (unlikely(ret != 0)) | 1831 | if (unlikely(ret != 0)) |
1844 | goto out; | 1832 | goto out; |
1845 | } | 1833 | } |
diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c index d764e82e799b..a37a94872a14 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_util.c +++ b/drivers/gpu/drm/ttm/ttm_bo_util.c | |||
@@ -50,7 +50,8 @@ void ttm_bo_free_old_node(struct ttm_buffer_object *bo) | |||
50 | } | 50 | } |
51 | 51 | ||
52 | int ttm_bo_move_ttm(struct ttm_buffer_object *bo, | 52 | int ttm_bo_move_ttm(struct ttm_buffer_object *bo, |
53 | bool evict, bool no_wait, struct ttm_mem_reg *new_mem) | 53 | bool evict, bool no_wait_reserve, |
54 | bool no_wait_gpu, struct ttm_mem_reg *new_mem) | ||
54 | { | 55 | { |
55 | struct ttm_tt *ttm = bo->ttm; | 56 | struct ttm_tt *ttm = bo->ttm; |
56 | struct ttm_mem_reg *old_mem = &bo->mem; | 57 | struct ttm_mem_reg *old_mem = &bo->mem; |
@@ -81,30 +82,51 @@ int ttm_bo_move_ttm(struct ttm_buffer_object *bo, | |||
81 | } | 82 | } |
82 | EXPORT_SYMBOL(ttm_bo_move_ttm); | 83 | EXPORT_SYMBOL(ttm_bo_move_ttm); |
83 | 84 | ||
85 | int ttm_mem_io_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) | ||
86 | { | ||
87 | int ret; | ||
88 | |||
89 | if (!mem->bus.io_reserved) { | ||
90 | mem->bus.io_reserved = true; | ||
91 | ret = bdev->driver->io_mem_reserve(bdev, mem); | ||
92 | if (unlikely(ret != 0)) | ||
93 | return ret; | ||
94 | } | ||
95 | return 0; | ||
96 | } | ||
97 | |||
98 | void ttm_mem_io_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) | ||
99 | { | ||
100 | if (bdev->driver->io_mem_reserve) { | ||
101 | if (mem->bus.io_reserved) { | ||
102 | mem->bus.io_reserved = false; | ||
103 | bdev->driver->io_mem_free(bdev, mem); | ||
104 | } | ||
105 | } | ||
106 | } | ||
107 | |||
84 | int ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem, | 108 | int ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem, |
85 | void **virtual) | 109 | void **virtual) |
86 | { | 110 | { |
87 | struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; | ||
88 | unsigned long bus_offset; | ||
89 | unsigned long bus_size; | ||
90 | unsigned long bus_base; | ||
91 | int ret; | 111 | int ret; |
92 | void *addr; | 112 | void *addr; |
93 | 113 | ||
94 | *virtual = NULL; | 114 | *virtual = NULL; |
95 | ret = ttm_bo_pci_offset(bdev, mem, &bus_base, &bus_offset, &bus_size); | 115 | ret = ttm_mem_io_reserve(bdev, mem); |
96 | if (ret || bus_size == 0) | 116 | if (ret) |
97 | return ret; | 117 | return ret; |
98 | 118 | ||
99 | if (!(man->flags & TTM_MEMTYPE_FLAG_NEEDS_IOREMAP)) | 119 | if (mem->bus.addr) { |
100 | addr = (void *)(((u8 *) man->io_addr) + bus_offset); | 120 | addr = mem->bus.addr; |
101 | else { | 121 | } else { |
102 | if (mem->placement & TTM_PL_FLAG_WC) | 122 | if (mem->placement & TTM_PL_FLAG_WC) |
103 | addr = ioremap_wc(bus_base + bus_offset, bus_size); | 123 | addr = ioremap_wc(mem->bus.base + mem->bus.offset, mem->bus.size); |
104 | else | 124 | else |
105 | addr = ioremap_nocache(bus_base + bus_offset, bus_size); | 125 | addr = ioremap_nocache(mem->bus.base + mem->bus.offset, mem->bus.size); |
106 | if (!addr) | 126 | if (!addr) { |
127 | ttm_mem_io_free(bdev, mem); | ||
107 | return -ENOMEM; | 128 | return -ENOMEM; |
129 | } | ||
108 | } | 130 | } |
109 | *virtual = addr; | 131 | *virtual = addr; |
110 | return 0; | 132 | return 0; |
@@ -117,8 +139,9 @@ void ttm_mem_reg_iounmap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem, | |||
117 | 139 | ||
118 | man = &bdev->man[mem->mem_type]; | 140 | man = &bdev->man[mem->mem_type]; |
119 | 141 | ||
120 | if (virtual && (man->flags & TTM_MEMTYPE_FLAG_NEEDS_IOREMAP)) | 142 | if (virtual && mem->bus.addr == NULL) |
121 | iounmap(virtual); | 143 | iounmap(virtual); |
144 | ttm_mem_io_free(bdev, mem); | ||
122 | } | 145 | } |
123 | 146 | ||
124 | static int ttm_copy_io_page(void *dst, void *src, unsigned long page) | 147 | static int ttm_copy_io_page(void *dst, void *src, unsigned long page) |
@@ -208,7 +231,8 @@ static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst, | |||
208 | } | 231 | } |
209 | 232 | ||
210 | int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, | 233 | int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, |
211 | bool evict, bool no_wait, struct ttm_mem_reg *new_mem) | 234 | bool evict, bool no_wait_reserve, bool no_wait_gpu, |
235 | struct ttm_mem_reg *new_mem) | ||
212 | { | 236 | { |
213 | struct ttm_bo_device *bdev = bo->bdev; | 237 | struct ttm_bo_device *bdev = bo->bdev; |
214 | struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type]; | 238 | struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type]; |
@@ -369,26 +393,23 @@ pgprot_t ttm_io_prot(uint32_t caching_flags, pgprot_t tmp) | |||
369 | EXPORT_SYMBOL(ttm_io_prot); | 393 | EXPORT_SYMBOL(ttm_io_prot); |
370 | 394 | ||
371 | static int ttm_bo_ioremap(struct ttm_buffer_object *bo, | 395 | static int ttm_bo_ioremap(struct ttm_buffer_object *bo, |
372 | unsigned long bus_base, | 396 | unsigned long offset, |
373 | unsigned long bus_offset, | 397 | unsigned long size, |
374 | unsigned long bus_size, | ||
375 | struct ttm_bo_kmap_obj *map) | 398 | struct ttm_bo_kmap_obj *map) |
376 | { | 399 | { |
377 | struct ttm_bo_device *bdev = bo->bdev; | ||
378 | struct ttm_mem_reg *mem = &bo->mem; | 400 | struct ttm_mem_reg *mem = &bo->mem; |
379 | struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; | ||
380 | 401 | ||
381 | if (!(man->flags & TTM_MEMTYPE_FLAG_NEEDS_IOREMAP)) { | 402 | if (bo->mem.bus.addr) { |
382 | map->bo_kmap_type = ttm_bo_map_premapped; | 403 | map->bo_kmap_type = ttm_bo_map_premapped; |
383 | map->virtual = (void *)(((u8 *) man->io_addr) + bus_offset); | 404 | map->virtual = (void *)(((u8 *)bo->mem.bus.addr) + offset); |
384 | } else { | 405 | } else { |
385 | map->bo_kmap_type = ttm_bo_map_iomap; | 406 | map->bo_kmap_type = ttm_bo_map_iomap; |
386 | if (mem->placement & TTM_PL_FLAG_WC) | 407 | if (mem->placement & TTM_PL_FLAG_WC) |
387 | map->virtual = ioremap_wc(bus_base + bus_offset, | 408 | map->virtual = ioremap_wc(bo->mem.bus.base + bo->mem.bus.offset + offset, |
388 | bus_size); | 409 | size); |
389 | else | 410 | else |
390 | map->virtual = ioremap_nocache(bus_base + bus_offset, | 411 | map->virtual = ioremap_nocache(bo->mem.bus.base + bo->mem.bus.offset + offset, |
391 | bus_size); | 412 | size); |
392 | } | 413 | } |
393 | return (!map->virtual) ? -ENOMEM : 0; | 414 | return (!map->virtual) ? -ENOMEM : 0; |
394 | } | 415 | } |
@@ -441,13 +462,12 @@ int ttm_bo_kmap(struct ttm_buffer_object *bo, | |||
441 | unsigned long start_page, unsigned long num_pages, | 462 | unsigned long start_page, unsigned long num_pages, |
442 | struct ttm_bo_kmap_obj *map) | 463 | struct ttm_bo_kmap_obj *map) |
443 | { | 464 | { |
465 | unsigned long offset, size; | ||
444 | int ret; | 466 | int ret; |
445 | unsigned long bus_base; | ||
446 | unsigned long bus_offset; | ||
447 | unsigned long bus_size; | ||
448 | 467 | ||
449 | BUG_ON(!list_empty(&bo->swap)); | 468 | BUG_ON(!list_empty(&bo->swap)); |
450 | map->virtual = NULL; | 469 | map->virtual = NULL; |
470 | map->bo = bo; | ||
451 | if (num_pages > bo->num_pages) | 471 | if (num_pages > bo->num_pages) |
452 | return -EINVAL; | 472 | return -EINVAL; |
453 | if (start_page > bo->num_pages) | 473 | if (start_page > bo->num_pages) |
@@ -456,16 +476,15 @@ int ttm_bo_kmap(struct ttm_buffer_object *bo, | |||
456 | if (num_pages > 1 && !DRM_SUSER(DRM_CURPROC)) | 476 | if (num_pages > 1 && !DRM_SUSER(DRM_CURPROC)) |
457 | return -EPERM; | 477 | return -EPERM; |
458 | #endif | 478 | #endif |
459 | ret = ttm_bo_pci_offset(bo->bdev, &bo->mem, &bus_base, | 479 | ret = ttm_mem_io_reserve(bo->bdev, &bo->mem); |
460 | &bus_offset, &bus_size); | ||
461 | if (ret) | 480 | if (ret) |
462 | return ret; | 481 | return ret; |
463 | if (bus_size == 0) { | 482 | if (!bo->mem.bus.is_iomem) { |
464 | return ttm_bo_kmap_ttm(bo, start_page, num_pages, map); | 483 | return ttm_bo_kmap_ttm(bo, start_page, num_pages, map); |
465 | } else { | 484 | } else { |
466 | bus_offset += start_page << PAGE_SHIFT; | 485 | offset = start_page << PAGE_SHIFT; |
467 | bus_size = num_pages << PAGE_SHIFT; | 486 | size = num_pages << PAGE_SHIFT; |
468 | return ttm_bo_ioremap(bo, bus_base, bus_offset, bus_size, map); | 487 | return ttm_bo_ioremap(bo, offset, size, map); |
469 | } | 488 | } |
470 | } | 489 | } |
471 | EXPORT_SYMBOL(ttm_bo_kmap); | 490 | EXPORT_SYMBOL(ttm_bo_kmap); |
@@ -477,6 +496,7 @@ void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map) | |||
477 | switch (map->bo_kmap_type) { | 496 | switch (map->bo_kmap_type) { |
478 | case ttm_bo_map_iomap: | 497 | case ttm_bo_map_iomap: |
479 | iounmap(map->virtual); | 498 | iounmap(map->virtual); |
499 | ttm_mem_io_free(map->bo->bdev, &map->bo->mem); | ||
480 | break; | 500 | break; |
481 | case ttm_bo_map_vmap: | 501 | case ttm_bo_map_vmap: |
482 | vunmap(map->virtual); | 502 | vunmap(map->virtual); |
@@ -494,39 +514,11 @@ void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map) | |||
494 | } | 514 | } |
495 | EXPORT_SYMBOL(ttm_bo_kunmap); | 515 | EXPORT_SYMBOL(ttm_bo_kunmap); |
496 | 516 | ||
497 | int ttm_bo_pfn_prot(struct ttm_buffer_object *bo, | ||
498 | unsigned long dst_offset, | ||
499 | unsigned long *pfn, pgprot_t *prot) | ||
500 | { | ||
501 | struct ttm_mem_reg *mem = &bo->mem; | ||
502 | struct ttm_bo_device *bdev = bo->bdev; | ||
503 | unsigned long bus_offset; | ||
504 | unsigned long bus_size; | ||
505 | unsigned long bus_base; | ||
506 | int ret; | ||
507 | ret = ttm_bo_pci_offset(bdev, mem, &bus_base, &bus_offset, | ||
508 | &bus_size); | ||
509 | if (ret) | ||
510 | return -EINVAL; | ||
511 | if (bus_size != 0) | ||
512 | *pfn = (bus_base + bus_offset + dst_offset) >> PAGE_SHIFT; | ||
513 | else | ||
514 | if (!bo->ttm) | ||
515 | return -EINVAL; | ||
516 | else | ||
517 | *pfn = page_to_pfn(ttm_tt_get_page(bo->ttm, | ||
518 | dst_offset >> | ||
519 | PAGE_SHIFT)); | ||
520 | *prot = (mem->placement & TTM_PL_FLAG_CACHED) ? | ||
521 | PAGE_KERNEL : ttm_io_prot(mem->placement, PAGE_KERNEL); | ||
522 | |||
523 | return 0; | ||
524 | } | ||
525 | |||
526 | int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo, | 517 | int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo, |
527 | void *sync_obj, | 518 | void *sync_obj, |
528 | void *sync_obj_arg, | 519 | void *sync_obj_arg, |
529 | bool evict, bool no_wait, | 520 | bool evict, bool no_wait_reserve, |
521 | bool no_wait_gpu, | ||
530 | struct ttm_mem_reg *new_mem) | 522 | struct ttm_mem_reg *new_mem) |
531 | { | 523 | { |
532 | struct ttm_bo_device *bdev = bo->bdev; | 524 | struct ttm_bo_device *bdev = bo->bdev; |
diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c index 668dbe8b8dd3..fe6cb77899f4 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_vm.c +++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c | |||
@@ -74,9 +74,6 @@ static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) | |||
74 | struct ttm_buffer_object *bo = (struct ttm_buffer_object *) | 74 | struct ttm_buffer_object *bo = (struct ttm_buffer_object *) |
75 | vma->vm_private_data; | 75 | vma->vm_private_data; |
76 | struct ttm_bo_device *bdev = bo->bdev; | 76 | struct ttm_bo_device *bdev = bo->bdev; |
77 | unsigned long bus_base; | ||
78 | unsigned long bus_offset; | ||
79 | unsigned long bus_size; | ||
80 | unsigned long page_offset; | 77 | unsigned long page_offset; |
81 | unsigned long page_last; | 78 | unsigned long page_last; |
82 | unsigned long pfn; | 79 | unsigned long pfn; |
@@ -84,7 +81,6 @@ static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) | |||
84 | struct page *page; | 81 | struct page *page; |
85 | int ret; | 82 | int ret; |
86 | int i; | 83 | int i; |
87 | bool is_iomem; | ||
88 | unsigned long address = (unsigned long)vmf->virtual_address; | 84 | unsigned long address = (unsigned long)vmf->virtual_address; |
89 | int retval = VM_FAULT_NOPAGE; | 85 | int retval = VM_FAULT_NOPAGE; |
90 | 86 | ||
@@ -101,8 +97,21 @@ static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) | |||
101 | return VM_FAULT_NOPAGE; | 97 | return VM_FAULT_NOPAGE; |
102 | } | 98 | } |
103 | 99 | ||
104 | if (bdev->driver->fault_reserve_notify) | 100 | if (bdev->driver->fault_reserve_notify) { |
105 | bdev->driver->fault_reserve_notify(bo); | 101 | ret = bdev->driver->fault_reserve_notify(bo); |
102 | switch (ret) { | ||
103 | case 0: | ||
104 | break; | ||
105 | case -EBUSY: | ||
106 | set_need_resched(); | ||
107 | case -ERESTARTSYS: | ||
108 | retval = VM_FAULT_NOPAGE; | ||
109 | goto out_unlock; | ||
110 | default: | ||
111 | retval = VM_FAULT_SIGBUS; | ||
112 | goto out_unlock; | ||
113 | } | ||
114 | } | ||
106 | 115 | ||
107 | /* | 116 | /* |
108 | * Wait for buffer data in transit, due to a pipelined | 117 | * Wait for buffer data in transit, due to a pipelined |
@@ -122,15 +131,12 @@ static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) | |||
122 | spin_unlock(&bo->lock); | 131 | spin_unlock(&bo->lock); |
123 | 132 | ||
124 | 133 | ||
125 | ret = ttm_bo_pci_offset(bdev, &bo->mem, &bus_base, &bus_offset, | 134 | ret = ttm_mem_io_reserve(bdev, &bo->mem); |
126 | &bus_size); | 135 | if (ret) { |
127 | if (unlikely(ret != 0)) { | ||
128 | retval = VM_FAULT_SIGBUS; | 136 | retval = VM_FAULT_SIGBUS; |
129 | goto out_unlock; | 137 | goto out_unlock; |
130 | } | 138 | } |
131 | 139 | ||
132 | is_iomem = (bus_size != 0); | ||
133 | |||
134 | page_offset = ((address - vma->vm_start) >> PAGE_SHIFT) + | 140 | page_offset = ((address - vma->vm_start) >> PAGE_SHIFT) + |
135 | bo->vm_node->start - vma->vm_pgoff; | 141 | bo->vm_node->start - vma->vm_pgoff; |
136 | page_last = ((vma->vm_end - vma->vm_start) >> PAGE_SHIFT) + | 142 | page_last = ((vma->vm_end - vma->vm_start) >> PAGE_SHIFT) + |
@@ -154,8 +160,7 @@ static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) | |||
154 | * vma->vm_page_prot when the object changes caching policy, with | 160 | * vma->vm_page_prot when the object changes caching policy, with |
155 | * the correct locks held. | 161 | * the correct locks held. |
156 | */ | 162 | */ |
157 | 163 | if (bo->mem.bus.is_iomem) { | |
158 | if (is_iomem) { | ||
159 | vma->vm_page_prot = ttm_io_prot(bo->mem.placement, | 164 | vma->vm_page_prot = ttm_io_prot(bo->mem.placement, |
160 | vma->vm_page_prot); | 165 | vma->vm_page_prot); |
161 | } else { | 166 | } else { |
@@ -171,10 +176,8 @@ static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) | |||
171 | */ | 176 | */ |
172 | 177 | ||
173 | for (i = 0; i < TTM_BO_VM_NUM_PREFAULT; ++i) { | 178 | for (i = 0; i < TTM_BO_VM_NUM_PREFAULT; ++i) { |
174 | 179 | if (bo->mem.bus.is_iomem) | |
175 | if (is_iomem) | 180 | pfn = ((bo->mem.bus.base + bo->mem.bus.offset) >> PAGE_SHIFT) + page_offset; |
176 | pfn = ((bus_base + bus_offset) >> PAGE_SHIFT) + | ||
177 | page_offset; | ||
178 | else { | 181 | else { |
179 | page = ttm_tt_get_page(ttm, page_offset); | 182 | page = ttm_tt_get_page(ttm, page_offset); |
180 | if (unlikely(!page && i == 0)) { | 183 | if (unlikely(!page && i == 0)) { |
@@ -198,7 +201,6 @@ static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) | |||
198 | retval = | 201 | retval = |
199 | (ret == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS; | 202 | (ret == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS; |
200 | goto out_unlock; | 203 | goto out_unlock; |
201 | |||
202 | } | 204 | } |
203 | 205 | ||
204 | address += PAGE_SIZE; | 206 | address += PAGE_SIZE; |
@@ -221,8 +223,7 @@ static void ttm_bo_vm_open(struct vm_area_struct *vma) | |||
221 | 223 | ||
222 | static void ttm_bo_vm_close(struct vm_area_struct *vma) | 224 | static void ttm_bo_vm_close(struct vm_area_struct *vma) |
223 | { | 225 | { |
224 | struct ttm_buffer_object *bo = | 226 | struct ttm_buffer_object *bo = (struct ttm_buffer_object *)vma->vm_private_data; |
225 | (struct ttm_buffer_object *)vma->vm_private_data; | ||
226 | 227 | ||
227 | ttm_bo_unref(&bo); | 228 | ttm_bo_unref(&bo); |
228 | vma->vm_private_data = NULL; | 229 | vma->vm_private_data = NULL; |