diff options
Diffstat (limited to 'drivers/gpu/drm/nouveau/nouveau_bo.c')
-rw-r--r-- | drivers/gpu/drm/nouveau/nouveau_bo.c | 243 |
1 files changed, 166 insertions, 77 deletions
diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c index 0cad6d834eb2..e342a418d434 100644 --- a/drivers/gpu/drm/nouveau/nouveau_bo.c +++ b/drivers/gpu/drm/nouveau/nouveau_bo.c | |||
@@ -33,10 +33,13 @@ | |||
33 | #include "nouveau_drv.h" | 33 | #include "nouveau_drv.h" |
34 | #include "nouveau_dma.h" | 34 | #include "nouveau_dma.h" |
35 | 35 | ||
36 | #include <linux/log2.h> | ||
37 | |||
36 | static void | 38 | static void |
37 | nouveau_bo_del_ttm(struct ttm_buffer_object *bo) | 39 | nouveau_bo_del_ttm(struct ttm_buffer_object *bo) |
38 | { | 40 | { |
39 | struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev); | 41 | struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev); |
42 | struct drm_device *dev = dev_priv->dev; | ||
40 | struct nouveau_bo *nvbo = nouveau_bo(bo); | 43 | struct nouveau_bo *nvbo = nouveau_bo(bo); |
41 | 44 | ||
42 | ttm_bo_kunmap(&nvbo->kmap); | 45 | ttm_bo_kunmap(&nvbo->kmap); |
@@ -44,12 +47,87 @@ nouveau_bo_del_ttm(struct ttm_buffer_object *bo) | |||
44 | if (unlikely(nvbo->gem)) | 47 | if (unlikely(nvbo->gem)) |
45 | DRM_ERROR("bo %p still attached to GEM object\n", bo); | 48 | DRM_ERROR("bo %p still attached to GEM object\n", bo); |
46 | 49 | ||
50 | if (nvbo->tile) | ||
51 | nv10_mem_expire_tiling(dev, nvbo->tile, NULL); | ||
52 | |||
47 | spin_lock(&dev_priv->ttm.bo_list_lock); | 53 | spin_lock(&dev_priv->ttm.bo_list_lock); |
48 | list_del(&nvbo->head); | 54 | list_del(&nvbo->head); |
49 | spin_unlock(&dev_priv->ttm.bo_list_lock); | 55 | spin_unlock(&dev_priv->ttm.bo_list_lock); |
50 | kfree(nvbo); | 56 | kfree(nvbo); |
51 | } | 57 | } |
52 | 58 | ||
59 | static void | ||
60 | nouveau_bo_fixup_align(struct drm_device *dev, | ||
61 | uint32_t tile_mode, uint32_t tile_flags, | ||
62 | int *align, int *size) | ||
63 | { | ||
64 | struct drm_nouveau_private *dev_priv = dev->dev_private; | ||
65 | |||
66 | /* | ||
67 | * Some of the tile_flags have a periodic structure of N*4096 bytes, | ||
68 | * align to to that as well as the page size. Overallocate memory to | ||
69 | * avoid corruption of other buffer objects. | ||
70 | */ | ||
71 | if (dev_priv->card_type == NV_50) { | ||
72 | uint32_t block_size = nouveau_mem_fb_amount(dev) >> 15; | ||
73 | int i; | ||
74 | |||
75 | switch (tile_flags) { | ||
76 | case 0x1800: | ||
77 | case 0x2800: | ||
78 | case 0x4800: | ||
79 | case 0x7a00: | ||
80 | *size = roundup(*size, block_size); | ||
81 | if (is_power_of_2(block_size)) { | ||
82 | *size += 3 * block_size; | ||
83 | for (i = 1; i < 10; i++) { | ||
84 | *align = 12 * i * block_size; | ||
85 | if (!(*align % 65536)) | ||
86 | break; | ||
87 | } | ||
88 | } else { | ||
89 | *size += 6 * block_size; | ||
90 | for (i = 1; i < 10; i++) { | ||
91 | *align = 8 * i * block_size; | ||
92 | if (!(*align % 65536)) | ||
93 | break; | ||
94 | } | ||
95 | } | ||
96 | break; | ||
97 | default: | ||
98 | break; | ||
99 | } | ||
100 | |||
101 | } else { | ||
102 | if (tile_mode) { | ||
103 | if (dev_priv->chipset >= 0x40) { | ||
104 | *align = 65536; | ||
105 | *size = roundup(*size, 64 * tile_mode); | ||
106 | |||
107 | } else if (dev_priv->chipset >= 0x30) { | ||
108 | *align = 32768; | ||
109 | *size = roundup(*size, 64 * tile_mode); | ||
110 | |||
111 | } else if (dev_priv->chipset >= 0x20) { | ||
112 | *align = 16384; | ||
113 | *size = roundup(*size, 64 * tile_mode); | ||
114 | |||
115 | } else if (dev_priv->chipset >= 0x10) { | ||
116 | *align = 16384; | ||
117 | *size = roundup(*size, 32 * tile_mode); | ||
118 | } | ||
119 | } | ||
120 | } | ||
121 | |||
122 | /* ALIGN works only on powers of two. */ | ||
123 | *size = roundup(*size, PAGE_SIZE); | ||
124 | |||
125 | if (dev_priv->card_type == NV_50) { | ||
126 | *size = roundup(*size, 65536); | ||
127 | *align = max(65536, *align); | ||
128 | } | ||
129 | } | ||
130 | |||
53 | int | 131 | int |
54 | nouveau_bo_new(struct drm_device *dev, struct nouveau_channel *chan, | 132 | nouveau_bo_new(struct drm_device *dev, struct nouveau_channel *chan, |
55 | int size, int align, uint32_t flags, uint32_t tile_mode, | 133 | int size, int align, uint32_t flags, uint32_t tile_mode, |
@@ -58,7 +136,7 @@ nouveau_bo_new(struct drm_device *dev, struct nouveau_channel *chan, | |||
58 | { | 136 | { |
59 | struct drm_nouveau_private *dev_priv = dev->dev_private; | 137 | struct drm_nouveau_private *dev_priv = dev->dev_private; |
60 | struct nouveau_bo *nvbo; | 138 | struct nouveau_bo *nvbo; |
61 | int ret, n = 0; | 139 | int ret = 0; |
62 | 140 | ||
63 | nvbo = kzalloc(sizeof(struct nouveau_bo), GFP_KERNEL); | 141 | nvbo = kzalloc(sizeof(struct nouveau_bo), GFP_KERNEL); |
64 | if (!nvbo) | 142 | if (!nvbo) |
@@ -70,59 +148,14 @@ nouveau_bo_new(struct drm_device *dev, struct nouveau_channel *chan, | |||
70 | nvbo->tile_mode = tile_mode; | 148 | nvbo->tile_mode = tile_mode; |
71 | nvbo->tile_flags = tile_flags; | 149 | nvbo->tile_flags = tile_flags; |
72 | 150 | ||
73 | /* | 151 | nouveau_bo_fixup_align(dev, tile_mode, tile_flags, &align, &size); |
74 | * Some of the tile_flags have a periodic structure of N*4096 bytes, | ||
75 | * align to to that as well as the page size. Overallocate memory to | ||
76 | * avoid corruption of other buffer objects. | ||
77 | */ | ||
78 | switch (tile_flags) { | ||
79 | case 0x1800: | ||
80 | case 0x2800: | ||
81 | case 0x4800: | ||
82 | case 0x7a00: | ||
83 | if (dev_priv->chipset >= 0xA0) { | ||
84 | /* This is based on high end cards with 448 bits | ||
85 | * memory bus, could be different elsewhere.*/ | ||
86 | size += 6 * 28672; | ||
87 | /* 8 * 28672 is the actual alignment requirement, | ||
88 | * but we must also align to page size. */ | ||
89 | align = 2 * 8 * 28672; | ||
90 | } else if (dev_priv->chipset >= 0x90) { | ||
91 | size += 3 * 16384; | ||
92 | align = 12 * 16834; | ||
93 | } else { | ||
94 | size += 3 * 8192; | ||
95 | /* 12 * 8192 is the actual alignment requirement, | ||
96 | * but we must also align to page size. */ | ||
97 | align = 2 * 12 * 8192; | ||
98 | } | ||
99 | break; | ||
100 | default: | ||
101 | break; | ||
102 | } | ||
103 | |||
104 | align >>= PAGE_SHIFT; | 152 | align >>= PAGE_SHIFT; |
105 | 153 | ||
106 | size = (size + (PAGE_SIZE - 1)) & ~(PAGE_SIZE - 1); | ||
107 | if (dev_priv->card_type == NV_50) { | ||
108 | size = (size + 65535) & ~65535; | ||
109 | if (align < (65536 / PAGE_SIZE)) | ||
110 | align = (65536 / PAGE_SIZE); | ||
111 | } | ||
112 | |||
113 | if (flags & TTM_PL_FLAG_VRAM) | ||
114 | nvbo->placements[n++] = TTM_PL_FLAG_VRAM | TTM_PL_MASK_CACHING; | ||
115 | if (flags & TTM_PL_FLAG_TT) | ||
116 | nvbo->placements[n++] = TTM_PL_FLAG_TT | TTM_PL_MASK_CACHING; | ||
117 | nvbo->placement.fpfn = 0; | 154 | nvbo->placement.fpfn = 0; |
118 | nvbo->placement.lpfn = mappable ? dev_priv->fb_mappable_pages : 0; | 155 | nvbo->placement.lpfn = mappable ? dev_priv->fb_mappable_pages : 0; |
119 | nvbo->placement.placement = nvbo->placements; | 156 | nouveau_bo_placement_set(nvbo, flags); |
120 | nvbo->placement.busy_placement = nvbo->placements; | ||
121 | nvbo->placement.num_placement = n; | ||
122 | nvbo->placement.num_busy_placement = n; | ||
123 | 157 | ||
124 | nvbo->channel = chan; | 158 | nvbo->channel = chan; |
125 | nouveau_bo_placement_set(nvbo, flags); | ||
126 | ret = ttm_bo_init(&dev_priv->ttm.bdev, &nvbo->bo, size, | 159 | ret = ttm_bo_init(&dev_priv->ttm.bdev, &nvbo->bo, size, |
127 | ttm_bo_type_device, &nvbo->placement, align, 0, | 160 | ttm_bo_type_device, &nvbo->placement, align, 0, |
128 | false, NULL, size, nouveau_bo_del_ttm); | 161 | false, NULL, size, nouveau_bo_del_ttm); |
@@ -421,6 +454,7 @@ nouveau_bo_evict_flags(struct ttm_buffer_object *bo, struct ttm_placement *pl) | |||
421 | /* GPU-assisted copy using NV_MEMORY_TO_MEMORY_FORMAT, can access | 454 | /* GPU-assisted copy using NV_MEMORY_TO_MEMORY_FORMAT, can access |
422 | * TTM_PL_{VRAM,TT} directly. | 455 | * TTM_PL_{VRAM,TT} directly. |
423 | */ | 456 | */ |
457 | |||
424 | static int | 458 | static int |
425 | nouveau_bo_move_accel_cleanup(struct nouveau_channel *chan, | 459 | nouveau_bo_move_accel_cleanup(struct nouveau_channel *chan, |
426 | struct nouveau_bo *nvbo, bool evict, bool no_wait, | 460 | struct nouveau_bo *nvbo, bool evict, bool no_wait, |
@@ -455,11 +489,12 @@ nouveau_bo_mem_ctxdma(struct nouveau_bo *nvbo, struct nouveau_channel *chan, | |||
455 | } | 489 | } |
456 | 490 | ||
457 | static int | 491 | static int |
458 | nouveau_bo_move_m2mf(struct ttm_buffer_object *bo, int evict, int no_wait, | 492 | nouveau_bo_move_m2mf(struct ttm_buffer_object *bo, int evict, bool intr, |
459 | struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem) | 493 | int no_wait, struct ttm_mem_reg *new_mem) |
460 | { | 494 | { |
461 | struct nouveau_bo *nvbo = nouveau_bo(bo); | 495 | struct nouveau_bo *nvbo = nouveau_bo(bo); |
462 | struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev); | 496 | struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev); |
497 | struct ttm_mem_reg *old_mem = &bo->mem; | ||
463 | struct nouveau_channel *chan; | 498 | struct nouveau_channel *chan; |
464 | uint64_t src_offset, dst_offset; | 499 | uint64_t src_offset, dst_offset; |
465 | uint32_t page_count; | 500 | uint32_t page_count; |
@@ -547,7 +582,7 @@ nouveau_bo_move_flipd(struct ttm_buffer_object *bo, bool evict, bool intr, | |||
547 | 582 | ||
548 | placement.fpfn = placement.lpfn = 0; | 583 | placement.fpfn = placement.lpfn = 0; |
549 | placement.num_placement = placement.num_busy_placement = 1; | 584 | placement.num_placement = placement.num_busy_placement = 1; |
550 | placement.placement = &placement_memtype; | 585 | placement.placement = placement.busy_placement = &placement_memtype; |
551 | 586 | ||
552 | tmp_mem = *new_mem; | 587 | tmp_mem = *new_mem; |
553 | tmp_mem.mm_node = NULL; | 588 | tmp_mem.mm_node = NULL; |
@@ -559,7 +594,7 @@ nouveau_bo_move_flipd(struct ttm_buffer_object *bo, bool evict, bool intr, | |||
559 | if (ret) | 594 | if (ret) |
560 | goto out; | 595 | goto out; |
561 | 596 | ||
562 | ret = nouveau_bo_move_m2mf(bo, true, no_wait, &bo->mem, &tmp_mem); | 597 | ret = nouveau_bo_move_m2mf(bo, true, intr, no_wait, &tmp_mem); |
563 | if (ret) | 598 | if (ret) |
564 | goto out; | 599 | goto out; |
565 | 600 | ||
@@ -585,7 +620,7 @@ nouveau_bo_move_flips(struct ttm_buffer_object *bo, bool evict, bool intr, | |||
585 | 620 | ||
586 | placement.fpfn = placement.lpfn = 0; | 621 | placement.fpfn = placement.lpfn = 0; |
587 | placement.num_placement = placement.num_busy_placement = 1; | 622 | placement.num_placement = placement.num_busy_placement = 1; |
588 | placement.placement = &placement_memtype; | 623 | placement.placement = placement.busy_placement = &placement_memtype; |
589 | 624 | ||
590 | tmp_mem = *new_mem; | 625 | tmp_mem = *new_mem; |
591 | tmp_mem.mm_node = NULL; | 626 | tmp_mem.mm_node = NULL; |
@@ -597,7 +632,7 @@ nouveau_bo_move_flips(struct ttm_buffer_object *bo, bool evict, bool intr, | |||
597 | if (ret) | 632 | if (ret) |
598 | goto out; | 633 | goto out; |
599 | 634 | ||
600 | ret = nouveau_bo_move_m2mf(bo, true, no_wait, &bo->mem, new_mem); | 635 | ret = nouveau_bo_move_m2mf(bo, evict, intr, no_wait, new_mem); |
601 | if (ret) | 636 | if (ret) |
602 | goto out; | 637 | goto out; |
603 | 638 | ||
@@ -612,52 +647,106 @@ out: | |||
612 | } | 647 | } |
613 | 648 | ||
614 | static int | 649 | static int |
615 | nouveau_bo_move(struct ttm_buffer_object *bo, bool evict, bool intr, | 650 | nouveau_bo_vm_bind(struct ttm_buffer_object *bo, struct ttm_mem_reg *new_mem, |
616 | bool no_wait, struct ttm_mem_reg *new_mem) | 651 | struct nouveau_tile_reg **new_tile) |
617 | { | 652 | { |
618 | struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev); | 653 | struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev); |
619 | struct nouveau_bo *nvbo = nouveau_bo(bo); | ||
620 | struct drm_device *dev = dev_priv->dev; | 654 | struct drm_device *dev = dev_priv->dev; |
621 | struct ttm_mem_reg *old_mem = &bo->mem; | 655 | struct nouveau_bo *nvbo = nouveau_bo(bo); |
656 | uint64_t offset; | ||
622 | int ret; | 657 | int ret; |
623 | 658 | ||
624 | if (dev_priv->card_type == NV_50 && new_mem->mem_type == TTM_PL_VRAM && | 659 | if (nvbo->no_vm || new_mem->mem_type != TTM_PL_VRAM) { |
625 | !nvbo->no_vm) { | 660 | /* Nothing to do. */ |
626 | uint64_t offset = new_mem->mm_node->start << PAGE_SHIFT; | 661 | *new_tile = NULL; |
662 | return 0; | ||
663 | } | ||
664 | |||
665 | offset = new_mem->mm_node->start << PAGE_SHIFT; | ||
627 | 666 | ||
667 | if (dev_priv->card_type == NV_50) { | ||
628 | ret = nv50_mem_vm_bind_linear(dev, | 668 | ret = nv50_mem_vm_bind_linear(dev, |
629 | offset + dev_priv->vm_vram_base, | 669 | offset + dev_priv->vm_vram_base, |
630 | new_mem->size, nvbo->tile_flags, | 670 | new_mem->size, nvbo->tile_flags, |
631 | offset); | 671 | offset); |
632 | if (ret) | 672 | if (ret) |
633 | return ret; | 673 | return ret; |
674 | |||
675 | } else if (dev_priv->card_type >= NV_10) { | ||
676 | *new_tile = nv10_mem_set_tiling(dev, offset, new_mem->size, | ||
677 | nvbo->tile_mode); | ||
634 | } | 678 | } |
635 | 679 | ||
680 | return 0; | ||
681 | } | ||
682 | |||
683 | static void | ||
684 | nouveau_bo_vm_cleanup(struct ttm_buffer_object *bo, | ||
685 | struct nouveau_tile_reg *new_tile, | ||
686 | struct nouveau_tile_reg **old_tile) | ||
687 | { | ||
688 | struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev); | ||
689 | struct drm_device *dev = dev_priv->dev; | ||
690 | |||
691 | if (dev_priv->card_type >= NV_10 && | ||
692 | dev_priv->card_type < NV_50) { | ||
693 | if (*old_tile) | ||
694 | nv10_mem_expire_tiling(dev, *old_tile, bo->sync_obj); | ||
695 | |||
696 | *old_tile = new_tile; | ||
697 | } | ||
698 | } | ||
699 | |||
700 | static int | ||
701 | nouveau_bo_move(struct ttm_buffer_object *bo, bool evict, bool intr, | ||
702 | bool no_wait, struct ttm_mem_reg *new_mem) | ||
703 | { | ||
704 | struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev); | ||
705 | struct nouveau_bo *nvbo = nouveau_bo(bo); | ||
706 | struct ttm_mem_reg *old_mem = &bo->mem; | ||
707 | struct nouveau_tile_reg *new_tile = NULL; | ||
708 | int ret = 0; | ||
709 | |||
710 | ret = nouveau_bo_vm_bind(bo, new_mem, &new_tile); | ||
711 | if (ret) | ||
712 | return ret; | ||
713 | |||
714 | /* Software copy if the card isn't up and running yet. */ | ||
636 | if (dev_priv->init_state != NOUVEAU_CARD_INIT_DONE || | 715 | if (dev_priv->init_state != NOUVEAU_CARD_INIT_DONE || |
637 | !dev_priv->channel) | 716 | !dev_priv->channel) { |
638 | return ttm_bo_move_memcpy(bo, evict, no_wait, new_mem); | 717 | ret = ttm_bo_move_memcpy(bo, evict, no_wait, new_mem); |
718 | goto out; | ||
719 | } | ||
639 | 720 | ||
721 | /* Fake bo copy. */ | ||
640 | if (old_mem->mem_type == TTM_PL_SYSTEM && !bo->ttm) { | 722 | if (old_mem->mem_type == TTM_PL_SYSTEM && !bo->ttm) { |
641 | BUG_ON(bo->mem.mm_node != NULL); | 723 | BUG_ON(bo->mem.mm_node != NULL); |
642 | bo->mem = *new_mem; | 724 | bo->mem = *new_mem; |
643 | new_mem->mm_node = NULL; | 725 | new_mem->mm_node = NULL; |
644 | return 0; | 726 | goto out; |
645 | } | 727 | } |
646 | 728 | ||
647 | if (new_mem->mem_type == TTM_PL_SYSTEM) { | 729 | /* Hardware assisted copy. */ |
648 | if (old_mem->mem_type == TTM_PL_SYSTEM) | 730 | if (new_mem->mem_type == TTM_PL_SYSTEM) |
649 | return ttm_bo_move_memcpy(bo, evict, no_wait, new_mem); | 731 | ret = nouveau_bo_move_flipd(bo, evict, intr, no_wait, new_mem); |
650 | if (nouveau_bo_move_flipd(bo, evict, intr, no_wait, new_mem)) | 732 | else if (old_mem->mem_type == TTM_PL_SYSTEM) |
651 | return ttm_bo_move_memcpy(bo, evict, no_wait, new_mem); | 733 | ret = nouveau_bo_move_flips(bo, evict, intr, no_wait, new_mem); |
652 | } else if (old_mem->mem_type == TTM_PL_SYSTEM) { | 734 | else |
653 | if (nouveau_bo_move_flips(bo, evict, intr, no_wait, new_mem)) | 735 | ret = nouveau_bo_move_m2mf(bo, evict, intr, no_wait, new_mem); |
654 | return ttm_bo_move_memcpy(bo, evict, no_wait, new_mem); | ||
655 | } else { | ||
656 | if (nouveau_bo_move_m2mf(bo, evict, no_wait, old_mem, new_mem)) | ||
657 | return ttm_bo_move_memcpy(bo, evict, no_wait, new_mem); | ||
658 | } | ||
659 | 736 | ||
660 | return 0; | 737 | if (!ret) |
738 | goto out; | ||
739 | |||
740 | /* Fallback to software copy. */ | ||
741 | ret = ttm_bo_move_memcpy(bo, evict, no_wait, new_mem); | ||
742 | |||
743 | out: | ||
744 | if (ret) | ||
745 | nouveau_bo_vm_cleanup(bo, NULL, &new_tile); | ||
746 | else | ||
747 | nouveau_bo_vm_cleanup(bo, new_tile, &nvbo->tile); | ||
748 | |||
749 | return ret; | ||
661 | } | 750 | } |
662 | 751 | ||
663 | static int | 752 | static int |