diff options
-rw-r--r-- | drivers/gpu/drm/radeon/ni.c | 30 | ||||
-rw-r--r-- | drivers/gpu/drm/radeon/radeon.h | 7 | ||||
-rw-r--r-- | drivers/gpu/drm/radeon/radeon_asic.h | 8 | ||||
-rw-r--r-- | drivers/gpu/drm/radeon/radeon_cs.c | 19 | ||||
-rw-r--r-- | drivers/gpu/drm/radeon/radeon_gart.c | 60 | ||||
-rw-r--r-- | drivers/gpu/drm/radeon/radeon_ring.c | 19 | ||||
-rw-r--r-- | drivers/gpu/drm/radeon/si.c | 52 |
7 files changed, 103 insertions, 92 deletions
diff --git a/drivers/gpu/drm/radeon/ni.c b/drivers/gpu/drm/radeon/ni.c index 170bd03d4dd8..7cead763be9e 100644 --- a/drivers/gpu/drm/radeon/ni.c +++ b/drivers/gpu/drm/radeon/ni.c | |||
@@ -1946,19 +1946,21 @@ uint32_t cayman_vm_page_flags(struct radeon_device *rdev, uint32_t flags) | |||
1946 | * cayman_vm_set_page - update the page tables using the CP | 1946 | * cayman_vm_set_page - update the page tables using the CP |
1947 | * | 1947 | * |
1948 | * @rdev: radeon_device pointer | 1948 | * @rdev: radeon_device pointer |
1949 | * @ib: indirect buffer to fill with commands | ||
1949 | * @pe: addr of the page entry | 1950 | * @pe: addr of the page entry |
1950 | * @addr: dst addr to write into pe | 1951 | * @addr: dst addr to write into pe |
1951 | * @count: number of page entries to update | 1952 | * @count: number of page entries to update |
1952 | * @incr: increase next addr by incr bytes | 1953 | * @incr: increase next addr by incr bytes |
1953 | * @flags: access flags | 1954 | * @flags: access flags |
1954 | * | 1955 | * |
1955 | * Update the page tables using the CP (cayman-si). | 1956 | * Update the page tables using the CP (cayman/TN). |
1956 | */ | 1957 | */ |
1957 | void cayman_vm_set_page(struct radeon_device *rdev, uint64_t pe, | 1958 | void cayman_vm_set_page(struct radeon_device *rdev, |
1959 | struct radeon_ib *ib, | ||
1960 | uint64_t pe, | ||
1958 | uint64_t addr, unsigned count, | 1961 | uint64_t addr, unsigned count, |
1959 | uint32_t incr, uint32_t flags) | 1962 | uint32_t incr, uint32_t flags) |
1960 | { | 1963 | { |
1961 | struct radeon_ring *ring = &rdev->ring[rdev->asic->vm.pt_ring_index]; | ||
1962 | uint32_t r600_flags = cayman_vm_page_flags(rdev, flags); | 1964 | uint32_t r600_flags = cayman_vm_page_flags(rdev, flags); |
1963 | uint64_t value; | 1965 | uint64_t value; |
1964 | unsigned ndw; | 1966 | unsigned ndw; |
@@ -1969,9 +1971,9 @@ void cayman_vm_set_page(struct radeon_device *rdev, uint64_t pe, | |||
1969 | if (ndw > 0x3FFF) | 1971 | if (ndw > 0x3FFF) |
1970 | ndw = 0x3FFF; | 1972 | ndw = 0x3FFF; |
1971 | 1973 | ||
1972 | radeon_ring_write(ring, PACKET3(PACKET3_ME_WRITE, ndw)); | 1974 | ib->ptr[ib->length_dw++] = PACKET3(PACKET3_ME_WRITE, ndw); |
1973 | radeon_ring_write(ring, pe); | 1975 | ib->ptr[ib->length_dw++] = pe; |
1974 | radeon_ring_write(ring, upper_32_bits(pe) & 0xff); | 1976 | ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff; |
1975 | for (; ndw > 1; ndw -= 2, --count, pe += 8) { | 1977 | for (; ndw > 1; ndw -= 2, --count, pe += 8) { |
1976 | if (flags & RADEON_VM_PAGE_SYSTEM) { | 1978 | if (flags & RADEON_VM_PAGE_SYSTEM) { |
1977 | value = radeon_vm_map_gart(rdev, addr); | 1979 | value = radeon_vm_map_gart(rdev, addr); |
@@ -1983,8 +1985,8 @@ void cayman_vm_set_page(struct radeon_device *rdev, uint64_t pe, | |||
1983 | } | 1985 | } |
1984 | addr += incr; | 1986 | addr += incr; |
1985 | value |= r600_flags; | 1987 | value |= r600_flags; |
1986 | radeon_ring_write(ring, value); | 1988 | ib->ptr[ib->length_dw++] = value; |
1987 | radeon_ring_write(ring, upper_32_bits(value)); | 1989 | ib->ptr[ib->length_dw++] = upper_32_bits(value); |
1988 | } | 1990 | } |
1989 | } | 1991 | } |
1990 | } else { | 1992 | } else { |
@@ -1994,9 +1996,9 @@ void cayman_vm_set_page(struct radeon_device *rdev, uint64_t pe, | |||
1994 | ndw = 0xFFFFE; | 1996 | ndw = 0xFFFFE; |
1995 | 1997 | ||
1996 | /* for non-physically contiguous pages (system) */ | 1998 | /* for non-physically contiguous pages (system) */ |
1997 | radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 0, ndw)); | 1999 | ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_WRITE, 0, 0, ndw); |
1998 | radeon_ring_write(ring, pe); | 2000 | ib->ptr[ib->length_dw++] = pe; |
1999 | radeon_ring_write(ring, upper_32_bits(pe) & 0xff); | 2001 | ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff; |
2000 | for (; ndw > 0; ndw -= 2, --count, pe += 8) { | 2002 | for (; ndw > 0; ndw -= 2, --count, pe += 8) { |
2001 | if (flags & RADEON_VM_PAGE_SYSTEM) { | 2003 | if (flags & RADEON_VM_PAGE_SYSTEM) { |
2002 | value = radeon_vm_map_gart(rdev, addr); | 2004 | value = radeon_vm_map_gart(rdev, addr); |
@@ -2008,10 +2010,12 @@ void cayman_vm_set_page(struct radeon_device *rdev, uint64_t pe, | |||
2008 | } | 2010 | } |
2009 | addr += incr; | 2011 | addr += incr; |
2010 | value |= r600_flags; | 2012 | value |= r600_flags; |
2011 | radeon_ring_write(ring, value); | 2013 | ib->ptr[ib->length_dw++] = value; |
2012 | radeon_ring_write(ring, upper_32_bits(value)); | 2014 | ib->ptr[ib->length_dw++] = upper_32_bits(value); |
2013 | } | 2015 | } |
2014 | } | 2016 | } |
2017 | while (ib->length_dw & 0x7) | ||
2018 | ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0); | ||
2015 | } | 2019 | } |
2016 | } | 2020 | } |
2017 | 2021 | ||
diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h index 6539d6cb4bc7..307d681b3443 100644 --- a/drivers/gpu/drm/radeon/radeon.h +++ b/drivers/gpu/drm/radeon/radeon.h | |||
@@ -780,6 +780,7 @@ int radeon_ib_get(struct radeon_device *rdev, int ring, | |||
780 | struct radeon_ib *ib, struct radeon_vm *vm, | 780 | struct radeon_ib *ib, struct radeon_vm *vm, |
781 | unsigned size); | 781 | unsigned size); |
782 | void radeon_ib_free(struct radeon_device *rdev, struct radeon_ib *ib); | 782 | void radeon_ib_free(struct radeon_device *rdev, struct radeon_ib *ib); |
783 | void radeon_ib_sync_to(struct radeon_ib *ib, struct radeon_fence *fence); | ||
783 | int radeon_ib_schedule(struct radeon_device *rdev, struct radeon_ib *ib, | 784 | int radeon_ib_schedule(struct radeon_device *rdev, struct radeon_ib *ib, |
784 | struct radeon_ib *const_ib); | 785 | struct radeon_ib *const_ib); |
785 | int radeon_ib_pool_init(struct radeon_device *rdev); | 786 | int radeon_ib_pool_init(struct radeon_device *rdev); |
@@ -1188,7 +1189,9 @@ struct radeon_asic { | |||
1188 | void (*fini)(struct radeon_device *rdev); | 1189 | void (*fini)(struct radeon_device *rdev); |
1189 | 1190 | ||
1190 | u32 pt_ring_index; | 1191 | u32 pt_ring_index; |
1191 | void (*set_page)(struct radeon_device *rdev, uint64_t pe, | 1192 | void (*set_page)(struct radeon_device *rdev, |
1193 | struct radeon_ib *ib, | ||
1194 | uint64_t pe, | ||
1192 | uint64_t addr, unsigned count, | 1195 | uint64_t addr, unsigned count, |
1193 | uint32_t incr, uint32_t flags); | 1196 | uint32_t incr, uint32_t flags); |
1194 | } vm; | 1197 | } vm; |
@@ -1810,7 +1813,7 @@ void radeon_ring_write(struct radeon_ring *ring, uint32_t v); | |||
1810 | #define radeon_gart_set_page(rdev, i, p) (rdev)->asic->gart.set_page((rdev), (i), (p)) | 1813 | #define radeon_gart_set_page(rdev, i, p) (rdev)->asic->gart.set_page((rdev), (i), (p)) |
1811 | #define radeon_asic_vm_init(rdev) (rdev)->asic->vm.init((rdev)) | 1814 | #define radeon_asic_vm_init(rdev) (rdev)->asic->vm.init((rdev)) |
1812 | #define radeon_asic_vm_fini(rdev) (rdev)->asic->vm.fini((rdev)) | 1815 | #define radeon_asic_vm_fini(rdev) (rdev)->asic->vm.fini((rdev)) |
1813 | #define radeon_asic_vm_set_page(rdev, pe, addr, count, incr, flags) ((rdev)->asic->vm.set_page((rdev), (pe), (addr), (count), (incr), (flags))) | 1816 | #define radeon_asic_vm_set_page(rdev, ib, pe, addr, count, incr, flags) ((rdev)->asic->vm.set_page((rdev), (ib), (pe), (addr), (count), (incr), (flags))) |
1814 | #define radeon_ring_start(rdev, r, cp) (rdev)->asic->ring[(r)].ring_start((rdev), (cp)) | 1817 | #define radeon_ring_start(rdev, r, cp) (rdev)->asic->ring[(r)].ring_start((rdev), (cp)) |
1815 | #define radeon_ring_test(rdev, r, cp) (rdev)->asic->ring[(r)].ring_test((rdev), (cp)) | 1818 | #define radeon_ring_test(rdev, r, cp) (rdev)->asic->ring[(r)].ring_test((rdev), (cp)) |
1816 | #define radeon_ib_test(rdev, r, cp) (rdev)->asic->ring[(r)].ib_test((rdev), (cp)) | 1819 | #define radeon_ib_test(rdev, r, cp) (rdev)->asic->ring[(r)].ib_test((rdev), (cp)) |
diff --git a/drivers/gpu/drm/radeon/radeon_asic.h b/drivers/gpu/drm/radeon/radeon_asic.h index e429e2574cae..f4134a823958 100644 --- a/drivers/gpu/drm/radeon/radeon_asic.h +++ b/drivers/gpu/drm/radeon/radeon_asic.h | |||
@@ -474,7 +474,9 @@ int cayman_vm_init(struct radeon_device *rdev); | |||
474 | void cayman_vm_fini(struct radeon_device *rdev); | 474 | void cayman_vm_fini(struct radeon_device *rdev); |
475 | void cayman_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm); | 475 | void cayman_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm); |
476 | uint32_t cayman_vm_page_flags(struct radeon_device *rdev, uint32_t flags); | 476 | uint32_t cayman_vm_page_flags(struct radeon_device *rdev, uint32_t flags); |
477 | void cayman_vm_set_page(struct radeon_device *rdev, uint64_t pe, | 477 | void cayman_vm_set_page(struct radeon_device *rdev, |
478 | struct radeon_ib *ib, | ||
479 | uint64_t pe, | ||
478 | uint64_t addr, unsigned count, | 480 | uint64_t addr, unsigned count, |
479 | uint32_t incr, uint32_t flags); | 481 | uint32_t incr, uint32_t flags); |
480 | int evergreen_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib); | 482 | int evergreen_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib); |
@@ -506,7 +508,9 @@ int si_irq_set(struct radeon_device *rdev); | |||
506 | int si_irq_process(struct radeon_device *rdev); | 508 | int si_irq_process(struct radeon_device *rdev); |
507 | int si_vm_init(struct radeon_device *rdev); | 509 | int si_vm_init(struct radeon_device *rdev); |
508 | void si_vm_fini(struct radeon_device *rdev); | 510 | void si_vm_fini(struct radeon_device *rdev); |
509 | void si_vm_set_page(struct radeon_device *rdev, uint64_t pe, | 511 | void si_vm_set_page(struct radeon_device *rdev, |
512 | struct radeon_ib *ib, | ||
513 | uint64_t pe, | ||
510 | uint64_t addr, unsigned count, | 514 | uint64_t addr, unsigned count, |
511 | uint32_t incr, uint32_t flags); | 515 | uint32_t incr, uint32_t flags); |
512 | void si_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm); | 516 | void si_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm); |
diff --git a/drivers/gpu/drm/radeon/radeon_cs.c b/drivers/gpu/drm/radeon/radeon_cs.c index 1d214b66650f..70d38241b083 100644 --- a/drivers/gpu/drm/radeon/radeon_cs.c +++ b/drivers/gpu/drm/radeon/radeon_cs.c | |||
@@ -125,18 +125,6 @@ static int radeon_cs_get_ring(struct radeon_cs_parser *p, u32 ring, s32 priority | |||
125 | return 0; | 125 | return 0; |
126 | } | 126 | } |
127 | 127 | ||
128 | static void radeon_cs_sync_to(struct radeon_cs_parser *p, | ||
129 | struct radeon_fence *fence) | ||
130 | { | ||
131 | struct radeon_fence *other; | ||
132 | |||
133 | if (!fence) | ||
134 | return; | ||
135 | |||
136 | other = p->ib.sync_to[fence->ring]; | ||
137 | p->ib.sync_to[fence->ring] = radeon_fence_later(fence, other); | ||
138 | } | ||
139 | |||
140 | static void radeon_cs_sync_rings(struct radeon_cs_parser *p) | 128 | static void radeon_cs_sync_rings(struct radeon_cs_parser *p) |
141 | { | 129 | { |
142 | int i; | 130 | int i; |
@@ -145,7 +133,7 @@ static void radeon_cs_sync_rings(struct radeon_cs_parser *p) | |||
145 | if (!p->relocs[i].robj) | 133 | if (!p->relocs[i].robj) |
146 | continue; | 134 | continue; |
147 | 135 | ||
148 | radeon_cs_sync_to(p, p->relocs[i].robj->tbo.sync_obj); | 136 | radeon_ib_sync_to(&p->ib, p->relocs[i].robj->tbo.sync_obj); |
149 | } | 137 | } |
150 | } | 138 | } |
151 | 139 | ||
@@ -472,8 +460,9 @@ static int radeon_cs_ib_vm_chunk(struct radeon_device *rdev, | |||
472 | goto out; | 460 | goto out; |
473 | } | 461 | } |
474 | radeon_cs_sync_rings(parser); | 462 | radeon_cs_sync_rings(parser); |
475 | radeon_cs_sync_to(parser, vm->fence); | 463 | radeon_ib_sync_to(&parser->ib, vm->fence); |
476 | radeon_cs_sync_to(parser, radeon_vm_grab_id(rdev, vm, parser->ring)); | 464 | radeon_ib_sync_to(&parser->ib, radeon_vm_grab_id( |
465 | rdev, vm, parser->ring)); | ||
477 | 466 | ||
478 | if ((rdev->family >= CHIP_TAHITI) && | 467 | if ((rdev->family >= CHIP_TAHITI) && |
479 | (parser->chunk_const_ib_idx != -1)) { | 468 | (parser->chunk_const_ib_idx != -1)) { |
diff --git a/drivers/gpu/drm/radeon/radeon_gart.c b/drivers/gpu/drm/radeon/radeon_gart.c index 6e24f84755b5..2c1341f63dc5 100644 --- a/drivers/gpu/drm/radeon/radeon_gart.c +++ b/drivers/gpu/drm/radeon/radeon_gart.c | |||
@@ -929,6 +929,7 @@ uint64_t radeon_vm_map_gart(struct radeon_device *rdev, uint64_t addr) | |||
929 | */ | 929 | */ |
930 | static int radeon_vm_update_pdes(struct radeon_device *rdev, | 930 | static int radeon_vm_update_pdes(struct radeon_device *rdev, |
931 | struct radeon_vm *vm, | 931 | struct radeon_vm *vm, |
932 | struct radeon_ib *ib, | ||
932 | uint64_t start, uint64_t end) | 933 | uint64_t start, uint64_t end) |
933 | { | 934 | { |
934 | static const uint32_t incr = RADEON_VM_PTE_COUNT * 8; | 935 | static const uint32_t incr = RADEON_VM_PTE_COUNT * 8; |
@@ -971,7 +972,7 @@ retry: | |||
971 | ((last_pt + incr * count) != pt)) { | 972 | ((last_pt + incr * count) != pt)) { |
972 | 973 | ||
973 | if (count) { | 974 | if (count) { |
974 | radeon_asic_vm_set_page(rdev, last_pde, | 975 | radeon_asic_vm_set_page(rdev, ib, last_pde, |
975 | last_pt, count, incr, | 976 | last_pt, count, incr, |
976 | RADEON_VM_PAGE_VALID); | 977 | RADEON_VM_PAGE_VALID); |
977 | } | 978 | } |
@@ -985,7 +986,7 @@ retry: | |||
985 | } | 986 | } |
986 | 987 | ||
987 | if (count) { | 988 | if (count) { |
988 | radeon_asic_vm_set_page(rdev, last_pde, last_pt, count, | 989 | radeon_asic_vm_set_page(rdev, ib, last_pde, last_pt, count, |
989 | incr, RADEON_VM_PAGE_VALID); | 990 | incr, RADEON_VM_PAGE_VALID); |
990 | 991 | ||
991 | } | 992 | } |
@@ -1009,6 +1010,7 @@ retry: | |||
1009 | */ | 1010 | */ |
1010 | static void radeon_vm_update_ptes(struct radeon_device *rdev, | 1011 | static void radeon_vm_update_ptes(struct radeon_device *rdev, |
1011 | struct radeon_vm *vm, | 1012 | struct radeon_vm *vm, |
1013 | struct radeon_ib *ib, | ||
1012 | uint64_t start, uint64_t end, | 1014 | uint64_t start, uint64_t end, |
1013 | uint64_t dst, uint32_t flags) | 1015 | uint64_t dst, uint32_t flags) |
1014 | { | 1016 | { |
@@ -1038,7 +1040,7 @@ static void radeon_vm_update_ptes(struct radeon_device *rdev, | |||
1038 | if ((last_pte + 8 * count) != pte) { | 1040 | if ((last_pte + 8 * count) != pte) { |
1039 | 1041 | ||
1040 | if (count) { | 1042 | if (count) { |
1041 | radeon_asic_vm_set_page(rdev, last_pte, | 1043 | radeon_asic_vm_set_page(rdev, ib, last_pte, |
1042 | last_dst, count, | 1044 | last_dst, count, |
1043 | RADEON_GPU_PAGE_SIZE, | 1045 | RADEON_GPU_PAGE_SIZE, |
1044 | flags); | 1046 | flags); |
@@ -1056,7 +1058,8 @@ static void radeon_vm_update_ptes(struct radeon_device *rdev, | |||
1056 | } | 1058 | } |
1057 | 1059 | ||
1058 | if (count) { | 1060 | if (count) { |
1059 | radeon_asic_vm_set_page(rdev, last_pte, last_dst, count, | 1061 | radeon_asic_vm_set_page(rdev, ib, last_pte, |
1062 | last_dst, count, | ||
1060 | RADEON_GPU_PAGE_SIZE, flags); | 1063 | RADEON_GPU_PAGE_SIZE, flags); |
1061 | } | 1064 | } |
1062 | } | 1065 | } |
@@ -1080,8 +1083,7 @@ int radeon_vm_bo_update_pte(struct radeon_device *rdev, | |||
1080 | struct ttm_mem_reg *mem) | 1083 | struct ttm_mem_reg *mem) |
1081 | { | 1084 | { |
1082 | unsigned ridx = rdev->asic->vm.pt_ring_index; | 1085 | unsigned ridx = rdev->asic->vm.pt_ring_index; |
1083 | struct radeon_ring *ring = &rdev->ring[ridx]; | 1086 | struct radeon_ib ib; |
1084 | struct radeon_semaphore *sem = NULL; | ||
1085 | struct radeon_bo_va *bo_va; | 1087 | struct radeon_bo_va *bo_va; |
1086 | unsigned nptes, npdes, ndw; | 1088 | unsigned nptes, npdes, ndw; |
1087 | uint64_t addr; | 1089 | uint64_t addr; |
@@ -1124,25 +1126,13 @@ int radeon_vm_bo_update_pte(struct radeon_device *rdev, | |||
1124 | bo_va->valid = false; | 1126 | bo_va->valid = false; |
1125 | } | 1127 | } |
1126 | 1128 | ||
1127 | if (vm->fence && radeon_fence_signaled(vm->fence)) { | ||
1128 | radeon_fence_unref(&vm->fence); | ||
1129 | } | ||
1130 | |||
1131 | if (vm->fence && vm->fence->ring != ridx) { | ||
1132 | r = radeon_semaphore_create(rdev, &sem); | ||
1133 | if (r) { | ||
1134 | return r; | ||
1135 | } | ||
1136 | } | ||
1137 | |||
1138 | nptes = radeon_bo_ngpu_pages(bo); | 1129 | nptes = radeon_bo_ngpu_pages(bo); |
1139 | 1130 | ||
1140 | /* assume two extra pdes in case the mapping overlaps the borders */ | 1131 | /* assume two extra pdes in case the mapping overlaps the borders */ |
1141 | npdes = (nptes >> RADEON_VM_BLOCK_SIZE) + 2; | 1132 | npdes = (nptes >> RADEON_VM_BLOCK_SIZE) + 2; |
1142 | 1133 | ||
1143 | /* estimate number of dw needed */ | 1134 | /* padding, etc. */ |
1144 | /* semaphore, fence and padding */ | 1135 | ndw = 64; |
1145 | ndw = 32; | ||
1146 | 1136 | ||
1147 | if (RADEON_VM_BLOCK_SIZE > 11) | 1137 | if (RADEON_VM_BLOCK_SIZE > 11) |
1148 | /* reserve space for one header for every 2k dwords */ | 1138 | /* reserve space for one header for every 2k dwords */ |
@@ -1161,33 +1151,31 @@ int radeon_vm_bo_update_pte(struct radeon_device *rdev, | |||
1161 | /* reserve space for pde addresses */ | 1151 | /* reserve space for pde addresses */ |
1162 | ndw += npdes * 2; | 1152 | ndw += npdes * 2; |
1163 | 1153 | ||
1164 | r = radeon_ring_lock(rdev, ring, ndw); | 1154 | /* update too big for an IB */ |
1165 | if (r) { | 1155 | if (ndw > 0xfffff) |
1166 | return r; | 1156 | return -ENOMEM; |
1167 | } | ||
1168 | 1157 | ||
1169 | if (sem && radeon_fence_need_sync(vm->fence, ridx)) { | 1158 | r = radeon_ib_get(rdev, ridx, &ib, NULL, ndw * 4); |
1170 | radeon_semaphore_sync_rings(rdev, sem, vm->fence->ring, ridx); | 1159 | ib.length_dw = 0; |
1171 | radeon_fence_note_sync(vm->fence, ridx); | ||
1172 | } | ||
1173 | 1160 | ||
1174 | r = radeon_vm_update_pdes(rdev, vm, bo_va->soffset, bo_va->eoffset); | 1161 | r = radeon_vm_update_pdes(rdev, vm, &ib, bo_va->soffset, bo_va->eoffset); |
1175 | if (r) { | 1162 | if (r) { |
1176 | radeon_ring_unlock_undo(rdev, ring); | 1163 | radeon_ib_free(rdev, &ib); |
1177 | return r; | 1164 | return r; |
1178 | } | 1165 | } |
1179 | 1166 | ||
1180 | radeon_vm_update_ptes(rdev, vm, bo_va->soffset, bo_va->eoffset, | 1167 | radeon_vm_update_ptes(rdev, vm, &ib, bo_va->soffset, bo_va->eoffset, |
1181 | addr, bo_va->flags); | 1168 | addr, bo_va->flags); |
1182 | 1169 | ||
1183 | radeon_fence_unref(&vm->fence); | 1170 | radeon_ib_sync_to(&ib, vm->fence); |
1184 | r = radeon_fence_emit(rdev, &vm->fence, ridx); | 1171 | r = radeon_ib_schedule(rdev, &ib, NULL); |
1185 | if (r) { | 1172 | if (r) { |
1186 | radeon_ring_unlock_undo(rdev, ring); | 1173 | radeon_ib_free(rdev, &ib); |
1187 | return r; | 1174 | return r; |
1188 | } | 1175 | } |
1189 | radeon_ring_unlock_commit(rdev, ring); | 1176 | radeon_fence_unref(&vm->fence); |
1190 | radeon_semaphore_free(rdev, &sem, vm->fence); | 1177 | vm->fence = radeon_fence_ref(ib.fence); |
1178 | radeon_ib_free(rdev, &ib); | ||
1191 | radeon_fence_unref(&vm->last_flush); | 1179 | radeon_fence_unref(&vm->last_flush); |
1192 | 1180 | ||
1193 | return 0; | 1181 | return 0; |
diff --git a/drivers/gpu/drm/radeon/radeon_ring.c b/drivers/gpu/drm/radeon/radeon_ring.c index cd72062d5a91..8d58e268ff6d 100644 --- a/drivers/gpu/drm/radeon/radeon_ring.c +++ b/drivers/gpu/drm/radeon/radeon_ring.c | |||
@@ -109,6 +109,25 @@ void radeon_ib_free(struct radeon_device *rdev, struct radeon_ib *ib) | |||
109 | } | 109 | } |
110 | 110 | ||
111 | /** | 111 | /** |
112 | * radeon_ib_sync_to - sync to fence before executing the IB | ||
113 | * | ||
114 | * @ib: IB object to add fence to | ||
115 | * @fence: fence to sync to | ||
116 | * | ||
117 | * Sync to the fence before executing the IB | ||
118 | */ | ||
119 | void radeon_ib_sync_to(struct radeon_ib *ib, struct radeon_fence *fence) | ||
120 | { | ||
121 | struct radeon_fence *other; | ||
122 | |||
123 | if (!fence) | ||
124 | return; | ||
125 | |||
126 | other = ib->sync_to[fence->ring]; | ||
127 | ib->sync_to[fence->ring] = radeon_fence_later(fence, other); | ||
128 | } | ||
129 | |||
130 | /** | ||
112 | * radeon_ib_schedule - schedule an IB (Indirect Buffer) on the ring | 131 | * radeon_ib_schedule - schedule an IB (Indirect Buffer) on the ring |
113 | * | 132 | * |
114 | * @rdev: radeon_device pointer | 133 | * @rdev: radeon_device pointer |
diff --git a/drivers/gpu/drm/radeon/si.c b/drivers/gpu/drm/radeon/si.c index cd83bc5bd235..a910cb92cfd0 100644 --- a/drivers/gpu/drm/radeon/si.c +++ b/drivers/gpu/drm/radeon/si.c | |||
@@ -3043,19 +3043,21 @@ void si_vm_fini(struct radeon_device *rdev) | |||
3043 | * si_vm_set_page - update the page tables using the CP | 3043 | * si_vm_set_page - update the page tables using the CP |
3044 | * | 3044 | * |
3045 | * @rdev: radeon_device pointer | 3045 | * @rdev: radeon_device pointer |
3046 | * @ib: indirect buffer to fill with commands | ||
3046 | * @pe: addr of the page entry | 3047 | * @pe: addr of the page entry |
3047 | * @addr: dst addr to write into pe | 3048 | * @addr: dst addr to write into pe |
3048 | * @count: number of page entries to update | 3049 | * @count: number of page entries to update |
3049 | * @incr: increase next addr by incr bytes | 3050 | * @incr: increase next addr by incr bytes |
3050 | * @flags: access flags | 3051 | * @flags: access flags |
3051 | * | 3052 | * |
3052 | * Update the page tables using the CP (cayman-si). | 3053 | * Update the page tables using the CP (SI). |
3053 | */ | 3054 | */ |
3054 | void si_vm_set_page(struct radeon_device *rdev, uint64_t pe, | 3055 | void si_vm_set_page(struct radeon_device *rdev, |
3056 | struct radeon_ib *ib, | ||
3057 | uint64_t pe, | ||
3055 | uint64_t addr, unsigned count, | 3058 | uint64_t addr, unsigned count, |
3056 | uint32_t incr, uint32_t flags) | 3059 | uint32_t incr, uint32_t flags) |
3057 | { | 3060 | { |
3058 | struct radeon_ring *ring = &rdev->ring[rdev->asic->vm.pt_ring_index]; | ||
3059 | uint32_t r600_flags = cayman_vm_page_flags(rdev, flags); | 3061 | uint32_t r600_flags = cayman_vm_page_flags(rdev, flags); |
3060 | uint64_t value; | 3062 | uint64_t value; |
3061 | unsigned ndw; | 3063 | unsigned ndw; |
@@ -3066,11 +3068,11 @@ void si_vm_set_page(struct radeon_device *rdev, uint64_t pe, | |||
3066 | if (ndw > 0x3FFE) | 3068 | if (ndw > 0x3FFE) |
3067 | ndw = 0x3FFE; | 3069 | ndw = 0x3FFE; |
3068 | 3070 | ||
3069 | radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, ndw)); | 3071 | ib->ptr[ib->length_dw++] = PACKET3(PACKET3_WRITE_DATA, ndw); |
3070 | radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | | 3072 | ib->ptr[ib->length_dw++] = (WRITE_DATA_ENGINE_SEL(0) | |
3071 | WRITE_DATA_DST_SEL(1))); | 3073 | WRITE_DATA_DST_SEL(1)); |
3072 | radeon_ring_write(ring, pe); | 3074 | ib->ptr[ib->length_dw++] = pe; |
3073 | radeon_ring_write(ring, upper_32_bits(pe)); | 3075 | ib->ptr[ib->length_dw++] = upper_32_bits(pe); |
3074 | for (; ndw > 2; ndw -= 2, --count, pe += 8) { | 3076 | for (; ndw > 2; ndw -= 2, --count, pe += 8) { |
3075 | if (flags & RADEON_VM_PAGE_SYSTEM) { | 3077 | if (flags & RADEON_VM_PAGE_SYSTEM) { |
3076 | value = radeon_vm_map_gart(rdev, addr); | 3078 | value = radeon_vm_map_gart(rdev, addr); |
@@ -3082,8 +3084,8 @@ void si_vm_set_page(struct radeon_device *rdev, uint64_t pe, | |||
3082 | } | 3084 | } |
3083 | addr += incr; | 3085 | addr += incr; |
3084 | value |= r600_flags; | 3086 | value |= r600_flags; |
3085 | radeon_ring_write(ring, value); | 3087 | ib->ptr[ib->length_dw++] = value; |
3086 | radeon_ring_write(ring, upper_32_bits(value)); | 3088 | ib->ptr[ib->length_dw++] = upper_32_bits(value); |
3087 | } | 3089 | } |
3088 | } | 3090 | } |
3089 | } else { | 3091 | } else { |
@@ -3095,9 +3097,9 @@ void si_vm_set_page(struct radeon_device *rdev, uint64_t pe, | |||
3095 | ndw = 0xFFFFE; | 3097 | ndw = 0xFFFFE; |
3096 | 3098 | ||
3097 | /* for non-physically contiguous pages (system) */ | 3099 | /* for non-physically contiguous pages (system) */ |
3098 | radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 0, ndw)); | 3100 | ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 0, ndw); |
3099 | radeon_ring_write(ring, pe); | 3101 | ib->ptr[ib->length_dw++] = pe; |
3100 | radeon_ring_write(ring, upper_32_bits(pe) & 0xff); | 3102 | ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff; |
3101 | for (; ndw > 0; ndw -= 2, --count, pe += 8) { | 3103 | for (; ndw > 0; ndw -= 2, --count, pe += 8) { |
3102 | if (flags & RADEON_VM_PAGE_SYSTEM) { | 3104 | if (flags & RADEON_VM_PAGE_SYSTEM) { |
3103 | value = radeon_vm_map_gart(rdev, addr); | 3105 | value = radeon_vm_map_gart(rdev, addr); |
@@ -3109,8 +3111,8 @@ void si_vm_set_page(struct radeon_device *rdev, uint64_t pe, | |||
3109 | } | 3111 | } |
3110 | addr += incr; | 3112 | addr += incr; |
3111 | value |= r600_flags; | 3113 | value |= r600_flags; |
3112 | radeon_ring_write(ring, value); | 3114 | ib->ptr[ib->length_dw++] = value; |
3113 | radeon_ring_write(ring, upper_32_bits(value)); | 3115 | ib->ptr[ib->length_dw++] = upper_32_bits(value); |
3114 | } | 3116 | } |
3115 | } | 3117 | } |
3116 | } else { | 3118 | } else { |
@@ -3124,20 +3126,22 @@ void si_vm_set_page(struct radeon_device *rdev, uint64_t pe, | |||
3124 | else | 3126 | else |
3125 | value = 0; | 3127 | value = 0; |
3126 | /* for physically contiguous pages (vram) */ | 3128 | /* for physically contiguous pages (vram) */ |
3127 | radeon_ring_write(ring, DMA_PTE_PDE_PACKET(ndw)); | 3129 | ib->ptr[ib->length_dw++] = DMA_PTE_PDE_PACKET(ndw); |
3128 | radeon_ring_write(ring, pe); /* dst addr */ | 3130 | ib->ptr[ib->length_dw++] = pe; /* dst addr */ |
3129 | radeon_ring_write(ring, upper_32_bits(pe) & 0xff); | 3131 | ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff; |
3130 | radeon_ring_write(ring, r600_flags); /* mask */ | 3132 | ib->ptr[ib->length_dw++] = r600_flags; /* mask */ |
3131 | radeon_ring_write(ring, 0); | 3133 | ib->ptr[ib->length_dw++] = 0; |
3132 | radeon_ring_write(ring, value); /* value */ | 3134 | ib->ptr[ib->length_dw++] = value; /* value */ |
3133 | radeon_ring_write(ring, upper_32_bits(value)); | 3135 | ib->ptr[ib->length_dw++] = upper_32_bits(value); |
3134 | radeon_ring_write(ring, incr); /* increment size */ | 3136 | ib->ptr[ib->length_dw++] = incr; /* increment size */ |
3135 | radeon_ring_write(ring, 0); | 3137 | ib->ptr[ib->length_dw++] = 0; |
3136 | pe += ndw * 4; | 3138 | pe += ndw * 4; |
3137 | addr += (ndw / 2) * incr; | 3139 | addr += (ndw / 2) * incr; |
3138 | count -= ndw / 2; | 3140 | count -= ndw / 2; |
3139 | } | 3141 | } |
3140 | } | 3142 | } |
3143 | while (ib->length_dw & 0x7) | ||
3144 | ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0); | ||
3141 | } | 3145 | } |
3142 | } | 3146 | } |
3143 | 3147 | ||