aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_object.c')
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_object.c176
1 files changed, 119 insertions, 57 deletions
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
index dcc6af97f59d..62cabfb5dff8 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
@@ -41,13 +41,13 @@ int amdgpu_ttm_init(struct amdgpu_device *adev);
41void amdgpu_ttm_fini(struct amdgpu_device *adev); 41void amdgpu_ttm_fini(struct amdgpu_device *adev);
42 42
43static u64 amdgpu_get_vis_part_size(struct amdgpu_device *adev, 43static u64 amdgpu_get_vis_part_size(struct amdgpu_device *adev,
44 struct ttm_mem_reg * mem) 44 struct ttm_mem_reg *mem)
45{ 45{
46 u64 ret = 0; 46 u64 ret = 0;
47 if (mem->start << PAGE_SHIFT < adev->mc.visible_vram_size) { 47 if (mem->start << PAGE_SHIFT < adev->mc.visible_vram_size) {
48 ret = (u64)((mem->start << PAGE_SHIFT) + mem->size) > 48 ret = (u64)((mem->start << PAGE_SHIFT) + mem->size) >
49 adev->mc.visible_vram_size ? 49 adev->mc.visible_vram_size ?
50 adev->mc.visible_vram_size - (mem->start << PAGE_SHIFT): 50 adev->mc.visible_vram_size - (mem->start << PAGE_SHIFT) :
51 mem->size; 51 mem->size;
52 } 52 }
53 return ret; 53 return ret;
@@ -112,82 +112,111 @@ bool amdgpu_ttm_bo_is_amdgpu_bo(struct ttm_buffer_object *bo)
112 return false; 112 return false;
113} 113}
114 114
115void amdgpu_ttm_placement_from_domain(struct amdgpu_bo *rbo, u32 domain) 115static void amdgpu_ttm_placement_init(struct amdgpu_device *adev,
116 struct ttm_placement *placement,
117 struct ttm_place *placements,
118 u32 domain, u64 flags)
116{ 119{
117 u32 c = 0, i; 120 u32 c = 0, i;
118 rbo->placement.placement = rbo->placements; 121
119 rbo->placement.busy_placement = rbo->placements; 122 placement->placement = placements;
123 placement->busy_placement = placements;
120 124
121 if (domain & AMDGPU_GEM_DOMAIN_VRAM) { 125 if (domain & AMDGPU_GEM_DOMAIN_VRAM) {
122 if (rbo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS && 126 if (flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS &&
123 rbo->adev->mc.visible_vram_size < rbo->adev->mc.real_vram_size) { 127 adev->mc.visible_vram_size < adev->mc.real_vram_size) {
124 rbo->placements[c].fpfn = 128 placements[c].fpfn =
125 rbo->adev->mc.visible_vram_size >> PAGE_SHIFT; 129 adev->mc.visible_vram_size >> PAGE_SHIFT;
126 rbo->placements[c++].flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_UNCACHED | 130 placements[c++].flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_UNCACHED |
127 TTM_PL_FLAG_VRAM; 131 TTM_PL_FLAG_VRAM;
128 } 132 }
129 rbo->placements[c].fpfn = 0; 133 placements[c].fpfn = 0;
130 rbo->placements[c++].flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_UNCACHED | 134 placements[c++].flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_UNCACHED |
131 TTM_PL_FLAG_VRAM; 135 TTM_PL_FLAG_VRAM;
132 } 136 }
133 137
134 if (domain & AMDGPU_GEM_DOMAIN_GTT) { 138 if (domain & AMDGPU_GEM_DOMAIN_GTT) {
135 if (rbo->flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC) { 139 if (flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC) {
136 rbo->placements[c].fpfn = 0; 140 placements[c].fpfn = 0;
137 rbo->placements[c++].flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_TT | 141 placements[c++].flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_TT |
138 TTM_PL_FLAG_UNCACHED; 142 TTM_PL_FLAG_UNCACHED;
139 } else { 143 } else {
140 rbo->placements[c].fpfn = 0; 144 placements[c].fpfn = 0;
141 rbo->placements[c++].flags = TTM_PL_FLAG_CACHED | TTM_PL_FLAG_TT; 145 placements[c++].flags = TTM_PL_FLAG_CACHED | TTM_PL_FLAG_TT;
142 } 146 }
143 } 147 }
144 148
145 if (domain & AMDGPU_GEM_DOMAIN_CPU) { 149 if (domain & AMDGPU_GEM_DOMAIN_CPU) {
146 if (rbo->flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC) { 150 if (flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC) {
147 rbo->placements[c].fpfn = 0; 151 placements[c].fpfn = 0;
148 rbo->placements[c++].flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_SYSTEM | 152 placements[c++].flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_SYSTEM |
149 TTM_PL_FLAG_UNCACHED; 153 TTM_PL_FLAG_UNCACHED;
150 } else { 154 } else {
151 rbo->placements[c].fpfn = 0; 155 placements[c].fpfn = 0;
152 rbo->placements[c++].flags = TTM_PL_FLAG_CACHED | TTM_PL_FLAG_SYSTEM; 156 placements[c++].flags = TTM_PL_FLAG_CACHED | TTM_PL_FLAG_SYSTEM;
153 } 157 }
154 } 158 }
155 159
156 if (domain & AMDGPU_GEM_DOMAIN_GDS) { 160 if (domain & AMDGPU_GEM_DOMAIN_GDS) {
157 rbo->placements[c++].flags = TTM_PL_FLAG_UNCACHED | 161 placements[c].fpfn = 0;
158 AMDGPU_PL_FLAG_GDS; 162 placements[c++].flags = TTM_PL_FLAG_UNCACHED |
163 AMDGPU_PL_FLAG_GDS;
159 } 164 }
160 if (domain & AMDGPU_GEM_DOMAIN_GWS) { 165 if (domain & AMDGPU_GEM_DOMAIN_GWS) {
161 rbo->placements[c++].flags = TTM_PL_FLAG_UNCACHED | 166 placements[c].fpfn = 0;
162 AMDGPU_PL_FLAG_GWS; 167 placements[c++].flags = TTM_PL_FLAG_UNCACHED |
168 AMDGPU_PL_FLAG_GWS;
163 } 169 }
164 if (domain & AMDGPU_GEM_DOMAIN_OA) { 170 if (domain & AMDGPU_GEM_DOMAIN_OA) {
165 rbo->placements[c++].flags = TTM_PL_FLAG_UNCACHED | 171 placements[c].fpfn = 0;
166 AMDGPU_PL_FLAG_OA; 172 placements[c++].flags = TTM_PL_FLAG_UNCACHED |
173 AMDGPU_PL_FLAG_OA;
167 } 174 }
168 175
169 if (!c) { 176 if (!c) {
170 rbo->placements[c].fpfn = 0; 177 placements[c].fpfn = 0;
171 rbo->placements[c++].flags = TTM_PL_MASK_CACHING | 178 placements[c++].flags = TTM_PL_MASK_CACHING |
172 TTM_PL_FLAG_SYSTEM; 179 TTM_PL_FLAG_SYSTEM;
173 } 180 }
174 rbo->placement.num_placement = c; 181 placement->num_placement = c;
175 rbo->placement.num_busy_placement = c; 182 placement->num_busy_placement = c;
176 183
177 for (i = 0; i < c; i++) { 184 for (i = 0; i < c; i++) {
178 if ((rbo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) && 185 if ((flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) &&
179 (rbo->placements[i].flags & TTM_PL_FLAG_VRAM) && 186 (placements[i].flags & TTM_PL_FLAG_VRAM) &&
180 !rbo->placements[i].fpfn) 187 !placements[i].fpfn)
181 rbo->placements[i].lpfn = 188 placements[i].lpfn =
182 rbo->adev->mc.visible_vram_size >> PAGE_SHIFT; 189 adev->mc.visible_vram_size >> PAGE_SHIFT;
183 else 190 else
184 rbo->placements[i].lpfn = 0; 191 placements[i].lpfn = 0;
185 } 192 }
186} 193}
187 194
188int amdgpu_bo_create(struct amdgpu_device *adev, 195void amdgpu_ttm_placement_from_domain(struct amdgpu_bo *rbo, u32 domain)
189 unsigned long size, int byte_align, bool kernel, u32 domain, u64 flags, 196{
190 struct sg_table *sg, struct amdgpu_bo **bo_ptr) 197 amdgpu_ttm_placement_init(rbo->adev, &rbo->placement,
198 rbo->placements, domain, rbo->flags);
199}
200
201static void amdgpu_fill_placement_to_bo(struct amdgpu_bo *bo,
202 struct ttm_placement *placement)
203{
204 BUG_ON(placement->num_placement > (AMDGPU_GEM_DOMAIN_MAX + 1));
205
206 memcpy(bo->placements, placement->placement,
207 placement->num_placement * sizeof(struct ttm_place));
208 bo->placement.num_placement = placement->num_placement;
209 bo->placement.num_busy_placement = placement->num_busy_placement;
210 bo->placement.placement = bo->placements;
211 bo->placement.busy_placement = bo->placements;
212}
213
214int amdgpu_bo_create_restricted(struct amdgpu_device *adev,
215 unsigned long size, int byte_align,
216 bool kernel, u32 domain, u64 flags,
217 struct sg_table *sg,
218 struct ttm_placement *placement,
219 struct amdgpu_bo **bo_ptr)
191{ 220{
192 struct amdgpu_bo *bo; 221 struct amdgpu_bo *bo;
193 enum ttm_bo_type type; 222 enum ttm_bo_type type;
@@ -241,7 +270,7 @@ int amdgpu_bo_create(struct amdgpu_device *adev,
241 AMDGPU_GEM_DOMAIN_OA); 270 AMDGPU_GEM_DOMAIN_OA);
242 271
243 bo->flags = flags; 272 bo->flags = flags;
244 amdgpu_ttm_placement_from_domain(bo, domain); 273 amdgpu_fill_placement_to_bo(bo, placement);
245 /* Kernel allocation are uninterruptible */ 274 /* Kernel allocation are uninterruptible */
246 down_read(&adev->pm.mclk_lock); 275 down_read(&adev->pm.mclk_lock);
247 r = ttm_bo_init(&adev->mman.bdev, &bo->tbo, size, type, 276 r = ttm_bo_init(&adev->mman.bdev, &bo->tbo, size, type,
@@ -258,6 +287,27 @@ int amdgpu_bo_create(struct amdgpu_device *adev,
258 return 0; 287 return 0;
259} 288}
260 289
290int amdgpu_bo_create(struct amdgpu_device *adev,
291 unsigned long size, int byte_align,
292 bool kernel, u32 domain, u64 flags,
293 struct sg_table *sg, struct amdgpu_bo **bo_ptr)
294{
295 struct ttm_placement placement = {0};
296 struct ttm_place placements[AMDGPU_GEM_DOMAIN_MAX + 1];
297
298 memset(&placements, 0,
299 (AMDGPU_GEM_DOMAIN_MAX + 1) * sizeof(struct ttm_place));
300
301 amdgpu_ttm_placement_init(adev, &placement,
302 placements, domain, flags);
303
304 return amdgpu_bo_create_restricted(adev, size, byte_align,
305 kernel, domain, flags,
306 sg,
307 &placement,
308 bo_ptr);
309}
310
261int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr) 311int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr)
262{ 312{
263 bool is_iomem; 313 bool is_iomem;
@@ -313,14 +363,19 @@ void amdgpu_bo_unref(struct amdgpu_bo **bo)
313 *bo = NULL; 363 *bo = NULL;
314} 364}
315 365
316int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain, u64 max_offset, 366int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain,
367 u64 min_offset, u64 max_offset,
317 u64 *gpu_addr) 368 u64 *gpu_addr)
318{ 369{
319 int r, i; 370 int r, i;
371 unsigned fpfn, lpfn;
320 372
321 if (amdgpu_ttm_tt_has_userptr(bo->tbo.ttm)) 373 if (amdgpu_ttm_tt_has_userptr(bo->tbo.ttm))
322 return -EPERM; 374 return -EPERM;
323 375
376 if (WARN_ON_ONCE(min_offset > max_offset))
377 return -EINVAL;
378
324 if (bo->pin_count) { 379 if (bo->pin_count) {
325 bo->pin_count++; 380 bo->pin_count++;
326 if (gpu_addr) 381 if (gpu_addr)
@@ -328,7 +383,6 @@ int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain, u64 max_offset,
328 383
329 if (max_offset != 0) { 384 if (max_offset != 0) {
330 u64 domain_start; 385 u64 domain_start;
331
332 if (domain == AMDGPU_GEM_DOMAIN_VRAM) 386 if (domain == AMDGPU_GEM_DOMAIN_VRAM)
333 domain_start = bo->adev->mc.vram_start; 387 domain_start = bo->adev->mc.vram_start;
334 else 388 else
@@ -343,13 +397,21 @@ int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain, u64 max_offset,
343 for (i = 0; i < bo->placement.num_placement; i++) { 397 for (i = 0; i < bo->placement.num_placement; i++) {
344 /* force to pin into visible video ram */ 398 /* force to pin into visible video ram */
345 if ((bo->placements[i].flags & TTM_PL_FLAG_VRAM) && 399 if ((bo->placements[i].flags & TTM_PL_FLAG_VRAM) &&
346 !(bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS) && 400 !(bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS) &&
347 (!max_offset || max_offset > bo->adev->mc.visible_vram_size)) 401 (!max_offset || max_offset > bo->adev->mc.visible_vram_size)) {
348 bo->placements[i].lpfn = 402 if (WARN_ON_ONCE(min_offset >
349 bo->adev->mc.visible_vram_size >> PAGE_SHIFT; 403 bo->adev->mc.visible_vram_size))
350 else 404 return -EINVAL;
351 bo->placements[i].lpfn = max_offset >> PAGE_SHIFT; 405 fpfn = min_offset >> PAGE_SHIFT;
352 406 lpfn = bo->adev->mc.visible_vram_size >> PAGE_SHIFT;
407 } else {
408 fpfn = min_offset >> PAGE_SHIFT;
409 lpfn = max_offset >> PAGE_SHIFT;
410 }
411 if (fpfn > bo->placements[i].fpfn)
412 bo->placements[i].fpfn = fpfn;
413 if (lpfn && lpfn < bo->placements[i].lpfn)
414 bo->placements[i].lpfn = lpfn;
353 bo->placements[i].flags |= TTM_PL_FLAG_NO_EVICT; 415 bo->placements[i].flags |= TTM_PL_FLAG_NO_EVICT;
354 } 416 }
355 417
@@ -370,7 +432,7 @@ int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain, u64 max_offset,
370 432
371int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain, u64 *gpu_addr) 433int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain, u64 *gpu_addr)
372{ 434{
373 return amdgpu_bo_pin_restricted(bo, domain, 0, gpu_addr); 435 return amdgpu_bo_pin_restricted(bo, domain, 0, 0, gpu_addr);
374} 436}
375 437
376int amdgpu_bo_unpin(struct amdgpu_bo *bo) 438int amdgpu_bo_unpin(struct amdgpu_bo *bo)
generated by cgit v1.2.2 (git 2.25.0) at 2025-04-27 19:36:05 -0400