aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorBen Widawsky <ben@bwidawsk.net>2013-08-14 05:38:36 -0400
committerDaniel Vetter <daniel.vetter@ffwll.ch>2013-09-04 11:34:41 -0400
commit27173f1f95db5e74ceb35fe9a2f2f348ea11bac9 (patch)
tree246e5e01a8ff70724d94261c97cd46d8dd2be93d
parenta2dc53e7dc4d22f20aecdfa10f85004de442e4d0 (diff)
drm/i915: Convert execbuf code to use vmas
In order to transition more of our code over to using a VMA instead of an <OBJ, VM> pair - we must have the vma accessible at execbuf time. Up until now, we've only had a VMA when actually binding an object. The previous patch helped handle the distinction on bound vs. unbound. This patch will help us catch leaks, and other issues before we actually shuffle a bunch of stuff around. This attempts to convert all the execbuf code to speak in vmas. Since the execbuf code is very self contained it was a nice isolated conversion. The meat of the code is about turning eb_objects into eb_vma, and then wiring up the rest of the code to use vmas instead of obj, vm pairs. Unfortunately, to do this, we must move the exec_list link from the obj structure. This list is reused in the eviction code, so we must also modify the eviction code to make this work. WARNING: This patch makes an already hotly profiled path slower. The cost is unavoidable. In reply to this mail, I will attach the extra data. v2: Release table lock early, and two a 2 phase vma lookup to avoid having to use a GFP_ATOMIC. (Chris) v3: s/obj_exec_list/obj_exec_link/ Updates to address commit 6d2b888569d366beb4be72cacfde41adee2c25e1 Author: Chris Wilson <chris@chris-wilson.co.uk> Date: Wed Aug 7 18:30:54 2013 +0100 drm/i915: List objects allocated from stolen memory in debugfs v4: Use obj = vma->obj for neatness in some places (Chris) need_reloc_mappable() should return false if ppgtt (Chris) Signed-off-by: Ben Widawsky <ben@bwidawsk.net> [danvet: Split out prep patches. Also remove a FIXME comment which is now taken care of.] Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
Diffstat
-rw-r--r--drivers/gpu/drm/i915/i915_drv.h16
-rw-r--r--drivers/gpu/drm/i915/i915_gem.c1
-rw-r--r--drivers/gpu/drm/i915/i915_gem_execbuffer.c320
3 files changed, 183 insertions, 154 deletions
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index 35874b3a86dc..b6494b24098d 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -568,6 +568,13 @@ struct i915_vma {
568 /** This vma's place in the batchbuffer or on the eviction list */ 568 /** This vma's place in the batchbuffer or on the eviction list */
569 struct list_head exec_list; 569 struct list_head exec_list;
570 570
571 /**
572 * Used for performing relocations during execbuffer insertion.
573 */
574 struct hlist_node exec_node;
575 unsigned long exec_handle;
576 struct drm_i915_gem_exec_object2 *exec_entry;
577
571}; 578};
572 579
573struct i915_ctx_hang_stats { 580struct i915_ctx_hang_stats {
@@ -1398,8 +1405,6 @@ struct drm_i915_gem_object {
1398 struct list_head ring_list; 1405 struct list_head ring_list;
1399 /** Used in execbuf to temporarily hold a ref */ 1406 /** Used in execbuf to temporarily hold a ref */
1400 struct list_head obj_exec_link; 1407 struct list_head obj_exec_link;
1401 /** This object's place in the batchbuffer or on the eviction list */
1402 struct list_head exec_list;
1403 1408
1404 /** 1409 /**
1405 * This is set if the object is on the active lists (has pending 1410 * This is set if the object is on the active lists (has pending
@@ -1485,13 +1490,6 @@ struct drm_i915_gem_object {
1485 void *dma_buf_vmapping; 1490 void *dma_buf_vmapping;
1486 int vmapping_count; 1491 int vmapping_count;
1487 1492
1488 /**
1489 * Used for performing relocations during execbuffer insertion.
1490 */
1491 struct hlist_node exec_node;
1492 unsigned long exec_handle;
1493 struct drm_i915_gem_exec_object2 *exec_entry;
1494
1495 struct intel_ring_buffer *ring; 1493 struct intel_ring_buffer *ring;
1496 1494
1497 /** Breadcrumb of last rendering to the buffer. */ 1495 /** Breadcrumb of last rendering to the buffer. */
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index f21a0c36a40b..c4c56f98d6e6 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -3978,7 +3978,6 @@ void i915_gem_object_init(struct drm_i915_gem_object *obj,
3978{ 3978{
3979 INIT_LIST_HEAD(&obj->global_list); 3979 INIT_LIST_HEAD(&obj->global_list);
3980 INIT_LIST_HEAD(&obj->ring_list); 3980 INIT_LIST_HEAD(&obj->ring_list);
3981 INIT_LIST_HEAD(&obj->exec_list);
3982 INIT_LIST_HEAD(&obj->obj_exec_link); 3981 INIT_LIST_HEAD(&obj->obj_exec_link);
3983 INIT_LIST_HEAD(&obj->vma_list); 3982 INIT_LIST_HEAD(&obj->vma_list);
3984 3983
diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
index bf345777ae9f..5dcfd4f59f87 100644
--- a/drivers/gpu/drm/i915/i915_gem_execbuffer.c
+++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
@@ -33,24 +33,24 @@
33#include "intel_drv.h" 33#include "intel_drv.h"
34#include <linux/dma_remapping.h> 34#include <linux/dma_remapping.h>
35 35
36struct eb_objects { 36struct eb_vmas {
37 struct list_head objects; 37 struct list_head vmas;
38 int and; 38 int and;
39 union { 39 union {
40 struct drm_i915_gem_object *lut[0]; 40 struct i915_vma *lut[0];
41 struct hlist_head buckets[0]; 41 struct hlist_head buckets[0];
42 }; 42 };
43}; 43};
44 44
45static struct eb_objects * 45static struct eb_vmas *
46eb_create(struct drm_i915_gem_execbuffer2 *args) 46eb_create(struct drm_i915_gem_execbuffer2 *args, struct i915_address_space *vm)
47{ 47{
48 struct eb_objects *eb = NULL; 48 struct eb_vmas *eb = NULL;
49 49
50 if (args->flags & I915_EXEC_HANDLE_LUT) { 50 if (args->flags & I915_EXEC_HANDLE_LUT) {
51 int size = args->buffer_count; 51 int size = args->buffer_count;
52 size *= sizeof(struct drm_i915_gem_object *); 52 size *= sizeof(struct i915_vma *);
53 size += sizeof(struct eb_objects); 53 size += sizeof(struct eb_vmas);
54 eb = kmalloc(size, GFP_TEMPORARY | __GFP_NOWARN | __GFP_NORETRY); 54 eb = kmalloc(size, GFP_TEMPORARY | __GFP_NOWARN | __GFP_NORETRY);
55 } 55 }
56 56
@@ -61,7 +61,7 @@ eb_create(struct drm_i915_gem_execbuffer2 *args)
61 while (count > 2*size) 61 while (count > 2*size)
62 count >>= 1; 62 count >>= 1;
63 eb = kzalloc(count*sizeof(struct hlist_head) + 63 eb = kzalloc(count*sizeof(struct hlist_head) +
64 sizeof(struct eb_objects), 64 sizeof(struct eb_vmas),
65 GFP_TEMPORARY); 65 GFP_TEMPORARY);
66 if (eb == NULL) 66 if (eb == NULL)
67 return eb; 67 return eb;
@@ -70,64 +70,97 @@ eb_create(struct drm_i915_gem_execbuffer2 *args)
70 } else 70 } else
71 eb->and = -args->buffer_count; 71 eb->and = -args->buffer_count;
72 72
73 INIT_LIST_HEAD(&eb->objects); 73 INIT_LIST_HEAD(&eb->vmas);
74 return eb; 74 return eb;
75} 75}
76 76
77static void 77static void
78eb_reset(struct eb_objects *eb) 78eb_reset(struct eb_vmas *eb)
79{ 79{
80 if (eb->and >= 0) 80 if (eb->and >= 0)
81 memset(eb->buckets, 0, (eb->and+1)*sizeof(struct hlist_head)); 81 memset(eb->buckets, 0, (eb->and+1)*sizeof(struct hlist_head));
82} 82}
83 83
84static int 84static int
85eb_lookup_objects(struct eb_objects *eb, 85eb_lookup_vmas(struct eb_vmas *eb,
86 struct drm_i915_gem_exec_object2 *exec, 86 struct drm_i915_gem_exec_object2 *exec,
87 const struct drm_i915_gem_execbuffer2 *args, 87 const struct drm_i915_gem_execbuffer2 *args,
88 struct drm_file *file) 88 struct i915_address_space *vm,
89 struct drm_file *file)
89{ 90{
90 int i; 91 struct drm_i915_gem_object *obj;
92 struct list_head objects;
93 int i, ret = 0;
91 94
95 INIT_LIST_HEAD(&objects);
92 spin_lock(&file->table_lock); 96 spin_lock(&file->table_lock);
97 /* Grab a reference to the object and release the lock so we can lookup
98 * or create the VMA without using GFP_ATOMIC */
93 for (i = 0; i < args->buffer_count; i++) { 99 for (i = 0; i < args->buffer_count; i++) {
94 struct drm_i915_gem_object *obj;
95
96 obj = to_intel_bo(idr_find(&file->object_idr, exec[i].handle)); 100 obj = to_intel_bo(idr_find(&file->object_idr, exec[i].handle));
97 if (obj == NULL) { 101 if (obj == NULL) {
98 spin_unlock(&file->table_lock); 102 spin_unlock(&file->table_lock);
99 DRM_DEBUG("Invalid object handle %d at index %d\n", 103 DRM_DEBUG("Invalid object handle %d at index %d\n",
100 exec[i].handle, i); 104 exec[i].handle, i);
101 return -ENOENT; 105 ret = -ENOENT;
106 goto out;
102 } 107 }
103 108
104 if (!list_empty(&obj->exec_list)) { 109 if (!list_empty(&obj->obj_exec_link)) {
105 spin_unlock(&file->table_lock); 110 spin_unlock(&file->table_lock);
106 DRM_DEBUG("Object %p [handle %d, index %d] appears more than once in object list\n", 111 DRM_DEBUG("Object %p [handle %d, index %d] appears more than once in object list\n",
107 obj, exec[i].handle, i); 112 obj, exec[i].handle, i);
108 return -EINVAL; 113 ret = -EINVAL;
114 goto out;
109 } 115 }
110 116
111 drm_gem_object_reference(&obj->base); 117 drm_gem_object_reference(&obj->base);
112 list_add_tail(&obj->exec_list, &eb->objects); 118 list_add_tail(&obj->obj_exec_link, &objects);
119 }
120 spin_unlock(&file->table_lock);
113 121
114 obj->exec_entry = &exec[i]; 122 i = 0;
123 list_for_each_entry(obj, &objects, obj_exec_link) {
124 struct i915_vma *vma;
125
126 vma = i915_gem_obj_lookup_or_create_vma(obj, vm);
127 if (IS_ERR(vma)) {
128 /* XXX: We don't need an error path fro vma because if
129 * the vma was created just for this execbuf, object
130 * unreference should kill it off.*/
131 DRM_DEBUG("Failed to lookup VMA\n");
132 ret = PTR_ERR(vma);
133 goto out;
134 }
135
136 list_add_tail(&vma->exec_list, &eb->vmas);
137
138 vma->exec_entry = &exec[i];
115 if (eb->and < 0) { 139 if (eb->and < 0) {
116 eb->lut[i] = obj; 140 eb->lut[i] = vma;
117 } else { 141 } else {
118 uint32_t handle = args->flags & I915_EXEC_HANDLE_LUT ? i : exec[i].handle; 142 uint32_t handle = args->flags & I915_EXEC_HANDLE_LUT ? i : exec[i].handle;
119 obj->exec_handle = handle; 143 vma->exec_handle = handle;
120 hlist_add_head(&obj->exec_node, 144 hlist_add_head(&vma->exec_node,
121 &eb->buckets[handle & eb->and]); 145 &eb->buckets[handle & eb->and]);
122 } 146 }
147 ++i;
123 } 148 }
124 spin_unlock(&file->table_lock);
125 149
126 return 0; 150
151out:
152 while (!list_empty(&objects)) {
153 obj = list_first_entry(&objects,
154 struct drm_i915_gem_object,
155 obj_exec_link);
156 list_del_init(&obj->obj_exec_link);
157 if (ret)
158 drm_gem_object_unreference(&obj->base);
159 }
160 return ret;
127} 161}
128 162
129static struct drm_i915_gem_object * 163static struct i915_vma *eb_get_vma(struct eb_vmas *eb, unsigned long handle)
130eb_get_object(struct eb_objects *eb, unsigned long handle)
131{ 164{
132 if (eb->and < 0) { 165 if (eb->and < 0) {
133 if (handle >= -eb->and) 166 if (handle >= -eb->and)
@@ -139,27 +172,25 @@ eb_get_object(struct eb_objects *eb, unsigned long handle)
139 172
140 head = &eb->buckets[handle & eb->and]; 173 head = &eb->buckets[handle & eb->and];
141 hlist_for_each(node, head) { 174 hlist_for_each(node, head) {
142 struct drm_i915_gem_object *obj; 175 struct i915_vma *vma;
143 176
144 obj = hlist_entry(node, struct drm_i915_gem_object, exec_node); 177 vma = hlist_entry(node, struct i915_vma, exec_node);
145 if (obj->exec_handle == handle) 178 if (vma->exec_handle == handle)
146 return obj; 179 return vma;
147 } 180 }
148 return NULL; 181 return NULL;
149 } 182 }
150} 183}
151 184
152static void 185static void eb_destroy(struct eb_vmas *eb) {
153eb_destroy(struct eb_objects *eb) 186 while (!list_empty(&eb->vmas)) {
154{ 187 struct i915_vma *vma;
155 while (!list_empty(&eb->objects)) {
156 struct drm_i915_gem_object *obj;
157 188
158 obj = list_first_entry(&eb->objects, 189 vma = list_first_entry(&eb->vmas,
159 struct drm_i915_gem_object, 190 struct i915_vma,
160 exec_list); 191 exec_list);
161 list_del_init(&obj->exec_list); 192 list_del_init(&vma->exec_list);
162 drm_gem_object_unreference(&obj->base); 193 drm_gem_object_unreference(&vma->obj->base);
163 } 194 }
164 kfree(eb); 195 kfree(eb);
165} 196}
@@ -223,22 +254,24 @@ relocate_entry_gtt(struct drm_i915_gem_object *obj,
223 254
224static int 255static int
225i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj, 256i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
226 struct eb_objects *eb, 257 struct eb_vmas *eb,
227 struct drm_i915_gem_relocation_entry *reloc, 258 struct drm_i915_gem_relocation_entry *reloc,
228 struct i915_address_space *vm) 259 struct i915_address_space *vm)
229{ 260{
230 struct drm_device *dev = obj->base.dev; 261 struct drm_device *dev = obj->base.dev;
231 struct drm_gem_object *target_obj; 262 struct drm_gem_object *target_obj;
232 struct drm_i915_gem_object *target_i915_obj; 263 struct drm_i915_gem_object *target_i915_obj;
264 struct i915_vma *target_vma;
233 uint32_t target_offset; 265 uint32_t target_offset;
234 int ret = -EINVAL; 266 int ret = -EINVAL;
235 267
236 /* we've already hold a reference to all valid objects */ 268 /* we've already hold a reference to all valid objects */
237 target_obj = &eb_get_object(eb, reloc->target_handle)->base; 269 target_vma = eb_get_vma(eb, reloc->target_handle);
238 if (unlikely(target_obj == NULL)) 270 if (unlikely(target_vma == NULL))
239 return -ENOENT; 271 return -ENOENT;
272 target_i915_obj = target_vma->obj;
273 target_obj = &target_vma->obj->base;
240 274
241 target_i915_obj = to_intel_bo(target_obj);
242 target_offset = i915_gem_obj_ggtt_offset(target_i915_obj); 275 target_offset = i915_gem_obj_ggtt_offset(target_i915_obj);
243 276
244 /* Sandybridge PPGTT errata: We need a global gtt mapping for MI and 277 /* Sandybridge PPGTT errata: We need a global gtt mapping for MI and
@@ -320,14 +353,13 @@ i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
320} 353}
321 354
322static int 355static int
323i915_gem_execbuffer_relocate_object(struct drm_i915_gem_object *obj, 356i915_gem_execbuffer_relocate_vma(struct i915_vma *vma,
324 struct eb_objects *eb, 357 struct eb_vmas *eb)
325 struct i915_address_space *vm)
326{ 358{
327#define N_RELOC(x) ((x) / sizeof(struct drm_i915_gem_relocation_entry)) 359#define N_RELOC(x) ((x) / sizeof(struct drm_i915_gem_relocation_entry))
328 struct drm_i915_gem_relocation_entry stack_reloc[N_RELOC(512)]; 360 struct drm_i915_gem_relocation_entry stack_reloc[N_RELOC(512)];
329 struct drm_i915_gem_relocation_entry __user *user_relocs; 361 struct drm_i915_gem_relocation_entry __user *user_relocs;
330 struct drm_i915_gem_exec_object2 *entry = obj->exec_entry; 362 struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
331 int remain, ret; 363 int remain, ret;
332 364
333 user_relocs = to_user_ptr(entry->relocs_ptr); 365 user_relocs = to_user_ptr(entry->relocs_ptr);
@@ -346,8 +378,8 @@ i915_gem_execbuffer_relocate_object(struct drm_i915_gem_object *obj,
346 do { 378 do {
347 u64 offset = r->presumed_offset; 379 u64 offset = r->presumed_offset;
348 380
349 ret = i915_gem_execbuffer_relocate_entry(obj, eb, r, 381 ret = i915_gem_execbuffer_relocate_entry(vma->obj, eb, r,
350 vm); 382 vma->vm);
351 if (ret) 383 if (ret)
352 return ret; 384 return ret;
353 385
@@ -368,17 +400,16 @@ i915_gem_execbuffer_relocate_object(struct drm_i915_gem_object *obj,
368} 400}
369 401
370static int 402static int
371i915_gem_execbuffer_relocate_object_slow(struct drm_i915_gem_object *obj, 403i915_gem_execbuffer_relocate_vma_slow(struct i915_vma *vma,
372 struct eb_objects *eb, 404 struct eb_vmas *eb,
373 struct drm_i915_gem_relocation_entry *relocs, 405 struct drm_i915_gem_relocation_entry *relocs)
374 struct i915_address_space *vm)
375{ 406{
376 const struct drm_i915_gem_exec_object2 *entry = obj->exec_entry; 407 const struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
377 int i, ret; 408 int i, ret;
378 409
379 for (i = 0; i < entry->relocation_count; i++) { 410 for (i = 0; i < entry->relocation_count; i++) {
380 ret = i915_gem_execbuffer_relocate_entry(obj, eb, &relocs[i], 411 ret = i915_gem_execbuffer_relocate_entry(vma->obj, eb, &relocs[i],
381 vm); 412 vma->vm);
382 if (ret) 413 if (ret)
383 return ret; 414 return ret;
384 } 415 }
@@ -387,10 +418,10 @@ i915_gem_execbuffer_relocate_object_slow(struct drm_i915_gem_object *obj,
387} 418}
388 419
389static int 420static int
390i915_gem_execbuffer_relocate(struct eb_objects *eb, 421i915_gem_execbuffer_relocate(struct eb_vmas *eb,
391 struct i915_address_space *vm) 422 struct i915_address_space *vm)
392{ 423{
393 struct drm_i915_gem_object *obj; 424 struct i915_vma *vma;
394 int ret = 0; 425 int ret = 0;
395 426
396 /* This is the fast path and we cannot handle a pagefault whilst 427 /* This is the fast path and we cannot handle a pagefault whilst
@@ -401,8 +432,8 @@ i915_gem_execbuffer_relocate(struct eb_objects *eb,
401 * lockdep complains vehemently. 432 * lockdep complains vehemently.
402 */ 433 */
403 pagefault_disable(); 434 pagefault_disable();
404 list_for_each_entry(obj, &eb->objects, exec_list) { 435 list_for_each_entry(vma, &eb->vmas, exec_list) {
405 ret = i915_gem_execbuffer_relocate_object(obj, eb, vm); 436 ret = i915_gem_execbuffer_relocate_vma(vma, eb);
406 if (ret) 437 if (ret)
407 break; 438 break;
408 } 439 }
@@ -415,31 +446,32 @@ i915_gem_execbuffer_relocate(struct eb_objects *eb,
415#define __EXEC_OBJECT_HAS_FENCE (1<<30) 446#define __EXEC_OBJECT_HAS_FENCE (1<<30)
416 447
417static int 448static int
418need_reloc_mappable(struct drm_i915_gem_object *obj) 449need_reloc_mappable(struct i915_vma *vma)
419{ 450{
420 struct drm_i915_gem_exec_object2 *entry = obj->exec_entry; 451 struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
421 return entry->relocation_count && !use_cpu_reloc(obj); 452 return entry->relocation_count && !use_cpu_reloc(vma->obj) &&
453 i915_is_ggtt(vma->vm);
422} 454}
423 455
424static int 456static int
425i915_gem_execbuffer_reserve_object(struct drm_i915_gem_object *obj, 457i915_gem_execbuffer_reserve_vma(struct i915_vma *vma,
426 struct intel_ring_buffer *ring, 458 struct intel_ring_buffer *ring,
427 struct i915_address_space *vm, 459 bool *need_reloc)
428 bool *need_reloc)
429{ 460{
430 struct drm_i915_private *dev_priv = obj->base.dev->dev_private; 461 struct drm_i915_private *dev_priv = ring->dev->dev_private;
431 struct drm_i915_gem_exec_object2 *entry = obj->exec_entry; 462 struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
432 bool has_fenced_gpu_access = INTEL_INFO(ring->dev)->gen < 4; 463 bool has_fenced_gpu_access = INTEL_INFO(ring->dev)->gen < 4;
433 bool need_fence, need_mappable; 464 bool need_fence, need_mappable;
465 struct drm_i915_gem_object *obj = vma->obj;
434 int ret; 466 int ret;
435 467
436 need_fence = 468 need_fence =
437 has_fenced_gpu_access && 469 has_fenced_gpu_access &&
438 entry->flags & EXEC_OBJECT_NEEDS_FENCE && 470 entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
439 obj->tiling_mode != I915_TILING_NONE; 471 obj->tiling_mode != I915_TILING_NONE;
440 need_mappable = need_fence || need_reloc_mappable(obj); 472 need_mappable = need_fence || need_reloc_mappable(vma);
441 473
442 ret = i915_gem_object_pin(obj, vm, entry->alignment, need_mappable, 474 ret = i915_gem_object_pin(obj, vma->vm, entry->alignment, need_mappable,
443 false); 475 false);
444 if (ret) 476 if (ret)
445 return ret; 477 return ret;
@@ -467,8 +499,8 @@ i915_gem_execbuffer_reserve_object(struct drm_i915_gem_object *obj,
467 obj->has_aliasing_ppgtt_mapping = 1; 499 obj->has_aliasing_ppgtt_mapping = 1;
468 } 500 }
469 501
470 if (entry->offset != i915_gem_obj_offset(obj, vm)) { 502 if (entry->offset != vma->node.start) {
471 entry->offset = i915_gem_obj_offset(obj, vm); 503 entry->offset = vma->node.start;
472 *need_reloc = true; 504 *need_reloc = true;
473 } 505 }
474 506
@@ -485,14 +517,15 @@ i915_gem_execbuffer_reserve_object(struct drm_i915_gem_object *obj,
485} 517}
486 518
487static void 519static void
488i915_gem_execbuffer_unreserve_object(struct drm_i915_gem_object *obj) 520i915_gem_execbuffer_unreserve_vma(struct i915_vma *vma)
489{ 521{
490 struct drm_i915_gem_exec_object2 *entry; 522 struct drm_i915_gem_exec_object2 *entry;
523 struct drm_i915_gem_object *obj = vma->obj;
491 524
492 if (!i915_gem_obj_bound_any(obj)) 525 if (!drm_mm_node_allocated(&vma->node))
493 return; 526 return;
494 527
495 entry = obj->exec_entry; 528 entry = vma->exec_entry;
496 529
497 if (entry->flags & __EXEC_OBJECT_HAS_FENCE) 530 if (entry->flags & __EXEC_OBJECT_HAS_FENCE)
498 i915_gem_object_unpin_fence(obj); 531 i915_gem_object_unpin_fence(obj);
@@ -505,41 +538,40 @@ i915_gem_execbuffer_unreserve_object(struct drm_i915_gem_object *obj)
505 538
506static int 539static int
507i915_gem_execbuffer_reserve(struct intel_ring_buffer *ring, 540i915_gem_execbuffer_reserve(struct intel_ring_buffer *ring,
508 struct list_head *objects, 541 struct list_head *vmas,
509 struct i915_address_space *vm,
510 bool *need_relocs) 542 bool *need_relocs)
511{ 543{
512 struct drm_i915_gem_object *obj; 544 struct drm_i915_gem_object *obj;
513 struct list_head ordered_objects; 545 struct i915_vma *vma;
546 struct list_head ordered_vmas;
514 bool has_fenced_gpu_access = INTEL_INFO(ring->dev)->gen < 4; 547 bool has_fenced_gpu_access = INTEL_INFO(ring->dev)->gen < 4;
515 int retry; 548 int retry;
516 549
517 INIT_LIST_HEAD(&ordered_objects); 550 INIT_LIST_HEAD(&ordered_vmas);
518 while (!list_empty(objects)) { 551 while (!list_empty(vmas)) {
519 struct drm_i915_gem_exec_object2 *entry; 552 struct drm_i915_gem_exec_object2 *entry;
520 bool need_fence, need_mappable; 553 bool need_fence, need_mappable;
521 554
522 obj = list_first_entry(objects, 555 vma = list_first_entry(vmas, struct i915_vma, exec_list);
523 struct drm_i915_gem_object, 556 obj = vma->obj;
524 exec_list); 557 entry = vma->exec_entry;
525 entry = obj->exec_entry;
526 558
527 need_fence = 559 need_fence =
528 has_fenced_gpu_access && 560 has_fenced_gpu_access &&
529 entry->flags & EXEC_OBJECT_NEEDS_FENCE && 561 entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
530 obj->tiling_mode != I915_TILING_NONE; 562 obj->tiling_mode != I915_TILING_NONE;
531 need_mappable = need_fence || need_reloc_mappable(obj); 563 need_mappable = need_fence || need_reloc_mappable(vma);
532 564
533 if (need_mappable) 565 if (need_mappable)
534 list_move(&obj->exec_list, &ordered_objects); 566 list_move(&vma->exec_list, &ordered_vmas);
535 else 567 else
536 list_move_tail(&obj->exec_list, &ordered_objects); 568 list_move_tail(&vma->exec_list, &ordered_vmas);
537 569
538 obj->base.pending_read_domains = I915_GEM_GPU_DOMAINS & ~I915_GEM_DOMAIN_COMMAND; 570 obj->base.pending_read_domains = I915_GEM_GPU_DOMAINS & ~I915_GEM_DOMAIN_COMMAND;
539 obj->base.pending_write_domain = 0; 571 obj->base.pending_write_domain = 0;
540 obj->pending_fenced_gpu_access = false; 572 obj->pending_fenced_gpu_access = false;
541 } 573 }
542 list_splice(&ordered_objects, objects); 574 list_splice(&ordered_vmas, vmas);
543 575
544 /* Attempt to pin all of the buffers into the GTT. 576 /* Attempt to pin all of the buffers into the GTT.
545 * This is done in 3 phases: 577 * This is done in 3 phases:
@@ -558,47 +590,47 @@ i915_gem_execbuffer_reserve(struct intel_ring_buffer *ring,
558 int ret = 0; 590 int ret = 0;
559 591
560 /* Unbind any ill-fitting objects or pin. */ 592 /* Unbind any ill-fitting objects or pin. */
561 list_for_each_entry(obj, objects, exec_list) { 593 list_for_each_entry(vma, vmas, exec_list) {
562 struct drm_i915_gem_exec_object2 *entry = obj->exec_entry; 594 struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
563 bool need_fence, need_mappable; 595 bool need_fence, need_mappable;
564 u32 obj_offset;
565 596
566 if (!i915_gem_obj_bound(obj, vm)) 597 obj = vma->obj;
598
599 if (!drm_mm_node_allocated(&vma->node))
567 continue; 600 continue;
568 601
569 obj_offset = i915_gem_obj_offset(obj, vm);
570 need_fence = 602 need_fence =
571 has_fenced_gpu_access && 603 has_fenced_gpu_access &&
572 entry->flags & EXEC_OBJECT_NEEDS_FENCE && 604 entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
573 obj->tiling_mode != I915_TILING_NONE; 605 obj->tiling_mode != I915_TILING_NONE;
574 need_mappable = need_fence || need_reloc_mappable(obj); 606 need_mappable = need_fence || need_reloc_mappable(vma);
575 607
576 WARN_ON((need_mappable || need_fence) && 608 WARN_ON((need_mappable || need_fence) &&
577 !i915_is_ggtt(vm)); 609 !i915_is_ggtt(vma->vm));
578 610
579 if ((entry->alignment && 611 if ((entry->alignment &&
580 obj_offset & (entry->alignment - 1)) || 612 vma->node.start & (entry->alignment - 1)) ||
581 (need_mappable && !obj->map_and_fenceable)) 613 (need_mappable && !obj->map_and_fenceable))
582 ret = i915_vma_unbind(i915_gem_obj_to_vma(obj, vm)); 614 ret = i915_vma_unbind(vma);
583 else 615 else
584 ret = i915_gem_execbuffer_reserve_object(obj, ring, vm, need_relocs); 616 ret = i915_gem_execbuffer_reserve_vma(vma, ring, need_relocs);
585 if (ret) 617 if (ret)
586 goto err; 618 goto err;
587 } 619 }
588 620
589 /* Bind fresh objects */ 621 /* Bind fresh objects */
590 list_for_each_entry(obj, objects, exec_list) { 622 list_for_each_entry(vma, vmas, exec_list) {
591 if (i915_gem_obj_bound(obj, vm)) 623 if (drm_mm_node_allocated(&vma->node))
592 continue; 624 continue;
593 625
594 ret = i915_gem_execbuffer_reserve_object(obj, ring, vm, need_relocs); 626 ret = i915_gem_execbuffer_reserve_vma(vma, ring, need_relocs);
595 if (ret) 627 if (ret)
596 goto err; 628 goto err;
597 } 629 }
598 630
599err: /* Decrement pin count for bound objects */ 631err: /* Decrement pin count for bound objects */
600 list_for_each_entry(obj, objects, exec_list) 632 list_for_each_entry(vma, vmas, exec_list)
601 i915_gem_execbuffer_unreserve_object(obj); 633 i915_gem_execbuffer_unreserve_vma(vma);
602 634
603 if (ret != -ENOSPC || retry++) 635 if (ret != -ENOSPC || retry++)
604 return ret; 636 return ret;
@@ -614,24 +646,27 @@ i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
614 struct drm_i915_gem_execbuffer2 *args, 646 struct drm_i915_gem_execbuffer2 *args,
615 struct drm_file *file, 647 struct drm_file *file,
616 struct intel_ring_buffer *ring, 648 struct intel_ring_buffer *ring,
617 struct eb_objects *eb, 649 struct eb_vmas *eb,
618 struct drm_i915_gem_exec_object2 *exec, 650 struct drm_i915_gem_exec_object2 *exec)
619 struct i915_address_space *vm)
620{ 651{
621 struct drm_i915_gem_relocation_entry *reloc; 652 struct drm_i915_gem_relocation_entry *reloc;
622 struct drm_i915_gem_object *obj; 653 struct i915_address_space *vm;
654 struct i915_vma *vma;
623 bool need_relocs; 655 bool need_relocs;
624 int *reloc_offset; 656 int *reloc_offset;
625 int i, total, ret; 657 int i, total, ret;
626 int count = args->buffer_count; 658 int count = args->buffer_count;
627 659
660 if (WARN_ON(list_empty(&eb->vmas)))
661 return 0;
662
663 vm = list_first_entry(&eb->vmas, struct i915_vma, exec_list)->vm;
664
628 /* We may process another execbuffer during the unlock... */ 665 /* We may process another execbuffer during the unlock... */
629 while (!list_empty(&eb->objects)) { 666 while (!list_empty(&eb->vmas)) {
630 obj = list_first_entry(&eb->objects, 667 vma = list_first_entry(&eb->vmas, struct i915_vma, exec_list);
631 struct drm_i915_gem_object, 668 list_del_init(&vma->exec_list);
632 exec_list); 669 drm_gem_object_unreference(&vma->obj->base);
633 list_del_init(&obj->exec_list);
634 drm_gem_object_unreference(&obj->base);
635 } 670 }
636 671
637 mutex_unlock(&dev->struct_mutex); 672 mutex_unlock(&dev->struct_mutex);
@@ -695,20 +730,19 @@ i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
695 730
696 /* reacquire the objects */ 731 /* reacquire the objects */
697 eb_reset(eb); 732 eb_reset(eb);
698 ret = eb_lookup_objects(eb, exec, args, file); 733 ret = eb_lookup_vmas(eb, exec, args, vm, file);
699 if (ret) 734 if (ret)
700 goto err; 735 goto err;
701 736
702 need_relocs = (args->flags & I915_EXEC_NO_RELOC) == 0; 737 need_relocs = (args->flags & I915_EXEC_NO_RELOC) == 0;
703 ret = i915_gem_execbuffer_reserve(ring, &eb->objects, vm, &need_relocs); 738 ret = i915_gem_execbuffer_reserve(ring, &eb->vmas, &need_relocs);
704 if (ret) 739 if (ret)
705 goto err; 740 goto err;
706 741
707 list_for_each_entry(obj, &eb->objects, exec_list) { 742 list_for_each_entry(vma, &eb->vmas, exec_list) {
708 int offset = obj->exec_entry - exec; 743 int offset = vma->exec_entry - exec;
709 ret = i915_gem_execbuffer_relocate_object_slow(obj, eb, 744 ret = i915_gem_execbuffer_relocate_vma_slow(vma, eb,
710 reloc + reloc_offset[offset], 745 reloc + reloc_offset[offset]);
711 vm);
712 if (ret) 746 if (ret)
713 goto err; 747 goto err;
714 } 748 }
@@ -727,14 +761,15 @@ err:
727 761
728static int 762static int
729i915_gem_execbuffer_move_to_gpu(struct intel_ring_buffer *ring, 763i915_gem_execbuffer_move_to_gpu(struct intel_ring_buffer *ring,
730 struct list_head *objects) 764 struct list_head *vmas)
731{ 765{
732 struct drm_i915_gem_object *obj; 766 struct i915_vma *vma;
733 uint32_t flush_domains = 0; 767 uint32_t flush_domains = 0;
734 bool flush_chipset = false; 768 bool flush_chipset = false;
735 int ret; 769 int ret;
736 770
737 list_for_each_entry(obj, objects, exec_list) { 771 list_for_each_entry(vma, vmas, exec_list) {
772 struct drm_i915_gem_object *obj = vma->obj;
738 ret = i915_gem_object_sync(obj, ring); 773 ret = i915_gem_object_sync(obj, ring);
739 if (ret) 774 if (ret)
740 return ret; 775 return ret;
@@ -809,13 +844,13 @@ validate_exec_list(struct drm_i915_gem_exec_object2 *exec,
809} 844}
810 845
811static void 846static void
812i915_gem_execbuffer_move_to_active(struct list_head *objects, 847i915_gem_execbuffer_move_to_active(struct list_head *vmas,
813 struct i915_address_space *vm,
814 struct intel_ring_buffer *ring) 848 struct intel_ring_buffer *ring)
815{ 849{
816 struct drm_i915_gem_object *obj; 850 struct i915_vma *vma;
817 851
818 list_for_each_entry(obj, objects, exec_list) { 852 list_for_each_entry(vma, vmas, exec_list) {
853 struct drm_i915_gem_object *obj = vma->obj;
819 u32 old_read = obj->base.read_domains; 854 u32 old_read = obj->base.read_domains;
820 u32 old_write = obj->base.write_domain; 855 u32 old_write = obj->base.write_domain;
821 856
@@ -825,8 +860,7 @@ i915_gem_execbuffer_move_to_active(struct list_head *objects,
825 obj->base.read_domains = obj->base.pending_read_domains; 860 obj->base.read_domains = obj->base.pending_read_domains;
826 obj->fenced_gpu_access = obj->pending_fenced_gpu_access; 861 obj->fenced_gpu_access = obj->pending_fenced_gpu_access;
827 862
828 /* FIXME: This lookup gets fixed later <-- danvet */ 863 list_move_tail(&vma->mm_list, &vma->vm->active_list);
829 list_move_tail(&i915_gem_obj_to_vma(obj, vm)->mm_list, &vm->active_list);
830 i915_gem_object_move_to_active(obj, ring); 864 i915_gem_object_move_to_active(obj, ring);
831 if (obj->base.write_domain) { 865 if (obj->base.write_domain) {
832 obj->dirty = 1; 866 obj->dirty = 1;
@@ -885,7 +919,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
885 struct i915_address_space *vm) 919 struct i915_address_space *vm)
886{ 920{
887 drm_i915_private_t *dev_priv = dev->dev_private; 921 drm_i915_private_t *dev_priv = dev->dev_private;
888 struct eb_objects *eb; 922 struct eb_vmas *eb;
889 struct drm_i915_gem_object *batch_obj; 923 struct drm_i915_gem_object *batch_obj;
890 struct drm_clip_rect *cliprects = NULL; 924 struct drm_clip_rect *cliprects = NULL;
891 struct intel_ring_buffer *ring; 925 struct intel_ring_buffer *ring;
@@ -1025,7 +1059,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
1025 goto pre_mutex_err; 1059 goto pre_mutex_err;
1026 } 1060 }
1027 1061
1028 eb = eb_create(args); 1062 eb = eb_create(args, vm);
1029 if (eb == NULL) { 1063 if (eb == NULL) {
1030 mutex_unlock(&dev->struct_mutex); 1064 mutex_unlock(&dev->struct_mutex);
1031 ret = -ENOMEM; 1065 ret = -ENOMEM;
@@ -1033,18 +1067,16 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
1033 } 1067 }
1034 1068
1035 /* Look up object handles */ 1069 /* Look up object handles */
1036 ret = eb_lookup_objects(eb, exec, args, file); 1070 ret = eb_lookup_vmas(eb, exec, args, vm, file);
1037 if (ret) 1071 if (ret)
1038 goto err; 1072 goto err;
1039 1073
1040 /* take note of the batch buffer before we might reorder the lists */ 1074 /* take note of the batch buffer before we might reorder the lists */
1041 batch_obj = list_entry(eb->objects.prev, 1075 batch_obj = list_entry(eb->vmas.prev, struct i915_vma, exec_list)->obj;
1042 struct drm_i915_gem_object,
1043 exec_list);
1044 1076
1045 /* Move the objects en-masse into the GTT, evicting if necessary. */ 1077 /* Move the objects en-masse into the GTT, evicting if necessary. */
1046 need_relocs = (args->flags & I915_EXEC_NO_RELOC) == 0; 1078 need_relocs = (args->flags & I915_EXEC_NO_RELOC) == 0;
1047 ret = i915_gem_execbuffer_reserve(ring, &eb->objects, vm, &need_relocs); 1079 ret = i915_gem_execbuffer_reserve(ring, &eb->vmas, &need_relocs);
1048 if (ret) 1080 if (ret)
1049 goto err; 1081 goto err;
1050 1082
@@ -1054,7 +1086,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
1054 if (ret) { 1086 if (ret) {
1055 if (ret == -EFAULT) { 1087 if (ret == -EFAULT) {
1056 ret = i915_gem_execbuffer_relocate_slow(dev, args, file, ring, 1088 ret = i915_gem_execbuffer_relocate_slow(dev, args, file, ring,
1057 eb, exec, vm); 1089 eb, exec);
1058 BUG_ON(!mutex_is_locked(&dev->struct_mutex)); 1090 BUG_ON(!mutex_is_locked(&dev->struct_mutex));
1059 } 1091 }
1060 if (ret) 1092 if (ret)
@@ -1076,7 +1108,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
1076 if (flags & I915_DISPATCH_SECURE && !batch_obj->has_global_gtt_mapping) 1108 if (flags & I915_DISPATCH_SECURE && !batch_obj->has_global_gtt_mapping)
1077 i915_gem_gtt_bind_object(batch_obj, batch_obj->cache_level); 1109 i915_gem_gtt_bind_object(batch_obj, batch_obj->cache_level);
1078 1110
1079 ret = i915_gem_execbuffer_move_to_gpu(ring, &eb->objects); 1111 ret = i915_gem_execbuffer_move_to_gpu(ring, &eb->vmas);
1080 if (ret) 1112 if (ret)
1081 goto err; 1113 goto err;
1082 1114
@@ -1131,7 +1163,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
1131 1163
1132 trace_i915_gem_ring_dispatch(ring, intel_ring_get_seqno(ring), flags); 1164 trace_i915_gem_ring_dispatch(ring, intel_ring_get_seqno(ring), flags);
1133 1165
1134 i915_gem_execbuffer_move_to_active(&eb->objects, vm, ring); 1166 i915_gem_execbuffer_move_to_active(&eb->vmas, ring);
1135 i915_gem_execbuffer_retire_commands(dev, file, ring, batch_obj); 1167 i915_gem_execbuffer_retire_commands(dev, file, ring, batch_obj);
1136 1168
1137err: 1169err:
generated by cgit v1.2.2 (git 2.25.0) at 2025-08-15 18:25:00 -0400