aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorChris Wilson <chris@chris-wilson.co.uk>2017-06-15 04:14:33 -0400
committerChris Wilson <chris@chris-wilson.co.uk>2017-06-15 05:50:35 -0400
commit650bc63568e4218508f206c14af92b5a3f77504f (patch)
treebb79bd075f2ba7f73b112a562e4e1fe1b5884264
parent28c7ef9ecca5984cbb416497f300826a870d9cf3 (diff)
drm/i915: Amalgamate execbuffer parameter structures
Combine the two slightly overlapping parameter structures we pass around the execbuffer routines into one. Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk> Reviewed-by: Joonas Lahtinen <joonas.lahtinen@linux.intel.com> Link: http://patchwork.freedesktop.org/patch/msgid/20170615081435.17699-1-chris@chris-wilson.co.uk
-rw-r--r--drivers/gpu/drm/i915/i915_gem_execbuffer.c549
1 files changed, 232 insertions, 317 deletions
diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
index 04211c970b9f..1c5a6a63a767 100644
--- a/drivers/gpu/drm/i915/i915_gem_execbuffer.c
+++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
@@ -50,70 +50,77 @@
50 50
51#define BATCH_OFFSET_BIAS (256*1024) 51#define BATCH_OFFSET_BIAS (256*1024)
52 52
53struct i915_execbuffer_params { 53#define __I915_EXEC_ILLEGAL_FLAGS \
54 struct drm_device *dev; 54 (__I915_EXEC_UNKNOWN_FLAGS | I915_EXEC_CONSTANTS_MASK)
55 struct drm_file *file;
56 struct i915_vma *batch;
57 u32 dispatch_flags;
58 u32 args_batch_start_offset;
59 struct intel_engine_cs *engine;
60 struct i915_gem_context *ctx;
61 struct drm_i915_gem_request *request;
62};
63 55
64struct eb_vmas { 56struct i915_execbuffer {
65 struct drm_i915_private *i915; 57 struct drm_i915_private *i915;
58 struct drm_file *file;
59 struct drm_i915_gem_execbuffer2 *args;
60 struct drm_i915_gem_exec_object2 *exec;
61 struct intel_engine_cs *engine;
62 struct i915_gem_context *ctx;
63 struct i915_address_space *vm;
64 struct i915_vma *batch;
65 struct drm_i915_gem_request *request;
66 u32 batch_start_offset;
67 u32 batch_len;
68 unsigned int dispatch_flags;
69 struct drm_i915_gem_exec_object2 shadow_exec_entry;
70 bool need_relocs;
66 struct list_head vmas; 71 struct list_head vmas;
72 struct reloc_cache {
73 struct drm_mm_node node;
74 unsigned long vaddr;
75 unsigned int page;
76 bool use_64bit_reloc : 1;
77 } reloc_cache;
67 int and; 78 int and;
68 union { 79 union {
69 struct i915_vma *lut[0]; 80 struct i915_vma **lut;
70 struct hlist_head buckets[0]; 81 struct hlist_head *buckets;
71 }; 82 };
72}; 83};
73 84
74static struct eb_vmas * 85static int eb_create(struct i915_execbuffer *eb)
75eb_create(struct drm_i915_private *i915,
76 struct drm_i915_gem_execbuffer2 *args)
77{ 86{
78 struct eb_vmas *eb = NULL; 87 eb->lut = NULL;
79 88 if (eb->args->flags & I915_EXEC_HANDLE_LUT) {
80 if (args->flags & I915_EXEC_HANDLE_LUT) { 89 unsigned int size = eb->args->buffer_count;
81 unsigned size = args->buffer_count;
82 size *= sizeof(struct i915_vma *); 90 size *= sizeof(struct i915_vma *);
83 size += sizeof(struct eb_vmas); 91 eb->lut = kmalloc(size,
84 eb = kmalloc(size, GFP_TEMPORARY | __GFP_NOWARN | __GFP_NORETRY); 92 GFP_TEMPORARY | __GFP_NOWARN | __GFP_NORETRY);
85 } 93 }
86 94
87 if (eb == NULL) { 95 if (!eb->lut) {
88 unsigned size = args->buffer_count; 96 unsigned int size = eb->args->buffer_count;
89 unsigned count = PAGE_SIZE / sizeof(struct hlist_head) / 2; 97 unsigned int count = PAGE_SIZE / sizeof(struct hlist_head) / 2;
90 BUILD_BUG_ON_NOT_POWER_OF_2(PAGE_SIZE / sizeof(struct hlist_head)); 98 BUILD_BUG_ON_NOT_POWER_OF_2(PAGE_SIZE / sizeof(struct hlist_head));
91 while (count > 2*size) 99 while (count > 2*size)
92 count >>= 1; 100 count >>= 1;
93 eb = kzalloc(count*sizeof(struct hlist_head) + 101 eb->lut = kzalloc(count * sizeof(struct hlist_head),
94 sizeof(struct eb_vmas), 102 GFP_TEMPORARY);
95 GFP_TEMPORARY); 103 if (!eb->lut)
96 if (eb == NULL) 104 return -ENOMEM;
97 return eb;
98 105
99 eb->and = count - 1; 106 eb->and = count - 1;
100 } else 107 } else {
101 eb->and = -args->buffer_count; 108 eb->and = -eb->args->buffer_count;
109 }
102 110
103 eb->i915 = i915;
104 INIT_LIST_HEAD(&eb->vmas); 111 INIT_LIST_HEAD(&eb->vmas);
105 return eb; 112 return 0;
106} 113}
107 114
108static void 115static void
109eb_reset(struct eb_vmas *eb) 116eb_reset(struct i915_execbuffer *eb)
110{ 117{
111 if (eb->and >= 0) 118 if (eb->and >= 0)
112 memset(eb->buckets, 0, (eb->and+1)*sizeof(struct hlist_head)); 119 memset(eb->buckets, 0, (eb->and+1)*sizeof(struct hlist_head));
113} 120}
114 121
115static struct i915_vma * 122static struct i915_vma *
116eb_get_batch(struct eb_vmas *eb) 123eb_get_batch(struct i915_execbuffer *eb)
117{ 124{
118 struct i915_vma *vma = list_entry(eb->vmas.prev, typeof(*vma), exec_list); 125 struct i915_vma *vma = list_entry(eb->vmas.prev, typeof(*vma), exec_list);
119 126
@@ -133,34 +140,30 @@ eb_get_batch(struct eb_vmas *eb)
133} 140}
134 141
135static int 142static int
136eb_lookup_vmas(struct eb_vmas *eb, 143eb_lookup_vmas(struct i915_execbuffer *eb)
137 struct drm_i915_gem_exec_object2 *exec,
138 const struct drm_i915_gem_execbuffer2 *args,
139 struct i915_address_space *vm,
140 struct drm_file *file)
141{ 144{
142 struct drm_i915_gem_object *obj; 145 struct drm_i915_gem_object *obj;
143 struct list_head objects; 146 struct list_head objects;
144 int i, ret; 147 int i, ret;
145 148
146 INIT_LIST_HEAD(&objects); 149 INIT_LIST_HEAD(&objects);
147 spin_lock(&file->table_lock); 150 spin_lock(&eb->file->table_lock);
148 /* Grab a reference to the object and release the lock so we can lookup 151 /* Grab a reference to the object and release the lock so we can lookup
149 * or create the VMA without using GFP_ATOMIC */ 152 * or create the VMA without using GFP_ATOMIC */
150 for (i = 0; i < args->buffer_count; i++) { 153 for (i = 0; i < eb->args->buffer_count; i++) {
151 obj = to_intel_bo(idr_find(&file->object_idr, exec[i].handle)); 154 obj = to_intel_bo(idr_find(&eb->file->object_idr, eb->exec[i].handle));
152 if (obj == NULL) { 155 if (obj == NULL) {
153 spin_unlock(&file->table_lock); 156 spin_unlock(&eb->file->table_lock);
154 DRM_DEBUG("Invalid object handle %d at index %d\n", 157 DRM_DEBUG("Invalid object handle %d at index %d\n",
155 exec[i].handle, i); 158 eb->exec[i].handle, i);
156 ret = -ENOENT; 159 ret = -ENOENT;
157 goto err; 160 goto err;
158 } 161 }
159 162
160 if (!list_empty(&obj->obj_exec_link)) { 163 if (!list_empty(&obj->obj_exec_link)) {
161 spin_unlock(&file->table_lock); 164 spin_unlock(&eb->file->table_lock);
162 DRM_DEBUG("Object %p [handle %d, index %d] appears more than once in object list\n", 165 DRM_DEBUG("Object %p [handle %d, index %d] appears more than once in object list\n",
163 obj, exec[i].handle, i); 166 obj, eb->exec[i].handle, i);
164 ret = -EINVAL; 167 ret = -EINVAL;
165 goto err; 168 goto err;
166 } 169 }
@@ -168,7 +171,7 @@ eb_lookup_vmas(struct eb_vmas *eb,
168 i915_gem_object_get(obj); 171 i915_gem_object_get(obj);
169 list_add_tail(&obj->obj_exec_link, &objects); 172 list_add_tail(&obj->obj_exec_link, &objects);
170 } 173 }
171 spin_unlock(&file->table_lock); 174 spin_unlock(&eb->file->table_lock);
172 175
173 i = 0; 176 i = 0;
174 while (!list_empty(&objects)) { 177 while (!list_empty(&objects)) {
@@ -186,7 +189,7 @@ eb_lookup_vmas(struct eb_vmas *eb,
186 * from the (obj, vm) we don't run the risk of creating 189 * from the (obj, vm) we don't run the risk of creating
187 * duplicated vmas for the same vm. 190 * duplicated vmas for the same vm.
188 */ 191 */
189 vma = i915_vma_instance(obj, vm, NULL); 192 vma = i915_vma_instance(obj, eb->vm, NULL);
190 if (unlikely(IS_ERR(vma))) { 193 if (unlikely(IS_ERR(vma))) {
191 DRM_DEBUG("Failed to lookup VMA\n"); 194 DRM_DEBUG("Failed to lookup VMA\n");
192 ret = PTR_ERR(vma); 195 ret = PTR_ERR(vma);
@@ -197,11 +200,13 @@ eb_lookup_vmas(struct eb_vmas *eb,
197 list_add_tail(&vma->exec_list, &eb->vmas); 200 list_add_tail(&vma->exec_list, &eb->vmas);
198 list_del_init(&obj->obj_exec_link); 201 list_del_init(&obj->obj_exec_link);
199 202
200 vma->exec_entry = &exec[i]; 203 vma->exec_entry = &eb->exec[i];
201 if (eb->and < 0) { 204 if (eb->and < 0) {
202 eb->lut[i] = vma; 205 eb->lut[i] = vma;
203 } else { 206 } else {
204 uint32_t handle = args->flags & I915_EXEC_HANDLE_LUT ? i : exec[i].handle; 207 u32 handle =
208 eb->args->flags & I915_EXEC_HANDLE_LUT ?
209 i : eb->exec[i].handle;
205 vma->exec_handle = handle; 210 vma->exec_handle = handle;
206 hlist_add_head(&vma->exec_node, 211 hlist_add_head(&vma->exec_node,
207 &eb->buckets[handle & eb->and]); 212 &eb->buckets[handle & eb->and]);
@@ -228,7 +233,7 @@ err:
228 return ret; 233 return ret;
229} 234}
230 235
231static struct i915_vma *eb_get_vma(struct eb_vmas *eb, unsigned long handle) 236static struct i915_vma *eb_get_vma(struct i915_execbuffer *eb, unsigned long handle)
232{ 237{
233 if (eb->and < 0) { 238 if (eb->and < 0) {
234 if (handle >= -eb->and) 239 if (handle >= -eb->and)
@@ -248,7 +253,7 @@ static struct i915_vma *eb_get_vma(struct eb_vmas *eb, unsigned long handle)
248} 253}
249 254
250static void 255static void
251i915_gem_execbuffer_unreserve_vma(struct i915_vma *vma) 256eb_unreserve_vma(struct i915_vma *vma)
252{ 257{
253 struct drm_i915_gem_exec_object2 *entry; 258 struct drm_i915_gem_exec_object2 *entry;
254 259
@@ -266,8 +271,10 @@ i915_gem_execbuffer_unreserve_vma(struct i915_vma *vma)
266 entry->flags &= ~(__EXEC_OBJECT_HAS_FENCE | __EXEC_OBJECT_HAS_PIN); 271 entry->flags &= ~(__EXEC_OBJECT_HAS_FENCE | __EXEC_OBJECT_HAS_PIN);
267} 272}
268 273
269static void eb_destroy(struct eb_vmas *eb) 274static void eb_destroy(struct i915_execbuffer *eb)
270{ 275{
276 i915_gem_context_put(eb->ctx);
277
271 while (!list_empty(&eb->vmas)) { 278 while (!list_empty(&eb->vmas)) {
272 struct i915_vma *vma; 279 struct i915_vma *vma;
273 280
@@ -275,11 +282,10 @@ static void eb_destroy(struct eb_vmas *eb)
275 struct i915_vma, 282 struct i915_vma,
276 exec_list); 283 exec_list);
277 list_del_init(&vma->exec_list); 284 list_del_init(&vma->exec_list);
278 i915_gem_execbuffer_unreserve_vma(vma); 285 eb_unreserve_vma(vma);
279 vma->exec_entry = NULL; 286 vma->exec_entry = NULL;
280 i915_vma_put(vma); 287 i915_vma_put(vma);
281 } 288 }
282 kfree(eb);
283} 289}
284 290
285static inline int use_cpu_reloc(struct drm_i915_gem_object *obj) 291static inline int use_cpu_reloc(struct drm_i915_gem_object *obj)
@@ -320,20 +326,11 @@ relocation_target(const struct drm_i915_gem_relocation_entry *reloc,
320 return gen8_canonical_addr((int)reloc->delta + target_offset); 326 return gen8_canonical_addr((int)reloc->delta + target_offset);
321} 327}
322 328
323struct reloc_cache {
324 struct drm_i915_private *i915;
325 struct drm_mm_node node;
326 unsigned long vaddr;
327 unsigned int page;
328 bool use_64bit_reloc;
329};
330
331static void reloc_cache_init(struct reloc_cache *cache, 329static void reloc_cache_init(struct reloc_cache *cache,
332 struct drm_i915_private *i915) 330 struct drm_i915_private *i915)
333{ 331{
334 cache->page = -1; 332 cache->page = -1;
335 cache->vaddr = 0; 333 cache->vaddr = 0;
336 cache->i915 = i915;
337 /* Must be a variable in the struct to allow GCC to unroll. */ 334 /* Must be a variable in the struct to allow GCC to unroll. */
338 cache->use_64bit_reloc = HAS_64BIT_RELOC(i915); 335 cache->use_64bit_reloc = HAS_64BIT_RELOC(i915);
339 cache->node.allocated = false; 336 cache->node.allocated = false;
@@ -351,7 +348,14 @@ static inline unsigned int unmask_flags(unsigned long p)
351 348
352#define KMAP 0x4 /* after CLFLUSH_FLAGS */ 349#define KMAP 0x4 /* after CLFLUSH_FLAGS */
353 350
354static void reloc_cache_fini(struct reloc_cache *cache) 351static inline struct i915_ggtt *cache_to_ggtt(struct reloc_cache *cache)
352{
353 struct drm_i915_private *i915 =
354 container_of(cache, struct i915_execbuffer, reloc_cache)->i915;
355 return &i915->ggtt;
356}
357
358static void reloc_cache_reset(struct reloc_cache *cache)
355{ 359{
356 void *vaddr; 360 void *vaddr;
357 361
@@ -369,7 +373,7 @@ static void reloc_cache_fini(struct reloc_cache *cache)
369 wmb(); 373 wmb();
370 io_mapping_unmap_atomic((void __iomem *)vaddr); 374 io_mapping_unmap_atomic((void __iomem *)vaddr);
371 if (cache->node.allocated) { 375 if (cache->node.allocated) {
372 struct i915_ggtt *ggtt = &cache->i915->ggtt; 376 struct i915_ggtt *ggtt = cache_to_ggtt(cache);
373 377
374 ggtt->base.clear_range(&ggtt->base, 378 ggtt->base.clear_range(&ggtt->base,
375 cache->node.start, 379 cache->node.start,
@@ -379,6 +383,9 @@ static void reloc_cache_fini(struct reloc_cache *cache)
379 i915_vma_unpin((struct i915_vma *)cache->node.mm); 383 i915_vma_unpin((struct i915_vma *)cache->node.mm);
380 } 384 }
381 } 385 }
386
387 cache->vaddr = 0;
388 cache->page = -1;
382} 389}
383 390
384static void *reloc_kmap(struct drm_i915_gem_object *obj, 391static void *reloc_kmap(struct drm_i915_gem_object *obj,
@@ -417,7 +424,7 @@ static void *reloc_iomap(struct drm_i915_gem_object *obj,
417 struct reloc_cache *cache, 424 struct reloc_cache *cache,
418 int page) 425 int page)
419{ 426{
420 struct i915_ggtt *ggtt = &cache->i915->ggtt; 427 struct i915_ggtt *ggtt = cache_to_ggtt(cache);
421 unsigned long offset; 428 unsigned long offset;
422 void *vaddr; 429 void *vaddr;
423 430
@@ -467,7 +474,8 @@ static void *reloc_iomap(struct drm_i915_gem_object *obj,
467 offset += page << PAGE_SHIFT; 474 offset += page << PAGE_SHIFT;
468 } 475 }
469 476
470 vaddr = (void __force *) io_mapping_map_atomic_wc(&cache->i915->ggtt.mappable, offset); 477 vaddr = (void __force *)io_mapping_map_atomic_wc(&ggtt->mappable,
478 offset);
471 cache->page = page; 479 cache->page = page;
472 cache->vaddr = (unsigned long)vaddr; 480 cache->vaddr = (unsigned long)vaddr;
473 481
@@ -546,12 +554,10 @@ repeat:
546} 554}
547 555
548static int 556static int
549i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj, 557eb_relocate_entry(struct drm_i915_gem_object *obj,
550 struct eb_vmas *eb, 558 struct i915_execbuffer *eb,
551 struct drm_i915_gem_relocation_entry *reloc, 559 struct drm_i915_gem_relocation_entry *reloc)
552 struct reloc_cache *cache)
553{ 560{
554 struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
555 struct drm_gem_object *target_obj; 561 struct drm_gem_object *target_obj;
556 struct drm_i915_gem_object *target_i915_obj; 562 struct drm_i915_gem_object *target_i915_obj;
557 struct i915_vma *target_vma; 563 struct i915_vma *target_vma;
@@ -570,8 +576,8 @@ i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
570 /* Sandybridge PPGTT errata: We need a global gtt mapping for MI and 576 /* Sandybridge PPGTT errata: We need a global gtt mapping for MI and
571 * pipe_control writes because the gpu doesn't properly redirect them 577 * pipe_control writes because the gpu doesn't properly redirect them
572 * through the ppgtt for non_secure batchbuffers. */ 578 * through the ppgtt for non_secure batchbuffers. */
573 if (unlikely(IS_GEN6(dev_priv) && 579 if (unlikely(IS_GEN6(eb->i915) &&
574 reloc->write_domain == I915_GEM_DOMAIN_INSTRUCTION)) { 580 reloc->write_domain == I915_GEM_DOMAIN_INSTRUCTION)) {
575 ret = i915_vma_bind(target_vma, target_i915_obj->cache_level, 581 ret = i915_vma_bind(target_vma, target_i915_obj->cache_level,
576 PIN_GLOBAL); 582 PIN_GLOBAL);
577 if (WARN_ONCE(ret, "Unexpected failure to bind target VMA!")) 583 if (WARN_ONCE(ret, "Unexpected failure to bind target VMA!"))
@@ -612,7 +618,7 @@ i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
612 618
613 /* Check that the relocation address is valid... */ 619 /* Check that the relocation address is valid... */
614 if (unlikely(reloc->offset > 620 if (unlikely(reloc->offset >
615 obj->base.size - (cache->use_64bit_reloc ? 8 : 4))) { 621 obj->base.size - (eb->reloc_cache.use_64bit_reloc ? 8 : 4))) {
616 DRM_DEBUG("Relocation beyond object bounds: " 622 DRM_DEBUG("Relocation beyond object bounds: "
617 "obj %p target %d offset %d size %d.\n", 623 "obj %p target %d offset %d size %d.\n",
618 obj, reloc->target_handle, 624 obj, reloc->target_handle,
@@ -628,7 +634,7 @@ i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
628 return -EINVAL; 634 return -EINVAL;
629 } 635 }
630 636
631 ret = relocate_entry(obj, reloc, cache, target_offset); 637 ret = relocate_entry(obj, reloc, &eb->reloc_cache, target_offset);
632 if (ret) 638 if (ret)
633 return ret; 639 return ret;
634 640
@@ -637,19 +643,15 @@ i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
637 return 0; 643 return 0;
638} 644}
639 645
640static int 646static int eb_relocate_vma(struct i915_vma *vma, struct i915_execbuffer *eb)
641i915_gem_execbuffer_relocate_vma(struct i915_vma *vma,
642 struct eb_vmas *eb)
643{ 647{
644#define N_RELOC(x) ((x) / sizeof(struct drm_i915_gem_relocation_entry)) 648#define N_RELOC(x) ((x) / sizeof(struct drm_i915_gem_relocation_entry))
645 struct drm_i915_gem_relocation_entry stack_reloc[N_RELOC(512)]; 649 struct drm_i915_gem_relocation_entry stack_reloc[N_RELOC(512)];
646 struct drm_i915_gem_relocation_entry __user *user_relocs; 650 struct drm_i915_gem_relocation_entry __user *user_relocs;
647 struct drm_i915_gem_exec_object2 *entry = vma->exec_entry; 651 struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
648 struct reloc_cache cache;
649 int remain, ret = 0; 652 int remain, ret = 0;
650 653
651 user_relocs = u64_to_user_ptr(entry->relocs_ptr); 654 user_relocs = u64_to_user_ptr(entry->relocs_ptr);
652 reloc_cache_init(&cache, eb->i915);
653 655
654 remain = entry->relocation_count; 656 remain = entry->relocation_count;
655 while (remain) { 657 while (remain) {
@@ -678,7 +680,7 @@ i915_gem_execbuffer_relocate_vma(struct i915_vma *vma,
678 do { 680 do {
679 u64 offset = r->presumed_offset; 681 u64 offset = r->presumed_offset;
680 682
681 ret = i915_gem_execbuffer_relocate_entry(vma->obj, eb, r, &cache); 683 ret = eb_relocate_entry(vma->obj, eb, r);
682 if (ret) 684 if (ret)
683 goto out; 685 goto out;
684 686
@@ -710,39 +712,35 @@ i915_gem_execbuffer_relocate_vma(struct i915_vma *vma,
710 } 712 }
711 713
712out: 714out:
713 reloc_cache_fini(&cache); 715 reloc_cache_reset(&eb->reloc_cache);
714 return ret; 716 return ret;
715#undef N_RELOC 717#undef N_RELOC
716} 718}
717 719
718static int 720static int
719i915_gem_execbuffer_relocate_vma_slow(struct i915_vma *vma, 721eb_relocate_vma_slow(struct i915_vma *vma,
720 struct eb_vmas *eb, 722 struct i915_execbuffer *eb,
721 struct drm_i915_gem_relocation_entry *relocs) 723 struct drm_i915_gem_relocation_entry *relocs)
722{ 724{
723 const struct drm_i915_gem_exec_object2 *entry = vma->exec_entry; 725 const struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
724 struct reloc_cache cache;
725 int i, ret = 0; 726 int i, ret = 0;
726 727
727 reloc_cache_init(&cache, eb->i915);
728 for (i = 0; i < entry->relocation_count; i++) { 728 for (i = 0; i < entry->relocation_count; i++) {
729 ret = i915_gem_execbuffer_relocate_entry(vma->obj, eb, &relocs[i], &cache); 729 ret = eb_relocate_entry(vma->obj, eb, &relocs[i]);
730 if (ret) 730 if (ret)
731 break; 731 break;
732 } 732 }
733 reloc_cache_fini(&cache); 733 reloc_cache_reset(&eb->reloc_cache);
734
735 return ret; 734 return ret;
736} 735}
737 736
738static int 737static int eb_relocate(struct i915_execbuffer *eb)
739i915_gem_execbuffer_relocate(struct eb_vmas *eb)
740{ 738{
741 struct i915_vma *vma; 739 struct i915_vma *vma;
742 int ret = 0; 740 int ret = 0;
743 741
744 list_for_each_entry(vma, &eb->vmas, exec_list) { 742 list_for_each_entry(vma, &eb->vmas, exec_list) {
745 ret = i915_gem_execbuffer_relocate_vma(vma, eb); 743 ret = eb_relocate_vma(vma, eb);
746 if (ret) 744 if (ret)
747 break; 745 break;
748 } 746 }
@@ -757,9 +755,9 @@ static bool only_mappable_for_reloc(unsigned int flags)
757} 755}
758 756
759static int 757static int
760i915_gem_execbuffer_reserve_vma(struct i915_vma *vma, 758eb_reserve_vma(struct i915_vma *vma,
761 struct intel_engine_cs *engine, 759 struct intel_engine_cs *engine,
762 bool *need_reloc) 760 bool *need_reloc)
763{ 761{
764 struct drm_i915_gem_object *obj = vma->obj; 762 struct drm_i915_gem_object *obj = vma->obj;
765 struct drm_i915_gem_exec_object2 *entry = vma->exec_entry; 763 struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
@@ -878,34 +876,27 @@ eb_vma_misplaced(struct i915_vma *vma)
878 return false; 876 return false;
879} 877}
880 878
881static int 879static int eb_reserve(struct i915_execbuffer *eb)
882i915_gem_execbuffer_reserve(struct intel_engine_cs *engine,
883 struct list_head *vmas,
884 struct i915_gem_context *ctx,
885 bool *need_relocs)
886{ 880{
881 const bool has_fenced_gpu_access = INTEL_GEN(eb->i915) < 4;
882 const bool needs_unfenced_map = INTEL_INFO(eb->i915)->unfenced_needs_alignment;
887 struct drm_i915_gem_object *obj; 883 struct drm_i915_gem_object *obj;
888 struct i915_vma *vma; 884 struct i915_vma *vma;
889 struct i915_address_space *vm;
890 struct list_head ordered_vmas; 885 struct list_head ordered_vmas;
891 struct list_head pinned_vmas; 886 struct list_head pinned_vmas;
892 bool has_fenced_gpu_access = INTEL_GEN(engine->i915) < 4;
893 bool needs_unfenced_map = INTEL_INFO(engine->i915)->unfenced_needs_alignment;
894 int retry; 887 int retry;
895 888
896 vm = list_first_entry(vmas, struct i915_vma, exec_list)->vm;
897
898 INIT_LIST_HEAD(&ordered_vmas); 889 INIT_LIST_HEAD(&ordered_vmas);
899 INIT_LIST_HEAD(&pinned_vmas); 890 INIT_LIST_HEAD(&pinned_vmas);
900 while (!list_empty(vmas)) { 891 while (!list_empty(&eb->vmas)) {
901 struct drm_i915_gem_exec_object2 *entry; 892 struct drm_i915_gem_exec_object2 *entry;
902 bool need_fence, need_mappable; 893 bool need_fence, need_mappable;
903 894
904 vma = list_first_entry(vmas, struct i915_vma, exec_list); 895 vma = list_first_entry(&eb->vmas, struct i915_vma, exec_list);
905 obj = vma->obj; 896 obj = vma->obj;
906 entry = vma->exec_entry; 897 entry = vma->exec_entry;
907 898
908 if (ctx->flags & CONTEXT_NO_ZEROMAP) 899 if (eb->ctx->flags & CONTEXT_NO_ZEROMAP)
909 entry->flags |= __EXEC_OBJECT_NEEDS_BIAS; 900 entry->flags |= __EXEC_OBJECT_NEEDS_BIAS;
910 901
911 if (!has_fenced_gpu_access) 902 if (!has_fenced_gpu_access)
@@ -927,8 +918,8 @@ i915_gem_execbuffer_reserve(struct intel_engine_cs *engine,
927 obj->base.pending_read_domains = I915_GEM_GPU_DOMAINS & ~I915_GEM_DOMAIN_COMMAND; 918 obj->base.pending_read_domains = I915_GEM_GPU_DOMAINS & ~I915_GEM_DOMAIN_COMMAND;
928 obj->base.pending_write_domain = 0; 919 obj->base.pending_write_domain = 0;
929 } 920 }
930 list_splice(&ordered_vmas, vmas); 921 list_splice(&ordered_vmas, &eb->vmas);
931 list_splice(&pinned_vmas, vmas); 922 list_splice(&pinned_vmas, &eb->vmas);
932 923
933 /* Attempt to pin all of the buffers into the GTT. 924 /* Attempt to pin all of the buffers into the GTT.
934 * This is done in 3 phases: 925 * This is done in 3 phases:
@@ -947,27 +938,24 @@ i915_gem_execbuffer_reserve(struct intel_engine_cs *engine,
947 int ret = 0; 938 int ret = 0;
948 939
949 /* Unbind any ill-fitting objects or pin. */ 940 /* Unbind any ill-fitting objects or pin. */
950 list_for_each_entry(vma, vmas, exec_list) { 941 list_for_each_entry(vma, &eb->vmas, exec_list) {
951 if (!drm_mm_node_allocated(&vma->node)) 942 if (!drm_mm_node_allocated(&vma->node))
952 continue; 943 continue;
953 944
954 if (eb_vma_misplaced(vma)) 945 if (eb_vma_misplaced(vma))
955 ret = i915_vma_unbind(vma); 946 ret = i915_vma_unbind(vma);
956 else 947 else
957 ret = i915_gem_execbuffer_reserve_vma(vma, 948 ret = eb_reserve_vma(vma, eb->engine, &eb->need_relocs);
958 engine,
959 need_relocs);
960 if (ret) 949 if (ret)
961 goto err; 950 goto err;
962 } 951 }
963 952
964 /* Bind fresh objects */ 953 /* Bind fresh objects */
965 list_for_each_entry(vma, vmas, exec_list) { 954 list_for_each_entry(vma, &eb->vmas, exec_list) {
966 if (drm_mm_node_allocated(&vma->node)) 955 if (drm_mm_node_allocated(&vma->node))
967 continue; 956 continue;
968 957
969 ret = i915_gem_execbuffer_reserve_vma(vma, engine, 958 ret = eb_reserve_vma(vma, eb->engine, &eb->need_relocs);
970 need_relocs);
971 if (ret) 959 if (ret)
972 goto err; 960 goto err;
973 } 961 }
@@ -977,39 +965,30 @@ err:
977 return ret; 965 return ret;
978 966
979 /* Decrement pin count for bound objects */ 967 /* Decrement pin count for bound objects */
980 list_for_each_entry(vma, vmas, exec_list) 968 list_for_each_entry(vma, &eb->vmas, exec_list)
981 i915_gem_execbuffer_unreserve_vma(vma); 969 eb_unreserve_vma(vma);
982 970
983 ret = i915_gem_evict_vm(vm, true); 971 ret = i915_gem_evict_vm(eb->vm, true);
984 if (ret) 972 if (ret)
985 return ret; 973 return ret;
986 } while (1); 974 } while (1);
987} 975}
988 976
989static int 977static int
990i915_gem_execbuffer_relocate_slow(struct drm_device *dev, 978eb_relocate_slow(struct i915_execbuffer *eb)
991 struct drm_i915_gem_execbuffer2 *args,
992 struct drm_file *file,
993 struct intel_engine_cs *engine,
994 struct eb_vmas *eb,
995 struct drm_i915_gem_exec_object2 *exec,
996 struct i915_gem_context *ctx)
997{ 979{
980 const unsigned int count = eb->args->buffer_count;
981 struct drm_device *dev = &eb->i915->drm;
998 struct drm_i915_gem_relocation_entry *reloc; 982 struct drm_i915_gem_relocation_entry *reloc;
999 struct i915_address_space *vm;
1000 struct i915_vma *vma; 983 struct i915_vma *vma;
1001 bool need_relocs;
1002 int *reloc_offset; 984 int *reloc_offset;
1003 int i, total, ret; 985 int i, total, ret;
1004 unsigned count = args->buffer_count;
1005
1006 vm = list_first_entry(&eb->vmas, struct i915_vma, exec_list)->vm;
1007 986
1008 /* We may process another execbuffer during the unlock... */ 987 /* We may process another execbuffer during the unlock... */
1009 while (!list_empty(&eb->vmas)) { 988 while (!list_empty(&eb->vmas)) {
1010 vma = list_first_entry(&eb->vmas, struct i915_vma, exec_list); 989 vma = list_first_entry(&eb->vmas, struct i915_vma, exec_list);
1011 list_del_init(&vma->exec_list); 990 list_del_init(&vma->exec_list);
1012 i915_gem_execbuffer_unreserve_vma(vma); 991 eb_unreserve_vma(vma);
1013 i915_vma_put(vma); 992 i915_vma_put(vma);
1014 } 993 }
1015 994
@@ -1017,7 +996,7 @@ i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
1017 996
1018 total = 0; 997 total = 0;
1019 for (i = 0; i < count; i++) 998 for (i = 0; i < count; i++)
1020 total += exec[i].relocation_count; 999 total += eb->exec[i].relocation_count;
1021 1000
1022 reloc_offset = kvmalloc_array(count, sizeof(*reloc_offset), GFP_KERNEL); 1001 reloc_offset = kvmalloc_array(count, sizeof(*reloc_offset), GFP_KERNEL);
1023 reloc = kvmalloc_array(total, sizeof(*reloc), GFP_KERNEL); 1002 reloc = kvmalloc_array(total, sizeof(*reloc), GFP_KERNEL);
@@ -1034,10 +1013,10 @@ i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
1034 u64 invalid_offset = (u64)-1; 1013 u64 invalid_offset = (u64)-1;
1035 int j; 1014 int j;
1036 1015
1037 user_relocs = u64_to_user_ptr(exec[i].relocs_ptr); 1016 user_relocs = u64_to_user_ptr(eb->exec[i].relocs_ptr);
1038 1017
1039 if (copy_from_user(reloc+total, user_relocs, 1018 if (copy_from_user(reloc+total, user_relocs,
1040 exec[i].relocation_count * sizeof(*reloc))) { 1019 eb->exec[i].relocation_count * sizeof(*reloc))) {
1041 ret = -EFAULT; 1020 ret = -EFAULT;
1042 mutex_lock(&dev->struct_mutex); 1021 mutex_lock(&dev->struct_mutex);
1043 goto err; 1022 goto err;
@@ -1052,7 +1031,7 @@ i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
1052 * happened we would make the mistake of assuming that the 1031 * happened we would make the mistake of assuming that the
1053 * relocations were valid. 1032 * relocations were valid.
1054 */ 1033 */
1055 for (j = 0; j < exec[i].relocation_count; j++) { 1034 for (j = 0; j < eb->exec[i].relocation_count; j++) {
1056 if (__copy_to_user(&user_relocs[j].presumed_offset, 1035 if (__copy_to_user(&user_relocs[j].presumed_offset,
1057 &invalid_offset, 1036 &invalid_offset,
1058 sizeof(invalid_offset))) { 1037 sizeof(invalid_offset))) {
@@ -1063,7 +1042,7 @@ i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
1063 } 1042 }
1064 1043
1065 reloc_offset[i] = total; 1044 reloc_offset[i] = total;
1066 total += exec[i].relocation_count; 1045 total += eb->exec[i].relocation_count;
1067 } 1046 }
1068 1047
1069 ret = i915_mutex_lock_interruptible(dev); 1048 ret = i915_mutex_lock_interruptible(dev);
@@ -1074,20 +1053,18 @@ i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
1074 1053
1075 /* reacquire the objects */ 1054 /* reacquire the objects */
1076 eb_reset(eb); 1055 eb_reset(eb);
1077 ret = eb_lookup_vmas(eb, exec, args, vm, file); 1056 ret = eb_lookup_vmas(eb);
1078 if (ret) 1057 if (ret)
1079 goto err; 1058 goto err;
1080 1059
1081 need_relocs = (args->flags & I915_EXEC_NO_RELOC) == 0; 1060 ret = eb_reserve(eb);
1082 ret = i915_gem_execbuffer_reserve(engine, &eb->vmas, ctx,
1083 &need_relocs);
1084 if (ret) 1061 if (ret)
1085 goto err; 1062 goto err;
1086 1063
1087 list_for_each_entry(vma, &eb->vmas, exec_list) { 1064 list_for_each_entry(vma, &eb->vmas, exec_list) {
1088 int offset = vma->exec_entry - exec; 1065 int idx = vma->exec_entry - eb->exec;
1089 ret = i915_gem_execbuffer_relocate_vma_slow(vma, eb, 1066
1090 reloc + reloc_offset[offset]); 1067 ret = eb_relocate_vma_slow(vma, eb, reloc + reloc_offset[idx]);
1091 if (ret) 1068 if (ret)
1092 goto err; 1069 goto err;
1093 } 1070 }
@@ -1105,13 +1082,12 @@ err:
1105} 1082}
1106 1083
1107static int 1084static int
1108i915_gem_execbuffer_move_to_gpu(struct drm_i915_gem_request *req, 1085eb_move_to_gpu(struct i915_execbuffer *eb)
1109 struct list_head *vmas)
1110{ 1086{
1111 struct i915_vma *vma; 1087 struct i915_vma *vma;
1112 int ret; 1088 int ret;
1113 1089
1114 list_for_each_entry(vma, vmas, exec_list) { 1090 list_for_each_entry(vma, &eb->vmas, exec_list) {
1115 struct drm_i915_gem_object *obj = vma->obj; 1091 struct drm_i915_gem_object *obj = vma->obj;
1116 1092
1117 if (vma->exec_entry->flags & EXEC_OBJECT_CAPTURE) { 1093 if (vma->exec_entry->flags & EXEC_OBJECT_CAPTURE) {
@@ -1121,9 +1097,9 @@ i915_gem_execbuffer_move_to_gpu(struct drm_i915_gem_request *req,
1121 if (unlikely(!capture)) 1097 if (unlikely(!capture))
1122 return -ENOMEM; 1098 return -ENOMEM;
1123 1099
1124 capture->next = req->capture_list; 1100 capture->next = eb->request->capture_list;
1125 capture->vma = vma; 1101 capture->vma = vma;
1126 req->capture_list = capture; 1102 eb->request->capture_list = capture;
1127 } 1103 }
1128 1104
1129 if (vma->exec_entry->flags & EXEC_OBJECT_ASYNC) 1105 if (vma->exec_entry->flags & EXEC_OBJECT_ASYNC)
@@ -1135,22 +1111,22 @@ i915_gem_execbuffer_move_to_gpu(struct drm_i915_gem_request *req,
1135 } 1111 }
1136 1112
1137 ret = i915_gem_request_await_object 1113 ret = i915_gem_request_await_object
1138 (req, obj, obj->base.pending_write_domain); 1114 (eb->request, obj, obj->base.pending_write_domain);
1139 if (ret) 1115 if (ret)
1140 return ret; 1116 return ret;
1141 } 1117 }
1142 1118
1143 /* Unconditionally flush any chipset caches (for streaming writes). */ 1119 /* Unconditionally flush any chipset caches (for streaming writes). */
1144 i915_gem_chipset_flush(req->engine->i915); 1120 i915_gem_chipset_flush(eb->i915);
1145 1121
1146 /* Unconditionally invalidate GPU caches and TLBs. */ 1122 /* Unconditionally invalidate GPU caches and TLBs. */
1147 return req->engine->emit_flush(req, EMIT_INVALIDATE); 1123 return eb->engine->emit_flush(eb->request, EMIT_INVALIDATE);
1148} 1124}
1149 1125
1150static bool 1126static bool
1151i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec) 1127i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec)
1152{ 1128{
1153 if (exec->flags & __I915_EXEC_UNKNOWN_FLAGS) 1129 if (exec->flags & __I915_EXEC_ILLEGAL_FLAGS)
1154 return false; 1130 return false;
1155 1131
1156 /* Kernel clipping was a DRI1 misfeature */ 1132 /* Kernel clipping was a DRI1 misfeature */
@@ -1247,22 +1223,24 @@ validate_exec_list(struct drm_device *dev,
1247 return 0; 1223 return 0;
1248} 1224}
1249 1225
1250static struct i915_gem_context * 1226static int eb_select_context(struct i915_execbuffer *eb)
1251i915_gem_validate_context(struct drm_device *dev, struct drm_file *file,
1252 struct intel_engine_cs *engine, const u32 ctx_id)
1253{ 1227{
1228 unsigned int ctx_id = i915_execbuffer2_get_context_id(*eb->args);
1254 struct i915_gem_context *ctx; 1229 struct i915_gem_context *ctx;
1255 1230
1256 ctx = i915_gem_context_lookup(file->driver_priv, ctx_id); 1231 ctx = i915_gem_context_lookup(eb->file->driver_priv, ctx_id);
1257 if (IS_ERR(ctx)) 1232 if (unlikely(IS_ERR(ctx)))
1258 return ctx; 1233 return PTR_ERR(ctx);
1259 1234
1260 if (i915_gem_context_is_banned(ctx)) { 1235 if (unlikely(i915_gem_context_is_banned(ctx))) {
1261 DRM_DEBUG("Context %u tried to submit while banned\n", ctx_id); 1236 DRM_DEBUG("Context %u tried to submit while banned\n", ctx_id);
1262 return ERR_PTR(-EIO); 1237 return -EIO;
1263 } 1238 }
1264 1239
1265 return ctx; 1240 eb->ctx = i915_gem_context_get(ctx);
1241 eb->vm = ctx->ppgtt ? &ctx->ppgtt->base : &eb->i915->ggtt.base;
1242
1243 return 0;
1266} 1244}
1267 1245
1268static bool gpu_write_needs_clflush(struct drm_i915_gem_object *obj) 1246static bool gpu_write_needs_clflush(struct drm_i915_gem_object *obj)
@@ -1327,12 +1305,11 @@ static void eb_export_fence(struct drm_i915_gem_object *obj,
1327} 1305}
1328 1306
1329static void 1307static void
1330i915_gem_execbuffer_move_to_active(struct list_head *vmas, 1308eb_move_to_active(struct i915_execbuffer *eb)
1331 struct drm_i915_gem_request *req)
1332{ 1309{
1333 struct i915_vma *vma; 1310 struct i915_vma *vma;
1334 1311
1335 list_for_each_entry(vma, vmas, exec_list) { 1312 list_for_each_entry(vma, &eb->vmas, exec_list) {
1336 struct drm_i915_gem_object *obj = vma->obj; 1313 struct drm_i915_gem_object *obj = vma->obj;
1337 1314
1338 obj->base.write_domain = obj->base.pending_write_domain; 1315 obj->base.write_domain = obj->base.pending_write_domain;
@@ -1342,8 +1319,8 @@ i915_gem_execbuffer_move_to_active(struct list_head *vmas,
1342 obj->base.pending_read_domains |= obj->base.read_domains; 1319 obj->base.pending_read_domains |= obj->base.read_domains;
1343 obj->base.read_domains = obj->base.pending_read_domains; 1320 obj->base.read_domains = obj->base.pending_read_domains;
1344 1321
1345 i915_vma_move_to_active(vma, req, vma->exec_entry->flags); 1322 i915_vma_move_to_active(vma, eb->request, vma->exec_entry->flags);
1346 eb_export_fence(obj, req, vma->exec_entry->flags); 1323 eb_export_fence(obj, eb->request, vma->exec_entry->flags);
1347 } 1324 }
1348} 1325}
1349 1326
@@ -1373,29 +1350,22 @@ i915_reset_gen7_sol_offsets(struct drm_i915_gem_request *req)
1373 return 0; 1350 return 0;
1374} 1351}
1375 1352
1376static struct i915_vma * 1353static struct i915_vma *eb_parse(struct i915_execbuffer *eb, bool is_master)
1377i915_gem_execbuffer_parse(struct intel_engine_cs *engine,
1378 struct drm_i915_gem_exec_object2 *shadow_exec_entry,
1379 struct drm_i915_gem_object *batch_obj,
1380 struct eb_vmas *eb,
1381 u32 batch_start_offset,
1382 u32 batch_len,
1383 bool is_master)
1384{ 1354{
1385 struct drm_i915_gem_object *shadow_batch_obj; 1355 struct drm_i915_gem_object *shadow_batch_obj;
1386 struct i915_vma *vma; 1356 struct i915_vma *vma;
1387 int ret; 1357 int ret;
1388 1358
1389 shadow_batch_obj = i915_gem_batch_pool_get(&engine->batch_pool, 1359 shadow_batch_obj = i915_gem_batch_pool_get(&eb->engine->batch_pool,
1390 PAGE_ALIGN(batch_len)); 1360 PAGE_ALIGN(eb->batch_len));
1391 if (IS_ERR(shadow_batch_obj)) 1361 if (IS_ERR(shadow_batch_obj))
1392 return ERR_CAST(shadow_batch_obj); 1362 return ERR_CAST(shadow_batch_obj);
1393 1363
1394 ret = intel_engine_cmd_parser(engine, 1364 ret = intel_engine_cmd_parser(eb->engine,
1395 batch_obj, 1365 eb->batch->obj,
1396 shadow_batch_obj, 1366 shadow_batch_obj,
1397 batch_start_offset, 1367 eb->batch_start_offset,
1398 batch_len, 1368 eb->batch_len,
1399 is_master); 1369 is_master);
1400 if (ret) { 1370 if (ret) {
1401 if (ret == -EACCES) /* unhandled chained batch */ 1371 if (ret == -EACCES) /* unhandled chained batch */
@@ -1409,9 +1379,8 @@ i915_gem_execbuffer_parse(struct intel_engine_cs *engine,
1409 if (IS_ERR(vma)) 1379 if (IS_ERR(vma))
1410 goto out; 1380 goto out;
1411 1381
1412 memset(shadow_exec_entry, 0, sizeof(*shadow_exec_entry)); 1382 vma->exec_entry =
1413 1383 memset(&eb->shadow_exec_entry, 0, sizeof(*vma->exec_entry));
1414 vma->exec_entry = shadow_exec_entry;
1415 vma->exec_entry->flags = __EXEC_OBJECT_HAS_PIN; 1384 vma->exec_entry->flags = __EXEC_OBJECT_HAS_PIN;
1416 i915_gem_object_get(shadow_batch_obj); 1385 i915_gem_object_get(shadow_batch_obj);
1417 list_add_tail(&vma->exec_list, &eb->vmas); 1386 list_add_tail(&vma->exec_list, &eb->vmas);
@@ -1430,46 +1399,33 @@ add_to_client(struct drm_i915_gem_request *req,
1430} 1399}
1431 1400
1432static int 1401static int
1433execbuf_submit(struct i915_execbuffer_params *params, 1402execbuf_submit(struct i915_execbuffer *eb)
1434 struct drm_i915_gem_execbuffer2 *args,
1435 struct list_head *vmas)
1436{ 1403{
1437 u64 exec_start, exec_len;
1438 int ret; 1404 int ret;
1439 1405
1440 ret = i915_gem_execbuffer_move_to_gpu(params->request, vmas); 1406 ret = eb_move_to_gpu(eb);
1441 if (ret) 1407 if (ret)
1442 return ret; 1408 return ret;
1443 1409
1444 ret = i915_switch_context(params->request); 1410 ret = i915_switch_context(eb->request);
1445 if (ret) 1411 if (ret)
1446 return ret; 1412 return ret;
1447 1413
1448 if (args->flags & I915_EXEC_CONSTANTS_MASK) { 1414 if (eb->args->flags & I915_EXEC_GEN7_SOL_RESET) {
1449 DRM_DEBUG("I915_EXEC_CONSTANTS_* unsupported\n"); 1415 ret = i915_reset_gen7_sol_offsets(eb->request);
1450 return -EINVAL;
1451 }
1452
1453 if (args->flags & I915_EXEC_GEN7_SOL_RESET) {
1454 ret = i915_reset_gen7_sol_offsets(params->request);
1455 if (ret) 1416 if (ret)
1456 return ret; 1417 return ret;
1457 } 1418 }
1458 1419
1459 exec_len = args->batch_len; 1420 ret = eb->engine->emit_bb_start(eb->request,
1460 exec_start = params->batch->node.start + 1421 eb->batch->node.start +
1461 params->args_batch_start_offset; 1422 eb->batch_start_offset,
1462 1423 eb->batch_len,
1463 if (exec_len == 0) 1424 eb->dispatch_flags);
1464 exec_len = params->batch->size - params->args_batch_start_offset;
1465
1466 ret = params->engine->emit_bb_start(params->request,
1467 exec_start, exec_len,
1468 params->dispatch_flags);
1469 if (ret) 1425 if (ret)
1470 return ret; 1426 return ret;
1471 1427
1472 i915_gem_execbuffer_move_to_active(vmas, params->request); 1428 eb_move_to_active(eb);
1473 1429
1474 return 0; 1430 return 0;
1475} 1431}
@@ -1551,27 +1507,16 @@ eb_select_engine(struct drm_i915_private *dev_priv,
1551} 1507}
1552 1508
1553static int 1509static int
1554i915_gem_do_execbuffer(struct drm_device *dev, void *data, 1510i915_gem_do_execbuffer(struct drm_device *dev,
1555 struct drm_file *file, 1511 struct drm_file *file,
1556 struct drm_i915_gem_execbuffer2 *args, 1512 struct drm_i915_gem_execbuffer2 *args,
1557 struct drm_i915_gem_exec_object2 *exec) 1513 struct drm_i915_gem_exec_object2 *exec)
1558{ 1514{
1559 struct drm_i915_private *dev_priv = to_i915(dev); 1515 struct i915_execbuffer eb;
1560 struct i915_ggtt *ggtt = &dev_priv->ggtt;
1561 struct eb_vmas *eb;
1562 struct drm_i915_gem_exec_object2 shadow_exec_entry;
1563 struct intel_engine_cs *engine;
1564 struct i915_gem_context *ctx;
1565 struct i915_address_space *vm;
1566 struct i915_execbuffer_params params_master; /* XXX: will be removed later */
1567 struct i915_execbuffer_params *params = &params_master;
1568 const u32 ctx_id = i915_execbuffer2_get_context_id(*args);
1569 u32 dispatch_flags;
1570 struct dma_fence *in_fence = NULL; 1516 struct dma_fence *in_fence = NULL;
1571 struct sync_file *out_fence = NULL; 1517 struct sync_file *out_fence = NULL;
1572 int out_fence_fd = -1; 1518 int out_fence_fd = -1;
1573 int ret; 1519 int ret;
1574 bool need_relocs;
1575 1520
1576 if (!i915_gem_check_execbuffer(args)) 1521 if (!i915_gem_check_execbuffer(args))
1577 return -EINVAL; 1522 return -EINVAL;
@@ -1580,37 +1525,42 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
1580 if (ret) 1525 if (ret)
1581 return ret; 1526 return ret;
1582 1527
1583 dispatch_flags = 0; 1528 eb.i915 = to_i915(dev);
1529 eb.file = file;
1530 eb.args = args;
1531 eb.exec = exec;
1532 eb.need_relocs = (args->flags & I915_EXEC_NO_RELOC) == 0;
1533 reloc_cache_init(&eb.reloc_cache, eb.i915);
1534
1535 eb.batch_start_offset = args->batch_start_offset;
1536 eb.batch_len = args->batch_len;
1537
1538 eb.dispatch_flags = 0;
1584 if (args->flags & I915_EXEC_SECURE) { 1539 if (args->flags & I915_EXEC_SECURE) {
1585 if (!drm_is_current_master(file) || !capable(CAP_SYS_ADMIN)) 1540 if (!drm_is_current_master(file) || !capable(CAP_SYS_ADMIN))
1586 return -EPERM; 1541 return -EPERM;
1587 1542
1588 dispatch_flags |= I915_DISPATCH_SECURE; 1543 eb.dispatch_flags |= I915_DISPATCH_SECURE;
1589 } 1544 }
1590 if (args->flags & I915_EXEC_IS_PINNED) 1545 if (args->flags & I915_EXEC_IS_PINNED)
1591 dispatch_flags |= I915_DISPATCH_PINNED; 1546 eb.dispatch_flags |= I915_DISPATCH_PINNED;
1592
1593 engine = eb_select_engine(dev_priv, file, args);
1594 if (!engine)
1595 return -EINVAL;
1596 1547
1597 if (args->buffer_count < 1) { 1548 eb.engine = eb_select_engine(eb.i915, file, args);
1598 DRM_DEBUG("execbuf with %d buffers\n", args->buffer_count); 1549 if (!eb.engine)
1599 return -EINVAL; 1550 return -EINVAL;
1600 }
1601 1551
1602 if (args->flags & I915_EXEC_RESOURCE_STREAMER) { 1552 if (args->flags & I915_EXEC_RESOURCE_STREAMER) {
1603 if (!HAS_RESOURCE_STREAMER(dev_priv)) { 1553 if (!HAS_RESOURCE_STREAMER(eb.i915)) {
1604 DRM_DEBUG("RS is only allowed for Haswell, Gen8 and above\n"); 1554 DRM_DEBUG("RS is only allowed for Haswell, Gen8 and above\n");
1605 return -EINVAL; 1555 return -EINVAL;
1606 } 1556 }
1607 if (engine->id != RCS) { 1557 if (eb.engine->id != RCS) {
1608 DRM_DEBUG("RS is not available on %s\n", 1558 DRM_DEBUG("RS is not available on %s\n",
1609 engine->name); 1559 eb.engine->name);
1610 return -EINVAL; 1560 return -EINVAL;
1611 } 1561 }
1612 1562
1613 dispatch_flags |= I915_DISPATCH_RS; 1563 eb.dispatch_flags |= I915_DISPATCH_RS;
1614 } 1564 }
1615 1565
1616 if (args->flags & I915_EXEC_FENCE_IN) { 1566 if (args->flags & I915_EXEC_FENCE_IN) {
@@ -1633,59 +1583,44 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
1633 * wakeref that we hold until the GPU has been idle for at least 1583 * wakeref that we hold until the GPU has been idle for at least
1634 * 100ms. 1584 * 100ms.
1635 */ 1585 */
1636 intel_runtime_pm_get(dev_priv); 1586 intel_runtime_pm_get(eb.i915);
1637 1587
1638 ret = i915_mutex_lock_interruptible(dev); 1588 ret = i915_mutex_lock_interruptible(dev);
1639 if (ret) 1589 if (ret)
1640 goto pre_mutex_err; 1590 goto pre_mutex_err;
1641 1591
1642 ctx = i915_gem_validate_context(dev, file, engine, ctx_id); 1592 ret = eb_select_context(&eb);
1643 if (IS_ERR(ctx)) { 1593 if (ret) {
1644 mutex_unlock(&dev->struct_mutex); 1594 mutex_unlock(&dev->struct_mutex);
1645 ret = PTR_ERR(ctx);
1646 goto pre_mutex_err; 1595 goto pre_mutex_err;
1647 } 1596 }
1648 1597
1649 i915_gem_context_get(ctx); 1598 if (eb_create(&eb)) {
1650 1599 i915_gem_context_put(eb.ctx);
1651 if (ctx->ppgtt)
1652 vm = &ctx->ppgtt->base;
1653 else
1654 vm = &ggtt->base;
1655
1656 memset(&params_master, 0x00, sizeof(params_master));
1657
1658 eb = eb_create(dev_priv, args);
1659 if (eb == NULL) {
1660 i915_gem_context_put(ctx);
1661 mutex_unlock(&dev->struct_mutex); 1600 mutex_unlock(&dev->struct_mutex);
1662 ret = -ENOMEM; 1601 ret = -ENOMEM;
1663 goto pre_mutex_err; 1602 goto pre_mutex_err;
1664 } 1603 }
1665 1604
1666 /* Look up object handles */ 1605 /* Look up object handles */
1667 ret = eb_lookup_vmas(eb, exec, args, vm, file); 1606 ret = eb_lookup_vmas(&eb);
1668 if (ret) 1607 if (ret)
1669 goto err; 1608 goto err;
1670 1609
1671 /* take note of the batch buffer before we might reorder the lists */ 1610 /* take note of the batch buffer before we might reorder the lists */
1672 params->batch = eb_get_batch(eb); 1611 eb.batch = eb_get_batch(&eb);
1673 1612
1674 /* Move the objects en-masse into the GTT, evicting if necessary. */ 1613 /* Move the objects en-masse into the GTT, evicting if necessary. */
1675 need_relocs = (args->flags & I915_EXEC_NO_RELOC) == 0; 1614 ret = eb_reserve(&eb);
1676 ret = i915_gem_execbuffer_reserve(engine, &eb->vmas, ctx,
1677 &need_relocs);
1678 if (ret) 1615 if (ret)
1679 goto err; 1616 goto err;
1680 1617
1681 /* The objects are in their final locations, apply the relocations. */ 1618 /* The objects are in their final locations, apply the relocations. */
1682 if (need_relocs) 1619 if (eb.need_relocs)
1683 ret = i915_gem_execbuffer_relocate(eb); 1620 ret = eb_relocate(&eb);
1684 if (ret) { 1621 if (ret) {
1685 if (ret == -EFAULT) { 1622 if (ret == -EFAULT) {
1686 ret = i915_gem_execbuffer_relocate_slow(dev, args, file, 1623 ret = eb_relocate_slow(&eb);
1687 engine,
1688 eb, exec, ctx);
1689 BUG_ON(!mutex_is_locked(&dev->struct_mutex)); 1624 BUG_ON(!mutex_is_locked(&dev->struct_mutex));
1690 } 1625 }
1691 if (ret) 1626 if (ret)
@@ -1693,28 +1628,22 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
1693 } 1628 }
1694 1629
1695 /* Set the pending read domains for the batch buffer to COMMAND */ 1630 /* Set the pending read domains for the batch buffer to COMMAND */
1696 if (params->batch->obj->base.pending_write_domain) { 1631 if (eb.batch->obj->base.pending_write_domain) {
1697 DRM_DEBUG("Attempting to use self-modifying batch buffer\n"); 1632 DRM_DEBUG("Attempting to use self-modifying batch buffer\n");
1698 ret = -EINVAL; 1633 ret = -EINVAL;
1699 goto err; 1634 goto err;
1700 } 1635 }
1701 if (args->batch_start_offset > params->batch->size || 1636 if (eb.batch_start_offset > eb.batch->size ||
1702 args->batch_len > params->batch->size - args->batch_start_offset) { 1637 eb.batch_len > eb.batch->size - eb.batch_start_offset) {
1703 DRM_DEBUG("Attempting to use out-of-bounds batch\n"); 1638 DRM_DEBUG("Attempting to use out-of-bounds batch\n");
1704 ret = -EINVAL; 1639 ret = -EINVAL;
1705 goto err; 1640 goto err;
1706 } 1641 }
1707 1642
1708 params->args_batch_start_offset = args->batch_start_offset; 1643 if (eb.engine->needs_cmd_parser && eb.batch_len) {
1709 if (engine->needs_cmd_parser && args->batch_len) {
1710 struct i915_vma *vma; 1644 struct i915_vma *vma;
1711 1645
1712 vma = i915_gem_execbuffer_parse(engine, &shadow_exec_entry, 1646 vma = eb_parse(&eb, drm_is_current_master(file));
1713 params->batch->obj,
1714 eb,
1715 args->batch_start_offset,
1716 args->batch_len,
1717 drm_is_current_master(file));
1718 if (IS_ERR(vma)) { 1647 if (IS_ERR(vma)) {
1719 ret = PTR_ERR(vma); 1648 ret = PTR_ERR(vma);
1720 goto err; 1649 goto err;
@@ -1730,19 +1659,21 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
1730 * specifically don't want that set on batches the 1659 * specifically don't want that set on batches the
1731 * command parser has accepted. 1660 * command parser has accepted.
1732 */ 1661 */
1733 dispatch_flags |= I915_DISPATCH_SECURE; 1662 eb.dispatch_flags |= I915_DISPATCH_SECURE;
1734 params->args_batch_start_offset = 0; 1663 eb.batch_start_offset = 0;
1735 params->batch = vma; 1664 eb.batch = vma;
1736 } 1665 }
1737 } 1666 }
1738 1667
1739 params->batch->obj->base.pending_read_domains |= I915_GEM_DOMAIN_COMMAND; 1668 eb.batch->obj->base.pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
1669 if (eb.batch_len == 0)
1670 eb.batch_len = eb.batch->size - eb.batch_start_offset;
1740 1671
1741 /* snb/ivb/vlv conflate the "batch in ppgtt" bit with the "non-secure 1672 /* snb/ivb/vlv conflate the "batch in ppgtt" bit with the "non-secure
1742 * batch" bit. Hence we need to pin secure batches into the global gtt. 1673 * batch" bit. Hence we need to pin secure batches into the global gtt.
1743 * hsw should have this fixed, but bdw mucks it up again. */ 1674 * hsw should have this fixed, but bdw mucks it up again. */
1744 if (dispatch_flags & I915_DISPATCH_SECURE) { 1675 if (eb.dispatch_flags & I915_DISPATCH_SECURE) {
1745 struct drm_i915_gem_object *obj = params->batch->obj; 1676 struct drm_i915_gem_object *obj = eb.batch->obj;
1746 struct i915_vma *vma; 1677 struct i915_vma *vma;
1747 1678
1748 /* 1679 /*
@@ -1761,25 +1692,24 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
1761 goto err; 1692 goto err;
1762 } 1693 }
1763 1694
1764 params->batch = vma; 1695 eb.batch = vma;
1765 } 1696 }
1766 1697
1767 /* Allocate a request for this batch buffer nice and early. */ 1698 /* Allocate a request for this batch buffer nice and early. */
1768 params->request = i915_gem_request_alloc(engine, ctx); 1699 eb.request = i915_gem_request_alloc(eb.engine, eb.ctx);
1769 if (IS_ERR(params->request)) { 1700 if (IS_ERR(eb.request)) {
1770 ret = PTR_ERR(params->request); 1701 ret = PTR_ERR(eb.request);
1771 goto err_batch_unpin; 1702 goto err_batch_unpin;
1772 } 1703 }
1773 1704
1774 if (in_fence) { 1705 if (in_fence) {
1775 ret = i915_gem_request_await_dma_fence(params->request, 1706 ret = i915_gem_request_await_dma_fence(eb.request, in_fence);
1776 in_fence);
1777 if (ret < 0) 1707 if (ret < 0)
1778 goto err_request; 1708 goto err_request;
1779 } 1709 }
1780 1710
1781 if (out_fence_fd != -1) { 1711 if (out_fence_fd != -1) {
1782 out_fence = sync_file_create(&params->request->fence); 1712 out_fence = sync_file_create(&eb.request->fence);
1783 if (!out_fence) { 1713 if (!out_fence) {
1784 ret = -ENOMEM; 1714 ret = -ENOMEM;
1785 goto err_request; 1715 goto err_request;
@@ -1792,26 +1722,13 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
1792 * inactive_list and lose its active reference. Hence we do not need 1722 * inactive_list and lose its active reference. Hence we do not need
1793 * to explicitly hold another reference here. 1723 * to explicitly hold another reference here.
1794 */ 1724 */
1795 params->request->batch = params->batch; 1725 eb.request->batch = eb.batch;
1796
1797 /*
1798 * Save assorted stuff away to pass through to *_submission().
1799 * NB: This data should be 'persistent' and not local as it will
1800 * kept around beyond the duration of the IOCTL once the GPU
1801 * scheduler arrives.
1802 */
1803 params->dev = dev;
1804 params->file = file;
1805 params->engine = engine;
1806 params->dispatch_flags = dispatch_flags;
1807 params->ctx = ctx;
1808 1726
1809 trace_i915_gem_request_queue(params->request, dispatch_flags); 1727 trace_i915_gem_request_queue(eb.request, eb.dispatch_flags);
1810 1728 ret = execbuf_submit(&eb);
1811 ret = execbuf_submit(params, args, &eb->vmas);
1812err_request: 1729err_request:
1813 __i915_add_request(params->request, ret == 0); 1730 __i915_add_request(eb.request, ret == 0);
1814 add_to_client(params->request, file); 1731 add_to_client(eb.request, file);
1815 1732
1816 if (out_fence) { 1733 if (out_fence) {
1817 if (ret == 0) { 1734 if (ret == 0) {
@@ -1831,19 +1748,17 @@ err_batch_unpin:
1831 * needs to be adjusted to also track the ggtt batch vma properly as 1748 * needs to be adjusted to also track the ggtt batch vma properly as
1832 * active. 1749 * active.
1833 */ 1750 */
1834 if (dispatch_flags & I915_DISPATCH_SECURE) 1751 if (eb.dispatch_flags & I915_DISPATCH_SECURE)
1835 i915_vma_unpin(params->batch); 1752 i915_vma_unpin(eb.batch);
1836err: 1753err:
1837 /* the request owns the ref now */ 1754 /* the request owns the ref now */
1838 i915_gem_context_put(ctx); 1755 eb_destroy(&eb);
1839 eb_destroy(eb);
1840
1841 mutex_unlock(&dev->struct_mutex); 1756 mutex_unlock(&dev->struct_mutex);
1842 1757
1843pre_mutex_err: 1758pre_mutex_err:
1844 /* intel_gpu_busy should also get a ref, so it will free when the device 1759 /* intel_gpu_busy should also get a ref, so it will free when the device
1845 * is really idle. */ 1760 * is really idle. */
1846 intel_runtime_pm_put(dev_priv); 1761 intel_runtime_pm_put(eb.i915);
1847 if (out_fence_fd != -1) 1762 if (out_fence_fd != -1)
1848 put_unused_fd(out_fence_fd); 1763 put_unused_fd(out_fence_fd);
1849err_in_fence: 1764err_in_fence:
@@ -1914,7 +1829,7 @@ i915_gem_execbuffer(struct drm_device *dev, void *data,
1914 exec2.flags = I915_EXEC_RENDER; 1829 exec2.flags = I915_EXEC_RENDER;
1915 i915_execbuffer2_set_context_id(exec2, 0); 1830 i915_execbuffer2_set_context_id(exec2, 0);
1916 1831
1917 ret = i915_gem_do_execbuffer(dev, data, file, &exec2, exec2_list); 1832 ret = i915_gem_do_execbuffer(dev, file, &exec2, exec2_list);
1918 if (!ret) { 1833 if (!ret) {
1919 struct drm_i915_gem_exec_object __user *user_exec_list = 1834 struct drm_i915_gem_exec_object __user *user_exec_list =
1920 u64_to_user_ptr(args->buffers_ptr); 1835 u64_to_user_ptr(args->buffers_ptr);
@@ -1973,7 +1888,7 @@ i915_gem_execbuffer2(struct drm_device *dev, void *data,
1973 return -EFAULT; 1888 return -EFAULT;
1974 } 1889 }
1975 1890
1976 ret = i915_gem_do_execbuffer(dev, data, file, args, exec2_list); 1891 ret = i915_gem_do_execbuffer(dev, file, args, exec2_list);
1977 if (!ret) { 1892 if (!ret) {
1978 /* Copy the new buffer offsets back to the user's exec list. */ 1893 /* Copy the new buffer offsets back to the user's exec list. */
1979 struct drm_i915_gem_exec_object2 __user *user_exec_list = 1894 struct drm_i915_gem_exec_object2 __user *user_exec_list =