aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/gpu/drm/i915/intel_lrc.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/gpu/drm/i915/intel_lrc.c')
-rw-r--r--drivers/gpu/drm/i915/intel_lrc.c964
1 files changed, 435 insertions, 529 deletions
diff --git a/drivers/gpu/drm/i915/intel_lrc.c b/drivers/gpu/drm/i915/intel_lrc.c
index 7f2d8415ed8b..414ddda43922 100644
--- a/drivers/gpu/drm/i915/intel_lrc.c
+++ b/drivers/gpu/drm/i915/intel_lrc.c
@@ -208,31 +208,27 @@
208} while (0) 208} while (0)
209 209
210enum { 210enum {
211 ADVANCED_CONTEXT = 0,
212 LEGACY_32B_CONTEXT,
213 ADVANCED_AD_CONTEXT,
214 LEGACY_64B_CONTEXT
215};
216#define GEN8_CTX_ADDRESSING_MODE_SHIFT 3
217#define GEN8_CTX_ADDRESSING_MODE(dev) (USES_FULL_48BIT_PPGTT(dev) ?\
218 LEGACY_64B_CONTEXT :\
219 LEGACY_32B_CONTEXT)
220enum {
221 FAULT_AND_HANG = 0, 211 FAULT_AND_HANG = 0,
222 FAULT_AND_HALT, /* Debug only */ 212 FAULT_AND_HALT, /* Debug only */
223 FAULT_AND_STREAM, 213 FAULT_AND_STREAM,
224 FAULT_AND_CONTINUE /* Unsupported */ 214 FAULT_AND_CONTINUE /* Unsupported */
225}; 215};
226#define GEN8_CTX_ID_SHIFT 32 216#define GEN8_CTX_ID_SHIFT 32
217#define GEN8_CTX_ID_WIDTH 21
227#define GEN8_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT 0x17 218#define GEN8_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT 0x17
228#define GEN9_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT 0x26 219#define GEN9_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT 0x26
229 220
230static int intel_lr_context_pin(struct intel_context *ctx, 221/* Typical size of the average request (2 pipecontrols and a MI_BB) */
222#define EXECLISTS_REQUEST_SIZE 64 /* bytes */
223
224static int execlists_context_deferred_alloc(struct i915_gem_context *ctx,
225 struct intel_engine_cs *engine);
226static int intel_lr_context_pin(struct i915_gem_context *ctx,
231 struct intel_engine_cs *engine); 227 struct intel_engine_cs *engine);
232 228
233/** 229/**
234 * intel_sanitize_enable_execlists() - sanitize i915.enable_execlists 230 * intel_sanitize_enable_execlists() - sanitize i915.enable_execlists
235 * @dev: DRM device. 231 * @dev_priv: i915 device private
236 * @enable_execlists: value of i915.enable_execlists module parameter. 232 * @enable_execlists: value of i915.enable_execlists module parameter.
237 * 233 *
238 * Only certain platforms support Execlists (the prerequisites being 234 * Only certain platforms support Execlists (the prerequisites being
@@ -240,23 +236,22 @@ static int intel_lr_context_pin(struct intel_context *ctx,
240 * 236 *
241 * Return: 1 if Execlists is supported and has to be enabled. 237 * Return: 1 if Execlists is supported and has to be enabled.
242 */ 238 */
243int intel_sanitize_enable_execlists(struct drm_device *dev, int enable_execlists) 239int intel_sanitize_enable_execlists(struct drm_i915_private *dev_priv, int enable_execlists)
244{ 240{
245 WARN_ON(i915.enable_ppgtt == -1);
246
247 /* On platforms with execlist available, vGPU will only 241 /* On platforms with execlist available, vGPU will only
248 * support execlist mode, no ring buffer mode. 242 * support execlist mode, no ring buffer mode.
249 */ 243 */
250 if (HAS_LOGICAL_RING_CONTEXTS(dev) && intel_vgpu_active(dev)) 244 if (HAS_LOGICAL_RING_CONTEXTS(dev_priv) && intel_vgpu_active(dev_priv))
251 return 1; 245 return 1;
252 246
253 if (INTEL_INFO(dev)->gen >= 9) 247 if (INTEL_GEN(dev_priv) >= 9)
254 return 1; 248 return 1;
255 249
256 if (enable_execlists == 0) 250 if (enable_execlists == 0)
257 return 0; 251 return 0;
258 252
259 if (HAS_LOGICAL_RING_CONTEXTS(dev) && USES_PPGTT(dev) && 253 if (HAS_LOGICAL_RING_CONTEXTS(dev_priv) &&
254 USES_PPGTT(dev_priv) &&
260 i915.use_mmio_flip >= 0) 255 i915.use_mmio_flip >= 0)
261 return 1; 256 return 1;
262 257
@@ -266,19 +261,17 @@ int intel_sanitize_enable_execlists(struct drm_device *dev, int enable_execlists
266static void 261static void
267logical_ring_init_platform_invariants(struct intel_engine_cs *engine) 262logical_ring_init_platform_invariants(struct intel_engine_cs *engine)
268{ 263{
269 struct drm_device *dev = engine->dev; 264 struct drm_i915_private *dev_priv = engine->i915;
270 265
271 if (IS_GEN8(dev) || IS_GEN9(dev)) 266 if (IS_GEN8(dev_priv) || IS_GEN9(dev_priv))
272 engine->idle_lite_restore_wa = ~0; 267 engine->idle_lite_restore_wa = ~0;
273 268
274 engine->disable_lite_restore_wa = (IS_SKL_REVID(dev, 0, SKL_REVID_B0) || 269 engine->disable_lite_restore_wa = (IS_SKL_REVID(dev_priv, 0, SKL_REVID_B0) ||
275 IS_BXT_REVID(dev, 0, BXT_REVID_A1)) && 270 IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) &&
276 (engine->id == VCS || engine->id == VCS2); 271 (engine->id == VCS || engine->id == VCS2);
277 272
278 engine->ctx_desc_template = GEN8_CTX_VALID; 273 engine->ctx_desc_template = GEN8_CTX_VALID;
279 engine->ctx_desc_template |= GEN8_CTX_ADDRESSING_MODE(dev) << 274 if (IS_GEN8(dev_priv))
280 GEN8_CTX_ADDRESSING_MODE_SHIFT;
281 if (IS_GEN8(dev))
282 engine->ctx_desc_template |= GEN8_CTX_L3LLC_COHERENT; 275 engine->ctx_desc_template |= GEN8_CTX_L3LLC_COHERENT;
283 engine->ctx_desc_template |= GEN8_CTX_PRIVILEGE; 276 engine->ctx_desc_template |= GEN8_CTX_PRIVILEGE;
284 277
@@ -297,7 +290,7 @@ logical_ring_init_platform_invariants(struct intel_engine_cs *engine)
297 * descriptor for a pinned context 290 * descriptor for a pinned context
298 * 291 *
299 * @ctx: Context to work on 292 * @ctx: Context to work on
300 * @ring: Engine the descriptor will be used with 293 * @engine: Engine the descriptor will be used with
301 * 294 *
302 * The context descriptor encodes various attributes of a context, 295 * The context descriptor encodes various attributes of a context,
303 * including its GTT address and some flags. Because it's fairly 296 * including its GTT address and some flags. Because it's fairly
@@ -305,62 +298,42 @@ logical_ring_init_platform_invariants(struct intel_engine_cs *engine)
305 * which remains valid until the context is unpinned. 298 * which remains valid until the context is unpinned.
306 * 299 *
307 * This is what a descriptor looks like, from LSB to MSB: 300 * This is what a descriptor looks like, from LSB to MSB:
308 * bits 0-11: flags, GEN8_CTX_* (cached in ctx_desc_template) 301 * bits 0-11: flags, GEN8_CTX_* (cached in ctx_desc_template)
309 * bits 12-31: LRCA, GTT address of (the HWSP of) this context 302 * bits 12-31: LRCA, GTT address of (the HWSP of) this context
310 * bits 32-51: ctx ID, a globally unique tag (the LRCA again!) 303 * bits 32-52: ctx ID, a globally unique tag
311 * bits 52-63: reserved, may encode the engine ID (for GuC) 304 * bits 53-54: mbz, reserved for use by hardware
305 * bits 55-63: group ID, currently unused and set to 0
312 */ 306 */
313static void 307static void
314intel_lr_context_descriptor_update(struct intel_context *ctx, 308intel_lr_context_descriptor_update(struct i915_gem_context *ctx,
315 struct intel_engine_cs *engine) 309 struct intel_engine_cs *engine)
316{ 310{
317 uint64_t lrca, desc; 311 struct intel_context *ce = &ctx->engine[engine->id];
312 u64 desc;
318 313
319 lrca = ctx->engine[engine->id].lrc_vma->node.start + 314 BUILD_BUG_ON(MAX_CONTEXT_HW_ID > (1<<GEN8_CTX_ID_WIDTH));
320 LRC_PPHWSP_PN * PAGE_SIZE;
321 315
322 desc = engine->ctx_desc_template; /* bits 0-11 */ 316 desc = ctx->desc_template; /* bits 3-4 */
323 desc |= lrca; /* bits 12-31 */ 317 desc |= engine->ctx_desc_template; /* bits 0-11 */
324 desc |= (lrca >> PAGE_SHIFT) << GEN8_CTX_ID_SHIFT; /* bits 32-51 */ 318 desc |= ce->lrc_vma->node.start + LRC_PPHWSP_PN * PAGE_SIZE;
319 /* bits 12-31 */
320 desc |= (u64)ctx->hw_id << GEN8_CTX_ID_SHIFT; /* bits 32-52 */
325 321
326 ctx->engine[engine->id].lrc_desc = desc; 322 ce->lrc_desc = desc;
327} 323}
328 324
329uint64_t intel_lr_context_descriptor(struct intel_context *ctx, 325uint64_t intel_lr_context_descriptor(struct i915_gem_context *ctx,
330 struct intel_engine_cs *engine) 326 struct intel_engine_cs *engine)
331{ 327{
332 return ctx->engine[engine->id].lrc_desc; 328 return ctx->engine[engine->id].lrc_desc;
333} 329}
334 330
335/**
336 * intel_execlists_ctx_id() - get the Execlists Context ID
337 * @ctx: Context to get the ID for
338 * @ring: Engine to get the ID for
339 *
340 * Do not confuse with ctx->id! Unfortunately we have a name overload
341 * here: the old context ID we pass to userspace as a handler so that
342 * they can refer to a context, and the new context ID we pass to the
343 * ELSP so that the GPU can inform us of the context status via
344 * interrupts.
345 *
346 * The context ID is a portion of the context descriptor, so we can
347 * just extract the required part from the cached descriptor.
348 *
349 * Return: 20-bits globally unique context ID.
350 */
351u32 intel_execlists_ctx_id(struct intel_context *ctx,
352 struct intel_engine_cs *engine)
353{
354 return intel_lr_context_descriptor(ctx, engine) >> GEN8_CTX_ID_SHIFT;
355}
356
357static void execlists_elsp_write(struct drm_i915_gem_request *rq0, 331static void execlists_elsp_write(struct drm_i915_gem_request *rq0,
358 struct drm_i915_gem_request *rq1) 332 struct drm_i915_gem_request *rq1)
359{ 333{
360 334
361 struct intel_engine_cs *engine = rq0->engine; 335 struct intel_engine_cs *engine = rq0->engine;
362 struct drm_device *dev = engine->dev; 336 struct drm_i915_private *dev_priv = rq0->i915;
363 struct drm_i915_private *dev_priv = dev->dev_private;
364 uint64_t desc[2]; 337 uint64_t desc[2];
365 338
366 if (rq1) { 339 if (rq1) {
@@ -431,6 +404,20 @@ static void execlists_submit_requests(struct drm_i915_gem_request *rq0,
431 spin_unlock_irq(&dev_priv->uncore.lock); 404 spin_unlock_irq(&dev_priv->uncore.lock);
432} 405}
433 406
407static inline void execlists_context_status_change(
408 struct drm_i915_gem_request *rq,
409 unsigned long status)
410{
411 /*
412 * Only used when GVT-g is enabled now. When GVT-g is disabled,
413 * The compiler should eliminate this function as dead-code.
414 */
415 if (!IS_ENABLED(CONFIG_DRM_I915_GVT))
416 return;
417
418 atomic_notifier_call_chain(&rq->ctx->status_notifier, status, rq);
419}
420
434static void execlists_context_unqueue(struct intel_engine_cs *engine) 421static void execlists_context_unqueue(struct intel_engine_cs *engine)
435{ 422{
436 struct drm_i915_gem_request *req0 = NULL, *req1 = NULL; 423 struct drm_i915_gem_request *req0 = NULL, *req1 = NULL;
@@ -442,7 +429,7 @@ static void execlists_context_unqueue(struct intel_engine_cs *engine)
442 * If irqs are not active generate a warning as batches that finish 429 * If irqs are not active generate a warning as batches that finish
443 * without the irqs may get lost and a GPU Hang may occur. 430 * without the irqs may get lost and a GPU Hang may occur.
444 */ 431 */
445 WARN_ON(!intel_irqs_enabled(engine->dev->dev_private)); 432 WARN_ON(!intel_irqs_enabled(engine->i915));
446 433
447 /* Try to read in pairs */ 434 /* Try to read in pairs */
448 list_for_each_entry_safe(cursor, tmp, &engine->execlist_queue, 435 list_for_each_entry_safe(cursor, tmp, &engine->execlist_queue,
@@ -453,10 +440,24 @@ static void execlists_context_unqueue(struct intel_engine_cs *engine)
453 /* Same ctx: ignore first request, as second request 440 /* Same ctx: ignore first request, as second request
454 * will update tail past first request's workload */ 441 * will update tail past first request's workload */
455 cursor->elsp_submitted = req0->elsp_submitted; 442 cursor->elsp_submitted = req0->elsp_submitted;
456 list_move_tail(&req0->execlist_link, 443 list_del(&req0->execlist_link);
457 &engine->execlist_retired_req_list); 444 i915_gem_request_unreference(req0);
458 req0 = cursor; 445 req0 = cursor;
459 } else { 446 } else {
447 if (IS_ENABLED(CONFIG_DRM_I915_GVT)) {
448 /*
449 * req0 (after merged) ctx requires single
450 * submission, stop picking
451 */
452 if (req0->ctx->execlists_force_single_submission)
453 break;
454 /*
455 * req0 ctx doesn't require single submission,
456 * but next req ctx requires, stop picking
457 */
458 if (cursor->ctx->execlists_force_single_submission)
459 break;
460 }
460 req1 = cursor; 461 req1 = cursor;
461 WARN_ON(req1->elsp_submitted); 462 WARN_ON(req1->elsp_submitted);
462 break; 463 break;
@@ -466,6 +467,12 @@ static void execlists_context_unqueue(struct intel_engine_cs *engine)
466 if (unlikely(!req0)) 467 if (unlikely(!req0))
467 return; 468 return;
468 469
470 execlists_context_status_change(req0, INTEL_CONTEXT_SCHEDULE_IN);
471
472 if (req1)
473 execlists_context_status_change(req1,
474 INTEL_CONTEXT_SCHEDULE_IN);
475
469 if (req0->elsp_submitted & engine->idle_lite_restore_wa) { 476 if (req0->elsp_submitted & engine->idle_lite_restore_wa) {
470 /* 477 /*
471 * WaIdleLiteRestore: make sure we never cause a lite restore 478 * WaIdleLiteRestore: make sure we never cause a lite restore
@@ -486,7 +493,7 @@ static void execlists_context_unqueue(struct intel_engine_cs *engine)
486} 493}
487 494
488static unsigned int 495static unsigned int
489execlists_check_remove_request(struct intel_engine_cs *engine, u32 request_id) 496execlists_check_remove_request(struct intel_engine_cs *engine, u32 ctx_id)
490{ 497{
491 struct drm_i915_gem_request *head_req; 498 struct drm_i915_gem_request *head_req;
492 499
@@ -496,19 +503,18 @@ execlists_check_remove_request(struct intel_engine_cs *engine, u32 request_id)
496 struct drm_i915_gem_request, 503 struct drm_i915_gem_request,
497 execlist_link); 504 execlist_link);
498 505
499 if (!head_req) 506 if (WARN_ON(!head_req || (head_req->ctx_hw_id != ctx_id)))
500 return 0; 507 return 0;
501
502 if (unlikely(intel_execlists_ctx_id(head_req->ctx, engine) != request_id))
503 return 0;
504 508
505 WARN(head_req->elsp_submitted == 0, "Never submitted head request\n"); 509 WARN(head_req->elsp_submitted == 0, "Never submitted head request\n");
506 510
507 if (--head_req->elsp_submitted > 0) 511 if (--head_req->elsp_submitted > 0)
508 return 0; 512 return 0;
509 513
510 list_move_tail(&head_req->execlist_link, 514 execlists_context_status_change(head_req, INTEL_CONTEXT_SCHEDULE_OUT);
511 &engine->execlist_retired_req_list); 515
516 list_del(&head_req->execlist_link);
517 i915_gem_request_unreference(head_req);
512 518
513 return 1; 519 return 1;
514} 520}
@@ -517,7 +523,7 @@ static u32
517get_context_status(struct intel_engine_cs *engine, unsigned int read_pointer, 523get_context_status(struct intel_engine_cs *engine, unsigned int read_pointer,
518 u32 *context_id) 524 u32 *context_id)
519{ 525{
520 struct drm_i915_private *dev_priv = engine->dev->dev_private; 526 struct drm_i915_private *dev_priv = engine->i915;
521 u32 status; 527 u32 status;
522 528
523 read_pointer %= GEN8_CSB_ENTRIES; 529 read_pointer %= GEN8_CSB_ENTRIES;
@@ -535,7 +541,7 @@ get_context_status(struct intel_engine_cs *engine, unsigned int read_pointer,
535 541
536/** 542/**
537 * intel_lrc_irq_handler() - handle Context Switch interrupts 543 * intel_lrc_irq_handler() - handle Context Switch interrupts
538 * @engine: Engine Command Streamer to handle. 544 * @data: tasklet handler passed in unsigned long
539 * 545 *
540 * Check the unread Context Status Buffers and manage the submission of new 546 * Check the unread Context Status Buffers and manage the submission of new
541 * contexts to the ELSP accordingly. 547 * contexts to the ELSP accordingly.
@@ -543,7 +549,7 @@ get_context_status(struct intel_engine_cs *engine, unsigned int read_pointer,
543static void intel_lrc_irq_handler(unsigned long data) 549static void intel_lrc_irq_handler(unsigned long data)
544{ 550{
545 struct intel_engine_cs *engine = (struct intel_engine_cs *)data; 551 struct intel_engine_cs *engine = (struct intel_engine_cs *)data;
546 struct drm_i915_private *dev_priv = engine->dev->dev_private; 552 struct drm_i915_private *dev_priv = engine->i915;
547 u32 status_pointer; 553 u32 status_pointer;
548 unsigned int read_pointer, write_pointer; 554 unsigned int read_pointer, write_pointer;
549 u32 csb[GEN8_CSB_ENTRIES][2]; 555 u32 csb[GEN8_CSB_ENTRIES][2];
@@ -612,11 +618,6 @@ static void execlists_context_queue(struct drm_i915_gem_request *request)
612 struct drm_i915_gem_request *cursor; 618 struct drm_i915_gem_request *cursor;
613 int num_elements = 0; 619 int num_elements = 0;
614 620
615 if (request->ctx != request->i915->kernel_context)
616 intel_lr_context_pin(request->ctx, engine);
617
618 i915_gem_request_reference(request);
619
620 spin_lock_bh(&engine->execlist_lock); 621 spin_lock_bh(&engine->execlist_lock);
621 622
622 list_for_each_entry(cursor, &engine->execlist_queue, execlist_link) 623 list_for_each_entry(cursor, &engine->execlist_queue, execlist_link)
@@ -633,12 +634,14 @@ static void execlists_context_queue(struct drm_i915_gem_request *request)
633 if (request->ctx == tail_req->ctx) { 634 if (request->ctx == tail_req->ctx) {
634 WARN(tail_req->elsp_submitted != 0, 635 WARN(tail_req->elsp_submitted != 0,
635 "More than 2 already-submitted reqs queued\n"); 636 "More than 2 already-submitted reqs queued\n");
636 list_move_tail(&tail_req->execlist_link, 637 list_del(&tail_req->execlist_link);
637 &engine->execlist_retired_req_list); 638 i915_gem_request_unreference(tail_req);
638 } 639 }
639 } 640 }
640 641
642 i915_gem_request_reference(request);
641 list_add_tail(&request->execlist_link, &engine->execlist_queue); 643 list_add_tail(&request->execlist_link, &engine->execlist_queue);
644 request->ctx_hw_id = request->ctx->hw_id;
642 if (num_elements == 0) 645 if (num_elements == 0)
643 execlists_context_unqueue(engine); 646 execlists_context_unqueue(engine);
644 647
@@ -698,9 +701,23 @@ static int execlists_move_to_gpu(struct drm_i915_gem_request *req,
698 701
699int intel_logical_ring_alloc_request_extras(struct drm_i915_gem_request *request) 702int intel_logical_ring_alloc_request_extras(struct drm_i915_gem_request *request)
700{ 703{
701 int ret = 0; 704 struct intel_engine_cs *engine = request->engine;
705 struct intel_context *ce = &request->ctx->engine[engine->id];
706 int ret;
702 707
703 request->ringbuf = request->ctx->engine[request->engine->id].ringbuf; 708 /* Flush enough space to reduce the likelihood of waiting after
709 * we start building the request - in which case we will just
710 * have to repeat work.
711 */
712 request->reserved_space += EXECLISTS_REQUEST_SIZE;
713
714 if (!ce->state) {
715 ret = execlists_context_deferred_alloc(request->ctx, engine);
716 if (ret)
717 return ret;
718 }
719
720 request->ringbuf = ce->ringbuf;
704 721
705 if (i915.enable_guc_submission) { 722 if (i915.enable_guc_submission) {
706 /* 723 /*
@@ -708,16 +725,39 @@ int intel_logical_ring_alloc_request_extras(struct drm_i915_gem_request *request
708 * going any further, as the i915_add_request() call 725 * going any further, as the i915_add_request() call
709 * later on mustn't fail ... 726 * later on mustn't fail ...
710 */ 727 */
711 struct intel_guc *guc = &request->i915->guc; 728 ret = i915_guc_wq_check_space(request);
712
713 ret = i915_guc_wq_check_space(guc->execbuf_client);
714 if (ret) 729 if (ret)
715 return ret; 730 return ret;
716 } 731 }
717 732
718 if (request->ctx != request->i915->kernel_context) 733 ret = intel_lr_context_pin(request->ctx, engine);
719 ret = intel_lr_context_pin(request->ctx, request->engine); 734 if (ret)
735 return ret;
736
737 ret = intel_ring_begin(request, 0);
738 if (ret)
739 goto err_unpin;
740
741 if (!ce->initialised) {
742 ret = engine->init_context(request);
743 if (ret)
744 goto err_unpin;
745
746 ce->initialised = true;
747 }
748
749 /* Note that after this point, we have committed to using
750 * this request as it is being used to both track the
751 * state of engine initialisation and liveness of the
752 * golden renderstate above. Think twice before you try
753 * to cancel/unwind this request now.
754 */
720 755
756 request->reserved_space -= EXECLISTS_REQUEST_SIZE;
757 return 0;
758
759err_unpin:
760 intel_lr_context_unpin(request->ctx, engine);
721 return ret; 761 return ret;
722} 762}
723 763
@@ -734,7 +774,6 @@ static int
734intel_logical_ring_advance_and_submit(struct drm_i915_gem_request *request) 774intel_logical_ring_advance_and_submit(struct drm_i915_gem_request *request)
735{ 775{
736 struct intel_ringbuffer *ringbuf = request->ringbuf; 776 struct intel_ringbuffer *ringbuf = request->ringbuf;
737 struct drm_i915_private *dev_priv = request->i915;
738 struct intel_engine_cs *engine = request->engine; 777 struct intel_engine_cs *engine = request->engine;
739 778
740 intel_logical_ring_advance(ringbuf); 779 intel_logical_ring_advance(ringbuf);
@@ -750,54 +789,28 @@ intel_logical_ring_advance_and_submit(struct drm_i915_gem_request *request)
750 intel_logical_ring_emit(ringbuf, MI_NOOP); 789 intel_logical_ring_emit(ringbuf, MI_NOOP);
751 intel_logical_ring_advance(ringbuf); 790 intel_logical_ring_advance(ringbuf);
752 791
753 if (intel_engine_stopped(engine)) 792 /* We keep the previous context alive until we retire the following
754 return 0; 793 * request. This ensures that any the context object is still pinned
755 794 * for any residual writes the HW makes into it on the context switch
756 if (engine->last_context != request->ctx) { 795 * into the next object following the breadcrumb. Otherwise, we may
757 if (engine->last_context) 796 * retire the context too early.
758 intel_lr_context_unpin(engine->last_context, engine); 797 */
759 if (request->ctx != request->i915->kernel_context) { 798 request->previous_context = engine->last_context;
760 intel_lr_context_pin(request->ctx, engine); 799 engine->last_context = request->ctx;
761 engine->last_context = request->ctx;
762 } else {
763 engine->last_context = NULL;
764 }
765 }
766 800
767 if (dev_priv->guc.execbuf_client) 801 if (i915.enable_guc_submission)
768 i915_guc_submit(dev_priv->guc.execbuf_client, request); 802 i915_guc_submit(request);
769 else 803 else
770 execlists_context_queue(request); 804 execlists_context_queue(request);
771 805
772 return 0; 806 return 0;
773} 807}
774 808
775int intel_logical_ring_reserve_space(struct drm_i915_gem_request *request)
776{
777 /*
778 * The first call merely notes the reserve request and is common for
779 * all back ends. The subsequent localised _begin() call actually
780 * ensures that the reservation is available. Without the begin, if
781 * the request creator immediately submitted the request without
782 * adding any commands to it then there might not actually be
783 * sufficient room for the submission commands.
784 */
785 intel_ring_reserved_space_reserve(request->ringbuf, MIN_SPACE_FOR_ADD_REQUEST);
786
787 return intel_ring_begin(request, 0);
788}
789
790/** 809/**
791 * execlists_submission() - submit a batchbuffer for execution, Execlists style 810 * execlists_submission() - submit a batchbuffer for execution, Execlists style
792 * @dev: DRM device. 811 * @params: execbuffer call parameters.
793 * @file: DRM file.
794 * @ring: Engine Command Streamer to submit to.
795 * @ctx: Context to employ for this submission.
796 * @args: execbuffer call arguments. 812 * @args: execbuffer call arguments.
797 * @vmas: list of vmas. 813 * @vmas: list of vmas.
798 * @batch_obj: the batchbuffer to submit.
799 * @exec_start: batchbuffer start virtual address pointer.
800 * @dispatch_flags: translated execbuffer call flags.
801 * 814 *
802 * This is the evil twin version of i915_gem_ringbuffer_submission. It abstracts 815 * This is the evil twin version of i915_gem_ringbuffer_submission. It abstracts
803 * away the submission details of the execbuffer ioctl call. 816 * away the submission details of the execbuffer ioctl call.
@@ -810,7 +823,7 @@ int intel_execlists_submission(struct i915_execbuffer_params *params,
810{ 823{
811 struct drm_device *dev = params->dev; 824 struct drm_device *dev = params->dev;
812 struct intel_engine_cs *engine = params->engine; 825 struct intel_engine_cs *engine = params->engine;
813 struct drm_i915_private *dev_priv = dev->dev_private; 826 struct drm_i915_private *dev_priv = to_i915(dev);
814 struct intel_ringbuffer *ringbuf = params->ctx->engine[engine->id].ringbuf; 827 struct intel_ringbuffer *ringbuf = params->ctx->engine[engine->id].ringbuf;
815 u64 exec_start; 828 u64 exec_start;
816 int instp_mode; 829 int instp_mode;
@@ -881,28 +894,18 @@ int intel_execlists_submission(struct i915_execbuffer_params *params,
881 return 0; 894 return 0;
882} 895}
883 896
884void intel_execlists_retire_requests(struct intel_engine_cs *engine) 897void intel_execlists_cancel_requests(struct intel_engine_cs *engine)
885{ 898{
886 struct drm_i915_gem_request *req, *tmp; 899 struct drm_i915_gem_request *req, *tmp;
887 struct list_head retired_list; 900 LIST_HEAD(cancel_list);
888 901
889 WARN_ON(!mutex_is_locked(&engine->dev->struct_mutex)); 902 WARN_ON(!mutex_is_locked(&engine->i915->drm.struct_mutex));
890 if (list_empty(&engine->execlist_retired_req_list))
891 return;
892 903
893 INIT_LIST_HEAD(&retired_list);
894 spin_lock_bh(&engine->execlist_lock); 904 spin_lock_bh(&engine->execlist_lock);
895 list_replace_init(&engine->execlist_retired_req_list, &retired_list); 905 list_replace_init(&engine->execlist_queue, &cancel_list);
896 spin_unlock_bh(&engine->execlist_lock); 906 spin_unlock_bh(&engine->execlist_lock);
897 907
898 list_for_each_entry_safe(req, tmp, &retired_list, execlist_link) { 908 list_for_each_entry_safe(req, tmp, &cancel_list, execlist_link) {
899 struct intel_context *ctx = req->ctx;
900 struct drm_i915_gem_object *ctx_obj =
901 ctx->engine[engine->id].state;
902
903 if (ctx_obj && (ctx != req->i915->kernel_context))
904 intel_lr_context_unpin(ctx, engine);
905
906 list_del(&req->execlist_link); 909 list_del(&req->execlist_link);
907 i915_gem_request_unreference(req); 910 i915_gem_request_unreference(req);
908 } 911 }
@@ -910,7 +913,7 @@ void intel_execlists_retire_requests(struct intel_engine_cs *engine)
910 913
911void intel_logical_ring_stop(struct intel_engine_cs *engine) 914void intel_logical_ring_stop(struct intel_engine_cs *engine)
912{ 915{
913 struct drm_i915_private *dev_priv = engine->dev->dev_private; 916 struct drm_i915_private *dev_priv = engine->i915;
914 int ret; 917 int ret;
915 918
916 if (!intel_engine_initialized(engine)) 919 if (!intel_engine_initialized(engine))
@@ -923,7 +926,10 @@ void intel_logical_ring_stop(struct intel_engine_cs *engine)
923 926
924 /* TODO: Is this correct with Execlists enabled? */ 927 /* TODO: Is this correct with Execlists enabled? */
925 I915_WRITE_MODE(engine, _MASKED_BIT_ENABLE(STOP_RING)); 928 I915_WRITE_MODE(engine, _MASKED_BIT_ENABLE(STOP_RING));
926 if (wait_for((I915_READ_MODE(engine) & MODE_IDLE) != 0, 1000)) { 929 if (intel_wait_for_register(dev_priv,
930 RING_MI_MODE(engine->mmio_base),
931 MODE_IDLE, MODE_IDLE,
932 1000)) {
927 DRM_ERROR("%s :timed out trying to stop ring\n", engine->name); 933 DRM_ERROR("%s :timed out trying to stop ring\n", engine->name);
928 return; 934 return;
929 } 935 }
@@ -946,25 +952,26 @@ int logical_ring_flush_all_caches(struct drm_i915_gem_request *req)
946 return 0; 952 return 0;
947} 953}
948 954
949static int intel_lr_context_do_pin(struct intel_context *ctx, 955static int intel_lr_context_pin(struct i915_gem_context *ctx,
950 struct intel_engine_cs *engine) 956 struct intel_engine_cs *engine)
951{ 957{
952 struct drm_device *dev = engine->dev; 958 struct drm_i915_private *dev_priv = ctx->i915;
953 struct drm_i915_private *dev_priv = dev->dev_private; 959 struct intel_context *ce = &ctx->engine[engine->id];
954 struct drm_i915_gem_object *ctx_obj = ctx->engine[engine->id].state;
955 struct intel_ringbuffer *ringbuf = ctx->engine[engine->id].ringbuf;
956 void *vaddr; 960 void *vaddr;
957 u32 *lrc_reg_state; 961 u32 *lrc_reg_state;
958 int ret; 962 int ret;
959 963
960 WARN_ON(!mutex_is_locked(&engine->dev->struct_mutex)); 964 lockdep_assert_held(&ctx->i915->drm.struct_mutex);
961 965
962 ret = i915_gem_obj_ggtt_pin(ctx_obj, GEN8_LR_CONTEXT_ALIGN, 966 if (ce->pin_count++)
963 PIN_OFFSET_BIAS | GUC_WOPCM_TOP); 967 return 0;
968
969 ret = i915_gem_obj_ggtt_pin(ce->state, GEN8_LR_CONTEXT_ALIGN,
970 PIN_OFFSET_BIAS | GUC_WOPCM_TOP);
964 if (ret) 971 if (ret)
965 return ret; 972 goto err;
966 973
967 vaddr = i915_gem_object_pin_map(ctx_obj); 974 vaddr = i915_gem_object_pin_map(ce->state);
968 if (IS_ERR(vaddr)) { 975 if (IS_ERR(vaddr)) {
969 ret = PTR_ERR(vaddr); 976 ret = PTR_ERR(vaddr);
970 goto unpin_ctx_obj; 977 goto unpin_ctx_obj;
@@ -972,65 +979,54 @@ static int intel_lr_context_do_pin(struct intel_context *ctx,
972 979
973 lrc_reg_state = vaddr + LRC_STATE_PN * PAGE_SIZE; 980 lrc_reg_state = vaddr + LRC_STATE_PN * PAGE_SIZE;
974 981
975 ret = intel_pin_and_map_ringbuffer_obj(engine->dev, ringbuf); 982 ret = intel_pin_and_map_ringbuffer_obj(dev_priv, ce->ringbuf);
976 if (ret) 983 if (ret)
977 goto unpin_map; 984 goto unpin_map;
978 985
979 ctx->engine[engine->id].lrc_vma = i915_gem_obj_to_ggtt(ctx_obj); 986 i915_gem_context_reference(ctx);
987 ce->lrc_vma = i915_gem_obj_to_ggtt(ce->state);
980 intel_lr_context_descriptor_update(ctx, engine); 988 intel_lr_context_descriptor_update(ctx, engine);
981 lrc_reg_state[CTX_RING_BUFFER_START+1] = ringbuf->vma->node.start; 989
982 ctx->engine[engine->id].lrc_reg_state = lrc_reg_state; 990 lrc_reg_state[CTX_RING_BUFFER_START+1] = ce->ringbuf->vma->node.start;
983 ctx_obj->dirty = true; 991 ce->lrc_reg_state = lrc_reg_state;
992 ce->state->dirty = true;
984 993
985 /* Invalidate GuC TLB. */ 994 /* Invalidate GuC TLB. */
986 if (i915.enable_guc_submission) 995 if (i915.enable_guc_submission)
987 I915_WRITE(GEN8_GTCR, GEN8_GTCR_INVALIDATE); 996 I915_WRITE(GEN8_GTCR, GEN8_GTCR_INVALIDATE);
988 997
989 return ret; 998 return 0;
990 999
991unpin_map: 1000unpin_map:
992 i915_gem_object_unpin_map(ctx_obj); 1001 i915_gem_object_unpin_map(ce->state);
993unpin_ctx_obj: 1002unpin_ctx_obj:
994 i915_gem_object_ggtt_unpin(ctx_obj); 1003 i915_gem_object_ggtt_unpin(ce->state);
995 1004err:
1005 ce->pin_count = 0;
996 return ret; 1006 return ret;
997} 1007}
998 1008
999static int intel_lr_context_pin(struct intel_context *ctx, 1009void intel_lr_context_unpin(struct i915_gem_context *ctx,
1000 struct intel_engine_cs *engine) 1010 struct intel_engine_cs *engine)
1001{ 1011{
1002 int ret = 0; 1012 struct intel_context *ce = &ctx->engine[engine->id];
1003 1013
1004 if (ctx->engine[engine->id].pin_count++ == 0) { 1014 lockdep_assert_held(&ctx->i915->drm.struct_mutex);
1005 ret = intel_lr_context_do_pin(ctx, engine); 1015 GEM_BUG_ON(ce->pin_count == 0);
1006 if (ret)
1007 goto reset_pin_count;
1008 1016
1009 i915_gem_context_reference(ctx); 1017 if (--ce->pin_count)
1010 } 1018 return;
1011 return ret;
1012 1019
1013reset_pin_count: 1020 intel_unpin_ringbuffer_obj(ce->ringbuf);
1014 ctx->engine[engine->id].pin_count = 0;
1015 return ret;
1016}
1017 1021
1018void intel_lr_context_unpin(struct intel_context *ctx, 1022 i915_gem_object_unpin_map(ce->state);
1019 struct intel_engine_cs *engine) 1023 i915_gem_object_ggtt_unpin(ce->state);
1020{
1021 struct drm_i915_gem_object *ctx_obj = ctx->engine[engine->id].state;
1022 1024
1023 WARN_ON(!mutex_is_locked(&ctx->i915->dev->struct_mutex)); 1025 ce->lrc_vma = NULL;
1024 if (--ctx->engine[engine->id].pin_count == 0) { 1026 ce->lrc_desc = 0;
1025 i915_gem_object_unpin_map(ctx_obj); 1027 ce->lrc_reg_state = NULL;
1026 intel_unpin_ringbuffer_obj(ctx->engine[engine->id].ringbuf);
1027 i915_gem_object_ggtt_unpin(ctx_obj);
1028 ctx->engine[engine->id].lrc_vma = NULL;
1029 ctx->engine[engine->id].lrc_desc = 0;
1030 ctx->engine[engine->id].lrc_reg_state = NULL;
1031 1028
1032 i915_gem_context_unreference(ctx); 1029 i915_gem_context_unreference(ctx);
1033 }
1034} 1030}
1035 1031
1036static int intel_logical_ring_workarounds_emit(struct drm_i915_gem_request *req) 1032static int intel_logical_ring_workarounds_emit(struct drm_i915_gem_request *req)
@@ -1038,9 +1034,7 @@ static int intel_logical_ring_workarounds_emit(struct drm_i915_gem_request *req)
1038 int ret, i; 1034 int ret, i;
1039 struct intel_engine_cs *engine = req->engine; 1035 struct intel_engine_cs *engine = req->engine;
1040 struct intel_ringbuffer *ringbuf = req->ringbuf; 1036 struct intel_ringbuffer *ringbuf = req->ringbuf;
1041 struct drm_device *dev = engine->dev; 1037 struct i915_workarounds *w = &req->i915->workarounds;
1042 struct drm_i915_private *dev_priv = dev->dev_private;
1043 struct i915_workarounds *w = &dev_priv->workarounds;
1044 1038
1045 if (w->count == 0) 1039 if (w->count == 0)
1046 return 0; 1040 return 0;
@@ -1103,7 +1097,7 @@ static inline int gen8_emit_flush_coherentl3_wa(struct intel_engine_cs *engine,
1103 uint32_t *const batch, 1097 uint32_t *const batch,
1104 uint32_t index) 1098 uint32_t index)
1105{ 1099{
1106 struct drm_i915_private *dev_priv = engine->dev->dev_private; 1100 struct drm_i915_private *dev_priv = engine->i915;
1107 uint32_t l3sqc4_flush = (0x40400000 | GEN8_LQSC_FLUSH_COHERENT_LINES); 1101 uint32_t l3sqc4_flush = (0x40400000 | GEN8_LQSC_FLUSH_COHERENT_LINES);
1108 1102
1109 /* 1103 /*
@@ -1165,7 +1159,7 @@ static inline int wa_ctx_end(struct i915_wa_ctx_bb *wa_ctx,
1165/** 1159/**
1166 * gen8_init_indirectctx_bb() - initialize indirect ctx batch with WA 1160 * gen8_init_indirectctx_bb() - initialize indirect ctx batch with WA
1167 * 1161 *
1168 * @ring: only applicable for RCS 1162 * @engine: only applicable for RCS
1169 * @wa_ctx: structure representing wa_ctx 1163 * @wa_ctx: structure representing wa_ctx
1170 * offset: specifies start of the batch, should be cache-aligned. This is updated 1164 * offset: specifies start of the batch, should be cache-aligned. This is updated
1171 * with the offset value received as input. 1165 * with the offset value received as input.
@@ -1202,7 +1196,7 @@ static int gen8_init_indirectctx_bb(struct intel_engine_cs *engine,
1202 wa_ctx_emit(batch, index, MI_ARB_ON_OFF | MI_ARB_DISABLE); 1196 wa_ctx_emit(batch, index, MI_ARB_ON_OFF | MI_ARB_DISABLE);
1203 1197
1204 /* WaFlushCoherentL3CacheLinesAtContextSwitch:bdw */ 1198 /* WaFlushCoherentL3CacheLinesAtContextSwitch:bdw */
1205 if (IS_BROADWELL(engine->dev)) { 1199 if (IS_BROADWELL(engine->i915)) {
1206 int rc = gen8_emit_flush_coherentl3_wa(engine, batch, index); 1200 int rc = gen8_emit_flush_coherentl3_wa(engine, batch, index);
1207 if (rc < 0) 1201 if (rc < 0)
1208 return rc; 1202 return rc;
@@ -1239,7 +1233,7 @@ static int gen8_init_indirectctx_bb(struct intel_engine_cs *engine,
1239/** 1233/**
1240 * gen8_init_perctx_bb() - initialize per ctx batch with WA 1234 * gen8_init_perctx_bb() - initialize per ctx batch with WA
1241 * 1235 *
1242 * @ring: only applicable for RCS 1236 * @engine: only applicable for RCS
1243 * @wa_ctx: structure representing wa_ctx 1237 * @wa_ctx: structure representing wa_ctx
1244 * offset: specifies start of the batch, should be cache-aligned. 1238 * offset: specifies start of the batch, should be cache-aligned.
1245 * size: size of the batch in DWORDS but HW expects in terms of cachelines 1239 * size: size of the batch in DWORDS but HW expects in terms of cachelines
@@ -1274,13 +1268,12 @@ static int gen9_init_indirectctx_bb(struct intel_engine_cs *engine,
1274 uint32_t *offset) 1268 uint32_t *offset)
1275{ 1269{
1276 int ret; 1270 int ret;
1277 struct drm_device *dev = engine->dev; 1271 struct drm_i915_private *dev_priv = engine->i915;
1278 struct drm_i915_private *dev_priv = dev->dev_private;
1279 uint32_t index = wa_ctx_start(wa_ctx, *offset, CACHELINE_DWORDS); 1272 uint32_t index = wa_ctx_start(wa_ctx, *offset, CACHELINE_DWORDS);
1280 1273
1281 /* WaDisableCtxRestoreArbitration:skl,bxt */ 1274 /* WaDisableCtxRestoreArbitration:skl,bxt */
1282 if (IS_SKL_REVID(dev, 0, SKL_REVID_D0) || 1275 if (IS_SKL_REVID(dev_priv, 0, SKL_REVID_D0) ||
1283 IS_BXT_REVID(dev, 0, BXT_REVID_A1)) 1276 IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
1284 wa_ctx_emit(batch, index, MI_ARB_ON_OFF | MI_ARB_DISABLE); 1277 wa_ctx_emit(batch, index, MI_ARB_ON_OFF | MI_ARB_DISABLE);
1285 1278
1286 /* WaFlushCoherentL3CacheLinesAtContextSwitch:skl,bxt */ 1279 /* WaFlushCoherentL3CacheLinesAtContextSwitch:skl,bxt */
@@ -1305,6 +1298,31 @@ static int gen9_init_indirectctx_bb(struct intel_engine_cs *engine,
1305 wa_ctx_emit(batch, index, 0); 1298 wa_ctx_emit(batch, index, 0);
1306 wa_ctx_emit(batch, index, 0); 1299 wa_ctx_emit(batch, index, 0);
1307 } 1300 }
1301
1302 /* WaMediaPoolStateCmdInWABB:bxt */
1303 if (HAS_POOLED_EU(engine->i915)) {
1304 /*
1305 * EU pool configuration is setup along with golden context
1306 * during context initialization. This value depends on
1307 * device type (2x6 or 3x6) and needs to be updated based
1308 * on which subslice is disabled especially for 2x6
1309 * devices, however it is safe to load default
1310 * configuration of 3x6 device instead of masking off
1311 * corresponding bits because HW ignores bits of a disabled
1312 * subslice and drops down to appropriate config. Please
1313 * see render_state_setup() in i915_gem_render_state.c for
1314 * possible configurations, to avoid duplication they are
1315 * not shown here again.
1316 */
1317 u32 eu_pool_config = 0x00777000;
1318 wa_ctx_emit(batch, index, GEN9_MEDIA_POOL_STATE);
1319 wa_ctx_emit(batch, index, GEN9_MEDIA_POOL_ENABLE);
1320 wa_ctx_emit(batch, index, eu_pool_config);
1321 wa_ctx_emit(batch, index, 0);
1322 wa_ctx_emit(batch, index, 0);
1323 wa_ctx_emit(batch, index, 0);
1324 }
1325
1308 /* Pad to end of cacheline */ 1326 /* Pad to end of cacheline */
1309 while (index % CACHELINE_DWORDS) 1327 while (index % CACHELINE_DWORDS)
1310 wa_ctx_emit(batch, index, MI_NOOP); 1328 wa_ctx_emit(batch, index, MI_NOOP);
@@ -1317,12 +1335,11 @@ static int gen9_init_perctx_bb(struct intel_engine_cs *engine,
1317 uint32_t *const batch, 1335 uint32_t *const batch,
1318 uint32_t *offset) 1336 uint32_t *offset)
1319{ 1337{
1320 struct drm_device *dev = engine->dev;
1321 uint32_t index = wa_ctx_start(wa_ctx, *offset, CACHELINE_DWORDS); 1338 uint32_t index = wa_ctx_start(wa_ctx, *offset, CACHELINE_DWORDS);
1322 1339
1323 /* WaSetDisablePixMaskCammingAndRhwoInCommonSliceChicken:skl,bxt */ 1340 /* WaSetDisablePixMaskCammingAndRhwoInCommonSliceChicken:skl,bxt */
1324 if (IS_SKL_REVID(dev, 0, SKL_REVID_B0) || 1341 if (IS_SKL_REVID(engine->i915, 0, SKL_REVID_B0) ||
1325 IS_BXT_REVID(dev, 0, BXT_REVID_A1)) { 1342 IS_BXT_REVID(engine->i915, 0, BXT_REVID_A1)) {
1326 wa_ctx_emit(batch, index, MI_LOAD_REGISTER_IMM(1)); 1343 wa_ctx_emit(batch, index, MI_LOAD_REGISTER_IMM(1));
1327 wa_ctx_emit_reg(batch, index, GEN9_SLICE_COMMON_ECO_CHICKEN0); 1344 wa_ctx_emit_reg(batch, index, GEN9_SLICE_COMMON_ECO_CHICKEN0);
1328 wa_ctx_emit(batch, index, 1345 wa_ctx_emit(batch, index,
@@ -1331,7 +1348,7 @@ static int gen9_init_perctx_bb(struct intel_engine_cs *engine,
1331 } 1348 }
1332 1349
1333 /* WaClearTdlStateAckDirtyBits:bxt */ 1350 /* WaClearTdlStateAckDirtyBits:bxt */
1334 if (IS_BXT_REVID(dev, 0, BXT_REVID_B0)) { 1351 if (IS_BXT_REVID(engine->i915, 0, BXT_REVID_B0)) {
1335 wa_ctx_emit(batch, index, MI_LOAD_REGISTER_IMM(4)); 1352 wa_ctx_emit(batch, index, MI_LOAD_REGISTER_IMM(4));
1336 1353
1337 wa_ctx_emit_reg(batch, index, GEN8_STATE_ACK); 1354 wa_ctx_emit_reg(batch, index, GEN8_STATE_ACK);
@@ -1350,8 +1367,8 @@ static int gen9_init_perctx_bb(struct intel_engine_cs *engine,
1350 } 1367 }
1351 1368
1352 /* WaDisableCtxRestoreArbitration:skl,bxt */ 1369 /* WaDisableCtxRestoreArbitration:skl,bxt */
1353 if (IS_SKL_REVID(dev, 0, SKL_REVID_D0) || 1370 if (IS_SKL_REVID(engine->i915, 0, SKL_REVID_D0) ||
1354 IS_BXT_REVID(dev, 0, BXT_REVID_A1)) 1371 IS_BXT_REVID(engine->i915, 0, BXT_REVID_A1))
1355 wa_ctx_emit(batch, index, MI_ARB_ON_OFF | MI_ARB_ENABLE); 1372 wa_ctx_emit(batch, index, MI_ARB_ON_OFF | MI_ARB_ENABLE);
1356 1373
1357 wa_ctx_emit(batch, index, MI_BATCH_BUFFER_END); 1374 wa_ctx_emit(batch, index, MI_BATCH_BUFFER_END);
@@ -1363,11 +1380,13 @@ static int lrc_setup_wa_ctx_obj(struct intel_engine_cs *engine, u32 size)
1363{ 1380{
1364 int ret; 1381 int ret;
1365 1382
1366 engine->wa_ctx.obj = i915_gem_alloc_object(engine->dev, 1383 engine->wa_ctx.obj = i915_gem_object_create(&engine->i915->drm,
1367 PAGE_ALIGN(size)); 1384 PAGE_ALIGN(size));
1368 if (!engine->wa_ctx.obj) { 1385 if (IS_ERR(engine->wa_ctx.obj)) {
1369 DRM_DEBUG_DRIVER("alloc LRC WA ctx backing obj failed.\n"); 1386 DRM_DEBUG_DRIVER("alloc LRC WA ctx backing obj failed.\n");
1370 return -ENOMEM; 1387 ret = PTR_ERR(engine->wa_ctx.obj);
1388 engine->wa_ctx.obj = NULL;
1389 return ret;
1371 } 1390 }
1372 1391
1373 ret = i915_gem_obj_ggtt_pin(engine->wa_ctx.obj, PAGE_SIZE, 0); 1392 ret = i915_gem_obj_ggtt_pin(engine->wa_ctx.obj, PAGE_SIZE, 0);
@@ -1401,9 +1420,9 @@ static int intel_init_workaround_bb(struct intel_engine_cs *engine)
1401 WARN_ON(engine->id != RCS); 1420 WARN_ON(engine->id != RCS);
1402 1421
1403 /* update this when WA for higher Gen are added */ 1422 /* update this when WA for higher Gen are added */
1404 if (INTEL_INFO(engine->dev)->gen > 9) { 1423 if (INTEL_GEN(engine->i915) > 9) {
1405 DRM_ERROR("WA batch buffer is not initialized for Gen%d\n", 1424 DRM_ERROR("WA batch buffer is not initialized for Gen%d\n",
1406 INTEL_INFO(engine->dev)->gen); 1425 INTEL_GEN(engine->i915));
1407 return 0; 1426 return 0;
1408 } 1427 }
1409 1428
@@ -1423,7 +1442,7 @@ static int intel_init_workaround_bb(struct intel_engine_cs *engine)
1423 batch = kmap_atomic(page); 1442 batch = kmap_atomic(page);
1424 offset = 0; 1443 offset = 0;
1425 1444
1426 if (INTEL_INFO(engine->dev)->gen == 8) { 1445 if (IS_GEN8(engine->i915)) {
1427 ret = gen8_init_indirectctx_bb(engine, 1446 ret = gen8_init_indirectctx_bb(engine,
1428 &wa_ctx->indirect_ctx, 1447 &wa_ctx->indirect_ctx,
1429 batch, 1448 batch,
@@ -1437,7 +1456,7 @@ static int intel_init_workaround_bb(struct intel_engine_cs *engine)
1437 &offset); 1456 &offset);
1438 if (ret) 1457 if (ret)
1439 goto out; 1458 goto out;
1440 } else if (INTEL_INFO(engine->dev)->gen == 9) { 1459 } else if (IS_GEN9(engine->i915)) {
1441 ret = gen9_init_indirectctx_bb(engine, 1460 ret = gen9_init_indirectctx_bb(engine,
1442 &wa_ctx->indirect_ctx, 1461 &wa_ctx->indirect_ctx,
1443 batch, 1462 batch,
@@ -1463,7 +1482,7 @@ out:
1463 1482
1464static void lrc_init_hws(struct intel_engine_cs *engine) 1483static void lrc_init_hws(struct intel_engine_cs *engine)
1465{ 1484{
1466 struct drm_i915_private *dev_priv = engine->dev->dev_private; 1485 struct drm_i915_private *dev_priv = engine->i915;
1467 1486
1468 I915_WRITE(RING_HWS_PGA(engine->mmio_base), 1487 I915_WRITE(RING_HWS_PGA(engine->mmio_base),
1469 (u32)engine->status_page.gfx_addr); 1488 (u32)engine->status_page.gfx_addr);
@@ -1472,8 +1491,7 @@ static void lrc_init_hws(struct intel_engine_cs *engine)
1472 1491
1473static int gen8_init_common_ring(struct intel_engine_cs *engine) 1492static int gen8_init_common_ring(struct intel_engine_cs *engine)
1474{ 1493{
1475 struct drm_device *dev = engine->dev; 1494 struct drm_i915_private *dev_priv = engine->i915;
1476 struct drm_i915_private *dev_priv = dev->dev_private;
1477 unsigned int next_context_status_buffer_hw; 1495 unsigned int next_context_status_buffer_hw;
1478 1496
1479 lrc_init_hws(engine); 1497 lrc_init_hws(engine);
@@ -1520,8 +1538,7 @@ static int gen8_init_common_ring(struct intel_engine_cs *engine)
1520 1538
1521static int gen8_init_render_ring(struct intel_engine_cs *engine) 1539static int gen8_init_render_ring(struct intel_engine_cs *engine)
1522{ 1540{
1523 struct drm_device *dev = engine->dev; 1541 struct drm_i915_private *dev_priv = engine->i915;
1524 struct drm_i915_private *dev_priv = dev->dev_private;
1525 int ret; 1542 int ret;
1526 1543
1527 ret = gen8_init_common_ring(engine); 1544 ret = gen8_init_common_ring(engine);
@@ -1598,7 +1615,7 @@ static int gen8_emit_bb_start(struct drm_i915_gem_request *req,
1598 if (req->ctx->ppgtt && 1615 if (req->ctx->ppgtt &&
1599 (intel_engine_flag(req->engine) & req->ctx->ppgtt->pd_dirty_rings)) { 1616 (intel_engine_flag(req->engine) & req->ctx->ppgtt->pd_dirty_rings)) {
1600 if (!USES_FULL_48BIT_PPGTT(req->i915) && 1617 if (!USES_FULL_48BIT_PPGTT(req->i915) &&
1601 !intel_vgpu_active(req->i915->dev)) { 1618 !intel_vgpu_active(req->i915)) {
1602 ret = intel_logical_ring_emit_pdps(req); 1619 ret = intel_logical_ring_emit_pdps(req);
1603 if (ret) 1620 if (ret)
1604 return ret; 1621 return ret;
@@ -1624,38 +1641,18 @@ static int gen8_emit_bb_start(struct drm_i915_gem_request *req,
1624 return 0; 1641 return 0;
1625} 1642}
1626 1643
1627static bool gen8_logical_ring_get_irq(struct intel_engine_cs *engine) 1644static void gen8_logical_ring_enable_irq(struct intel_engine_cs *engine)
1628{ 1645{
1629 struct drm_device *dev = engine->dev; 1646 struct drm_i915_private *dev_priv = engine->i915;
1630 struct drm_i915_private *dev_priv = dev->dev_private; 1647 I915_WRITE_IMR(engine,
1631 unsigned long flags; 1648 ~(engine->irq_enable_mask | engine->irq_keep_mask));
1632 1649 POSTING_READ_FW(RING_IMR(engine->mmio_base));
1633 if (WARN_ON(!intel_irqs_enabled(dev_priv)))
1634 return false;
1635
1636 spin_lock_irqsave(&dev_priv->irq_lock, flags);
1637 if (engine->irq_refcount++ == 0) {
1638 I915_WRITE_IMR(engine,
1639 ~(engine->irq_enable_mask | engine->irq_keep_mask));
1640 POSTING_READ(RING_IMR(engine->mmio_base));
1641 }
1642 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1643
1644 return true;
1645} 1650}
1646 1651
1647static void gen8_logical_ring_put_irq(struct intel_engine_cs *engine) 1652static void gen8_logical_ring_disable_irq(struct intel_engine_cs *engine)
1648{ 1653{
1649 struct drm_device *dev = engine->dev; 1654 struct drm_i915_private *dev_priv = engine->i915;
1650 struct drm_i915_private *dev_priv = dev->dev_private; 1655 I915_WRITE_IMR(engine, ~engine->irq_keep_mask);
1651 unsigned long flags;
1652
1653 spin_lock_irqsave(&dev_priv->irq_lock, flags);
1654 if (--engine->irq_refcount == 0) {
1655 I915_WRITE_IMR(engine, ~engine->irq_keep_mask);
1656 POSTING_READ(RING_IMR(engine->mmio_base));
1657 }
1658 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1659} 1656}
1660 1657
1661static int gen8_emit_flush(struct drm_i915_gem_request *request, 1658static int gen8_emit_flush(struct drm_i915_gem_request *request,
@@ -1664,8 +1661,7 @@ static int gen8_emit_flush(struct drm_i915_gem_request *request,
1664{ 1661{
1665 struct intel_ringbuffer *ringbuf = request->ringbuf; 1662 struct intel_ringbuffer *ringbuf = request->ringbuf;
1666 struct intel_engine_cs *engine = ringbuf->engine; 1663 struct intel_engine_cs *engine = ringbuf->engine;
1667 struct drm_device *dev = engine->dev; 1664 struct drm_i915_private *dev_priv = request->i915;
1668 struct drm_i915_private *dev_priv = dev->dev_private;
1669 uint32_t cmd; 1665 uint32_t cmd;
1670 int ret; 1666 int ret;
1671 1667
@@ -1734,7 +1730,7 @@ static int gen8_emit_flush_render(struct drm_i915_gem_request *request,
1734 * On GEN9: before VF_CACHE_INVALIDATE we need to emit a NULL 1730 * On GEN9: before VF_CACHE_INVALIDATE we need to emit a NULL
1735 * pipe control. 1731 * pipe control.
1736 */ 1732 */
1737 if (IS_GEN9(engine->dev)) 1733 if (IS_GEN9(request->i915))
1738 vf_flush_wa = true; 1734 vf_flush_wa = true;
1739 1735
1740 /* WaForGAMHang:kbl */ 1736 /* WaForGAMHang:kbl */
@@ -1793,16 +1789,6 @@ static int gen8_emit_flush_render(struct drm_i915_gem_request *request,
1793 return 0; 1789 return 0;
1794} 1790}
1795 1791
1796static u32 gen8_get_seqno(struct intel_engine_cs *engine)
1797{
1798 return intel_read_status_page(engine, I915_GEM_HWS_INDEX);
1799}
1800
1801static void gen8_set_seqno(struct intel_engine_cs *engine, u32 seqno)
1802{
1803 intel_write_status_page(engine, I915_GEM_HWS_INDEX, seqno);
1804}
1805
1806static void bxt_a_seqno_barrier(struct intel_engine_cs *engine) 1792static void bxt_a_seqno_barrier(struct intel_engine_cs *engine)
1807{ 1793{
1808 /* 1794 /*
@@ -1818,14 +1804,6 @@ static void bxt_a_seqno_barrier(struct intel_engine_cs *engine)
1818 intel_flush_status_page(engine, I915_GEM_HWS_INDEX); 1804 intel_flush_status_page(engine, I915_GEM_HWS_INDEX);
1819} 1805}
1820 1806
1821static void bxt_a_set_seqno(struct intel_engine_cs *engine, u32 seqno)
1822{
1823 intel_write_status_page(engine, I915_GEM_HWS_INDEX, seqno);
1824
1825 /* See bxt_a_get_seqno() explaining the reason for the clflush. */
1826 intel_flush_status_page(engine, I915_GEM_HWS_INDEX);
1827}
1828
1829/* 1807/*
1830 * Reserve space for 2 NOOPs at the end of each request to be 1808 * Reserve space for 2 NOOPs at the end of each request to be
1831 * used as a workaround for not being allowed to do lite 1809 * used as a workaround for not being allowed to do lite
@@ -1833,11 +1811,6 @@ static void bxt_a_set_seqno(struct intel_engine_cs *engine, u32 seqno)
1833 */ 1811 */
1834#define WA_TAIL_DWORDS 2 1812#define WA_TAIL_DWORDS 2
1835 1813
1836static inline u32 hws_seqno_address(struct intel_engine_cs *engine)
1837{
1838 return engine->status_page.gfx_addr + I915_GEM_HWS_INDEX_ADDR;
1839}
1840
1841static int gen8_emit_request(struct drm_i915_gem_request *request) 1814static int gen8_emit_request(struct drm_i915_gem_request *request)
1842{ 1815{
1843 struct intel_ringbuffer *ringbuf = request->ringbuf; 1816 struct intel_ringbuffer *ringbuf = request->ringbuf;
@@ -1853,10 +1826,10 @@ static int gen8_emit_request(struct drm_i915_gem_request *request)
1853 intel_logical_ring_emit(ringbuf, 1826 intel_logical_ring_emit(ringbuf,
1854 (MI_FLUSH_DW + 1) | MI_FLUSH_DW_OP_STOREDW); 1827 (MI_FLUSH_DW + 1) | MI_FLUSH_DW_OP_STOREDW);
1855 intel_logical_ring_emit(ringbuf, 1828 intel_logical_ring_emit(ringbuf,
1856 hws_seqno_address(request->engine) | 1829 intel_hws_seqno_address(request->engine) |
1857 MI_FLUSH_DW_USE_GTT); 1830 MI_FLUSH_DW_USE_GTT);
1858 intel_logical_ring_emit(ringbuf, 0); 1831 intel_logical_ring_emit(ringbuf, 0);
1859 intel_logical_ring_emit(ringbuf, i915_gem_request_get_seqno(request)); 1832 intel_logical_ring_emit(ringbuf, request->seqno);
1860 intel_logical_ring_emit(ringbuf, MI_USER_INTERRUPT); 1833 intel_logical_ring_emit(ringbuf, MI_USER_INTERRUPT);
1861 intel_logical_ring_emit(ringbuf, MI_NOOP); 1834 intel_logical_ring_emit(ringbuf, MI_NOOP);
1862 return intel_logical_ring_advance_and_submit(request); 1835 return intel_logical_ring_advance_and_submit(request);
@@ -1883,7 +1856,8 @@ static int gen8_emit_request_render(struct drm_i915_gem_request *request)
1883 (PIPE_CONTROL_GLOBAL_GTT_IVB | 1856 (PIPE_CONTROL_GLOBAL_GTT_IVB |
1884 PIPE_CONTROL_CS_STALL | 1857 PIPE_CONTROL_CS_STALL |
1885 PIPE_CONTROL_QW_WRITE)); 1858 PIPE_CONTROL_QW_WRITE));
1886 intel_logical_ring_emit(ringbuf, hws_seqno_address(request->engine)); 1859 intel_logical_ring_emit(ringbuf,
1860 intel_hws_seqno_address(request->engine));
1887 intel_logical_ring_emit(ringbuf, 0); 1861 intel_logical_ring_emit(ringbuf, 0);
1888 intel_logical_ring_emit(ringbuf, i915_gem_request_get_seqno(request)); 1862 intel_logical_ring_emit(ringbuf, i915_gem_request_get_seqno(request));
1889 /* We're thrashing one dword of HWS. */ 1863 /* We're thrashing one dword of HWS. */
@@ -1945,7 +1919,7 @@ static int gen8_init_rcs_context(struct drm_i915_gem_request *req)
1945/** 1919/**
1946 * intel_logical_ring_cleanup() - deallocate the Engine Command Streamer 1920 * intel_logical_ring_cleanup() - deallocate the Engine Command Streamer
1947 * 1921 *
1948 * @ring: Engine Command Streamer. 1922 * @engine: Engine Command Streamer.
1949 * 1923 *
1950 */ 1924 */
1951void intel_logical_ring_cleanup(struct intel_engine_cs *engine) 1925void intel_logical_ring_cleanup(struct intel_engine_cs *engine)
@@ -1962,7 +1936,7 @@ void intel_logical_ring_cleanup(struct intel_engine_cs *engine)
1962 if (WARN_ON(test_bit(TASKLET_STATE_SCHED, &engine->irq_tasklet.state))) 1936 if (WARN_ON(test_bit(TASKLET_STATE_SCHED, &engine->irq_tasklet.state)))
1963 tasklet_kill(&engine->irq_tasklet); 1937 tasklet_kill(&engine->irq_tasklet);
1964 1938
1965 dev_priv = engine->dev->dev_private; 1939 dev_priv = engine->i915;
1966 1940
1967 if (engine->buffer) { 1941 if (engine->buffer) {
1968 intel_logical_ring_stop(engine); 1942 intel_logical_ring_stop(engine);
@@ -1975,36 +1949,34 @@ void intel_logical_ring_cleanup(struct intel_engine_cs *engine)
1975 i915_cmd_parser_fini_ring(engine); 1949 i915_cmd_parser_fini_ring(engine);
1976 i915_gem_batch_pool_fini(&engine->batch_pool); 1950 i915_gem_batch_pool_fini(&engine->batch_pool);
1977 1951
1952 intel_engine_fini_breadcrumbs(engine);
1953
1978 if (engine->status_page.obj) { 1954 if (engine->status_page.obj) {
1979 i915_gem_object_unpin_map(engine->status_page.obj); 1955 i915_gem_object_unpin_map(engine->status_page.obj);
1980 engine->status_page.obj = NULL; 1956 engine->status_page.obj = NULL;
1981 } 1957 }
1958 intel_lr_context_unpin(dev_priv->kernel_context, engine);
1982 1959
1983 engine->idle_lite_restore_wa = 0; 1960 engine->idle_lite_restore_wa = 0;
1984 engine->disable_lite_restore_wa = false; 1961 engine->disable_lite_restore_wa = false;
1985 engine->ctx_desc_template = 0; 1962 engine->ctx_desc_template = 0;
1986 1963
1987 lrc_destroy_wa_ctx_obj(engine); 1964 lrc_destroy_wa_ctx_obj(engine);
1988 engine->dev = NULL; 1965 engine->i915 = NULL;
1989} 1966}
1990 1967
1991static void 1968static void
1992logical_ring_default_vfuncs(struct drm_device *dev, 1969logical_ring_default_vfuncs(struct intel_engine_cs *engine)
1993 struct intel_engine_cs *engine)
1994{ 1970{
1995 /* Default vfuncs which can be overriden by each engine. */ 1971 /* Default vfuncs which can be overriden by each engine. */
1996 engine->init_hw = gen8_init_common_ring; 1972 engine->init_hw = gen8_init_common_ring;
1997 engine->emit_request = gen8_emit_request; 1973 engine->emit_request = gen8_emit_request;
1998 engine->emit_flush = gen8_emit_flush; 1974 engine->emit_flush = gen8_emit_flush;
1999 engine->irq_get = gen8_logical_ring_get_irq; 1975 engine->irq_enable = gen8_logical_ring_enable_irq;
2000 engine->irq_put = gen8_logical_ring_put_irq; 1976 engine->irq_disable = gen8_logical_ring_disable_irq;
2001 engine->emit_bb_start = gen8_emit_bb_start; 1977 engine->emit_bb_start = gen8_emit_bb_start;
2002 engine->get_seqno = gen8_get_seqno; 1978 if (IS_BXT_REVID(engine->i915, 0, BXT_REVID_A1))
2003 engine->set_seqno = gen8_set_seqno;
2004 if (IS_BXT_REVID(dev, 0, BXT_REVID_A1)) {
2005 engine->irq_seqno_barrier = bxt_a_seqno_barrier; 1979 engine->irq_seqno_barrier = bxt_a_seqno_barrier;
2006 engine->set_seqno = bxt_a_set_seqno;
2007 }
2008} 1980}
2009 1981
2010static inline void 1982static inline void
@@ -2033,60 +2005,28 @@ lrc_setup_hws(struct intel_engine_cs *engine,
2033} 2005}
2034 2006
2035static int 2007static int
2036logical_ring_init(struct drm_device *dev, struct intel_engine_cs *engine) 2008logical_ring_init(struct intel_engine_cs *engine)
2037{ 2009{
2038 struct drm_i915_private *dev_priv = to_i915(dev); 2010 struct i915_gem_context *dctx = engine->i915->kernel_context;
2039 struct intel_context *dctx = dev_priv->kernel_context;
2040 enum forcewake_domains fw_domains;
2041 int ret; 2011 int ret;
2042 2012
2043 /* Intentionally left blank. */ 2013 ret = intel_engine_init_breadcrumbs(engine);
2044 engine->buffer = NULL; 2014 if (ret)
2045 2015 goto error;
2046 engine->dev = dev;
2047 INIT_LIST_HEAD(&engine->active_list);
2048 INIT_LIST_HEAD(&engine->request_list);
2049 i915_gem_batch_pool_init(dev, &engine->batch_pool);
2050 init_waitqueue_head(&engine->irq_queue);
2051
2052 INIT_LIST_HEAD(&engine->buffers);
2053 INIT_LIST_HEAD(&engine->execlist_queue);
2054 INIT_LIST_HEAD(&engine->execlist_retired_req_list);
2055 spin_lock_init(&engine->execlist_lock);
2056
2057 tasklet_init(&engine->irq_tasklet,
2058 intel_lrc_irq_handler, (unsigned long)engine);
2059
2060 logical_ring_init_platform_invariants(engine);
2061
2062 fw_domains = intel_uncore_forcewake_for_reg(dev_priv,
2063 RING_ELSP(engine),
2064 FW_REG_WRITE);
2065
2066 fw_domains |= intel_uncore_forcewake_for_reg(dev_priv,
2067 RING_CONTEXT_STATUS_PTR(engine),
2068 FW_REG_READ | FW_REG_WRITE);
2069
2070 fw_domains |= intel_uncore_forcewake_for_reg(dev_priv,
2071 RING_CONTEXT_STATUS_BUF_BASE(engine),
2072 FW_REG_READ);
2073
2074 engine->fw_domains = fw_domains;
2075 2016
2076 ret = i915_cmd_parser_init_ring(engine); 2017 ret = i915_cmd_parser_init_ring(engine);
2077 if (ret) 2018 if (ret)
2078 goto error; 2019 goto error;
2079 2020
2080 ret = intel_lr_context_deferred_alloc(dctx, engine); 2021 ret = execlists_context_deferred_alloc(dctx, engine);
2081 if (ret) 2022 if (ret)
2082 goto error; 2023 goto error;
2083 2024
2084 /* As this is the default context, always pin it */ 2025 /* As this is the default context, always pin it */
2085 ret = intel_lr_context_do_pin(dctx, engine); 2026 ret = intel_lr_context_pin(dctx, engine);
2086 if (ret) { 2027 if (ret) {
2087 DRM_ERROR( 2028 DRM_ERROR("Failed to pin context for %s: %d\n",
2088 "Failed to pin and map ringbuffer %s: %d\n", 2029 engine->name, ret);
2089 engine->name, ret);
2090 goto error; 2030 goto error;
2091 } 2031 }
2092 2032
@@ -2104,26 +2044,16 @@ error:
2104 return ret; 2044 return ret;
2105} 2045}
2106 2046
2107static int logical_render_ring_init(struct drm_device *dev) 2047static int logical_render_ring_init(struct intel_engine_cs *engine)
2108{ 2048{
2109 struct drm_i915_private *dev_priv = dev->dev_private; 2049 struct drm_i915_private *dev_priv = engine->i915;
2110 struct intel_engine_cs *engine = &dev_priv->engine[RCS];
2111 int ret; 2050 int ret;
2112 2051
2113 engine->name = "render ring"; 2052 if (HAS_L3_DPF(dev_priv))
2114 engine->id = RCS;
2115 engine->exec_id = I915_EXEC_RENDER;
2116 engine->guc_id = GUC_RENDER_ENGINE;
2117 engine->mmio_base = RENDER_RING_BASE;
2118
2119 logical_ring_default_irqs(engine, GEN8_RCS_IRQ_SHIFT);
2120 if (HAS_L3_DPF(dev))
2121 engine->irq_keep_mask |= GT_RENDER_L3_PARITY_ERROR_INTERRUPT; 2053 engine->irq_keep_mask |= GT_RENDER_L3_PARITY_ERROR_INTERRUPT;
2122 2054
2123 logical_ring_default_vfuncs(dev, engine);
2124
2125 /* Override some for render ring. */ 2055 /* Override some for render ring. */
2126 if (INTEL_INFO(dev)->gen >= 9) 2056 if (INTEL_GEN(dev_priv) >= 9)
2127 engine->init_hw = gen9_init_render_ring; 2057 engine->init_hw = gen9_init_render_ring;
2128 else 2058 else
2129 engine->init_hw = gen8_init_render_ring; 2059 engine->init_hw = gen8_init_render_ring;
@@ -2132,9 +2062,7 @@ static int logical_render_ring_init(struct drm_device *dev)
2132 engine->emit_flush = gen8_emit_flush_render; 2062 engine->emit_flush = gen8_emit_flush_render;
2133 engine->emit_request = gen8_emit_request_render; 2063 engine->emit_request = gen8_emit_request_render;
2134 2064
2135 engine->dev = dev; 2065 ret = intel_init_pipe_control(engine, 4096);
2136
2137 ret = intel_init_pipe_control(engine);
2138 if (ret) 2066 if (ret)
2139 return ret; 2067 return ret;
2140 2068
@@ -2149,7 +2077,7 @@ static int logical_render_ring_init(struct drm_device *dev)
2149 ret); 2077 ret);
2150 } 2078 }
2151 2079
2152 ret = logical_ring_init(dev, engine); 2080 ret = logical_ring_init(engine);
2153 if (ret) { 2081 if (ret) {
2154 lrc_destroy_wa_ctx_obj(engine); 2082 lrc_destroy_wa_ctx_obj(engine);
2155 } 2083 }
@@ -2157,133 +2085,164 @@ static int logical_render_ring_init(struct drm_device *dev)
2157 return ret; 2085 return ret;
2158} 2086}
2159 2087
2160static int logical_bsd_ring_init(struct drm_device *dev) 2088static const struct logical_ring_info {
2161{ 2089 const char *name;
2162 struct drm_i915_private *dev_priv = dev->dev_private; 2090 unsigned exec_id;
2163 struct intel_engine_cs *engine = &dev_priv->engine[VCS]; 2091 unsigned guc_id;
2164 2092 u32 mmio_base;
2165 engine->name = "bsd ring"; 2093 unsigned irq_shift;
2166 engine->id = VCS; 2094 int (*init)(struct intel_engine_cs *engine);
2167 engine->exec_id = I915_EXEC_BSD; 2095} logical_rings[] = {
2168 engine->guc_id = GUC_VIDEO_ENGINE; 2096 [RCS] = {
2169 engine->mmio_base = GEN6_BSD_RING_BASE; 2097 .name = "render ring",
2170 2098 .exec_id = I915_EXEC_RENDER,
2171 logical_ring_default_irqs(engine, GEN8_VCS1_IRQ_SHIFT); 2099 .guc_id = GUC_RENDER_ENGINE,
2172 logical_ring_default_vfuncs(dev, engine); 2100 .mmio_base = RENDER_RING_BASE,
2173 2101 .irq_shift = GEN8_RCS_IRQ_SHIFT,
2174 return logical_ring_init(dev, engine); 2102 .init = logical_render_ring_init,
2175} 2103 },
2104 [BCS] = {
2105 .name = "blitter ring",
2106 .exec_id = I915_EXEC_BLT,
2107 .guc_id = GUC_BLITTER_ENGINE,
2108 .mmio_base = BLT_RING_BASE,
2109 .irq_shift = GEN8_BCS_IRQ_SHIFT,
2110 .init = logical_ring_init,
2111 },
2112 [VCS] = {
2113 .name = "bsd ring",
2114 .exec_id = I915_EXEC_BSD,
2115 .guc_id = GUC_VIDEO_ENGINE,
2116 .mmio_base = GEN6_BSD_RING_BASE,
2117 .irq_shift = GEN8_VCS1_IRQ_SHIFT,
2118 .init = logical_ring_init,
2119 },
2120 [VCS2] = {
2121 .name = "bsd2 ring",
2122 .exec_id = I915_EXEC_BSD,
2123 .guc_id = GUC_VIDEO_ENGINE2,
2124 .mmio_base = GEN8_BSD2_RING_BASE,
2125 .irq_shift = GEN8_VCS2_IRQ_SHIFT,
2126 .init = logical_ring_init,
2127 },
2128 [VECS] = {
2129 .name = "video enhancement ring",
2130 .exec_id = I915_EXEC_VEBOX,
2131 .guc_id = GUC_VIDEOENHANCE_ENGINE,
2132 .mmio_base = VEBOX_RING_BASE,
2133 .irq_shift = GEN8_VECS_IRQ_SHIFT,
2134 .init = logical_ring_init,
2135 },
2136};
2176 2137
2177static int logical_bsd2_ring_init(struct drm_device *dev) 2138static struct intel_engine_cs *
2139logical_ring_setup(struct drm_i915_private *dev_priv, enum intel_engine_id id)
2178{ 2140{
2179 struct drm_i915_private *dev_priv = dev->dev_private; 2141 const struct logical_ring_info *info = &logical_rings[id];
2180 struct intel_engine_cs *engine = &dev_priv->engine[VCS2]; 2142 struct intel_engine_cs *engine = &dev_priv->engine[id];
2143 enum forcewake_domains fw_domains;
2181 2144
2182 engine->name = "bsd2 ring"; 2145 engine->id = id;
2183 engine->id = VCS2; 2146 engine->name = info->name;
2184 engine->exec_id = I915_EXEC_BSD; 2147 engine->exec_id = info->exec_id;
2185 engine->guc_id = GUC_VIDEO_ENGINE2; 2148 engine->guc_id = info->guc_id;
2186 engine->mmio_base = GEN8_BSD2_RING_BASE; 2149 engine->mmio_base = info->mmio_base;
2187 2150
2188 logical_ring_default_irqs(engine, GEN8_VCS2_IRQ_SHIFT); 2151 engine->i915 = dev_priv;
2189 logical_ring_default_vfuncs(dev, engine);
2190 2152
2191 return logical_ring_init(dev, engine); 2153 /* Intentionally left blank. */
2192} 2154 engine->buffer = NULL;
2193 2155
2194static int logical_blt_ring_init(struct drm_device *dev) 2156 fw_domains = intel_uncore_forcewake_for_reg(dev_priv,
2195{ 2157 RING_ELSP(engine),
2196 struct drm_i915_private *dev_priv = dev->dev_private; 2158 FW_REG_WRITE);
2197 struct intel_engine_cs *engine = &dev_priv->engine[BCS];
2198 2159
2199 engine->name = "blitter ring"; 2160 fw_domains |= intel_uncore_forcewake_for_reg(dev_priv,
2200 engine->id = BCS; 2161 RING_CONTEXT_STATUS_PTR(engine),
2201 engine->exec_id = I915_EXEC_BLT; 2162 FW_REG_READ | FW_REG_WRITE);
2202 engine->guc_id = GUC_BLITTER_ENGINE;
2203 engine->mmio_base = BLT_RING_BASE;
2204 2163
2205 logical_ring_default_irqs(engine, GEN8_BCS_IRQ_SHIFT); 2164 fw_domains |= intel_uncore_forcewake_for_reg(dev_priv,
2206 logical_ring_default_vfuncs(dev, engine); 2165 RING_CONTEXT_STATUS_BUF_BASE(engine),
2166 FW_REG_READ);
2207 2167
2208 return logical_ring_init(dev, engine); 2168 engine->fw_domains = fw_domains;
2209}
2210 2169
2211static int logical_vebox_ring_init(struct drm_device *dev) 2170 INIT_LIST_HEAD(&engine->active_list);
2212{ 2171 INIT_LIST_HEAD(&engine->request_list);
2213 struct drm_i915_private *dev_priv = dev->dev_private; 2172 INIT_LIST_HEAD(&engine->buffers);
2214 struct intel_engine_cs *engine = &dev_priv->engine[VECS]; 2173 INIT_LIST_HEAD(&engine->execlist_queue);
2174 spin_lock_init(&engine->execlist_lock);
2215 2175
2216 engine->name = "video enhancement ring"; 2176 tasklet_init(&engine->irq_tasklet,
2217 engine->id = VECS; 2177 intel_lrc_irq_handler, (unsigned long)engine);
2218 engine->exec_id = I915_EXEC_VEBOX;
2219 engine->guc_id = GUC_VIDEOENHANCE_ENGINE;
2220 engine->mmio_base = VEBOX_RING_BASE;
2221 2178
2222 logical_ring_default_irqs(engine, GEN8_VECS_IRQ_SHIFT); 2179 logical_ring_init_platform_invariants(engine);
2223 logical_ring_default_vfuncs(dev, engine); 2180 logical_ring_default_vfuncs(engine);
2181 logical_ring_default_irqs(engine, info->irq_shift);
2182
2183 intel_engine_init_hangcheck(engine);
2184 i915_gem_batch_pool_init(&dev_priv->drm, &engine->batch_pool);
2224 2185
2225 return logical_ring_init(dev, engine); 2186 return engine;
2226} 2187}
2227 2188
2228/** 2189/**
2229 * intel_logical_rings_init() - allocate, populate and init the Engine Command Streamers 2190 * intel_logical_rings_init() - allocate, populate and init the Engine Command Streamers
2230 * @dev: DRM device. 2191 * @dev: DRM device.
2231 * 2192 *
2232 * This function inits the engines for an Execlists submission style (the equivalent in the 2193 * This function inits the engines for an Execlists submission style (the
2233 * legacy ringbuffer submission world would be i915_gem_init_engines). It does it only for 2194 * equivalent in the legacy ringbuffer submission world would be
2234 * those engines that are present in the hardware. 2195 * i915_gem_init_engines). It does it only for those engines that are present in
2196 * the hardware.
2235 * 2197 *
2236 * Return: non-zero if the initialization failed. 2198 * Return: non-zero if the initialization failed.
2237 */ 2199 */
2238int intel_logical_rings_init(struct drm_device *dev) 2200int intel_logical_rings_init(struct drm_device *dev)
2239{ 2201{
2240 struct drm_i915_private *dev_priv = dev->dev_private; 2202 struct drm_i915_private *dev_priv = to_i915(dev);
2203 unsigned int mask = 0;
2204 unsigned int i;
2241 int ret; 2205 int ret;
2242 2206
2243 ret = logical_render_ring_init(dev); 2207 WARN_ON(INTEL_INFO(dev_priv)->ring_mask &
2244 if (ret) 2208 GENMASK(sizeof(mask) * BITS_PER_BYTE - 1, I915_NUM_ENGINES));
2245 return ret;
2246 2209
2247 if (HAS_BSD(dev)) { 2210 for (i = 0; i < ARRAY_SIZE(logical_rings); i++) {
2248 ret = logical_bsd_ring_init(dev); 2211 if (!HAS_ENGINE(dev_priv, i))
2249 if (ret) 2212 continue;
2250 goto cleanup_render_ring;
2251 }
2252 2213
2253 if (HAS_BLT(dev)) { 2214 if (!logical_rings[i].init)
2254 ret = logical_blt_ring_init(dev); 2215 continue;
2255 if (ret)
2256 goto cleanup_bsd_ring;
2257 }
2258 2216
2259 if (HAS_VEBOX(dev)) { 2217 ret = logical_rings[i].init(logical_ring_setup(dev_priv, i));
2260 ret = logical_vebox_ring_init(dev);
2261 if (ret) 2218 if (ret)
2262 goto cleanup_blt_ring; 2219 goto cleanup;
2220
2221 mask |= ENGINE_MASK(i);
2263 } 2222 }
2264 2223
2265 if (HAS_BSD2(dev)) { 2224 /*
2266 ret = logical_bsd2_ring_init(dev); 2225 * Catch failures to update logical_rings table when the new engines
2267 if (ret) 2226 * are added to the driver by a warning and disabling the forgotten
2268 goto cleanup_vebox_ring; 2227 * engines.
2228 */
2229 if (WARN_ON(mask != INTEL_INFO(dev_priv)->ring_mask)) {
2230 struct intel_device_info *info =
2231 (struct intel_device_info *)&dev_priv->info;
2232 info->ring_mask = mask;
2269 } 2233 }
2270 2234
2271 return 0; 2235 return 0;
2272 2236
2273cleanup_vebox_ring: 2237cleanup:
2274 intel_logical_ring_cleanup(&dev_priv->engine[VECS]); 2238 for (i = 0; i < I915_NUM_ENGINES; i++)
2275cleanup_blt_ring: 2239 intel_logical_ring_cleanup(&dev_priv->engine[i]);
2276 intel_logical_ring_cleanup(&dev_priv->engine[BCS]);
2277cleanup_bsd_ring:
2278 intel_logical_ring_cleanup(&dev_priv->engine[VCS]);
2279cleanup_render_ring:
2280 intel_logical_ring_cleanup(&dev_priv->engine[RCS]);
2281 2240
2282 return ret; 2241 return ret;
2283} 2242}
2284 2243
2285static u32 2244static u32
2286make_rpcs(struct drm_device *dev) 2245make_rpcs(struct drm_i915_private *dev_priv)
2287{ 2246{
2288 u32 rpcs = 0; 2247 u32 rpcs = 0;
2289 2248
@@ -2291,7 +2250,7 @@ make_rpcs(struct drm_device *dev)
2291 * No explicit RPCS request is needed to ensure full 2250 * No explicit RPCS request is needed to ensure full
2292 * slice/subslice/EU enablement prior to Gen9. 2251 * slice/subslice/EU enablement prior to Gen9.
2293 */ 2252 */
2294 if (INTEL_INFO(dev)->gen < 9) 2253 if (INTEL_GEN(dev_priv) < 9)
2295 return 0; 2254 return 0;
2296 2255
2297 /* 2256 /*
@@ -2300,24 +2259,24 @@ make_rpcs(struct drm_device *dev)
2300 * must make an explicit request through RPCS for full 2259 * must make an explicit request through RPCS for full
2301 * enablement. 2260 * enablement.
2302 */ 2261 */
2303 if (INTEL_INFO(dev)->has_slice_pg) { 2262 if (INTEL_INFO(dev_priv)->has_slice_pg) {
2304 rpcs |= GEN8_RPCS_S_CNT_ENABLE; 2263 rpcs |= GEN8_RPCS_S_CNT_ENABLE;
2305 rpcs |= INTEL_INFO(dev)->slice_total << 2264 rpcs |= INTEL_INFO(dev_priv)->slice_total <<
2306 GEN8_RPCS_S_CNT_SHIFT; 2265 GEN8_RPCS_S_CNT_SHIFT;
2307 rpcs |= GEN8_RPCS_ENABLE; 2266 rpcs |= GEN8_RPCS_ENABLE;
2308 } 2267 }
2309 2268
2310 if (INTEL_INFO(dev)->has_subslice_pg) { 2269 if (INTEL_INFO(dev_priv)->has_subslice_pg) {
2311 rpcs |= GEN8_RPCS_SS_CNT_ENABLE; 2270 rpcs |= GEN8_RPCS_SS_CNT_ENABLE;
2312 rpcs |= INTEL_INFO(dev)->subslice_per_slice << 2271 rpcs |= INTEL_INFO(dev_priv)->subslice_per_slice <<
2313 GEN8_RPCS_SS_CNT_SHIFT; 2272 GEN8_RPCS_SS_CNT_SHIFT;
2314 rpcs |= GEN8_RPCS_ENABLE; 2273 rpcs |= GEN8_RPCS_ENABLE;
2315 } 2274 }
2316 2275
2317 if (INTEL_INFO(dev)->has_eu_pg) { 2276 if (INTEL_INFO(dev_priv)->has_eu_pg) {
2318 rpcs |= INTEL_INFO(dev)->eu_per_subslice << 2277 rpcs |= INTEL_INFO(dev_priv)->eu_per_subslice <<
2319 GEN8_RPCS_EU_MIN_SHIFT; 2278 GEN8_RPCS_EU_MIN_SHIFT;
2320 rpcs |= INTEL_INFO(dev)->eu_per_subslice << 2279 rpcs |= INTEL_INFO(dev_priv)->eu_per_subslice <<
2321 GEN8_RPCS_EU_MAX_SHIFT; 2280 GEN8_RPCS_EU_MAX_SHIFT;
2322 rpcs |= GEN8_RPCS_ENABLE; 2281 rpcs |= GEN8_RPCS_ENABLE;
2323 } 2282 }
@@ -2329,9 +2288,9 @@ static u32 intel_lr_indirect_ctx_offset(struct intel_engine_cs *engine)
2329{ 2288{
2330 u32 indirect_ctx_offset; 2289 u32 indirect_ctx_offset;
2331 2290
2332 switch (INTEL_INFO(engine->dev)->gen) { 2291 switch (INTEL_GEN(engine->i915)) {
2333 default: 2292 default:
2334 MISSING_CASE(INTEL_INFO(engine->dev)->gen); 2293 MISSING_CASE(INTEL_GEN(engine->i915));
2335 /* fall through */ 2294 /* fall through */
2336 case 9: 2295 case 9:
2337 indirect_ctx_offset = 2296 indirect_ctx_offset =
@@ -2347,13 +2306,12 @@ static u32 intel_lr_indirect_ctx_offset(struct intel_engine_cs *engine)
2347} 2306}
2348 2307
2349static int 2308static int
2350populate_lr_context(struct intel_context *ctx, 2309populate_lr_context(struct i915_gem_context *ctx,
2351 struct drm_i915_gem_object *ctx_obj, 2310 struct drm_i915_gem_object *ctx_obj,
2352 struct intel_engine_cs *engine, 2311 struct intel_engine_cs *engine,
2353 struct intel_ringbuffer *ringbuf) 2312 struct intel_ringbuffer *ringbuf)
2354{ 2313{
2355 struct drm_device *dev = engine->dev; 2314 struct drm_i915_private *dev_priv = ctx->i915;
2356 struct drm_i915_private *dev_priv = dev->dev_private;
2357 struct i915_hw_ppgtt *ppgtt = ctx->ppgtt; 2315 struct i915_hw_ppgtt *ppgtt = ctx->ppgtt;
2358 void *vaddr; 2316 void *vaddr;
2359 u32 *reg_state; 2317 u32 *reg_state;
@@ -2391,7 +2349,7 @@ populate_lr_context(struct intel_context *ctx,
2391 RING_CONTEXT_CONTROL(engine), 2349 RING_CONTEXT_CONTROL(engine),
2392 _MASKED_BIT_ENABLE(CTX_CTRL_INHIBIT_SYN_CTX_SWITCH | 2350 _MASKED_BIT_ENABLE(CTX_CTRL_INHIBIT_SYN_CTX_SWITCH |
2393 CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT | 2351 CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT |
2394 (HAS_RESOURCE_STREAMER(dev) ? 2352 (HAS_RESOURCE_STREAMER(dev_priv) ?
2395 CTX_CTRL_RS_CTX_ENABLE : 0))); 2353 CTX_CTRL_RS_CTX_ENABLE : 0)));
2396 ASSIGN_CTX_REG(reg_state, CTX_RING_HEAD, RING_HEAD(engine->mmio_base), 2354 ASSIGN_CTX_REG(reg_state, CTX_RING_HEAD, RING_HEAD(engine->mmio_base),
2397 0); 2355 0);
@@ -2480,7 +2438,7 @@ populate_lr_context(struct intel_context *ctx,
2480 if (engine->id == RCS) { 2438 if (engine->id == RCS) {
2481 reg_state[CTX_LRI_HEADER_2] = MI_LOAD_REGISTER_IMM(1); 2439 reg_state[CTX_LRI_HEADER_2] = MI_LOAD_REGISTER_IMM(1);
2482 ASSIGN_CTX_REG(reg_state, CTX_R_PWR_CLK_STATE, GEN8_R_PWR_CLK_STATE, 2440 ASSIGN_CTX_REG(reg_state, CTX_R_PWR_CLK_STATE, GEN8_R_PWR_CLK_STATE,
2483 make_rpcs(dev)); 2441 make_rpcs(dev_priv));
2484 } 2442 }
2485 2443
2486 i915_gem_object_unpin_map(ctx_obj); 2444 i915_gem_object_unpin_map(ctx_obj);
@@ -2489,39 +2447,8 @@ populate_lr_context(struct intel_context *ctx,
2489} 2447}
2490 2448
2491/** 2449/**
2492 * intel_lr_context_free() - free the LRC specific bits of a context
2493 * @ctx: the LR context to free.
2494 *
2495 * The real context freeing is done in i915_gem_context_free: this only
2496 * takes care of the bits that are LRC related: the per-engine backing
2497 * objects and the logical ringbuffer.
2498 */
2499void intel_lr_context_free(struct intel_context *ctx)
2500{
2501 int i;
2502
2503 for (i = I915_NUM_ENGINES; --i >= 0; ) {
2504 struct intel_ringbuffer *ringbuf = ctx->engine[i].ringbuf;
2505 struct drm_i915_gem_object *ctx_obj = ctx->engine[i].state;
2506
2507 if (!ctx_obj)
2508 continue;
2509
2510 if (ctx == ctx->i915->kernel_context) {
2511 intel_unpin_ringbuffer_obj(ringbuf);
2512 i915_gem_object_ggtt_unpin(ctx_obj);
2513 i915_gem_object_unpin_map(ctx_obj);
2514 }
2515
2516 WARN_ON(ctx->engine[i].pin_count);
2517 intel_ringbuffer_free(ringbuf);
2518 drm_gem_object_unreference(&ctx_obj->base);
2519 }
2520}
2521
2522/**
2523 * intel_lr_context_size() - return the size of the context for an engine 2450 * intel_lr_context_size() - return the size of the context for an engine
2524 * @ring: which engine to find the context size for 2451 * @engine: which engine to find the context size for
2525 * 2452 *
2526 * Each engine may require a different amount of space for a context image, 2453 * Each engine may require a different amount of space for a context image,
2527 * so when allocating (or copying) an image, this function can be used to 2454 * so when allocating (or copying) an image, this function can be used to
@@ -2537,11 +2464,11 @@ uint32_t intel_lr_context_size(struct intel_engine_cs *engine)
2537{ 2464{
2538 int ret = 0; 2465 int ret = 0;
2539 2466
2540 WARN_ON(INTEL_INFO(engine->dev)->gen < 8); 2467 WARN_ON(INTEL_GEN(engine->i915) < 8);
2541 2468
2542 switch (engine->id) { 2469 switch (engine->id) {
2543 case RCS: 2470 case RCS:
2544 if (INTEL_INFO(engine->dev)->gen >= 9) 2471 if (INTEL_GEN(engine->i915) >= 9)
2545 ret = GEN9_LR_CONTEXT_RENDER_SIZE; 2472 ret = GEN9_LR_CONTEXT_RENDER_SIZE;
2546 else 2473 else
2547 ret = GEN8_LR_CONTEXT_RENDER_SIZE; 2474 ret = GEN8_LR_CONTEXT_RENDER_SIZE;
@@ -2558,9 +2485,9 @@ uint32_t intel_lr_context_size(struct intel_engine_cs *engine)
2558} 2485}
2559 2486
2560/** 2487/**
2561 * intel_lr_context_deferred_alloc() - create the LRC specific bits of a context 2488 * execlists_context_deferred_alloc() - create the LRC specific bits of a context
2562 * @ctx: LR context to create. 2489 * @ctx: LR context to create.
2563 * @ring: engine to be used with the context. 2490 * @engine: engine to be used with the context.
2564 * 2491 *
2565 * This function can be called more than once, with different engines, if we plan 2492 * This function can be called more than once, with different engines, if we plan
2566 * to use the context with them. The context backing objects and the ringbuffers 2493 * to use the context with them. The context backing objects and the ringbuffers
@@ -2570,31 +2497,29 @@ uint32_t intel_lr_context_size(struct intel_engine_cs *engine)
2570 * 2497 *
2571 * Return: non-zero on error. 2498 * Return: non-zero on error.
2572 */ 2499 */
2573 2500static int execlists_context_deferred_alloc(struct i915_gem_context *ctx,
2574int intel_lr_context_deferred_alloc(struct intel_context *ctx, 2501 struct intel_engine_cs *engine)
2575 struct intel_engine_cs *engine)
2576{ 2502{
2577 struct drm_device *dev = engine->dev;
2578 struct drm_i915_gem_object *ctx_obj; 2503 struct drm_i915_gem_object *ctx_obj;
2504 struct intel_context *ce = &ctx->engine[engine->id];
2579 uint32_t context_size; 2505 uint32_t context_size;
2580 struct intel_ringbuffer *ringbuf; 2506 struct intel_ringbuffer *ringbuf;
2581 int ret; 2507 int ret;
2582 2508
2583 WARN_ON(ctx->legacy_hw_ctx.rcs_state != NULL); 2509 WARN_ON(ce->state);
2584 WARN_ON(ctx->engine[engine->id].state);
2585 2510
2586 context_size = round_up(intel_lr_context_size(engine), 4096); 2511 context_size = round_up(intel_lr_context_size(engine), 4096);
2587 2512
2588 /* One extra page as the sharing data between driver and GuC */ 2513 /* One extra page as the sharing data between driver and GuC */
2589 context_size += PAGE_SIZE * LRC_PPHWSP_PN; 2514 context_size += PAGE_SIZE * LRC_PPHWSP_PN;
2590 2515
2591 ctx_obj = i915_gem_alloc_object(dev, context_size); 2516 ctx_obj = i915_gem_object_create(&ctx->i915->drm, context_size);
2592 if (!ctx_obj) { 2517 if (IS_ERR(ctx_obj)) {
2593 DRM_DEBUG_DRIVER("Alloc LRC backing obj failed.\n"); 2518 DRM_DEBUG_DRIVER("Alloc LRC backing obj failed.\n");
2594 return -ENOMEM; 2519 return PTR_ERR(ctx_obj);
2595 } 2520 }
2596 2521
2597 ringbuf = intel_engine_create_ringbuffer(engine, 4 * PAGE_SIZE); 2522 ringbuf = intel_engine_create_ringbuffer(engine, ctx->ring_size);
2598 if (IS_ERR(ringbuf)) { 2523 if (IS_ERR(ringbuf)) {
2599 ret = PTR_ERR(ringbuf); 2524 ret = PTR_ERR(ringbuf);
2600 goto error_deref_obj; 2525 goto error_deref_obj;
@@ -2606,48 +2531,29 @@ int intel_lr_context_deferred_alloc(struct intel_context *ctx,
2606 goto error_ringbuf; 2531 goto error_ringbuf;
2607 } 2532 }
2608 2533
2609 ctx->engine[engine->id].ringbuf = ringbuf; 2534 ce->ringbuf = ringbuf;
2610 ctx->engine[engine->id].state = ctx_obj; 2535 ce->state = ctx_obj;
2536 ce->initialised = engine->init_context == NULL;
2611 2537
2612 if (ctx != ctx->i915->kernel_context && engine->init_context) {
2613 struct drm_i915_gem_request *req;
2614
2615 req = i915_gem_request_alloc(engine, ctx);
2616 if (IS_ERR(req)) {
2617 ret = PTR_ERR(req);
2618 DRM_ERROR("ring create req: %d\n", ret);
2619 goto error_ringbuf;
2620 }
2621
2622 ret = engine->init_context(req);
2623 i915_add_request_no_flush(req);
2624 if (ret) {
2625 DRM_ERROR("ring init context: %d\n",
2626 ret);
2627 goto error_ringbuf;
2628 }
2629 }
2630 return 0; 2538 return 0;
2631 2539
2632error_ringbuf: 2540error_ringbuf:
2633 intel_ringbuffer_free(ringbuf); 2541 intel_ringbuffer_free(ringbuf);
2634error_deref_obj: 2542error_deref_obj:
2635 drm_gem_object_unreference(&ctx_obj->base); 2543 drm_gem_object_unreference(&ctx_obj->base);
2636 ctx->engine[engine->id].ringbuf = NULL; 2544 ce->ringbuf = NULL;
2637 ctx->engine[engine->id].state = NULL; 2545 ce->state = NULL;
2638 return ret; 2546 return ret;
2639} 2547}
2640 2548
2641void intel_lr_context_reset(struct drm_i915_private *dev_priv, 2549void intel_lr_context_reset(struct drm_i915_private *dev_priv,
2642 struct intel_context *ctx) 2550 struct i915_gem_context *ctx)
2643{ 2551{
2644 struct intel_engine_cs *engine; 2552 struct intel_engine_cs *engine;
2645 2553
2646 for_each_engine(engine, dev_priv) { 2554 for_each_engine(engine, dev_priv) {
2647 struct drm_i915_gem_object *ctx_obj = 2555 struct intel_context *ce = &ctx->engine[engine->id];
2648 ctx->engine[engine->id].state; 2556 struct drm_i915_gem_object *ctx_obj = ce->state;
2649 struct intel_ringbuffer *ringbuf =
2650 ctx->engine[engine->id].ringbuf;
2651 void *vaddr; 2557 void *vaddr;
2652 uint32_t *reg_state; 2558 uint32_t *reg_state;
2653 2559
@@ -2666,7 +2572,7 @@ void intel_lr_context_reset(struct drm_i915_private *dev_priv,
2666 2572
2667 i915_gem_object_unpin_map(ctx_obj); 2573 i915_gem_object_unpin_map(ctx_obj);
2668 2574
2669 ringbuf->head = 0; 2575 ce->ringbuf->head = 0;
2670 ringbuf->tail = 0; 2576 ce->ringbuf->tail = 0;
2671 } 2577 }
2672} 2578}