diff options
Diffstat (limited to 'drivers/gpu/drm/i915/intel_lrc.c')
-rw-r--r-- | drivers/gpu/drm/i915/intel_lrc.c | 964 |
1 files changed, 435 insertions, 529 deletions
diff --git a/drivers/gpu/drm/i915/intel_lrc.c b/drivers/gpu/drm/i915/intel_lrc.c index 7f2d8415ed8b..414ddda43922 100644 --- a/drivers/gpu/drm/i915/intel_lrc.c +++ b/drivers/gpu/drm/i915/intel_lrc.c | |||
@@ -208,31 +208,27 @@ | |||
208 | } while (0) | 208 | } while (0) |
209 | 209 | ||
210 | enum { | 210 | enum { |
211 | ADVANCED_CONTEXT = 0, | ||
212 | LEGACY_32B_CONTEXT, | ||
213 | ADVANCED_AD_CONTEXT, | ||
214 | LEGACY_64B_CONTEXT | ||
215 | }; | ||
216 | #define GEN8_CTX_ADDRESSING_MODE_SHIFT 3 | ||
217 | #define GEN8_CTX_ADDRESSING_MODE(dev) (USES_FULL_48BIT_PPGTT(dev) ?\ | ||
218 | LEGACY_64B_CONTEXT :\ | ||
219 | LEGACY_32B_CONTEXT) | ||
220 | enum { | ||
221 | FAULT_AND_HANG = 0, | 211 | FAULT_AND_HANG = 0, |
222 | FAULT_AND_HALT, /* Debug only */ | 212 | FAULT_AND_HALT, /* Debug only */ |
223 | FAULT_AND_STREAM, | 213 | FAULT_AND_STREAM, |
224 | FAULT_AND_CONTINUE /* Unsupported */ | 214 | FAULT_AND_CONTINUE /* Unsupported */ |
225 | }; | 215 | }; |
226 | #define GEN8_CTX_ID_SHIFT 32 | 216 | #define GEN8_CTX_ID_SHIFT 32 |
217 | #define GEN8_CTX_ID_WIDTH 21 | ||
227 | #define GEN8_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT 0x17 | 218 | #define GEN8_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT 0x17 |
228 | #define GEN9_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT 0x26 | 219 | #define GEN9_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT 0x26 |
229 | 220 | ||
230 | static int intel_lr_context_pin(struct intel_context *ctx, | 221 | /* Typical size of the average request (2 pipecontrols and a MI_BB) */ |
222 | #define EXECLISTS_REQUEST_SIZE 64 /* bytes */ | ||
223 | |||
224 | static int execlists_context_deferred_alloc(struct i915_gem_context *ctx, | ||
225 | struct intel_engine_cs *engine); | ||
226 | static int intel_lr_context_pin(struct i915_gem_context *ctx, | ||
231 | struct intel_engine_cs *engine); | 227 | struct intel_engine_cs *engine); |
232 | 228 | ||
233 | /** | 229 | /** |
234 | * intel_sanitize_enable_execlists() - sanitize i915.enable_execlists | 230 | * intel_sanitize_enable_execlists() - sanitize i915.enable_execlists |
235 | * @dev: DRM device. | 231 | * @dev_priv: i915 device private |
236 | * @enable_execlists: value of i915.enable_execlists module parameter. | 232 | * @enable_execlists: value of i915.enable_execlists module parameter. |
237 | * | 233 | * |
238 | * Only certain platforms support Execlists (the prerequisites being | 234 | * Only certain platforms support Execlists (the prerequisites being |
@@ -240,23 +236,22 @@ static int intel_lr_context_pin(struct intel_context *ctx, | |||
240 | * | 236 | * |
241 | * Return: 1 if Execlists is supported and has to be enabled. | 237 | * Return: 1 if Execlists is supported and has to be enabled. |
242 | */ | 238 | */ |
243 | int intel_sanitize_enable_execlists(struct drm_device *dev, int enable_execlists) | 239 | int intel_sanitize_enable_execlists(struct drm_i915_private *dev_priv, int enable_execlists) |
244 | { | 240 | { |
245 | WARN_ON(i915.enable_ppgtt == -1); | ||
246 | |||
247 | /* On platforms with execlist available, vGPU will only | 241 | /* On platforms with execlist available, vGPU will only |
248 | * support execlist mode, no ring buffer mode. | 242 | * support execlist mode, no ring buffer mode. |
249 | */ | 243 | */ |
250 | if (HAS_LOGICAL_RING_CONTEXTS(dev) && intel_vgpu_active(dev)) | 244 | if (HAS_LOGICAL_RING_CONTEXTS(dev_priv) && intel_vgpu_active(dev_priv)) |
251 | return 1; | 245 | return 1; |
252 | 246 | ||
253 | if (INTEL_INFO(dev)->gen >= 9) | 247 | if (INTEL_GEN(dev_priv) >= 9) |
254 | return 1; | 248 | return 1; |
255 | 249 | ||
256 | if (enable_execlists == 0) | 250 | if (enable_execlists == 0) |
257 | return 0; | 251 | return 0; |
258 | 252 | ||
259 | if (HAS_LOGICAL_RING_CONTEXTS(dev) && USES_PPGTT(dev) && | 253 | if (HAS_LOGICAL_RING_CONTEXTS(dev_priv) && |
254 | USES_PPGTT(dev_priv) && | ||
260 | i915.use_mmio_flip >= 0) | 255 | i915.use_mmio_flip >= 0) |
261 | return 1; | 256 | return 1; |
262 | 257 | ||
@@ -266,19 +261,17 @@ int intel_sanitize_enable_execlists(struct drm_device *dev, int enable_execlists | |||
266 | static void | 261 | static void |
267 | logical_ring_init_platform_invariants(struct intel_engine_cs *engine) | 262 | logical_ring_init_platform_invariants(struct intel_engine_cs *engine) |
268 | { | 263 | { |
269 | struct drm_device *dev = engine->dev; | 264 | struct drm_i915_private *dev_priv = engine->i915; |
270 | 265 | ||
271 | if (IS_GEN8(dev) || IS_GEN9(dev)) | 266 | if (IS_GEN8(dev_priv) || IS_GEN9(dev_priv)) |
272 | engine->idle_lite_restore_wa = ~0; | 267 | engine->idle_lite_restore_wa = ~0; |
273 | 268 | ||
274 | engine->disable_lite_restore_wa = (IS_SKL_REVID(dev, 0, SKL_REVID_B0) || | 269 | engine->disable_lite_restore_wa = (IS_SKL_REVID(dev_priv, 0, SKL_REVID_B0) || |
275 | IS_BXT_REVID(dev, 0, BXT_REVID_A1)) && | 270 | IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) && |
276 | (engine->id == VCS || engine->id == VCS2); | 271 | (engine->id == VCS || engine->id == VCS2); |
277 | 272 | ||
278 | engine->ctx_desc_template = GEN8_CTX_VALID; | 273 | engine->ctx_desc_template = GEN8_CTX_VALID; |
279 | engine->ctx_desc_template |= GEN8_CTX_ADDRESSING_MODE(dev) << | 274 | if (IS_GEN8(dev_priv)) |
280 | GEN8_CTX_ADDRESSING_MODE_SHIFT; | ||
281 | if (IS_GEN8(dev)) | ||
282 | engine->ctx_desc_template |= GEN8_CTX_L3LLC_COHERENT; | 275 | engine->ctx_desc_template |= GEN8_CTX_L3LLC_COHERENT; |
283 | engine->ctx_desc_template |= GEN8_CTX_PRIVILEGE; | 276 | engine->ctx_desc_template |= GEN8_CTX_PRIVILEGE; |
284 | 277 | ||
@@ -297,7 +290,7 @@ logical_ring_init_platform_invariants(struct intel_engine_cs *engine) | |||
297 | * descriptor for a pinned context | 290 | * descriptor for a pinned context |
298 | * | 291 | * |
299 | * @ctx: Context to work on | 292 | * @ctx: Context to work on |
300 | * @ring: Engine the descriptor will be used with | 293 | * @engine: Engine the descriptor will be used with |
301 | * | 294 | * |
302 | * The context descriptor encodes various attributes of a context, | 295 | * The context descriptor encodes various attributes of a context, |
303 | * including its GTT address and some flags. Because it's fairly | 296 | * including its GTT address and some flags. Because it's fairly |
@@ -305,62 +298,42 @@ logical_ring_init_platform_invariants(struct intel_engine_cs *engine) | |||
305 | * which remains valid until the context is unpinned. | 298 | * which remains valid until the context is unpinned. |
306 | * | 299 | * |
307 | * This is what a descriptor looks like, from LSB to MSB: | 300 | * This is what a descriptor looks like, from LSB to MSB: |
308 | * bits 0-11: flags, GEN8_CTX_* (cached in ctx_desc_template) | 301 | * bits 0-11: flags, GEN8_CTX_* (cached in ctx_desc_template) |
309 | * bits 12-31: LRCA, GTT address of (the HWSP of) this context | 302 | * bits 12-31: LRCA, GTT address of (the HWSP of) this context |
310 | * bits 32-51: ctx ID, a globally unique tag (the LRCA again!) | 303 | * bits 32-52: ctx ID, a globally unique tag |
311 | * bits 52-63: reserved, may encode the engine ID (for GuC) | 304 | * bits 53-54: mbz, reserved for use by hardware |
305 | * bits 55-63: group ID, currently unused and set to 0 | ||
312 | */ | 306 | */ |
313 | static void | 307 | static void |
314 | intel_lr_context_descriptor_update(struct intel_context *ctx, | 308 | intel_lr_context_descriptor_update(struct i915_gem_context *ctx, |
315 | struct intel_engine_cs *engine) | 309 | struct intel_engine_cs *engine) |
316 | { | 310 | { |
317 | uint64_t lrca, desc; | 311 | struct intel_context *ce = &ctx->engine[engine->id]; |
312 | u64 desc; | ||
318 | 313 | ||
319 | lrca = ctx->engine[engine->id].lrc_vma->node.start + | 314 | BUILD_BUG_ON(MAX_CONTEXT_HW_ID > (1<<GEN8_CTX_ID_WIDTH)); |
320 | LRC_PPHWSP_PN * PAGE_SIZE; | ||
321 | 315 | ||
322 | desc = engine->ctx_desc_template; /* bits 0-11 */ | 316 | desc = ctx->desc_template; /* bits 3-4 */ |
323 | desc |= lrca; /* bits 12-31 */ | 317 | desc |= engine->ctx_desc_template; /* bits 0-11 */ |
324 | desc |= (lrca >> PAGE_SHIFT) << GEN8_CTX_ID_SHIFT; /* bits 32-51 */ | 318 | desc |= ce->lrc_vma->node.start + LRC_PPHWSP_PN * PAGE_SIZE; |
319 | /* bits 12-31 */ | ||
320 | desc |= (u64)ctx->hw_id << GEN8_CTX_ID_SHIFT; /* bits 32-52 */ | ||
325 | 321 | ||
326 | ctx->engine[engine->id].lrc_desc = desc; | 322 | ce->lrc_desc = desc; |
327 | } | 323 | } |
328 | 324 | ||
329 | uint64_t intel_lr_context_descriptor(struct intel_context *ctx, | 325 | uint64_t intel_lr_context_descriptor(struct i915_gem_context *ctx, |
330 | struct intel_engine_cs *engine) | 326 | struct intel_engine_cs *engine) |
331 | { | 327 | { |
332 | return ctx->engine[engine->id].lrc_desc; | 328 | return ctx->engine[engine->id].lrc_desc; |
333 | } | 329 | } |
334 | 330 | ||
335 | /** | ||
336 | * intel_execlists_ctx_id() - get the Execlists Context ID | ||
337 | * @ctx: Context to get the ID for | ||
338 | * @ring: Engine to get the ID for | ||
339 | * | ||
340 | * Do not confuse with ctx->id! Unfortunately we have a name overload | ||
341 | * here: the old context ID we pass to userspace as a handler so that | ||
342 | * they can refer to a context, and the new context ID we pass to the | ||
343 | * ELSP so that the GPU can inform us of the context status via | ||
344 | * interrupts. | ||
345 | * | ||
346 | * The context ID is a portion of the context descriptor, so we can | ||
347 | * just extract the required part from the cached descriptor. | ||
348 | * | ||
349 | * Return: 20-bits globally unique context ID. | ||
350 | */ | ||
351 | u32 intel_execlists_ctx_id(struct intel_context *ctx, | ||
352 | struct intel_engine_cs *engine) | ||
353 | { | ||
354 | return intel_lr_context_descriptor(ctx, engine) >> GEN8_CTX_ID_SHIFT; | ||
355 | } | ||
356 | |||
357 | static void execlists_elsp_write(struct drm_i915_gem_request *rq0, | 331 | static void execlists_elsp_write(struct drm_i915_gem_request *rq0, |
358 | struct drm_i915_gem_request *rq1) | 332 | struct drm_i915_gem_request *rq1) |
359 | { | 333 | { |
360 | 334 | ||
361 | struct intel_engine_cs *engine = rq0->engine; | 335 | struct intel_engine_cs *engine = rq0->engine; |
362 | struct drm_device *dev = engine->dev; | 336 | struct drm_i915_private *dev_priv = rq0->i915; |
363 | struct drm_i915_private *dev_priv = dev->dev_private; | ||
364 | uint64_t desc[2]; | 337 | uint64_t desc[2]; |
365 | 338 | ||
366 | if (rq1) { | 339 | if (rq1) { |
@@ -431,6 +404,20 @@ static void execlists_submit_requests(struct drm_i915_gem_request *rq0, | |||
431 | spin_unlock_irq(&dev_priv->uncore.lock); | 404 | spin_unlock_irq(&dev_priv->uncore.lock); |
432 | } | 405 | } |
433 | 406 | ||
407 | static inline void execlists_context_status_change( | ||
408 | struct drm_i915_gem_request *rq, | ||
409 | unsigned long status) | ||
410 | { | ||
411 | /* | ||
412 | * Only used when GVT-g is enabled now. When GVT-g is disabled, | ||
413 | * The compiler should eliminate this function as dead-code. | ||
414 | */ | ||
415 | if (!IS_ENABLED(CONFIG_DRM_I915_GVT)) | ||
416 | return; | ||
417 | |||
418 | atomic_notifier_call_chain(&rq->ctx->status_notifier, status, rq); | ||
419 | } | ||
420 | |||
434 | static void execlists_context_unqueue(struct intel_engine_cs *engine) | 421 | static void execlists_context_unqueue(struct intel_engine_cs *engine) |
435 | { | 422 | { |
436 | struct drm_i915_gem_request *req0 = NULL, *req1 = NULL; | 423 | struct drm_i915_gem_request *req0 = NULL, *req1 = NULL; |
@@ -442,7 +429,7 @@ static void execlists_context_unqueue(struct intel_engine_cs *engine) | |||
442 | * If irqs are not active generate a warning as batches that finish | 429 | * If irqs are not active generate a warning as batches that finish |
443 | * without the irqs may get lost and a GPU Hang may occur. | 430 | * without the irqs may get lost and a GPU Hang may occur. |
444 | */ | 431 | */ |
445 | WARN_ON(!intel_irqs_enabled(engine->dev->dev_private)); | 432 | WARN_ON(!intel_irqs_enabled(engine->i915)); |
446 | 433 | ||
447 | /* Try to read in pairs */ | 434 | /* Try to read in pairs */ |
448 | list_for_each_entry_safe(cursor, tmp, &engine->execlist_queue, | 435 | list_for_each_entry_safe(cursor, tmp, &engine->execlist_queue, |
@@ -453,10 +440,24 @@ static void execlists_context_unqueue(struct intel_engine_cs *engine) | |||
453 | /* Same ctx: ignore first request, as second request | 440 | /* Same ctx: ignore first request, as second request |
454 | * will update tail past first request's workload */ | 441 | * will update tail past first request's workload */ |
455 | cursor->elsp_submitted = req0->elsp_submitted; | 442 | cursor->elsp_submitted = req0->elsp_submitted; |
456 | list_move_tail(&req0->execlist_link, | 443 | list_del(&req0->execlist_link); |
457 | &engine->execlist_retired_req_list); | 444 | i915_gem_request_unreference(req0); |
458 | req0 = cursor; | 445 | req0 = cursor; |
459 | } else { | 446 | } else { |
447 | if (IS_ENABLED(CONFIG_DRM_I915_GVT)) { | ||
448 | /* | ||
449 | * req0 (after merged) ctx requires single | ||
450 | * submission, stop picking | ||
451 | */ | ||
452 | if (req0->ctx->execlists_force_single_submission) | ||
453 | break; | ||
454 | /* | ||
455 | * req0 ctx doesn't require single submission, | ||
456 | * but next req ctx requires, stop picking | ||
457 | */ | ||
458 | if (cursor->ctx->execlists_force_single_submission) | ||
459 | break; | ||
460 | } | ||
460 | req1 = cursor; | 461 | req1 = cursor; |
461 | WARN_ON(req1->elsp_submitted); | 462 | WARN_ON(req1->elsp_submitted); |
462 | break; | 463 | break; |
@@ -466,6 +467,12 @@ static void execlists_context_unqueue(struct intel_engine_cs *engine) | |||
466 | if (unlikely(!req0)) | 467 | if (unlikely(!req0)) |
467 | return; | 468 | return; |
468 | 469 | ||
470 | execlists_context_status_change(req0, INTEL_CONTEXT_SCHEDULE_IN); | ||
471 | |||
472 | if (req1) | ||
473 | execlists_context_status_change(req1, | ||
474 | INTEL_CONTEXT_SCHEDULE_IN); | ||
475 | |||
469 | if (req0->elsp_submitted & engine->idle_lite_restore_wa) { | 476 | if (req0->elsp_submitted & engine->idle_lite_restore_wa) { |
470 | /* | 477 | /* |
471 | * WaIdleLiteRestore: make sure we never cause a lite restore | 478 | * WaIdleLiteRestore: make sure we never cause a lite restore |
@@ -486,7 +493,7 @@ static void execlists_context_unqueue(struct intel_engine_cs *engine) | |||
486 | } | 493 | } |
487 | 494 | ||
488 | static unsigned int | 495 | static unsigned int |
489 | execlists_check_remove_request(struct intel_engine_cs *engine, u32 request_id) | 496 | execlists_check_remove_request(struct intel_engine_cs *engine, u32 ctx_id) |
490 | { | 497 | { |
491 | struct drm_i915_gem_request *head_req; | 498 | struct drm_i915_gem_request *head_req; |
492 | 499 | ||
@@ -496,19 +503,18 @@ execlists_check_remove_request(struct intel_engine_cs *engine, u32 request_id) | |||
496 | struct drm_i915_gem_request, | 503 | struct drm_i915_gem_request, |
497 | execlist_link); | 504 | execlist_link); |
498 | 505 | ||
499 | if (!head_req) | 506 | if (WARN_ON(!head_req || (head_req->ctx_hw_id != ctx_id))) |
500 | return 0; | 507 | return 0; |
501 | |||
502 | if (unlikely(intel_execlists_ctx_id(head_req->ctx, engine) != request_id)) | ||
503 | return 0; | ||
504 | 508 | ||
505 | WARN(head_req->elsp_submitted == 0, "Never submitted head request\n"); | 509 | WARN(head_req->elsp_submitted == 0, "Never submitted head request\n"); |
506 | 510 | ||
507 | if (--head_req->elsp_submitted > 0) | 511 | if (--head_req->elsp_submitted > 0) |
508 | return 0; | 512 | return 0; |
509 | 513 | ||
510 | list_move_tail(&head_req->execlist_link, | 514 | execlists_context_status_change(head_req, INTEL_CONTEXT_SCHEDULE_OUT); |
511 | &engine->execlist_retired_req_list); | 515 | |
516 | list_del(&head_req->execlist_link); | ||
517 | i915_gem_request_unreference(head_req); | ||
512 | 518 | ||
513 | return 1; | 519 | return 1; |
514 | } | 520 | } |
@@ -517,7 +523,7 @@ static u32 | |||
517 | get_context_status(struct intel_engine_cs *engine, unsigned int read_pointer, | 523 | get_context_status(struct intel_engine_cs *engine, unsigned int read_pointer, |
518 | u32 *context_id) | 524 | u32 *context_id) |
519 | { | 525 | { |
520 | struct drm_i915_private *dev_priv = engine->dev->dev_private; | 526 | struct drm_i915_private *dev_priv = engine->i915; |
521 | u32 status; | 527 | u32 status; |
522 | 528 | ||
523 | read_pointer %= GEN8_CSB_ENTRIES; | 529 | read_pointer %= GEN8_CSB_ENTRIES; |
@@ -535,7 +541,7 @@ get_context_status(struct intel_engine_cs *engine, unsigned int read_pointer, | |||
535 | 541 | ||
536 | /** | 542 | /** |
537 | * intel_lrc_irq_handler() - handle Context Switch interrupts | 543 | * intel_lrc_irq_handler() - handle Context Switch interrupts |
538 | * @engine: Engine Command Streamer to handle. | 544 | * @data: tasklet handler passed in unsigned long |
539 | * | 545 | * |
540 | * Check the unread Context Status Buffers and manage the submission of new | 546 | * Check the unread Context Status Buffers and manage the submission of new |
541 | * contexts to the ELSP accordingly. | 547 | * contexts to the ELSP accordingly. |
@@ -543,7 +549,7 @@ get_context_status(struct intel_engine_cs *engine, unsigned int read_pointer, | |||
543 | static void intel_lrc_irq_handler(unsigned long data) | 549 | static void intel_lrc_irq_handler(unsigned long data) |
544 | { | 550 | { |
545 | struct intel_engine_cs *engine = (struct intel_engine_cs *)data; | 551 | struct intel_engine_cs *engine = (struct intel_engine_cs *)data; |
546 | struct drm_i915_private *dev_priv = engine->dev->dev_private; | 552 | struct drm_i915_private *dev_priv = engine->i915; |
547 | u32 status_pointer; | 553 | u32 status_pointer; |
548 | unsigned int read_pointer, write_pointer; | 554 | unsigned int read_pointer, write_pointer; |
549 | u32 csb[GEN8_CSB_ENTRIES][2]; | 555 | u32 csb[GEN8_CSB_ENTRIES][2]; |
@@ -612,11 +618,6 @@ static void execlists_context_queue(struct drm_i915_gem_request *request) | |||
612 | struct drm_i915_gem_request *cursor; | 618 | struct drm_i915_gem_request *cursor; |
613 | int num_elements = 0; | 619 | int num_elements = 0; |
614 | 620 | ||
615 | if (request->ctx != request->i915->kernel_context) | ||
616 | intel_lr_context_pin(request->ctx, engine); | ||
617 | |||
618 | i915_gem_request_reference(request); | ||
619 | |||
620 | spin_lock_bh(&engine->execlist_lock); | 621 | spin_lock_bh(&engine->execlist_lock); |
621 | 622 | ||
622 | list_for_each_entry(cursor, &engine->execlist_queue, execlist_link) | 623 | list_for_each_entry(cursor, &engine->execlist_queue, execlist_link) |
@@ -633,12 +634,14 @@ static void execlists_context_queue(struct drm_i915_gem_request *request) | |||
633 | if (request->ctx == tail_req->ctx) { | 634 | if (request->ctx == tail_req->ctx) { |
634 | WARN(tail_req->elsp_submitted != 0, | 635 | WARN(tail_req->elsp_submitted != 0, |
635 | "More than 2 already-submitted reqs queued\n"); | 636 | "More than 2 already-submitted reqs queued\n"); |
636 | list_move_tail(&tail_req->execlist_link, | 637 | list_del(&tail_req->execlist_link); |
637 | &engine->execlist_retired_req_list); | 638 | i915_gem_request_unreference(tail_req); |
638 | } | 639 | } |
639 | } | 640 | } |
640 | 641 | ||
642 | i915_gem_request_reference(request); | ||
641 | list_add_tail(&request->execlist_link, &engine->execlist_queue); | 643 | list_add_tail(&request->execlist_link, &engine->execlist_queue); |
644 | request->ctx_hw_id = request->ctx->hw_id; | ||
642 | if (num_elements == 0) | 645 | if (num_elements == 0) |
643 | execlists_context_unqueue(engine); | 646 | execlists_context_unqueue(engine); |
644 | 647 | ||
@@ -698,9 +701,23 @@ static int execlists_move_to_gpu(struct drm_i915_gem_request *req, | |||
698 | 701 | ||
699 | int intel_logical_ring_alloc_request_extras(struct drm_i915_gem_request *request) | 702 | int intel_logical_ring_alloc_request_extras(struct drm_i915_gem_request *request) |
700 | { | 703 | { |
701 | int ret = 0; | 704 | struct intel_engine_cs *engine = request->engine; |
705 | struct intel_context *ce = &request->ctx->engine[engine->id]; | ||
706 | int ret; | ||
702 | 707 | ||
703 | request->ringbuf = request->ctx->engine[request->engine->id].ringbuf; | 708 | /* Flush enough space to reduce the likelihood of waiting after |
709 | * we start building the request - in which case we will just | ||
710 | * have to repeat work. | ||
711 | */ | ||
712 | request->reserved_space += EXECLISTS_REQUEST_SIZE; | ||
713 | |||
714 | if (!ce->state) { | ||
715 | ret = execlists_context_deferred_alloc(request->ctx, engine); | ||
716 | if (ret) | ||
717 | return ret; | ||
718 | } | ||
719 | |||
720 | request->ringbuf = ce->ringbuf; | ||
704 | 721 | ||
705 | if (i915.enable_guc_submission) { | 722 | if (i915.enable_guc_submission) { |
706 | /* | 723 | /* |
@@ -708,16 +725,39 @@ int intel_logical_ring_alloc_request_extras(struct drm_i915_gem_request *request | |||
708 | * going any further, as the i915_add_request() call | 725 | * going any further, as the i915_add_request() call |
709 | * later on mustn't fail ... | 726 | * later on mustn't fail ... |
710 | */ | 727 | */ |
711 | struct intel_guc *guc = &request->i915->guc; | 728 | ret = i915_guc_wq_check_space(request); |
712 | |||
713 | ret = i915_guc_wq_check_space(guc->execbuf_client); | ||
714 | if (ret) | 729 | if (ret) |
715 | return ret; | 730 | return ret; |
716 | } | 731 | } |
717 | 732 | ||
718 | if (request->ctx != request->i915->kernel_context) | 733 | ret = intel_lr_context_pin(request->ctx, engine); |
719 | ret = intel_lr_context_pin(request->ctx, request->engine); | 734 | if (ret) |
735 | return ret; | ||
736 | |||
737 | ret = intel_ring_begin(request, 0); | ||
738 | if (ret) | ||
739 | goto err_unpin; | ||
740 | |||
741 | if (!ce->initialised) { | ||
742 | ret = engine->init_context(request); | ||
743 | if (ret) | ||
744 | goto err_unpin; | ||
745 | |||
746 | ce->initialised = true; | ||
747 | } | ||
748 | |||
749 | /* Note that after this point, we have committed to using | ||
750 | * this request as it is being used to both track the | ||
751 | * state of engine initialisation and liveness of the | ||
752 | * golden renderstate above. Think twice before you try | ||
753 | * to cancel/unwind this request now. | ||
754 | */ | ||
720 | 755 | ||
756 | request->reserved_space -= EXECLISTS_REQUEST_SIZE; | ||
757 | return 0; | ||
758 | |||
759 | err_unpin: | ||
760 | intel_lr_context_unpin(request->ctx, engine); | ||
721 | return ret; | 761 | return ret; |
722 | } | 762 | } |
723 | 763 | ||
@@ -734,7 +774,6 @@ static int | |||
734 | intel_logical_ring_advance_and_submit(struct drm_i915_gem_request *request) | 774 | intel_logical_ring_advance_and_submit(struct drm_i915_gem_request *request) |
735 | { | 775 | { |
736 | struct intel_ringbuffer *ringbuf = request->ringbuf; | 776 | struct intel_ringbuffer *ringbuf = request->ringbuf; |
737 | struct drm_i915_private *dev_priv = request->i915; | ||
738 | struct intel_engine_cs *engine = request->engine; | 777 | struct intel_engine_cs *engine = request->engine; |
739 | 778 | ||
740 | intel_logical_ring_advance(ringbuf); | 779 | intel_logical_ring_advance(ringbuf); |
@@ -750,54 +789,28 @@ intel_logical_ring_advance_and_submit(struct drm_i915_gem_request *request) | |||
750 | intel_logical_ring_emit(ringbuf, MI_NOOP); | 789 | intel_logical_ring_emit(ringbuf, MI_NOOP); |
751 | intel_logical_ring_advance(ringbuf); | 790 | intel_logical_ring_advance(ringbuf); |
752 | 791 | ||
753 | if (intel_engine_stopped(engine)) | 792 | /* We keep the previous context alive until we retire the following |
754 | return 0; | 793 | * request. This ensures that any the context object is still pinned |
755 | 794 | * for any residual writes the HW makes into it on the context switch | |
756 | if (engine->last_context != request->ctx) { | 795 | * into the next object following the breadcrumb. Otherwise, we may |
757 | if (engine->last_context) | 796 | * retire the context too early. |
758 | intel_lr_context_unpin(engine->last_context, engine); | 797 | */ |
759 | if (request->ctx != request->i915->kernel_context) { | 798 | request->previous_context = engine->last_context; |
760 | intel_lr_context_pin(request->ctx, engine); | 799 | engine->last_context = request->ctx; |
761 | engine->last_context = request->ctx; | ||
762 | } else { | ||
763 | engine->last_context = NULL; | ||
764 | } | ||
765 | } | ||
766 | 800 | ||
767 | if (dev_priv->guc.execbuf_client) | 801 | if (i915.enable_guc_submission) |
768 | i915_guc_submit(dev_priv->guc.execbuf_client, request); | 802 | i915_guc_submit(request); |
769 | else | 803 | else |
770 | execlists_context_queue(request); | 804 | execlists_context_queue(request); |
771 | 805 | ||
772 | return 0; | 806 | return 0; |
773 | } | 807 | } |
774 | 808 | ||
775 | int intel_logical_ring_reserve_space(struct drm_i915_gem_request *request) | ||
776 | { | ||
777 | /* | ||
778 | * The first call merely notes the reserve request and is common for | ||
779 | * all back ends. The subsequent localised _begin() call actually | ||
780 | * ensures that the reservation is available. Without the begin, if | ||
781 | * the request creator immediately submitted the request without | ||
782 | * adding any commands to it then there might not actually be | ||
783 | * sufficient room for the submission commands. | ||
784 | */ | ||
785 | intel_ring_reserved_space_reserve(request->ringbuf, MIN_SPACE_FOR_ADD_REQUEST); | ||
786 | |||
787 | return intel_ring_begin(request, 0); | ||
788 | } | ||
789 | |||
790 | /** | 809 | /** |
791 | * execlists_submission() - submit a batchbuffer for execution, Execlists style | 810 | * execlists_submission() - submit a batchbuffer for execution, Execlists style |
792 | * @dev: DRM device. | 811 | * @params: execbuffer call parameters. |
793 | * @file: DRM file. | ||
794 | * @ring: Engine Command Streamer to submit to. | ||
795 | * @ctx: Context to employ for this submission. | ||
796 | * @args: execbuffer call arguments. | 812 | * @args: execbuffer call arguments. |
797 | * @vmas: list of vmas. | 813 | * @vmas: list of vmas. |
798 | * @batch_obj: the batchbuffer to submit. | ||
799 | * @exec_start: batchbuffer start virtual address pointer. | ||
800 | * @dispatch_flags: translated execbuffer call flags. | ||
801 | * | 814 | * |
802 | * This is the evil twin version of i915_gem_ringbuffer_submission. It abstracts | 815 | * This is the evil twin version of i915_gem_ringbuffer_submission. It abstracts |
803 | * away the submission details of the execbuffer ioctl call. | 816 | * away the submission details of the execbuffer ioctl call. |
@@ -810,7 +823,7 @@ int intel_execlists_submission(struct i915_execbuffer_params *params, | |||
810 | { | 823 | { |
811 | struct drm_device *dev = params->dev; | 824 | struct drm_device *dev = params->dev; |
812 | struct intel_engine_cs *engine = params->engine; | 825 | struct intel_engine_cs *engine = params->engine; |
813 | struct drm_i915_private *dev_priv = dev->dev_private; | 826 | struct drm_i915_private *dev_priv = to_i915(dev); |
814 | struct intel_ringbuffer *ringbuf = params->ctx->engine[engine->id].ringbuf; | 827 | struct intel_ringbuffer *ringbuf = params->ctx->engine[engine->id].ringbuf; |
815 | u64 exec_start; | 828 | u64 exec_start; |
816 | int instp_mode; | 829 | int instp_mode; |
@@ -881,28 +894,18 @@ int intel_execlists_submission(struct i915_execbuffer_params *params, | |||
881 | return 0; | 894 | return 0; |
882 | } | 895 | } |
883 | 896 | ||
884 | void intel_execlists_retire_requests(struct intel_engine_cs *engine) | 897 | void intel_execlists_cancel_requests(struct intel_engine_cs *engine) |
885 | { | 898 | { |
886 | struct drm_i915_gem_request *req, *tmp; | 899 | struct drm_i915_gem_request *req, *tmp; |
887 | struct list_head retired_list; | 900 | LIST_HEAD(cancel_list); |
888 | 901 | ||
889 | WARN_ON(!mutex_is_locked(&engine->dev->struct_mutex)); | 902 | WARN_ON(!mutex_is_locked(&engine->i915->drm.struct_mutex)); |
890 | if (list_empty(&engine->execlist_retired_req_list)) | ||
891 | return; | ||
892 | 903 | ||
893 | INIT_LIST_HEAD(&retired_list); | ||
894 | spin_lock_bh(&engine->execlist_lock); | 904 | spin_lock_bh(&engine->execlist_lock); |
895 | list_replace_init(&engine->execlist_retired_req_list, &retired_list); | 905 | list_replace_init(&engine->execlist_queue, &cancel_list); |
896 | spin_unlock_bh(&engine->execlist_lock); | 906 | spin_unlock_bh(&engine->execlist_lock); |
897 | 907 | ||
898 | list_for_each_entry_safe(req, tmp, &retired_list, execlist_link) { | 908 | list_for_each_entry_safe(req, tmp, &cancel_list, execlist_link) { |
899 | struct intel_context *ctx = req->ctx; | ||
900 | struct drm_i915_gem_object *ctx_obj = | ||
901 | ctx->engine[engine->id].state; | ||
902 | |||
903 | if (ctx_obj && (ctx != req->i915->kernel_context)) | ||
904 | intel_lr_context_unpin(ctx, engine); | ||
905 | |||
906 | list_del(&req->execlist_link); | 909 | list_del(&req->execlist_link); |
907 | i915_gem_request_unreference(req); | 910 | i915_gem_request_unreference(req); |
908 | } | 911 | } |
@@ -910,7 +913,7 @@ void intel_execlists_retire_requests(struct intel_engine_cs *engine) | |||
910 | 913 | ||
911 | void intel_logical_ring_stop(struct intel_engine_cs *engine) | 914 | void intel_logical_ring_stop(struct intel_engine_cs *engine) |
912 | { | 915 | { |
913 | struct drm_i915_private *dev_priv = engine->dev->dev_private; | 916 | struct drm_i915_private *dev_priv = engine->i915; |
914 | int ret; | 917 | int ret; |
915 | 918 | ||
916 | if (!intel_engine_initialized(engine)) | 919 | if (!intel_engine_initialized(engine)) |
@@ -923,7 +926,10 @@ void intel_logical_ring_stop(struct intel_engine_cs *engine) | |||
923 | 926 | ||
924 | /* TODO: Is this correct with Execlists enabled? */ | 927 | /* TODO: Is this correct with Execlists enabled? */ |
925 | I915_WRITE_MODE(engine, _MASKED_BIT_ENABLE(STOP_RING)); | 928 | I915_WRITE_MODE(engine, _MASKED_BIT_ENABLE(STOP_RING)); |
926 | if (wait_for((I915_READ_MODE(engine) & MODE_IDLE) != 0, 1000)) { | 929 | if (intel_wait_for_register(dev_priv, |
930 | RING_MI_MODE(engine->mmio_base), | ||
931 | MODE_IDLE, MODE_IDLE, | ||
932 | 1000)) { | ||
927 | DRM_ERROR("%s :timed out trying to stop ring\n", engine->name); | 933 | DRM_ERROR("%s :timed out trying to stop ring\n", engine->name); |
928 | return; | 934 | return; |
929 | } | 935 | } |
@@ -946,25 +952,26 @@ int logical_ring_flush_all_caches(struct drm_i915_gem_request *req) | |||
946 | return 0; | 952 | return 0; |
947 | } | 953 | } |
948 | 954 | ||
949 | static int intel_lr_context_do_pin(struct intel_context *ctx, | 955 | static int intel_lr_context_pin(struct i915_gem_context *ctx, |
950 | struct intel_engine_cs *engine) | 956 | struct intel_engine_cs *engine) |
951 | { | 957 | { |
952 | struct drm_device *dev = engine->dev; | 958 | struct drm_i915_private *dev_priv = ctx->i915; |
953 | struct drm_i915_private *dev_priv = dev->dev_private; | 959 | struct intel_context *ce = &ctx->engine[engine->id]; |
954 | struct drm_i915_gem_object *ctx_obj = ctx->engine[engine->id].state; | ||
955 | struct intel_ringbuffer *ringbuf = ctx->engine[engine->id].ringbuf; | ||
956 | void *vaddr; | 960 | void *vaddr; |
957 | u32 *lrc_reg_state; | 961 | u32 *lrc_reg_state; |
958 | int ret; | 962 | int ret; |
959 | 963 | ||
960 | WARN_ON(!mutex_is_locked(&engine->dev->struct_mutex)); | 964 | lockdep_assert_held(&ctx->i915->drm.struct_mutex); |
961 | 965 | ||
962 | ret = i915_gem_obj_ggtt_pin(ctx_obj, GEN8_LR_CONTEXT_ALIGN, | 966 | if (ce->pin_count++) |
963 | PIN_OFFSET_BIAS | GUC_WOPCM_TOP); | 967 | return 0; |
968 | |||
969 | ret = i915_gem_obj_ggtt_pin(ce->state, GEN8_LR_CONTEXT_ALIGN, | ||
970 | PIN_OFFSET_BIAS | GUC_WOPCM_TOP); | ||
964 | if (ret) | 971 | if (ret) |
965 | return ret; | 972 | goto err; |
966 | 973 | ||
967 | vaddr = i915_gem_object_pin_map(ctx_obj); | 974 | vaddr = i915_gem_object_pin_map(ce->state); |
968 | if (IS_ERR(vaddr)) { | 975 | if (IS_ERR(vaddr)) { |
969 | ret = PTR_ERR(vaddr); | 976 | ret = PTR_ERR(vaddr); |
970 | goto unpin_ctx_obj; | 977 | goto unpin_ctx_obj; |
@@ -972,65 +979,54 @@ static int intel_lr_context_do_pin(struct intel_context *ctx, | |||
972 | 979 | ||
973 | lrc_reg_state = vaddr + LRC_STATE_PN * PAGE_SIZE; | 980 | lrc_reg_state = vaddr + LRC_STATE_PN * PAGE_SIZE; |
974 | 981 | ||
975 | ret = intel_pin_and_map_ringbuffer_obj(engine->dev, ringbuf); | 982 | ret = intel_pin_and_map_ringbuffer_obj(dev_priv, ce->ringbuf); |
976 | if (ret) | 983 | if (ret) |
977 | goto unpin_map; | 984 | goto unpin_map; |
978 | 985 | ||
979 | ctx->engine[engine->id].lrc_vma = i915_gem_obj_to_ggtt(ctx_obj); | 986 | i915_gem_context_reference(ctx); |
987 | ce->lrc_vma = i915_gem_obj_to_ggtt(ce->state); | ||
980 | intel_lr_context_descriptor_update(ctx, engine); | 988 | intel_lr_context_descriptor_update(ctx, engine); |
981 | lrc_reg_state[CTX_RING_BUFFER_START+1] = ringbuf->vma->node.start; | 989 | |
982 | ctx->engine[engine->id].lrc_reg_state = lrc_reg_state; | 990 | lrc_reg_state[CTX_RING_BUFFER_START+1] = ce->ringbuf->vma->node.start; |
983 | ctx_obj->dirty = true; | 991 | ce->lrc_reg_state = lrc_reg_state; |
992 | ce->state->dirty = true; | ||
984 | 993 | ||
985 | /* Invalidate GuC TLB. */ | 994 | /* Invalidate GuC TLB. */ |
986 | if (i915.enable_guc_submission) | 995 | if (i915.enable_guc_submission) |
987 | I915_WRITE(GEN8_GTCR, GEN8_GTCR_INVALIDATE); | 996 | I915_WRITE(GEN8_GTCR, GEN8_GTCR_INVALIDATE); |
988 | 997 | ||
989 | return ret; | 998 | return 0; |
990 | 999 | ||
991 | unpin_map: | 1000 | unpin_map: |
992 | i915_gem_object_unpin_map(ctx_obj); | 1001 | i915_gem_object_unpin_map(ce->state); |
993 | unpin_ctx_obj: | 1002 | unpin_ctx_obj: |
994 | i915_gem_object_ggtt_unpin(ctx_obj); | 1003 | i915_gem_object_ggtt_unpin(ce->state); |
995 | 1004 | err: | |
1005 | ce->pin_count = 0; | ||
996 | return ret; | 1006 | return ret; |
997 | } | 1007 | } |
998 | 1008 | ||
999 | static int intel_lr_context_pin(struct intel_context *ctx, | 1009 | void intel_lr_context_unpin(struct i915_gem_context *ctx, |
1000 | struct intel_engine_cs *engine) | 1010 | struct intel_engine_cs *engine) |
1001 | { | 1011 | { |
1002 | int ret = 0; | 1012 | struct intel_context *ce = &ctx->engine[engine->id]; |
1003 | 1013 | ||
1004 | if (ctx->engine[engine->id].pin_count++ == 0) { | 1014 | lockdep_assert_held(&ctx->i915->drm.struct_mutex); |
1005 | ret = intel_lr_context_do_pin(ctx, engine); | 1015 | GEM_BUG_ON(ce->pin_count == 0); |
1006 | if (ret) | ||
1007 | goto reset_pin_count; | ||
1008 | 1016 | ||
1009 | i915_gem_context_reference(ctx); | 1017 | if (--ce->pin_count) |
1010 | } | 1018 | return; |
1011 | return ret; | ||
1012 | 1019 | ||
1013 | reset_pin_count: | 1020 | intel_unpin_ringbuffer_obj(ce->ringbuf); |
1014 | ctx->engine[engine->id].pin_count = 0; | ||
1015 | return ret; | ||
1016 | } | ||
1017 | 1021 | ||
1018 | void intel_lr_context_unpin(struct intel_context *ctx, | 1022 | i915_gem_object_unpin_map(ce->state); |
1019 | struct intel_engine_cs *engine) | 1023 | i915_gem_object_ggtt_unpin(ce->state); |
1020 | { | ||
1021 | struct drm_i915_gem_object *ctx_obj = ctx->engine[engine->id].state; | ||
1022 | 1024 | ||
1023 | WARN_ON(!mutex_is_locked(&ctx->i915->dev->struct_mutex)); | 1025 | ce->lrc_vma = NULL; |
1024 | if (--ctx->engine[engine->id].pin_count == 0) { | 1026 | ce->lrc_desc = 0; |
1025 | i915_gem_object_unpin_map(ctx_obj); | 1027 | ce->lrc_reg_state = NULL; |
1026 | intel_unpin_ringbuffer_obj(ctx->engine[engine->id].ringbuf); | ||
1027 | i915_gem_object_ggtt_unpin(ctx_obj); | ||
1028 | ctx->engine[engine->id].lrc_vma = NULL; | ||
1029 | ctx->engine[engine->id].lrc_desc = 0; | ||
1030 | ctx->engine[engine->id].lrc_reg_state = NULL; | ||
1031 | 1028 | ||
1032 | i915_gem_context_unreference(ctx); | 1029 | i915_gem_context_unreference(ctx); |
1033 | } | ||
1034 | } | 1030 | } |
1035 | 1031 | ||
1036 | static int intel_logical_ring_workarounds_emit(struct drm_i915_gem_request *req) | 1032 | static int intel_logical_ring_workarounds_emit(struct drm_i915_gem_request *req) |
@@ -1038,9 +1034,7 @@ static int intel_logical_ring_workarounds_emit(struct drm_i915_gem_request *req) | |||
1038 | int ret, i; | 1034 | int ret, i; |
1039 | struct intel_engine_cs *engine = req->engine; | 1035 | struct intel_engine_cs *engine = req->engine; |
1040 | struct intel_ringbuffer *ringbuf = req->ringbuf; | 1036 | struct intel_ringbuffer *ringbuf = req->ringbuf; |
1041 | struct drm_device *dev = engine->dev; | 1037 | struct i915_workarounds *w = &req->i915->workarounds; |
1042 | struct drm_i915_private *dev_priv = dev->dev_private; | ||
1043 | struct i915_workarounds *w = &dev_priv->workarounds; | ||
1044 | 1038 | ||
1045 | if (w->count == 0) | 1039 | if (w->count == 0) |
1046 | return 0; | 1040 | return 0; |
@@ -1103,7 +1097,7 @@ static inline int gen8_emit_flush_coherentl3_wa(struct intel_engine_cs *engine, | |||
1103 | uint32_t *const batch, | 1097 | uint32_t *const batch, |
1104 | uint32_t index) | 1098 | uint32_t index) |
1105 | { | 1099 | { |
1106 | struct drm_i915_private *dev_priv = engine->dev->dev_private; | 1100 | struct drm_i915_private *dev_priv = engine->i915; |
1107 | uint32_t l3sqc4_flush = (0x40400000 | GEN8_LQSC_FLUSH_COHERENT_LINES); | 1101 | uint32_t l3sqc4_flush = (0x40400000 | GEN8_LQSC_FLUSH_COHERENT_LINES); |
1108 | 1102 | ||
1109 | /* | 1103 | /* |
@@ -1165,7 +1159,7 @@ static inline int wa_ctx_end(struct i915_wa_ctx_bb *wa_ctx, | |||
1165 | /** | 1159 | /** |
1166 | * gen8_init_indirectctx_bb() - initialize indirect ctx batch with WA | 1160 | * gen8_init_indirectctx_bb() - initialize indirect ctx batch with WA |
1167 | * | 1161 | * |
1168 | * @ring: only applicable for RCS | 1162 | * @engine: only applicable for RCS |
1169 | * @wa_ctx: structure representing wa_ctx | 1163 | * @wa_ctx: structure representing wa_ctx |
1170 | * offset: specifies start of the batch, should be cache-aligned. This is updated | 1164 | * offset: specifies start of the batch, should be cache-aligned. This is updated |
1171 | * with the offset value received as input. | 1165 | * with the offset value received as input. |
@@ -1202,7 +1196,7 @@ static int gen8_init_indirectctx_bb(struct intel_engine_cs *engine, | |||
1202 | wa_ctx_emit(batch, index, MI_ARB_ON_OFF | MI_ARB_DISABLE); | 1196 | wa_ctx_emit(batch, index, MI_ARB_ON_OFF | MI_ARB_DISABLE); |
1203 | 1197 | ||
1204 | /* WaFlushCoherentL3CacheLinesAtContextSwitch:bdw */ | 1198 | /* WaFlushCoherentL3CacheLinesAtContextSwitch:bdw */ |
1205 | if (IS_BROADWELL(engine->dev)) { | 1199 | if (IS_BROADWELL(engine->i915)) { |
1206 | int rc = gen8_emit_flush_coherentl3_wa(engine, batch, index); | 1200 | int rc = gen8_emit_flush_coherentl3_wa(engine, batch, index); |
1207 | if (rc < 0) | 1201 | if (rc < 0) |
1208 | return rc; | 1202 | return rc; |
@@ -1239,7 +1233,7 @@ static int gen8_init_indirectctx_bb(struct intel_engine_cs *engine, | |||
1239 | /** | 1233 | /** |
1240 | * gen8_init_perctx_bb() - initialize per ctx batch with WA | 1234 | * gen8_init_perctx_bb() - initialize per ctx batch with WA |
1241 | * | 1235 | * |
1242 | * @ring: only applicable for RCS | 1236 | * @engine: only applicable for RCS |
1243 | * @wa_ctx: structure representing wa_ctx | 1237 | * @wa_ctx: structure representing wa_ctx |
1244 | * offset: specifies start of the batch, should be cache-aligned. | 1238 | * offset: specifies start of the batch, should be cache-aligned. |
1245 | * size: size of the batch in DWORDS but HW expects in terms of cachelines | 1239 | * size: size of the batch in DWORDS but HW expects in terms of cachelines |
@@ -1274,13 +1268,12 @@ static int gen9_init_indirectctx_bb(struct intel_engine_cs *engine, | |||
1274 | uint32_t *offset) | 1268 | uint32_t *offset) |
1275 | { | 1269 | { |
1276 | int ret; | 1270 | int ret; |
1277 | struct drm_device *dev = engine->dev; | 1271 | struct drm_i915_private *dev_priv = engine->i915; |
1278 | struct drm_i915_private *dev_priv = dev->dev_private; | ||
1279 | uint32_t index = wa_ctx_start(wa_ctx, *offset, CACHELINE_DWORDS); | 1272 | uint32_t index = wa_ctx_start(wa_ctx, *offset, CACHELINE_DWORDS); |
1280 | 1273 | ||
1281 | /* WaDisableCtxRestoreArbitration:skl,bxt */ | 1274 | /* WaDisableCtxRestoreArbitration:skl,bxt */ |
1282 | if (IS_SKL_REVID(dev, 0, SKL_REVID_D0) || | 1275 | if (IS_SKL_REVID(dev_priv, 0, SKL_REVID_D0) || |
1283 | IS_BXT_REVID(dev, 0, BXT_REVID_A1)) | 1276 | IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) |
1284 | wa_ctx_emit(batch, index, MI_ARB_ON_OFF | MI_ARB_DISABLE); | 1277 | wa_ctx_emit(batch, index, MI_ARB_ON_OFF | MI_ARB_DISABLE); |
1285 | 1278 | ||
1286 | /* WaFlushCoherentL3CacheLinesAtContextSwitch:skl,bxt */ | 1279 | /* WaFlushCoherentL3CacheLinesAtContextSwitch:skl,bxt */ |
@@ -1305,6 +1298,31 @@ static int gen9_init_indirectctx_bb(struct intel_engine_cs *engine, | |||
1305 | wa_ctx_emit(batch, index, 0); | 1298 | wa_ctx_emit(batch, index, 0); |
1306 | wa_ctx_emit(batch, index, 0); | 1299 | wa_ctx_emit(batch, index, 0); |
1307 | } | 1300 | } |
1301 | |||
1302 | /* WaMediaPoolStateCmdInWABB:bxt */ | ||
1303 | if (HAS_POOLED_EU(engine->i915)) { | ||
1304 | /* | ||
1305 | * EU pool configuration is setup along with golden context | ||
1306 | * during context initialization. This value depends on | ||
1307 | * device type (2x6 or 3x6) and needs to be updated based | ||
1308 | * on which subslice is disabled especially for 2x6 | ||
1309 | * devices, however it is safe to load default | ||
1310 | * configuration of 3x6 device instead of masking off | ||
1311 | * corresponding bits because HW ignores bits of a disabled | ||
1312 | * subslice and drops down to appropriate config. Please | ||
1313 | * see render_state_setup() in i915_gem_render_state.c for | ||
1314 | * possible configurations, to avoid duplication they are | ||
1315 | * not shown here again. | ||
1316 | */ | ||
1317 | u32 eu_pool_config = 0x00777000; | ||
1318 | wa_ctx_emit(batch, index, GEN9_MEDIA_POOL_STATE); | ||
1319 | wa_ctx_emit(batch, index, GEN9_MEDIA_POOL_ENABLE); | ||
1320 | wa_ctx_emit(batch, index, eu_pool_config); | ||
1321 | wa_ctx_emit(batch, index, 0); | ||
1322 | wa_ctx_emit(batch, index, 0); | ||
1323 | wa_ctx_emit(batch, index, 0); | ||
1324 | } | ||
1325 | |||
1308 | /* Pad to end of cacheline */ | 1326 | /* Pad to end of cacheline */ |
1309 | while (index % CACHELINE_DWORDS) | 1327 | while (index % CACHELINE_DWORDS) |
1310 | wa_ctx_emit(batch, index, MI_NOOP); | 1328 | wa_ctx_emit(batch, index, MI_NOOP); |
@@ -1317,12 +1335,11 @@ static int gen9_init_perctx_bb(struct intel_engine_cs *engine, | |||
1317 | uint32_t *const batch, | 1335 | uint32_t *const batch, |
1318 | uint32_t *offset) | 1336 | uint32_t *offset) |
1319 | { | 1337 | { |
1320 | struct drm_device *dev = engine->dev; | ||
1321 | uint32_t index = wa_ctx_start(wa_ctx, *offset, CACHELINE_DWORDS); | 1338 | uint32_t index = wa_ctx_start(wa_ctx, *offset, CACHELINE_DWORDS); |
1322 | 1339 | ||
1323 | /* WaSetDisablePixMaskCammingAndRhwoInCommonSliceChicken:skl,bxt */ | 1340 | /* WaSetDisablePixMaskCammingAndRhwoInCommonSliceChicken:skl,bxt */ |
1324 | if (IS_SKL_REVID(dev, 0, SKL_REVID_B0) || | 1341 | if (IS_SKL_REVID(engine->i915, 0, SKL_REVID_B0) || |
1325 | IS_BXT_REVID(dev, 0, BXT_REVID_A1)) { | 1342 | IS_BXT_REVID(engine->i915, 0, BXT_REVID_A1)) { |
1326 | wa_ctx_emit(batch, index, MI_LOAD_REGISTER_IMM(1)); | 1343 | wa_ctx_emit(batch, index, MI_LOAD_REGISTER_IMM(1)); |
1327 | wa_ctx_emit_reg(batch, index, GEN9_SLICE_COMMON_ECO_CHICKEN0); | 1344 | wa_ctx_emit_reg(batch, index, GEN9_SLICE_COMMON_ECO_CHICKEN0); |
1328 | wa_ctx_emit(batch, index, | 1345 | wa_ctx_emit(batch, index, |
@@ -1331,7 +1348,7 @@ static int gen9_init_perctx_bb(struct intel_engine_cs *engine, | |||
1331 | } | 1348 | } |
1332 | 1349 | ||
1333 | /* WaClearTdlStateAckDirtyBits:bxt */ | 1350 | /* WaClearTdlStateAckDirtyBits:bxt */ |
1334 | if (IS_BXT_REVID(dev, 0, BXT_REVID_B0)) { | 1351 | if (IS_BXT_REVID(engine->i915, 0, BXT_REVID_B0)) { |
1335 | wa_ctx_emit(batch, index, MI_LOAD_REGISTER_IMM(4)); | 1352 | wa_ctx_emit(batch, index, MI_LOAD_REGISTER_IMM(4)); |
1336 | 1353 | ||
1337 | wa_ctx_emit_reg(batch, index, GEN8_STATE_ACK); | 1354 | wa_ctx_emit_reg(batch, index, GEN8_STATE_ACK); |
@@ -1350,8 +1367,8 @@ static int gen9_init_perctx_bb(struct intel_engine_cs *engine, | |||
1350 | } | 1367 | } |
1351 | 1368 | ||
1352 | /* WaDisableCtxRestoreArbitration:skl,bxt */ | 1369 | /* WaDisableCtxRestoreArbitration:skl,bxt */ |
1353 | if (IS_SKL_REVID(dev, 0, SKL_REVID_D0) || | 1370 | if (IS_SKL_REVID(engine->i915, 0, SKL_REVID_D0) || |
1354 | IS_BXT_REVID(dev, 0, BXT_REVID_A1)) | 1371 | IS_BXT_REVID(engine->i915, 0, BXT_REVID_A1)) |
1355 | wa_ctx_emit(batch, index, MI_ARB_ON_OFF | MI_ARB_ENABLE); | 1372 | wa_ctx_emit(batch, index, MI_ARB_ON_OFF | MI_ARB_ENABLE); |
1356 | 1373 | ||
1357 | wa_ctx_emit(batch, index, MI_BATCH_BUFFER_END); | 1374 | wa_ctx_emit(batch, index, MI_BATCH_BUFFER_END); |
@@ -1363,11 +1380,13 @@ static int lrc_setup_wa_ctx_obj(struct intel_engine_cs *engine, u32 size) | |||
1363 | { | 1380 | { |
1364 | int ret; | 1381 | int ret; |
1365 | 1382 | ||
1366 | engine->wa_ctx.obj = i915_gem_alloc_object(engine->dev, | 1383 | engine->wa_ctx.obj = i915_gem_object_create(&engine->i915->drm, |
1367 | PAGE_ALIGN(size)); | 1384 | PAGE_ALIGN(size)); |
1368 | if (!engine->wa_ctx.obj) { | 1385 | if (IS_ERR(engine->wa_ctx.obj)) { |
1369 | DRM_DEBUG_DRIVER("alloc LRC WA ctx backing obj failed.\n"); | 1386 | DRM_DEBUG_DRIVER("alloc LRC WA ctx backing obj failed.\n"); |
1370 | return -ENOMEM; | 1387 | ret = PTR_ERR(engine->wa_ctx.obj); |
1388 | engine->wa_ctx.obj = NULL; | ||
1389 | return ret; | ||
1371 | } | 1390 | } |
1372 | 1391 | ||
1373 | ret = i915_gem_obj_ggtt_pin(engine->wa_ctx.obj, PAGE_SIZE, 0); | 1392 | ret = i915_gem_obj_ggtt_pin(engine->wa_ctx.obj, PAGE_SIZE, 0); |
@@ -1401,9 +1420,9 @@ static int intel_init_workaround_bb(struct intel_engine_cs *engine) | |||
1401 | WARN_ON(engine->id != RCS); | 1420 | WARN_ON(engine->id != RCS); |
1402 | 1421 | ||
1403 | /* update this when WA for higher Gen are added */ | 1422 | /* update this when WA for higher Gen are added */ |
1404 | if (INTEL_INFO(engine->dev)->gen > 9) { | 1423 | if (INTEL_GEN(engine->i915) > 9) { |
1405 | DRM_ERROR("WA batch buffer is not initialized for Gen%d\n", | 1424 | DRM_ERROR("WA batch buffer is not initialized for Gen%d\n", |
1406 | INTEL_INFO(engine->dev)->gen); | 1425 | INTEL_GEN(engine->i915)); |
1407 | return 0; | 1426 | return 0; |
1408 | } | 1427 | } |
1409 | 1428 | ||
@@ -1423,7 +1442,7 @@ static int intel_init_workaround_bb(struct intel_engine_cs *engine) | |||
1423 | batch = kmap_atomic(page); | 1442 | batch = kmap_atomic(page); |
1424 | offset = 0; | 1443 | offset = 0; |
1425 | 1444 | ||
1426 | if (INTEL_INFO(engine->dev)->gen == 8) { | 1445 | if (IS_GEN8(engine->i915)) { |
1427 | ret = gen8_init_indirectctx_bb(engine, | 1446 | ret = gen8_init_indirectctx_bb(engine, |
1428 | &wa_ctx->indirect_ctx, | 1447 | &wa_ctx->indirect_ctx, |
1429 | batch, | 1448 | batch, |
@@ -1437,7 +1456,7 @@ static int intel_init_workaround_bb(struct intel_engine_cs *engine) | |||
1437 | &offset); | 1456 | &offset); |
1438 | if (ret) | 1457 | if (ret) |
1439 | goto out; | 1458 | goto out; |
1440 | } else if (INTEL_INFO(engine->dev)->gen == 9) { | 1459 | } else if (IS_GEN9(engine->i915)) { |
1441 | ret = gen9_init_indirectctx_bb(engine, | 1460 | ret = gen9_init_indirectctx_bb(engine, |
1442 | &wa_ctx->indirect_ctx, | 1461 | &wa_ctx->indirect_ctx, |
1443 | batch, | 1462 | batch, |
@@ -1463,7 +1482,7 @@ out: | |||
1463 | 1482 | ||
1464 | static void lrc_init_hws(struct intel_engine_cs *engine) | 1483 | static void lrc_init_hws(struct intel_engine_cs *engine) |
1465 | { | 1484 | { |
1466 | struct drm_i915_private *dev_priv = engine->dev->dev_private; | 1485 | struct drm_i915_private *dev_priv = engine->i915; |
1467 | 1486 | ||
1468 | I915_WRITE(RING_HWS_PGA(engine->mmio_base), | 1487 | I915_WRITE(RING_HWS_PGA(engine->mmio_base), |
1469 | (u32)engine->status_page.gfx_addr); | 1488 | (u32)engine->status_page.gfx_addr); |
@@ -1472,8 +1491,7 @@ static void lrc_init_hws(struct intel_engine_cs *engine) | |||
1472 | 1491 | ||
1473 | static int gen8_init_common_ring(struct intel_engine_cs *engine) | 1492 | static int gen8_init_common_ring(struct intel_engine_cs *engine) |
1474 | { | 1493 | { |
1475 | struct drm_device *dev = engine->dev; | 1494 | struct drm_i915_private *dev_priv = engine->i915; |
1476 | struct drm_i915_private *dev_priv = dev->dev_private; | ||
1477 | unsigned int next_context_status_buffer_hw; | 1495 | unsigned int next_context_status_buffer_hw; |
1478 | 1496 | ||
1479 | lrc_init_hws(engine); | 1497 | lrc_init_hws(engine); |
@@ -1520,8 +1538,7 @@ static int gen8_init_common_ring(struct intel_engine_cs *engine) | |||
1520 | 1538 | ||
1521 | static int gen8_init_render_ring(struct intel_engine_cs *engine) | 1539 | static int gen8_init_render_ring(struct intel_engine_cs *engine) |
1522 | { | 1540 | { |
1523 | struct drm_device *dev = engine->dev; | 1541 | struct drm_i915_private *dev_priv = engine->i915; |
1524 | struct drm_i915_private *dev_priv = dev->dev_private; | ||
1525 | int ret; | 1542 | int ret; |
1526 | 1543 | ||
1527 | ret = gen8_init_common_ring(engine); | 1544 | ret = gen8_init_common_ring(engine); |
@@ -1598,7 +1615,7 @@ static int gen8_emit_bb_start(struct drm_i915_gem_request *req, | |||
1598 | if (req->ctx->ppgtt && | 1615 | if (req->ctx->ppgtt && |
1599 | (intel_engine_flag(req->engine) & req->ctx->ppgtt->pd_dirty_rings)) { | 1616 | (intel_engine_flag(req->engine) & req->ctx->ppgtt->pd_dirty_rings)) { |
1600 | if (!USES_FULL_48BIT_PPGTT(req->i915) && | 1617 | if (!USES_FULL_48BIT_PPGTT(req->i915) && |
1601 | !intel_vgpu_active(req->i915->dev)) { | 1618 | !intel_vgpu_active(req->i915)) { |
1602 | ret = intel_logical_ring_emit_pdps(req); | 1619 | ret = intel_logical_ring_emit_pdps(req); |
1603 | if (ret) | 1620 | if (ret) |
1604 | return ret; | 1621 | return ret; |
@@ -1624,38 +1641,18 @@ static int gen8_emit_bb_start(struct drm_i915_gem_request *req, | |||
1624 | return 0; | 1641 | return 0; |
1625 | } | 1642 | } |
1626 | 1643 | ||
1627 | static bool gen8_logical_ring_get_irq(struct intel_engine_cs *engine) | 1644 | static void gen8_logical_ring_enable_irq(struct intel_engine_cs *engine) |
1628 | { | 1645 | { |
1629 | struct drm_device *dev = engine->dev; | 1646 | struct drm_i915_private *dev_priv = engine->i915; |
1630 | struct drm_i915_private *dev_priv = dev->dev_private; | 1647 | I915_WRITE_IMR(engine, |
1631 | unsigned long flags; | 1648 | ~(engine->irq_enable_mask | engine->irq_keep_mask)); |
1632 | 1649 | POSTING_READ_FW(RING_IMR(engine->mmio_base)); | |
1633 | if (WARN_ON(!intel_irqs_enabled(dev_priv))) | ||
1634 | return false; | ||
1635 | |||
1636 | spin_lock_irqsave(&dev_priv->irq_lock, flags); | ||
1637 | if (engine->irq_refcount++ == 0) { | ||
1638 | I915_WRITE_IMR(engine, | ||
1639 | ~(engine->irq_enable_mask | engine->irq_keep_mask)); | ||
1640 | POSTING_READ(RING_IMR(engine->mmio_base)); | ||
1641 | } | ||
1642 | spin_unlock_irqrestore(&dev_priv->irq_lock, flags); | ||
1643 | |||
1644 | return true; | ||
1645 | } | 1650 | } |
1646 | 1651 | ||
1647 | static void gen8_logical_ring_put_irq(struct intel_engine_cs *engine) | 1652 | static void gen8_logical_ring_disable_irq(struct intel_engine_cs *engine) |
1648 | { | 1653 | { |
1649 | struct drm_device *dev = engine->dev; | 1654 | struct drm_i915_private *dev_priv = engine->i915; |
1650 | struct drm_i915_private *dev_priv = dev->dev_private; | 1655 | I915_WRITE_IMR(engine, ~engine->irq_keep_mask); |
1651 | unsigned long flags; | ||
1652 | |||
1653 | spin_lock_irqsave(&dev_priv->irq_lock, flags); | ||
1654 | if (--engine->irq_refcount == 0) { | ||
1655 | I915_WRITE_IMR(engine, ~engine->irq_keep_mask); | ||
1656 | POSTING_READ(RING_IMR(engine->mmio_base)); | ||
1657 | } | ||
1658 | spin_unlock_irqrestore(&dev_priv->irq_lock, flags); | ||
1659 | } | 1656 | } |
1660 | 1657 | ||
1661 | static int gen8_emit_flush(struct drm_i915_gem_request *request, | 1658 | static int gen8_emit_flush(struct drm_i915_gem_request *request, |
@@ -1664,8 +1661,7 @@ static int gen8_emit_flush(struct drm_i915_gem_request *request, | |||
1664 | { | 1661 | { |
1665 | struct intel_ringbuffer *ringbuf = request->ringbuf; | 1662 | struct intel_ringbuffer *ringbuf = request->ringbuf; |
1666 | struct intel_engine_cs *engine = ringbuf->engine; | 1663 | struct intel_engine_cs *engine = ringbuf->engine; |
1667 | struct drm_device *dev = engine->dev; | 1664 | struct drm_i915_private *dev_priv = request->i915; |
1668 | struct drm_i915_private *dev_priv = dev->dev_private; | ||
1669 | uint32_t cmd; | 1665 | uint32_t cmd; |
1670 | int ret; | 1666 | int ret; |
1671 | 1667 | ||
@@ -1734,7 +1730,7 @@ static int gen8_emit_flush_render(struct drm_i915_gem_request *request, | |||
1734 | * On GEN9: before VF_CACHE_INVALIDATE we need to emit a NULL | 1730 | * On GEN9: before VF_CACHE_INVALIDATE we need to emit a NULL |
1735 | * pipe control. | 1731 | * pipe control. |
1736 | */ | 1732 | */ |
1737 | if (IS_GEN9(engine->dev)) | 1733 | if (IS_GEN9(request->i915)) |
1738 | vf_flush_wa = true; | 1734 | vf_flush_wa = true; |
1739 | 1735 | ||
1740 | /* WaForGAMHang:kbl */ | 1736 | /* WaForGAMHang:kbl */ |
@@ -1793,16 +1789,6 @@ static int gen8_emit_flush_render(struct drm_i915_gem_request *request, | |||
1793 | return 0; | 1789 | return 0; |
1794 | } | 1790 | } |
1795 | 1791 | ||
1796 | static u32 gen8_get_seqno(struct intel_engine_cs *engine) | ||
1797 | { | ||
1798 | return intel_read_status_page(engine, I915_GEM_HWS_INDEX); | ||
1799 | } | ||
1800 | |||
1801 | static void gen8_set_seqno(struct intel_engine_cs *engine, u32 seqno) | ||
1802 | { | ||
1803 | intel_write_status_page(engine, I915_GEM_HWS_INDEX, seqno); | ||
1804 | } | ||
1805 | |||
1806 | static void bxt_a_seqno_barrier(struct intel_engine_cs *engine) | 1792 | static void bxt_a_seqno_barrier(struct intel_engine_cs *engine) |
1807 | { | 1793 | { |
1808 | /* | 1794 | /* |
@@ -1818,14 +1804,6 @@ static void bxt_a_seqno_barrier(struct intel_engine_cs *engine) | |||
1818 | intel_flush_status_page(engine, I915_GEM_HWS_INDEX); | 1804 | intel_flush_status_page(engine, I915_GEM_HWS_INDEX); |
1819 | } | 1805 | } |
1820 | 1806 | ||
1821 | static void bxt_a_set_seqno(struct intel_engine_cs *engine, u32 seqno) | ||
1822 | { | ||
1823 | intel_write_status_page(engine, I915_GEM_HWS_INDEX, seqno); | ||
1824 | |||
1825 | /* See bxt_a_get_seqno() explaining the reason for the clflush. */ | ||
1826 | intel_flush_status_page(engine, I915_GEM_HWS_INDEX); | ||
1827 | } | ||
1828 | |||
1829 | /* | 1807 | /* |
1830 | * Reserve space for 2 NOOPs at the end of each request to be | 1808 | * Reserve space for 2 NOOPs at the end of each request to be |
1831 | * used as a workaround for not being allowed to do lite | 1809 | * used as a workaround for not being allowed to do lite |
@@ -1833,11 +1811,6 @@ static void bxt_a_set_seqno(struct intel_engine_cs *engine, u32 seqno) | |||
1833 | */ | 1811 | */ |
1834 | #define WA_TAIL_DWORDS 2 | 1812 | #define WA_TAIL_DWORDS 2 |
1835 | 1813 | ||
1836 | static inline u32 hws_seqno_address(struct intel_engine_cs *engine) | ||
1837 | { | ||
1838 | return engine->status_page.gfx_addr + I915_GEM_HWS_INDEX_ADDR; | ||
1839 | } | ||
1840 | |||
1841 | static int gen8_emit_request(struct drm_i915_gem_request *request) | 1814 | static int gen8_emit_request(struct drm_i915_gem_request *request) |
1842 | { | 1815 | { |
1843 | struct intel_ringbuffer *ringbuf = request->ringbuf; | 1816 | struct intel_ringbuffer *ringbuf = request->ringbuf; |
@@ -1853,10 +1826,10 @@ static int gen8_emit_request(struct drm_i915_gem_request *request) | |||
1853 | intel_logical_ring_emit(ringbuf, | 1826 | intel_logical_ring_emit(ringbuf, |
1854 | (MI_FLUSH_DW + 1) | MI_FLUSH_DW_OP_STOREDW); | 1827 | (MI_FLUSH_DW + 1) | MI_FLUSH_DW_OP_STOREDW); |
1855 | intel_logical_ring_emit(ringbuf, | 1828 | intel_logical_ring_emit(ringbuf, |
1856 | hws_seqno_address(request->engine) | | 1829 | intel_hws_seqno_address(request->engine) | |
1857 | MI_FLUSH_DW_USE_GTT); | 1830 | MI_FLUSH_DW_USE_GTT); |
1858 | intel_logical_ring_emit(ringbuf, 0); | 1831 | intel_logical_ring_emit(ringbuf, 0); |
1859 | intel_logical_ring_emit(ringbuf, i915_gem_request_get_seqno(request)); | 1832 | intel_logical_ring_emit(ringbuf, request->seqno); |
1860 | intel_logical_ring_emit(ringbuf, MI_USER_INTERRUPT); | 1833 | intel_logical_ring_emit(ringbuf, MI_USER_INTERRUPT); |
1861 | intel_logical_ring_emit(ringbuf, MI_NOOP); | 1834 | intel_logical_ring_emit(ringbuf, MI_NOOP); |
1862 | return intel_logical_ring_advance_and_submit(request); | 1835 | return intel_logical_ring_advance_and_submit(request); |
@@ -1883,7 +1856,8 @@ static int gen8_emit_request_render(struct drm_i915_gem_request *request) | |||
1883 | (PIPE_CONTROL_GLOBAL_GTT_IVB | | 1856 | (PIPE_CONTROL_GLOBAL_GTT_IVB | |
1884 | PIPE_CONTROL_CS_STALL | | 1857 | PIPE_CONTROL_CS_STALL | |
1885 | PIPE_CONTROL_QW_WRITE)); | 1858 | PIPE_CONTROL_QW_WRITE)); |
1886 | intel_logical_ring_emit(ringbuf, hws_seqno_address(request->engine)); | 1859 | intel_logical_ring_emit(ringbuf, |
1860 | intel_hws_seqno_address(request->engine)); | ||
1887 | intel_logical_ring_emit(ringbuf, 0); | 1861 | intel_logical_ring_emit(ringbuf, 0); |
1888 | intel_logical_ring_emit(ringbuf, i915_gem_request_get_seqno(request)); | 1862 | intel_logical_ring_emit(ringbuf, i915_gem_request_get_seqno(request)); |
1889 | /* We're thrashing one dword of HWS. */ | 1863 | /* We're thrashing one dword of HWS. */ |
@@ -1945,7 +1919,7 @@ static int gen8_init_rcs_context(struct drm_i915_gem_request *req) | |||
1945 | /** | 1919 | /** |
1946 | * intel_logical_ring_cleanup() - deallocate the Engine Command Streamer | 1920 | * intel_logical_ring_cleanup() - deallocate the Engine Command Streamer |
1947 | * | 1921 | * |
1948 | * @ring: Engine Command Streamer. | 1922 | * @engine: Engine Command Streamer. |
1949 | * | 1923 | * |
1950 | */ | 1924 | */ |
1951 | void intel_logical_ring_cleanup(struct intel_engine_cs *engine) | 1925 | void intel_logical_ring_cleanup(struct intel_engine_cs *engine) |
@@ -1962,7 +1936,7 @@ void intel_logical_ring_cleanup(struct intel_engine_cs *engine) | |||
1962 | if (WARN_ON(test_bit(TASKLET_STATE_SCHED, &engine->irq_tasklet.state))) | 1936 | if (WARN_ON(test_bit(TASKLET_STATE_SCHED, &engine->irq_tasklet.state))) |
1963 | tasklet_kill(&engine->irq_tasklet); | 1937 | tasklet_kill(&engine->irq_tasklet); |
1964 | 1938 | ||
1965 | dev_priv = engine->dev->dev_private; | 1939 | dev_priv = engine->i915; |
1966 | 1940 | ||
1967 | if (engine->buffer) { | 1941 | if (engine->buffer) { |
1968 | intel_logical_ring_stop(engine); | 1942 | intel_logical_ring_stop(engine); |
@@ -1975,36 +1949,34 @@ void intel_logical_ring_cleanup(struct intel_engine_cs *engine) | |||
1975 | i915_cmd_parser_fini_ring(engine); | 1949 | i915_cmd_parser_fini_ring(engine); |
1976 | i915_gem_batch_pool_fini(&engine->batch_pool); | 1950 | i915_gem_batch_pool_fini(&engine->batch_pool); |
1977 | 1951 | ||
1952 | intel_engine_fini_breadcrumbs(engine); | ||
1953 | |||
1978 | if (engine->status_page.obj) { | 1954 | if (engine->status_page.obj) { |
1979 | i915_gem_object_unpin_map(engine->status_page.obj); | 1955 | i915_gem_object_unpin_map(engine->status_page.obj); |
1980 | engine->status_page.obj = NULL; | 1956 | engine->status_page.obj = NULL; |
1981 | } | 1957 | } |
1958 | intel_lr_context_unpin(dev_priv->kernel_context, engine); | ||
1982 | 1959 | ||
1983 | engine->idle_lite_restore_wa = 0; | 1960 | engine->idle_lite_restore_wa = 0; |
1984 | engine->disable_lite_restore_wa = false; | 1961 | engine->disable_lite_restore_wa = false; |
1985 | engine->ctx_desc_template = 0; | 1962 | engine->ctx_desc_template = 0; |
1986 | 1963 | ||
1987 | lrc_destroy_wa_ctx_obj(engine); | 1964 | lrc_destroy_wa_ctx_obj(engine); |
1988 | engine->dev = NULL; | 1965 | engine->i915 = NULL; |
1989 | } | 1966 | } |
1990 | 1967 | ||
1991 | static void | 1968 | static void |
1992 | logical_ring_default_vfuncs(struct drm_device *dev, | 1969 | logical_ring_default_vfuncs(struct intel_engine_cs *engine) |
1993 | struct intel_engine_cs *engine) | ||
1994 | { | 1970 | { |
1995 | /* Default vfuncs which can be overriden by each engine. */ | 1971 | /* Default vfuncs which can be overriden by each engine. */ |
1996 | engine->init_hw = gen8_init_common_ring; | 1972 | engine->init_hw = gen8_init_common_ring; |
1997 | engine->emit_request = gen8_emit_request; | 1973 | engine->emit_request = gen8_emit_request; |
1998 | engine->emit_flush = gen8_emit_flush; | 1974 | engine->emit_flush = gen8_emit_flush; |
1999 | engine->irq_get = gen8_logical_ring_get_irq; | 1975 | engine->irq_enable = gen8_logical_ring_enable_irq; |
2000 | engine->irq_put = gen8_logical_ring_put_irq; | 1976 | engine->irq_disable = gen8_logical_ring_disable_irq; |
2001 | engine->emit_bb_start = gen8_emit_bb_start; | 1977 | engine->emit_bb_start = gen8_emit_bb_start; |
2002 | engine->get_seqno = gen8_get_seqno; | 1978 | if (IS_BXT_REVID(engine->i915, 0, BXT_REVID_A1)) |
2003 | engine->set_seqno = gen8_set_seqno; | ||
2004 | if (IS_BXT_REVID(dev, 0, BXT_REVID_A1)) { | ||
2005 | engine->irq_seqno_barrier = bxt_a_seqno_barrier; | 1979 | engine->irq_seqno_barrier = bxt_a_seqno_barrier; |
2006 | engine->set_seqno = bxt_a_set_seqno; | ||
2007 | } | ||
2008 | } | 1980 | } |
2009 | 1981 | ||
2010 | static inline void | 1982 | static inline void |
@@ -2033,60 +2005,28 @@ lrc_setup_hws(struct intel_engine_cs *engine, | |||
2033 | } | 2005 | } |
2034 | 2006 | ||
2035 | static int | 2007 | static int |
2036 | logical_ring_init(struct drm_device *dev, struct intel_engine_cs *engine) | 2008 | logical_ring_init(struct intel_engine_cs *engine) |
2037 | { | 2009 | { |
2038 | struct drm_i915_private *dev_priv = to_i915(dev); | 2010 | struct i915_gem_context *dctx = engine->i915->kernel_context; |
2039 | struct intel_context *dctx = dev_priv->kernel_context; | ||
2040 | enum forcewake_domains fw_domains; | ||
2041 | int ret; | 2011 | int ret; |
2042 | 2012 | ||
2043 | /* Intentionally left blank. */ | 2013 | ret = intel_engine_init_breadcrumbs(engine); |
2044 | engine->buffer = NULL; | 2014 | if (ret) |
2045 | 2015 | goto error; | |
2046 | engine->dev = dev; | ||
2047 | INIT_LIST_HEAD(&engine->active_list); | ||
2048 | INIT_LIST_HEAD(&engine->request_list); | ||
2049 | i915_gem_batch_pool_init(dev, &engine->batch_pool); | ||
2050 | init_waitqueue_head(&engine->irq_queue); | ||
2051 | |||
2052 | INIT_LIST_HEAD(&engine->buffers); | ||
2053 | INIT_LIST_HEAD(&engine->execlist_queue); | ||
2054 | INIT_LIST_HEAD(&engine->execlist_retired_req_list); | ||
2055 | spin_lock_init(&engine->execlist_lock); | ||
2056 | |||
2057 | tasklet_init(&engine->irq_tasklet, | ||
2058 | intel_lrc_irq_handler, (unsigned long)engine); | ||
2059 | |||
2060 | logical_ring_init_platform_invariants(engine); | ||
2061 | |||
2062 | fw_domains = intel_uncore_forcewake_for_reg(dev_priv, | ||
2063 | RING_ELSP(engine), | ||
2064 | FW_REG_WRITE); | ||
2065 | |||
2066 | fw_domains |= intel_uncore_forcewake_for_reg(dev_priv, | ||
2067 | RING_CONTEXT_STATUS_PTR(engine), | ||
2068 | FW_REG_READ | FW_REG_WRITE); | ||
2069 | |||
2070 | fw_domains |= intel_uncore_forcewake_for_reg(dev_priv, | ||
2071 | RING_CONTEXT_STATUS_BUF_BASE(engine), | ||
2072 | FW_REG_READ); | ||
2073 | |||
2074 | engine->fw_domains = fw_domains; | ||
2075 | 2016 | ||
2076 | ret = i915_cmd_parser_init_ring(engine); | 2017 | ret = i915_cmd_parser_init_ring(engine); |
2077 | if (ret) | 2018 | if (ret) |
2078 | goto error; | 2019 | goto error; |
2079 | 2020 | ||
2080 | ret = intel_lr_context_deferred_alloc(dctx, engine); | 2021 | ret = execlists_context_deferred_alloc(dctx, engine); |
2081 | if (ret) | 2022 | if (ret) |
2082 | goto error; | 2023 | goto error; |
2083 | 2024 | ||
2084 | /* As this is the default context, always pin it */ | 2025 | /* As this is the default context, always pin it */ |
2085 | ret = intel_lr_context_do_pin(dctx, engine); | 2026 | ret = intel_lr_context_pin(dctx, engine); |
2086 | if (ret) { | 2027 | if (ret) { |
2087 | DRM_ERROR( | 2028 | DRM_ERROR("Failed to pin context for %s: %d\n", |
2088 | "Failed to pin and map ringbuffer %s: %d\n", | 2029 | engine->name, ret); |
2089 | engine->name, ret); | ||
2090 | goto error; | 2030 | goto error; |
2091 | } | 2031 | } |
2092 | 2032 | ||
@@ -2104,26 +2044,16 @@ error: | |||
2104 | return ret; | 2044 | return ret; |
2105 | } | 2045 | } |
2106 | 2046 | ||
2107 | static int logical_render_ring_init(struct drm_device *dev) | 2047 | static int logical_render_ring_init(struct intel_engine_cs *engine) |
2108 | { | 2048 | { |
2109 | struct drm_i915_private *dev_priv = dev->dev_private; | 2049 | struct drm_i915_private *dev_priv = engine->i915; |
2110 | struct intel_engine_cs *engine = &dev_priv->engine[RCS]; | ||
2111 | int ret; | 2050 | int ret; |
2112 | 2051 | ||
2113 | engine->name = "render ring"; | 2052 | if (HAS_L3_DPF(dev_priv)) |
2114 | engine->id = RCS; | ||
2115 | engine->exec_id = I915_EXEC_RENDER; | ||
2116 | engine->guc_id = GUC_RENDER_ENGINE; | ||
2117 | engine->mmio_base = RENDER_RING_BASE; | ||
2118 | |||
2119 | logical_ring_default_irqs(engine, GEN8_RCS_IRQ_SHIFT); | ||
2120 | if (HAS_L3_DPF(dev)) | ||
2121 | engine->irq_keep_mask |= GT_RENDER_L3_PARITY_ERROR_INTERRUPT; | 2053 | engine->irq_keep_mask |= GT_RENDER_L3_PARITY_ERROR_INTERRUPT; |
2122 | 2054 | ||
2123 | logical_ring_default_vfuncs(dev, engine); | ||
2124 | |||
2125 | /* Override some for render ring. */ | 2055 | /* Override some for render ring. */ |
2126 | if (INTEL_INFO(dev)->gen >= 9) | 2056 | if (INTEL_GEN(dev_priv) >= 9) |
2127 | engine->init_hw = gen9_init_render_ring; | 2057 | engine->init_hw = gen9_init_render_ring; |
2128 | else | 2058 | else |
2129 | engine->init_hw = gen8_init_render_ring; | 2059 | engine->init_hw = gen8_init_render_ring; |
@@ -2132,9 +2062,7 @@ static int logical_render_ring_init(struct drm_device *dev) | |||
2132 | engine->emit_flush = gen8_emit_flush_render; | 2062 | engine->emit_flush = gen8_emit_flush_render; |
2133 | engine->emit_request = gen8_emit_request_render; | 2063 | engine->emit_request = gen8_emit_request_render; |
2134 | 2064 | ||
2135 | engine->dev = dev; | 2065 | ret = intel_init_pipe_control(engine, 4096); |
2136 | |||
2137 | ret = intel_init_pipe_control(engine); | ||
2138 | if (ret) | 2066 | if (ret) |
2139 | return ret; | 2067 | return ret; |
2140 | 2068 | ||
@@ -2149,7 +2077,7 @@ static int logical_render_ring_init(struct drm_device *dev) | |||
2149 | ret); | 2077 | ret); |
2150 | } | 2078 | } |
2151 | 2079 | ||
2152 | ret = logical_ring_init(dev, engine); | 2080 | ret = logical_ring_init(engine); |
2153 | if (ret) { | 2081 | if (ret) { |
2154 | lrc_destroy_wa_ctx_obj(engine); | 2082 | lrc_destroy_wa_ctx_obj(engine); |
2155 | } | 2083 | } |
@@ -2157,133 +2085,164 @@ static int logical_render_ring_init(struct drm_device *dev) | |||
2157 | return ret; | 2085 | return ret; |
2158 | } | 2086 | } |
2159 | 2087 | ||
2160 | static int logical_bsd_ring_init(struct drm_device *dev) | 2088 | static const struct logical_ring_info { |
2161 | { | 2089 | const char *name; |
2162 | struct drm_i915_private *dev_priv = dev->dev_private; | 2090 | unsigned exec_id; |
2163 | struct intel_engine_cs *engine = &dev_priv->engine[VCS]; | 2091 | unsigned guc_id; |
2164 | 2092 | u32 mmio_base; | |
2165 | engine->name = "bsd ring"; | 2093 | unsigned irq_shift; |
2166 | engine->id = VCS; | 2094 | int (*init)(struct intel_engine_cs *engine); |
2167 | engine->exec_id = I915_EXEC_BSD; | 2095 | } logical_rings[] = { |
2168 | engine->guc_id = GUC_VIDEO_ENGINE; | 2096 | [RCS] = { |
2169 | engine->mmio_base = GEN6_BSD_RING_BASE; | 2097 | .name = "render ring", |
2170 | 2098 | .exec_id = I915_EXEC_RENDER, | |
2171 | logical_ring_default_irqs(engine, GEN8_VCS1_IRQ_SHIFT); | 2099 | .guc_id = GUC_RENDER_ENGINE, |
2172 | logical_ring_default_vfuncs(dev, engine); | 2100 | .mmio_base = RENDER_RING_BASE, |
2173 | 2101 | .irq_shift = GEN8_RCS_IRQ_SHIFT, | |
2174 | return logical_ring_init(dev, engine); | 2102 | .init = logical_render_ring_init, |
2175 | } | 2103 | }, |
2104 | [BCS] = { | ||
2105 | .name = "blitter ring", | ||
2106 | .exec_id = I915_EXEC_BLT, | ||
2107 | .guc_id = GUC_BLITTER_ENGINE, | ||
2108 | .mmio_base = BLT_RING_BASE, | ||
2109 | .irq_shift = GEN8_BCS_IRQ_SHIFT, | ||
2110 | .init = logical_ring_init, | ||
2111 | }, | ||
2112 | [VCS] = { | ||
2113 | .name = "bsd ring", | ||
2114 | .exec_id = I915_EXEC_BSD, | ||
2115 | .guc_id = GUC_VIDEO_ENGINE, | ||
2116 | .mmio_base = GEN6_BSD_RING_BASE, | ||
2117 | .irq_shift = GEN8_VCS1_IRQ_SHIFT, | ||
2118 | .init = logical_ring_init, | ||
2119 | }, | ||
2120 | [VCS2] = { | ||
2121 | .name = "bsd2 ring", | ||
2122 | .exec_id = I915_EXEC_BSD, | ||
2123 | .guc_id = GUC_VIDEO_ENGINE2, | ||
2124 | .mmio_base = GEN8_BSD2_RING_BASE, | ||
2125 | .irq_shift = GEN8_VCS2_IRQ_SHIFT, | ||
2126 | .init = logical_ring_init, | ||
2127 | }, | ||
2128 | [VECS] = { | ||
2129 | .name = "video enhancement ring", | ||
2130 | .exec_id = I915_EXEC_VEBOX, | ||
2131 | .guc_id = GUC_VIDEOENHANCE_ENGINE, | ||
2132 | .mmio_base = VEBOX_RING_BASE, | ||
2133 | .irq_shift = GEN8_VECS_IRQ_SHIFT, | ||
2134 | .init = logical_ring_init, | ||
2135 | }, | ||
2136 | }; | ||
2176 | 2137 | ||
2177 | static int logical_bsd2_ring_init(struct drm_device *dev) | 2138 | static struct intel_engine_cs * |
2139 | logical_ring_setup(struct drm_i915_private *dev_priv, enum intel_engine_id id) | ||
2178 | { | 2140 | { |
2179 | struct drm_i915_private *dev_priv = dev->dev_private; | 2141 | const struct logical_ring_info *info = &logical_rings[id]; |
2180 | struct intel_engine_cs *engine = &dev_priv->engine[VCS2]; | 2142 | struct intel_engine_cs *engine = &dev_priv->engine[id]; |
2143 | enum forcewake_domains fw_domains; | ||
2181 | 2144 | ||
2182 | engine->name = "bsd2 ring"; | 2145 | engine->id = id; |
2183 | engine->id = VCS2; | 2146 | engine->name = info->name; |
2184 | engine->exec_id = I915_EXEC_BSD; | 2147 | engine->exec_id = info->exec_id; |
2185 | engine->guc_id = GUC_VIDEO_ENGINE2; | 2148 | engine->guc_id = info->guc_id; |
2186 | engine->mmio_base = GEN8_BSD2_RING_BASE; | 2149 | engine->mmio_base = info->mmio_base; |
2187 | 2150 | ||
2188 | logical_ring_default_irqs(engine, GEN8_VCS2_IRQ_SHIFT); | 2151 | engine->i915 = dev_priv; |
2189 | logical_ring_default_vfuncs(dev, engine); | ||
2190 | 2152 | ||
2191 | return logical_ring_init(dev, engine); | 2153 | /* Intentionally left blank. */ |
2192 | } | 2154 | engine->buffer = NULL; |
2193 | 2155 | ||
2194 | static int logical_blt_ring_init(struct drm_device *dev) | 2156 | fw_domains = intel_uncore_forcewake_for_reg(dev_priv, |
2195 | { | 2157 | RING_ELSP(engine), |
2196 | struct drm_i915_private *dev_priv = dev->dev_private; | 2158 | FW_REG_WRITE); |
2197 | struct intel_engine_cs *engine = &dev_priv->engine[BCS]; | ||
2198 | 2159 | ||
2199 | engine->name = "blitter ring"; | 2160 | fw_domains |= intel_uncore_forcewake_for_reg(dev_priv, |
2200 | engine->id = BCS; | 2161 | RING_CONTEXT_STATUS_PTR(engine), |
2201 | engine->exec_id = I915_EXEC_BLT; | 2162 | FW_REG_READ | FW_REG_WRITE); |
2202 | engine->guc_id = GUC_BLITTER_ENGINE; | ||
2203 | engine->mmio_base = BLT_RING_BASE; | ||
2204 | 2163 | ||
2205 | logical_ring_default_irqs(engine, GEN8_BCS_IRQ_SHIFT); | 2164 | fw_domains |= intel_uncore_forcewake_for_reg(dev_priv, |
2206 | logical_ring_default_vfuncs(dev, engine); | 2165 | RING_CONTEXT_STATUS_BUF_BASE(engine), |
2166 | FW_REG_READ); | ||
2207 | 2167 | ||
2208 | return logical_ring_init(dev, engine); | 2168 | engine->fw_domains = fw_domains; |
2209 | } | ||
2210 | 2169 | ||
2211 | static int logical_vebox_ring_init(struct drm_device *dev) | 2170 | INIT_LIST_HEAD(&engine->active_list); |
2212 | { | 2171 | INIT_LIST_HEAD(&engine->request_list); |
2213 | struct drm_i915_private *dev_priv = dev->dev_private; | 2172 | INIT_LIST_HEAD(&engine->buffers); |
2214 | struct intel_engine_cs *engine = &dev_priv->engine[VECS]; | 2173 | INIT_LIST_HEAD(&engine->execlist_queue); |
2174 | spin_lock_init(&engine->execlist_lock); | ||
2215 | 2175 | ||
2216 | engine->name = "video enhancement ring"; | 2176 | tasklet_init(&engine->irq_tasklet, |
2217 | engine->id = VECS; | 2177 | intel_lrc_irq_handler, (unsigned long)engine); |
2218 | engine->exec_id = I915_EXEC_VEBOX; | ||
2219 | engine->guc_id = GUC_VIDEOENHANCE_ENGINE; | ||
2220 | engine->mmio_base = VEBOX_RING_BASE; | ||
2221 | 2178 | ||
2222 | logical_ring_default_irqs(engine, GEN8_VECS_IRQ_SHIFT); | 2179 | logical_ring_init_platform_invariants(engine); |
2223 | logical_ring_default_vfuncs(dev, engine); | 2180 | logical_ring_default_vfuncs(engine); |
2181 | logical_ring_default_irqs(engine, info->irq_shift); | ||
2182 | |||
2183 | intel_engine_init_hangcheck(engine); | ||
2184 | i915_gem_batch_pool_init(&dev_priv->drm, &engine->batch_pool); | ||
2224 | 2185 | ||
2225 | return logical_ring_init(dev, engine); | 2186 | return engine; |
2226 | } | 2187 | } |
2227 | 2188 | ||
2228 | /** | 2189 | /** |
2229 | * intel_logical_rings_init() - allocate, populate and init the Engine Command Streamers | 2190 | * intel_logical_rings_init() - allocate, populate and init the Engine Command Streamers |
2230 | * @dev: DRM device. | 2191 | * @dev: DRM device. |
2231 | * | 2192 | * |
2232 | * This function inits the engines for an Execlists submission style (the equivalent in the | 2193 | * This function inits the engines for an Execlists submission style (the |
2233 | * legacy ringbuffer submission world would be i915_gem_init_engines). It does it only for | 2194 | * equivalent in the legacy ringbuffer submission world would be |
2234 | * those engines that are present in the hardware. | 2195 | * i915_gem_init_engines). It does it only for those engines that are present in |
2196 | * the hardware. | ||
2235 | * | 2197 | * |
2236 | * Return: non-zero if the initialization failed. | 2198 | * Return: non-zero if the initialization failed. |
2237 | */ | 2199 | */ |
2238 | int intel_logical_rings_init(struct drm_device *dev) | 2200 | int intel_logical_rings_init(struct drm_device *dev) |
2239 | { | 2201 | { |
2240 | struct drm_i915_private *dev_priv = dev->dev_private; | 2202 | struct drm_i915_private *dev_priv = to_i915(dev); |
2203 | unsigned int mask = 0; | ||
2204 | unsigned int i; | ||
2241 | int ret; | 2205 | int ret; |
2242 | 2206 | ||
2243 | ret = logical_render_ring_init(dev); | 2207 | WARN_ON(INTEL_INFO(dev_priv)->ring_mask & |
2244 | if (ret) | 2208 | GENMASK(sizeof(mask) * BITS_PER_BYTE - 1, I915_NUM_ENGINES)); |
2245 | return ret; | ||
2246 | 2209 | ||
2247 | if (HAS_BSD(dev)) { | 2210 | for (i = 0; i < ARRAY_SIZE(logical_rings); i++) { |
2248 | ret = logical_bsd_ring_init(dev); | 2211 | if (!HAS_ENGINE(dev_priv, i)) |
2249 | if (ret) | 2212 | continue; |
2250 | goto cleanup_render_ring; | ||
2251 | } | ||
2252 | 2213 | ||
2253 | if (HAS_BLT(dev)) { | 2214 | if (!logical_rings[i].init) |
2254 | ret = logical_blt_ring_init(dev); | 2215 | continue; |
2255 | if (ret) | ||
2256 | goto cleanup_bsd_ring; | ||
2257 | } | ||
2258 | 2216 | ||
2259 | if (HAS_VEBOX(dev)) { | 2217 | ret = logical_rings[i].init(logical_ring_setup(dev_priv, i)); |
2260 | ret = logical_vebox_ring_init(dev); | ||
2261 | if (ret) | 2218 | if (ret) |
2262 | goto cleanup_blt_ring; | 2219 | goto cleanup; |
2220 | |||
2221 | mask |= ENGINE_MASK(i); | ||
2263 | } | 2222 | } |
2264 | 2223 | ||
2265 | if (HAS_BSD2(dev)) { | 2224 | /* |
2266 | ret = logical_bsd2_ring_init(dev); | 2225 | * Catch failures to update logical_rings table when the new engines |
2267 | if (ret) | 2226 | * are added to the driver by a warning and disabling the forgotten |
2268 | goto cleanup_vebox_ring; | 2227 | * engines. |
2228 | */ | ||
2229 | if (WARN_ON(mask != INTEL_INFO(dev_priv)->ring_mask)) { | ||
2230 | struct intel_device_info *info = | ||
2231 | (struct intel_device_info *)&dev_priv->info; | ||
2232 | info->ring_mask = mask; | ||
2269 | } | 2233 | } |
2270 | 2234 | ||
2271 | return 0; | 2235 | return 0; |
2272 | 2236 | ||
2273 | cleanup_vebox_ring: | 2237 | cleanup: |
2274 | intel_logical_ring_cleanup(&dev_priv->engine[VECS]); | 2238 | for (i = 0; i < I915_NUM_ENGINES; i++) |
2275 | cleanup_blt_ring: | 2239 | intel_logical_ring_cleanup(&dev_priv->engine[i]); |
2276 | intel_logical_ring_cleanup(&dev_priv->engine[BCS]); | ||
2277 | cleanup_bsd_ring: | ||
2278 | intel_logical_ring_cleanup(&dev_priv->engine[VCS]); | ||
2279 | cleanup_render_ring: | ||
2280 | intel_logical_ring_cleanup(&dev_priv->engine[RCS]); | ||
2281 | 2240 | ||
2282 | return ret; | 2241 | return ret; |
2283 | } | 2242 | } |
2284 | 2243 | ||
2285 | static u32 | 2244 | static u32 |
2286 | make_rpcs(struct drm_device *dev) | 2245 | make_rpcs(struct drm_i915_private *dev_priv) |
2287 | { | 2246 | { |
2288 | u32 rpcs = 0; | 2247 | u32 rpcs = 0; |
2289 | 2248 | ||
@@ -2291,7 +2250,7 @@ make_rpcs(struct drm_device *dev) | |||
2291 | * No explicit RPCS request is needed to ensure full | 2250 | * No explicit RPCS request is needed to ensure full |
2292 | * slice/subslice/EU enablement prior to Gen9. | 2251 | * slice/subslice/EU enablement prior to Gen9. |
2293 | */ | 2252 | */ |
2294 | if (INTEL_INFO(dev)->gen < 9) | 2253 | if (INTEL_GEN(dev_priv) < 9) |
2295 | return 0; | 2254 | return 0; |
2296 | 2255 | ||
2297 | /* | 2256 | /* |
@@ -2300,24 +2259,24 @@ make_rpcs(struct drm_device *dev) | |||
2300 | * must make an explicit request through RPCS for full | 2259 | * must make an explicit request through RPCS for full |
2301 | * enablement. | 2260 | * enablement. |
2302 | */ | 2261 | */ |
2303 | if (INTEL_INFO(dev)->has_slice_pg) { | 2262 | if (INTEL_INFO(dev_priv)->has_slice_pg) { |
2304 | rpcs |= GEN8_RPCS_S_CNT_ENABLE; | 2263 | rpcs |= GEN8_RPCS_S_CNT_ENABLE; |
2305 | rpcs |= INTEL_INFO(dev)->slice_total << | 2264 | rpcs |= INTEL_INFO(dev_priv)->slice_total << |
2306 | GEN8_RPCS_S_CNT_SHIFT; | 2265 | GEN8_RPCS_S_CNT_SHIFT; |
2307 | rpcs |= GEN8_RPCS_ENABLE; | 2266 | rpcs |= GEN8_RPCS_ENABLE; |
2308 | } | 2267 | } |
2309 | 2268 | ||
2310 | if (INTEL_INFO(dev)->has_subslice_pg) { | 2269 | if (INTEL_INFO(dev_priv)->has_subslice_pg) { |
2311 | rpcs |= GEN8_RPCS_SS_CNT_ENABLE; | 2270 | rpcs |= GEN8_RPCS_SS_CNT_ENABLE; |
2312 | rpcs |= INTEL_INFO(dev)->subslice_per_slice << | 2271 | rpcs |= INTEL_INFO(dev_priv)->subslice_per_slice << |
2313 | GEN8_RPCS_SS_CNT_SHIFT; | 2272 | GEN8_RPCS_SS_CNT_SHIFT; |
2314 | rpcs |= GEN8_RPCS_ENABLE; | 2273 | rpcs |= GEN8_RPCS_ENABLE; |
2315 | } | 2274 | } |
2316 | 2275 | ||
2317 | if (INTEL_INFO(dev)->has_eu_pg) { | 2276 | if (INTEL_INFO(dev_priv)->has_eu_pg) { |
2318 | rpcs |= INTEL_INFO(dev)->eu_per_subslice << | 2277 | rpcs |= INTEL_INFO(dev_priv)->eu_per_subslice << |
2319 | GEN8_RPCS_EU_MIN_SHIFT; | 2278 | GEN8_RPCS_EU_MIN_SHIFT; |
2320 | rpcs |= INTEL_INFO(dev)->eu_per_subslice << | 2279 | rpcs |= INTEL_INFO(dev_priv)->eu_per_subslice << |
2321 | GEN8_RPCS_EU_MAX_SHIFT; | 2280 | GEN8_RPCS_EU_MAX_SHIFT; |
2322 | rpcs |= GEN8_RPCS_ENABLE; | 2281 | rpcs |= GEN8_RPCS_ENABLE; |
2323 | } | 2282 | } |
@@ -2329,9 +2288,9 @@ static u32 intel_lr_indirect_ctx_offset(struct intel_engine_cs *engine) | |||
2329 | { | 2288 | { |
2330 | u32 indirect_ctx_offset; | 2289 | u32 indirect_ctx_offset; |
2331 | 2290 | ||
2332 | switch (INTEL_INFO(engine->dev)->gen) { | 2291 | switch (INTEL_GEN(engine->i915)) { |
2333 | default: | 2292 | default: |
2334 | MISSING_CASE(INTEL_INFO(engine->dev)->gen); | 2293 | MISSING_CASE(INTEL_GEN(engine->i915)); |
2335 | /* fall through */ | 2294 | /* fall through */ |
2336 | case 9: | 2295 | case 9: |
2337 | indirect_ctx_offset = | 2296 | indirect_ctx_offset = |
@@ -2347,13 +2306,12 @@ static u32 intel_lr_indirect_ctx_offset(struct intel_engine_cs *engine) | |||
2347 | } | 2306 | } |
2348 | 2307 | ||
2349 | static int | 2308 | static int |
2350 | populate_lr_context(struct intel_context *ctx, | 2309 | populate_lr_context(struct i915_gem_context *ctx, |
2351 | struct drm_i915_gem_object *ctx_obj, | 2310 | struct drm_i915_gem_object *ctx_obj, |
2352 | struct intel_engine_cs *engine, | 2311 | struct intel_engine_cs *engine, |
2353 | struct intel_ringbuffer *ringbuf) | 2312 | struct intel_ringbuffer *ringbuf) |
2354 | { | 2313 | { |
2355 | struct drm_device *dev = engine->dev; | 2314 | struct drm_i915_private *dev_priv = ctx->i915; |
2356 | struct drm_i915_private *dev_priv = dev->dev_private; | ||
2357 | struct i915_hw_ppgtt *ppgtt = ctx->ppgtt; | 2315 | struct i915_hw_ppgtt *ppgtt = ctx->ppgtt; |
2358 | void *vaddr; | 2316 | void *vaddr; |
2359 | u32 *reg_state; | 2317 | u32 *reg_state; |
@@ -2391,7 +2349,7 @@ populate_lr_context(struct intel_context *ctx, | |||
2391 | RING_CONTEXT_CONTROL(engine), | 2349 | RING_CONTEXT_CONTROL(engine), |
2392 | _MASKED_BIT_ENABLE(CTX_CTRL_INHIBIT_SYN_CTX_SWITCH | | 2350 | _MASKED_BIT_ENABLE(CTX_CTRL_INHIBIT_SYN_CTX_SWITCH | |
2393 | CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT | | 2351 | CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT | |
2394 | (HAS_RESOURCE_STREAMER(dev) ? | 2352 | (HAS_RESOURCE_STREAMER(dev_priv) ? |
2395 | CTX_CTRL_RS_CTX_ENABLE : 0))); | 2353 | CTX_CTRL_RS_CTX_ENABLE : 0))); |
2396 | ASSIGN_CTX_REG(reg_state, CTX_RING_HEAD, RING_HEAD(engine->mmio_base), | 2354 | ASSIGN_CTX_REG(reg_state, CTX_RING_HEAD, RING_HEAD(engine->mmio_base), |
2397 | 0); | 2355 | 0); |
@@ -2480,7 +2438,7 @@ populate_lr_context(struct intel_context *ctx, | |||
2480 | if (engine->id == RCS) { | 2438 | if (engine->id == RCS) { |
2481 | reg_state[CTX_LRI_HEADER_2] = MI_LOAD_REGISTER_IMM(1); | 2439 | reg_state[CTX_LRI_HEADER_2] = MI_LOAD_REGISTER_IMM(1); |
2482 | ASSIGN_CTX_REG(reg_state, CTX_R_PWR_CLK_STATE, GEN8_R_PWR_CLK_STATE, | 2440 | ASSIGN_CTX_REG(reg_state, CTX_R_PWR_CLK_STATE, GEN8_R_PWR_CLK_STATE, |
2483 | make_rpcs(dev)); | 2441 | make_rpcs(dev_priv)); |
2484 | } | 2442 | } |
2485 | 2443 | ||
2486 | i915_gem_object_unpin_map(ctx_obj); | 2444 | i915_gem_object_unpin_map(ctx_obj); |
@@ -2489,39 +2447,8 @@ populate_lr_context(struct intel_context *ctx, | |||
2489 | } | 2447 | } |
2490 | 2448 | ||
2491 | /** | 2449 | /** |
2492 | * intel_lr_context_free() - free the LRC specific bits of a context | ||
2493 | * @ctx: the LR context to free. | ||
2494 | * | ||
2495 | * The real context freeing is done in i915_gem_context_free: this only | ||
2496 | * takes care of the bits that are LRC related: the per-engine backing | ||
2497 | * objects and the logical ringbuffer. | ||
2498 | */ | ||
2499 | void intel_lr_context_free(struct intel_context *ctx) | ||
2500 | { | ||
2501 | int i; | ||
2502 | |||
2503 | for (i = I915_NUM_ENGINES; --i >= 0; ) { | ||
2504 | struct intel_ringbuffer *ringbuf = ctx->engine[i].ringbuf; | ||
2505 | struct drm_i915_gem_object *ctx_obj = ctx->engine[i].state; | ||
2506 | |||
2507 | if (!ctx_obj) | ||
2508 | continue; | ||
2509 | |||
2510 | if (ctx == ctx->i915->kernel_context) { | ||
2511 | intel_unpin_ringbuffer_obj(ringbuf); | ||
2512 | i915_gem_object_ggtt_unpin(ctx_obj); | ||
2513 | i915_gem_object_unpin_map(ctx_obj); | ||
2514 | } | ||
2515 | |||
2516 | WARN_ON(ctx->engine[i].pin_count); | ||
2517 | intel_ringbuffer_free(ringbuf); | ||
2518 | drm_gem_object_unreference(&ctx_obj->base); | ||
2519 | } | ||
2520 | } | ||
2521 | |||
2522 | /** | ||
2523 | * intel_lr_context_size() - return the size of the context for an engine | 2450 | * intel_lr_context_size() - return the size of the context for an engine |
2524 | * @ring: which engine to find the context size for | 2451 | * @engine: which engine to find the context size for |
2525 | * | 2452 | * |
2526 | * Each engine may require a different amount of space for a context image, | 2453 | * Each engine may require a different amount of space for a context image, |
2527 | * so when allocating (or copying) an image, this function can be used to | 2454 | * so when allocating (or copying) an image, this function can be used to |
@@ -2537,11 +2464,11 @@ uint32_t intel_lr_context_size(struct intel_engine_cs *engine) | |||
2537 | { | 2464 | { |
2538 | int ret = 0; | 2465 | int ret = 0; |
2539 | 2466 | ||
2540 | WARN_ON(INTEL_INFO(engine->dev)->gen < 8); | 2467 | WARN_ON(INTEL_GEN(engine->i915) < 8); |
2541 | 2468 | ||
2542 | switch (engine->id) { | 2469 | switch (engine->id) { |
2543 | case RCS: | 2470 | case RCS: |
2544 | if (INTEL_INFO(engine->dev)->gen >= 9) | 2471 | if (INTEL_GEN(engine->i915) >= 9) |
2545 | ret = GEN9_LR_CONTEXT_RENDER_SIZE; | 2472 | ret = GEN9_LR_CONTEXT_RENDER_SIZE; |
2546 | else | 2473 | else |
2547 | ret = GEN8_LR_CONTEXT_RENDER_SIZE; | 2474 | ret = GEN8_LR_CONTEXT_RENDER_SIZE; |
@@ -2558,9 +2485,9 @@ uint32_t intel_lr_context_size(struct intel_engine_cs *engine) | |||
2558 | } | 2485 | } |
2559 | 2486 | ||
2560 | /** | 2487 | /** |
2561 | * intel_lr_context_deferred_alloc() - create the LRC specific bits of a context | 2488 | * execlists_context_deferred_alloc() - create the LRC specific bits of a context |
2562 | * @ctx: LR context to create. | 2489 | * @ctx: LR context to create. |
2563 | * @ring: engine to be used with the context. | 2490 | * @engine: engine to be used with the context. |
2564 | * | 2491 | * |
2565 | * This function can be called more than once, with different engines, if we plan | 2492 | * This function can be called more than once, with different engines, if we plan |
2566 | * to use the context with them. The context backing objects and the ringbuffers | 2493 | * to use the context with them. The context backing objects and the ringbuffers |
@@ -2570,31 +2497,29 @@ uint32_t intel_lr_context_size(struct intel_engine_cs *engine) | |||
2570 | * | 2497 | * |
2571 | * Return: non-zero on error. | 2498 | * Return: non-zero on error. |
2572 | */ | 2499 | */ |
2573 | 2500 | static int execlists_context_deferred_alloc(struct i915_gem_context *ctx, | |
2574 | int intel_lr_context_deferred_alloc(struct intel_context *ctx, | 2501 | struct intel_engine_cs *engine) |
2575 | struct intel_engine_cs *engine) | ||
2576 | { | 2502 | { |
2577 | struct drm_device *dev = engine->dev; | ||
2578 | struct drm_i915_gem_object *ctx_obj; | 2503 | struct drm_i915_gem_object *ctx_obj; |
2504 | struct intel_context *ce = &ctx->engine[engine->id]; | ||
2579 | uint32_t context_size; | 2505 | uint32_t context_size; |
2580 | struct intel_ringbuffer *ringbuf; | 2506 | struct intel_ringbuffer *ringbuf; |
2581 | int ret; | 2507 | int ret; |
2582 | 2508 | ||
2583 | WARN_ON(ctx->legacy_hw_ctx.rcs_state != NULL); | 2509 | WARN_ON(ce->state); |
2584 | WARN_ON(ctx->engine[engine->id].state); | ||
2585 | 2510 | ||
2586 | context_size = round_up(intel_lr_context_size(engine), 4096); | 2511 | context_size = round_up(intel_lr_context_size(engine), 4096); |
2587 | 2512 | ||
2588 | /* One extra page as the sharing data between driver and GuC */ | 2513 | /* One extra page as the sharing data between driver and GuC */ |
2589 | context_size += PAGE_SIZE * LRC_PPHWSP_PN; | 2514 | context_size += PAGE_SIZE * LRC_PPHWSP_PN; |
2590 | 2515 | ||
2591 | ctx_obj = i915_gem_alloc_object(dev, context_size); | 2516 | ctx_obj = i915_gem_object_create(&ctx->i915->drm, context_size); |
2592 | if (!ctx_obj) { | 2517 | if (IS_ERR(ctx_obj)) { |
2593 | DRM_DEBUG_DRIVER("Alloc LRC backing obj failed.\n"); | 2518 | DRM_DEBUG_DRIVER("Alloc LRC backing obj failed.\n"); |
2594 | return -ENOMEM; | 2519 | return PTR_ERR(ctx_obj); |
2595 | } | 2520 | } |
2596 | 2521 | ||
2597 | ringbuf = intel_engine_create_ringbuffer(engine, 4 * PAGE_SIZE); | 2522 | ringbuf = intel_engine_create_ringbuffer(engine, ctx->ring_size); |
2598 | if (IS_ERR(ringbuf)) { | 2523 | if (IS_ERR(ringbuf)) { |
2599 | ret = PTR_ERR(ringbuf); | 2524 | ret = PTR_ERR(ringbuf); |
2600 | goto error_deref_obj; | 2525 | goto error_deref_obj; |
@@ -2606,48 +2531,29 @@ int intel_lr_context_deferred_alloc(struct intel_context *ctx, | |||
2606 | goto error_ringbuf; | 2531 | goto error_ringbuf; |
2607 | } | 2532 | } |
2608 | 2533 | ||
2609 | ctx->engine[engine->id].ringbuf = ringbuf; | 2534 | ce->ringbuf = ringbuf; |
2610 | ctx->engine[engine->id].state = ctx_obj; | 2535 | ce->state = ctx_obj; |
2536 | ce->initialised = engine->init_context == NULL; | ||
2611 | 2537 | ||
2612 | if (ctx != ctx->i915->kernel_context && engine->init_context) { | ||
2613 | struct drm_i915_gem_request *req; | ||
2614 | |||
2615 | req = i915_gem_request_alloc(engine, ctx); | ||
2616 | if (IS_ERR(req)) { | ||
2617 | ret = PTR_ERR(req); | ||
2618 | DRM_ERROR("ring create req: %d\n", ret); | ||
2619 | goto error_ringbuf; | ||
2620 | } | ||
2621 | |||
2622 | ret = engine->init_context(req); | ||
2623 | i915_add_request_no_flush(req); | ||
2624 | if (ret) { | ||
2625 | DRM_ERROR("ring init context: %d\n", | ||
2626 | ret); | ||
2627 | goto error_ringbuf; | ||
2628 | } | ||
2629 | } | ||
2630 | return 0; | 2538 | return 0; |
2631 | 2539 | ||
2632 | error_ringbuf: | 2540 | error_ringbuf: |
2633 | intel_ringbuffer_free(ringbuf); | 2541 | intel_ringbuffer_free(ringbuf); |
2634 | error_deref_obj: | 2542 | error_deref_obj: |
2635 | drm_gem_object_unreference(&ctx_obj->base); | 2543 | drm_gem_object_unreference(&ctx_obj->base); |
2636 | ctx->engine[engine->id].ringbuf = NULL; | 2544 | ce->ringbuf = NULL; |
2637 | ctx->engine[engine->id].state = NULL; | 2545 | ce->state = NULL; |
2638 | return ret; | 2546 | return ret; |
2639 | } | 2547 | } |
2640 | 2548 | ||
2641 | void intel_lr_context_reset(struct drm_i915_private *dev_priv, | 2549 | void intel_lr_context_reset(struct drm_i915_private *dev_priv, |
2642 | struct intel_context *ctx) | 2550 | struct i915_gem_context *ctx) |
2643 | { | 2551 | { |
2644 | struct intel_engine_cs *engine; | 2552 | struct intel_engine_cs *engine; |
2645 | 2553 | ||
2646 | for_each_engine(engine, dev_priv) { | 2554 | for_each_engine(engine, dev_priv) { |
2647 | struct drm_i915_gem_object *ctx_obj = | 2555 | struct intel_context *ce = &ctx->engine[engine->id]; |
2648 | ctx->engine[engine->id].state; | 2556 | struct drm_i915_gem_object *ctx_obj = ce->state; |
2649 | struct intel_ringbuffer *ringbuf = | ||
2650 | ctx->engine[engine->id].ringbuf; | ||
2651 | void *vaddr; | 2557 | void *vaddr; |
2652 | uint32_t *reg_state; | 2558 | uint32_t *reg_state; |
2653 | 2559 | ||
@@ -2666,7 +2572,7 @@ void intel_lr_context_reset(struct drm_i915_private *dev_priv, | |||
2666 | 2572 | ||
2667 | i915_gem_object_unpin_map(ctx_obj); | 2573 | i915_gem_object_unpin_map(ctx_obj); |
2668 | 2574 | ||
2669 | ringbuf->head = 0; | 2575 | ce->ringbuf->head = 0; |
2670 | ringbuf->tail = 0; | 2576 | ce->ringbuf->tail = 0; |
2671 | } | 2577 | } |
2672 | } | 2578 | } |