summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2019-11-11 19:27:46 -0500
committerLinus Torvalds <torvalds@linux-foundation.org>2019-11-11 19:27:46 -0500
commit100d46bd72ec689a5582c2f5f4deadc5bcb92d60 (patch)
treeb16833998b423f438d2190fc83792e668f8ebbd1
parentde620fb99ef2bd52b2c5bc52656e89dcfc0e223a (diff)
parentea0b163b13ffc52818c079adb00d55e227a6da6f (diff)
Merge Intel Gen8/Gen9 graphics fixes from Jon Bloomfield.
This fixes two different classes of bugs in the Intel graphics hardware: MMIO register read hang: "On Intels Gen8 and Gen9 Graphics hardware, a read of specific graphics MMIO registers when the product is in certain low power states causes a system hang. There are two potential triggers for DoS: a) H/W corruption of the RC6 save/restore vector b) Hard hang within the MIPI hardware This prevents the DoS in two areas of the hardware: 1) Detect corruption of RC6 address on exit from low-power state, and if we find it corrupted, disable RC6 and RPM 2) Permanently lower the MIPI MMIO timeout" Blitter command streamer unrestricted memory accesses: "On Intels Gen9 Graphics hardware the Blitter Command Streamer (BCS) allows writing to Memory Mapped Input Output (MMIO) that should be blocked. With modifications of page tables, this can lead to privilege escalation. This exposure is limited to the Guest Physical Address space and does not allow for access outside of the graphics virtual machine. This series establishes a software parser into the Blitter command stream to scan for, and prevent, reads or writes to MMIO's that should not be accessible to non-privileged contexts. Much of the command parser infrastructure has existed for some time, and is used on Ivybridge/Haswell/Valleyview derived products to allow the use of features normally blocked by hardware. In this legacy context, the command parser is employed to allow normally unprivileged submissions to be run with elevated privileges in order to grant access to a limited set of extra capabilities. In this mode the parser is optional; In the event that the parser finds any construct that it cannot properly validate (e.g. nested command buffers), it simply aborts the scan and submits the buffer in non-privileged mode. For Gen9 Graphics, this series makes the parser mandatory for all Blitter submissions. The incoming user buffer is first copied to a kernel owned buffer, and parsed. If all checks are successful the kernel owned buffer is mapped READ-ONLY and submitted on behalf of the user. If any checks fail, or the parser is unable to complete the scan (nested buffers), it is forcibly rejected. The successfully scanned buffer is executed with NORMAL user privileges (key difference from legacy usage). Modern usermode does not use the Blitter on later hardware, having switched over to using the 3D engine instead for performance reasons. There are however some legacy usermode apps that rely on Blitter, notably the SNA X-Server. There are no known usermode applications that require nested command buffers on the Blitter, so the forcible rejection of such buffers in this patch series is considered an acceptable limitation" * Intel graphics fixes in emailed bundle from Jon Bloomfield <jon.bloomfield@intel.com>: drm/i915/cmdparser: Fix jump whitelist clearing drm/i915/gen8+: Add RC6 CTX corruption WA drm/i915: Lower RM timeout to avoid DSI hard hangs drm/i915/cmdparser: Ignore Length operands during command matching drm/i915/cmdparser: Add support for backward jumps drm/i915/cmdparser: Use explicit goto for error paths drm/i915: Add gen9 BCS cmdparsing drm/i915: Allow parsing of unsized batches drm/i915: Support ro ppgtt mapped cmdparser shadow buffers drm/i915: Add support for mandatory cmdparsing drm/i915: Remove Master tables from cmdparser drm/i915: Disable Secure Batches for gen6+ drm/i915: Rename gen7 cmdparser tables
Diffstat
-rw-r--r--drivers/gpu/drm/i915/gem/i915_gem_context.c5
-rw-r--r--drivers/gpu/drm/i915/gem/i915_gem_context_types.h7
-rw-r--r--drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c111
-rw-r--r--drivers/gpu/drm/i915/gt/intel_engine_types.h13
-rw-r--r--drivers/gpu/drm/i915/gt/intel_gt_pm.c8
-rw-r--r--drivers/gpu/drm/i915/i915_cmd_parser.c435
-rw-r--r--drivers/gpu/drm/i915/i915_drv.c4
-rw-r--r--drivers/gpu/drm/i915/i915_drv.h31
-rw-r--r--drivers/gpu/drm/i915/i915_gem.c16
-rw-r--r--drivers/gpu/drm/i915/i915_getparam.c2
-rw-r--r--drivers/gpu/drm/i915/i915_reg.h10
-rw-r--r--drivers/gpu/drm/i915/intel_pm.c122
-rw-r--r--drivers/gpu/drm/i915/intel_pm.h3
13 files changed, 595 insertions, 172 deletions
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_context.c b/drivers/gpu/drm/i915/gem/i915_gem_context.c
index 1cdfe05514c3..e41fd94ae5a9 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_context.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_context.c
@@ -319,6 +319,8 @@ static void i915_gem_context_free(struct i915_gem_context *ctx)
319 free_engines(rcu_access_pointer(ctx->engines)); 319 free_engines(rcu_access_pointer(ctx->engines));
320 mutex_destroy(&ctx->engines_mutex); 320 mutex_destroy(&ctx->engines_mutex);
321 321
322 kfree(ctx->jump_whitelist);
323
322 if (ctx->timeline) 324 if (ctx->timeline)
323 intel_timeline_put(ctx->timeline); 325 intel_timeline_put(ctx->timeline);
324 326
@@ -441,6 +443,9 @@ __create_context(struct drm_i915_private *i915)
441 for (i = 0; i < ARRAY_SIZE(ctx->hang_timestamp); i++) 443 for (i = 0; i < ARRAY_SIZE(ctx->hang_timestamp); i++)
442 ctx->hang_timestamp[i] = jiffies - CONTEXT_FAST_HANG_JIFFIES; 444 ctx->hang_timestamp[i] = jiffies - CONTEXT_FAST_HANG_JIFFIES;
443 445
446 ctx->jump_whitelist = NULL;
447 ctx->jump_whitelist_cmds = 0;
448
444 return ctx; 449 return ctx;
445 450
446err_free: 451err_free:
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_context_types.h b/drivers/gpu/drm/i915/gem/i915_gem_context_types.h
index 260d59cc3de8..00537b9d7006 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_context_types.h
+++ b/drivers/gpu/drm/i915/gem/i915_gem_context_types.h
@@ -192,6 +192,13 @@ struct i915_gem_context {
192 * per vm, which may be one per context or shared with the global GTT) 192 * per vm, which may be one per context or shared with the global GTT)
193 */ 193 */
194 struct radix_tree_root handles_vma; 194 struct radix_tree_root handles_vma;
195
196 /** jump_whitelist: Bit array for tracking cmds during cmdparsing
197 * Guarded by struct_mutex
198 */
199 unsigned long *jump_whitelist;
200 /** jump_whitelist_cmds: No of cmd slots available */
201 u32 jump_whitelist_cmds;
195}; 202};
196 203
197#endif /* __I915_GEM_CONTEXT_TYPES_H__ */ 204#endif /* __I915_GEM_CONTEXT_TYPES_H__ */
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c
index b5f6937369ea..e635e1e5f4d3 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c
@@ -296,7 +296,9 @@ static inline u64 gen8_noncanonical_addr(u64 address)
296 296
297static inline bool eb_use_cmdparser(const struct i915_execbuffer *eb) 297static inline bool eb_use_cmdparser(const struct i915_execbuffer *eb)
298{ 298{
299 return intel_engine_needs_cmd_parser(eb->engine) && eb->batch_len; 299 return intel_engine_requires_cmd_parser(eb->engine) ||
300 (intel_engine_using_cmd_parser(eb->engine) &&
301 eb->args->batch_len);
300} 302}
301 303
302static int eb_create(struct i915_execbuffer *eb) 304static int eb_create(struct i915_execbuffer *eb)
@@ -1955,40 +1957,94 @@ static int i915_reset_gen7_sol_offsets(struct i915_request *rq)
1955 return 0; 1957 return 0;
1956} 1958}
1957 1959
1958static struct i915_vma *eb_parse(struct i915_execbuffer *eb, bool is_master) 1960static struct i915_vma *
1961shadow_batch_pin(struct i915_execbuffer *eb, struct drm_i915_gem_object *obj)
1962{
1963 struct drm_i915_private *dev_priv = eb->i915;
1964 struct i915_vma * const vma = *eb->vma;
1965 struct i915_address_space *vm;
1966 u64 flags;
1967
1968 /*
1969 * PPGTT backed shadow buffers must be mapped RO, to prevent
1970 * post-scan tampering
1971 */
1972 if (CMDPARSER_USES_GGTT(dev_priv)) {
1973 flags = PIN_GLOBAL;
1974 vm = &dev_priv->ggtt.vm;
1975 } else if (vma->vm->has_read_only) {
1976 flags = PIN_USER;
1977 vm = vma->vm;
1978 i915_gem_object_set_readonly(obj);
1979 } else {
1980 DRM_DEBUG("Cannot prevent post-scan tampering without RO capable vm\n");
1981 return ERR_PTR(-EINVAL);
1982 }
1983
1984 return i915_gem_object_pin(obj, vm, NULL, 0, 0, flags);
1985}
1986
1987static struct i915_vma *eb_parse(struct i915_execbuffer *eb)
1959{ 1988{
1960 struct intel_engine_pool_node *pool; 1989 struct intel_engine_pool_node *pool;
1961 struct i915_vma *vma; 1990 struct i915_vma *vma;
1991 u64 batch_start;
1992 u64 shadow_batch_start;
1962 int err; 1993 int err;
1963 1994
1964 pool = intel_engine_pool_get(&eb->engine->pool, eb->batch_len); 1995 pool = intel_engine_pool_get(&eb->engine->pool, eb->batch_len);
1965 if (IS_ERR(pool)) 1996 if (IS_ERR(pool))
1966 return ERR_CAST(pool); 1997 return ERR_CAST(pool);
1967 1998
1968 err = intel_engine_cmd_parser(eb->engine, 1999 vma = shadow_batch_pin(eb, pool->obj);
2000 if (IS_ERR(vma))
2001 goto err;
2002
2003 batch_start = gen8_canonical_addr(eb->batch->node.start) +
2004 eb->batch_start_offset;
2005
2006 shadow_batch_start = gen8_canonical_addr(vma->node.start);
2007
2008 err = intel_engine_cmd_parser(eb->gem_context,
2009 eb->engine,
1969 eb->batch->obj, 2010 eb->batch->obj,
1970 pool->obj, 2011 batch_start,
1971 eb->batch_start_offset, 2012 eb->batch_start_offset,
1972 eb->batch_len, 2013 eb->batch_len,
1973 is_master); 2014 pool->obj,
2015 shadow_batch_start);
2016
1974 if (err) { 2017 if (err) {
1975 if (err == -EACCES) /* unhandled chained batch */ 2018 i915_vma_unpin(vma);
2019
2020 /*
2021 * Unsafe GGTT-backed buffers can still be submitted safely
2022 * as non-secure.
2023 * For PPGTT backing however, we have no choice but to forcibly
2024 * reject unsafe buffers
2025 */
2026 if (CMDPARSER_USES_GGTT(eb->i915) && (err == -EACCES))
2027 /* Execute original buffer non-secure */
1976 vma = NULL; 2028 vma = NULL;
1977 else 2029 else
1978 vma = ERR_PTR(err); 2030 vma = ERR_PTR(err);
1979 goto err; 2031 goto err;
1980 } 2032 }
1981 2033
1982 vma = i915_gem_object_ggtt_pin(pool->obj, NULL, 0, 0, 0);
1983 if (IS_ERR(vma))
1984 goto err;
1985
1986 eb->vma[eb->buffer_count] = i915_vma_get(vma); 2034 eb->vma[eb->buffer_count] = i915_vma_get(vma);
1987 eb->flags[eb->buffer_count] = 2035 eb->flags[eb->buffer_count] =
1988 __EXEC_OBJECT_HAS_PIN | __EXEC_OBJECT_HAS_REF; 2036 __EXEC_OBJECT_HAS_PIN | __EXEC_OBJECT_HAS_REF;
1989 vma->exec_flags = &eb->flags[eb->buffer_count]; 2037 vma->exec_flags = &eb->flags[eb->buffer_count];
1990 eb->buffer_count++; 2038 eb->buffer_count++;
1991 2039
2040 eb->batch_start_offset = 0;
2041 eb->batch = vma;
2042
2043 if (CMDPARSER_USES_GGTT(eb->i915))
2044 eb->batch_flags |= I915_DISPATCH_SECURE;
2045
2046 /* eb->batch_len unchanged */
2047
1992 vma->private = pool; 2048 vma->private = pool;
1993 return vma; 2049 return vma;
1994 2050
@@ -2421,6 +2477,7 @@ i915_gem_do_execbuffer(struct drm_device *dev,
2421 struct drm_i915_gem_exec_object2 *exec, 2477 struct drm_i915_gem_exec_object2 *exec,
2422 struct drm_syncobj **fences) 2478 struct drm_syncobj **fences)
2423{ 2479{
2480 struct drm_i915_private *i915 = to_i915(dev);
2424 struct i915_execbuffer eb; 2481 struct i915_execbuffer eb;
2425 struct dma_fence *in_fence = NULL; 2482 struct dma_fence *in_fence = NULL;
2426 struct dma_fence *exec_fence = NULL; 2483 struct dma_fence *exec_fence = NULL;
@@ -2432,7 +2489,7 @@ i915_gem_do_execbuffer(struct drm_device *dev,
2432 BUILD_BUG_ON(__EXEC_OBJECT_INTERNAL_FLAGS & 2489 BUILD_BUG_ON(__EXEC_OBJECT_INTERNAL_FLAGS &
2433 ~__EXEC_OBJECT_UNKNOWN_FLAGS); 2490 ~__EXEC_OBJECT_UNKNOWN_FLAGS);
2434 2491
2435 eb.i915 = to_i915(dev); 2492 eb.i915 = i915;
2436 eb.file = file; 2493 eb.file = file;
2437 eb.args = args; 2494 eb.args = args;
2438 if (DBG_FORCE_RELOC || !(args->flags & I915_EXEC_NO_RELOC)) 2495 if (DBG_FORCE_RELOC || !(args->flags & I915_EXEC_NO_RELOC))
@@ -2452,8 +2509,15 @@ i915_gem_do_execbuffer(struct drm_device *dev,
2452 2509
2453 eb.batch_flags = 0; 2510 eb.batch_flags = 0;
2454 if (args->flags & I915_EXEC_SECURE) { 2511 if (args->flags & I915_EXEC_SECURE) {
2512 if (INTEL_GEN(i915) >= 11)
2513 return -ENODEV;
2514
2515 /* Return -EPERM to trigger fallback code on old binaries. */
2516 if (!HAS_SECURE_BATCHES(i915))
2517 return -EPERM;
2518
2455 if (!drm_is_current_master(file) || !capable(CAP_SYS_ADMIN)) 2519 if (!drm_is_current_master(file) || !capable(CAP_SYS_ADMIN))
2456 return -EPERM; 2520 return -EPERM;
2457 2521
2458 eb.batch_flags |= I915_DISPATCH_SECURE; 2522 eb.batch_flags |= I915_DISPATCH_SECURE;
2459 } 2523 }
@@ -2530,34 +2594,19 @@ i915_gem_do_execbuffer(struct drm_device *dev,
2530 goto err_vma; 2594 goto err_vma;
2531 } 2595 }
2532 2596
2597 if (eb.batch_len == 0)
2598 eb.batch_len = eb.batch->size - eb.batch_start_offset;
2599
2533 if (eb_use_cmdparser(&eb)) { 2600 if (eb_use_cmdparser(&eb)) {
2534 struct i915_vma *vma; 2601 struct i915_vma *vma;
2535 2602
2536 vma = eb_parse(&eb, drm_is_current_master(file)); 2603 vma = eb_parse(&eb);
2537 if (IS_ERR(vma)) { 2604 if (IS_ERR(vma)) {
2538 err = PTR_ERR(vma); 2605 err = PTR_ERR(vma);
2539 goto err_vma; 2606 goto err_vma;
2540 } 2607 }
2541
2542 if (vma) {
2543 /*
2544 * Batch parsed and accepted:
2545 *
2546 * Set the DISPATCH_SECURE bit to remove the NON_SECURE
2547 * bit from MI_BATCH_BUFFER_START commands issued in
2548 * the dispatch_execbuffer implementations. We
2549 * specifically don't want that set on batches the
2550 * command parser has accepted.
2551 */
2552 eb.batch_flags |= I915_DISPATCH_SECURE;
2553 eb.batch_start_offset = 0;
2554 eb.batch = vma;
2555 }
2556 } 2608 }
2557 2609
2558 if (eb.batch_len == 0)
2559 eb.batch_len = eb.batch->size - eb.batch_start_offset;
2560
2561 /* 2610 /*
2562 * snb/ivb/vlv conflate the "batch in ppgtt" bit with the "non-secure 2611 * snb/ivb/vlv conflate the "batch in ppgtt" bit with the "non-secure
2563 * batch" bit. Hence we need to pin secure batches into the global gtt. 2612 * batch" bit. Hence we need to pin secure batches into the global gtt.
diff --git a/drivers/gpu/drm/i915/gt/intel_engine_types.h b/drivers/gpu/drm/i915/gt/intel_engine_types.h
index a82cea95c2f2..9dd8c299cb2d 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_types.h
+++ b/drivers/gpu/drm/i915/gt/intel_engine_types.h
@@ -475,12 +475,13 @@ struct intel_engine_cs {
475 475
476 struct intel_engine_hangcheck hangcheck; 476 struct intel_engine_hangcheck hangcheck;
477 477
478#define I915_ENGINE_NEEDS_CMD_PARSER BIT(0) 478#define I915_ENGINE_USING_CMD_PARSER BIT(0)
479#define I915_ENGINE_SUPPORTS_STATS BIT(1) 479#define I915_ENGINE_SUPPORTS_STATS BIT(1)
480#define I915_ENGINE_HAS_PREEMPTION BIT(2) 480#define I915_ENGINE_HAS_PREEMPTION BIT(2)
481#define I915_ENGINE_HAS_SEMAPHORES BIT(3) 481#define I915_ENGINE_HAS_SEMAPHORES BIT(3)
482#define I915_ENGINE_NEEDS_BREADCRUMB_TASKLET BIT(4) 482#define I915_ENGINE_NEEDS_BREADCRUMB_TASKLET BIT(4)
483#define I915_ENGINE_IS_VIRTUAL BIT(5) 483#define I915_ENGINE_IS_VIRTUAL BIT(5)
484#define I915_ENGINE_REQUIRES_CMD_PARSER BIT(7)
484 unsigned int flags; 485 unsigned int flags;
485 486
486 /* 487 /*
@@ -541,9 +542,15 @@ struct intel_engine_cs {
541}; 542};
542 543
543static inline bool 544static inline bool
544intel_engine_needs_cmd_parser(const struct intel_engine_cs *engine) 545intel_engine_using_cmd_parser(const struct intel_engine_cs *engine)
545{ 546{
546 return engine->flags & I915_ENGINE_NEEDS_CMD_PARSER; 547 return engine->flags & I915_ENGINE_USING_CMD_PARSER;
548}
549
550static inline bool
551intel_engine_requires_cmd_parser(const struct intel_engine_cs *engine)
552{
553 return engine->flags & I915_ENGINE_REQUIRES_CMD_PARSER;
547} 554}
548 555
549static inline bool 556static inline bool
diff --git a/drivers/gpu/drm/i915/gt/intel_gt_pm.c b/drivers/gpu/drm/i915/gt/intel_gt_pm.c
index 1363e069ec83..fac75afed35b 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt_pm.c
+++ b/drivers/gpu/drm/i915/gt/intel_gt_pm.c
@@ -38,6 +38,9 @@ static int __gt_unpark(struct intel_wakeref *wf)
38 gt->awake = intel_display_power_get(i915, POWER_DOMAIN_GT_IRQ); 38 gt->awake = intel_display_power_get(i915, POWER_DOMAIN_GT_IRQ);
39 GEM_BUG_ON(!gt->awake); 39 GEM_BUG_ON(!gt->awake);
40 40
41 if (NEEDS_RC6_CTX_CORRUPTION_WA(i915))
42 intel_uncore_forcewake_get(&i915->uncore, FORCEWAKE_ALL);
43
41 intel_enable_gt_powersave(i915); 44 intel_enable_gt_powersave(i915);
42 45
43 i915_update_gfx_val(i915); 46 i915_update_gfx_val(i915);
@@ -67,6 +70,11 @@ static int __gt_park(struct intel_wakeref *wf)
67 if (INTEL_GEN(i915) >= 6) 70 if (INTEL_GEN(i915) >= 6)
68 gen6_rps_idle(i915); 71 gen6_rps_idle(i915);
69 72
73 if (NEEDS_RC6_CTX_CORRUPTION_WA(i915)) {
74 i915_rc6_ctx_wa_check(i915);
75 intel_uncore_forcewake_put(&i915->uncore, FORCEWAKE_ALL);
76 }
77
70 /* Everything switched off, flush any residual interrupt just in case */ 78 /* Everything switched off, flush any residual interrupt just in case */
71 intel_synchronize_irq(i915); 79 intel_synchronize_irq(i915);
72 80
diff --git a/drivers/gpu/drm/i915/i915_cmd_parser.c b/drivers/gpu/drm/i915/i915_cmd_parser.c
index 24555102e198..f24096e27bef 100644
--- a/drivers/gpu/drm/i915/i915_cmd_parser.c
+++ b/drivers/gpu/drm/i915/i915_cmd_parser.c
@@ -53,13 +53,11 @@
53 * granting userspace undue privileges. There are three categories of privilege. 53 * granting userspace undue privileges. There are three categories of privilege.
54 * 54 *
55 * First, commands which are explicitly defined as privileged or which should 55 * First, commands which are explicitly defined as privileged or which should
56 * only be used by the kernel driver. The parser generally rejects such 56 * only be used by the kernel driver. The parser rejects such commands
57 * commands, though it may allow some from the drm master process.
58 * 57 *
59 * Second, commands which access registers. To support correct/enhanced 58 * Second, commands which access registers. To support correct/enhanced
60 * userspace functionality, particularly certain OpenGL extensions, the parser 59 * userspace functionality, particularly certain OpenGL extensions, the parser
61 * provides a whitelist of registers which userspace may safely access (for both 60 * provides a whitelist of registers which userspace may safely access
62 * normal and drm master processes).
63 * 61 *
64 * Third, commands which access privileged memory (i.e. GGTT, HWS page, etc). 62 * Third, commands which access privileged memory (i.e. GGTT, HWS page, etc).
65 * The parser always rejects such commands. 63 * The parser always rejects such commands.
@@ -84,9 +82,9 @@
84 * in the per-engine command tables. 82 * in the per-engine command tables.
85 * 83 *
86 * Other command table entries map fairly directly to high level categories 84 * Other command table entries map fairly directly to high level categories
87 * mentioned above: rejected, master-only, register whitelist. The parser 85 * mentioned above: rejected, register whitelist. The parser implements a number
88 * implements a number of checks, including the privileged memory checks, via a 86 * of checks, including the privileged memory checks, via a general bitmasking
89 * general bitmasking mechanism. 87 * mechanism.
90 */ 88 */
91 89
92/* 90/*
@@ -104,8 +102,6 @@ struct drm_i915_cmd_descriptor {
104 * CMD_DESC_REJECT: The command is never allowed 102 * CMD_DESC_REJECT: The command is never allowed
105 * CMD_DESC_REGISTER: The command should be checked against the 103 * CMD_DESC_REGISTER: The command should be checked against the
106 * register whitelist for the appropriate ring 104 * register whitelist for the appropriate ring
107 * CMD_DESC_MASTER: The command is allowed if the submitting process
108 * is the DRM master
109 */ 105 */
110 u32 flags; 106 u32 flags;
111#define CMD_DESC_FIXED (1<<0) 107#define CMD_DESC_FIXED (1<<0)
@@ -113,7 +109,6 @@ struct drm_i915_cmd_descriptor {
113#define CMD_DESC_REJECT (1<<2) 109#define CMD_DESC_REJECT (1<<2)
114#define CMD_DESC_REGISTER (1<<3) 110#define CMD_DESC_REGISTER (1<<3)
115#define CMD_DESC_BITMASK (1<<4) 111#define CMD_DESC_BITMASK (1<<4)
116#define CMD_DESC_MASTER (1<<5)
117 112
118 /* 113 /*
119 * The command's unique identification bits and the bitmask to get them. 114 * The command's unique identification bits and the bitmask to get them.
@@ -194,7 +189,7 @@ struct drm_i915_cmd_table {
194#define CMD(op, opm, f, lm, fl, ...) \ 189#define CMD(op, opm, f, lm, fl, ...) \
195 { \ 190 { \
196 .flags = (fl) | ((f) ? CMD_DESC_FIXED : 0), \ 191 .flags = (fl) | ((f) ? CMD_DESC_FIXED : 0), \
197 .cmd = { (op), ~0u << (opm) }, \ 192 .cmd = { (op & ~0u << (opm)), ~0u << (opm) }, \
198 .length = { (lm) }, \ 193 .length = { (lm) }, \
199 __VA_ARGS__ \ 194 __VA_ARGS__ \
200 } 195 }
@@ -209,14 +204,13 @@ struct drm_i915_cmd_table {
209#define R CMD_DESC_REJECT 204#define R CMD_DESC_REJECT
210#define W CMD_DESC_REGISTER 205#define W CMD_DESC_REGISTER
211#define B CMD_DESC_BITMASK 206#define B CMD_DESC_BITMASK
212#define M CMD_DESC_MASTER
213 207
214/* Command Mask Fixed Len Action 208/* Command Mask Fixed Len Action
215 ---------------------------------------------------------- */ 209 ---------------------------------------------------------- */
216static const struct drm_i915_cmd_descriptor common_cmds[] = { 210static const struct drm_i915_cmd_descriptor gen7_common_cmds[] = {
217 CMD( MI_NOOP, SMI, F, 1, S ), 211 CMD( MI_NOOP, SMI, F, 1, S ),
218 CMD( MI_USER_INTERRUPT, SMI, F, 1, R ), 212 CMD( MI_USER_INTERRUPT, SMI, F, 1, R ),
219 CMD( MI_WAIT_FOR_EVENT, SMI, F, 1, M ), 213 CMD( MI_WAIT_FOR_EVENT, SMI, F, 1, R ),
220 CMD( MI_ARB_CHECK, SMI, F, 1, S ), 214 CMD( MI_ARB_CHECK, SMI, F, 1, S ),
221 CMD( MI_REPORT_HEAD, SMI, F, 1, S ), 215 CMD( MI_REPORT_HEAD, SMI, F, 1, S ),
222 CMD( MI_SUSPEND_FLUSH, SMI, F, 1, S ), 216 CMD( MI_SUSPEND_FLUSH, SMI, F, 1, S ),
@@ -246,7 +240,7 @@ static const struct drm_i915_cmd_descriptor common_cmds[] = {
246 CMD( MI_BATCH_BUFFER_START, SMI, !F, 0xFF, S ), 240 CMD( MI_BATCH_BUFFER_START, SMI, !F, 0xFF, S ),
247}; 241};
248 242
249static const struct drm_i915_cmd_descriptor render_cmds[] = { 243static const struct drm_i915_cmd_descriptor gen7_render_cmds[] = {
250 CMD( MI_FLUSH, SMI, F, 1, S ), 244 CMD( MI_FLUSH, SMI, F, 1, S ),
251 CMD( MI_ARB_ON_OFF, SMI, F, 1, R ), 245 CMD( MI_ARB_ON_OFF, SMI, F, 1, R ),
252 CMD( MI_PREDICATE, SMI, F, 1, S ), 246 CMD( MI_PREDICATE, SMI, F, 1, S ),
@@ -313,7 +307,7 @@ static const struct drm_i915_cmd_descriptor hsw_render_cmds[] = {
313 CMD( MI_URB_ATOMIC_ALLOC, SMI, F, 1, S ), 307 CMD( MI_URB_ATOMIC_ALLOC, SMI, F, 1, S ),
314 CMD( MI_SET_APPID, SMI, F, 1, S ), 308 CMD( MI_SET_APPID, SMI, F, 1, S ),
315 CMD( MI_RS_CONTEXT, SMI, F, 1, S ), 309 CMD( MI_RS_CONTEXT, SMI, F, 1, S ),
316 CMD( MI_LOAD_SCAN_LINES_INCL, SMI, !F, 0x3F, M ), 310 CMD( MI_LOAD_SCAN_LINES_INCL, SMI, !F, 0x3F, R ),
317 CMD( MI_LOAD_SCAN_LINES_EXCL, SMI, !F, 0x3F, R ), 311 CMD( MI_LOAD_SCAN_LINES_EXCL, SMI, !F, 0x3F, R ),
318 CMD( MI_LOAD_REGISTER_REG, SMI, !F, 0xFF, W, 312 CMD( MI_LOAD_REGISTER_REG, SMI, !F, 0xFF, W,
319 .reg = { .offset = 1, .mask = 0x007FFFFC, .step = 1 } ), 313 .reg = { .offset = 1, .mask = 0x007FFFFC, .step = 1 } ),
@@ -330,7 +324,7 @@ static const struct drm_i915_cmd_descriptor hsw_render_cmds[] = {
330 CMD( GFX_OP_3DSTATE_BINDING_TABLE_EDIT_PS, S3D, !F, 0x1FF, S ), 324 CMD( GFX_OP_3DSTATE_BINDING_TABLE_EDIT_PS, S3D, !F, 0x1FF, S ),
331}; 325};
332 326
333static const struct drm_i915_cmd_descriptor video_cmds[] = { 327static const struct drm_i915_cmd_descriptor gen7_video_cmds[] = {
334 CMD( MI_ARB_ON_OFF, SMI, F, 1, R ), 328 CMD( MI_ARB_ON_OFF, SMI, F, 1, R ),
335 CMD( MI_SET_APPID, SMI, F, 1, S ), 329 CMD( MI_SET_APPID, SMI, F, 1, S ),
336 CMD( MI_STORE_DWORD_IMM, SMI, !F, 0xFF, B, 330 CMD( MI_STORE_DWORD_IMM, SMI, !F, 0xFF, B,
@@ -374,7 +368,7 @@ static const struct drm_i915_cmd_descriptor video_cmds[] = {
374 CMD( MFX_WAIT, SMFX, F, 1, S ), 368 CMD( MFX_WAIT, SMFX, F, 1, S ),
375}; 369};
376 370
377static const struct drm_i915_cmd_descriptor vecs_cmds[] = { 371static const struct drm_i915_cmd_descriptor gen7_vecs_cmds[] = {
378 CMD( MI_ARB_ON_OFF, SMI, F, 1, R ), 372 CMD( MI_ARB_ON_OFF, SMI, F, 1, R ),
379 CMD( MI_SET_APPID, SMI, F, 1, S ), 373 CMD( MI_SET_APPID, SMI, F, 1, S ),
380 CMD( MI_STORE_DWORD_IMM, SMI, !F, 0xFF, B, 374 CMD( MI_STORE_DWORD_IMM, SMI, !F, 0xFF, B,
@@ -412,7 +406,7 @@ static const struct drm_i915_cmd_descriptor vecs_cmds[] = {
412 }}, ), 406 }}, ),
413}; 407};
414 408
415static const struct drm_i915_cmd_descriptor blt_cmds[] = { 409static const struct drm_i915_cmd_descriptor gen7_blt_cmds[] = {
416 CMD( MI_DISPLAY_FLIP, SMI, !F, 0xFF, R ), 410 CMD( MI_DISPLAY_FLIP, SMI, !F, 0xFF, R ),
417 CMD( MI_STORE_DWORD_IMM, SMI, !F, 0x3FF, B, 411 CMD( MI_STORE_DWORD_IMM, SMI, !F, 0x3FF, B,
418 .bits = {{ 412 .bits = {{
@@ -446,10 +440,64 @@ static const struct drm_i915_cmd_descriptor blt_cmds[] = {
446}; 440};
447 441
448static const struct drm_i915_cmd_descriptor hsw_blt_cmds[] = { 442static const struct drm_i915_cmd_descriptor hsw_blt_cmds[] = {
449 CMD( MI_LOAD_SCAN_LINES_INCL, SMI, !F, 0x3F, M ), 443 CMD( MI_LOAD_SCAN_LINES_INCL, SMI, !F, 0x3F, R ),
450 CMD( MI_LOAD_SCAN_LINES_EXCL, SMI, !F, 0x3F, R ), 444 CMD( MI_LOAD_SCAN_LINES_EXCL, SMI, !F, 0x3F, R ),
451}; 445};
452 446
447/*
448 * For Gen9 we can still rely on the h/w to enforce cmd security, and only
449 * need to re-enforce the register access checks. We therefore only need to
450 * teach the cmdparser how to find the end of each command, and identify
451 * register accesses. The table doesn't need to reject any commands, and so
452 * the only commands listed here are:
453 * 1) Those that touch registers
454 * 2) Those that do not have the default 8-bit length
455 *
456 * Note that the default MI length mask chosen for this table is 0xFF, not
457 * the 0x3F used on older devices. This is because the vast majority of MI
458 * cmds on Gen9 use a standard 8-bit Length field.
459 * All the Gen9 blitter instructions are standard 0xFF length mask, and
460 * none allow access to non-general registers, so in fact no BLT cmds are
461 * included in the table at all.
462 *
463 */
464static const struct drm_i915_cmd_descriptor gen9_blt_cmds[] = {
465 CMD( MI_NOOP, SMI, F, 1, S ),
466 CMD( MI_USER_INTERRUPT, SMI, F, 1, S ),
467 CMD( MI_WAIT_FOR_EVENT, SMI, F, 1, S ),
468 CMD( MI_FLUSH, SMI, F, 1, S ),
469 CMD( MI_ARB_CHECK, SMI, F, 1, S ),
470 CMD( MI_REPORT_HEAD, SMI, F, 1, S ),
471 CMD( MI_ARB_ON_OFF, SMI, F, 1, S ),
472 CMD( MI_SUSPEND_FLUSH, SMI, F, 1, S ),
473 CMD( MI_LOAD_SCAN_LINES_INCL, SMI, !F, 0x3F, S ),
474 CMD( MI_LOAD_SCAN_LINES_EXCL, SMI, !F, 0x3F, S ),
475 CMD( MI_STORE_DWORD_IMM, SMI, !F, 0x3FF, S ),
476 CMD( MI_LOAD_REGISTER_IMM(1), SMI, !F, 0xFF, W,
477 .reg = { .offset = 1, .mask = 0x007FFFFC, .step = 2 } ),
478 CMD( MI_UPDATE_GTT, SMI, !F, 0x3FF, S ),
479 CMD( MI_STORE_REGISTER_MEM_GEN8, SMI, F, 4, W,
480 .reg = { .offset = 1, .mask = 0x007FFFFC } ),
481 CMD( MI_FLUSH_DW, SMI, !F, 0x3F, S ),
482 CMD( MI_LOAD_REGISTER_MEM_GEN8, SMI, F, 4, W,
483 .reg = { .offset = 1, .mask = 0x007FFFFC } ),
484 CMD( MI_LOAD_REGISTER_REG, SMI, !F, 0xFF, W,
485 .reg = { .offset = 1, .mask = 0x007FFFFC, .step = 1 } ),
486
487 /*
488 * We allow BB_START but apply further checks. We just sanitize the
489 * basic fields here.
490 */
491#define MI_BB_START_OPERAND_MASK GENMASK(SMI-1, 0)
492#define MI_BB_START_OPERAND_EXPECT (MI_BATCH_PPGTT_HSW | 1)
493 CMD( MI_BATCH_BUFFER_START_GEN8, SMI, !F, 0xFF, B,
494 .bits = {{
495 .offset = 0,
496 .mask = MI_BB_START_OPERAND_MASK,
497 .expected = MI_BB_START_OPERAND_EXPECT,
498 }}, ),
499};
500
453static const struct drm_i915_cmd_descriptor noop_desc = 501static const struct drm_i915_cmd_descriptor noop_desc =
454 CMD(MI_NOOP, SMI, F, 1, S); 502 CMD(MI_NOOP, SMI, F, 1, S);
455 503
@@ -463,40 +511,44 @@ static const struct drm_i915_cmd_descriptor noop_desc =
463#undef R 511#undef R
464#undef W 512#undef W
465#undef B 513#undef B
466#undef M
467 514
468static const struct drm_i915_cmd_table gen7_render_cmds[] = { 515static const struct drm_i915_cmd_table gen7_render_cmd_table[] = {
469 { common_cmds, ARRAY_SIZE(common_cmds) }, 516 { gen7_common_cmds, ARRAY_SIZE(gen7_common_cmds) },
470 { render_cmds, ARRAY_SIZE(render_cmds) }, 517 { gen7_render_cmds, ARRAY_SIZE(gen7_render_cmds) },
471}; 518};
472 519
473static const struct drm_i915_cmd_table hsw_render_ring_cmds[] = { 520static const struct drm_i915_cmd_table hsw_render_ring_cmd_table[] = {
474 { common_cmds, ARRAY_SIZE(common_cmds) }, 521 { gen7_common_cmds, ARRAY_SIZE(gen7_common_cmds) },
475 { render_cmds, ARRAY_SIZE(render_cmds) }, 522 { gen7_render_cmds, ARRAY_SIZE(gen7_render_cmds) },
476 { hsw_render_cmds, ARRAY_SIZE(hsw_render_cmds) }, 523 { hsw_render_cmds, ARRAY_SIZE(hsw_render_cmds) },
477}; 524};
478 525
479static const struct drm_i915_cmd_table gen7_video_cmds[] = { 526static const struct drm_i915_cmd_table gen7_video_cmd_table[] = {
480 { common_cmds, ARRAY_SIZE(common_cmds) }, 527 { gen7_common_cmds, ARRAY_SIZE(gen7_common_cmds) },
481 { video_cmds, ARRAY_SIZE(video_cmds) }, 528 { gen7_video_cmds, ARRAY_SIZE(gen7_video_cmds) },
482}; 529};
483 530
484static const struct drm_i915_cmd_table hsw_vebox_cmds[] = { 531static const struct drm_i915_cmd_table hsw_vebox_cmd_table[] = {
485 { common_cmds, ARRAY_SIZE(common_cmds) }, 532 { gen7_common_cmds, ARRAY_SIZE(gen7_common_cmds) },
486 { vecs_cmds, ARRAY_SIZE(vecs_cmds) }, 533 { gen7_vecs_cmds, ARRAY_SIZE(gen7_vecs_cmds) },
487}; 534};
488 535
489static const struct drm_i915_cmd_table gen7_blt_cmds[] = { 536static const struct drm_i915_cmd_table gen7_blt_cmd_table[] = {
490 { common_cmds, ARRAY_SIZE(common_cmds) }, 537 { gen7_common_cmds, ARRAY_SIZE(gen7_common_cmds) },
491 { blt_cmds, ARRAY_SIZE(blt_cmds) }, 538 { gen7_blt_cmds, ARRAY_SIZE(gen7_blt_cmds) },
492}; 539};
493 540
494static const struct drm_i915_cmd_table hsw_blt_ring_cmds[] = { 541static const struct drm_i915_cmd_table hsw_blt_ring_cmd_table[] = {
495 { common_cmds, ARRAY_SIZE(common_cmds) }, 542 { gen7_common_cmds, ARRAY_SIZE(gen7_common_cmds) },
496 { blt_cmds, ARRAY_SIZE(blt_cmds) }, 543 { gen7_blt_cmds, ARRAY_SIZE(gen7_blt_cmds) },
497 { hsw_blt_cmds, ARRAY_SIZE(hsw_blt_cmds) }, 544 { hsw_blt_cmds, ARRAY_SIZE(hsw_blt_cmds) },
498}; 545};
499 546
547static const struct drm_i915_cmd_table gen9_blt_cmd_table[] = {
548 { gen9_blt_cmds, ARRAY_SIZE(gen9_blt_cmds) },
549};
550
551
500/* 552/*
501 * Register whitelists, sorted by increasing register offset. 553 * Register whitelists, sorted by increasing register offset.
502 */ 554 */
@@ -612,17 +664,27 @@ static const struct drm_i915_reg_descriptor gen7_blt_regs[] = {
612 REG64_IDX(RING_TIMESTAMP, BLT_RING_BASE), 664 REG64_IDX(RING_TIMESTAMP, BLT_RING_BASE),
613}; 665};
614 666
615static const struct drm_i915_reg_descriptor ivb_master_regs[] = { 667static const struct drm_i915_reg_descriptor gen9_blt_regs[] = {
616 REG32(FORCEWAKE_MT), 668 REG64_IDX(RING_TIMESTAMP, RENDER_RING_BASE),
617 REG32(DERRMR), 669 REG64_IDX(RING_TIMESTAMP, BSD_RING_BASE),
618 REG32(GEN7_PIPE_DE_LOAD_SL(PIPE_A)), 670 REG32(BCS_SWCTRL),
619 REG32(GEN7_PIPE_DE_LOAD_SL(PIPE_B)), 671 REG64_IDX(RING_TIMESTAMP, BLT_RING_BASE),
620 REG32(GEN7_PIPE_DE_LOAD_SL(PIPE_C)), 672 REG64_IDX(BCS_GPR, 0),
621}; 673 REG64_IDX(BCS_GPR, 1),
622 674 REG64_IDX(BCS_GPR, 2),
623static const struct drm_i915_reg_descriptor hsw_master_regs[] = { 675 REG64_IDX(BCS_GPR, 3),
624 REG32(FORCEWAKE_MT), 676 REG64_IDX(BCS_GPR, 4),
625 REG32(DERRMR), 677 REG64_IDX(BCS_GPR, 5),
678 REG64_IDX(BCS_GPR, 6),
679 REG64_IDX(BCS_GPR, 7),
680 REG64_IDX(BCS_GPR, 8),
681 REG64_IDX(BCS_GPR, 9),
682 REG64_IDX(BCS_GPR, 10),
683 REG64_IDX(BCS_GPR, 11),
684 REG64_IDX(BCS_GPR, 12),
685 REG64_IDX(BCS_GPR, 13),
686 REG64_IDX(BCS_GPR, 14),
687 REG64_IDX(BCS_GPR, 15),
626}; 688};
627 689
628#undef REG64 690#undef REG64
@@ -631,28 +693,27 @@ static const struct drm_i915_reg_descriptor hsw_master_regs[] = {
631struct drm_i915_reg_table { 693struct drm_i915_reg_table {
632 const struct drm_i915_reg_descriptor *regs; 694 const struct drm_i915_reg_descriptor *regs;
633 int num_regs; 695 int num_regs;
634 bool master;
635}; 696};
636 697
637static const struct drm_i915_reg_table ivb_render_reg_tables[] = { 698static const struct drm_i915_reg_table ivb_render_reg_tables[] = {
638 { gen7_render_regs, ARRAY_SIZE(gen7_render_regs), false }, 699 { gen7_render_regs, ARRAY_SIZE(gen7_render_regs) },
639 { ivb_master_regs, ARRAY_SIZE(ivb_master_regs), true },
640}; 700};
641 701
642static const struct drm_i915_reg_table ivb_blt_reg_tables[] = { 702static const struct drm_i915_reg_table ivb_blt_reg_tables[] = {
643 { gen7_blt_regs, ARRAY_SIZE(gen7_blt_regs), false }, 703 { gen7_blt_regs, ARRAY_SIZE(gen7_blt_regs) },
644 { ivb_master_regs, ARRAY_SIZE(ivb_master_regs), true },
645}; 704};
646 705
647static const struct drm_i915_reg_table hsw_render_reg_tables[] = { 706static const struct drm_i915_reg_table hsw_render_reg_tables[] = {
648 { gen7_render_regs, ARRAY_SIZE(gen7_render_regs), false }, 707 { gen7_render_regs, ARRAY_SIZE(gen7_render_regs) },
649 { hsw_render_regs, ARRAY_SIZE(hsw_render_regs), false }, 708 { hsw_render_regs, ARRAY_SIZE(hsw_render_regs) },
650 { hsw_master_regs, ARRAY_SIZE(hsw_master_regs), true },
651}; 709};
652 710
653static const struct drm_i915_reg_table hsw_blt_reg_tables[] = { 711static const struct drm_i915_reg_table hsw_blt_reg_tables[] = {
654 { gen7_blt_regs, ARRAY_SIZE(gen7_blt_regs), false }, 712 { gen7_blt_regs, ARRAY_SIZE(gen7_blt_regs) },
655 { hsw_master_regs, ARRAY_SIZE(hsw_master_regs), true }, 713};
714
715static const struct drm_i915_reg_table gen9_blt_reg_tables[] = {
716 { gen9_blt_regs, ARRAY_SIZE(gen9_blt_regs) },
656}; 717};
657 718
658static u32 gen7_render_get_cmd_length_mask(u32 cmd_header) 719static u32 gen7_render_get_cmd_length_mask(u32 cmd_header)
@@ -710,6 +771,17 @@ static u32 gen7_blt_get_cmd_length_mask(u32 cmd_header)
710 return 0; 771 return 0;
711} 772}
712 773
774static u32 gen9_blt_get_cmd_length_mask(u32 cmd_header)
775{
776 u32 client = cmd_header >> INSTR_CLIENT_SHIFT;
777
778 if (client == INSTR_MI_CLIENT || client == INSTR_BC_CLIENT)
779 return 0xFF;
780
781 DRM_DEBUG_DRIVER("CMD: Abnormal blt cmd length! 0x%08X\n", cmd_header);
782 return 0;
783}
784
713static bool validate_cmds_sorted(const struct intel_engine_cs *engine, 785static bool validate_cmds_sorted(const struct intel_engine_cs *engine,
714 const struct drm_i915_cmd_table *cmd_tables, 786 const struct drm_i915_cmd_table *cmd_tables,
715 int cmd_table_count) 787 int cmd_table_count)
@@ -867,18 +939,19 @@ void intel_engine_init_cmd_parser(struct intel_engine_cs *engine)
867 int cmd_table_count; 939 int cmd_table_count;
868 int ret; 940 int ret;
869 941
870 if (!IS_GEN(engine->i915, 7)) 942 if (!IS_GEN(engine->i915, 7) && !(IS_GEN(engine->i915, 9) &&
943 engine->class == COPY_ENGINE_CLASS))
871 return; 944 return;
872 945
873 switch (engine->class) { 946 switch (engine->class) {
874 case RENDER_CLASS: 947 case RENDER_CLASS:
875 if (IS_HASWELL(engine->i915)) { 948 if (IS_HASWELL(engine->i915)) {
876 cmd_tables = hsw_render_ring_cmds; 949 cmd_tables = hsw_render_ring_cmd_table;
877 cmd_table_count = 950 cmd_table_count =
878 ARRAY_SIZE(hsw_render_ring_cmds); 951 ARRAY_SIZE(hsw_render_ring_cmd_table);
879 } else { 952 } else {
880 cmd_tables = gen7_render_cmds; 953 cmd_tables = gen7_render_cmd_table;
881 cmd_table_count = ARRAY_SIZE(gen7_render_cmds); 954 cmd_table_count = ARRAY_SIZE(gen7_render_cmd_table);
882 } 955 }
883 956
884 if (IS_HASWELL(engine->i915)) { 957 if (IS_HASWELL(engine->i915)) {
@@ -888,36 +961,46 @@ void intel_engine_init_cmd_parser(struct intel_engine_cs *engine)
888 engine->reg_tables = ivb_render_reg_tables; 961 engine->reg_tables = ivb_render_reg_tables;
889 engine->reg_table_count = ARRAY_SIZE(ivb_render_reg_tables); 962 engine->reg_table_count = ARRAY_SIZE(ivb_render_reg_tables);
890 } 963 }
891
892 engine->get_cmd_length_mask = gen7_render_get_cmd_length_mask; 964 engine->get_cmd_length_mask = gen7_render_get_cmd_length_mask;
893 break; 965 break;
894 case VIDEO_DECODE_CLASS: 966 case VIDEO_DECODE_CLASS:
895 cmd_tables = gen7_video_cmds; 967 cmd_tables = gen7_video_cmd_table;
896 cmd_table_count = ARRAY_SIZE(gen7_video_cmds); 968 cmd_table_count = ARRAY_SIZE(gen7_video_cmd_table);
897 engine->get_cmd_length_mask = gen7_bsd_get_cmd_length_mask; 969 engine->get_cmd_length_mask = gen7_bsd_get_cmd_length_mask;
898 break; 970 break;
899 case COPY_ENGINE_CLASS: 971 case COPY_ENGINE_CLASS:
900 if (IS_HASWELL(engine->i915)) { 972 engine->get_cmd_length_mask = gen7_blt_get_cmd_length_mask;
901 cmd_tables = hsw_blt_ring_cmds; 973 if (IS_GEN(engine->i915, 9)) {
902 cmd_table_count = ARRAY_SIZE(hsw_blt_ring_cmds); 974 cmd_tables = gen9_blt_cmd_table;
975 cmd_table_count = ARRAY_SIZE(gen9_blt_cmd_table);
976 engine->get_cmd_length_mask =
977 gen9_blt_get_cmd_length_mask;
978
979 /* BCS Engine unsafe without parser */
980 engine->flags |= I915_ENGINE_REQUIRES_CMD_PARSER;
981 } else if (IS_HASWELL(engine->i915)) {
982 cmd_tables = hsw_blt_ring_cmd_table;
983 cmd_table_count = ARRAY_SIZE(hsw_blt_ring_cmd_table);
903 } else { 984 } else {
904 cmd_tables = gen7_blt_cmds; 985 cmd_tables = gen7_blt_cmd_table;
905 cmd_table_count = ARRAY_SIZE(gen7_blt_cmds); 986 cmd_table_count = ARRAY_SIZE(gen7_blt_cmd_table);
906 } 987 }
907 988
908 if (IS_HASWELL(engine->i915)) { 989 if (IS_GEN(engine->i915, 9)) {
990 engine->reg_tables = gen9_blt_reg_tables;
991 engine->reg_table_count =
992 ARRAY_SIZE(gen9_blt_reg_tables);
993 } else if (IS_HASWELL(engine->i915)) {
909 engine->reg_tables = hsw_blt_reg_tables; 994 engine->reg_tables = hsw_blt_reg_tables;
910 engine->reg_table_count = ARRAY_SIZE(hsw_blt_reg_tables); 995 engine->reg_table_count = ARRAY_SIZE(hsw_blt_reg_tables);
911 } else { 996 } else {
912 engine->reg_tables = ivb_blt_reg_tables; 997 engine->reg_tables = ivb_blt_reg_tables;
913 engine->reg_table_count = ARRAY_SIZE(ivb_blt_reg_tables); 998 engine->reg_table_count = ARRAY_SIZE(ivb_blt_reg_tables);
914 } 999 }
915
916 engine->get_cmd_length_mask = gen7_blt_get_cmd_length_mask;
917 break; 1000 break;
918 case VIDEO_ENHANCEMENT_CLASS: 1001 case VIDEO_ENHANCEMENT_CLASS:
919 cmd_tables = hsw_vebox_cmds; 1002 cmd_tables = hsw_vebox_cmd_table;
920 cmd_table_count = ARRAY_SIZE(hsw_vebox_cmds); 1003 cmd_table_count = ARRAY_SIZE(hsw_vebox_cmd_table);
921 /* VECS can use the same length_mask function as VCS */ 1004 /* VECS can use the same length_mask function as VCS */
922 engine->get_cmd_length_mask = gen7_bsd_get_cmd_length_mask; 1005 engine->get_cmd_length_mask = gen7_bsd_get_cmd_length_mask;
923 break; 1006 break;
@@ -943,7 +1026,7 @@ void intel_engine_init_cmd_parser(struct intel_engine_cs *engine)
943 return; 1026 return;
944 } 1027 }
945 1028
946 engine->flags |= I915_ENGINE_NEEDS_CMD_PARSER; 1029 engine->flags |= I915_ENGINE_USING_CMD_PARSER;
947} 1030}
948 1031
949/** 1032/**
@@ -955,7 +1038,7 @@ void intel_engine_init_cmd_parser(struct intel_engine_cs *engine)
955 */ 1038 */
956void intel_engine_cleanup_cmd_parser(struct intel_engine_cs *engine) 1039void intel_engine_cleanup_cmd_parser(struct intel_engine_cs *engine)
957{ 1040{
958 if (!intel_engine_needs_cmd_parser(engine)) 1041 if (!intel_engine_using_cmd_parser(engine))
959 return; 1042 return;
960 1043
961 fini_hash_table(engine); 1044 fini_hash_table(engine);
@@ -1029,22 +1112,16 @@ __find_reg(const struct drm_i915_reg_descriptor *table, int count, u32 addr)
1029} 1112}
1030 1113
1031static const struct drm_i915_reg_descriptor * 1114static const struct drm_i915_reg_descriptor *
1032find_reg(const struct intel_engine_cs *engine, bool is_master, u32 addr) 1115find_reg(const struct intel_engine_cs *engine, u32 addr)
1033{ 1116{
1034 const struct drm_i915_reg_table *table = engine->reg_tables; 1117 const struct drm_i915_reg_table *table = engine->reg_tables;
1118 const struct drm_i915_reg_descriptor *reg = NULL;
1035 int count = engine->reg_table_count; 1119 int count = engine->reg_table_count;
1036 1120
1037 for (; count > 0; ++table, --count) { 1121 for (; !reg && (count > 0); ++table, --count)
1038 if (!table->master || is_master) { 1122 reg = __find_reg(table->regs, table->num_regs, addr);
1039 const struct drm_i915_reg_descriptor *reg;
1040 1123
1041 reg = __find_reg(table->regs, table->num_regs, addr); 1124 return reg;
1042 if (reg != NULL)
1043 return reg;
1044 }
1045 }
1046
1047 return NULL;
1048} 1125}
1049 1126
1050/* Returns a vmap'd pointer to dst_obj, which the caller must unmap */ 1127/* Returns a vmap'd pointer to dst_obj, which the caller must unmap */
@@ -1128,8 +1205,7 @@ static u32 *copy_batch(struct drm_i915_gem_object *dst_obj,
1128 1205
1129static bool check_cmd(const struct intel_engine_cs *engine, 1206static bool check_cmd(const struct intel_engine_cs *engine,
1130 const struct drm_i915_cmd_descriptor *desc, 1207 const struct drm_i915_cmd_descriptor *desc,
1131 const u32 *cmd, u32 length, 1208 const u32 *cmd, u32 length)
1132 const bool is_master)
1133{ 1209{
1134 if (desc->flags & CMD_DESC_SKIP) 1210 if (desc->flags & CMD_DESC_SKIP)
1135 return true; 1211 return true;
@@ -1139,12 +1215,6 @@ static bool check_cmd(const struct intel_engine_cs *engine,
1139 return false; 1215 return false;
1140 } 1216 }
1141 1217
1142 if ((desc->flags & CMD_DESC_MASTER) && !is_master) {
1143 DRM_DEBUG_DRIVER("CMD: Rejected master-only command: 0x%08X\n",
1144 *cmd);
1145 return false;
1146 }
1147
1148 if (desc->flags & CMD_DESC_REGISTER) { 1218 if (desc->flags & CMD_DESC_REGISTER) {
1149 /* 1219 /*
1150 * Get the distance between individual register offset 1220 * Get the distance between individual register offset
@@ -1158,7 +1228,7 @@ static bool check_cmd(const struct intel_engine_cs *engine,
1158 offset += step) { 1228 offset += step) {
1159 const u32 reg_addr = cmd[offset] & desc->reg.mask; 1229 const u32 reg_addr = cmd[offset] & desc->reg.mask;
1160 const struct drm_i915_reg_descriptor *reg = 1230 const struct drm_i915_reg_descriptor *reg =
1161 find_reg(engine, is_master, reg_addr); 1231 find_reg(engine, reg_addr);
1162 1232
1163 if (!reg) { 1233 if (!reg) {
1164 DRM_DEBUG_DRIVER("CMD: Rejected register 0x%08X in command: 0x%08X (%s)\n", 1234 DRM_DEBUG_DRIVER("CMD: Rejected register 0x%08X in command: 0x%08X (%s)\n",
@@ -1236,16 +1306,112 @@ static bool check_cmd(const struct intel_engine_cs *engine,
1236 return true; 1306 return true;
1237} 1307}
1238 1308
1309static int check_bbstart(const struct i915_gem_context *ctx,
1310 u32 *cmd, u32 offset, u32 length,
1311 u32 batch_len,
1312 u64 batch_start,
1313 u64 shadow_batch_start)
1314{
1315 u64 jump_offset, jump_target;
1316 u32 target_cmd_offset, target_cmd_index;
1317
1318 /* For igt compatibility on older platforms */
1319 if (CMDPARSER_USES_GGTT(ctx->i915)) {
1320 DRM_DEBUG("CMD: Rejecting BB_START for ggtt based submission\n");
1321 return -EACCES;
1322 }
1323
1324 if (length != 3) {
1325 DRM_DEBUG("CMD: Recursive BB_START with bad length(%u)\n",
1326 length);
1327 return -EINVAL;
1328 }
1329
1330 jump_target = *(u64*)(cmd+1);
1331 jump_offset = jump_target - batch_start;
1332
1333 /*
1334 * Any underflow of jump_target is guaranteed to be outside the range
1335 * of a u32, so >= test catches both too large and too small
1336 */
1337 if (jump_offset >= batch_len) {
1338 DRM_DEBUG("CMD: BB_START to 0x%llx jumps out of BB\n",
1339 jump_target);
1340 return -EINVAL;
1341 }
1342
1343 /*
1344 * This cannot overflow a u32 because we already checked jump_offset
1345 * is within the BB, and the batch_len is a u32
1346 */
1347 target_cmd_offset = lower_32_bits(jump_offset);
1348 target_cmd_index = target_cmd_offset / sizeof(u32);
1349
1350 *(u64*)(cmd + 1) = shadow_batch_start + target_cmd_offset;
1351
1352 if (target_cmd_index == offset)
1353 return 0;
1354
1355 if (ctx->jump_whitelist_cmds <= target_cmd_index) {
1356 DRM_DEBUG("CMD: Rejecting BB_START - truncated whitelist array\n");
1357 return -EINVAL;
1358 } else if (!test_bit(target_cmd_index, ctx->jump_whitelist)) {
1359 DRM_DEBUG("CMD: BB_START to 0x%llx not a previously executed cmd\n",
1360 jump_target);
1361 return -EINVAL;
1362 }
1363
1364 return 0;
1365}
1366
1367static void init_whitelist(struct i915_gem_context *ctx, u32 batch_len)
1368{
1369 const u32 batch_cmds = DIV_ROUND_UP(batch_len, sizeof(u32));
1370 const u32 exact_size = BITS_TO_LONGS(batch_cmds);
1371 u32 next_size = BITS_TO_LONGS(roundup_pow_of_two(batch_cmds));
1372 unsigned long *next_whitelist;
1373
1374 if (CMDPARSER_USES_GGTT(ctx->i915))
1375 return;
1376
1377 if (batch_cmds <= ctx->jump_whitelist_cmds) {
1378 bitmap_zero(ctx->jump_whitelist, batch_cmds);
1379 return;
1380 }
1381
1382again:
1383 next_whitelist = kcalloc(next_size, sizeof(long), GFP_KERNEL);
1384 if (next_whitelist) {
1385 kfree(ctx->jump_whitelist);
1386 ctx->jump_whitelist = next_whitelist;
1387 ctx->jump_whitelist_cmds =
1388 next_size * BITS_PER_BYTE * sizeof(long);
1389 return;
1390 }
1391
1392 if (next_size > exact_size) {
1393 next_size = exact_size;
1394 goto again;
1395 }
1396
1397 DRM_DEBUG("CMD: Failed to extend whitelist. BB_START may be disallowed\n");
1398 bitmap_zero(ctx->jump_whitelist, ctx->jump_whitelist_cmds);
1399
1400 return;
1401}
1402
1239#define LENGTH_BIAS 2 1403#define LENGTH_BIAS 2
1240 1404
1241/** 1405/**
1242 * i915_parse_cmds() - parse a submitted batch buffer for privilege violations 1406 * i915_parse_cmds() - parse a submitted batch buffer for privilege violations
1407 * @ctx: the context in which the batch is to execute
1243 * @engine: the engine on which the batch is to execute 1408 * @engine: the engine on which the batch is to execute
1244 * @batch_obj: the batch buffer in question 1409 * @batch_obj: the batch buffer in question
1245 * @shadow_batch_obj: copy of the batch buffer in question 1410 * @batch_start: Canonical base address of batch
1246 * @batch_start_offset: byte offset in the batch at which execution starts 1411 * @batch_start_offset: byte offset in the batch at which execution starts
1247 * @batch_len: length of the commands in batch_obj 1412 * @batch_len: length of the commands in batch_obj
1248 * @is_master: is the submitting process the drm master? 1413 * @shadow_batch_obj: copy of the batch buffer in question
1414 * @shadow_batch_start: Canonical base address of shadow_batch_obj
1249 * 1415 *
1250 * Parses the specified batch buffer looking for privilege violations as 1416 * Parses the specified batch buffer looking for privilege violations as
1251 * described in the overview. 1417 * described in the overview.
@@ -1253,14 +1419,17 @@ static bool check_cmd(const struct intel_engine_cs *engine,
1253 * Return: non-zero if the parser finds violations or otherwise fails; -EACCES 1419 * Return: non-zero if the parser finds violations or otherwise fails; -EACCES
1254 * if the batch appears legal but should use hardware parsing 1420 * if the batch appears legal but should use hardware parsing
1255 */ 1421 */
1256int intel_engine_cmd_parser(struct intel_engine_cs *engine, 1422
1423int intel_engine_cmd_parser(struct i915_gem_context *ctx,
1424 struct intel_engine_cs *engine,
1257 struct drm_i915_gem_object *batch_obj, 1425 struct drm_i915_gem_object *batch_obj,
1258 struct drm_i915_gem_object *shadow_batch_obj, 1426 u64 batch_start,
1259 u32 batch_start_offset, 1427 u32 batch_start_offset,
1260 u32 batch_len, 1428 u32 batch_len,
1261 bool is_master) 1429 struct drm_i915_gem_object *shadow_batch_obj,
1430 u64 shadow_batch_start)
1262{ 1431{
1263 u32 *cmd, *batch_end; 1432 u32 *cmd, *batch_end, offset = 0;
1264 struct drm_i915_cmd_descriptor default_desc = noop_desc; 1433 struct drm_i915_cmd_descriptor default_desc = noop_desc;
1265 const struct drm_i915_cmd_descriptor *desc = &default_desc; 1434 const struct drm_i915_cmd_descriptor *desc = &default_desc;
1266 bool needs_clflush_after = false; 1435 bool needs_clflush_after = false;
@@ -1274,6 +1443,8 @@ int intel_engine_cmd_parser(struct intel_engine_cs *engine,
1274 return PTR_ERR(cmd); 1443 return PTR_ERR(cmd);
1275 } 1444 }
1276 1445
1446 init_whitelist(ctx, batch_len);
1447
1277 /* 1448 /*
1278 * We use the batch length as size because the shadow object is as 1449 * We use the batch length as size because the shadow object is as
1279 * large or larger and copy_batch() will write MI_NOPs to the extra 1450 * large or larger and copy_batch() will write MI_NOPs to the extra
@@ -1283,31 +1454,15 @@ int intel_engine_cmd_parser(struct intel_engine_cs *engine,
1283 do { 1454 do {
1284 u32 length; 1455 u32 length;
1285 1456
1286 if (*cmd == MI_BATCH_BUFFER_END) { 1457 if (*cmd == MI_BATCH_BUFFER_END)
1287 if (needs_clflush_after) {
1288 void *ptr = page_mask_bits(shadow_batch_obj->mm.mapping);
1289 drm_clflush_virt_range(ptr,
1290 (void *)(cmd + 1) - ptr);
1291 }
1292 break; 1458 break;
1293 }
1294 1459
1295 desc = find_cmd(engine, *cmd, desc, &default_desc); 1460 desc = find_cmd(engine, *cmd, desc, &default_desc);
1296 if (!desc) { 1461 if (!desc) {
1297 DRM_DEBUG_DRIVER("CMD: Unrecognized command: 0x%08X\n", 1462 DRM_DEBUG_DRIVER("CMD: Unrecognized command: 0x%08X\n",
1298 *cmd); 1463 *cmd);
1299 ret = -EINVAL; 1464 ret = -EINVAL;
1300 break; 1465 goto err;
1301 }
1302
1303 /*
1304 * If the batch buffer contains a chained batch, return an
1305 * error that tells the caller to abort and dispatch the
1306 * workload as a non-secure batch.
1307 */
1308 if (desc->cmd.value == MI_BATCH_BUFFER_START) {
1309 ret = -EACCES;
1310 break;
1311 } 1466 }
1312 1467
1313 if (desc->flags & CMD_DESC_FIXED) 1468 if (desc->flags & CMD_DESC_FIXED)
@@ -1321,22 +1476,43 @@ int intel_engine_cmd_parser(struct intel_engine_cs *engine,
1321 length, 1476 length,
1322 batch_end - cmd); 1477 batch_end - cmd);
1323 ret = -EINVAL; 1478 ret = -EINVAL;
1324 break; 1479 goto err;
1325 } 1480 }
1326 1481
1327 if (!check_cmd(engine, desc, cmd, length, is_master)) { 1482 if (!check_cmd(engine, desc, cmd, length)) {
1328 ret = -EACCES; 1483 ret = -EACCES;
1484 goto err;
1485 }
1486
1487 if (desc->cmd.value == MI_BATCH_BUFFER_START) {
1488 ret = check_bbstart(ctx, cmd, offset, length,
1489 batch_len, batch_start,
1490 shadow_batch_start);
1491
1492 if (ret)
1493 goto err;
1329 break; 1494 break;
1330 } 1495 }
1331 1496
1497 if (ctx->jump_whitelist_cmds > offset)
1498 set_bit(offset, ctx->jump_whitelist);
1499
1332 cmd += length; 1500 cmd += length;
1501 offset += length;
1333 if (cmd >= batch_end) { 1502 if (cmd >= batch_end) {
1334 DRM_DEBUG_DRIVER("CMD: Got to the end of the buffer w/o a BBE cmd!\n"); 1503 DRM_DEBUG_DRIVER("CMD: Got to the end of the buffer w/o a BBE cmd!\n");
1335 ret = -EINVAL; 1504 ret = -EINVAL;
1336 break; 1505 goto err;
1337 } 1506 }
1338 } while (1); 1507 } while (1);
1339 1508
1509 if (needs_clflush_after) {
1510 void *ptr = page_mask_bits(shadow_batch_obj->mm.mapping);
1511
1512 drm_clflush_virt_range(ptr, (void *)(cmd + 1) - ptr);
1513 }
1514
1515err:
1340 i915_gem_object_unpin_map(shadow_batch_obj); 1516 i915_gem_object_unpin_map(shadow_batch_obj);
1341 return ret; 1517 return ret;
1342} 1518}
@@ -1357,7 +1533,7 @@ int i915_cmd_parser_get_version(struct drm_i915_private *dev_priv)
1357 1533
1358 /* If the command parser is not enabled, report 0 - unsupported */ 1534 /* If the command parser is not enabled, report 0 - unsupported */
1359 for_each_uabi_engine(engine, dev_priv) { 1535 for_each_uabi_engine(engine, dev_priv) {
1360 if (intel_engine_needs_cmd_parser(engine)) { 1536 if (intel_engine_using_cmd_parser(engine)) {
1361 active = true; 1537 active = true;
1362 break; 1538 break;
1363 } 1539 }
@@ -1382,6 +1558,7 @@ int i915_cmd_parser_get_version(struct drm_i915_private *dev_priv)
1382 * the parser enabled. 1558 * the parser enabled.
1383 * 9. Don't whitelist or handle oacontrol specially, as ownership 1559 * 9. Don't whitelist or handle oacontrol specially, as ownership
1384 * for oacontrol state is moving to i915-perf. 1560 * for oacontrol state is moving to i915-perf.
1561 * 10. Support for Gen9 BCS Parsing
1385 */ 1562 */
1386 return 9; 1563 return 10;
1387} 1564}
diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c
index bb6f86c7067a..fe4d7cabfdf1 100644
--- a/drivers/gpu/drm/i915/i915_drv.c
+++ b/drivers/gpu/drm/i915/i915_drv.c
@@ -1850,6 +1850,8 @@ static int i915_drm_suspend_late(struct drm_device *dev, bool hibernation)
1850 1850
1851 i915_gem_suspend_late(dev_priv); 1851 i915_gem_suspend_late(dev_priv);
1852 1852
1853 i915_rc6_ctx_wa_suspend(dev_priv);
1854
1853 intel_uncore_suspend(&dev_priv->uncore); 1855 intel_uncore_suspend(&dev_priv->uncore);
1854 1856
1855 intel_power_domains_suspend(dev_priv, 1857 intel_power_domains_suspend(dev_priv,
@@ -2053,6 +2055,8 @@ static int i915_drm_resume_early(struct drm_device *dev)
2053 2055
2054 intel_power_domains_resume(dev_priv); 2056 intel_power_domains_resume(dev_priv);
2055 2057
2058 i915_rc6_ctx_wa_resume(dev_priv);
2059
2056 intel_gt_sanitize(&dev_priv->gt, true); 2060 intel_gt_sanitize(&dev_priv->gt, true);
2057 2061
2058 enable_rpm_wakeref_asserts(&dev_priv->runtime_pm); 2062 enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index 953e1d12c23c..89b6112bd66b 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -593,6 +593,8 @@ struct intel_rps {
593 593
594struct intel_rc6 { 594struct intel_rc6 {
595 bool enabled; 595 bool enabled;
596 bool ctx_corrupted;
597 intel_wakeref_t ctx_corrupted_wakeref;
596 u64 prev_hw_residency[4]; 598 u64 prev_hw_residency[4];
597 u64 cur_residency[4]; 599 u64 cur_residency[4];
598}; 600};
@@ -2075,9 +2077,16 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915,
2075#define VEBOX_MASK(dev_priv) \ 2077#define VEBOX_MASK(dev_priv) \
2076 ENGINE_INSTANCES_MASK(dev_priv, VECS0, I915_MAX_VECS) 2078 ENGINE_INSTANCES_MASK(dev_priv, VECS0, I915_MAX_VECS)
2077 2079
2080/*
2081 * The Gen7 cmdparser copies the scanned buffer to the ggtt for execution
2082 * All later gens can run the final buffer from the ppgtt
2083 */
2084#define CMDPARSER_USES_GGTT(dev_priv) IS_GEN(dev_priv, 7)
2085
2078#define HAS_LLC(dev_priv) (INTEL_INFO(dev_priv)->has_llc) 2086#define HAS_LLC(dev_priv) (INTEL_INFO(dev_priv)->has_llc)
2079#define HAS_SNOOP(dev_priv) (INTEL_INFO(dev_priv)->has_snoop) 2087#define HAS_SNOOP(dev_priv) (INTEL_INFO(dev_priv)->has_snoop)
2080#define HAS_EDRAM(dev_priv) ((dev_priv)->edram_size_mb) 2088#define HAS_EDRAM(dev_priv) ((dev_priv)->edram_size_mb)
2089#define HAS_SECURE_BATCHES(dev_priv) (INTEL_GEN(dev_priv) < 6)
2081#define HAS_WT(dev_priv) ((IS_HASWELL(dev_priv) || \ 2090#define HAS_WT(dev_priv) ((IS_HASWELL(dev_priv) || \
2082 IS_BROADWELL(dev_priv)) && HAS_EDRAM(dev_priv)) 2091 IS_BROADWELL(dev_priv)) && HAS_EDRAM(dev_priv))
2083 2092
@@ -2110,10 +2119,12 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915,
2110/* Early gen2 have a totally busted CS tlb and require pinned batches. */ 2119/* Early gen2 have a totally busted CS tlb and require pinned batches. */
2111#define HAS_BROKEN_CS_TLB(dev_priv) (IS_I830(dev_priv) || IS_I845G(dev_priv)) 2120#define HAS_BROKEN_CS_TLB(dev_priv) (IS_I830(dev_priv) || IS_I845G(dev_priv))
2112 2121
2122#define NEEDS_RC6_CTX_CORRUPTION_WA(dev_priv) \
2123 (IS_BROADWELL(dev_priv) || IS_GEN(dev_priv, 9))
2124
2113/* WaRsDisableCoarsePowerGating:skl,cnl */ 2125/* WaRsDisableCoarsePowerGating:skl,cnl */
2114#define NEEDS_WaRsDisableCoarsePowerGating(dev_priv) \ 2126#define NEEDS_WaRsDisableCoarsePowerGating(dev_priv) \
2115 (IS_CANNONLAKE(dev_priv) || \ 2127 (IS_CANNONLAKE(dev_priv) || IS_GEN(dev_priv, 9))
2116 IS_SKL_GT3(dev_priv) || IS_SKL_GT4(dev_priv))
2117 2128
2118#define HAS_GMBUS_IRQ(dev_priv) (INTEL_GEN(dev_priv) >= 4) 2129#define HAS_GMBUS_IRQ(dev_priv) (INTEL_GEN(dev_priv) >= 4)
2119#define HAS_GMBUS_BURST_READ(dev_priv) (INTEL_GEN(dev_priv) >= 10 || \ 2130#define HAS_GMBUS_BURST_READ(dev_priv) (INTEL_GEN(dev_priv) >= 10 || \
@@ -2284,6 +2295,14 @@ int i915_gem_object_unbind(struct drm_i915_gem_object *obj,
2284 unsigned long flags); 2295 unsigned long flags);
2285#define I915_GEM_OBJECT_UNBIND_ACTIVE BIT(0) 2296#define I915_GEM_OBJECT_UNBIND_ACTIVE BIT(0)
2286 2297
2298struct i915_vma * __must_check
2299i915_gem_object_pin(struct drm_i915_gem_object *obj,
2300 struct i915_address_space *vm,
2301 const struct i915_ggtt_view *view,
2302 u64 size,
2303 u64 alignment,
2304 u64 flags);
2305
2287void i915_gem_runtime_suspend(struct drm_i915_private *dev_priv); 2306void i915_gem_runtime_suspend(struct drm_i915_private *dev_priv);
2288 2307
2289static inline int __must_check 2308static inline int __must_check
@@ -2393,12 +2412,14 @@ const char *i915_cache_level_str(struct drm_i915_private *i915, int type);
2393int i915_cmd_parser_get_version(struct drm_i915_private *dev_priv); 2412int i915_cmd_parser_get_version(struct drm_i915_private *dev_priv);
2394void intel_engine_init_cmd_parser(struct intel_engine_cs *engine); 2413void intel_engine_init_cmd_parser(struct intel_engine_cs *engine);
2395void intel_engine_cleanup_cmd_parser(struct intel_engine_cs *engine); 2414void intel_engine_cleanup_cmd_parser(struct intel_engine_cs *engine);
2396int intel_engine_cmd_parser(struct intel_engine_cs *engine, 2415int intel_engine_cmd_parser(struct i915_gem_context *cxt,
2416 struct intel_engine_cs *engine,
2397 struct drm_i915_gem_object *batch_obj, 2417 struct drm_i915_gem_object *batch_obj,
2398 struct drm_i915_gem_object *shadow_batch_obj, 2418 u64 user_batch_start,
2399 u32 batch_start_offset, 2419 u32 batch_start_offset,
2400 u32 batch_len, 2420 u32 batch_len,
2401 bool is_master); 2421 struct drm_i915_gem_object *shadow_batch_obj,
2422 u64 shadow_batch_start);
2402 2423
2403/* intel_device_info.c */ 2424/* intel_device_info.c */
2404static inline struct intel_device_info * 2425static inline struct intel_device_info *
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index d0f94f239919..98305d987ac1 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -964,6 +964,20 @@ i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
964{ 964{
965 struct drm_i915_private *dev_priv = to_i915(obj->base.dev); 965 struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
966 struct i915_address_space *vm = &dev_priv->ggtt.vm; 966 struct i915_address_space *vm = &dev_priv->ggtt.vm;
967
968 return i915_gem_object_pin(obj, vm, view, size, alignment,
969 flags | PIN_GLOBAL);
970}
971
972struct i915_vma *
973i915_gem_object_pin(struct drm_i915_gem_object *obj,
974 struct i915_address_space *vm,
975 const struct i915_ggtt_view *view,
976 u64 size,
977 u64 alignment,
978 u64 flags)
979{
980 struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
967 struct i915_vma *vma; 981 struct i915_vma *vma;
968 int ret; 982 int ret;
969 983
@@ -1038,7 +1052,7 @@ i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
1038 return ERR_PTR(ret); 1052 return ERR_PTR(ret);
1039 } 1053 }
1040 1054
1041 ret = i915_vma_pin(vma, size, alignment, flags | PIN_GLOBAL); 1055 ret = i915_vma_pin(vma, size, alignment, flags);
1042 if (ret) 1056 if (ret)
1043 return ERR_PTR(ret); 1057 return ERR_PTR(ret);
1044 1058
diff --git a/drivers/gpu/drm/i915/i915_getparam.c b/drivers/gpu/drm/i915/i915_getparam.c
index 5d9101376a3d..9f1517af5b7f 100644
--- a/drivers/gpu/drm/i915/i915_getparam.c
+++ b/drivers/gpu/drm/i915/i915_getparam.c
@@ -62,7 +62,7 @@ int i915_getparam_ioctl(struct drm_device *dev, void *data,
62 value = !!(i915->caps.scheduler & I915_SCHEDULER_CAP_SEMAPHORES); 62 value = !!(i915->caps.scheduler & I915_SCHEDULER_CAP_SEMAPHORES);
63 break; 63 break;
64 case I915_PARAM_HAS_SECURE_BATCHES: 64 case I915_PARAM_HAS_SECURE_BATCHES:
65 value = capable(CAP_SYS_ADMIN); 65 value = HAS_SECURE_BATCHES(i915) && capable(CAP_SYS_ADMIN);
66 break; 66 break;
67 case I915_PARAM_CMD_PARSER_VERSION: 67 case I915_PARAM_CMD_PARSER_VERSION:
68 value = i915_cmd_parser_get_version(i915); 68 value = i915_cmd_parser_get_version(i915);
diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h
index 2abd199093c5..f8ee9aba3955 100644
--- a/drivers/gpu/drm/i915/i915_reg.h
+++ b/drivers/gpu/drm/i915/i915_reg.h
@@ -471,6 +471,8 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
471#define ECOCHK_PPGTT_WT_HSW (0x2 << 3) 471#define ECOCHK_PPGTT_WT_HSW (0x2 << 3)
472#define ECOCHK_PPGTT_WB_HSW (0x3 << 3) 472#define ECOCHK_PPGTT_WB_HSW (0x3 << 3)
473 473
474#define GEN8_RC6_CTX_INFO _MMIO(0x8504)
475
474#define GAC_ECO_BITS _MMIO(0x14090) 476#define GAC_ECO_BITS _MMIO(0x14090)
475#define ECOBITS_SNB_BIT (1 << 13) 477#define ECOBITS_SNB_BIT (1 << 13)
476#define ECOBITS_PPGTT_CACHE64B (3 << 8) 478#define ECOBITS_PPGTT_CACHE64B (3 << 8)
@@ -555,6 +557,10 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
555 */ 557 */
556#define BCS_SWCTRL _MMIO(0x22200) 558#define BCS_SWCTRL _MMIO(0x22200)
557 559
560/* There are 16 GPR registers */
561#define BCS_GPR(n) _MMIO(0x22600 + (n) * 8)
562#define BCS_GPR_UDW(n) _MMIO(0x22600 + (n) * 8 + 4)
563
558#define GPGPU_THREADS_DISPATCHED _MMIO(0x2290) 564#define GPGPU_THREADS_DISPATCHED _MMIO(0x2290)
559#define GPGPU_THREADS_DISPATCHED_UDW _MMIO(0x2290 + 4) 565#define GPGPU_THREADS_DISPATCHED_UDW _MMIO(0x2290 + 4)
560#define HS_INVOCATION_COUNT _MMIO(0x2300) 566#define HS_INVOCATION_COUNT _MMIO(0x2300)
@@ -7211,6 +7217,10 @@ enum {
7211#define TGL_DMC_DEBUG_DC5_COUNT _MMIO(0x101084) 7217#define TGL_DMC_DEBUG_DC5_COUNT _MMIO(0x101084)
7212#define TGL_DMC_DEBUG_DC6_COUNT _MMIO(0x101088) 7218#define TGL_DMC_DEBUG_DC6_COUNT _MMIO(0x101088)
7213 7219
7220/* Display Internal Timeout Register */
7221#define RM_TIMEOUT _MMIO(0x42060)
7222#define MMIO_TIMEOUT_US(us) ((us) << 0)
7223
7214/* interrupts */ 7224/* interrupts */
7215#define DE_MASTER_IRQ_CONTROL (1 << 31) 7225#define DE_MASTER_IRQ_CONTROL (1 << 31)
7216#define DE_SPRITEB_FLIP_DONE (1 << 29) 7226#define DE_SPRITEB_FLIP_DONE (1 << 29)
diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c
index 75ee027abb80..2efe1d12d5a9 100644
--- a/drivers/gpu/drm/i915/intel_pm.c
+++ b/drivers/gpu/drm/i915/intel_pm.c
@@ -126,6 +126,14 @@ static void bxt_init_clock_gating(struct drm_i915_private *dev_priv)
126 */ 126 */
127 I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) | 127 I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) |
128 PWM1_GATING_DIS | PWM2_GATING_DIS); 128 PWM1_GATING_DIS | PWM2_GATING_DIS);
129
130 /*
131 * Lower the display internal timeout.
132 * This is needed to avoid any hard hangs when DSI port PLL
133 * is off and a MMIO access is attempted by any privilege
134 * application, using batch buffers or any other means.
135 */
136 I915_WRITE(RM_TIMEOUT, MMIO_TIMEOUT_US(950));
129} 137}
130 138
131static void glk_init_clock_gating(struct drm_i915_private *dev_priv) 139static void glk_init_clock_gating(struct drm_i915_private *dev_priv)
@@ -8544,6 +8552,100 @@ static void intel_init_emon(struct drm_i915_private *dev_priv)
8544 dev_priv->ips.corr = (lcfuse & LCFUSE_HIV_MASK); 8552 dev_priv->ips.corr = (lcfuse & LCFUSE_HIV_MASK);
8545} 8553}
8546 8554
8555static bool i915_rc6_ctx_corrupted(struct drm_i915_private *dev_priv)
8556{
8557 return !I915_READ(GEN8_RC6_CTX_INFO);
8558}
8559
8560static void i915_rc6_ctx_wa_init(struct drm_i915_private *i915)
8561{
8562 if (!NEEDS_RC6_CTX_CORRUPTION_WA(i915))
8563 return;
8564
8565 if (i915_rc6_ctx_corrupted(i915)) {
8566 DRM_INFO("RC6 context corrupted, disabling runtime power management\n");
8567 i915->gt_pm.rc6.ctx_corrupted = true;
8568 i915->gt_pm.rc6.ctx_corrupted_wakeref =
8569 intel_runtime_pm_get(&i915->runtime_pm);
8570 }
8571}
8572
8573static void i915_rc6_ctx_wa_cleanup(struct drm_i915_private *i915)
8574{
8575 if (i915->gt_pm.rc6.ctx_corrupted) {
8576 intel_runtime_pm_put(&i915->runtime_pm,
8577 i915->gt_pm.rc6.ctx_corrupted_wakeref);
8578 i915->gt_pm.rc6.ctx_corrupted = false;
8579 }
8580}
8581
8582/**
8583 * i915_rc6_ctx_wa_suspend - system suspend sequence for the RC6 CTX WA
8584 * @i915: i915 device
8585 *
8586 * Perform any steps needed to clean up the RC6 CTX WA before system suspend.
8587 */
8588void i915_rc6_ctx_wa_suspend(struct drm_i915_private *i915)
8589{
8590 if (i915->gt_pm.rc6.ctx_corrupted)
8591 intel_runtime_pm_put(&i915->runtime_pm,
8592 i915->gt_pm.rc6.ctx_corrupted_wakeref);
8593}
8594
8595/**
8596 * i915_rc6_ctx_wa_resume - system resume sequence for the RC6 CTX WA
8597 * @i915: i915 device
8598 *
8599 * Perform any steps needed to re-init the RC6 CTX WA after system resume.
8600 */
8601void i915_rc6_ctx_wa_resume(struct drm_i915_private *i915)
8602{
8603 if (!i915->gt_pm.rc6.ctx_corrupted)
8604 return;
8605
8606 if (i915_rc6_ctx_corrupted(i915)) {
8607 i915->gt_pm.rc6.ctx_corrupted_wakeref =
8608 intel_runtime_pm_get(&i915->runtime_pm);
8609 return;
8610 }
8611
8612 DRM_INFO("RC6 context restored, re-enabling runtime power management\n");
8613 i915->gt_pm.rc6.ctx_corrupted = false;
8614}
8615
8616static void intel_disable_rc6(struct drm_i915_private *dev_priv);
8617
8618/**
8619 * i915_rc6_ctx_wa_check - check for a new RC6 CTX corruption
8620 * @i915: i915 device
8621 *
8622 * Check if an RC6 CTX corruption has happened since the last check and if so
8623 * disable RC6 and runtime power management.
8624 *
8625 * Return false if no context corruption has happened since the last call of
8626 * this function, true otherwise.
8627*/
8628bool i915_rc6_ctx_wa_check(struct drm_i915_private *i915)
8629{
8630 if (!NEEDS_RC6_CTX_CORRUPTION_WA(i915))
8631 return false;
8632
8633 if (i915->gt_pm.rc6.ctx_corrupted)
8634 return false;
8635
8636 if (!i915_rc6_ctx_corrupted(i915))
8637 return false;
8638
8639 DRM_NOTE("RC6 context corruption, disabling runtime power management\n");
8640
8641 intel_disable_rc6(i915);
8642 i915->gt_pm.rc6.ctx_corrupted = true;
8643 i915->gt_pm.rc6.ctx_corrupted_wakeref =
8644 intel_runtime_pm_get_noresume(&i915->runtime_pm);
8645
8646 return true;
8647}
8648
8547void intel_init_gt_powersave(struct drm_i915_private *dev_priv) 8649void intel_init_gt_powersave(struct drm_i915_private *dev_priv)
8548{ 8650{
8549 struct intel_rps *rps = &dev_priv->gt_pm.rps; 8651 struct intel_rps *rps = &dev_priv->gt_pm.rps;
@@ -8557,6 +8659,8 @@ void intel_init_gt_powersave(struct drm_i915_private *dev_priv)
8557 pm_runtime_get(&dev_priv->drm.pdev->dev); 8659 pm_runtime_get(&dev_priv->drm.pdev->dev);
8558 } 8660 }
8559 8661
8662 i915_rc6_ctx_wa_init(dev_priv);
8663
8560 /* Initialize RPS limits (for userspace) */ 8664 /* Initialize RPS limits (for userspace) */
8561 if (IS_CHERRYVIEW(dev_priv)) 8665 if (IS_CHERRYVIEW(dev_priv))
8562 cherryview_init_gt_powersave(dev_priv); 8666 cherryview_init_gt_powersave(dev_priv);
@@ -8595,6 +8699,8 @@ void intel_cleanup_gt_powersave(struct drm_i915_private *dev_priv)
8595 if (IS_VALLEYVIEW(dev_priv)) 8699 if (IS_VALLEYVIEW(dev_priv))
8596 valleyview_cleanup_gt_powersave(dev_priv); 8700 valleyview_cleanup_gt_powersave(dev_priv);
8597 8701
8702 i915_rc6_ctx_wa_cleanup(dev_priv);
8703
8598 if (!HAS_RC6(dev_priv)) 8704 if (!HAS_RC6(dev_priv))
8599 pm_runtime_put(&dev_priv->drm.pdev->dev); 8705 pm_runtime_put(&dev_priv->drm.pdev->dev);
8600} 8706}
@@ -8623,7 +8729,7 @@ static inline void intel_disable_llc_pstate(struct drm_i915_private *i915)
8623 i915->gt_pm.llc_pstate.enabled = false; 8729 i915->gt_pm.llc_pstate.enabled = false;
8624} 8730}
8625 8731
8626static void intel_disable_rc6(struct drm_i915_private *dev_priv) 8732static void __intel_disable_rc6(struct drm_i915_private *dev_priv)
8627{ 8733{
8628 lockdep_assert_held(&dev_priv->gt_pm.rps.lock); 8734 lockdep_assert_held(&dev_priv->gt_pm.rps.lock);
8629 8735
@@ -8642,6 +8748,15 @@ static void intel_disable_rc6(struct drm_i915_private *dev_priv)
8642 dev_priv->gt_pm.rc6.enabled = false; 8748 dev_priv->gt_pm.rc6.enabled = false;
8643} 8749}
8644 8750
8751static void intel_disable_rc6(struct drm_i915_private *dev_priv)
8752{
8753 struct intel_rps *rps = &dev_priv->gt_pm.rps;
8754
8755 mutex_lock(&rps->lock);
8756 __intel_disable_rc6(dev_priv);
8757 mutex_unlock(&rps->lock);
8758}
8759
8645static void intel_disable_rps(struct drm_i915_private *dev_priv) 8760static void intel_disable_rps(struct drm_i915_private *dev_priv)
8646{ 8761{
8647 lockdep_assert_held(&dev_priv->gt_pm.rps.lock); 8762 lockdep_assert_held(&dev_priv->gt_pm.rps.lock);
@@ -8667,7 +8782,7 @@ void intel_disable_gt_powersave(struct drm_i915_private *dev_priv)
8667{ 8782{
8668 mutex_lock(&dev_priv->gt_pm.rps.lock); 8783 mutex_lock(&dev_priv->gt_pm.rps.lock);
8669 8784
8670 intel_disable_rc6(dev_priv); 8785 __intel_disable_rc6(dev_priv);
8671 intel_disable_rps(dev_priv); 8786 intel_disable_rps(dev_priv);
8672 if (HAS_LLC(dev_priv)) 8787 if (HAS_LLC(dev_priv))
8673 intel_disable_llc_pstate(dev_priv); 8788 intel_disable_llc_pstate(dev_priv);
@@ -8694,6 +8809,9 @@ static void intel_enable_rc6(struct drm_i915_private *dev_priv)
8694 if (dev_priv->gt_pm.rc6.enabled) 8809 if (dev_priv->gt_pm.rc6.enabled)
8695 return; 8810 return;
8696 8811
8812 if (dev_priv->gt_pm.rc6.ctx_corrupted)
8813 return;
8814
8697 if (IS_CHERRYVIEW(dev_priv)) 8815 if (IS_CHERRYVIEW(dev_priv))
8698 cherryview_enable_rc6(dev_priv); 8816 cherryview_enable_rc6(dev_priv);
8699 else if (IS_VALLEYVIEW(dev_priv)) 8817 else if (IS_VALLEYVIEW(dev_priv))
diff --git a/drivers/gpu/drm/i915/intel_pm.h b/drivers/gpu/drm/i915/intel_pm.h
index e3573e1e16e3..0f7390c850ec 100644
--- a/drivers/gpu/drm/i915/intel_pm.h
+++ b/drivers/gpu/drm/i915/intel_pm.h
@@ -36,6 +36,9 @@ void intel_cleanup_gt_powersave(struct drm_i915_private *dev_priv);
36void intel_sanitize_gt_powersave(struct drm_i915_private *dev_priv); 36void intel_sanitize_gt_powersave(struct drm_i915_private *dev_priv);
37void intel_enable_gt_powersave(struct drm_i915_private *dev_priv); 37void intel_enable_gt_powersave(struct drm_i915_private *dev_priv);
38void intel_disable_gt_powersave(struct drm_i915_private *dev_priv); 38void intel_disable_gt_powersave(struct drm_i915_private *dev_priv);
39bool i915_rc6_ctx_wa_check(struct drm_i915_private *i915);
40void i915_rc6_ctx_wa_suspend(struct drm_i915_private *i915);
41void i915_rc6_ctx_wa_resume(struct drm_i915_private *i915);
39void gen6_rps_busy(struct drm_i915_private *dev_priv); 42void gen6_rps_busy(struct drm_i915_private *dev_priv);
40void gen6_rps_idle(struct drm_i915_private *dev_priv); 43void gen6_rps_idle(struct drm_i915_private *dev_priv);
41void gen6_rps_boost(struct i915_request *rq); 44void gen6_rps_boost(struct i915_request *rq);
generated by cgit v1.2.2 (git 2.25.0) at 2025-09-10 07:21:08 -0400